You've already forked libopenshot
mirror of
https://github.com/OpenShot/libopenshot.git
synced 2026-06-08 22:17:28 -07:00
Add Object Mask effect with EdgeSAM and XMem propagation
- Add ObjectMask effect for rendering and exposing generated object masks - Add EdgeSAM preprocessing path for prompt-based seed and reseed masks - Add XMem ONNX propagation between seed frames for single-object tracking - Support positive/negative points, positive rect prompts, and prompt keyframes - Store generated masks in protobuf data for playback and downstream effects - Add mask color/alpha plus stroke color/alpha/width effect controls - Register ObjectMask with effect discovery and clip preprocessing jobs - Add focused ObjectMask effect tests
This commit is contained in:
Jonathan Thomas
@@ -98,10 +98,12 @@ set(OPENSHOT_CV_SOURCES
|
||||
CVStabilization.cpp
|
||||
ClipProcessingJobs.cpp
|
||||
CVObjectDetection.cpp
|
||||
CVObjectMask.cpp
|
||||
TrackedObjectBBox.cpp
|
||||
effects/Stabilizer.cpp
|
||||
effects/Tracker.cpp
|
||||
effects/ObjectDetection.cpp
|
||||
effects/ObjectMask.cpp
|
||||
effects/Outline.cpp
|
||||
./sort_filter/sort.cpp
|
||||
./sort_filter/Hungarian.cpp
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* @file
|
||||
* @brief Header file for CVObjectMask class
|
||||
* @author Jonathan Thomas <jonathan@openshot.org>
|
||||
*
|
||||
* @ref License
|
||||
*/
|
||||
|
||||
// Copyright (c) 2026 OpenShot Studios, LLC
|
||||
//
|
||||
// SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#define int64 opencv_broken_int
|
||||
#define uint64 opencv_broken_uint
|
||||
#include <opencv2/core.hpp>
|
||||
#include <opencv2/dnn.hpp>
|
||||
#include <opencv2/opencv.hpp>
|
||||
#undef uint64
|
||||
#undef int64
|
||||
|
||||
#include "Clip.h"
|
||||
#include "Json.h"
|
||||
#include "ProcessingController.h"
|
||||
|
||||
namespace pb_objdetect {
|
||||
class Frame;
|
||||
}
|
||||
|
||||
namespace openshot
|
||||
{
|
||||
struct CVObjectMaskFrameData {
|
||||
size_t frameId = 0;
|
||||
cv::Rect_<float> box;
|
||||
float score = 0.0f;
|
||||
int objectId = 1;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
std::vector<uint32_t> rle;
|
||||
|
||||
bool HasMask() const { return width > 0 && height > 0 && !rle.empty(); }
|
||||
};
|
||||
|
||||
struct CVObjectMaskPromptSet {
|
||||
std::vector<cv::Point2f> positivePoints;
|
||||
std::vector<cv::Point2f> negativePoints;
|
||||
cv::Point2f rectTopLeft = cv::Point2f(-1.0f, -1.0f);
|
||||
cv::Point2f rectBottomRight = cv::Point2f(-1.0f, -1.0f);
|
||||
bool hasRect = false;
|
||||
|
||||
bool HasPositivePrompt() const { return hasRect || !positivePoints.empty(); }
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Preprocess a clip into EdgeSAM object masks stored in the object-detection protobuf format.
|
||||
*/
|
||||
class CVObjectMask
|
||||
{
|
||||
private:
|
||||
cv::dnn::Net encoder;
|
||||
cv::dnn::Net decoder;
|
||||
|
||||
std::string encoderModelPath;
|
||||
std::string decoderModelPath;
|
||||
std::string xmemModelDir;
|
||||
std::string xmemEncodeKeyModelPath;
|
||||
std::string xmemEncodeValueModelPath;
|
||||
std::string xmemDecodeModelPath;
|
||||
std::string protobufDataPath;
|
||||
std::string processingDevice = "CPU";
|
||||
|
||||
std::map<size_t, CVObjectMaskPromptSet> promptKeyframes;
|
||||
int promptSlots = 10;
|
||||
float maskThreshold = 0.0f;
|
||||
int modelSize = 1024;
|
||||
int maskSize = 256;
|
||||
|
||||
size_t start = 0;
|
||||
size_t end = 0;
|
||||
bool error = false;
|
||||
|
||||
ProcessingController* processingController;
|
||||
|
||||
void SetProcessingDevice();
|
||||
cv::Mat CreateEdgeSAMSeedMask(const cv::Mat& frame, const CVObjectMaskPromptSet& prompts);
|
||||
void AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, const CVObjectMaskFrameData& frameData);
|
||||
|
||||
public:
|
||||
std::map<size_t, CVObjectMaskFrameData> masksData;
|
||||
|
||||
CVObjectMask(std::string processInfoJson, ProcessingController& processingController);
|
||||
|
||||
static std::string ValidateONNXModels(std::string encoderPath, std::string decoderPath);
|
||||
|
||||
void maskClip(openshot::Clip& video, size_t start = 0, size_t end = 0, bool process_interval = false);
|
||||
bool SaveObjMaskData();
|
||||
|
||||
void SetJson(const std::string value);
|
||||
void SetJsonValue(const Json::Value root);
|
||||
};
|
||||
}
|
||||
@@ -41,6 +41,9 @@ void ClipProcessingJobs::processClip(Clip& clip, std::string json){
|
||||
if(processingType == "ObjectDetection"){
|
||||
t = std::thread(&ClipProcessingJobs::detectObjectsClip, this, std::ref(clip), std::ref(this->processingController));
|
||||
}
|
||||
if(processingType == "ObjectMask"){
|
||||
t = std::thread(&ClipProcessingJobs::maskObjectClip, this, std::ref(clip), std::ref(this->processingController));
|
||||
}
|
||||
}
|
||||
|
||||
// Apply object tracking to clip
|
||||
@@ -87,6 +90,21 @@ void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& con
|
||||
}
|
||||
}
|
||||
|
||||
// Apply object segmentation mask to clip
|
||||
void ClipProcessingJobs::maskObjectClip(Clip& clip, ProcessingController& controller){
|
||||
CVObjectMask objectMask(processInfoJson, controller);
|
||||
objectMask.maskClip(clip);
|
||||
|
||||
if(controller.ShouldStop()){
|
||||
controller.SetFinished(true);
|
||||
return;
|
||||
}
|
||||
else{
|
||||
objectMask.SaveObjMaskData();
|
||||
controller.SetFinished(true);
|
||||
}
|
||||
}
|
||||
|
||||
void ClipProcessingJobs::stabilizeClip(Clip& clip, ProcessingController& controller){
|
||||
// create CVStabilization object
|
||||
CVStabilization stabilizer(processInfoJson, controller);
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
#include "CVStabilization.h"
|
||||
#include "CVTracker.h"
|
||||
#include "CVObjectDetection.h"
|
||||
#include "CVObjectMask.h"
|
||||
#endif
|
||||
|
||||
#include <thread>
|
||||
@@ -51,6 +52,8 @@ class ClipProcessingJobs{
|
||||
void stabilizeClip(Clip& clip, ProcessingController& controller);
|
||||
// Apply object detection to clip
|
||||
void detectObjectsClip(Clip& clip, ProcessingController& controller);
|
||||
// Apply object segmentation mask to clip
|
||||
void maskObjectClip(Clip& clip, ProcessingController& controller);
|
||||
|
||||
|
||||
public:
|
||||
|
||||
@@ -149,6 +149,9 @@ EffectBase* EffectInfo::CreateEffect(std::string effect_type) {
|
||||
|
||||
else if(effect_type == "ObjectDetection")
|
||||
return new ObjectDetection();
|
||||
|
||||
else if(effect_type == "ObjectMask")
|
||||
return new ObjectMask();
|
||||
#endif
|
||||
|
||||
return NULL;
|
||||
@@ -205,6 +208,7 @@ Json::Value EffectInfo::JsonValue() {
|
||||
root.append(Stabilizer().JsonInfo());
|
||||
root.append(Tracker().JsonInfo());
|
||||
root.append(ObjectDetection().JsonInfo());
|
||||
root.append(ObjectMask().JsonInfo());
|
||||
#endif
|
||||
|
||||
// return JsonValue
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
#ifdef USE_OPENCV
|
||||
#include "effects/Outline.h"
|
||||
#include "effects/ObjectDetection.h"
|
||||
#include "effects/ObjectMask.h"
|
||||
#include "effects/Tracker.h"
|
||||
#include "effects/Stabilizer.h"
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
/**
|
||||
* @file
|
||||
* @brief Source file for Object Mask effect class
|
||||
* @author Jonathan Thomas <jonathan@openshot.org>
|
||||
*
|
||||
* @ref License
|
||||
*/
|
||||
|
||||
// Copyright (c) 2026 OpenShot Studios, LLC
|
||||
//
|
||||
// SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
#include "effects/ObjectMask.h"
|
||||
|
||||
#include "Exceptions.h"
|
||||
#include "Frame.h"
|
||||
#include "objdetectdata.pb.h"
|
||||
|
||||
#define int64 opencv_broken_int
|
||||
#define uint64 opencv_broken_uint
|
||||
#include <opencv2/core.hpp>
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#undef uint64
|
||||
#undef int64
|
||||
|
||||
#include <QColor>
|
||||
#include <QImage>
|
||||
#include <QPainter>
|
||||
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
|
||||
using namespace openshot;
|
||||
|
||||
namespace {
|
||||
|
||||
QImage AlphaMaskImageFromRLE(const ObjectMaskFrameData& mask)
|
||||
{
|
||||
QImage image(mask.width, mask.height, QImage::Format_ARGB32_Premultiplied);
|
||||
image.fill(Qt::transparent);
|
||||
if (!mask.HasData())
|
||||
return image;
|
||||
|
||||
QRgb* data = reinterpret_cast<QRgb*>(image.bits());
|
||||
const int total = mask.width * mask.height;
|
||||
int offset = 0;
|
||||
bool value = false;
|
||||
for (uint32_t count : mask.rle) {
|
||||
const int end = std::min(total, offset + static_cast<int>(count));
|
||||
if (value)
|
||||
std::fill(data + offset, data + end, qRgba(255, 255, 255, 255));
|
||||
offset = end;
|
||||
value = !value;
|
||||
if (offset >= total)
|
||||
break;
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
cv::Mat BinaryMaskFromImage(const QImage& image)
|
||||
{
|
||||
QImage rgba = image.convertToFormat(QImage::Format_RGBA8888);
|
||||
cv::Mat binary(rgba.height(), rgba.width(), CV_8UC1, cv::Scalar(0));
|
||||
for (int y = 0; y < rgba.height(); ++y) {
|
||||
const uchar* source = rgba.constScanLine(y);
|
||||
uchar* target = binary.ptr<uchar>(y);
|
||||
for (int x = 0; x < rgba.width(); ++x)
|
||||
target[x] = source[x * 4 + 3] > 0 ? 255 : 0;
|
||||
}
|
||||
return binary;
|
||||
}
|
||||
|
||||
QImage StrokeImageFromMask(const QImage& alphaMask, int width)
|
||||
{
|
||||
QImage result(alphaMask.size(), QImage::Format_ARGB32_Premultiplied);
|
||||
result.fill(Qt::transparent);
|
||||
if (width <= 0)
|
||||
return result;
|
||||
|
||||
cv::Mat binary = BinaryMaskFromImage(alphaMask);
|
||||
cv::Mat dilated;
|
||||
const int kernelSize = std::max(1, width * 2 + 1);
|
||||
cv::Mat kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(kernelSize, kernelSize));
|
||||
cv::dilate(binary, dilated, kernel);
|
||||
cv::Mat edge = dilated - binary;
|
||||
|
||||
for (int y = 0; y < edge.rows; ++y) {
|
||||
const uchar* edgeRow = edge.ptr<uchar>(y);
|
||||
QRgb* target = reinterpret_cast<QRgb*>(result.scanLine(y));
|
||||
for (int x = 0; x < edge.cols; ++x) {
|
||||
if (edgeRow[x])
|
||||
target[x] = qRgba(255, 255, 255, 255);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ObjectMask::ObjectMask()
|
||||
: draw_mask(1.0)
|
||||
, mask_color(83, 160, 237, 255)
|
||||
, mask_alpha(120.0 / 255.0)
|
||||
, stroke_color(255, 255, 255, 255)
|
||||
, stroke_alpha(1.0)
|
||||
, stroke_width(3.0)
|
||||
{
|
||||
init_effect_details();
|
||||
}
|
||||
|
||||
void ObjectMask::init_effect_details()
|
||||
{
|
||||
InitEffectInfo();
|
||||
info.class_name = "ObjectMask";
|
||||
info.name = "Object Mask";
|
||||
info.description = "Create and draw a segmentation mask for a prompted object.";
|
||||
info.has_audio = false;
|
||||
info.has_video = true;
|
||||
info.has_tracked_object = true;
|
||||
}
|
||||
|
||||
std::shared_ptr<Frame> ObjectMask::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
|
||||
{
|
||||
std::shared_ptr<QImage> frame_image = frame->GetImage();
|
||||
if (!frame_image || frame_image->isNull() || draw_mask.GetValue(frame_number) != 1)
|
||||
return frame;
|
||||
|
||||
auto mask_it = masksData.find(frame_number);
|
||||
if (mask_it == masksData.end() || !mask_it->second.HasData())
|
||||
return frame;
|
||||
|
||||
QImage alpha_mask = AlphaMaskImageFromRLE(mask_it->second)
|
||||
.scaled(frame_image->size(), Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
|
||||
std::vector<int> mask_rgba = mask_color.GetColorRGBA(frame_number);
|
||||
QColor overlay_color(mask_rgba[0], mask_rgba[1], mask_rgba[2], 255 * mask_alpha.GetValue(frame_number));
|
||||
|
||||
QImage overlay(frame_image->size(), QImage::Format_ARGB32_Premultiplied);
|
||||
overlay.fill(Qt::transparent);
|
||||
QPainter overlay_painter(&overlay);
|
||||
overlay_painter.setCompositionMode(QPainter::CompositionMode_Source);
|
||||
overlay_painter.fillRect(overlay.rect(), overlay_color);
|
||||
overlay_painter.setCompositionMode(QPainter::CompositionMode_DestinationIn);
|
||||
overlay_painter.drawImage(0, 0, alpha_mask);
|
||||
overlay_painter.end();
|
||||
|
||||
QPainter painter(frame_image.get());
|
||||
painter.drawImage(0, 0, overlay);
|
||||
|
||||
const int strokeWidth = static_cast<int>(std::round(stroke_width.GetValue(frame_number)));
|
||||
if (strokeWidth > 0 && stroke_alpha.GetValue(frame_number) > 0.0) {
|
||||
QImage stroke_mask = StrokeImageFromMask(alpha_mask, strokeWidth);
|
||||
std::vector<int> stroke_rgba = stroke_color.GetColorRGBA(frame_number);
|
||||
QColor stroke_qcolor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha.GetValue(frame_number));
|
||||
|
||||
QImage stroke_overlay(frame_image->size(), QImage::Format_ARGB32_Premultiplied);
|
||||
stroke_overlay.fill(Qt::transparent);
|
||||
QPainter stroke_painter(&stroke_overlay);
|
||||
stroke_painter.setCompositionMode(QPainter::CompositionMode_Source);
|
||||
stroke_painter.fillRect(stroke_overlay.rect(), stroke_qcolor);
|
||||
stroke_painter.setCompositionMode(QPainter::CompositionMode_DestinationIn);
|
||||
stroke_painter.drawImage(0, 0, stroke_mask);
|
||||
stroke_painter.end();
|
||||
painter.drawImage(0, 0, stroke_overlay);
|
||||
}
|
||||
painter.end();
|
||||
|
||||
return frame;
|
||||
}
|
||||
|
||||
bool ObjectMask::LoadObjMaskData(std::string inputFilePath)
|
||||
{
|
||||
pb_objdetect::ObjDetect objMessage;
|
||||
std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
|
||||
if (!objMessage.ParseFromIstream(&input))
|
||||
return false;
|
||||
|
||||
masksData.clear();
|
||||
trackedObjects.clear();
|
||||
|
||||
auto trackedObject = std::make_shared<TrackedObjectBBox>(83, 160, 237, 255);
|
||||
trackedObject->Id(Id().empty() ? "Object Mask" : Id() + "-1");
|
||||
trackedObject->ParentClip(this->ParentClip());
|
||||
trackedObject->draw_box = Keyframe(0.0);
|
||||
trackedObject->draw_text = Keyframe(0.0);
|
||||
trackedObject->draw_mask = draw_mask;
|
||||
trackedObject->mask_alpha = mask_alpha;
|
||||
trackedObject->mask_color = mask_color;
|
||||
trackedObject->stroke = stroke_color;
|
||||
trackedObject->stroke_alpha = stroke_alpha;
|
||||
trackedObject->stroke_width = stroke_width;
|
||||
|
||||
for (int frameIndex = 0; frameIndex < objMessage.frame_size(); ++frameIndex) {
|
||||
const auto& pbFrame = objMessage.frame(frameIndex);
|
||||
if (pbFrame.bounding_box_size() <= 0)
|
||||
continue;
|
||||
|
||||
const auto& box = pbFrame.bounding_box(0);
|
||||
ObjectMaskFrameData mask;
|
||||
mask.box = BBox(box.x() + box.w() / 2.0f, box.y() + box.h() / 2.0f, box.w(), box.h(), 0.0f);
|
||||
mask.score = box.confidence();
|
||||
if (box.has_mask()) {
|
||||
mask.width = box.mask().width();
|
||||
mask.height = box.mask().height();
|
||||
for (int rleIndex = 0; rleIndex < box.mask().rle_size(); ++rleIndex)
|
||||
mask.rle.push_back(box.mask().rle(rleIndex));
|
||||
}
|
||||
masksData[pbFrame.id()] = mask;
|
||||
|
||||
trackedObject->AddBox(pbFrame.id(), mask.box.cx, mask.box.cy, mask.box.width, mask.box.height, 0.0f);
|
||||
if (mask.HasData()) {
|
||||
ObjectMaskData trackedMask;
|
||||
trackedMask.width = mask.width;
|
||||
trackedMask.height = mask.height;
|
||||
trackedMask.rle = mask.rle;
|
||||
trackedObject->AddMask(pbFrame.id(), trackedMask);
|
||||
}
|
||||
}
|
||||
|
||||
if (!masksData.empty())
|
||||
trackedObjects[1] = trackedObject;
|
||||
|
||||
google::protobuf::ShutdownProtobufLibrary();
|
||||
return true;
|
||||
}
|
||||
|
||||
std::shared_ptr<QImage> ObjectMask::TrackedObjectMask(std::shared_ptr<QImage> target_image, int64_t frame_number) const
|
||||
{
|
||||
if (!target_image || target_image->isNull())
|
||||
return {};
|
||||
|
||||
auto mask_it = masksData.find(frame_number);
|
||||
if (mask_it == masksData.end() || !mask_it->second.HasData())
|
||||
return {};
|
||||
|
||||
QImage alpha_mask = AlphaMaskImageFromRLE(mask_it->second)
|
||||
.scaled(target_image->size(), Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
|
||||
|
||||
auto mask_image = std::make_shared<QImage>(
|
||||
target_image->width(), target_image->height(), QImage::Format_RGBA8888_Premultiplied);
|
||||
mask_image->fill(QColor(0, 0, 0, 255));
|
||||
QPainter painter(mask_image.get());
|
||||
painter.drawImage(0, 0, alpha_mask);
|
||||
painter.end();
|
||||
return mask_image;
|
||||
}
|
||||
|
||||
std::string ObjectMask::Json() const
|
||||
{
|
||||
return JsonValue().toStyledString();
|
||||
}
|
||||
|
||||
Json::Value ObjectMask::JsonValue() const
|
||||
{
|
||||
Json::Value root = EffectBase::JsonValue();
|
||||
root["type"] = info.class_name;
|
||||
root["protobuf_data_path"] = protobuf_data_path;
|
||||
root["draw_mask"] = draw_mask.JsonValue();
|
||||
root["mask_color"] = mask_color.JsonValue();
|
||||
root["mask_alpha"] = mask_alpha.JsonValue();
|
||||
root["stroke_color"] = stroke_color.JsonValue();
|
||||
root["stroke_alpha"] = stroke_alpha.JsonValue();
|
||||
root["stroke_width"] = stroke_width.JsonValue();
|
||||
return root;
|
||||
}
|
||||
|
||||
void ObjectMask::SetJson(const std::string value)
|
||||
{
|
||||
try {
|
||||
SetJsonValue(openshot::stringToJson(value));
|
||||
} catch (const std::exception&) {
|
||||
throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
|
||||
}
|
||||
}
|
||||
|
||||
void ObjectMask::SetJsonValue(const Json::Value root)
|
||||
{
|
||||
EffectBase::SetJsonValue(root);
|
||||
|
||||
if (!root["draw_mask"].isNull())
|
||||
draw_mask.SetJsonValue(root["draw_mask"]);
|
||||
if (!root["mask_color"].isNull())
|
||||
mask_color.SetJsonValue(root["mask_color"]);
|
||||
if (!root["mask_alpha"].isNull())
|
||||
mask_alpha.SetJsonValue(root["mask_alpha"]);
|
||||
if (!root["stroke_color"].isNull())
|
||||
stroke_color.SetJsonValue(root["stroke_color"]);
|
||||
if (!root["stroke"].isNull())
|
||||
stroke_color.SetJsonValue(root["stroke"]);
|
||||
if (!root["stroke_alpha"].isNull())
|
||||
stroke_alpha.SetJsonValue(root["stroke_alpha"]);
|
||||
if (!root["stroke_width"].isNull())
|
||||
stroke_width.SetJsonValue(root["stroke_width"]);
|
||||
|
||||
if (!root["protobuf_data_path"].isNull()) {
|
||||
std::string new_path = root["protobuf_data_path"].asString();
|
||||
if (protobuf_data_path != new_path || masksData.empty()) {
|
||||
protobuf_data_path = new_path;
|
||||
if (!LoadObjMaskData(protobuf_data_path))
|
||||
throw InvalidFile("Invalid object mask protobuf data path", "");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string ObjectMask::PropertiesJSON(int64_t requested_frame) const
|
||||
{
|
||||
Json::Value root = BasePropertiesJSON(requested_frame);
|
||||
root["protobuf_data_path"] = add_property_json("Object Mask Data", 0.0, "string", protobuf_data_path, NULL, -1, -1, false, requested_frame);
|
||||
|
||||
root["draw_mask"] = add_property_json("Draw Mask", draw_mask.GetValue(requested_frame), "int", "", &draw_mask, 0, 1, false, requested_frame);
|
||||
root["draw_mask"]["choices"].append(add_property_choice_json("Yes", true, draw_mask.GetValue(requested_frame)));
|
||||
root["draw_mask"]["choices"].append(add_property_choice_json("No", false, draw_mask.GetValue(requested_frame)));
|
||||
|
||||
root["mask_color"] = add_property_json("Mask Color", 0.0, "color", "", NULL, 0, 255, false, requested_frame);
|
||||
root["mask_color"]["red"] = add_property_json("Red", mask_color.red.GetValue(requested_frame), "float", "", &mask_color.red, 0, 255, false, requested_frame);
|
||||
root["mask_color"]["blue"] = add_property_json("Blue", mask_color.blue.GetValue(requested_frame), "float", "", &mask_color.blue, 0, 255, false, requested_frame);
|
||||
root["mask_color"]["green"] = add_property_json("Green", mask_color.green.GetValue(requested_frame), "float", "", &mask_color.green, 0, 255, false, requested_frame);
|
||||
root["mask_alpha"] = add_property_json("Mask Alpha", mask_alpha.GetValue(requested_frame), "float", "", &mask_alpha, 0.0, 1.0, false, requested_frame);
|
||||
|
||||
root["stroke_color"] = add_property_json("Stroke Color", 0.0, "color", "", NULL, 0, 255, false, requested_frame);
|
||||
root["stroke_color"]["red"] = add_property_json("Red", stroke_color.red.GetValue(requested_frame), "float", "", &stroke_color.red, 0, 255, false, requested_frame);
|
||||
root["stroke_color"]["blue"] = add_property_json("Blue", stroke_color.blue.GetValue(requested_frame), "float", "", &stroke_color.blue, 0, 255, false, requested_frame);
|
||||
root["stroke_color"]["green"] = add_property_json("Green", stroke_color.green.GetValue(requested_frame), "float", "", &stroke_color.green, 0, 255, false, requested_frame);
|
||||
root["stroke_alpha"] = add_property_json("Stroke Alpha", stroke_alpha.GetValue(requested_frame), "float", "", &stroke_alpha, 0.0, 1.0, false, requested_frame);
|
||||
root["stroke_width"] = add_property_json("Stroke Width", stroke_width.GetValue(requested_frame), "int", "", &stroke_width, 0, 50, false, requested_frame);
|
||||
|
||||
return root.toStyledString();
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* @file
|
||||
* @brief Header file for Object Mask effect class
|
||||
* @author Jonathan Thomas <jonathan@openshot.org>
|
||||
*
|
||||
* @ref License
|
||||
*/
|
||||
|
||||
// Copyright (c) 2026 OpenShot Studios, LLC
|
||||
//
|
||||
// SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
#ifndef OPENSHOT_OBJECT_MASK_EFFECT_H
|
||||
#define OPENSHOT_OBJECT_MASK_EFFECT_H
|
||||
|
||||
#include "Color.h"
|
||||
#include "EffectBase.h"
|
||||
#include "KeyFrame.h"
|
||||
#include "TrackedObjectBBox.h"
|
||||
|
||||
#include <map>
|
||||
|
||||
namespace openshot
|
||||
{
|
||||
class Frame;
|
||||
|
||||
struct ObjectMaskFrameData {
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
std::vector<uint32_t> rle;
|
||||
BBox box;
|
||||
float score = 0.0f;
|
||||
|
||||
bool HasData() const { return width > 0 && height > 0 && !rle.empty(); }
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Display and expose a preprocessed segmentation mask for an object.
|
||||
*/
|
||||
class ObjectMask : public EffectBase
|
||||
{
|
||||
private:
|
||||
std::string protobuf_data_path;
|
||||
std::map<size_t, ObjectMaskFrameData> masksData;
|
||||
|
||||
void init_effect_details();
|
||||
|
||||
public:
|
||||
Keyframe draw_mask;
|
||||
Color mask_color;
|
||||
Keyframe mask_alpha;
|
||||
Color stroke_color;
|
||||
Keyframe stroke_alpha;
|
||||
Keyframe stroke_width;
|
||||
|
||||
ObjectMask();
|
||||
|
||||
std::shared_ptr<Frame> GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number) override;
|
||||
std::shared_ptr<openshot::Frame> GetFrame(int64_t frame_number) override { return GetFrame(std::make_shared<Frame>(), frame_number); }
|
||||
|
||||
bool LoadObjMaskData(std::string inputFilePath);
|
||||
std::shared_ptr<QImage> TrackedObjectMask(std::shared_ptr<QImage> target_image, int64_t frame_number) const override;
|
||||
|
||||
std::string Json() const override;
|
||||
void SetJson(const std::string value) override;
|
||||
Json::Value JsonValue() const override;
|
||||
void SetJsonValue(const Json::Value root) override;
|
||||
|
||||
std::string PropertiesJSON(int64_t requested_frame) const override;
|
||||
};
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -91,6 +91,7 @@ if($CACHE{HAVE_OPENCV})
|
||||
CVTracker
|
||||
CVStabilizer
|
||||
CVOutline
|
||||
ObjectMask
|
||||
# CVObjectDetection
|
||||
)
|
||||
endif()
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* @file
|
||||
* @brief Unit tests for Object Mask effect
|
||||
* @author Jonathan Thomas <jonathan@openshot.org>
|
||||
*
|
||||
* @ref License
|
||||
*/
|
||||
|
||||
// Copyright (c) 2026 OpenShot Studios, LLC
|
||||
//
|
||||
// SPDX-License-Identifier: LGPL-3.0-or-later
|
||||
|
||||
#include "openshot_catch.h"
|
||||
|
||||
#include "EffectInfo.h"
|
||||
#include "Frame.h"
|
||||
#include "Json.h"
|
||||
#include "effects/ObjectMask.h"
|
||||
|
||||
#include <QColor>
|
||||
#include <QImage>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <unistd.h>
|
||||
|
||||
using namespace openshot;
|
||||
|
||||
static std::shared_ptr<Frame> make_object_mask_frame(int64_t number, int width, int height) {
|
||||
auto frame = std::make_shared<Frame>(number, width, height, "#000000");
|
||||
frame->GetImage()->fill(QColor(64, 64, 64, 255));
|
||||
return frame;
|
||||
}
|
||||
|
||||
static std::string temp_object_mask_path() {
|
||||
char path[] = "/tmp/libopenshot_object_mask_XXXXXX";
|
||||
int fd = mkstemp(path);
|
||||
REQUIRE(fd != -1);
|
||||
close(fd);
|
||||
std::remove(path);
|
||||
return std::string(path) + ".data";
|
||||
}
|
||||
|
||||
static void append_varint(std::string& output, uint64_t value) {
|
||||
while (value >= 0x80) {
|
||||
output.push_back(static_cast<char>((value & 0x7f) | 0x80));
|
||||
value >>= 7;
|
||||
}
|
||||
output.push_back(static_cast<char>(value));
|
||||
}
|
||||
|
||||
static void append_fixed32_float(std::string& output, float value) {
|
||||
uint32_t bits;
|
||||
std::memcpy(&bits, &value, sizeof(float));
|
||||
for (int i = 0; i < 4; ++i)
|
||||
output.push_back(static_cast<char>((bits >> (8 * i)) & 0xff));
|
||||
}
|
||||
|
||||
static void append_length_delimited(std::string& output, uint32_t field_number, const std::string& value) {
|
||||
append_varint(output, (field_number << 3) | 2);
|
||||
append_varint(output, static_cast<uint32_t>(value.size()));
|
||||
output.append(value);
|
||||
}
|
||||
|
||||
static std::string create_object_mask_data() {
|
||||
const std::string path = temp_object_mask_path();
|
||||
|
||||
std::string mask;
|
||||
append_varint(mask, 8);
|
||||
append_varint(mask, 4);
|
||||
append_varint(mask, 16);
|
||||
append_varint(mask, 4);
|
||||
for (uint32_t count : {0u, 6u, 10u}) {
|
||||
append_varint(mask, 24);
|
||||
append_varint(mask, count);
|
||||
}
|
||||
|
||||
std::string box;
|
||||
append_varint(box, 13);
|
||||
append_fixed32_float(box, 0.0f);
|
||||
append_varint(box, 21);
|
||||
append_fixed32_float(box, 0.0f);
|
||||
append_varint(box, 29);
|
||||
append_fixed32_float(box, 1.0f);
|
||||
append_varint(box, 37);
|
||||
append_fixed32_float(box, 1.0f);
|
||||
append_varint(box, 40);
|
||||
append_varint(box, 0);
|
||||
append_varint(box, 53);
|
||||
append_fixed32_float(box, 0.95f);
|
||||
append_varint(box, 56);
|
||||
append_varint(box, 1);
|
||||
append_length_delimited(box, 8, mask);
|
||||
|
||||
std::string frame;
|
||||
append_varint(frame, 8);
|
||||
append_varint(frame, 1);
|
||||
append_length_delimited(frame, 2, box);
|
||||
|
||||
std::string data;
|
||||
append_length_delimited(data, 1, frame);
|
||||
append_length_delimited(data, 3, "object mask");
|
||||
|
||||
std::ofstream output(path, std::ios::out | std::ios::binary);
|
||||
output.write(data.data(), static_cast<std::streamsize>(data.size()));
|
||||
REQUIRE(output.good());
|
||||
return path;
|
||||
}
|
||||
|
||||
TEST_CASE("ObjectMask effect is registered", "[effect][object_mask]") {
|
||||
std::unique_ptr<EffectBase> effect(EffectInfo().CreateEffect("ObjectMask"));
|
||||
REQUIRE(effect != nullptr);
|
||||
CHECK(effect->info.name == "Object Mask");
|
||||
CHECK(effect->info.has_video);
|
||||
CHECK(effect->info.has_tracked_object);
|
||||
}
|
||||
|
||||
TEST_CASE("ObjectMask loads protobuf masks and exposes style controls", "[effect][object_mask]") {
|
||||
const std::string protobuf_path = create_object_mask_data();
|
||||
|
||||
ObjectMask effect;
|
||||
Json::Value config;
|
||||
config["protobuf_data_path"] = protobuf_path;
|
||||
config["mask_alpha"] = Keyframe(0.5).JsonValue();
|
||||
config["stroke_width"] = Keyframe(2.0).JsonValue();
|
||||
effect.SetJsonValue(config);
|
||||
|
||||
Json::Value properties = stringToJson(effect.PropertiesJSON(1));
|
||||
CHECK(properties["draw_mask"]["name"].asString() == "Draw Mask");
|
||||
CHECK(properties["mask_color"]["name"].asString() == "Mask Color");
|
||||
CHECK(properties["mask_alpha"]["value"].asDouble() == Approx(0.5));
|
||||
CHECK(properties["stroke_color"]["name"].asString() == "Stroke Color");
|
||||
CHECK(properties["stroke_alpha"]["name"].asString() == "Stroke Alpha");
|
||||
CHECK(properties["stroke_width"]["value"].asDouble() == Approx(2.0));
|
||||
|
||||
auto generated_mask = effect.TrackedObjectMask(std::make_shared<QImage>(4, 4, QImage::Format_RGBA8888_Premultiplied), 1);
|
||||
REQUIRE(generated_mask != nullptr);
|
||||
CHECK(generated_mask->pixelColor(0, 0).red() == 255);
|
||||
CHECK(generated_mask->pixelColor(3, 3).red() == 0);
|
||||
|
||||
Json::Value no_stroke;
|
||||
no_stroke["stroke_width"] = Keyframe(0.0).JsonValue();
|
||||
effect.SetJsonValue(no_stroke);
|
||||
|
||||
auto frame = make_object_mask_frame(1, 4, 4);
|
||||
auto output = effect.GetFrame(frame, 1)->GetImage();
|
||||
CHECK(output->pixelColor(0, 0) != QColor(64, 64, 64, 255));
|
||||
CHECK(output->pixelColor(3, 3) == QColor(64, 64, 64, 255));
|
||||
|
||||
std::remove(protobuf_path.c_str());
|
||||
}
|
||||
Reference in New Issue
Block a user