Migrate object detection to YOLOv5 ONNX

- Replace YOLOv3 Darknet loading with YOLOv5 ONNX model loading and validation.
- Parse YOLOv5 outputs with top class candidates for smoother labels.
- Improve SORT tracking with class-score smoothing and stricter geometry gates.
- Prevent active tracks from hopping to adjacent objects or tiny nested detections.
- Keep object ids stable through protobuf load/save and add an object-detection debug example.
- Add class-based default colors and matching border/background defaults.
- Add All Objects pseudo-selection support for tracked-object properties.
- Honor tracked-object corner radius when using tracker/object detector masks.
- Add regression tests for tracking stability, all-object styling, masks, and keyframes.

examples/Example_opencv.cpp

@@ -261,24 +261,21 @@ string objectDetectionJson(bool onlyProtoPath){
     string protobufDataPath = "example_object_detection.data";
     // Define processing device
     string processingDevice = "GPU";
-    // Set path to model configuration file
-    string modelConfiguration = "yolov3.cfg";
-    // Set path to model weights
-    string modelWeights = "yolov3.weights";
+    // Set path to YOLOv5 ONNX model
+    string model = "Yolo5/yolov5s.onnx";
     // Set path to class names file
-    string classesFile = "obj.names";
+    string classesFile = "Yolo5/obj.names";
 
     // Construct all the composition of the JSON string
     string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath);
     string processing_device = jsonFormat("processing_device", processingDevice);
-    string model_configuration = jsonFormat("model_configuration", modelConfiguration);
-    string model_weights = jsonFormat("model_weights", modelWeights);
+    string model_path = jsonFormat("model", model);
     string classes_file = jsonFormat("classes_file", classesFile);
 
     // Return only the the protobuf path in JSON format
     if(onlyProtoPath)
         return "{" + protobuf_data_path + "}";
     else
-        return "{" + protobuf_data_path + "," + processing_device + "," + model_configuration + ","
-                + model_weights + "," + classes_file + "}";
+        return "{" + protobuf_data_path + "," + processing_device + "," + model_path + ","
+                + classes_file + "}";
 }

src/CVObjectDetection.cpp

@@ -14,6 +14,7 @@
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <algorithm>
 
 #include "CVObjectDetection.h"
 #include "Exceptions.h"
@@ -26,10 +27,10 @@ using namespace openshot;
 using google::protobuf::util::TimeUtil;
 
 CVObjectDetection::CVObjectDetection(std::string processInfoJson, ProcessingController &processingController)
-: processingController(&processingController), processingDevice("CPU"){
-    SetJson(processInfoJson);
-    confThreshold = 0.5;
+: processingController(&processingController), processingDevice("CPU"), inpWidth(640), inpHeight(640){
+    confThreshold = 0.25;
     nmsThreshold = 0.1;
+    SetJson(processInfoJson);
 }
 
 void CVObjectDetection::setProcessingDevice(){
@@ -56,15 +57,48 @@ void CVObjectDetection::detectObjectsClip(openshot::Clip &video, size_t _start,
 
     processingController->SetError(false, "");
 
+    if(modelPath.empty()) {
+        processingController->SetError(true, "Missing path to YOLOv5 ONNX model file");
+        error = true;
+        return;
+    }
+    if(classesFile.empty()) {
+        processingController->SetError(true, "Missing path to class name file");
+        error = true;
+        return;
+    }
+
+    std::ifstream model_file(modelPath);
+    if(!model_file.good()){
+        processingController->SetError(true, "Incorrect path to YOLOv5 ONNX model file");
+        error = true;
+        return;
+    }
+    std::ifstream classes_file(classesFile);
+    if(!classes_file.good()){
+        processingController->SetError(true, "Incorrect path to class name file");
+        error = true;
+        return;
+    }
+
     // Load names of classes
-    std::ifstream ifs(classesFile.c_str());
+    classNames.clear();
     std::string line;
-    while (std::getline(ifs, line)) classNames.push_back(line);
+    while (std::getline(classes_file, line)) classNames.push_back(line);
 
     // Load the network
-    if(classesFile == "" || modelConfiguration == "" || modelWeights == "")
+    try {
+        net = cv::dnn::readNetFromONNX(modelPath);
+    } catch (const cv::Exception& e) {
+        std::string error_text = std::string("Failed to load model: ") + e.what();
+        if (error_text.find("Unsupported data type: FLOAT16") != std::string::npos) {
+            error_text = "Failed to load ONNX model: FLOAT16 is not supported by this OpenCV build. "
+                         "Please use an FP32 ONNX model.";
+        }
+        processingController->SetError(true, error_text);
+        error = true;
         return;
-    net = cv::dnn::readNetFromDarknet(modelConfiguration, modelWeights);
+    }
     setProcessingDevice();
 
     size_t frame_number;
@@ -99,17 +133,19 @@ void CVObjectDetection::DetectObjects(const cv::Mat &frame, size_t frameId){
     cv::Mat blob;
 
     // Create a 4D blob from the frame.
-    int inpWidth, inpHeight;
-    inpWidth = inpHeight = 416;
-
     cv::dnn::blobFromImage(frame, blob, 1/255.0, cv::Size(inpWidth, inpHeight), cv::Scalar(0,0,0), true, false);
 
-    //Sets the input to the network
-    net.setInput(blob);
-
-    // Runs the forward pass to get output of the output layers
     std::vector<cv::Mat> outs;
-    net.forward(outs, getOutputsNames(net));
+    try {
+        // Sets the input to the network
+        net.setInput(blob);
+        // Runs the forward pass to get output of the output layers
+        net.forward(outs, getOutputsNames(net));
+    } catch (const cv::Exception& e) {
+        processingController->SetError(true, std::string("Object detection inference failed: ") + e.what());
+        error = true;
+        return;
+    }
 
     // Remove the bounding boxes with low confidence
     postprocess(frame.size(), outs, frameId);
@@ -123,33 +159,71 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
     std::vector<int> classIds;
     std::vector<float> confidences;
     std::vector<cv::Rect> boxes;
+    std::vector<std::vector<ClassScore>> detectionClassScores;
     std::vector<int> objectIds;
+    const int maxClassCandidates = 5;
+
+    for (size_t i = 0; i < outs.size(); ++i) {
+        cv::Mat det = outs[i];
+
+        // YOLOv5 ONNX output is usually [1, num_boxes, num_classes + 5].
+        if (det.dims == 3) {
+            det = det.reshape(1, det.size[1]);
+        }
+        if (det.dims != 2 || det.cols < 6) {
+            continue;
+        }
+
+        const float xFactor = static_cast<float>(frameDims.width) / static_cast<float>(inpWidth);
+        const float yFactor = static_cast<float>(frameDims.height) / static_cast<float>(inpHeight);
+
+        float* data = reinterpret_cast<float*>(det.data);
+        for (int j = 0; j < det.rows; ++j, data += det.cols) {
+            std::vector<ClassScore> rowClassScores;
+            rowClassScores.reserve(maxClassCandidates);
+            for (int classIndex = 5; classIndex < det.cols; ++classIndex) {
+                const float classConfidence = data[classIndex] * data[4];
+                if (rowClassScores.size() < static_cast<size_t>(maxClassCandidates)) {
+                    rowClassScores.emplace_back(classIndex - 5, classConfidence);
+                    std::sort(rowClassScores.begin(), rowClassScores.end(),
+                        [](const ClassScore& a, const ClassScore& b) { return a.score > b.score; });
+                } else if (classConfidence > rowClassScores.back().score) {
+                    rowClassScores.back() = ClassScore(classIndex - 5, classConfidence);
+                    std::sort(rowClassScores.begin(), rowClassScores.end(),
+                        [](const ClassScore& a, const ClassScore& b) { return a.score > b.score; });
+                }
+            }
+            if (rowClassScores.empty()) {
+                continue;
+            }
+
+            float confidence = rowClassScores.front().score;
+
+            if (confidence > confThreshold) {
+                int centerX = 0;
+                int centerY = 0;
+                int width = 0;
+                int height = 0;
+
+                if (data[0] > 1.0f || data[1] > 1.0f || data[2] > 1.0f || data[3] > 1.0f) {
+                    centerX = static_cast<int>(data[0] * xFactor);
+                    centerY = static_cast<int>(data[1] * yFactor);
+                    width = static_cast<int>(data[2] * xFactor);
+                    height = static_cast<int>(data[3] * yFactor);
+                } else {
+                    centerX = static_cast<int>(data[0] * frameDims.width);
+                    centerY = static_cast<int>(data[1] * frameDims.height);
+                    width = static_cast<int>(data[2] * frameDims.width);
+                    height = static_cast<int>(data[3] * frameDims.height);
+                }
 
-    for (size_t i = 0; i < outs.size(); ++i)
-    {
-        // Scan through all the bounding boxes output from the network and keep only the
-        // ones with high confidence scores. Assign the box's class label as the class
-        // with the highest score for the box.
-        float* data = (float*)outs[i].data;
-        for (int j = 0; j < outs[i].rows; ++j, data += outs[i].cols)
-        {
-            cv::Mat scores = outs[i].row(j).colRange(5, outs[i].cols);
-            cv::Point classIdPoint;
-            double confidence;
-            // Get the value and location of the maximum score
-            cv::minMaxLoc(scores, 0, &confidence, 0, &classIdPoint);
-            if (confidence > confThreshold)
-            {
-                int centerX = (int)(data[0] * frameDims.width);
-                int centerY = (int)(data[1] * frameDims.height);
-                int width = (int)(data[2] * frameDims.width);
-                int height = (int)(data[3] * frameDims.height);
                 int left = centerX - width / 2;
                 int top = centerY - height / 2;
 
-                classIds.push_back(classIdPoint.x);
-                confidences.push_back((float)confidence);
+                classIds.push_back(rowClassScores.front().classId);
+                confidences.push_back(confidence);
                 boxes.push_back(cv::Rect(left, top, width, height));
+                detectionClassScores.push_back(rowClassScores);
             }
         }
     }
@@ -161,9 +235,16 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
 
     // Pass boxes to SORT algorithm
     std::vector<cv::Rect> sortBoxes;
-    for(auto box : boxes)
-        sortBoxes.push_back(box);
-    sort.update(sortBoxes, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2)), confidences, classIds);
+    std::vector<float> sortConfidences;
+    std::vector<int> sortClassIds;
+    std::vector<std::vector<ClassScore>> sortClassScores;
+    for(auto index : indices) {
+        sortBoxes.push_back(boxes[index]);
+        sortConfidences.push_back(confidences[index]);
+        sortClassIds.push_back(classIds[index]);
+        sortClassScores.push_back(detectionClassScores[index]);
+    }
+    sort.update(sortBoxes, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2)), sortConfidences, sortClassIds, sortClassScores);
 
     // Clear data vectors
     boxes.clear(); confidences.clear(); classIds.clear(); objectIds.clear();
@@ -180,8 +261,8 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
     // Remove boxes based on controids distance
     for(uint i = 0; i<boxes.size(); i++){
         for(uint j = i+1; j<boxes.size(); j++){
-            int xc_1 = boxes[i].x + (int)(boxes[i].width/2), yc_1 = boxes[i].y + (int)(boxes[i].width/2);
-            int xc_2 = boxes[j].x + (int)(boxes[j].width/2), yc_2 = boxes[j].y + (int)(boxes[j].width/2);
+            int xc_1 = boxes[i].x + (int)(boxes[i].width/2), yc_1 = boxes[i].y + (int)(boxes[i].height/2);
+            int xc_2 = boxes[j].x + (int)(boxes[j].width/2), yc_2 = boxes[j].y + (int)(boxes[j].height/2);
 
             if(fabs(xc_1 - xc_2) < 10 && fabs(yc_1 - yc_2) < 10){
                 if(classIds[i] == classIds[j]){
@@ -272,8 +353,6 @@ bool CVObjectDetection::iou(cv::Rect pred_box, cv::Rect sort_box){
 // Get the names of the output layers
 std::vector<cv::String> CVObjectDetection::getOutputsNames(const cv::dnn::Net& net)
 {
-    static std::vector<cv::String> names;
-
     //Get the indices of the output layers, i.e. the layers with unconnected outputs
     std::vector<int> outLayers = net.getUnconnectedOutLayers();
 
@@ -281,6 +360,7 @@ std::vector<cv::String> CVObjectDetection::getOutputsNames(const cv::dnn::Net& n
     std::vector<cv::String> layersNames = net.getLayerNames();
 
     // Get the names of the output layers in names
+    std::vector<cv::String> names;
     names.resize(outLayers.size());
     for (size_t i = 0; i < outLayers.size(); ++i)
         names[i] = layersNames[outLayers[i] - 1];
@@ -299,6 +379,11 @@ CVDetectionData CVObjectDetection::GetDetectionData(size_t frameId){
 }
 
 bool CVObjectDetection::SaveObjDetectedData(){
+    if(protobuf_data_path.empty()) {
+        cerr << "Missing path to object detection protobuf data file." << endl;
+        return false;
+    }
+
     // Create tracker message
     pb_objdetect::ObjDetect objMessage;
 
@@ -311,7 +396,6 @@ bool CVObjectDetection::SaveObjDetectedData(){
     // Iterate over all frames data and save in protobuf message
     for(std::map<size_t,CVDetectionData>::iterator it=detectionsData.begin(); it!=detectionsData.end(); ++it){
         CVDetectionData dData = it->second;
-        pb_objdetect::Frame* pbFrameData;
         AddFrameDataToProto(objMessage.add_frame(), dData);
     }
 
@@ -380,37 +464,49 @@ void CVObjectDetection::SetJsonValue(const Json::Value root) {
 	if (!root["protobuf_data_path"].isNull()){
 		protobuf_data_path = (root["protobuf_data_path"].asString());
 	}
+
     if (!root["processing-device"].isNull()){
 		processingDevice = (root["processing-device"].asString());
 	}
-    if (!root["model-config"].isNull()){
-		modelConfiguration = (root["model-config"].asString());
-        std::ifstream infile(modelConfiguration);
-        if(!infile.good()){
-            processingController->SetError(true, "Incorrect path to model config file");
-            error = true;
-        }
-
-	}
-    if (!root["model-weights"].isNull()){
-		modelWeights= (root["model-weights"].asString());
-        std::ifstream infile(modelWeights);
-        if(!infile.good()){
-            processingController->SetError(true, "Incorrect path to model weight file");
-            error = true;
-        }
-
+    if (!root["processing_device"].isNull()){
+		processingDevice = (root["processing_device"].asString());
 	}
     if (!root["class-names"].isNull()){
 		classesFile = (root["class-names"].asString());
-
-        std::ifstream infile(classesFile);
-        if(!infile.good()){
-            processingController->SetError(true, "Incorrect path to class name file");
-            error = true;
-        }
-
 	}
+    if (!root["classes_file"].isNull()){
+		classesFile = (root["classes_file"].asString());
+	}
+    if (!root["model"].isNull()){
+        modelPath = (root["model"].asString());
+    }
+    if (!root["model_path"].isNull()){
+        modelPath = (root["model_path"].asString());
+    }
+    if (!root["input-width"].isNull()){
+        inpWidth = root["input-width"].asInt();
+    }
+    if (!root["input_width"].isNull()){
+        inpWidth = root["input_width"].asInt();
+    }
+    if (!root["input-height"].isNull()){
+        inpHeight = root["input-height"].asInt();
+    }
+    if (!root["input_height"].isNull()){
+        inpHeight = root["input_height"].asInt();
+    }
+    if (!root["confidence-threshold"].isNull()){
+        confThreshold = root["confidence-threshold"].asFloat();
+    }
+    if (!root["confidence_threshold"].isNull()){
+        confThreshold = root["confidence_threshold"].asFloat();
+    }
+    if (!root["nms-threshold"].isNull()){
+        nmsThreshold = root["nms-threshold"].asFloat();
+    }
+    if (!root["nms_threshold"].isNull()){
+        nmsThreshold = root["nms_threshold"].asFloat();
+    }
 }
 
 /*
@@ -421,6 +517,11 @@ void CVObjectDetection::SetJsonValue(const Json::Value root) {
 
 // Load protobuf data file
 bool CVObjectDetection::_LoadObjDetectdData(){
+    if(protobuf_data_path.empty()) {
+        cerr << "Missing path to object detection protobuf data file." << endl;
+        return false;
+    }
+
     // Create tracker message
     pb_objdetect::ObjDetect objMessage;
 
@@ -472,6 +573,7 @@ bool CVObjectDetection::_LoadObjDetectdData(){
 
             // Push back data into vectors
             boxes.push_back(box); classIds.push_back(classId); confidences.push_back(confidence);
+            objectIds.push_back(objectId);
         }
 
         // Assign data to object detector map

src/CVObjectDetection.h

@@ -59,7 +59,7 @@ namespace openshot
     /**
      * @brief This class runs trought a clip to detect objects and returns the bounding boxes and its properties.
      *
-     * Object detection is performed using YoloV3 model with OpenCV DNN module
+     * Object detection is performed using a YOLOv5 ONNX model with OpenCV DNN module.
      */
     class CVObjectDetection{
 
@@ -70,10 +70,11 @@ namespace openshot
         float confThreshold, nmsThreshold;
 
         std::string classesFile;
-        std::string modelConfiguration;
-        std::string modelWeights;
+        std::string modelPath;
         std::string processingDevice;
         std::string protobuf_data_path;
+        int inpWidth;
+        int inpHeight;
 
         SortTracker sort;
 

src/EffectBase.cpp

@@ -544,7 +544,7 @@ std::shared_ptr<QImage> EffectBase::TrackedObjectMask(std::shared_ptr<QImage> ta
 	mask_image->fill(QColor(0, 0, 0, 255));
 
 	QPainter painter(mask_image.get());
-	painter.setRenderHint(QPainter::Antialiasing, false);
+	painter.setRenderHint(QPainter::Antialiasing, true);
 	painter.setPen(Qt::NoPen);
 	painter.setBrush(QBrush(QColor(255, 255, 255, 255)));
 
@@ -564,16 +564,17 @@ std::shared_ptr<QImage> EffectBase::TrackedObjectMask(std::shared_ptr<QImage> ta
 		const double y = (box.cy - box.height / 2.0) * target_image->height();
 		const double w = box.width * target_image->width();
 		const double h = box.height * target_image->height();
+		const double corner = bbox->background_corner.GetValue(frame_number);
 		QRectF rect(x, y, w, h);
 
 		if (std::abs(box.angle) > 0.0001f) {
 			painter.save();
 			painter.translate(rect.center());
 			painter.rotate(box.angle);
-			painter.drawRect(QRectF(-w / 2.0, -h / 2.0, w, h));
+			painter.drawRoundedRect(QRectF(-w / 2.0, -h / 2.0, w, h), corner, corner);
 			painter.restore();
 		} else {
-			painter.drawRect(rect);
+			painter.drawRoundedRect(rect, corner, corner);
 		}
 		drew_any_box = true;
 	}

src/TrackedObjectBBox.cpp

@@ -39,7 +39,7 @@ TrackedObjectBBox::TrackedObjectBBox(int Red, int Green, int Blue, int Alfa)
 	  background_alpha(0.0), background_corner(12),
 	  stroke_width(2) , stroke_alpha(0.7),
 	  stroke(Red, Green, Blue, Alfa),
-	  background(0, 0, 255, Alfa)
+	  background(Red, Green, Blue, Alfa)
 {
 	this->TimeScale = 1.0;
 }

src/effects/ObjectDetection.cpp

@@ -32,6 +32,22 @@ using namespace std;
 using namespace openshot;
 
 namespace {
+bool is_all_objects_key(const std::string& name)
+{
+    const QString normalized = QString::fromStdString(name).trimmed().toLower();
+    return normalized == "all" || normalized == "*" || normalized == "-1";
+}
+
+std::shared_ptr<TrackedObjectBBox> make_all_objects_properties(
+	const std::shared_ptr<TrackedObjectBase>& source)
+{
+	auto properties = std::make_shared<TrackedObjectBBox>();
+	if (source)
+		properties->SetJsonValue(source->JsonValue());
+	properties->Id("All Objects");
+	return properties;
+}
+
 cv::Scalar default_class_color(const std::string& class_name, int index)
 {
     const QString normalized = QString::fromStdString(class_name).trimmed().toLower();
@@ -249,6 +265,7 @@ bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath)
                     /*alpha=*/0
                 );
                 tmpObj.stroke_alpha = Keyframe(1.0);
+                tmpObj.background_alpha = Keyframe(0.15);
                 tmpObj.AddBox(frameId, x + w/2, y + h/2, w, h, 0.0);
 
 				auto ptr = std::make_shared<TrackedObjectBBox>(tmpObj);
@@ -271,9 +288,9 @@ bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath)
 
     google::protobuf::ShutdownProtobufLibrary();
 
-    // Finally, pick a default selectedObjectIndex if we have any
+    // Default to the pseudo-selection that edits every tracked object.
     if (!trackedObjects.empty()) {
-        selectedObjectIndex = trackedObjects.begin()->first;
+        selectedObjectIndex = -1;
     }
 
     return true;
@@ -294,6 +311,12 @@ std::string ObjectDetection::GetVisibleObjects(int64_t frame_number) const{
 	}
 	DetectionData detections = detectionsData.at(frame_number);
 
+	if (!trackedObjects.empty()) {
+		root["visible_objects_index"].append(-1);
+		root["visible_objects_id"].append("All Objects");
+		root["visible_class_names"].append("All Objects");
+	}
+
 	// Iterate through the tracked objects
 	for(int i = 0; i<detections.boxes.size(); i++){
 		// Does not show boxes with confidence below the threshold
@@ -348,7 +371,7 @@ std::shared_ptr<QImage> ObjectDetection::TrackedObjectMask(std::shared_ptr<QImag
 	mask_image->fill(QColor(0, 0, 0, 255));
 
 	QPainter painter(mask_image.get());
-	painter.setRenderHint(QPainter::Antialiasing, false);
+	painter.setRenderHint(QPainter::Antialiasing, true);
 	painter.setPen(Qt::NoPen);
 	painter.setBrush(QBrush(QColor(255, 255, 255, 255)));
 
@@ -387,7 +410,8 @@ std::shared_ptr<QImage> ObjectDetection::TrackedObjectMask(std::shared_ptr<QImag
 		const double y = (box.cy - box.height / 2.0) * target_image->height();
 		const double w = box.width * target_image->width();
 		const double h = box.height * target_image->height();
-		painter.drawRect(QRectF(x, y, w, h));
+		const double corner = tracked_object->background_corner.GetValue(frame_number);
+		painter.drawRoundedRect(QRectF(x, y, w, h), corner, corner);
 		drew_any_box = true;
 	}
 
@@ -457,6 +481,7 @@ void ObjectDetection::SetJsonValue(const Json::Value root)
 		std::string new_path = root["protobuf_data_path"].asString();
 		if (protobuf_data_path != new_path || trackedObjects.empty()) {
 			protobuf_data_path = new_path;
+			allObjectsProperties.reset();
 			if (!LoadObjDetectdData(protobuf_data_path)) {
 				throw InvalidFile("Invalid protobuf data path", "");
 			}
@@ -491,6 +516,25 @@ void ObjectDetection::SetJsonValue(const Json::Value root)
 		const auto memberNames = root["objects"].getMemberNames();
 		for (const auto& name : memberNames)
 		{
+			if (is_all_objects_key(name)) {
+				if (!allObjectsProperties) {
+					std::shared_ptr<TrackedObjectBase> firstObject;
+					if (!trackedObjects.empty())
+						firstObject = trackedObjects.begin()->second;
+					allObjectsProperties = make_all_objects_properties(firstObject);
+				}
+				allObjectsProperties->SetJsonValue(root["objects"][name]);
+				for (auto& trackedObject : trackedObjects) {
+					if (trackedObject.second)
+						trackedObject.second->SetJsonValue(root["objects"][name]);
+				}
+			}
+		}
+
+		for (const auto& name : memberNames)
+		{
+			if (is_all_objects_key(name))
+				continue;
 			// Determine the numeric index of this object
 			int index = -1;
 			bool numeric_key = std::all_of(name.begin(), name.end(), ::isdigit);
@@ -534,7 +578,19 @@ std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const {
 	Json::Value root = BasePropertiesJSON(requested_frame);
 
 	Json::Value objects;
-	if(trackedObjects.count(selectedObjectIndex) != 0){
+	if(selectedObjectIndex == -1 && !trackedObjects.empty()){
+		auto selectedObject = allObjectsProperties ? allObjectsProperties : trackedObjects.begin()->second;
+		if (selectedObject){
+			Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
+			trackedObjectJSON["box_id"]["memo"] = "All Objects";
+			trackedObjectJSON.removeMember("x1");
+			trackedObjectJSON.removeMember("y1");
+			trackedObjectJSON.removeMember("x2");
+			trackedObjectJSON.removeMember("y2");
+			objects["all"] = trackedObjectJSON;
+		}
+	}
+	else if(trackedObjects.count(selectedObjectIndex) != 0){
 		auto selectedObject = trackedObjects.at(selectedObjectIndex);
 		if (selectedObject){
 			Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
@@ -544,18 +600,10 @@ std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const {
 	}
 	root["objects"] = objects;
 
-	root["selected_object_index"] = add_property_json("Selected Object", selectedObjectIndex, "int", "", NULL, 0, 200, false, requested_frame);
-	root["confidence_threshold"] = add_property_json("Confidence Theshold", confidence_threshold, "float", "", NULL, 0, 1, false, requested_frame);
+	root["selected_object_index"] = add_property_json("Selected Object", selectedObjectIndex, "int", "", NULL, -1, 200, false, requested_frame);
+	root["confidence_threshold"] = add_property_json("Confidence Threshold", confidence_threshold, "float", "", NULL, 0, 1, false, requested_frame);
 	root["class_filter"] = add_property_json("Class Filter", 0.0, "string", class_filter, NULL, -1, -1, false, requested_frame);
 
-	root["display_box_text"] = add_property_json("Draw All Text", display_box_text.GetValue(requested_frame), "int", "", &display_box_text, 0, 1, false, requested_frame);
-	root["display_box_text"]["choices"].append(add_property_choice_json("Yes", true, display_box_text.GetValue(requested_frame)));
-	root["display_box_text"]["choices"].append(add_property_choice_json("No", false, display_box_text.GetValue(requested_frame)));
-
-	root["display_boxes"] = add_property_json("Draw All Boxes", display_boxes.GetValue(requested_frame), "int", "", &display_boxes, 0, 1, false, requested_frame);
-	root["display_boxes"]["choices"].append(add_property_choice_json("Yes", true, display_boxes.GetValue(requested_frame)));
-	root["display_boxes"]["choices"].append(add_property_choice_json("No", false, display_boxes.GetValue(requested_frame)));
-
 	// Return formatted string
 	return root.toStyledString();
 }

src/effects/ObjectDetection.h

@@ -23,6 +23,10 @@
 #include "Json.h"
 #include "KeyFrame.h"
 
+namespace openshot {
+	class TrackedObjectBBox;
+}
+
 // Struct that stores the detected bounding boxes for all the clip frames
 struct DetectionData{
     DetectionData(){}
@@ -69,12 +73,15 @@ namespace openshot
         Keyframe display_boxes;
 
         /// Minimum confidence value to display the detected objects
-        float confidence_threshold = 0.5;
+        float confidence_threshold = 0.25;
 
         /// Contain the user selected classes for visualization
         std::vector<std::string> display_classes;
         std::string class_filter;
 
+        /// Last explicit "All Objects" settings, used for stable UI readback
+        std::shared_ptr<TrackedObjectBBox> allObjectsProperties;
+
         /// Init effect settings
         void init_effect_details();
 

src/sort_filter/KalmanTracker.cpp

@@ -7,6 +7,7 @@
 
 #include "KalmanTracker.h"
 #include <ctime>
+#include <algorithm>
 
 using namespace std;
 using namespace cv;
@@ -99,6 +100,42 @@ void KalmanTracker::update(
 	// detect_times.push_back(dt);
 }
 
+void KalmanTracker::update_class_scores(const std::vector<ClassScore>& classScores, int fallbackClassId, float fallbackConfidence)
+{
+	const double decay = 0.82;
+	const double update_weight = 1.0 - decay;
+	const bool had_history = !classScoreHistory.empty();
+
+	for (auto it = classScoreHistory.begin(); it != classScoreHistory.end();) {
+		it->second *= decay;
+		if (it->second < 0.0001)
+			it = classScoreHistory.erase(it);
+		else
+			++it;
+	}
+
+	const double candidate_weight = had_history ? update_weight : 1.0;
+	if (classScores.empty()) {
+		classScoreHistory[fallbackClassId] += fallbackConfidence * candidate_weight;
+	} else {
+		for (const auto& candidate : classScores) {
+			if (candidate.classId < 0 || candidate.score <= 0.0f)
+				continue;
+			classScoreHistory[candidate.classId] += candidate.score * candidate_weight;
+		}
+	}
+
+	if (classScoreHistory.empty()) {
+		classId = fallbackClassId;
+		return;
+	}
+
+	auto best = std::max_element(
+		classScoreHistory.begin(), classScoreHistory.end(),
+		[](const auto& a, const auto& b) { return a.second < b.second; });
+	classId = best->first;
+}
+
 // Return the current state vector
 StateType KalmanTracker::get_state()
 {

src/sort_filter/KalmanTracker.h

@@ -11,9 +11,20 @@
 #include "opencv2/video/tracking.hpp"
 #include "opencv2/highgui/highgui.hpp"
 
+#include <map>
+#include <vector>
+
 
 #define StateType cv::Rect_<float>
 
+struct ClassScore
+{
+	int classId = 0;
+	float score = 0.0f;
+	ClassScore() {}
+	ClassScore(int _classId, float _score) : classId(_classId), score(_score) {}
+};
+
 /// This class represents the internel state of individual tracked objects observed as bounding box.
 class KalmanTracker
 {
@@ -27,7 +38,7 @@ public:
 		m_age = 0;
 		m_id = 0;
 	}
-	KalmanTracker(StateType initRect, float confidence, int classId, int objectId) : confidence(confidence), classId(classId)
+	KalmanTracker(StateType initRect, float confidence, int classId, int objectId, const std::vector<ClassScore>& classScores = {}) : confidence(confidence), classId(classId)
 	{
 		init_kf(initRect);
 		m_time_since_update = 0;
@@ -35,6 +46,7 @@ public:
 		m_hit_streak = 0;
 		m_age = 0;
 		m_id = objectId;
+		update_class_scores(classScores, classId, confidence);
 	}
 
 	~KalmanTracker()
@@ -45,6 +57,7 @@ public:
 	StateType predict();
 	StateType predict2();
 	void update(StateType stateMat);
+	void update_class_scores(const std::vector<ClassScore>& classScores, int fallbackClassId, float fallbackConfidence);
 
 	StateType get_state();
 	StateType get_rect_xysr(float cx, float cy, float s, float r);
@@ -56,6 +69,7 @@ public:
 	int m_id;
 	float confidence;
 	int classId;
+	std::map<int, double> classScoreHistory;
 
 private:
 	void init_kf(StateType stateMat);
@@ -66,4 +80,4 @@ private:
 	std::vector<StateType> m_history;
 };
 
-#endif
+#endif

src/sort_filter/sort.cpp

@@ -3,9 +3,70 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
 
 #include "sort.hpp"
+#include <cmath>
 
 using namespace std;
 
+namespace {
+double box_diagonal(const cv::Rect_<float>& box)
+{
+	return std::sqrt(box.width * box.width + box.height * box.height);
+}
+
+double box_area(const cv::Rect_<float>& box)
+{
+	return std::max(0.0f, box.width) * std::max(0.0f, box.height);
+}
+
+double aspect_ratio(const cv::Rect_<float>& box)
+{
+	if (box.width <= 0.0f || box.height <= 0.0f)
+		return 0.0;
+	return static_cast<double>(box.width) / static_cast<double>(box.height);
+}
+
+bool box_shape_matches(const cv::Rect_<float>& predicted_box, const cv::Rect_<float>& detection_box, double iou)
+{
+	const double predicted_area = box_area(predicted_box);
+	const double detection_area = box_area(detection_box);
+	if (predicted_area <= 1.0 || detection_area <= 1.0)
+		return false;
+
+	const double area_ratio = detection_area / predicted_area;
+	const double predicted_aspect = aspect_ratio(predicted_box);
+	const double detection_aspect = aspect_ratio(detection_box);
+	const double aspect_ratio_delta = (predicted_aspect > 0.0 && detection_aspect > 0.0)
+		? std::max(predicted_aspect / detection_aspect, detection_aspect / predicted_aspect)
+		: 999.0;
+
+	if (iou >= 0.70)
+		return area_ratio >= 0.20 && area_ratio <= 5.00 && aspect_ratio_delta <= 4.00;
+	return area_ratio >= 0.35 && area_ratio <= 2.80 && aspect_ratio_delta <= 2.75;
+}
+
+bool detection_matches_track_gate(
+	const KalmanTracker& tracker,
+	const cv::Rect_<float>& predicted_box,
+	const TrackingBox& detection,
+	double iou,
+	double centroid_distance)
+{
+	if (!box_shape_matches(predicted_box, detection.box, iou))
+		return false;
+
+	const double scale = std::max(box_diagonal(predicted_box), box_diagonal(detection.box));
+	const bool missed_previous_frame = tracker.m_time_since_update > 1;
+
+	if (missed_previous_frame) {
+		const double reacquire_distance = std::max(12.0, scale * 0.25);
+		return iou >= 0.35 || centroid_distance <= reacquire_distance;
+	}
+
+	const double local_distance = std::max(12.0, scale * 0.22);
+	return iou >= 0.20 || centroid_distance <= local_distance;
+}
+}
+
 // Constructor
 SortTracker::SortTracker(int max_age, int min_hits, int max_missed, double min_iou, double nms_iou_thresh, double min_conf)
 {
@@ -64,8 +125,9 @@ void apply_nms(vector<TrackingBox>& detections, double nms_iou_thresh) {
         for (size_t j = i + 1; j < detections.size(); ++j) {
             if (suppressed[j]) continue;
 
-            if (detections[i].classId == detections[j].classId &&
-                SortTracker::GetIOU(detections[i].box, detections[j].box) > nms_iou_thresh) {
+            double iou = SortTracker::GetIOU(detections[i].box, detections[j].box);
+            if ((detections[i].classId == detections[j].classId && iou > nms_iou_thresh) ||
+                iou > 0.85) {
                 suppressed[j] = true;
             }
         }
@@ -81,9 +143,12 @@ void apply_nms(vector<TrackingBox>& detections, double nms_iou_thresh) {
     detections = filtered;
 }
 
-void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double image_diagonal, std::vector<float> confidences, std::vector<int> classIds)
+void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double image_diagonal, std::vector<float> confidences, std::vector<int> classIds, std::vector<std::vector<ClassScore>> classScores)
 {
 	vector<TrackingBox> detections;
+	dead_trackers_id.clear();
+	if (classScores.size() != detections_cv.size())
+		classScores.resize(detections_cv.size());
 	if (trackers.size() == 0) // the first frame met
 	{
 		alive_tracker = false;
@@ -97,9 +162,15 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 			tb.box = cv::Rect_<float>(detections_cv[i]);
 			tb.classId = classIds[i];
 			tb.confidence = confidences[i];
+			tb.classScores = classScores[i];
 			detections.push_back(tb);
+		}
 
-			KalmanTracker trk = KalmanTracker(detections.back().box, detections.back().confidence, detections.back().classId, _next_id++);
+		apply_nms(detections, _nms_iou_thresh);
+
+		for (const auto& detection : detections)
+		{
+			KalmanTracker trk = KalmanTracker(detection.box, detection.confidence, detection.classId, _next_id++, detection.classScores);
 			trackers.push_back(trk);
 		}
 		return;
@@ -114,20 +185,12 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 			tb.box = cv::Rect_<float>(detections_cv[i]);
 			tb.classId = classIds[i];
 			tb.confidence = confidences[i];
+			tb.classScores = classScores[i];
 			detections.push_back(tb);
 		}
 
 		// Apply NMS to remove duplicates
 		apply_nms(detections, _nms_iou_thresh);
-
-		for (auto it = frameTrackingResult.begin(); it != frameTrackingResult.end(); it++)
-		{
-			int frame_age = frame_count - it->frame;
-			if (frame_age >= _max_age || frame_age < 0)
-			{
-				dead_trackers_id.push_back(it->id);
-			}
-		}
 	}
 
 	///////////////////////////////////////
@@ -150,67 +213,86 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 
 	cost_matrix.clear();
 	cost_matrix.resize(trkNum, vector<double>(detNum, 0));
-
-	for (unsigned int i = 0; i < trkNum; i++) // compute cost matrix using 1 - IOU with gating
-	{
-		for (unsigned int j = 0; j < detNum; j++)
-		{
-			double iou = GetIOU(predictedBoxes[i], detections[j].box);
-			double dist = GetCentroidsDistance(predictedBoxes[i], detections[j].box) / image_diagonal;
-			if (trackers[i].classId != detections[j].classId || dist > max_centroid_dist_norm)
-			{
-				cost_matrix[i][j] = 1e9; // large cost for gating
-			}
-			else
-			{
-				cost_matrix[i][j] = 1 - iou + (1 - detections[j].confidence) * 0.1; // slight penalty for low conf
-			}
-		}
-	}
-
-	HungarianAlgorithm HungAlgo;
 	assignment.clear();
-	HungAlgo.Solve(cost_matrix, assignment);
-	// find matches, unmatched_detections and unmatched_predictions
+	matchedPairs.clear();
 	unmatchedTrajectories.clear();
 	unmatchedDetections.clear();
 	allItems.clear();
 	matchedItems.clear();
 
-	if (detNum > trkNum) //	there are unmatched detections
+	if (trkNum == 0)
 	{
-		for (unsigned int n = 0; n < detNum; n++)
-			allItems.insert(n);
-
-		for (unsigned int i = 0; i < trkNum; ++i)
-			matchedItems.insert(assignment[i]);
-
-		set_difference(allItems.begin(), allItems.end(),
-					   matchedItems.begin(), matchedItems.end(),
-					   insert_iterator<set<int>>(unmatchedDetections, unmatchedDetections.begin()));
+		for (auto& detection : detections)
+		{
+			KalmanTracker tracker = KalmanTracker(detection.box, detection.confidence, detection.classId, _next_id++, detection.classScores);
+			trackers.push_back(tracker);
+		}
 	}
-	else if (detNum < trkNum) // there are unmatched trajectory/predictions
+	else if (detNum == 0)
 	{
 		for (unsigned int i = 0; i < trkNum; ++i)
-			if (assignment[i] == -1) // unassigned label will be set as -1 in the assignment algorithm
-				unmatchedTrajectories.insert(i);
+			unmatchedTrajectories.insert(i);
 	}
 	else
-		;
-
-	// filter out matched with low IOU
-	matchedPairs.clear();
-	for (unsigned int i = 0; i < trkNum; ++i)
 	{
-		if (assignment[i] == -1) // pass over invalid values
-			continue;
-		if (cost_matrix[i][assignment[i]] > 1 - _min_iou)
+
+		for (unsigned int i = 0; i < trkNum; i++) // compute cost matrix using 1 - IOU with gating
 		{
-			unmatchedTrajectories.insert(i);
-			unmatchedDetections.insert(assignment[i]);
+			for (unsigned int j = 0; j < detNum; j++)
+			{
+				double iou = GetIOU(predictedBoxes[i], detections[j].box);
+				double centroid_distance = GetCentroidsDistance(predictedBoxes[i], detections[j].box);
+				if (!detection_matches_track_gate(trackers[i], predictedBoxes[i], detections[j], iou, centroid_distance))
+				{
+					cost_matrix[i][j] = 1e9; // large cost for gating
+				}
+				else
+				{
+					const double scale = std::max(1.0, std::max(box_diagonal(predictedBoxes[i]), box_diagonal(detections[j].box)));
+					const double distance_penalty = std::min(1.0, centroid_distance / scale) * 0.35;
+					cost_matrix[i][j] = 1 - iou + distance_penalty + (1 - detections[j].confidence) * 0.1;
+				}
+			}
+		}
+
+		HungarianAlgorithm HungAlgo;
+		HungAlgo.Solve(cost_matrix, assignment);
+
+		// find matches, unmatched_detections and unmatched_predictions
+		if (detNum > trkNum) //	there are unmatched detections
+		{
+			for (unsigned int n = 0; n < detNum; n++)
+				allItems.insert(n);
+
+			for (unsigned int i = 0; i < trkNum; ++i)
+				matchedItems.insert(assignment[i]);
+
+			set_difference(allItems.begin(), allItems.end(),
+						   matchedItems.begin(), matchedItems.end(),
+						   insert_iterator<set<int>>(unmatchedDetections, unmatchedDetections.begin()));
+		}
+		else if (detNum < trkNum) // there are unmatched trajectory/predictions
+		{
+			for (unsigned int i = 0; i < trkNum; ++i)
+				if (assignment[i] == -1) // unassigned label will be set as -1 in the assignment algorithm
+					unmatchedTrajectories.insert(i);
 		}
 		else
-			matchedPairs.push_back(cv::Point(i, assignment[i]));
+			;
+
+		// filter out matched with low IOU
+		for (unsigned int i = 0; i < trkNum; ++i)
+		{
+			if (assignment[i] == -1) // pass over invalid values
+				continue;
+			if (cost_matrix[i][assignment[i]] >= 1e8)
+			{
+				unmatchedTrajectories.insert(i);
+				unmatchedDetections.insert(assignment[i]);
+			}
+			else
+				matchedPairs.push_back(cv::Point(i, assignment[i]));
+		}
 	}
 
 	for (unsigned int i = 0; i < matchedPairs.size(); i++)
@@ -218,30 +300,17 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 		int trkIdx = matchedPairs[i].x;
 		int detIdx = matchedPairs[i].y;
 		trackers[trkIdx].update(detections[detIdx].box);
-		trackers[trkIdx].classId = detections[detIdx].classId;
+		trackers[trkIdx].update_class_scores(detections[detIdx].classScores, detections[detIdx].classId, detections[detIdx].confidence);
 		trackers[trkIdx].confidence = detections[detIdx].confidence;
 	}
 
 	// create and initialise new trackers for unmatched detections
 	for (auto umd : unmatchedDetections)
 	{
-		KalmanTracker tracker = KalmanTracker(detections[umd].box, detections[umd].confidence, detections[umd].classId, _next_id++);
+		KalmanTracker tracker = KalmanTracker(detections[umd].box, detections[umd].confidence, detections[umd].classId, _next_id++, detections[umd].classScores);
 		trackers.push_back(tracker);
 	}
 
-	for (auto it2 = dead_trackers_id.begin(); it2 != dead_trackers_id.end(); it2++)
-	{
-		for (unsigned int i = 0; i < trackers.size();)
-		{
-			if (trackers[i].m_id == (*it2))
-			{
-				trackers.erase(trackers.begin() + i);
-				continue;
-			}
-			i++;
-		}
-	}
-
 	// get trackers' output
 	frameTrackingResult.clear();
 	for (unsigned int i = 0; i < trackers.size();)
@@ -251,7 +320,10 @@ void SortTracker::update(vector<cv::Rect> detections_cv, int frame_count, double
 		{
 			alive_tracker = true;
 			TrackingBox res;
-			res.box = trackers[i].get_state();
+			if (trackers[i].m_time_since_update > 0 && i < predictedBoxes.size())
+				res.box = predictedBoxes[i];
+			else
+				res.box = trackers[i].get_state();
 			res.id = trackers[i].m_id;
 			res.frame = frame_count;
 			res.classId = trackers[i].classId;

src/sort_filter/sort.hpp

@@ -26,6 +26,7 @@ typedef struct TrackingBox
 	int classId = 0;
 	int id = 0;
 	cv::Rect_<float> box = cv::Rect_<float>(0.0, 0.0, 0.0, 0.0);
+	std::vector<ClassScore> classScores;
 	TrackingBox() {}
 	TrackingBox(int _frame, float _confidence, int _classId, int _id) : frame(_frame), confidence(_confidence), classId(_classId), id(_id) {}
 } TrackingBox;
@@ -34,11 +35,11 @@ class SortTracker
 {
 public:
 	// Constructor
-	SortTracker(int max_age = 50, int min_hits = 5, int max_missed = 7, double min_iou = 0.1, double nms_iou_thresh = 0.5, double min_conf = 0.3);
+	SortTracker(int max_age = 50, int min_hits = 5, int max_missed = 3, double min_iou = 0.1, double nms_iou_thresh = 0.5, double min_conf = 0.3);
 	// Initialize tracker
 
 	// Update position based on the new frame
-	void update(std::vector<cv::Rect> detection, int frame_count, double image_diagonal, std::vector<float> confidences, std::vector<int> classIds);
+	void update(std::vector<cv::Rect> detection, int frame_count, double image_diagonal, std::vector<float> confidences, std::vector<int> classIds, std::vector<std::vector<ClassScore>> classScores = {});
 	static double GetIOU(cv::Rect_<float> bb_test, cv::Rect_<float> bb_gt);
 	double GetCentroidsDistance(cv::Rect_<float> bb_test, cv::Rect_<float> bb_gt);
 	std::vector<KalmanTracker> trackers;

tests/CVObjectDetection.cpp

@@ -25,9 +25,8 @@ using namespace openshot;
 
 std::string effectInfo =(" {\"protobuf_data_path\": \"objdetector.data\", "
                          "  \"processing_device\": \"GPU\", "
-                         "  \"model_configuration\": \"~/yolo/yolov3.cfg\", "
-                         "  \"model_weights\": \"~/yolo/yolov3.weights\", "
-                         "  \"classes_file\": \"~/yolo/obj.names\"} ");
+                         "  \"model\": \"~/yolo/Yolo5/yolov5s.onnx\", "
+                         "  \"classes_file\": \"~/yolo/Yolo5/obj.names\"} ");
 
 // Just for the stabilizer constructor, it won't be used
 ProcessingController processingController;

tests/CVTracker.cpp

@@ -13,6 +13,7 @@
 
 #include <sstream>
 #include <memory>
+#include <cmath>
 
 #include "openshot_catch.h"
 
@@ -20,6 +21,7 @@
 #include "CVTracker.h"  // for FrameData, CVTracker
 #include "ProcessingController.h"
 #include "Exceptions.h"
+#include "sort_filter/sort.hpp"
 
 using namespace openshot;
 
@@ -181,6 +183,166 @@ TEST_CASE( "Track_FrameSizeChangeDoesNotCrash", "[libopenshot][opencv][tracker]"
     CHECK(fd.y2 <= 1.0f);
 }
 
+TEST_CASE( "KalmanTracker smooths class scores", "[libopenshot][opencv][tracker]" )
+{
+    KalmanTracker tracker(
+        cv::Rect_<float>(0.0f, 0.0f, 10.0f, 10.0f),
+        0.9f, 1, 42,
+        { ClassScore(1, 0.9f), ClassScore(2, 0.1f) }
+    );
+
+    CHECK(tracker.classId == 1);
+    CHECK(tracker.confidence == Approx(0.9f));
+
+    tracker.update_class_scores({ ClassScore(1, 0.1f), ClassScore(2, 0.9f) }, 2, 0.9f);
+    CHECK(tracker.classId == 1);
+
+    tracker.update_class_scores({ ClassScore(1, 0.1f), ClassScore(2, 0.9f) }, 2, 0.9f);
+    CHECK(tracker.classId == 1);
+
+    tracker.update_class_scores({ ClassScore(1, 0.1f), ClassScore(2, 0.9f) }, 2, 0.9f);
+    tracker.update_class_scores({ ClassScore(1, 0.1f), ClassScore(2, 0.9f) }, 2, 0.9f);
+    CHECK(tracker.classId == 2);
+}
+
+TEST_CASE( "SortTracker does not reacquire a missed track onto a nearby object", "[libopenshot][opencv][tracker]" )
+{
+    SortTracker sort(50, 1, 7, 0.1, 0.5, 0.0);
+    const double diagonal = std::sqrt(1920.0 * 1920.0 + 1080.0 * 1080.0);
+
+    sort.update(
+        { cv::Rect(100, 100, 60, 60) },
+        1,
+        diagonal,
+        { 0.95f },
+        { 2 },
+        { { ClassScore(2, 0.95f) } }
+    );
+    sort.update(
+        { cv::Rect(100, 100, 60, 60) },
+        2,
+        diagonal,
+        { 0.95f },
+        { 2 },
+        { { ClassScore(2, 0.95f) } }
+    );
+    REQUIRE(sort.frameTrackingResult.size() == 1);
+    const int first_id = sort.frameTrackingResult[0].id;
+
+    sort.update({}, 3, diagonal, {}, {}, {});
+    REQUIRE(sort.frameTrackingResult.size() == 1);
+    CHECK(sort.frameTrackingResult[0].id == first_id);
+
+    sort.update(
+        { cv::Rect(100, 145, 60, 60) },
+        4,
+        diagonal,
+        { 0.95f },
+        { 2 },
+        { { ClassScore(2, 0.95f) } }
+    );
+
+    bool original_track_coasted = false;
+    for (const auto& result : sort.frameTrackingResult) {
+        if (result.id == first_id) {
+            original_track_coasted = true;
+            CHECK(result.box.y < 130.0f);
+        }
+    }
+    CHECK(original_track_coasted);
+    CHECK(sort.trackers.size() >= 2);
+}
+
+TEST_CASE( "SortTracker rejects adjacent-object handoff for active track", "[libopenshot][opencv][tracker]" )
+{
+    SortTracker sort(50, 1, 3, 0.1, 0.5, 0.0);
+    const double diagonal = std::sqrt(960.0 * 960.0 + 540.0 * 540.0);
+
+    sort.update(
+        { cv::Rect(299, 181, 112, 97) },
+        1,
+        diagonal,
+        { 0.80f },
+        { 2 },
+        { { ClassScore(2, 0.80f) } }
+    );
+    sort.update(
+        { cv::Rect(299, 181, 112, 97) },
+        2,
+        diagonal,
+        { 0.80f },
+        { 2 },
+        { { ClassScore(2, 0.80f) } }
+    );
+    REQUIRE(sort.frameTrackingResult.size() == 1);
+    const int first_id = sort.frameTrackingResult[0].id;
+
+    sort.update(
+        { cv::Rect(248, 156, 103, 71) },
+        3,
+        diagonal,
+        { 0.77f },
+        { 2 },
+        { { ClassScore(2, 0.77f) } }
+    );
+
+    bool original_track_did_not_jump = false;
+    for (const auto& result : sort.frameTrackingResult) {
+        if (result.id == first_id) {
+            original_track_did_not_jump = true;
+            CHECK(result.box.x > 285.0f);
+            CHECK(result.box.y > 170.0f);
+        }
+    }
+    CHECK(original_track_did_not_jump);
+    CHECK(sort.trackers.size() >= 2);
+}
+
+TEST_CASE( "SortTracker rejects tiny nested detection for vehicle track", "[libopenshot][opencv][tracker]" )
+{
+    SortTracker sort(50, 1, 3, 0.1, 0.5, 0.0);
+    const double diagonal = std::sqrt(960.0 * 960.0 + 540.0 * 540.0);
+
+    sort.update(
+        { cv::Rect(520, 178, 123, 91) },
+        1,
+        diagonal,
+        { 0.77f },
+        { 2 },
+        { { ClassScore(2, 0.77f) } }
+    );
+    sort.update(
+        { cv::Rect(520, 178, 123, 91) },
+        2,
+        diagonal,
+        { 0.77f },
+        { 2 },
+        { { ClassScore(2, 0.77f) } }
+    );
+    REQUIRE(sort.frameTrackingResult.size() == 1);
+    const int car_id = sort.frameTrackingResult[0].id;
+
+    sort.update(
+        { cv::Rect(592, 198, 30, 13) },
+        3,
+        diagonal,
+        { 0.36f },
+        { 0 },
+        { { ClassScore(0, 0.36f), ClassScore(2, 0.15f) } }
+    );
+
+    bool car_track_did_not_shrink = false;
+    for (const auto& result : sort.frameTrackingResult) {
+        if (result.id == car_id) {
+            car_track_did_not_shrink = true;
+            CHECK(result.box.width > 90.0f);
+            CHECK(result.box.height > 70.0f);
+        }
+    }
+    CHECK(car_track_did_not_shrink);
+    CHECK(sort.trackers.size() >= 2);
+}
+
 
 TEST_CASE( "SaveLoad_Protobuf", "[libopenshot][opencv][tracker]" )
 {

tests/EffectMask.cpp

@@ -24,10 +24,12 @@
 #include "effects/Blur.h"
 #include "effects/Brightness.h"
 #include "effects/Hue.h"
+#include "effects/ObjectDetection.h"
 #include "effects/Pixelate.h"
 #include "effects/Saturation.h"
 #include "effects/Sharpen.h"
 #include "effects/Tracker.h"
+#include "TrackedObjectBBox.h"
 #include "QtImageReader.h"
 #include "openshot_catch.h"
 
@@ -149,6 +151,116 @@ TEST_CASE("EffectBase common mask blend applies to ProcessFrame", "[effect][mask
 	CHECK(out_image->pixelColor(3, 0).red() == 80);
 }
 
+TEST_CASE("ObjectDetection all object update applies sparse style keys", "[effect][object_detection]") {
+	ObjectDetection effect;
+	auto first = std::make_shared<TrackedObjectBBox>(10, 20, 30, 255);
+	auto second = std::make_shared<TrackedObjectBBox>(200, 210, 220, 255);
+	first->AddBox(1, 0.25f, 0.25f, 0.2f, 0.2f, 0.0f);
+	second->AddBox(1, 0.75f, 0.75f, 0.2f, 0.2f, 0.0f);
+	effect.trackedObjects[1] = first;
+	effect.trackedObjects[2] = second;
+
+	Json::Value update;
+	update["objects"]["all"]["background_alpha"] = Keyframe(0.0).JsonValue();
+	update["objects"]["all"]["stroke_width"] = Keyframe(5.0).JsonValue();
+	effect.SetJsonValue(update);
+
+	CHECK(first->background_alpha.GetValue(1) == Approx(0.0));
+	CHECK(second->background_alpha.GetValue(1) == Approx(0.0));
+	CHECK(first->stroke_width.GetValue(1) == Approx(5.0));
+	CHECK(second->stroke_width.GetValue(1) == Approx(5.0));
+
+	CHECK(first->stroke.red.GetValue(1) == Approx(10.0));
+	CHECK(second->stroke.red.GetValue(1) == Approx(200.0));
+	CHECK(first->background.red.GetValue(1) == Approx(10.0));
+	CHECK(second->background.red.GetValue(1) == Approx(200.0));
+}
+
+TEST_CASE("ObjectDetection individual style overrides all object style", "[effect][object_detection]") {
+	ObjectDetection effect;
+	auto first = std::make_shared<TrackedObjectBBox>(10, 20, 30, 255);
+	auto second = std::make_shared<TrackedObjectBBox>(200, 210, 220, 255);
+	first->Id("effect-1");
+	second->Id("effect-2");
+	first->AddBox(1, 0.25f, 0.25f, 0.2f, 0.2f, 0.0f);
+	second->AddBox(1, 0.75f, 0.75f, 0.2f, 0.2f, 0.0f);
+	effect.trackedObjects[1] = first;
+	effect.trackedObjects[2] = second;
+	effect.selectedObjectIndex = 1;
+
+	Json::Value update;
+	update["objects"]["effect-1"]["stroke"]["red"] = Keyframe(255.0).JsonValue();
+	update["objects"]["effect-1"]["stroke"]["green"] = Keyframe(0.0).JsonValue();
+	update["objects"]["effect-1"]["stroke"]["blue"] = Keyframe(0.0).JsonValue();
+	update["objects"]["all"]["stroke"]["red"] = Keyframe(0.0).JsonValue();
+	update["objects"]["all"]["stroke"]["green"] = Keyframe(255.0).JsonValue();
+	update["objects"]["all"]["stroke"]["blue"] = Keyframe(0.0).JsonValue();
+	effect.SetJsonValue(update);
+
+	CHECK(first->stroke.red.GetValue(1) == Approx(255.0));
+	CHECK(first->stroke.green.GetValue(1) == Approx(0.0));
+	CHECK(first->stroke.blue.GetValue(1) == Approx(0.0));
+	CHECK(second->stroke.red.GetValue(1) == Approx(0.0));
+	CHECK(second->stroke.green.GetValue(1) == Approx(255.0));
+	CHECK(second->stroke.blue.GetValue(1) == Approx(0.0));
+
+	Json::Value props = stringToJson(effect.PropertiesJSON(1));
+	REQUIRE(props["objects"].isMember("effect-1"));
+	CHECK(props["objects"]["effect-1"]["stroke"]["red"]["value"].asDouble() == Approx(255.0));
+	CHECK(props["objects"]["effect-1"]["stroke"]["green"]["value"].asDouble() == Approx(0.0));
+	CHECK(props["objects"]["effect-1"]["stroke"]["blue"]["value"].asDouble() == Approx(0.0));
+}
+
+TEST_CASE("ObjectDetection all object selection exposes tracked object properties", "[effect][object_detection]") {
+	ObjectDetection effect;
+	auto tracked = std::make_shared<TrackedObjectBBox>(10, 20, 30, 255);
+	tracked->AddBox(1, 0.25f, 0.25f, 0.2f, 0.2f, 0.0f);
+	effect.trackedObjects[1] = tracked;
+	effect.selectedObjectIndex = -1;
+
+	Json::Value props = stringToJson(effect.PropertiesJSON(1));
+	REQUIRE(props["objects"].isMember("all"));
+	CHECK(props["selected_object_index"]["min"].asInt() == -1);
+	CHECK(props["objects"]["all"].isMember("background_alpha"));
+	CHECK(props["objects"]["all"].isMember("stroke"));
+	CHECK(props["objects"]["all"].isMember("delta_x"));
+	CHECK(props["objects"]["all"].isMember("scale_x"));
+	CHECK_FALSE(props["objects"]["all"].isMember("x1"));
+	CHECK_FALSE(props.isMember("display_box_text"));
+	CHECK_FALSE(props.isMember("display_boxes"));
+}
+
+TEST_CASE("ObjectDetection all object properties keep explicit all values", "[effect][object_detection]") {
+	ObjectDetection effect;
+	auto first = std::make_shared<TrackedObjectBBox>(10, 20, 30, 255);
+	auto second = std::make_shared<TrackedObjectBBox>(200, 210, 220, 255);
+	first->Id("effect-1");
+	second->Id("effect-2");
+	first->AddBox(1, 0.25f, 0.25f, 0.2f, 0.2f, 0.0f);
+	second->AddBox(1, 0.75f, 0.75f, 0.2f, 0.2f, 0.0f);
+	effect.trackedObjects[1] = first;
+	effect.trackedObjects[2] = second;
+
+	Json::Value update;
+	update["objects"]["all"]["stroke_width"] = Keyframe(6.0).JsonValue();
+	update["objects"]["all"]["stroke_alpha"] = Keyframe(0.25).JsonValue();
+	update["objects"]["effect-1"]["stroke_width"] = Keyframe(3.0).JsonValue();
+	update["objects"]["effect-1"]["stroke_alpha"] = Keyframe(0.75).JsonValue();
+	effect.SetJsonValue(update);
+
+	effect.selectedObjectIndex = -1;
+	Json::Value all_props = stringToJson(effect.PropertiesJSON(1));
+	REQUIRE(all_props["objects"].isMember("all"));
+	CHECK(all_props["objects"]["all"]["stroke_width"]["value"].asDouble() == Approx(6.0));
+	CHECK(all_props["objects"]["all"]["stroke_alpha"]["value"].asDouble() == Approx(0.25));
+
+	effect.selectedObjectIndex = 1;
+	Json::Value first_props = stringToJson(effect.PropertiesJSON(1));
+	REQUIRE(first_props["objects"].isMember("effect-1"));
+	CHECK(first_props["objects"]["effect-1"]["stroke_width"]["value"].asDouble() == Approx(3.0));
+	CHECK(first_props["objects"]["effect-1"]["stroke_alpha"]["value"].asDouble() == Approx(0.75));
+}
+
 TEST_CASE("EffectBase mask fields serialize and deserialize", "[effect][mask][json]") {
 	const std::string mask_path = create_mask_png({255, 0});
 
@@ -196,6 +308,30 @@ TEST_CASE("EffectBase can use tracker effect bbox as generated mask source", "[e
 	CHECK(out_image->pixelColor(3, 0).red() == 80);
 }
 
+TEST_CASE("EffectBase tracker mask source honors corner radius", "[effect][mask][tracker]") {
+	Clip parent_clip;
+	Tracker tracker;
+	tracker.Id("tracker-source");
+	tracker.trackedData->SetBaseFPS(Fraction(30, 1));
+	tracker.trackedData->AddBox(1, 0.5f, 0.5f, 1.0f, 1.0f, 0.0f);
+	tracker.trackedData->background_corner = Keyframe(12.0);
+
+	Brightness brightness(Keyframe(0.8), Keyframe(0.0));
+	brightness.MaskSourceId("tracker-source");
+
+	parent_clip.AddEffect(&tracker);
+	parent_clip.AddEffect(&brightness);
+
+	auto frame = std::make_shared<Frame>(1, 20, 20, "#000000");
+	frame->GetImage()->fill(QColor(80, 80, 80, 255));
+
+	auto out = brightness.ProcessFrame(frame, 1);
+	auto out_image = out->GetImage();
+
+	CHECK(out_image->pixelColor(0, 0).red() == 80);
+	CHECK(out_image->pixelColor(10, 10).red() > 80);
+}
+
 TEST_CASE("EffectBase tracker mask source with no visible bbox preserves original frame", "[effect][mask][tracker]") {
 	Clip parent_clip;
 	Tracker tracker;

tests/KeyFrame.cpp

@@ -597,9 +597,9 @@ TEST_CASE( "TrackedObjectBBox init", "[libopenshot][keyframe]" )
 	CHECK(kfb.stroke.blue.GetInt(1) == 0);
 	CHECK(kfb.stroke.alpha.GetInt(1) == 212);
 
-	CHECK(kfb.background.red.GetInt(1) == 0);
-	CHECK(kfb.background.green.GetInt(1) == 0);
-	CHECK(kfb.background.blue.GetInt(1) == 255);
+	CHECK(kfb.background.red.GetInt(1) == 62);
+	CHECK(kfb.background.green.GetInt(1) == 143);
+	CHECK(kfb.background.blue.GetInt(1) == 0);
 	CHECK(kfb.background.alpha.GetInt(1) == 212);
 }