diff --git a/.gitignore b/.gitignore index 58848653..5d00d580 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /.metadata/ tags *~ + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3280edde..3ce3829b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,7 @@ windows-builder-x64: - $env:LIBOPENSHOT_AUDIO_DIR = "$CI_PROJECT_DIR\build\install-x64" - $env:UNITTEST_DIR = "C:\msys64\usr" - $env:RESVGDIR = "C:\msys64\usr" - - $env:Path = "C:\msys64\mingw64\bin;C:\msys64\mingw64\lib;C:\msys64\usr\lib\cmake\UnitTest++;C:\msys64\home\jonathan\depot_tools;C:\msys64\usr;C:\msys64\usr\lib;" + $env:Path; + - $env:Path = "C:\msys64\mingw64\bin;C:\msys64\mingw64\lib;C:\msys64\usr\lib\cmake\UnitTest++;C:\msys64\home\jonathan\depot_tools;C:\msys64\usr;C:\msys64\usr\lib;C:\msys64\usr\local\x64\mingw\bin;C:\msys64\usr\local;" + $env:Path; - New-Item -ItemType Directory -Force -Path build - cd build - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MSYS Makefiles" -DCMAKE_MAKE_PROGRAM=mingw32-make -D"CMAKE_BUILD_TYPE:STRING=Release" ../ @@ -97,7 +97,7 @@ windows-builder-x86: - $env:LIBOPENSHOT_AUDIO_DIR = "$CI_PROJECT_DIR\build\install-x86" - $env:UNITTEST_DIR = "C:\msys32\usr" - $env:RESVGDIR = "C:\msys32\usr" - - $env:Path = "C:\msys32\mingw32\bin;C:\msys32\mingw32\lib;C:\msys32\usr\lib\cmake\UnitTest++;C:\msys32\home\jonathan\depot_tools;C:\msys32\usr;C:\msys32\usr\lib;" + $env:Path; + - $env:Path = "C:\msys32\mingw32\bin;C:\msys32\mingw32\lib;C:\msys32\usr\lib\cmake\UnitTest++;C:\msys32\home\jonathan\depot_tools;C:\msys32\usr;C:\msys32\usr\lib;C:\msys32\usr\local\x64\mingw\bin;C:\msys32\usr\local;" + $env:Path; - New-Item -ItemType Directory -Force -Path build - cd build - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MSYS Makefiles" -DCMAKE_MAKE_PROGRAM=mingw32-make -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32" ../ diff --git a/CMakeLists.txt b/CMakeLists.txt index 294639bf..ff71dde7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,7 @@ option(ENABLE_COVERAGE "Scan test coverage using gcov and report" OFF) option(ENABLE_DOCS "Build API documentation (requires Doxygen)" ON) option(APPIMAGE_BUILD "Build to install in an AppImage (Linux only)" OFF) option(ENABLE_MAGICK "Use ImageMagick, if available" ON) +option(ENABLE_OPENCV "Build with OpenCV algorithms (requires Boost, Protobuf 3)" ON) # Legacy commandline override if (DISABLE_TESTS) diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index a129280d..67e38618 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -114,6 +114,15 @@ %} #endif +#ifdef USE_OPENCV + %{ + #include "ClipProcessingJobs.h" + #include "effects/Stabilizer.h" + #include "effects/Tracker.h" + #include "effects/ObjectDetection.h" + %} +#endif + /* Generic language independent exception handler. */ %include "exception.i" %exception { @@ -269,6 +278,10 @@ %include "ZmqLogger.h" %include "AudioDeviceInfo.h" +#ifdef USE_OPENCV + %include "ClipProcessingJobs.h" +#endif + #ifdef USE_IMAGEMAGICK %include "ImageReader.h" %include "ImageWriter.h" @@ -291,6 +304,11 @@ %include "effects/Saturation.h" %include "effects/Shift.h" %include "effects/Wave.h" +#ifdef USE_OPENCV + %include "effects/Stabilizer.h" + %include "effects/Tracker.h" + %include "effects/ObjectDetection.h" +#endif /* Wrap std templates (list, vector, etc...) */ diff --git a/examples/Example_opencv.cpp b/examples/Example_opencv.cpp new file mode 100644 index 00000000..e820708a --- /dev/null +++ b/examples/Example_opencv.cpp @@ -0,0 +1,297 @@ +/** + * @file + * @brief Source file for Example Executable (example app for libopenshot) + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include +#include +#include +#include "../../include/CVTracker.h" +#include "../../include/CVStabilization.h" +#include "../../include/CVObjectDetection.h" + +#include "../../include/OpenShot.h" +#include "../../include/CrashHandler.h" + +using namespace openshot; +using namespace std; + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +The following methods are just for getting JSON info to the pre-processing effects +*/ + +string jsonFormat(string key, string value, string type="string"); // Format variables to the needed JSON format +string trackerJson(cv::Rect2d r, bool onlyProtoPath); // Set variable values for tracker effect +string stabilizerJson(bool onlyProtoPath); // Set variable values for stabilizer effect +string objectDetectionJson(bool onlyProtoPath); // Set variable values for object detector effect + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + +// Show the pre-processed clip on the screen +void displayClip(openshot::Clip &r9){ + + // Opencv display window + cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); + + // Get video lenght + int videoLenght = r9.Reader()->info.video_length; + + // Loop through the clip and show it with the effects, if any + for (long int frame = 0; frame < videoLenght; frame++) + { + int frame_number = frame; + // Get the frame + std::shared_ptr f = r9.GetFrame(frame_number); + // Grab OpenCV::Mat image + cv::Mat cvimage = f->GetImageCV(); + + // Display the frame + cv::imshow("Display Image", cvimage); + + // Press ESC on keyboard to exit + char c=(char)cv::waitKey(25); + if(c==27) + break; + } + // Destroy all remaining windows + cv::destroyAllWindows(); +} + +int main(int argc, char* argv[]) { + + // Set pre-processing effects + bool TRACK_DATA = true; + bool SMOOTH_VIDEO = false; + bool OBJECT_DETECTION_DATA = false; + + // Get media path + std::stringstream path; + path << TEST_MEDIA_PATH << ((OBJECT_DETECTION_DATA) ? "run.mp4" : "test.avi"); + // run.mp4 --> Used for object detector + // test.avi --> Used for tracker and stabilizer + + // Thread controller just for the pre-processing constructors, it won't be used + ProcessingController processingController; + + // Open clip + openshot::Clip r9(path.str()); + r9.Open(); + + // Apply tracking effect on the clip + if(TRACK_DATA){ + + // Take the bounding box coordinates + cv::Mat roi = r9.GetFrame(0)->GetImageCV(); + cv::Rect2d r = cv::selectROI(roi); + cv::destroyAllWindows(); + + // Create a tracker object by passing a JSON string and a thread controller, this last one won't be used + // JSON info: path to save the tracked data, type of tracker and bbox coordinates + CVTracker tracker(trackerJson(r, false), processingController); + + // Start the tracking + tracker.trackClip(r9, 0, 100, true); + // Save the tracked data + tracker.SaveTrackedData(); + + // Create a tracker effect + EffectBase* e = EffectInfo().CreateEffect("Tracker"); + + // Pass a JSON string with the saved tracked data + // The effect will read and save the tracking in a map:: + e->SetJson(trackerJson(r, true)); + // Add the effect to the clip + r9.AddEffect(e); + } + + // Apply stabilizer effect on the clip + if(SMOOTH_VIDEO){ + + // Create a stabilizer object by passing a JSON string and a thread controller, this last one won't be used + // JSON info: path to save the stabilized data and smoothing window value + CVStabilization stabilizer(stabilizerJson(false), processingController); + + // Start the stabilization + stabilizer.stabilizeClip(r9, 0, 100, true); + // Save the stabilization data + stabilizer.SaveStabilizedData(); + + // Create a stabilizer effect + EffectBase* e = EffectInfo().CreateEffect("Stabilizer"); + + // Pass a JSON string with the saved stabilized data + // The effect will read and save the stabilization in a map:: + e->SetJson(stabilizerJson(true)); + // Add the effect to the clip + r9.AddEffect(e); + } + + // Apply object detection effect on the clip + if(OBJECT_DETECTION_DATA){ + + // Create a object detection object by passing a JSON string and a thread controller, this last one won't be used + // JSON info: path to save the detection data, processing devicee, model weights, model configuration and class names + CVObjectDetection objectDetection(objectDetectionJson(false), processingController); + + // Start the object detection + objectDetection.detectObjectsClip(r9, 0, 100, true); + // Save the object detection data + objectDetection.SaveObjDetectedData(); + + // Create a object detector effect + EffectBase* e = EffectInfo().CreateEffect("Object Detector"); + + // Pass a JSON string with the saved detections data + // The effect will read and save the detections in a map:: + + e->SetJson(objectDetectionJson(true)); + // Add the effect to the clip + r9.AddEffect(e); + } + + // Show the pre-processed clip on the screen + displayClip(r9); + + // Close timeline + r9.Close(); + + std::cout << "Completed successfully!" << std::endl; + + return 0; +} + + + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| + +The following methods are just for getting JSON info to the pre-processing effects + +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + + + +string jsonFormat(string key, string value, string type){ + stringstream jsonFormatMessage; + jsonFormatMessage << ( "\"" + key + "\": " ); + + if(type == "string") + jsonFormatMessage << ( "\"" + value + "\"" ); + if(type == "rstring") + jsonFormatMessage << value; + if(type == "int") + jsonFormatMessage << stoi(value); + if(type == "float") + jsonFormatMessage << (float)stof(value); + if(type == "double") + jsonFormatMessage << (double)stof(value); + if (type == "bool") + jsonFormatMessage << ((value == "true" || value == "1") ? "true" : "false"); + + return jsonFormatMessage.str(); +} + +// Return JSON string for the tracker effect +string trackerJson(cv::Rect2d r, bool onlyProtoPath){ + + // Define path to save tracked data + string protobufDataPath = "kcf_tracker.data"; + // Set the tracker + string tracker = "KCF"; + + // Construct all the composition of the JSON string + string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath); + string trackerType = jsonFormat("tracker_type", tracker); + string bboxCoords = jsonFormat( + "bbox", + "{" + jsonFormat("x", to_string(r.x), "int") + + "," + jsonFormat("y", to_string(r.y), "int") + + "," + jsonFormat("w", to_string(r.width), "int") + + "," + jsonFormat("h", to_string(r.height), "int") + + "}", + "rstring"); + + // Return only the the protobuf path in JSON format + if(onlyProtoPath) + return "{" + protobuf_data_path + "}"; + // Return all the parameters for the pre-processing effect + else + return "{" + protobuf_data_path + "," + trackerType + "," + bboxCoords + "}"; +} + +// Return JSON string for the stabilizer effect +string stabilizerJson(bool onlyProtoPath){ + + // Define path to save stabilized data + string protobufDataPath = "example_stabilizer.data"; + // Set smoothing window value + string smoothingWindow = "30"; + + // Construct all the composition of the JSON string + string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath); + string smoothing_window = jsonFormat("smoothing_window", smoothingWindow, "int"); + + // Return only the the protobuf path in JSON format + if(onlyProtoPath) + return "{" + protobuf_data_path + "}"; + // Return all the parameters for the pre-processing effect + else + return "{" + protobuf_data_path + "," + smoothing_window + "}"; +} + +string objectDetectionJson(bool onlyProtoPath){ + + // Define path to save object detection data + string protobufDataPath = "example_object_detection.data"; + // Define processing device + string processingDevice = "GPU"; + // Set path to model configuration file + string modelConfiguration = "yolov3.cfg"; + // Set path to model weights + string modelWeights = "yolov3.weights"; + // Set path to class names file + string classesFile = "obj.names"; + + // Construct all the composition of the JSON string + string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath); + string processing_device = jsonFormat("processing_device", processingDevice); + string model_configuration = jsonFormat("model_configuration", modelConfiguration); + string model_weights = jsonFormat("model_weights", modelWeights); + string classes_file = jsonFormat("classes_file", classesFile); + + // Return only the the protobuf path in JSON format + if(onlyProtoPath) + return "{" + protobuf_data_path + "}"; + else + return "{" + protobuf_data_path + "," + processing_device + "," + model_configuration + "," + + model_weights + "," + classes_file + "}"; +} diff --git a/examples/run.mp4 b/examples/run.mp4 new file mode 100644 index 00000000..d5499580 Binary files /dev/null and b/examples/run.mp4 differ diff --git a/examples/test.avi b/examples/test.avi new file mode 100644 index 00000000..4d6b8fc6 Binary files /dev/null and b/examples/test.avi differ diff --git a/examples/test_video.mp4 b/examples/test_video.mp4 new file mode 100644 index 00000000..b648984c Binary files /dev/null and b/examples/test_video.mp4 differ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ee9adf56..9f4a1376 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -99,6 +99,19 @@ set(OPENSHOT_SOURCES TimelineBase.cpp Timeline.cpp) +# OpenCV related classes +set(OPENSHOT_CV_SOURCES + CVTracker.cpp + CVStabilization.cpp + ClipProcessingJobs.cpp + CVObjectDetection.cpp + effects/Stabilizer.cpp + effects/Tracker.cpp + effects/ObjectDetection.cpp + ./sort_filter/sort.cpp + ./sort_filter/Hungarian.cpp + ./sort_filter/KalmanTracker.cpp) + # Video effects set(EFFECTS_SOURCES effects/Bars.cpp @@ -375,6 +388,33 @@ if (ENABLE_BLACKMAGIC) endif() endif() +################## OPENCV ################### +if(ENABLE_OPENCV) + find_package(OpenCV 4) + if(NOT OpenCV_FOUND) + set(ENABLE_OPENCV FALSE CACHE BOOL + "Build with OpenCV algorithms (requires Boost, Protobuf 3)" FORCE) + else() + add_subdirectory(protobuf_messages) + # Add OpenCV source files + target_sources(openshot PRIVATE + ${OPENSHOT_CV_SOURCES} + ) + target_compile_definitions(openshot PUBLIC USE_OPENCV=1) + target_link_libraries(openshot PUBLIC + opencv_core + opencv_video + opencv_highgui + opencv_dnn + opencv_tracking + openshot_protobuf + ) + set(HAVE_OPENCV TRUE CACHE BOOL "Building with OpenCV effects" FORCE) + mark_as_advanced(HAVE_OPENCV) + endif() +endif() +add_feature_info("OpenCV algorithms" ENABLE_OPENCV "Use OpenCV algorithms") + ############### LINK LIBRARY ################# # Link remaining dependency libraries if(DEFINED PROFILER) @@ -402,6 +442,9 @@ install( DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/libopenshot FILES_MATCHING PATTERN "*.h" ) +install(FILES ${ProtobufHeaders} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/libopenshot +) ############### CPACK PACKAGING ############## if(MINGW) diff --git a/src/CVObjectDetection.cpp b/src/CVObjectDetection.cpp new file mode 100644 index 00000000..c391197a --- /dev/null +++ b/src/CVObjectDetection.cpp @@ -0,0 +1,488 @@ +/** + * @file + * @brief Source file for CVObjectDetection class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "CVObjectDetection.h" +#include + +using namespace std; +using namespace openshot; +using google::protobuf::util::TimeUtil; + +CVObjectDetection::CVObjectDetection(std::string processInfoJson, ProcessingController &processingController) +: processingController(&processingController), processingDevice("CPU"){ + SetJson(processInfoJson); + confThreshold = 0.5; + nmsThreshold = 0.1; +} + +void CVObjectDetection::setProcessingDevice(){ + if(processingDevice == "GPU"){ + net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + } + else if(processingDevice == "CPU"){ + net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); + } +} + +void CVObjectDetection::detectObjectsClip(openshot::Clip &video, size_t _start, size_t _end, bool process_interval) +{ + + start = _start; end = _end; + + video.Open(); + + if(error){ + return; + } + + processingController->SetError(false, ""); + + // Load names of classes + std::ifstream ifs(classesFile.c_str()); + std::string line; + while (std::getline(ifs, line)) classNames.push_back(line); + + // Load the network + if(classesFile == "" || modelConfiguration == "" || modelWeights == "") + return; + net = cv::dnn::readNetFromDarknet(modelConfiguration, modelWeights); + setProcessingDevice(); + + size_t frame_number; + if(!process_interval || end == 0 || end-start == 0){ + // Get total number of frames in video + start = video.Start() * video.Reader()->info.fps.ToInt(); + end = video.End() * video.Reader()->info.fps.ToInt(); + } + + for (frame_number = start; frame_number <= end; frame_number++) + { + // Stop the feature tracker process + if(processingController->ShouldStop()){ + return; + } + + std::shared_ptr f = video.GetFrame(frame_number); + + // Grab OpenCV Mat image + cv::Mat cvimage = f->GetImageCV(); + + DetectObjects(cvimage, frame_number); + + // Update progress + processingController->SetProgress(uint(100*(frame_number-start)/(end-start))); + + // std::cout<<"Frame: "< outs; + net.forward(outs, getOutputsNames(net)); + + // Remove the bounding boxes with low confidence + postprocess(frame.size(), outs, frameId); + +} + + +// Remove the bounding boxes with low confidence using non-maxima suppression +void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector& outs, size_t frameId) +{ + std::vector classIds; + std::vector confidences; + std::vector boxes; + + for (size_t i = 0; i < outs.size(); ++i) + { + // Scan through all the bounding boxes output from the network and keep only the + // ones with high confidence scores. Assign the box's class label as the class + // with the highest score for the box. + float* data = (float*)outs[i].data; + for (int j = 0; j < outs[i].rows; ++j, data += outs[i].cols) + { + cv::Mat scores = outs[i].row(j).colRange(5, outs[i].cols); + cv::Point classIdPoint; + double confidence; + // Get the value and location of the maximum score + cv::minMaxLoc(scores, 0, &confidence, 0, &classIdPoint); + if (confidence > confThreshold) + { + int centerX = (int)(data[0] * frameDims.width); + int centerY = (int)(data[1] * frameDims.height); + int width = (int)(data[2] * frameDims.width); + int height = (int)(data[3] * frameDims.height); + int left = centerX - width / 2; + int top = centerY - height / 2; + + classIds.push_back(classIdPoint.x); + confidences.push_back((float)confidence); + boxes.push_back(cv::Rect(left, top, width, height)); + } + } + } + + // Perform non maximum suppression to eliminate redundant overlapping boxes with + // lower confidences + std::vector indices; + cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); + + // Pass boxes to SORT algorithm + std::vector sortBoxes; + for(auto box : boxes) + sortBoxes.push_back(box); + sort.update(sortBoxes, frameId, sqrt(pow(frameDims.width,2) + pow(frameDims.height, 2)), confidences, classIds); + + // Clear data vectors + boxes.clear(); confidences.clear(); classIds.clear(); + // Get SORT predicted boxes + for(auto TBox : sort.frameTrackingResult){ + if(TBox.frame == frameId){ + boxes.push_back(TBox.box); + confidences.push_back(TBox.confidence); + classIds.push_back(TBox.classId); + } + } + + // Remove boxes based on controids distance + for(uint i = 0; i= confidences[j]){ + boxes.erase(boxes.begin() + j); + classIds.erase(classIds.begin() + j); + confidences.erase(confidences.begin() + j); + break; + } + else{ + boxes.erase(boxes.begin() + i); + classIds.erase(classIds.begin() + i); + confidences.erase(confidences.begin() + i); + i = 0; + break; + } + } + } + } + } + + // Remove boxes based in IOU score + for(uint i = 0; i= confidences[j]){ + boxes.erase(boxes.begin() + j); + classIds.erase(classIds.begin() + j); + confidences.erase(confidences.begin() + j); + break; + } + else{ + boxes.erase(boxes.begin() + i); + classIds.erase(classIds.begin() + i); + confidences.erase(confidences.begin() + i); + i = 0; + break; + } + } + } + } + } + + // Normalize boxes coordinates + std::vector> normalized_boxes; + for(auto box : boxes){ + cv::Rect_ normalized_box; + normalized_box.x = (box.x)/(float)frameDims.width; + normalized_box.y = (box.y)/(float)frameDims.height; + normalized_box.width = (box.width)/(float)frameDims.width; + normalized_box.height = (box.height)/(float)frameDims.height; + normalized_boxes.push_back(normalized_box); + } + + detectionsData[frameId] = CVDetectionData(classIds, confidences, normalized_boxes, frameId); +} + +// Compute IOU between 2 boxes +bool CVObjectDetection::iou(cv::Rect pred_box, cv::Rect sort_box){ + // Determine the (x, y)-coordinates of the intersection rectangle + int xA = std::max(pred_box.x, sort_box.x); + int yA = std::max(pred_box.y, sort_box.y); + int xB = std::min(pred_box.x + pred_box.width, sort_box.x + sort_box.width); + int yB = std::min(pred_box.y + pred_box.height, sort_box.y + sort_box.height); + + // Compute the area of intersection rectangle + int interArea = std::max(0, xB - xA + 1) * std::max(0, yB - yA + 1); + + // Compute the area of both the prediction and ground-truth rectangles + int boxAArea = (pred_box.width + 1) * (pred_box.height + 1); + int boxBArea = (sort_box.width + 1) * (sort_box.height + 1); + + // Compute the intersection over union by taking the intersection + float iou = interArea / (float)(boxAArea + boxBArea - interArea); + + // If IOU is above this value the boxes are very close (probably a variation of the same bounding box) + if(iou > 0.5) + return true; + return false; +} + +// Get the names of the output layers +std::vector CVObjectDetection::getOutputsNames(const cv::dnn::Net& net) +{ + static std::vector names; + + //Get the indices of the output layers, i.e. the layers with unconnected outputs + std::vector outLayers = net.getUnconnectedOutLayers(); + + //get the names of all the layers in the network + std::vector layersNames = net.getLayerNames(); + + // Get the names of the output layers in names + names.resize(outLayers.size()); + for (size_t i = 0; i < outLayers.size(); ++i) + names[i] = layersNames[outLayers[i] - 1]; + return names; +} + +CVDetectionData CVObjectDetection::GetDetectionData(size_t frameId){ + // Check if the stabilizer info for the requested frame exists + if ( detectionsData.find(frameId) == detectionsData.end() ) { + + return CVDetectionData(); + } else { + + return detectionsData[frameId]; + } +} + +bool CVObjectDetection::SaveObjDetectedData(){ + // Create tracker message + pb_objdetect::ObjDetect objMessage; + + //Save class names in protobuf message + for(int i = 0; iassign(classNames.at(i)); + } + + // Iterate over all frames data and save in protobuf message + for(std::map::iterator it=detectionsData.begin(); it!=detectionsData.end(); ++it){ + CVDetectionData dData = it->second; + pb_objdetect::Frame* pbFrameData; + AddFrameDataToProto(objMessage.add_frame(), dData); + } + + // Add timestamp + *objMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL)); + + { + // Write the new message to disk. + std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary); + if (!objMessage.SerializeToOstream(&output)) { + cerr << "Failed to write protobuf message." << endl; + return false; + } + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; + +} + +// Add frame object detection into protobuf message. +void CVObjectDetection::AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, CVDetectionData& dData) { + + // Save frame number and rotation + pbFrameData->set_id(dData.frameId); + + for(size_t i = 0; i < dData.boxes.size(); i++){ + pb_objdetect::Frame_Box* box = pbFrameData->add_bounding_box(); + + // Save bounding box data + box->set_x(dData.boxes.at(i).x); + box->set_y(dData.boxes.at(i).y); + box->set_w(dData.boxes.at(i).width); + box->set_h(dData.boxes.at(i).height); + box->set_classid(dData.classIds.at(i)); + box->set_confidence(dData.confidences.at(i)); + + } +} + +// Load JSON string into this object +void CVObjectDetection::SetJson(const std::string value) { + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + // throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + std::cout<<"JSON is invalid (missing keys or invalid data types)"<SetError(true, "Incorrect path to model config file"); + error = true; + } + + } + if (!root["model-weights"].isNull()){ + modelWeights= (root["model-weights"].asString()); + std::ifstream infile(modelWeights); + if(!infile.good()){ + processingController->SetError(true, "Incorrect path to model weight file"); + error = true; + } + + } + if (!root["class-names"].isNull()){ + classesFile = (root["class-names"].asString()); + + std::ifstream infile(classesFile); + if(!infile.good()){ + processingController->SetError(true, "Incorrect path to class name file"); + error = true; + } + + } +} + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||| + ONLY FOR MAKE TEST +|||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + +// Load protobuf data file +bool CVObjectDetection::_LoadObjDetectdData(){ + // Create tracker message + pb_objdetect::ObjDetect objMessage; + + { + // Read the existing tracker message. + fstream input(protobuf_data_path, ios::in | ios::binary); + if (!objMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + } + + // Make sure classNames and detectionsData are empty + classNames.clear(); detectionsData.clear(); + + // Get all classes names and assign a color to them + for(int i = 0; i < objMessage.classnames_size(); i++){ + classNames.push_back(objMessage.classnames(i)); + } + + // Iterate over all frames of the saved message + for (size_t i = 0; i < objMessage.frame_size(); i++) { + // Create protobuf message reader + const pb_objdetect::Frame& pbFrameData = objMessage.frame(i); + + // Get frame Id + size_t id = pbFrameData.id(); + + // Load bounding box data + const google::protobuf::RepeatedPtrField &pBox = pbFrameData.bounding_box(); + + // Construct data vectors related to detections in the current frame + std::vector classIds; std::vector confidences; std::vector> boxes; + + for(int i = 0; i < pbFrameData.bounding_box_size(); i++){ + // Get bounding box coordinates + float x = pBox.Get(i).x(); float y = pBox.Get(i).y(); + float w = pBox.Get(i).w(); float h = pBox.Get(i).h(); + // Create OpenCV rectangle with the bouding box info + cv::Rect_ box(x, y, w, h); + + // Get class Id (which will be assign to a class name) and prediction confidence + int classId = pBox.Get(i).classid(); float confidence = pBox.Get(i).confidence(); + + // Push back data into vectors + boxes.push_back(box); classIds.push_back(classId); confidences.push_back(confidence); + } + + // Assign data to object detector map + detectionsData[id] = CVDetectionData(classIds, confidences, boxes, id); + } + + // Show the time stamp from the last update in object detector data file + if (objMessage.has_last_updated()) + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(objMessage.last_updated()) << endl; + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} diff --git a/src/CVObjectDetection.h b/src/CVObjectDetection.h new file mode 100644 index 00000000..ae3dbf7d --- /dev/null +++ b/src/CVObjectDetection.h @@ -0,0 +1,133 @@ +/** + * @file + * @brief Header file for CVObjectDetection class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#pragma once + +#define int64 opencv_broken_int +#define uint64 opencv_broken_uint +#include +#include +#include +#undef uint64 +#undef int64 +#include "Json.h" +#include "ProcessingController.h" +#include "Clip.h" +#include "protobuf_messages/objdetectdata.pb.h" + +#include "sort_filter/sort.hpp" + +namespace openshot +{ + // Stores the detected object bounding boxes and its properties. + struct CVDetectionData{ + CVDetectionData(){} + CVDetectionData(std::vector _classIds, std::vector _confidences, std::vector> _boxes, size_t _frameId){ + classIds = _classIds; + confidences = _confidences; + boxes = _boxes; + frameId = _frameId; + } + size_t frameId; + std::vector classIds; + std::vector confidences; + std::vector> boxes; + }; + + /** + * @brief This class runs trought a clip to detect objects and returns the bounding boxes and its properties. + * + * Object detection is performed using YoloV3 model with OpenCV DNN module + */ + class CVObjectDetection{ + + private: + + cv::dnn::Net net; + std::vector classNames; + float confThreshold, nmsThreshold; + + std::string classesFile; + std::string modelConfiguration; + std::string modelWeights; + std::string processingDevice; + std::string protobuf_data_path; + + SortTracker sort; + + uint progress; + + size_t start; + size_t end; + + bool error = false; + + /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes + ProcessingController *processingController; + + void setProcessingDevice(); + + // Detect onbects on a single frame + void DetectObjects(const cv::Mat &frame, size_t frame_number); + + bool iou(cv::Rect pred_box, cv::Rect sort_box); + + // Remove the bounding boxes with low confidence using non-maxima suppression + void postprocess(const cv::Size &frameDims, const std::vector& out, size_t frame_number); + + // Get the names of the output layers + std::vector getOutputsNames(const cv::dnn::Net& net); + + public: + + std::map detectionsData; + + CVObjectDetection(std::string processInfoJson, ProcessingController &processingController); + + // Iterate over a clip object and run inference for each video frame + void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false); + + CVDetectionData GetDetectionData(size_t frameId); + + /// Protobuf Save and Load methods + // Save protobuf file + bool SaveObjDetectedData(); + // Add frame object detection data into protobuf message. + void AddFrameDataToProto(pb_objdetect::Frame* pbFrameData, CVDetectionData& dData); + + /// Get and Set JSON methods + void SetJson(const std::string value); ///< Load JSON string into this object + void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object + + // Load protobuf file (ONLY FOR MAKE TEST) + bool _LoadObjDetectdData(); + }; + +} diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp new file mode 100644 index 00000000..864f9b58 --- /dev/null +++ b/src/CVStabilization.cpp @@ -0,0 +1,435 @@ +/** + * @file + * @brief Source file for CVStabilization class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "CVStabilization.h" +#include + +using namespace std; +using namespace openshot; +using google::protobuf::util::TimeUtil; + +// Set default smoothing window value to compute stabilization +CVStabilization::CVStabilization(std::string processInfoJson, ProcessingController &processingController) +: processingController(&processingController){ + SetJson(processInfoJson); + start = 0; + end = 0; +} + +// Process clip and store necessary stabilization data +void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){ + + if(error){ + return; + } + processingController->SetError(false, ""); + + start = _start; end = _end; + // Compute max and average transformation parameters + avr_dx=0; avr_dy=0; avr_da=0; max_dx=0; max_dy=0; max_da=0; + + video.Open(); + // Save original video width and height + cv::Size readerDims(video.Reader()->info.width, video.Reader()->info.height); + + size_t frame_number; + if(!process_interval || end == 0 || end-start == 0){ + // Get total number of frames in video + start = video.Start() * video.Reader()->info.fps.ToInt(); + end = video.End() * video.Reader()->info.fps.ToInt(); + } + + // Extract and track opticalflow features for each frame + for (frame_number = start; frame_number <= end; frame_number++) + { + // Stop the feature tracker process + if(processingController->ShouldStop()){ + return; + } + + std::shared_ptr f = video.GetFrame(frame_number); + + // Grab OpenCV Mat image + cv::Mat cvimage = f->GetImageCV(); + // Resize frame to original video width and height if they differ + if(cvimage.size().width != readerDims.width || cvimage.size().height != readerDims.height) + cv::resize(cvimage, cvimage, cv::Size(readerDims.width, readerDims.height)); + cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); + + if(!TrackFrameFeatures(cvimage, frame_number)){ + prev_to_cur_transform.push_back(TransformParam(0, 0, 0)); + } + + // Update progress + processingController->SetProgress(uint(100*(frame_number-start)/(end-start))); + } + // Show average and max transformation parameters + std::cout<<"\nAVERAGE DX: "< trajectory = ComputeFramesTrajectory(); + + // Calculate and save smoothed trajectory data + trajectoryData = SmoothTrajectory(trajectory); + + // Calculate and save transformation data + transformationData = GenNewCamPosition(trajectoryData); + + // Normalize smoothed trajectory data + for(auto &dataToNormalize : trajectoryData){ + dataToNormalize.second.x/=readerDims.width; + dataToNormalize.second.y/=readerDims.height; + } + // Normalize transformation data + for(auto &dataToNormalize : transformationData){ + dataToNormalize.second.dx/=readerDims.width; + dataToNormalize.second.dy/=readerDims.height; + } +} + +// Track current frame features and find the relative transformation +bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ + // Check if there are black frames + if(cv::countNonZero(frame) < 1){ + return false; + } + + // Initialize prev_grey if not + if(prev_grey.empty()){ + prev_grey = frame; + return true; + } + + // OpticalFlow features vector + std::vector prev_corner, cur_corner; + std::vector prev_corner2, cur_corner2; + std::vector status; + std::vector err; + // Extract new image features + cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30); + // Track features + cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err); + // Remove untracked features + for(size_t i=0; i < status.size(); i++) { + if(status[i]) { + prev_corner2.push_back(prev_corner[i]); + cur_corner2.push_back(cur_corner[i]); + } + } + // In case no feature was detected + if(prev_corner2.empty() || cur_corner2.empty()){ + last_T = cv::Mat(); + // prev_grey = cv::Mat(); + return false; + } + + // Translation + rotation only + cv::Mat T = cv::estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing + + double da, dx, dy; + // If T has nothing inside return (probably a segment where there is nothing to stabilize) + if(T.size().width == 0 || T.size().height == 0){ + return false; + } + else{ + // If no transformation is found, just use the last known good transform + if(T.data == NULL){ + if(!last_T.empty()) + last_T.copyTo(T); + else + return false; + } + // Decompose T + dx = T.at(0,2); + dy = T.at(1,2); + da = atan2(T.at(1,0), T.at(0,0)); + } + + // Filter transformations parameters, if they are higher than these: return + if(dx > 200 || dy > 200 || da > 0.1){ + return false; + } + + // Keep computing average and max transformation parameters + avr_dx+=fabs(dx); + avr_dy+=fabs(dy); + avr_da+=fabs(da); + if(fabs(dx) > max_dx) + max_dx = dx; + if(fabs(dy) > max_dy) + max_dy = dy; + if(fabs(da) > max_da) + max_da = da; + + T.copyTo(last_T); + + prev_to_cur_transform.push_back(TransformParam(dx, dy, da)); + frame.copyTo(prev_grey); + + // Show processing info + cout << "Frame: " << frameNum << " - good optical flow: " << prev_corner2.size() << endl; + + return true; +} + +std::vector CVStabilization::ComputeFramesTrajectory(){ + + // Accumulated frame to frame transform + double a = 0; + double x = 0; + double y = 0; + + vector trajectory; // trajectory at all frames + + // Compute global camera trajectory. First frame is the origin + for(size_t i=0; i < prev_to_cur_transform.size(); i++) { + x += prev_to_cur_transform[i].dx; + y += prev_to_cur_transform[i].dy; + a += prev_to_cur_transform[i].da; + + // Save trajectory data to vector + trajectory.push_back(CamTrajectory(x,y,a)); + } + return trajectory; +} + +std::map CVStabilization::SmoothTrajectory(std::vector &trajectory){ + + std::map smoothed_trajectory; // trajectory at all frames + + for(size_t i=0; i < trajectory.size(); i++) { + double sum_x = 0; + double sum_y = 0; + double sum_a = 0; + int count = 0; + + for(int j=-smoothingWindow; j <= smoothingWindow; j++) { + if(i+j < trajectory.size()) { + sum_x += trajectory[i+j].x; + sum_y += trajectory[i+j].y; + sum_a += trajectory[i+j].a; + + count++; + } + } + + double avg_a = sum_a / count; + double avg_x = sum_x / count; + double avg_y = sum_y / count; + + // Add smoothed trajectory data to map + smoothed_trajectory[i + start] = CamTrajectory(avg_x, avg_y, avg_a); + } + return smoothed_trajectory; +} + +// Generate new transformations parameters for each frame to follow the smoothed trajectory +std::map CVStabilization::GenNewCamPosition(std::map &smoothed_trajectory){ + std::map new_prev_to_cur_transform; + + // Accumulated frame to frame transform + double a = 0; + double x = 0; + double y = 0; + + for(size_t i=0; i < prev_to_cur_transform.size(); i++) { + x += prev_to_cur_transform[i].dx; + y += prev_to_cur_transform[i].dy; + a += prev_to_cur_transform[i].da; + + // target - current + double diff_x = smoothed_trajectory[i + start].x - x; + double diff_y = smoothed_trajectory[i + start].y - y; + double diff_a = smoothed_trajectory[i + start].a - a; + + double dx = prev_to_cur_transform[i].dx + diff_x; + double dy = prev_to_cur_transform[i].dy + diff_y; + double da = prev_to_cur_transform[i].da + diff_a; + + // Add transformation data to map + new_prev_to_cur_transform[i + start] = TransformParam(dx, dy, da); + } + return new_prev_to_cur_transform; +} + +// Save stabilization data to protobuf file +bool CVStabilization::SaveStabilizedData(){ + // Create stabilization message + pb_stabilize::Stabilization stabilizationMessage; + + std::map::iterator trajData = trajectoryData.begin(); + std::map::iterator transData = transformationData.begin(); + + // Iterate over all frames data and save in protobuf message + for(; trajData != trajectoryData.end(); ++trajData, ++transData){ + AddFrameDataToProto(stabilizationMessage.add_frame(), trajData->second, transData->second, trajData->first); + } + // Add timestamp + *stabilizationMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL)); + + // Write the new message to disk. + std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary); + if (!stabilizationMessage.SerializeToOstream(&output)) { + cerr << "Failed to write protobuf message." << endl; + return false; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + +// Add frame stabilization data into protobuf message +void CVStabilization::AddFrameDataToProto(pb_stabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number){ + + // Save frame number + pbFrameData->set_id(frame_number); + + // Save camera trajectory data + pbFrameData->set_a(trajData.a); + pbFrameData->set_x(trajData.x); + pbFrameData->set_y(trajData.y); + + // Save transformation data + pbFrameData->set_da(transData.da); + pbFrameData->set_dx(transData.dx); + pbFrameData->set_dy(transData.dy); +} + +TransformParam CVStabilization::GetTransformParamData(size_t frameId){ + + // Check if the stabilizer info for the requested frame exists + if ( transformationData.find(frameId) == transformationData.end() ) { + + return TransformParam(); + } else { + + return transformationData[frameId]; + } +} + +CamTrajectory CVStabilization::GetCamTrajectoryTrackedData(size_t frameId){ + + // Check if the stabilizer info for the requested frame exists + if ( trajectoryData.find(frameId) == trajectoryData.end() ) { + + return CamTrajectory(); + } else { + + return trajectoryData[frameId]; + } +} + +// Load JSON string into this object +void CVStabilization::SetJson(const std::string value) { + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +// Load Json::Value into this object +void CVStabilization::SetJsonValue(const Json::Value root) { + + // Set data from Json (if key is found) + if (!root["protobuf_data_path"].isNull()){ + protobuf_data_path = (root["protobuf_data_path"].asString()); + } + if (!root["smoothing-window"].isNull()){ + smoothingWindow = (root["smoothing-window"].asInt()); + } +} + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||| + ONLY FOR MAKE TEST +|||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + +// Load protobuf data file +bool CVStabilization::_LoadStabilizedData(){ + // Create stabilization message + pb_stabilize::Stabilization stabilizationMessage; + // Read the existing tracker message. + fstream input(protobuf_data_path, ios::in | ios::binary); + if (!stabilizationMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + + // Make sure the data maps are empty + transformationData.clear(); + trajectoryData.clear(); + + // Iterate over all frames of the saved message and assign to the data maps + for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) { + const pb_stabilize::Frame& pbFrameData = stabilizationMessage.frame(i); + + // Load frame number + size_t id = pbFrameData.id(); + + // Load camera trajectory data + float x = pbFrameData.x(); + float y = pbFrameData.y(); + float a = pbFrameData.a(); + + // Assign data to trajectory map + trajectoryData[id] = CamTrajectory(x,y,a); + + // Load transformation data + float dx = pbFrameData.dx(); + float dy = pbFrameData.dy(); + float da = pbFrameData.da(); + + // Assing data to transformation map + transformationData[id] = TransformParam(dx,dy,da); + } + + // Show the time stamp from the last update in stabilization data file + if (stabilizationMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} diff --git a/src/CVStabilization.h b/src/CVStabilization.h new file mode 100644 index 00000000..7c41ac7c --- /dev/null +++ b/src/CVStabilization.h @@ -0,0 +1,142 @@ +/** + * @file + * @brief Header file for CVStabilization class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_STABILIZATION_H +#define OPENSHOT_STABILIZATION_H + +#define int64 opencv_broken_int +#define uint64 opencv_broken_uint +#include +#include +#undef uint64 +#undef int64 +#include +#include "protobuf_messages/stabilizedata.pb.h" +#include "ProcessingController.h" +#include "Clip.h" +#include "Json.h" + +// Store the relative transformation parameters between consecutive frames +struct TransformParam +{ + TransformParam() {} + TransformParam(double _dx, double _dy, double _da) { + dx = _dx; + dy = _dy; + da = _da; + } + + double dx; + double dy; + double da; // angle +}; + +// Stores the global camera trajectory for one frame +struct CamTrajectory +{ + CamTrajectory() {} + CamTrajectory(double _x, double _y, double _a) { + x = _x; + y = _y; + a = _a; + } + + double x; + double y; + double a; // angle +}; + + +/** + * @brief This class stabilizes a video frame using optical flow + * + * The relative motion between two consecutive frames is computed to obtain the global camera trajectory. + * The camera trajectory is then smoothed to reduce jittering. + */ +class CVStabilization { + + private: + + int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning + + size_t start; + size_t end; + double avr_dx, avr_dy, avr_da, max_dx, max_dy, max_da; + + cv::Mat last_T; + cv::Mat prev_grey; + std::vector prev_to_cur_transform; // Previous to current + std::string protobuf_data_path; + + uint progress; + bool error = false; + + /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes + ProcessingController *processingController; + + // Track current frame features and find the relative transformation + bool TrackFrameFeatures(cv::Mat frame, size_t frameNum); + + std::vector ComputeFramesTrajectory(); + std::map SmoothTrajectory(std::vector &trajectory); + + // Generate new transformations parameters for each frame to follow the smoothed trajectory + std::map GenNewCamPosition(std::map &smoothed_trajectory); + + public: + + std::map trajectoryData; // Save camera trajectory data + std::map transformationData; // Save transormation data + + // Set default smoothing window value to compute stabilization + CVStabilization(std::string processInfoJson, ProcessingController &processingController); + + // Process clip and store necessary stabilization data + void stabilizeClip(openshot::Clip& video, size_t _start=0, size_t _end=0, bool process_interval=false); + + /// Protobuf Save and Load methods + // Save stabilization data to protobuf file + bool SaveStabilizedData(); + // Add frame stabilization data into protobuf message + void AddFrameDataToProto(pb_stabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number); + + // Return requested struct info for a given frame + TransformParam GetTransformParamData(size_t frameId); + CamTrajectory GetCamTrajectoryTrackedData(size_t frameId); + + /// Get and Set JSON methods + void SetJson(const std::string value); ///< Load JSON string into this object + void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object + + // Load protobuf data file (ONLY FOR MAKE TEST) + bool _LoadStabilizedData(); +}; + +#endif diff --git a/src/CVTracker.cpp b/src/CVTracker.cpp new file mode 100644 index 00000000..f8644859 --- /dev/null +++ b/src/CVTracker.cpp @@ -0,0 +1,361 @@ +/** + * @file + * @brief Track an object selected by the user + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "CVTracker.h" +#include + +using namespace std; +using namespace openshot; +using google::protobuf::util::TimeUtil; + + +// Constructor +CVTracker::CVTracker(std::string processInfoJson, ProcessingController &processingController) +: processingController(&processingController), json_interval(false){ + SetJson(processInfoJson); + start = 0; + end = 0; +} + +// Set desirable tracker method +cv::Ptr CVTracker::selectTracker(std::string trackerType){ + cv::Ptr t; + + if (trackerType == "BOOSTING") + t = cv::TrackerBoosting::create(); + if (trackerType == "MIL") + t = cv::TrackerMIL::create(); + if (trackerType == "KCF") + t = cv::TrackerKCF::create(); + if (trackerType == "TLD") + t = cv::TrackerTLD::create(); + if (trackerType == "MEDIANFLOW") + t = cv::TrackerMedianFlow::create(); + if (trackerType == "MOSSE") + t = cv::TrackerMOSSE::create(); + if (trackerType == "CSRT") + t = cv::TrackerCSRT::create(); + + return t; +} + +// Track object in the hole clip or in a given interval +void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){ + + video.Open(); + if(!json_interval){ + start = _start; end = _end; + + if(!process_interval || end <= 0 || end-start == 0){ + // Get total number of frames in video + start = video.Start() * video.Reader()->info.fps.ToInt(); + end = video.End() * video.Reader()->info.fps.ToInt(); + } + } + else{ + start = start + video.Start() * video.Reader()->info.fps.ToInt(); + end = video.End() * video.Reader()->info.fps.ToInt(); + } + + if(error){ + return; + } + + processingController->SetError(false, ""); + bool trackerInit = false; + + size_t frame; + // Loop through video + for (frame = start; frame <= end; frame++) + { + + // Stop the feature tracker process + if(processingController->ShouldStop()){ + return; + } + + size_t frame_number = frame; + // Get current frame + std::shared_ptr f = video.GetFrame(frame_number); + + // Grab OpenCV Mat image + cv::Mat cvimage = f->GetImageCV(); + + // Pass the first frame to initialize the tracker + if(!trackerInit){ + + // Initialize the tracker + initTracker(cvimage, frame_number); + + trackerInit = true; + } + else{ + // Update the object tracker according to frame + trackerInit = trackFrame(cvimage, frame_number); + + // Draw box on image + FrameData fd = GetTrackedData(frame_number); + + } + // Update progress + processingController->SetProgress(uint(100*(frame_number-start)/(end-start))); + } +} + +// Initialize the tracker +bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){ + + // Create new tracker object + tracker = selectTracker(trackerType); + + // Correct if bounding box contains negative proportions (width and/or height < 0) + if(bbox.width < 0){ + bbox.x = bbox.x - abs(bbox.width); + bbox.width = abs(bbox.width); + } + if(bbox.height < 0){ + bbox.y = bbox.y - abs(bbox.height); + bbox.height = abs(bbox.height); + } + + // Initialize tracker + tracker->init(frame, bbox); + + float fw = frame.size().width; + float fh = frame.size().height; + + // Add new frame data + trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw, + (bbox.y)/fh, + (bbox.x+bbox.width)/fw, + (bbox.y+bbox.height)/fh); + + return true; +} + +// Update the object tracker according to frame +bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){ + // Update the tracking result + bool ok = tracker->update(frame, bbox); + + // Add frame number and box coords if tracker finds the object + // Otherwise add only frame number + if (ok) + { + float fw = frame.size().width; + float fh = frame.size().height; + + std::vector bboxes = {bbox}; + std::vector confidence = {1.0}; + std::vector classId = {1}; + + sort.update(bboxes, frameId, sqrt(pow(frame.rows, 2) + pow(frame.cols, 2)), confidence, classId); + + for(auto TBox : sort.frameTrackingResult) + bbox = TBox.box; + + // Add new frame data + trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw, + (bbox.y)/fh, + (bbox.x+bbox.width)/fw, + (bbox.y+bbox.height)/fh); + } + else + { + // Add new frame data + trackedDataById[frameId] = FrameData(frameId); + } + + return ok; +} + +bool CVTracker::SaveTrackedData(){ + // Create tracker message + pb_tracker::Tracker trackerMessage; + + // Iterate over all frames data and save in protobuf message + for(std::map::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){ + FrameData fData = it->second; + pb_tracker::Frame* pbFrameData; + AddFrameDataToProto(trackerMessage.add_frame(), fData); + } + + // Add timestamp + *trackerMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL)); + + { + // Write the new message to disk. + std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary); + if (!trackerMessage.SerializeToOstream(&output)) { + cerr << "Failed to write protobuf message." << endl; + return false; + } + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; + +} + +// Add frame tracked data into protobuf message. +void CVTracker::AddFrameDataToProto(pb_tracker::Frame* pbFrameData, FrameData& fData) { + + // Save frame number and rotation + pbFrameData->set_id(fData.frame_id); + pbFrameData->set_rotation(0); + + pb_tracker::Frame::Box* box = pbFrameData->mutable_bounding_box(); + // Save bounding box data + box->set_x1(fData.x1); + box->set_y1(fData.y1); + box->set_x2(fData.x2); + box->set_y2(fData.y2); +} + +// Get tracker info for the desired frame +FrameData CVTracker::GetTrackedData(size_t frameId){ + + // Check if the tracker info for the requested frame exists + if ( trackedDataById.find(frameId) == trackedDataById.end() ) { + + return FrameData(); + } else { + + return trackedDataById[frameId]; + } + +} + +// Load JSON string into this object +void CVTracker::SetJson(const std::string value) { + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw openshot::InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +// Load Json::Value into this object +void CVTracker::SetJsonValue(const Json::Value root) { + + // Set data from Json (if key is found) + if (!root["protobuf_data_path"].isNull()){ + protobuf_data_path = (root["protobuf_data_path"].asString()); + } + if (!root["tracker-type"].isNull()){ + trackerType = (root["tracker-type"].asString()); + } + + if (!root["region"].isNull()){ + double x = root["region"]["x"].asDouble(); + double y = root["region"]["y"].asDouble(); + double w = root["region"]["width"].asDouble(); + double h = root["region"]["height"].asDouble(); + cv::Rect2d prev_bbox(x,y,w,h); + bbox = prev_bbox; + } + else{ + processingController->SetError(true, "No initial bounding box selected"); + error = true; + } + + if (!root["region"]["first-frame"].isNull()){ + start = root["region"]["first-frame"].asInt64(); + json_interval = true; + } + else{ + processingController->SetError(true, "No first-frame"); + error = true; + } +} + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||| + ONLY FOR MAKE TEST +|||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + +// Load protobuf data file +bool CVTracker::_LoadTrackedData(){ + // Create tracker message + pb_tracker::Tracker trackerMessage; + + { + // Read the existing tracker message. + fstream input(protobuf_data_path, ios::in | ios::binary); + if (!trackerMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + } + + // Make sure the trackedData is empty + trackedDataById.clear(); + + // Iterate over all frames of the saved message + for (size_t i = 0; i < trackerMessage.frame_size(); i++) { + const pb_tracker::Frame& pbFrameData = trackerMessage.frame(i); + + // Load frame and rotation data + size_t id = pbFrameData.id(); + float rotation = pbFrameData.rotation(); + + // Load bounding box data + const pb_tracker::Frame::Box& box = pbFrameData.bounding_box(); + float x1 = box.x1(); + float y1 = box.y1(); + float x2 = box.x2(); + float y2 = box.y2(); + + // Assign data to tracker map + trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2); + } + + // Show the time stamp from the last update in tracker data file + if (trackerMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + diff --git a/src/CVTracker.h b/src/CVTracker.h new file mode 100644 index 00000000..918072ce --- /dev/null +++ b/src/CVTracker.h @@ -0,0 +1,144 @@ +/** + * @file + * @brief Track an object selected by the user + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_CVTRACKER_H +#define OPENSHOT_CVTRACKER_H + +#define int64 opencv_broken_int +#define uint64 opencv_broken_uint +#include +#include +#include +#undef uint64 +#undef int64 + +#include +#include "Clip.h" +#include "KeyFrame.h" +#include "Frame.h" +#include "Json.h" +#include "ProcessingController.h" +#include "protobuf_messages/trackerdata.pb.h" + +#include "sort_filter/sort.hpp" + +namespace openshot +{ + + // Store the tracked object information for one frame + struct FrameData{ + size_t frame_id = -1; + float rotation = 0; + float x1 = -1; + float y1 = -1; + float x2 = -1; + float y2 = -1; + + // Constructors + FrameData() + {} + + FrameData( size_t _frame_id) + {frame_id = _frame_id;} + + FrameData( size_t _frame_id , float _rotation, float _x1, float _y1, float _x2, float _y2) + { + frame_id = _frame_id; + rotation = _rotation; + x1 = _x1; + y1 = _y1; + x2 = _x2; + y2 = _y2; + } + }; + + /** + * @brief The tracker class will receive one bounding box provided by the user and then iterate over the clip frames + * to return the object position in all the frames. + */ + class CVTracker { + private: + std::map trackedDataById; // Save tracked data + std::string trackerType; // Name of the chosen tracker + cv::Ptr tracker; // Pointer of the selected tracker + + cv::Rect2d bbox; // Bounding box coords + SortTracker sort; + + std::string protobuf_data_path; // Path to protobuf data file + + uint progress; // Pre-processing effect progress + + /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes + ProcessingController *processingController; + + bool json_interval; + size_t start; + size_t end; + + bool error = false; + + // Initialize the tracker + bool initTracker(cv::Mat &frame, size_t frameId); + + // Update the object tracker according to frame + bool trackFrame(cv::Mat &frame, size_t frameId); + + public: + + // Constructor + CVTracker(std::string processInfoJson, ProcessingController &processingController); + + // Set desirable tracker method + cv::Ptr selectTracker(std::string trackerType); + + // Track object in the hole clip or in a given interval + // If start, end and process_interval are passed as argument, clip will be processed in [start,end) + void trackClip(openshot::Clip& video, size_t _start=0, size_t _end=0, bool process_interval=false); + + // Get tracked data for a given frame + FrameData GetTrackedData(size_t frameId); + + /// Protobuf Save and Load methods + // Save protobuf file + bool SaveTrackedData(); + // Add frame tracked data into protobuf message. + void AddFrameDataToProto(pb_tracker::Frame* pbFrameData, FrameData& fData); + + /// Get and Set JSON methods + void SetJson(const std::string value); ///< Load JSON string into this object + void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object + + // Load protobuf file (ONLY FOR MAKE TEST) + bool _LoadTrackedData(); + }; +} + +#endif diff --git a/src/Clip.cpp b/src/Clip.cpp index df746751..b98df04b 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -983,11 +983,11 @@ void Clip::SetJsonValue(const Json::Value root) { for (const auto existing_effect : root["effects"]) { // Create Effect EffectBase *e = NULL; - if (!existing_effect["type"].isNull()) { - // Create instance of effect - if ( (e = EffectInfo().CreateEffect(existing_effect["type"].asString())) ) { + // Create instance of effect + if ( (e = EffectInfo().CreateEffect(existing_effect["type"].asString()))) { + // Load Json into Effect e->SetJsonValue(existing_effect); diff --git a/src/Clip.h b/src/Clip.h index e8b75c79..ca303c16 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -31,6 +31,16 @@ #ifndef OPENSHOT_CLIP_H #define OPENSHOT_CLIP_H +#ifdef USE_OPENCV + #define int64 opencv_broken_int + #define uint64 opencv_broken_uint + #include + #include + #undef uint64 + #undef int64 + +#endif + #include #include #include @@ -46,7 +56,9 @@ #include "ReaderBase.h" #include "JuceHeader.h" + namespace openshot { + class EffectInfo; /// Comparison method for sorting effect pointers (by Position, Layer, and Order). Effects are sorted /// from lowest layer to top layer (since that is sequence clips are combined), and then by @@ -148,6 +160,8 @@ namespace openshot { /// Reverse an audio buffer void reverse_buffer(juce::AudioSampleBuffer* buffer); + + public: openshot::GravityType gravity; ///< The gravity of a clip determines where it snaps to its parent openshot::ScaleType scale; ///< The scale determines how a clip should be resized to fit its parent @@ -155,6 +169,12 @@ namespace openshot { openshot::FrameDisplayType display; ///< The format to display the frame number (if any) openshot::VolumeMixType mixing; ///< What strategy should be followed when mixing audio with other clips + #ifdef USE_OPENCV + bool COMPILED_WITH_CV = true; + #else + bool COMPILED_WITH_CV = false; + #endif + /// Default Constructor Clip(); @@ -178,8 +198,6 @@ namespace openshot { /// Return the type name of the class std::string Name() override { return "Clip"; }; - - /// @brief Add an effect to the clip /// @param effect Add an effect to the clip. An effect can modify the audio or video of an openshot::Frame. void AddEffect(openshot::EffectBase* effect); diff --git a/src/ClipProcessingJobs.cpp b/src/ClipProcessingJobs.cpp new file mode 100644 index 00000000..fb64fd78 --- /dev/null +++ b/src/ClipProcessingJobs.cpp @@ -0,0 +1,115 @@ +#include "ClipProcessingJobs.h" + +// Constructor responsible to choose processing type and apply to clip +ClipProcessingJobs::ClipProcessingJobs(std::string processingType, std::string processInfoJson) : +processingType(processingType), processInfoJson(processInfoJson){ +} + +void ClipProcessingJobs::processClip(Clip& clip, std::string json){ + processInfoJson = json; + + // Process clip and save processed data + if(processingType == "Stabilizer"){ + t = std::thread(&ClipProcessingJobs::stabilizeClip, this, std::ref(clip), std::ref(this->processingController)); + } + if(processingType == "Tracker"){ + t = std::thread(&ClipProcessingJobs::trackClip, this, std::ref(clip), std::ref(this->processingController)); + } + if(processingType == "Object Detector"){ + t = std::thread(&ClipProcessingJobs::detectObjectsClip, this, std::ref(clip), std::ref(this->processingController)); + } +} + +// Apply object tracking to clip +void ClipProcessingJobs::trackClip(Clip& clip, ProcessingController& controller){ + + // Create CVTracker object + CVTracker tracker(processInfoJson, controller); + // Start tracking + tracker.trackClip(clip); + + // Thread controller. If effect processing is done, save data + // Else, kill thread + if(controller.ShouldStop()){ + controller.SetFinished(true); + return; + } + else{ + // Save stabilization data + tracker.SaveTrackedData(); + // tells to UI that the processing finished + controller.SetFinished(true); + } + +} + +// Apply object detection to clip +void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& controller){ + // create CVObjectDetection object + CVObjectDetection objDetector(processInfoJson, controller); + // Start object detection process + objDetector.detectObjectsClip(clip); + + // Thread controller. If effect processing is done, save data + // Else, kill thread + if(controller.ShouldStop()){ + controller.SetFinished(true); + return; + } + else{ + // Save object detection data + objDetector.SaveObjDetectedData(); + // tells to UI that the processing finished + controller.SetFinished(true); + } +} + +void ClipProcessingJobs::stabilizeClip(Clip& clip, ProcessingController& controller){ + // create CVStabilization object + CVStabilization stabilizer(processInfoJson, controller); + // Start stabilization process + stabilizer.stabilizeClip(clip); + + // Thread controller. If effect processing is done, save data + // Else, kill thread + if(controller.ShouldStop()){ + controller.SetFinished(true); + return; + } + else{ + // Save stabilization data + stabilizer.SaveStabilizedData(); + // tells to UI that the processing finished + controller.SetFinished(true); + } +} + +// Get processing progress while iterating on the clip +int ClipProcessingJobs::GetProgress(){ + + return (int)processingController.GetProgress(); +} + +// Check if processing finished +bool ClipProcessingJobs::IsDone(){ + + if(processingController.GetFinished()){ + t.join(); + } + return processingController.GetFinished(); +} + +// stop preprocessing before finishing it +void ClipProcessingJobs::CancelProcessing(){ + processingController.CancelProcessing(); +} + +// check if there is an error with the config +bool ClipProcessingJobs::GetError(){ + return processingController.GetError(); +} + +// get the error message +std::string ClipProcessingJobs::GetErrorMessage(){ + return processingController.GetErrorMessage(); +} \ No newline at end of file diff --git a/src/ClipProcessingJobs.h b/src/ClipProcessingJobs.h new file mode 100644 index 00000000..2a34d46e --- /dev/null +++ b/src/ClipProcessingJobs.h @@ -0,0 +1,88 @@ +/** + * @file + * @brief Header for the ClipProcessingJobs class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + + +#ifdef USE_OPENCV + #define int64 opencv_broken_int + #define uint64 opencv_broken_uint + #include + #include + #undef uint64 + #undef int64 + + #include "CVStabilization.h" + #include "CVTracker.h" + #include "CVObjectDetection.h" +#endif + +#include +#include "ProcessingController.h" +#include "Clip.h" + +using namespace openshot; + +// Constructor responsible to choose processing type and apply to clip +class ClipProcessingJobs{ + private: + std::string processInfoJson; + std::string processingType; + + bool processingDone = false; + bool stopProcessing = false; + uint processingProgress = 0; + + std::thread t; + + /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes + ProcessingController processingController; + + // Apply object tracking to clip + void trackClip(Clip& clip, ProcessingController& controller); + // Apply stabilization to clip + void stabilizeClip(Clip& clip, ProcessingController& controller); + // Apply object detection to clip + void detectObjectsClip(Clip& clip, ProcessingController& controller); + + + public: + // Constructor + ClipProcessingJobs(std::string processingType, std::string processInfoJson); + // Process clip accordingly to processingType + void processClip(Clip& clip, std::string json); + + // Thread related variables and methods + int GetProgress(); + bool IsDone(); + void CancelProcessing(); + bool GetError(); + std::string GetErrorMessage(); + + +}; \ No newline at end of file diff --git a/src/EffectInfo.cpp b/src/EffectInfo.cpp index c68db07e..fb292dde 100644 --- a/src/EffectInfo.cpp +++ b/src/EffectInfo.cpp @@ -87,6 +87,18 @@ EffectBase* EffectInfo::CreateEffect(std::string effect_type) { else if (effect_type == "Wave") return new Wave(); + + #ifdef USE_OPENCV + else if(effect_type == "Stabilizer") + return new Stabilizer(); + + else if(effect_type == "Tracker") + return new Tracker(); + + else if(effect_type == "Object Detector") + return new ObjectDetection(); + #endif + return NULL; } @@ -113,6 +125,12 @@ Json::Value EffectInfo::JsonValue() { root.append(Shift().JsonInfo()); root.append(Wave().JsonInfo()); + #ifdef USE_OPENCV + root.append(Stabilizer().JsonInfo()); + root.append(Tracker().JsonInfo()); + root.append(ObjectDetection().JsonInfo()); + #endif + // return JsonValue return root; diff --git a/src/EffectInfo.h b/src/EffectInfo.h index 0e64327b..e3e20b47 100644 --- a/src/EffectInfo.h +++ b/src/EffectInfo.h @@ -34,9 +34,10 @@ #include "Effects.h" + namespace openshot { - + class Clip; /** * @brief This class returns a listing of all effects supported by libopenshot * diff --git a/src/Effects.h b/src/Effects.h index 2acddd69..e4abc958 100644 --- a/src/Effects.h +++ b/src/Effects.h @@ -48,5 +48,12 @@ #include "effects/Shift.h" #include "effects/Wave.h" +#ifdef USE_OPENCV +#include "effects/ObjectDetection.h" +#include "effects/Tracker.h" +#include "effects/Stabilizer.h" +#endif + + #endif diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index b51d0b06..59276e81 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -817,6 +817,9 @@ void FFmpegReader::UpdateVideoInfo() { } } +bool FFmpegReader::GetIsDurationKnown() { + return this->is_duration_known; +} std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { // Check for open reader (or throw exception) diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index dde0cb37..78572cac 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -272,6 +272,9 @@ namespace openshot { /// Open File - which is called by the constructor automatically void Open() override; + + /// Return true if frame can be read with GetFrame() + bool GetIsDurationKnown(); }; } diff --git a/src/Frame.cpp b/src/Frame.cpp index d19ab4e5..d25def6d 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -121,6 +121,9 @@ Frame::~Frame() { // Clear all pointers image.reset(); audio.reset(); + #ifdef USE_OPENCV + imagecv.release(); + #endif } // Display the frame image to the screen (primarily used for debugging reasons) @@ -935,6 +938,55 @@ std::shared_ptr Frame::GetImage() return image; } +#ifdef USE_OPENCV + +// Convert Qimage to Mat +cv::Mat Frame::Qimage2mat( std::shared_ptr& qimage) { + + cv::Mat mat = cv::Mat(qimage->height(), qimage->width(), CV_8UC4, (uchar*)qimage->constBits(), qimage->bytesPerLine()).clone(); + cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 ); + int from_to[] = { 0,0, 1,1, 2,2 }; + cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 ); + cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR); + return mat2; +}; + +// Get pointer to OpenCV image object +cv::Mat Frame::GetImageCV() +{ + // Check for blank image + if (!image) + // Fill with black + AddColor(width, height, color); + + // if (imagecv.empty()) + // Convert Qimage to Mat + imagecv = Qimage2mat(image); + + return imagecv; +} + +std::shared_ptr Frame::Mat2Qimage(cv::Mat img){ + cv::cvtColor(img, img, cv::COLOR_BGR2RGB); + QImage qimg((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888); + + std::shared_ptr imgIn = std::make_shared(qimg.copy()); + + // Always convert to RGBA8888 (if different) + if (imgIn->format() != QImage::Format_RGBA8888_Premultiplied) + *imgIn = imgIn->convertToFormat(QImage::Format_RGBA8888_Premultiplied); + + return imgIn; +} + +// Set pointer to OpenCV image object +void Frame::SetImageCV(cv::Mat _image) +{ + imagecv = _image; + image = Mat2Qimage(_image); +} +#endif + #ifdef USE_IMAGEMAGICK // Get pointer to ImageMagick image object std::shared_ptr Frame::GetMagickImage() diff --git a/src/Frame.h b/src/Frame.h index b3416d38..75976f17 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -26,11 +26,20 @@ * * You should have received a copy of the GNU Lesser General Public License * along with OpenShot Library. If not, see . - */ + */ #ifndef OPENSHOT_FRAME_H #define OPENSHOT_FRAME_H +#ifdef USE_OPENCV + #define int64 opencv_broken_int + #define uint64 opencv_broken_uint + #include + #undef uint64 + #undef int64 +#endif + + #include #include #include @@ -114,6 +123,10 @@ namespace openshot std::string color; int64_t max_audio_sample; ///< The max audio sample count added to this frame +#ifdef USE_OPENCV + cv::Mat imagecv; ///< OpenCV image. It will always be in BGR format +#endif + /// Constrain a color value from 0 to 255 int constrain(int color_value); @@ -149,7 +162,7 @@ namespace openshot /// Add (or replace) pixel data to the frame void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_); - + /// Add (or replace) pixel data to the frame void AddImage(std::shared_ptr new_image); @@ -278,6 +291,20 @@ namespace openshot /// Play audio samples for this frame void Play(); + +#ifdef USE_OPENCV + /// Convert Qimage to Mat + cv::Mat Qimage2mat( std::shared_ptr& qimage); + + /// Convert OpenCV Mat to QImage + std::shared_ptr Mat2Qimage(cv::Mat img); + + /// Get pointer to OpenCV Mat image object + cv::Mat GetImageCV(); + + /// Set pointer to OpenCV image object + void SetImageCV(cv::Mat _image); +#endif }; } diff --git a/src/OpenShot.h b/src/OpenShot.h index c1428342..268c6058 100644 --- a/src/OpenShot.h +++ b/src/OpenShot.h @@ -141,5 +141,11 @@ #include "TimelineBase.h" #include "Timeline.h" #include "Settings.h" +#ifdef USE_OPENCV + #include "ClipProcessingJobs.h" + #include "CVStabilization.h" + #include "CVTracker.h" + #include "CVObjectDetection.h" +#endif #endif diff --git a/src/OpenShotVersion.h.in b/src/OpenShotVersion.h.in index 5e86e8ce..e0829c00 100644 --- a/src/OpenShotVersion.h.in +++ b/src/OpenShotVersion.h.in @@ -49,6 +49,7 @@ #cmakedefine AVUTIL_VERSION_STR "@AVUTIL_VERSION_STR@" #cmakedefine01 HAVE_IMAGEMAGICK #cmakedefine01 HAVE_RESVG +#cmakedefine01 HAVE_OPENCV #cmakedefine01 APPIMAGE_BUILD #include diff --git a/src/ProcessingController.h b/src/ProcessingController.h new file mode 100644 index 00000000..98c77888 --- /dev/null +++ b/src/ProcessingController.h @@ -0,0 +1,113 @@ +/** + * @file + * @brief This is a message class for thread safe comunication between ClipProcessingJobs and OpenCV classes + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_PROCESSINGCONTROLLER_H +#define OPENSHOT_PROCESSINGCONTROLLER_H + +#include +#include +#include + + +class ProcessingController{ + private: + uint processingProgress; + bool processingFinished; + bool stopProcessing; + bool error = true; + std::string error_message; + + std::mutex mtxProgress; + std::mutex mtxFinished; + std::mutex mtxStop; + std::mutex mtxerror; + + public: + + ProcessingController(){ + processingProgress = 0; + stopProcessing = false; + processingFinished = false; + } + + int GetFinished(){ + std::lock_guard lck (mtxFinished); + bool f = processingFinished; + return f; + } + + void SetFinished(bool f){ + std::lock_guard lck (mtxFinished); + processingFinished = f; + } + + void SetProgress(uint p){ + std::lock_guard lck (mtxProgress); + processingProgress = p; + } + + int GetProgress(){ + std::lock_guard lck (mtxProgress); + uint p = processingProgress; + return p; + } + + void CancelProcessing(){ + std::lock_guard lck (mtxStop); + stopProcessing = true; + } + + bool ShouldStop(){ + std::lock_guard lck (mtxStop); + bool s = stopProcessing; + return s; + } + + void SetError(bool err, std::string message){ + std::lock_guard lck (mtxerror); + error = err; + error_message = message; + } + + bool GetError(){ + std::lock_guard lck (mtxerror); + bool e = error; + return e; + } + + std::string GetErrorMessage(){ + std::lock_guard lck (mtxerror); + std::string message = error_message; + return message; + } + +}; + +#endif \ No newline at end of file diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index fd92655e..879c7421 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -194,10 +194,10 @@ namespace openshot // Stop video/audio playback void PlayerPrivate::stopPlayback(int timeOutMilliseconds) { - if (isThreadRunning()) stopThread(timeOutMilliseconds); if (audioPlayback->isThreadRunning() && reader->info.has_audio) audioPlayback->stopThread(timeOutMilliseconds); if (videoCache->isThreadRunning() && reader->info.has_video) videoCache->stopThread(timeOutMilliseconds); if (videoPlayback->isThreadRunning() && reader->info.has_video) videoPlayback->stopThread(timeOutMilliseconds); + if (isThreadRunning()) stopThread(timeOutMilliseconds); } } diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 1ec52dd7..82b3c146 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -94,43 +94,43 @@ namespace openshot while (!threadShouldExit() && is_playing) { - // Cache frames before the other threads need them - // Cache frames up to the max frames. Reset to current position - // if cache gets too far away from display frame. Cache frames - // even when player is paused (i.e. speed 0). - while ((position - current_display_frame) < max_frames) - { - // Only cache up till the max_frames amount... then sleep - try + // Cache frames before the other threads need them + // Cache frames up to the max frames. Reset to current position + // if cache gets too far away from display frame. Cache frames + // even when player is paused (i.e. speed 0). + while (((position - current_display_frame) < max_frames) && is_playing) { - if (reader) { - ZmqLogger::Instance()->AppendDebugMethod("VideoCacheThread::run (cache frame)", "position", position, "current_display_frame", current_display_frame, "max_frames", max_frames, "needed_frames", (position - current_display_frame)); + // Only cache up till the max_frames amount... then sleep + try + { + if (reader) { + ZmqLogger::Instance()->AppendDebugMethod("VideoCacheThread::run (cache frame)", "position", position, "current_display_frame", current_display_frame, "max_frames", max_frames, "needed_frames", (position - current_display_frame)); - // Force the frame to be generated - if (reader->GetCache()->GetSmallestFrame()) { - int64_t smallest_cached_frame = reader->GetCache()->GetSmallestFrame()->number; - if (smallest_cached_frame > current_display_frame) { - // Cache position has gotten too far away from current display frame. - // Reset the position to the current display frame. - position = current_display_frame; + // Force the frame to be generated + if (reader->GetCache()->GetSmallestFrame()) { + int64_t smallest_cached_frame = reader->GetCache()->GetSmallestFrame()->number; + if (smallest_cached_frame > current_display_frame) { + // Cache position has gotten too far away from current display frame. + // Reset the position to the current display frame. + position = current_display_frame; + } } + reader->GetFrame(position); } - reader->GetFrame(position); + + } + catch (const OutOfBoundsFrame & e) + { + // Ignore out of bounds frame exceptions } - } - catch (const OutOfBoundsFrame & e) - { - // Ignore out of bounds frame exceptions + // Increment frame number + position++; } - // Increment frame number - position++; - } - - // Sleep for 1 frame length - std::this_thread::sleep_for(frame_duration); - } + // Sleep for 1 frame length + std::this_thread::sleep_for(frame_duration); + } return; } diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp new file mode 100644 index 00000000..7c84e716 --- /dev/null +++ b/src/effects/ObjectDetection.cpp @@ -0,0 +1,280 @@ +/** + * @file + * @brief Source file for Object Detection effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "effects/ObjectDetection.h" +#include "effects/Tracker.h" + +using namespace std; +using namespace openshot; + +/// Blank constructor, useful when using Json to load the effect properties +ObjectDetection::ObjectDetection(std::string clipObDetectDataPath) +{ + // Init effect properties + init_effect_details(); + + // Tries to load the tracker data from protobuf + LoadObjDetectdData(clipObDetectDataPath); +} + +// Default constructor +ObjectDetection::ObjectDetection() +{ + // Init effect properties + init_effect_details(); + +} + +// Init effect settings +void ObjectDetection::init_effect_details() +{ + /// Initialize the values of the EffectInfo struct. + InitEffectInfo(); + + /// Set the effect info + info.class_name = "Object Detector"; + info.name = "Object Detector"; + info.description = "Detect objects through the video."; + info.has_audio = false; + info.has_video = true; +} + +// This method is required for all derived classes of EffectBase, and returns a +// modified openshot::Frame object +std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, int64_t frame_number) +{ + // Get the frame's image + cv::Mat cv_image = frame->GetImageCV(); + + // Check if frame isn't NULL + if(cv_image.empty()){ + return frame; + } + + // Check if track data exists for the requested frame + if (detectionsData.find(frame_number) != detectionsData.end()) { + float fw = cv_image.size().width; + float fh = cv_image.size().height; + + DetectionData detections = detectionsData[frame_number]; + for(int i = 0; i bb_nrml = detections.boxes.at(i); + cv::Rect2d box((int)(bb_nrml.x*fw), + (int)(bb_nrml.y*fh), + (int)(bb_nrml.width*fw), + (int)(bb_nrml.height*fh)); + drawPred(detections.classIds.at(i), detections.confidences.at(i), + box, cv_image); + } + } + + // Set image with drawn box to frame + // If the input image is NULL or doesn't have tracking data, it's returned as it came + frame->SetImageCV(cv_image); + + return frame; +} + +void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame) +{ + + //Draw a rectangle displaying the bounding box + cv::rectangle(frame, box, classesColor[classId], 2); + + //Get the label for the class name and its confidence + std::string label = cv::format("%.2f", conf); + if (!classNames.empty()) + { + CV_Assert(classId < (int)classNames.size()); + label = classNames[classId] + ":" + label; + } + + //Display the label at the top of the bounding box + int baseLine; + cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + double left = box.x; + double top = std::max((int)box.y, labelSize.height); + + cv::rectangle(frame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine), classesColor[classId], cv::FILLED); + putText(frame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1); +} + +// Load protobuf data file +bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){ + // Create tracker message + pb_objdetect::ObjDetect objMessage; + + { + // Read the existing tracker message. + fstream input(inputFilePath, ios::in | ios::binary); + if (!objMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + } + + // Make sure classNames and detectionsData are empty + classNames.clear(); + detectionsData.clear(); + + // Seed to generate same random numbers + std::srand(1); + // Get all classes names and assign a color to them + for(int i = 0; i < objMessage.classnames_size(); i++){ + classNames.push_back(objMessage.classnames(i)); + classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50)); + } + + // Iterate over all frames of the saved message + for (size_t i = 0; i < objMessage.frame_size(); i++) { + // Create protobuf message reader + const pb_objdetect::Frame& pbFrameData = objMessage.frame(i); + + // Get frame Id + size_t id = pbFrameData.id(); + + // Load bounding box data + const google::protobuf::RepeatedPtrField &pBox = pbFrameData.bounding_box(); + + // Construct data vectors related to detections in the current frame + std::vector classIds; + std::vector confidences; + std::vector> boxes; + + for(int i = 0; i < pbFrameData.bounding_box_size(); i++){ + // Get bounding box coordinates + float x = pBox.Get(i).x(); + float y = pBox.Get(i).y(); + float w = pBox.Get(i).w(); + float h = pBox.Get(i).h(); + // Get class Id (which will be assign to a class name) + int classId = pBox.Get(i).classid(); + // Get prediction confidence + float confidence = pBox.Get(i).confidence(); + + // Create OpenCV rectangle with the bouding box info + cv::Rect_ box(x, y, w, h); + + // Push back data into vectors + boxes.push_back(box); + classIds.push_back(classId); + confidences.push_back(confidence); + } + + // Assign data to object detector map + detectionsData[id] = DetectionData(classIds, confidences, boxes, id); + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + +// Get tracker info for the desired frame +DetectionData ObjectDetection::GetTrackedData(size_t frameId){ + + // Check if the tracker info for the requested frame exists + if ( detectionsData.find(frameId) == detectionsData.end() ) { + return DetectionData(); + } else { + return detectionsData[frameId]; + } + +} + +// Generate JSON string of this object +std::string ObjectDetection::Json() const { + + // Return formatted string + return JsonValue().toStyledString(); +} + +// Generate Json::Value for this object +Json::Value ObjectDetection::JsonValue() const { + + // Create root json object + Json::Value root = EffectBase::JsonValue(); // get parent properties + root["type"] = info.class_name; + root["protobuf_data_path"] = protobuf_data_path; + + // return JsonValue + return root; +} + +// Load JSON string into this object +void ObjectDetection::SetJson(const std::string value) { + + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +// Load Json::Value into this object +void ObjectDetection::SetJsonValue(const Json::Value root) { + + // Set parent data + EffectBase::SetJsonValue(root); + // Set data from Json (if key is found) + if (!root["protobuf_data_path"].isNull()){ + protobuf_data_path = (root["protobuf_data_path"].asString()); + + if(!LoadObjDetectdData(protobuf_data_path)){ + std::cout<<"Invalid protobuf data path"; + protobuf_data_path = ""; + } + } +} + +// Get all properties for a specific frame +std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const { + + // Generate JSON properties list + Json::Value root; + root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame); + root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame); + root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); + + // Return formatted string + return root.toStyledString(); +} diff --git a/src/effects/ObjectDetection.h b/src/effects/ObjectDetection.h new file mode 100644 index 00000000..31518c86 --- /dev/null +++ b/src/effects/ObjectDetection.h @@ -0,0 +1,118 @@ +/** + * @file + * @brief Header file for Object Detection effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_OBJECT_DETECTION_EFFECT_H +#define OPENSHOT_OBJECT_DETECTION_EFFECT_H + +#include "../EffectBase.h" + +#include +#include +#include +#include +#include "../Color.h" +#include "../Json.h" +#include "../KeyFrame.h" +#include "protobuf_messages/objdetectdata.pb.h" + +// Struct that stores the detected bounding boxes for all the clip frames +struct DetectionData{ + DetectionData(){} + DetectionData(std::vector _classIds, std::vector _confidences, std::vector> _boxes, size_t _frameId){ + classIds = _classIds; + confidences = _confidences; + boxes = _boxes; + frameId = _frameId; + } + size_t frameId; + std::vector classIds; + std::vector confidences; + std::vector> boxes; +}; + +namespace openshot +{ + /** + * @brief This effect displays all the detected objects on a clip. + */ + class ObjectDetection : public EffectBase + { + private: + std::string protobuf_data_path; + std::map detectionsData; + std::vector classNames; + + std::vector classesColor; + + /// Init effect settings + void init_effect_details(); + + void drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame); + + public: + + /// Blank constructor, useful when using Json to load the effect properties + ObjectDetection(std::string clipTrackerDataPath); + + /// Default constructor + ObjectDetection(); + + /// @brief This method is required for all derived classes of EffectBase, and returns a + /// modified openshot::Frame object + /// + /// The frame object is passed into this method, and a frame_number is passed in which + /// tells the effect which settings to use from its keyframes (starting at 1). + /// + /// @returns The modified openshot::Frame object + /// @param frame The frame object that needs the effect applied to it + /// @param frame_number The frame number (starting at 1) of the effect on the timeline. + std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + + std::shared_ptr GetFrame(int64_t frame_number) override { return GetFrame(std::shared_ptr (new Frame()), frame_number); } + + // Load protobuf data file + bool LoadObjDetectdData(std::string inputFilePath); + + DetectionData GetTrackedData(size_t frameId); + + /// Get and Set JSON methods + std::string Json() const override; ///< Generate JSON string of this object + void SetJson(const std::string value) override; ///< Load JSON string into this object + Json::Value JsonValue() const override; ///< Generate Json::Value for this object + void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object + + /// Get all properties for a specific frame (perfect for a UI to display the current state + /// of all properties at any time) + std::string PropertiesJSON(int64_t requested_frame) const override; + }; + +} + +#endif diff --git a/src/effects/Stabilizer.cpp b/src/effects/Stabilizer.cpp new file mode 100644 index 00000000..8c507784 --- /dev/null +++ b/src/effects/Stabilizer.cpp @@ -0,0 +1,243 @@ +/** + * @file + * @brief Source file for Stabilizer effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "effects/Stabilizer.h" +#include + +using namespace std; +using namespace openshot; +using google::protobuf::util::TimeUtil; + +/// Blank constructor, useful when using Json to load the effect properties +Stabilizer::Stabilizer(std::string clipStabilizedDataPath):protobuf_data_path(clipStabilizedDataPath) +{ + // Init effect properties + init_effect_details(); + // Tries to load the stabilization data from protobuf + LoadStabilizedData(clipStabilizedDataPath); +} + +// Default constructor +Stabilizer::Stabilizer() +{ + // Init effect properties + init_effect_details(); +} + +// Init effect settings +void Stabilizer::init_effect_details() +{ + + /// Initialize the values of the EffectInfo struct. + InitEffectInfo(); + + /// Set the effect info + info.class_name = "Stabilizer"; + info.name = "Stabilizer"; + info.description = "Stabilize video clip to remove undesired shaking and jitter."; + info.has_audio = false; + info.has_video = true; + protobuf_data_path = ""; + zoom = 1.0; +} + +// This method is required for all derived classes of EffectBase, and returns a +// modified openshot::Frame object +std::shared_ptr Stabilizer::GetFrame(std::shared_ptr frame, int64_t frame_number) +{ + + // Grab OpenCV Mat image + cv::Mat frame_image = frame->GetImageCV(); + + // If frame is NULL, return itself + if(!frame_image.empty()){ + + // Check if track data exists for the requested frame + if(transformationData.find(frame_number) != transformationData.end()){ + + float zoom_value = zoom.GetValue(frame_number); + + // Create empty rotation matrix + cv::Mat T(2,3,CV_64F); + + // Set rotation matrix values + T.at(0,0) = cos(transformationData[frame_number].da); + T.at(0,1) = -sin(transformationData[frame_number].da); + T.at(1,0) = sin(transformationData[frame_number].da); + T.at(1,1) = cos(transformationData[frame_number].da); + + T.at(0,2) = transformationData[frame_number].dx * frame_image.size().width; + T.at(1,2) = transformationData[frame_number].dy * frame_image.size().height; + + // Apply rotation matrix to image + cv::Mat frame_stabilized; + cv::warpAffine(frame_image, frame_stabilized, T, frame_image.size()); + + // Scale up the image to remove black borders + cv::Mat T_scale = cv::getRotationMatrix2D(cv::Point2f(frame_stabilized.cols/2, frame_stabilized.rows/2), 0, zoom_value); + cv::warpAffine(frame_stabilized, frame_stabilized, T_scale, frame_stabilized.size()); + frame_image = frame_stabilized; + } + } + // Set stabilized image to frame + // If the input image is NULL or doesn't have tracking data, it's returned as it came + frame->SetImageCV(frame_image); + return frame; +} + +// Load protobuf data file +bool Stabilizer::LoadStabilizedData(std::string inputFilePath){ + + // Create stabilization message + pb_stabilize::Stabilization stabilizationMessage; + + // Read the existing tracker message. + fstream input(inputFilePath, ios::in | ios::binary); + if (!stabilizationMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + + // Make sure the data maps are empty + transformationData.clear(); + trajectoryData.clear(); + + // Iterate over all frames of the saved message and assign to the data maps + for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) { + + // Create stabilization message + const pb_stabilize::Frame& pbFrameData = stabilizationMessage.frame(i); + + // Load frame number + size_t id = pbFrameData.id(); + + // Load camera trajectory data + float x = pbFrameData.x(); + float y = pbFrameData.y(); + float a = pbFrameData.a(); + + // Assign data to trajectory map + trajectoryData[i] = EffectCamTrajectory(x,y,a); + + // Load transformation data + float dx = pbFrameData.dx(); + float dy = pbFrameData.dy(); + float da = pbFrameData.da(); + + // Assing data to transformation map + transformationData[id] = EffectTransformParam(dx,dy,da); + } + + // Show the time stamp from the last update in stabilization data file + if (stabilizationMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + + + +// Generate JSON string of this object +std::string Stabilizer::Json() const { + + // Return formatted string + return JsonValue().toStyledString(); +} + +// Generate Json::Value for this object +Json::Value Stabilizer::JsonValue() const { + + // Create root json object + Json::Value root = EffectBase::JsonValue(); // get parent properties + root["type"] = info.class_name; + root["protobuf_data_path"] = protobuf_data_path; + root["zoom"] = zoom.JsonValue(); + + // return JsonValue + return root; +} + +// Load JSON string into this object +void Stabilizer::SetJson(const std::string value) { + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +// Load Json::Value into this object +void Stabilizer::SetJsonValue(const Json::Value root) { + + // Set parent data + EffectBase::SetJsonValue(root); + + // Set data from Json (if key is found) + if (!root["protobuf_data_path"].isNull()){ + protobuf_data_path = (root["protobuf_data_path"].asString()); + + if(!LoadStabilizedData(protobuf_data_path)){ + std::cout<<"Invalid protobuf data path"; + protobuf_data_path = ""; + } + } + if(!root["zoom"].isNull()) + zoom.SetJsonValue(root["zoom"]); +} + +// Get all properties for a specific frame +std::string Stabilizer::PropertiesJSON(int64_t requested_frame) const { + + // Generate JSON properties list + Json::Value root; + root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame); + root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame); + root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); + + root["zoom"] = add_property_json("Zoom", zoom.GetValue(requested_frame), "float", "", &zoom, 0.0, 2.0, false, requested_frame); + + // Return formatted string + return root.toStyledString(); +} diff --git a/src/effects/Stabilizer.h b/src/effects/Stabilizer.h new file mode 100644 index 00000000..0d24b6c0 --- /dev/null +++ b/src/effects/Stabilizer.h @@ -0,0 +1,134 @@ +/** + * @file + * @brief Header file for Stabilizer effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_STABILIZER_EFFECT_H +#define OPENSHOT_STABILIZER_EFFECT_H + +#include "../EffectBase.h" + +#include +#include +#include +#include "../Color.h" +#include "../Json.h" +#include "../KeyFrame.h" +#include "protobuf_messages/stabilizedata.pb.h" + +// Store the relative transformation parameters between consecutive frames +struct EffectTransformParam +{ + EffectTransformParam() {} + EffectTransformParam(double _dx, double _dy, double _da) { + dx = _dx; + dy = _dy; + da = _da; + } + + double dx; + double dy; + double da; // angle +}; + +// Stores the global camera trajectory for one frame +struct EffectCamTrajectory +{ + EffectCamTrajectory() {} + EffectCamTrajectory(double _x, double _y, double _a) { + x = _x; + y = _y; + a = _a; + } + + double x; + double y; + double a; // angle +}; + + +namespace openshot +{ + + /** + * @brief This class stabilizes video clip to remove undesired shaking and jitter. + * + * Adding stabilization is useful to increase video quality overall, since it removes + * from subtle to harsh unexpected camera movements. + */ + class Stabilizer : public EffectBase + { + private: + /// Init effect settings + void init_effect_details(); + std::string protobuf_data_path; + Keyframe zoom; + + public: + std::string teste; + std::map trajectoryData; // Save camera trajectory data + std::map transformationData; // Save transormation data + + /// Blank constructor, useful when using Json to load the effect properties + Stabilizer(std::string clipTrackerDataPath); + + /// Default constructor + Stabilizer(); + + /// @brief This method is required for all derived classes of EffectBase, and returns a + /// modified openshot::Frame object + /// + /// The frame object is passed into this method, and a frame_number is passed in which + /// tells the effect which settings to use from its keyframes (starting at 1). + /// + /// @returns The modified openshot::Frame object + /// @param frame The frame object that needs the effect applied to it + /// @param frame_number The frame number (starting at 1) of the effect on the timeline. + std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + + std::shared_ptr GetFrame(int64_t frame_number) override { + return GetFrame(std::make_shared(), frame_number); + }; + + // Load protobuf data file + bool LoadStabilizedData(std::string inputFilePath); + + /// Get and Set JSON methods + std::string Json() const override; ///< Generate JSON string of this object + void SetJson(const std::string value) override; ///< Load JSON string into this object + Json::Value JsonValue() const override; ///< Generate Json::Value for this object + void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object + + /// Get all properties for a specific frame (perfect for a UI to display the current state + /// of all properties at any time) + std::string PropertiesJSON(int64_t requested_frame) const override; + }; + +} + +#endif diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp new file mode 100644 index 00000000..6b4130a3 --- /dev/null +++ b/src/effects/Tracker.cpp @@ -0,0 +1,229 @@ +/** + * @file + * @brief Source file for Tracker effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "effects/Tracker.h" +#include + +using namespace std; +using namespace openshot; +using google::protobuf::util::TimeUtil; + +/// Blank constructor, useful when using Json to load the effect properties +Tracker::Tracker(std::string clipTrackerDataPath) +{ + // Init effect properties + init_effect_details(); + + // Tries to load the tracker data from protobuf + LoadTrackedData(clipTrackerDataPath); +} + +// Default constructor +Tracker::Tracker() +{ + // Init effect properties + init_effect_details(); + +} + +// Init effect settings +void Tracker::init_effect_details() +{ + /// Initialize the values of the EffectInfo struct. + InitEffectInfo(); + + /// Set the effect info + info.class_name = "Tracker"; + info.name = "Tracker"; + info.description = "Track the selected bounding box through the video."; + info.has_audio = false; + info.has_video = true; +} + +// This method is required for all derived classes of EffectBase, and returns a +// modified openshot::Frame object +std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t frame_number) +{ + + // Get the frame's image + cv::Mat frame_image = frame->GetImageCV(); + + // Check if frame isn't NULL + if(!frame_image.empty()){ + + // Check if track data exists for the requested frame + if (trackedDataById.find(frame_number) != trackedDataById.end()) { + + float fw = frame_image.size().width; + float fh = frame_image.size().height; + + // Draw box on image + EffectFrameData fd = trackedDataById[frame_number]; + cv::Rect2d box((int)(fd.x1*fw), + (int)(fd.y1*fh), + (int)((fd.x2-fd.x1)*fw), + (int)((fd.y2-fd.y1)*fh)); + cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); + } + } + + // Set image with drawn box to frame + // If the input image is NULL or doesn't have tracking data, it's returned as it came + frame->SetImageCV(frame_image); + + return frame; +} + +// Load protobuf data file +bool Tracker::LoadTrackedData(std::string inputFilePath){ + // Create tracker message + pb_tracker::Tracker trackerMessage; + + { + // Read the existing tracker message. + fstream input(inputFilePath, ios::in | ios::binary); + if (!trackerMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + } + + // Make sure the trackedData is empty + trackedDataById.clear(); + + // Iterate over all frames of the saved message + for (size_t i = 0; i < trackerMessage.frame_size(); i++) { + const pb_tracker::Frame& pbFrameData = trackerMessage.frame(i); + + // Load frame and rotation data + size_t id = pbFrameData.id(); + float rotation = pbFrameData.rotation(); + + // Load bounding box data + const pb_tracker::Frame::Box& box = pbFrameData.bounding_box(); + float x1 = box.x1(); + float y1 = box.y1(); + float x2 = box.x2(); + float y2 = box.y2(); + + // Assign data to tracker map + trackedDataById[id] = EffectFrameData(id, rotation, x1, y1, x2, y2); + } + + // Show the time stamp from the last update in tracker data file + if (trackerMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + +// Get tracker info for the desired frame +EffectFrameData Tracker::GetTrackedData(size_t frameId){ + + // Check if the tracker info for the requested frame exists + if ( trackedDataById.find(frameId) == trackedDataById.end() ) { + return EffectFrameData(); + } else { + return trackedDataById[frameId]; + } + +} + +// Generate JSON string of this object +std::string Tracker::Json() const { + + // Return formatted string + return JsonValue().toStyledString(); +} + +// Generate Json::Value for this object +Json::Value Tracker::JsonValue() const { + + // Create root json object + Json::Value root = EffectBase::JsonValue(); // get parent properties + root["type"] = info.class_name; + root["protobuf_data_path"] = protobuf_data_path; + + // return JsonValue + return root; +} + +// Load JSON string into this object +void Tracker::SetJson(const std::string value) { + + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +// Load Json::Value into this object +void Tracker::SetJsonValue(const Json::Value root) { + + // Set parent data + EffectBase::SetJsonValue(root); + // Set data from Json (if key is found) + if (!root["protobuf_data_path"].isNull()){ + protobuf_data_path = (root["protobuf_data_path"].asString()); + + if(!LoadTrackedData(protobuf_data_path)){ + std::cout<<"Invalid protobuf data path"; + protobuf_data_path = ""; + } + } +} + +// Get all properties for a specific frame +std::string Tracker::PropertiesJSON(int64_t requested_frame) const { + + // Generate JSON properties list + Json::Value root; + root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame); + root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame); + root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); + + // Return formatted string + return root.toStyledString(); +} diff --git a/src/effects/Tracker.h b/src/effects/Tracker.h new file mode 100644 index 00000000..ad39b6b9 --- /dev/null +++ b/src/effects/Tracker.h @@ -0,0 +1,128 @@ +/** + * @file + * @brief Header file for Tracker effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_TRACKER_EFFECT_H +#define OPENSHOT_TRACKER_EFFECT_H + +#include "../EffectBase.h" + +#include +#include +#include +#include +#include "../Color.h" +#include "../Json.h" +#include "../KeyFrame.h" +#include "protobuf_messages/trackerdata.pb.h" + +// Tracking info struct +struct EffectFrameData{ + size_t frame_id = -1; + float rotation = 0; + float x1 = -1; + float y1 = -1; + float x2 = -1; + float y2 = -1; + + // Constructors + EffectFrameData() + {} + + EffectFrameData( int _frame_id) + {frame_id = _frame_id;} + + EffectFrameData( int _frame_id , float _rotation, float _x1, float _y1, float _x2, float _y2) + { + frame_id = _frame_id; + rotation = _rotation; + x1 = _x1; + y1 = _y1; + x2 = _x2; + y2 = _y2; + } +}; + + +namespace openshot +{ + /** + * @brief This class track a given object through the clip and, when called, draws a box surrounding it. + * + * Tracking is useful to better visualize and follow the movement of an object through video. + */ + class Tracker : public EffectBase + { + private: + /// Init effect settings + void init_effect_details(); + std::string protobuf_data_path; + + public: + + std::map trackedDataById; // Save object tracking box data + + /// Blank constructor, useful when using Json to load the effect properties + Tracker(std::string clipTrackerDataPath); + + /// Default constructor + Tracker(); + + /// @brief This method is required for all derived classes of EffectBase, and returns a + /// modified openshot::Frame object + /// + /// The frame object is passed into this method, and a frame_number is passed in which + /// tells the effect which settings to use from its keyframes (starting at 1). + /// + /// @returns The modified openshot::Frame object + /// @param frame The frame object that needs the effect applied to it + /// @param frame_number The frame number (starting at 1) of the effect on the timeline. + std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + std::shared_ptr GetFrame(int64_t frame_number) override { return GetFrame(std::shared_ptr (new Frame()), frame_number); } + + // Load protobuf data file + bool LoadTrackedData(std::string inputFilePath); + + // Get tracker info for the desired frame + EffectFrameData GetTrackedData(size_t frameId); + + /// Get and Set JSON methods + std::string Json() const override; ///< Generate JSON string of this object + void SetJson(const std::string value) override; ///< Load JSON string into this object + Json::Value JsonValue() const override; ///< Generate Json::Value for this object + void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object + + /// Get all properties for a specific frame (perfect for a UI to display the current state + /// of all properties at any time) + std::string PropertiesJSON(int64_t requested_frame) const override; + }; + +} + +#endif diff --git a/src/protobuf_messages/CMakeLists.txt b/src/protobuf_messages/CMakeLists.txt new file mode 100644 index 00000000..e62fca64 --- /dev/null +++ b/src/protobuf_messages/CMakeLists.txt @@ -0,0 +1,65 @@ +####################### CMakeLists.txt (libopenshot) ######################### +# @brief CMake build file for libopenshot (used to generate makefiles) +# @author Jonathan Thomas +# @author FeRD (Frank Dana) +# +# @section LICENSE +# +# Copyright (c) 2008-2020 OpenShot Studios, LLC +# . This file is part of +# OpenShot Library (libopenshot), an open-source project dedicated to +# delivering high quality video editing and animation solutions to the +# world. For more information visit . +# +# OpenShot Library (libopenshot) is free software: you can redistribute it +# and/or modify it under the terms of the GNU Lesser General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# OpenShot Library (libopenshot) is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with OpenShot Library. If not, see . +################################################################################ + +# Dependencies +find_package(Protobuf 3 REQUIRED) + +# Create a target for libprotobuf, if necessary (CMake < 3.9) +if(NOT TARGET protobuf::libprotobuf) + add_library(protobuf_TARGET INTERFACE) + target_include_directories(protobuf_TARGET + INTERFACE ${Protobuf_INCLUDE_DIRS}) + target_link_libraries(protobuf_TARGET INTERFACE ${Protobuf_LIBRARIES}) + add_library(protobuf::libprotobuf ALIAS protobuf_TARGET) +endif() + +file(GLOB ProtoFiles "${CMAKE_CURRENT_SOURCE_DIR}/*.proto") +PROTOBUF_GENERATE_CPP(ProtoSources ProtoHeaders ${ProtoFiles}) +add_library(openshot_protobuf SHARED ${ProtoSources} ${ProtoHeaders}) + +set(ProtobufMessagePath ${CMAKE_CURRENT_BINARY_DIR} + CACHE INTERNAL "Path to generated protobuf header files.") + +target_link_libraries(openshot_protobuf protobuf::libprotobuf) + +# Set SONAME and other library properties +set_target_properties(openshot_protobuf PROPERTIES + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_SO_VERSION} + INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib" + ) + +# Install protobuf library and generated headers +install(TARGETS openshot_protobuf + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/libopenshot + ) +install(FILES ${ProtoHeaders} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/libopenshot/protobuf_messages +) diff --git a/src/protobuf_messages/objdetectdata.proto b/src/protobuf_messages/objdetectdata.proto new file mode 100644 index 00000000..49ad94af --- /dev/null +++ b/src/protobuf_messages/objdetectdata.proto @@ -0,0 +1,32 @@ + +// [START declaration] +syntax = "proto3"; +package pb_objdetect; + +import "google/protobuf/timestamp.proto"; +// [END declaration] + +// [START messages] +message Frame { + int32 id = 1; // Frame ID. + + message Box{ + float x = 1; + float y = 2; + float w = 3; + float h = 4; + int32 classId = 5; + float confidence = 6; + } + + repeated Box bounding_box = 2; +} + +message ObjDetect { + repeated Frame frame = 1; + + google.protobuf.Timestamp last_updated = 2; + + repeated string classNames = 3; +} +// [END messages] diff --git a/src/protobuf_messages/stabilizedata.proto b/src/protobuf_messages/stabilizedata.proto new file mode 100644 index 00000000..13a4b08c --- /dev/null +++ b/src/protobuf_messages/stabilizedata.proto @@ -0,0 +1,31 @@ + +// [START declaration] +syntax = "proto3"; +package pb_stabilize; + +import "google/protobuf/timestamp.proto"; +// [END declaration] + +// [START messages] +message Frame { + int32 id = 1; // Frame ID. + + // TransformParam: frame smothed transformation + float dx = 2; + float dy = 3; + float da = 4; + + // CamTrajectory: Shaky camera trajectory. + // Can be used to run different smoothing techniques. + float x = 5; + float y = 6; + float a = 7; + +} + +message Stabilization { + repeated Frame frame = 1; + + google.protobuf.Timestamp last_updated = 2; +} +// [END messages] diff --git a/src/protobuf_messages/trackerdata.proto b/src/protobuf_messages/trackerdata.proto new file mode 100644 index 00000000..24b35bb6 --- /dev/null +++ b/src/protobuf_messages/trackerdata.proto @@ -0,0 +1,29 @@ + +// [START declaration] +syntax = "proto3"; +package pb_tracker; + +import "google/protobuf/timestamp.proto"; +// [END declaration] + +// [START messages] +message Frame { + int32 id = 1; // Frame ID. + float rotation = 2; + + message Box { + float x1 = 1; + float y1 = 2; + float x2 = 3; + float y2 = 4; + } + + Box bounding_box = 3; +} + +message Tracker { + repeated Frame frame = 1; + + google.protobuf.Timestamp last_updated = 2; +} +// [END messages] diff --git a/src/sort_filter/Hungarian.cpp b/src/sort_filter/Hungarian.cpp new file mode 100644 index 00000000..6442d2da --- /dev/null +++ b/src/sort_filter/Hungarian.cpp @@ -0,0 +1,454 @@ +/////////////////////////////////////////////////////////////////////////////// +// Hungarian.cpp: Implementation file for Class HungarianAlgorithm. +// +// This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren. +// The original implementation is a few mex-functions for use in MATLAB, found here: +// http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem +// +// Both this code and the orignal code are published under the BSD license. +// by Cong Ma, 2016 +// + +#include "Hungarian.h" + +using namespace std; + +HungarianAlgorithm::HungarianAlgorithm() {} +HungarianAlgorithm::~HungarianAlgorithm() {} + +//********************************************************// +// A single function wrapper for solving assignment problem. +//********************************************************// +double HungarianAlgorithm::Solve( + vector> &DistMatrix, + vector &Assignment) +{ + unsigned int nRows = DistMatrix.size(); + unsigned int nCols = DistMatrix[0].size(); + + double *distMatrixIn = new double[nRows * nCols]; + int *assignment = new int[nRows]; + double cost = 0.0; + + // Fill in the distMatrixIn. Mind the index is "i + nRows * j". + // Here the cost matrix of size MxN is defined as a double precision array of N*M elements. + // In the solving functions matrices are seen to be saved MATLAB-internally in row-order. + // (i.e. the matrix [1 2; 3 4] will be stored as a vector [1 3 2 4], NOT [1 2 3 4]). + for (unsigned int i = 0; i < nRows; i++) + for (unsigned int j = 0; j < nCols; j++) + distMatrixIn[i + nRows * j] = DistMatrix[i][j]; + + // call solving function + assignmentoptimal(assignment, &cost, distMatrixIn, nRows, nCols); + + Assignment.clear(); + for (unsigned int r = 0; r < nRows; r++) + Assignment.push_back(assignment[r]); + + delete[] distMatrixIn; + delete[] assignment; + return cost; +} + +//********************************************************// +// Solve optimal solution for assignment problem using Munkres algorithm, also known as Hungarian Algorithm. +//********************************************************// +void HungarianAlgorithm::assignmentoptimal( + int *assignment, + double *cost, + double *distMatrixIn, + int nOfRows, + int nOfColumns) +{ + double *distMatrix, *distMatrixTemp, *distMatrixEnd, *columnEnd, value, minValue; + bool *coveredColumns, *coveredRows, *starMatrix, *newStarMatrix, *primeMatrix; + int nOfElements, minDim, row, col; + + /* initialization */ + *cost = 0; + for (row = 0; row < nOfRows; row++) + assignment[row] = -1; + + /* generate working copy of distance Matrix */ + /* check if all matrix elements are positive */ + nOfElements = nOfRows * nOfColumns; + distMatrix = (double *)malloc(nOfElements * sizeof(double)); + distMatrixEnd = distMatrix + nOfElements; + + for (row = 0; row < nOfElements; row++) + { + value = distMatrixIn[row]; + if (value < 0) + cerr << "All matrix elements have to be non-negative." << endl; + distMatrix[row] = value; + } + + /* memory allocation */ + coveredColumns = (bool *)calloc(nOfColumns, sizeof(bool)); + coveredRows = (bool *)calloc(nOfRows, sizeof(bool)); + starMatrix = (bool *)calloc(nOfElements, sizeof(bool)); + primeMatrix = (bool *)calloc(nOfElements, sizeof(bool)); + newStarMatrix = (bool *)calloc(nOfElements, sizeof(bool)); /* used in step4 */ + + /* preliminary steps */ + if (nOfRows <= nOfColumns) + { + minDim = nOfRows; + + for (row = 0; row < nOfRows; row++) + { + /* find the smallest element in the row */ + distMatrixTemp = distMatrix + row; + minValue = *distMatrixTemp; + distMatrixTemp += nOfRows; + while (distMatrixTemp < distMatrixEnd) + { + value = *distMatrixTemp; + if (value < minValue) + minValue = value; + distMatrixTemp += nOfRows; + } + + /* subtract the smallest element from each element of the row */ + distMatrixTemp = distMatrix + row; + while (distMatrixTemp < distMatrixEnd) + { + *distMatrixTemp -= minValue; + distMatrixTemp += nOfRows; + } + } + + /* Steps 1 and 2a */ + for (row = 0; row < nOfRows; row++) + for (col = 0; col < nOfColumns; col++) + if (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON) + if (!coveredColumns[col]) + { + starMatrix[row + nOfRows * col] = true; + coveredColumns[col] = true; + break; + } + } + else /* if(nOfRows > nOfColumns) */ + { + minDim = nOfColumns; + + for (col = 0; col < nOfColumns; col++) + { + /* find the smallest element in the column */ + distMatrixTemp = distMatrix + nOfRows * col; + columnEnd = distMatrixTemp + nOfRows; + + minValue = *distMatrixTemp++; + while (distMatrixTemp < columnEnd) + { + value = *distMatrixTemp++; + if (value < minValue) + minValue = value; + } + + /* subtract the smallest element from each element of the column */ + distMatrixTemp = distMatrix + nOfRows * col; + while (distMatrixTemp < columnEnd) + *distMatrixTemp++ -= minValue; + } + + /* Steps 1 and 2a */ + for (col = 0; col < nOfColumns; col++) + for (row = 0; row < nOfRows; row++) + if (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON) + if (!coveredRows[row]) + { + starMatrix[row + nOfRows * col] = true; + coveredColumns[col] = true; + coveredRows[row] = true; + break; + } + for (row = 0; row < nOfRows; row++) + coveredRows[row] = false; + } + + /* move to step 2b */ + step2b(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); + + /* compute cost and remove invalid assignments */ + computeassignmentcost(assignment, cost, distMatrixIn, nOfRows); + + /* free allocated memory */ + free(distMatrix); + free(coveredColumns); + free(coveredRows); + free(starMatrix); + free(primeMatrix); + free(newStarMatrix); + + return; +} + +/********************************************************/ +void HungarianAlgorithm::buildassignmentvector( + int *assignment, + bool *starMatrix, + int nOfRows, + int nOfColumns) +{ + for (int row = 0; row < nOfRows; row++) + for (int col = 0; col < nOfColumns; col++) + if (starMatrix[row + nOfRows * col]) + { +#ifdef ONE_INDEXING + assignment[row] = col + 1; /* MATLAB-Indexing */ +#else + assignment[row] = col; +#endif + break; + } +} + +/********************************************************/ +void HungarianAlgorithm::computeassignmentcost( + int *assignment, + double *cost, + double *distMatrix, + int nOfRows) +{ + int row, col; + + for (row = 0; row < nOfRows; row++) + { + col = assignment[row]; + if (col >= 0) + *cost += distMatrix[row + nOfRows * col]; + } +} + +/********************************************************/ +void HungarianAlgorithm::step2a( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + bool *starMatrixTemp, *columnEnd; + int col; + + /* cover every column containing a starred zero */ + for (col = 0; col < nOfColumns; col++) + { + starMatrixTemp = starMatrix + nOfRows * col; + columnEnd = starMatrixTemp + nOfRows; + while (starMatrixTemp < columnEnd) + { + if (*starMatrixTemp++) + { + coveredColumns[col] = true; + break; + } + } + } + + /* move to step 3 */ + step2b(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} + +/********************************************************/ +void HungarianAlgorithm::step2b( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + int col, nOfCoveredColumns; + + /* count covered columns */ + nOfCoveredColumns = 0; + for (col = 0; col < nOfColumns; col++) + if (coveredColumns[col]) + nOfCoveredColumns++; + + if (nOfCoveredColumns == minDim) + { + /* algorithm finished */ + buildassignmentvector(assignment, starMatrix, nOfRows, nOfColumns); + } + else + { + /* move to step 3 */ + step3(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); + } +} + +/********************************************************/ +void HungarianAlgorithm::step3( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + bool zerosFound; + int row, col, starCol; + + zerosFound = true; + while (zerosFound) + { + zerosFound = false; + for (col = 0; col < nOfColumns; col++) + if (!coveredColumns[col]) + for (row = 0; row < nOfRows; row++) + if ((!coveredRows[row]) && (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON)) + { + /* prime zero */ + primeMatrix[row + nOfRows * col] = true; + + /* find starred zero in current row */ + for (starCol = 0; starCol < nOfColumns; starCol++) + if (starMatrix[row + nOfRows * starCol]) + break; + + if (starCol == nOfColumns) /* no starred zero found */ + { + /* move to step 4 */ + step4(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim, row, col); + return; + } + else + { + coveredRows[row] = true; + coveredColumns[starCol] = false; + zerosFound = true; + break; + } + } + } + + /* move to step 5 */ + step5(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} + +/********************************************************/ +void HungarianAlgorithm::step4( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim, + int row, + int col) +{ + int n, starRow, starCol, primeRow, primeCol; + int nOfElements = nOfRows * nOfColumns; + + /* generate temporary copy of starMatrix */ + for (n = 0; n < nOfElements; n++) + newStarMatrix[n] = starMatrix[n]; + + /* star current zero */ + newStarMatrix[row + nOfRows * col] = true; + + /* find starred zero in current column */ + starCol = col; + for (starRow = 0; starRow < nOfRows; starRow++) + if (starMatrix[starRow + nOfRows * starCol]) + break; + + while (starRow < nOfRows) + { + /* unstar the starred zero */ + newStarMatrix[starRow + nOfRows * starCol] = false; + + /* find primed zero in current row */ + primeRow = starRow; + for (primeCol = 0; primeCol < nOfColumns; primeCol++) + if (primeMatrix[primeRow + nOfRows * primeCol]) + break; + + /* star the primed zero */ + newStarMatrix[primeRow + nOfRows * primeCol] = true; + + /* find starred zero in current column */ + starCol = primeCol; + for (starRow = 0; starRow < nOfRows; starRow++) + if (starMatrix[starRow + nOfRows * starCol]) + break; + } + + /* use temporary copy as new starMatrix */ + /* delete all primes, uncover all rows */ + for (n = 0; n < nOfElements; n++) + { + primeMatrix[n] = false; + starMatrix[n] = newStarMatrix[n]; + } + for (n = 0; n < nOfRows; n++) + coveredRows[n] = false; + + /* move to step 2a */ + step2a(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} + +/********************************************************/ +void HungarianAlgorithm::step5( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + double h, value; + int row, col; + + /* find smallest uncovered element h */ + h = DBL_MAX; + for (row = 0; row < nOfRows; row++) + if (!coveredRows[row]) + for (col = 0; col < nOfColumns; col++) + if (!coveredColumns[col]) + { + value = distMatrix[row + nOfRows * col]; + if (value < h) + h = value; + } + + /* add h to each covered row */ + for (row = 0; row < nOfRows; row++) + if (coveredRows[row]) + for (col = 0; col < nOfColumns; col++) + distMatrix[row + nOfRows * col] += h; + + /* subtract h from each uncovered column */ + for (col = 0; col < nOfColumns; col++) + if (!coveredColumns[col]) + for (row = 0; row < nOfRows; row++) + distMatrix[row + nOfRows * col] -= h; + + /* move to step 3 */ + step3(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} diff --git a/src/sort_filter/Hungarian.h b/src/sort_filter/Hungarian.h new file mode 100644 index 00000000..3bd834e7 --- /dev/null +++ b/src/sort_filter/Hungarian.h @@ -0,0 +1,34 @@ +/////////////////////////////////////////////////////////////////////////////// +// Hungarian.h: Header file for Class HungarianAlgorithm. +// +// This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren. +// The original implementation is a few mex-functions for use in MATLAB, found here: +// http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem +// +// Both this code and the orignal code are published under the BSD license. +// by Cong Ma, 2016 +// + +#include +#include +#include +#include +#include + +class HungarianAlgorithm +{ +public: + HungarianAlgorithm(); + ~HungarianAlgorithm(); + double Solve(std::vector> &DistMatrix, std::vector &Assignment); + +private: + void assignmentoptimal(int *assignment, double *cost, double *distMatrix, int nOfRows, int nOfColumns); + void buildassignmentvector(int *assignment, bool *starMatrix, int nOfRows, int nOfColumns); + void computeassignmentcost(int *assignment, double *cost, double *distMatrix, int nOfRows); + void step2a(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step2b(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step3(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step4(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim, int row, int col); + void step5(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); +}; diff --git a/src/sort_filter/KalmanTracker.cpp b/src/sort_filter/KalmanTracker.cpp new file mode 100644 index 00000000..bb8519c2 --- /dev/null +++ b/src/sort_filter/KalmanTracker.cpp @@ -0,0 +1,118 @@ +/////////////////////////////////////////////////////////////////////////////// +// KalmanTracker.cpp: KalmanTracker Class Implementation Declaration + +#include "KalmanTracker.h" +#include + +using namespace std; +using namespace cv; + +int KalmanTracker::kf_count = 0; + +// initialize Kalman filter +void KalmanTracker::init_kf( + StateType stateMat) +{ + int stateNum = 7; + int measureNum = 4; + kf = KalmanFilter(stateNum, measureNum, 0); + + measurement = Mat::zeros(measureNum, 1, CV_32F); + + kf.transitionMatrix = (Mat_(7, 7) << 1, 0, 0, 0, 1, 0, 0, + + 0, 1, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 1, + 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 1); + + setIdentity(kf.measurementMatrix); + setIdentity(kf.processNoiseCov, Scalar::all(1e-1)); + setIdentity(kf.measurementNoiseCov, Scalar::all(1e-4)); + setIdentity(kf.errorCovPost, Scalar::all(1e-2)); + + // initialize state vector with bounding box in [cx,cy,s,r] style + kf.statePost.at(0, 0) = stateMat.x + stateMat.width / 2; + kf.statePost.at(1, 0) = stateMat.y + stateMat.height / 2; + kf.statePost.at(2, 0) = stateMat.area(); + kf.statePost.at(3, 0) = stateMat.width / stateMat.height; +} + +// Predict the estimated bounding box. +StateType KalmanTracker::predict() +{ + // predict + Mat p = kf.predict(); + m_age += 1; + + if (m_time_since_update > 0) + m_hit_streak = 0; + m_time_since_update += 1; + + StateType predictBox = get_rect_xysr(p.at(0, 0), p.at(1, 0), p.at(2, 0), p.at(3, 0)); + + m_history.push_back(predictBox); + return m_history.back(); +} + +StateType KalmanTracker::predict2() +{ + // predict + Mat p = kf.predict(); + + StateType predictBox = get_rect_xysr(p.at(0, 0), p.at(1, 0), p.at(2, 0), p.at(3, 0)); + + return predictBox; +} + +// Update the state vector with observed bounding box. +void KalmanTracker::update( + StateType stateMat) +{ + m_time_since_update = 0; + m_history.clear(); + m_hits += 1; + m_hit_streak += 1; + + // measurement + measurement.at(0, 0) = stateMat.x + stateMat.width / 2; + measurement.at(1, 0) = stateMat.y + stateMat.height / 2; + measurement.at(2, 0) = stateMat.area(); + measurement.at(3, 0) = stateMat.width / stateMat.height; + + // update + kf.correct(measurement); + // time_t now = time(0); + // convert now to string form + + // detect_times.push_back(dt); +} + +// Return the current state vector +StateType KalmanTracker::get_state() +{ + Mat s = kf.statePost; + return get_rect_xysr(s.at(0, 0), s.at(1, 0), s.at(2, 0), s.at(3, 0)); +} + +// Convert bounding box from [cx,cy,s,r] to [x,y,w,h] style. +StateType KalmanTracker::get_rect_xysr( + float cx, + float cy, + float s, + float r) +{ + float w = sqrt(s * r); + float h = s / w; + float x = (cx - w / 2); + float y = (cy - h / 2); + + if (x < 0 && cx > 0) + x = 0; + if (y < 0 && cy > 0) + y = 0; + + return StateType(x, y, w, h); +} diff --git a/src/sort_filter/KalmanTracker.h b/src/sort_filter/KalmanTracker.h new file mode 100644 index 00000000..03360f0d --- /dev/null +++ b/src/sort_filter/KalmanTracker.h @@ -0,0 +1,69 @@ +/////////////////////////////////////////////////////////////////////////////// +// KalmanTracker.h: KalmanTracker Class Declaration + +#ifndef KALMAN_H +#define KALMAN_H 2 + +#include "opencv2/video/tracking.hpp" +#include "opencv2/highgui/highgui.hpp" + + +#define StateType cv::Rect_ + +// This class represents the internel state of individual tracked objects observed as bounding box. +class KalmanTracker +{ +public: + KalmanTracker() + { + init_kf(StateType()); + m_time_since_update = 0; + m_hits = 0; + m_hit_streak = 0; + m_age = 0; + m_id = kf_count; + //kf_count++; + } + KalmanTracker(StateType initRect, float confidence, int classId) : confidence(confidence), classId(classId) + { + init_kf(initRect); + m_time_since_update = 0; + m_hits = 0; + m_hit_streak = 0; + m_age = 0; + m_id = kf_count; + kf_count++; + } + + ~KalmanTracker() + { + m_history.clear(); + } + + StateType predict(); + StateType predict2(); + void update(StateType stateMat); + + StateType get_state(); + StateType get_rect_xysr(float cx, float cy, float s, float r); + + static int kf_count; + + int m_time_since_update; + int m_hits; + int m_hit_streak; + int m_age; + int m_id; + float confidence; + int classId; + +private: + void init_kf(StateType stateMat); + + cv::KalmanFilter kf; + cv::Mat measurement; + + std::vector m_history; +}; + +#endif \ No newline at end of file diff --git a/src/sort_filter/sort.cpp b/src/sort_filter/sort.cpp new file mode 100644 index 00000000..186a2825 --- /dev/null +++ b/src/sort_filter/sort.cpp @@ -0,0 +1,211 @@ +#include "sort.hpp" + +using namespace std; + +// Constructor +SortTracker::SortTracker(int max_age, int min_hits) +{ + _min_hits = min_hits; + _max_age = max_age; + alive_tracker = true; +} + +// Computes IOU between two bounding boxes +double SortTracker::GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt) +{ + float in = (bb_test & bb_gt).area(); + float un = bb_test.area() + bb_gt.area() - in; + + if (un < DBL_EPSILON) + return 0; + + return (double)(in / un); +} + +// Computes centroid distance between two bounding boxes +double SortTracker::GetCentroidsDistance( + cv::Rect_ bb_test, + cv::Rect_ bb_gt) +{ + float bb_test_centroid_x = (bb_test.x + bb_test.width / 2); + float bb_test_centroid_y = (bb_test.y + bb_test.height / 2); + + float bb_gt_centroid_x = (bb_gt.x + bb_gt.width / 2); + float bb_gt_centroid_y = (bb_gt.y + bb_gt.height / 2); + + double distance = (double)sqrt(pow(bb_gt_centroid_x - bb_test_centroid_x, 2) + pow(bb_gt_centroid_y - bb_test_centroid_y, 2)); + + return distance; +} + +void SortTracker::update(vector detections_cv, int frame_count, double image_diagonal, std::vector confidences, std::vector classIds) +{ + vector detections; + if (trackers.size() == 0) // the first frame met + { + alive_tracker = false; + // initialize kalman trackers using first detections. + for (unsigned int i = 0; i < detections_cv.size(); i++) + { + TrackingBox tb; + + tb.box = cv::Rect_(detections_cv[i]); + tb.classId = classIds[i]; + tb.confidence = confidences[i]; + detections.push_back(tb); + + KalmanTracker trk = KalmanTracker(detections[i].box, detections[i].confidence, detections[i].classId); + trackers.push_back(trk); + } + return; + } + else + { + for (unsigned int i = 0; i < detections_cv.size(); i++) + { + TrackingBox tb; + tb.box = cv::Rect_(detections_cv[i]); + tb.classId = classIds[i]; + tb.confidence = confidences[i]; + detections.push_back(tb); + } + for (auto it = frameTrackingResult.begin(); it != frameTrackingResult.end(); it++) + { + int frame_age = frame_count - it->frame; + if (frame_age >= _max_age || frame_age < 0) + { + dead_trackers_id.push_back(it->id); + } + } + } + + /////////////////////////////////////// + // 3.1. get predicted locations from existing trackers. + predictedBoxes.clear(); + for (unsigned int i = 0; i < trackers.size();) + { + cv::Rect_ pBox = trackers[i].predict(); + if (pBox.x >= 0 && pBox.y >= 0) + { + predictedBoxes.push_back(pBox); + i++; + continue; + } + trackers.erase(trackers.begin() + i); + } + + trkNum = predictedBoxes.size(); + detNum = detections.size(); + + centroid_dist_matrix.clear(); + centroid_dist_matrix.resize(trkNum, vector(detNum, 0)); + + for (unsigned int i = 0; i < trkNum; i++) // compute iou matrix as a distance matrix + { + for (unsigned int j = 0; j < detNum; j++) + { + // use 1-iou because the hungarian algorithm computes a minimum-cost assignment. + double distance = SortTracker::GetCentroidsDistance(predictedBoxes[i], detections[j].box) / image_diagonal; + centroid_dist_matrix[i][j] = distance; + } + } + + HungarianAlgorithm HungAlgo; + assignment.clear(); + HungAlgo.Solve(centroid_dist_matrix, assignment); + // find matches, unmatched_detections and unmatched_predictions + unmatchedTrajectories.clear(); + unmatchedDetections.clear(); + allItems.clear(); + matchedItems.clear(); + + if (detNum > trkNum) // there are unmatched detections + { + for (unsigned int n = 0; n < detNum; n++) + allItems.insert(n); + + for (unsigned int i = 0; i < trkNum; ++i) + matchedItems.insert(assignment[i]); + + set_difference(allItems.begin(), allItems.end(), + matchedItems.begin(), matchedItems.end(), + insert_iterator>(unmatchedDetections, unmatchedDetections.begin())); + } + else if (detNum < trkNum) // there are unmatched trajectory/predictions + { + for (unsigned int i = 0; i < trkNum; ++i) + if (assignment[i] == -1) // unassigned label will be set as -1 in the assignment algorithm + unmatchedTrajectories.insert(i); + } + else + ; + + // filter out matched with low IOU + matchedPairs.clear(); + for (unsigned int i = 0; i < trkNum; ++i) + { + if (assignment[i] == -1) // pass over invalid values + continue; + if (centroid_dist_matrix[i][assignment[i]] > max_centroid_dist_norm) + { + unmatchedTrajectories.insert(i); + unmatchedDetections.insert(assignment[i]); + } + else + matchedPairs.push_back(cv::Point(i, assignment[i])); + } + + for (unsigned int i = 0; i < matchedPairs.size(); i++) + { + int trkIdx = matchedPairs[i].x; + int detIdx = matchedPairs[i].y; + trackers[trkIdx].update(detections[detIdx].box); + trackers[trkIdx].classId = detections[detIdx].classId; + trackers[trkIdx].confidence = detections[detIdx].confidence; + } + + // create and initialise new trackers for unmatched detections + for (auto umd : unmatchedDetections) + { + KalmanTracker tracker = KalmanTracker(detections[umd].box, detections[umd].confidence, detections[umd].classId); + trackers.push_back(tracker); + } + + for (auto it2 = dead_trackers_id.begin(); it2 != dead_trackers_id.end(); it2++) + { + for (unsigned int i = 0; i < trackers.size();) + { + if (trackers[i].m_id == (*it2)) + { + trackers.erase(trackers.begin() + i); + continue; + } + i++; + } + } + + // get trackers' output + frameTrackingResult.clear(); + for (unsigned int i = 0; i < trackers.size();) + { + if ((trackers[i].m_time_since_update < 1 && trackers[i].m_hit_streak >= _min_hits) || frame_count <= _min_hits) + { + alive_tracker = true; + TrackingBox res; + res.box = trackers[i].get_state(); + res.id = trackers[i].m_id; + res.frame = frame_count; + res.classId = trackers[i].classId; + res.confidence = trackers[i].confidence; + frameTrackingResult.push_back(res); + } + + // remove dead tracklet + if (trackers[i].m_time_since_update >= _max_age) + { + trackers.erase(trackers.begin() + i); + continue; + } + i++; + } +} diff --git a/src/sort_filter/sort.hpp b/src/sort_filter/sort.hpp new file mode 100644 index 00000000..1482a28a --- /dev/null +++ b/src/sort_filter/sort.hpp @@ -0,0 +1,61 @@ + +#include "KalmanTracker.h" +#include "Hungarian.h" + +#include +#include +#include // to format image names using setw() and setfill() +#include + +#include "opencv2/video/tracking.hpp" +#include "opencv2/highgui/highgui.hpp" + +#ifndef _OPENCV_KCFTRACKER_HPP_ +#define _OPENCV_KCFTRACKER_HPP_ +#endif +#pragma once + +typedef struct TrackingBox +{ + int frame = 0; + float confidence = 0; + int classId = 0; + int id = 0; + cv::Rect_ box = cv::Rect_(0.0, 0.0, 0.0, 0.0); + TrackingBox() {} + TrackingBox(int _frame, float _confidence, int _classId, int _id) : frame(_frame), confidence(_confidence), classId(_classId), id(_id) {} +} TrackingBox; + +class SortTracker +{ +public: + // Constructor + SortTracker(int max_age = 7, int min_hits = 2); + // Initialize tracker + + // Update position based on the new frame + void update(std::vector detection, int frame_count, double image_diagonal, std::vector confidences, std::vector classIds); + double GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt); + double GetCentroidsDistance(cv::Rect_ bb_test, cv::Rect_ bb_gt); + std::vector trackers; + + double max_centroid_dist_norm = 0.15; + + std::vector> predictedBoxes; + std::vector> centroid_dist_matrix; + std::vector assignment; + std::set unmatchedDetections; + std::set unmatchedTrajectories; + std::set allItems; + std::set matchedItems; + std::vector matchedPairs; + + std::vector frameTrackingResult; + std::vector dead_trackers_id; + + unsigned int trkNum = 0; + unsigned int detNum = 0; + int _min_hits; + int _max_age; + bool alive_tracker; +}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 50a85dc9..c649db9e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -56,7 +56,6 @@ if(ENABLE_BLACKMAGIC) endif() endif() - ############### SET TEST SOURCE FILES ################# set(OPENSHOT_TEST_FILES Cache_Tests.cpp @@ -75,17 +74,31 @@ set(OPENSHOT_TEST_FILES Point_Tests.cpp QtImageReader_Tests.cpp Settings_Tests.cpp - Timeline_Tests.cpp ) + Timeline_Tests.cpp) + +########## SET OPENCV RELATED TEST FILES ############### +if(ENABLE_OPENCV) + list(APPEND OPENSHOT_TEST_FILES + CVTracker_Tests.cpp + CVStabilizer_Tests.cpp + # CVObjectDetection_Tests.cpp + ) +endif() ################ TESTER EXECUTABLE ################# # Create unit test executable (openshot-test) message (STATUS "Tests enabled, test executable will be built as tests/openshot-test") + add_executable(openshot-test tests.cpp - ${OPENSHOT_TEST_FILES} ) + ${OPENSHOT_TEST_FILES} +) # Link libraries to the new executable -target_link_libraries(openshot-test openshot ${UnitTest++_LIBRARIES}) +target_link_libraries(openshot-test + openshot + ${UnitTest++_LIBRARIES} +) ##### RUNNING TESTS (make os_test / make test) ##### # Hook up the 'make os_test' target to the 'openshot-test' executable, @@ -93,4 +106,3 @@ target_link_libraries(openshot-test openshot ${UnitTest++_LIBRARIES}) if(NOT ENABLE_COVERAGE) add_custom_target(os_test COMMAND openshot-test) endif() - diff --git a/tests/CVObjectDetection_Tests.cpp b/tests/CVObjectDetection_Tests.cpp new file mode 100644 index 00000000..e6c997a0 --- /dev/null +++ b/tests/CVObjectDetection_Tests.cpp @@ -0,0 +1,140 @@ +/** + * @file + * @brief Unit tests for openshot::Frame + * @author Jonathan Thomas + * @author FeRD (Frank Dana) + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include +#include + +#include "UnitTest++.h" +// Prevent name clashes with juce::UnitTest +#define DONT_SET_USING_JUCE_NAMESPACE 1 +#include "Clip.h" +#include "CVObjectDetection.h" +#include "ProcessingController.h" +#include "Json.h" + +using namespace openshot; + +std::string effectInfo =(" {\"protobuf_data_path\": \"objdetector.data\", " + " \"processing_device\": \"GPU\", " + " \"model_configuration\": \"~/yolo/yolov3.cfg\", " + " \"model_weights\": \"~/yolo/yolov3.weights\", " + " \"classes_file\": \"~/yolo/obj.names\"} "); + +SUITE(CVObjectDetection_Tests) +{ + + // Just for the stabilizer constructor, it won't be used + ProcessingController processingController; + + TEST(DetectObject_Video) + { + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "run.mp4"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + //TODO remove hardcoded path + CVObjectDetection objectDetector(effectInfo, processingController); + + objectDetector.detectObjectsClip(c1, 0, 20, true); + + CVDetectionData dd = objectDetector.GetDetectionData(20); + + float x1 = dd.boxes.at(20).x; + float y1 = dd.boxes.at(20).y; + float x2 = x1 + dd.boxes.at(20).width; + float y2 = y1 + dd.boxes.at(20).height; + float confidence = dd.confidences.at(20); + int classId = dd.classIds.at(20); + + CHECK_EQUAL((int) (x1 * 720), 106); + CHECK_EQUAL((int) (y1 * 400), 21); + CHECK_EQUAL((int) (x2 * 720), 628); + CHECK_EQUAL((int) (y2 * 400), 429); + CHECK_EQUAL((int) (confidence * 1000), 554); + CHECK_EQUAL(classId, 0); + + } + + + TEST(SaveLoad_Protobuf) + { + + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "run.mp4"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + //TODO remove hardcoded path + CVObjectDetection objectDetector_1(effectInfo ,processingController); + + objectDetector_1.detectObjectsClip(c1, 0, 20, true); + + CVDetectionData dd_1 = objectDetector_1.GetDetectionData(20); + + float x1_1 = dd_1.boxes.at(20).x; + float y1_1 = dd_1.boxes.at(20).y; + float x2_1 = x1_1 + dd_1.boxes.at(20).width; + float y2_1 = y1_1 + dd_1.boxes.at(20).height; + float confidence_1 = dd_1.confidences.at(20); + int classId_1 = dd_1.classIds.at(20); + + objectDetector_1.SaveObjDetectedData(); + + CVObjectDetection objectDetector_2(effectInfo, processingController); + + objectDetector_2._LoadObjDetectdData(); + + CVDetectionData dd_2 = objectDetector_2.GetDetectionData(20); + + float x1_2 = dd_2.boxes.at(20).x; + float y1_2 = dd_2.boxes.at(20).y; + float x2_2 = x1_2 + dd_2.boxes.at(20).width; + float y2_2 = y1_2 + dd_2.boxes.at(20).height; + float confidence_2 = dd_2.confidences.at(20); + int classId_2 = dd_2.classIds.at(20); + + CHECK_EQUAL((int) (x1_1 * 720), (int) (x1_2 * 720)); + CHECK_EQUAL((int) (y1_1 * 400), (int) (y1_2 * 400)); + CHECK_EQUAL((int) (x2_1 * 720), (int) (x2_2 * 720)); + CHECK_EQUAL((int) (y2_1 * 400), (int) (y2_2 * 400)); + CHECK_EQUAL((int) (confidence_1 * 1000), (int) (confidence_2 * 1000)); + CHECK_EQUAL(classId_1, classId_2); + + } + +} // SUITE(Frame_Tests) diff --git a/tests/CVStabilizer_Tests.cpp b/tests/CVStabilizer_Tests.cpp new file mode 100644 index 00000000..4d1f9305 --- /dev/null +++ b/tests/CVStabilizer_Tests.cpp @@ -0,0 +1,142 @@ +/** + * @file + * @brief Unit tests for openshot::Frame + * @author Jonathan Thomas + * @author FeRD (Frank Dana) + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include +#include + +#include "UnitTest++.h" +// Prevent name clashes with juce::UnitTest +#define DONT_SET_USING_JUCE_NAMESPACE 1 +#include "Clip.h" +#include "CVStabilization.h" // for TransformParam, CamTrajectory, CVStabilization +#include "ProcessingController.h" + +using namespace openshot; + +SUITE(CVStabilizer_Tests) +{ + + // Just for the stabilizer constructor, it won't be used + ProcessingController processingController; + + TEST(Stabilize_Video) + { + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + std::string json_data = R"proto( + { + "protobuf_data_path": "stabilizer.data", + "smoothing-window": 30 + } )proto"; + + // Create stabilizer + CVStabilization stabilizer(json_data, processingController); + + // Stabilize clip for frames 0-21 + stabilizer.stabilizeClip(c1, 0, 21, true); + + // Get stabilized data + TransformParam tp = stabilizer.GetTransformParamData(20); + CamTrajectory ct = stabilizer.GetCamTrajectoryTrackedData(20); + + // // Compare if stabilized data is equal to pre-tested ones + int dx = tp.dx*1000; + int dy = tp.dy*1000; + int da = tp.da*1000; + int x = ct.x*1000; + int y = ct.y*1000; + int a = ct.a*1000; + + CHECK_EQUAL((int) (58), dx); + CHECK_EQUAL((int) (-88), dy); + CHECK_EQUAL((int) (7), da); + CHECK_EQUAL((int) (0), x); + CHECK_EQUAL((int) (-1), y); + CHECK_EQUAL((int) (0), a); + } + + + TEST(SaveLoad_Protobuf) + { + + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + std::string json_data = R"proto( + { + "protobuf_data_path": "stabilizer.data", + "smoothing-window": 30 + } )proto"; + + // Create first stabilizer + CVStabilization stabilizer_1(json_data, processingController); + + // Stabilize clip for frames 0-20 + stabilizer_1.stabilizeClip(c1, 0, 20+1, true); + + // Get stabilized data + TransformParam tp_1 = stabilizer_1.GetTransformParamData(20); + CamTrajectory ct_1 = stabilizer_1.GetCamTrajectoryTrackedData(20); + + // Save stabilized data + stabilizer_1.SaveStabilizedData(); + + // Create second stabilizer + CVStabilization stabilizer_2(json_data, processingController); + + // Load stabilized data from first stabilizer protobuf data + stabilizer_2._LoadStabilizedData(); + + // Get stabilized data + TransformParam tp_2 = stabilizer_2.GetTransformParamData(20); + CamTrajectory ct_2 = stabilizer_2.GetCamTrajectoryTrackedData(20); + + // Compare first stabilizer data with second stabilizer data + CHECK_EQUAL((int) (tp_1.dx * 10000), (int) (tp_2.dx *10000)); + CHECK_EQUAL((int) (tp_1.dy * 10000), (int) (tp_2.dy * 10000)); + CHECK_EQUAL((int) (tp_1.da * 10000), (int) (tp_2.da * 10000)); + CHECK_EQUAL((int) (ct_1.x * 10000), (int) (ct_2.x * 10000)); + CHECK_EQUAL((int) (ct_1.y * 10000), (int) (ct_2.y * 10000)); + CHECK_EQUAL((int) (ct_1.a * 10000), (int) (ct_2.a * 10000)); + } + +} // SUITE(Frame_Tests) diff --git a/tests/CVTracker_Tests.cpp b/tests/CVTracker_Tests.cpp new file mode 100644 index 00000000..3229d1ca --- /dev/null +++ b/tests/CVTracker_Tests.cpp @@ -0,0 +1,151 @@ +/** + * @file + * @brief Unit tests for openshot::Frame + * @author Jonathan Thomas + * @author FeRD (Frank Dana) + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include +#include + +#include "UnitTest++.h" +// Prevent name clashes with juce::UnitTest +#define DONT_SET_USING_JUCE_NAMESPACE 1 +#include "Clip.h" +#include "CVTracker.h" // for FrameData, CVTracker +#include "ProcessingController.h" + +using namespace openshot; + +SUITE(CVTracker_Tests) +{ + + // Just for the tracker constructor, it won't be used + ProcessingController processingController; + + TEST(Track_Video) + { + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + std::string json_data = R"proto( + { + "protobuf_data_path": "kcf_tracker.data", + "tracker-type": "KCF", + "region": {"x": 294, "y": 102, "width": 180, "height": 166, "first-frame": 0} + } )proto"; + + // Create tracker + CVTracker kcfTracker(json_data, processingController); + + // Track clip for frames 0-20 + kcfTracker.trackClip(c1, 0, 20, true); + // Get tracked data + FrameData fd = kcfTracker.GetTrackedData(20); + float x = fd.x1; + float y = fd.y1; + float width = fd.x2 - x; + float height = fd.y2 - y; + + // Compare if tracked data is equal to pre-tested ones + CHECK_EQUAL(259, (int)(x * 640)); + CHECK_EQUAL(131, (int)(y * 360)); + CHECK_EQUAL(180, (int)(width * 640)); + CHECK_EQUAL(166, (int)(height * 360)); + } + + + TEST(SaveLoad_Protobuf) + { + + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + std::string json_data = R"proto( + { + "protobuf_data_path": "kcf_tracker.data", + "tracker-type": "KCF", + "region": {"x": 294, "y": 102, "width": 180, "height": 166, "first-frame": 0} + } )proto"; + + + // Create first tracker + CVTracker kcfTracker_1(json_data, processingController); + + // Track clip for frames 0-20 + kcfTracker_1.trackClip(c1, 0, 20, true); + + // Get tracked data + FrameData fd_1 = kcfTracker_1.GetTrackedData(20); + + float x_1 = fd_1.x1; + float y_1 = fd_1.y1; + float width_1 = fd_1.x2 - x_1; + float height_1 = fd_1.y2 - y_1; + + // Save tracked data + kcfTracker_1.SaveTrackedData(); + + std::string proto_data_1 = R"proto( + { + "protobuf_data_path": "kcf_tracker.data", + "tracker_type": "", + "region": {"x": -1, "y": -1, "width": -1, "height": -1, "first-frame": 0} + } )proto"; + + // Create second tracker + CVTracker kcfTracker_2(proto_data_1, processingController); + + // Load tracked data from first tracker protobuf data + kcfTracker_2._LoadTrackedData(); + + // Get tracked data + FrameData fd_2 = kcfTracker_2.GetTrackedData(20); + + float x_2 = fd_2.x1; + float y_2 = fd_2.y1; + float width_2 = fd_2.x2 - x_2; + float height_2 = fd_2.y2 - y_2; + + // Compare first tracker data with second tracker data + CHECK_EQUAL((int)(x_1 * 640), (int)(x_2 * 640)); + CHECK_EQUAL((int)(y_1 * 360), (int)(y_2 * 360)); + CHECK_EQUAL((int)(width_1 * 640), (int)(width_2 * 640)); + CHECK_EQUAL((int)(height_1 * 360), (int)(height_2 * 360)); + } + +} // SUITE(Frame_Tests) diff --git a/tests/Frame_Tests.cpp b/tests/Frame_Tests.cpp index 0568efe9..9038f8b8 100644 --- a/tests/Frame_Tests.cpp +++ b/tests/Frame_Tests.cpp @@ -41,6 +41,10 @@ #include +#ifdef USE_OPENCV +#include +#endif + using namespace openshot; SUITE(Frame_Tests) @@ -152,4 +156,28 @@ TEST(Copy_Constructor) CHECK_EQUAL(f1.GetAudioSamplesCount(), f2.GetAudioSamplesCount()); } +#ifdef USE_OPENCV +TEST(Convert_Image) +{ + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + Clip c1(path.str()); + c1.Open(); + + // Get first frame + auto f1 = c1.GetFrame(1); + + // Get first Mat image + cv::Mat cvimage = f1->GetImageCV(); + + CHECK(!cvimage.empty()); + + CHECK_EQUAL(1, f1->number); + CHECK_EQUAL(f1->GetWidth(), cvimage.cols); + CHECK_EQUAL(f1->GetHeight(), cvimage.rows); + CHECK_EQUAL(3, cvimage.channels()); +} +#endif + } // SUITE(Frame_Tests)