diff --git a/include/CVStabilization.h b/include/CVStabilization.h index 62fd174e..279ffb6f 100644 --- a/include/CVStabilization.h +++ b/include/CVStabilization.h @@ -41,7 +41,9 @@ #undef int64 #include #include "stabilizedata.pb.h" +#include "ProcessingController.h" #include "Clip.h" +#include "Json.h" using namespace std; using google::protobuf::util::TimeUtil; @@ -82,9 +84,17 @@ class CVStabilization { cv::Mat cur, cur_grey; cv::Mat prev, prev_grey; std::vector prev_to_cur_transform; // Previous to current + std::string protobuf_data_path; + + bool smoothingWindowSet = false; + + uint progress; + + /// Will handle a Thread saflly comutication between ClipProcessingJobs and the processing effect classes + ProcessingController *processingController; // Track current frame features and find the relative transformation - void TrackFrameFeatures(cv::Mat frame, int frameNum); + void TrackFrameFeatures(cv::Mat frame, size_t frameNum); std::vector ComputeFramesTrajectory(); std::map SmoothTrajectory(std::vector &trajectory); @@ -94,26 +104,34 @@ class CVStabilization { public: - const int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning + int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning std::map trajectoryData; // Save camera trajectory data std::map transformationData; // Save transormation data // Set default smoothing window value to compute stabilization - CVStabilization(); + CVStabilization(std::string processInfoJson, ProcessingController &processingController); // Set desirable smoothing window value to compute stabilization - CVStabilization(int _smoothingWindow); + void setSmoothingWindow(int _smoothingWindow); // Process clip and store necessary stabilization data - void ProcessClip(openshot::Clip &video); + void stabilizeClip(openshot::Clip& video, size_t start=0, size_t end=0, bool process_interval=false); /// Protobuf Save and Load methods // Save stabilization data to protobuf file - bool SaveStabilizedData(std::string outputFilePath); + bool SaveStabilizedData(); // Add frame stabilization data into protobuf message void AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number); // Load protobuf data file - bool LoadStabilizedData(std::string inputFilePath); + bool LoadStabilizedData(); + + // Return requested struct info for a given frame + TransformParam GetTransformParamData(size_t frameId); + CamTrajectory GetCamTrajectoryTrackedData(size_t frameId); + + /// Get and Set JSON methods + void SetJson(const std::string value); ///< Load JSON string into this object + void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object }; diff --git a/include/CVTracker.h b/include/CVTracker.h index 796efc6e..45ea41cb 100644 --- a/include/CVTracker.h +++ b/include/CVTracker.h @@ -15,6 +15,8 @@ #include "Clip.h" #include "KeyFrame.h" #include "Frame.h" +#include "Json.h" +#include "ProcessingController.h" #include "trackerdata.pb.h" using namespace std; @@ -22,7 +24,7 @@ using google::protobuf::util::TimeUtil; // Tracking info struct struct FrameData{ - int frame_id = -1; + size_t frame_id = -1; float rotation = 0; int x1 = -1; int y1 = -1; @@ -33,10 +35,10 @@ struct FrameData{ FrameData() {} - FrameData( int _frame_id) + FrameData( size_t _frame_id) {frame_id = _frame_id;} - FrameData( int _frame_id , float _rotation, int _x1, int _y1, int _x2, int _y2) + FrameData( size_t _frame_id , float _rotation, int _x1, int _y1, int _x2, int _y2) { frame_id = _frame_id; rotation = _rotation; @@ -47,39 +49,53 @@ struct FrameData{ } }; -class CVTracker { - public: - - std::map trackedDataById; // Save tracked data +class CVTracker { + private: + std::map trackedDataById; // Save tracked data std::string trackerType; // Name of the chosen tracker - cv::Ptr tracker; // Pointer of the selected tracker + cv::Ptr tracker; // Pointer of the selected tracker + cv::Rect2d bbox; // Bounding box coords - /// Class constructors - // Expects a tracker type, if none is passed, set default as KCF - CVTracker(); - CVTracker(std::string trackerType); + std::string protobuf_data_path; // Path to protobuf data file + + uint progress; // Pre-processing effect progress + + /// Will handle a Thread saflly comutication between ClipProcessingJobs and the processing effect classes + ProcessingController *processingController; + + // Initialize the tracker + bool initTracker(cv::Mat &frame, size_t frameId); + + // Update the object tracker according to frame + bool trackFrame(cv::Mat &frame, size_t frameId); + + public: + + // Constructor + CVTracker(std::string processInfoJson, ProcessingController &processingController); // Set desirable tracker method - cv::Ptr select_tracker(std::string trackerType); + cv::Ptr selectTracker(std::string trackerType); - // Track object in the hole clip - void trackClip(openshot::Clip& video); - // Initialize the tracker - bool initTracker(cv::Rect2d bbox, cv::Mat &frame, int frameId); - // Update the object tracker according to frame - bool trackFrame(cv::Mat &frame, int frameId); + // Track object in the hole clip or in a given interval + // If start, end and process_interval are passed as argument, clip will be processed in [start,end) + void trackClip(openshot::Clip& video, size_t start=0, size_t end=0, bool process_interval=false); + // Get tracked data for a given frame + FrameData GetTrackedData(size_t frameId); + /// Protobuf Save and Load methods // Save protobuf file - bool SaveTrackedData(std::string outputFilePath); + bool SaveTrackedData(); // Add frame tracked data into protobuf message. void AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, FrameData& fData); // Load protobuf file - bool LoadTrackedData(std::string inputFilePath); + bool LoadTrackedData(); - // Get tracked data for a given frame - FrameData GetTrackedData(int frameId); + /// Get and Set JSON methods + void SetJson(const std::string value); ///< Load JSON string into this object + void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object }; diff --git a/include/ClipProcessingJobs.h b/include/ClipProcessingJobs.h index 7887e1a3..1c071857 100644 --- a/include/ClipProcessingJobs.h +++ b/include/ClipProcessingJobs.h @@ -41,31 +41,42 @@ #include "CVTracker.h" #endif +#include +#include "ProcessingController.h" #include "Clip.h" using namespace openshot; // Constructor responsible to choose processing type and apply to clip class ClipProcessingJobs{ - private: - int processingProgress; + std::string processInfoJson; + std::string processingType; + bool processingDone = false; bool stopProcessing = false; + uint processingProgress = 0; - public: - ClipProcessingJobs(std::string processingType, Clip& videoClip); + std::thread t; - int GetProgress(); - - void CancelProcessing(); + /// Will handle a Thread saflly comutication between ClipProcessingJobs and the processing effect classes + ProcessingController processingController; // Apply object tracking to clip - std::string trackVideo(Clip& videoClip); + void trackClip(Clip& clip, ProcessingController& controller); // Apply stabilization to clip - std::string stabilizeVideo(Clip& videoClip); + void stabilizeClip(Clip& clip, ProcessingController& controller); + public: + // Constructor + ClipProcessingJobs(std::string processingType, std::string processInfoJson); + // Process clip accordingly to processingType + void processClip(Clip& clip); + // Thread related variables and methods + int GetProgress(); + bool IsDone(); + void CancelProcessing(); }; \ No newline at end of file diff --git a/include/EffectInfo.h b/include/EffectInfo.h index 901e4dba..e3e20b47 100644 --- a/include/EffectInfo.h +++ b/include/EffectInfo.h @@ -49,7 +49,6 @@ namespace openshot public: // Create an instance of an effect (factory style) EffectBase* CreateEffect(std::string effect_type); - EffectBase* CreateEffect(std::string effect_type, std::string pb_data_path); /// JSON methods static std::string Json(); ///< Generate JSON string of this object diff --git a/include/ProcessingController.h b/include/ProcessingController.h new file mode 100644 index 00000000..05e24e18 --- /dev/null +++ b/include/ProcessingController.h @@ -0,0 +1,62 @@ +#ifndef OPENSHOT_PROCESSINGCONTROLLER_H +#define OPENSHOT_PROCESSINGCONTROLLER_H + +#include +#include +#include + + +class ProcessingController{ + private: + uint processingProgress; + bool processingFinished; + bool stopProcessing; + + std::mutex mtxProgress; + std::mutex mtxFinished; + std::mutex mtxStop; + + public: + + ProcessingController(){ + processingProgress = 0; + stopProcessing = false; + processingFinished = false; + } + + int GetFinished(){ + std::lock_guard lck (mtxFinished); + bool f = processingFinished; + return f; + } + + void SetFinished(bool f){ + std::lock_guard lck (mtxFinished); + processingFinished = f; + } + + void SetProgress(uint p){ + std::lock_guard lck (mtxProgress); + processingProgress = p; + } + + int GetProgress(){ + std::lock_guard lck (mtxProgress); + uint p = processingProgress; + return p; + } + + void CancelProcessing(){ + std::lock_guard lck (mtxStop); + stopProcessing = true; + } + + bool ShouldStop(){ + std::lock_guard lck (mtxStop); + bool s = stopProcessing; + return s; + } + +}; + +#endif \ No newline at end of file diff --git a/include/effects/Tracker.h b/include/effects/Tracker.h index b9bde100..76323b52 100644 --- a/include/effects/Tracker.h +++ b/include/effects/Tracker.h @@ -89,7 +89,7 @@ namespace openshot private: /// Init effect settings void init_effect_details(); - + std::string protobuf_data_path; public: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1ad4eae2..0ba7cc5d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -191,7 +191,8 @@ set(OPENSHOT_SOURCES set(OPENSHOT_CV_SOURCES CVTracker.cpp CVStabilization.cpp - ClipProcessingJobs.cpp) + ClipProcessingJobs.cpp + CVObjectDetection.cpp) # Compiled Protobuf messages set(PROTOBUF_MESSAGES @@ -213,9 +214,9 @@ set(EFFECTS_SOURCES effects/Pixelate.cpp effects/Saturation.cpp effects/Shift.cpp - effects/Wave.cpp + effects/Stabilizer.cpp effects/Tracker.cpp - effects/Stabilizer.cpp) + effects/Wave.cpp) # Qt video player components set(QT_PLAYER_SOURCES diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp index 01477988..c267f3b3 100644 --- a/src/CVStabilization.cpp +++ b/src/CVStabilization.cpp @@ -31,25 +31,39 @@ #include "../include/CVStabilization.h" // Set default smoothing window value to compute stabilization -CVStabilization::CVStabilization():smoothingWindow(30) {} - -// Set desirable smoothing window value to compute stabilization -CVStabilization::CVStabilization(int _smoothingWindow): smoothingWindow(_smoothingWindow){} +CVStabilization::CVStabilization(std::string processInfoJson, ProcessingController &processingController) +: smoothingWindow(30), processingController(&processingController){ + SetJson(processInfoJson); +} // Process clip and store necessary stabilization data -void CVStabilization::ProcessClip(openshot::Clip &video){ - // Get total number of frames in video - int videoLenght = video.Reader()->info.video_length; +void CVStabilization::stabilizeClip(openshot::Clip& video, size_t start, size_t end, bool process_interval){ + size_t frame_number; + + smoothingWindowSet = true; // Certificate that smoothing window value won't change + + if(!process_interval || end == 0 || end-start <= 0){ + // Get total number of frames in video + end = video.Reader()->info.video_length; + } // Extract and track opticalflow features for each frame - for (long int frame_number = 0; frame_number <= videoLenght; frame_number++) + for (frame_number = start; frame_number <= end; frame_number++) { + // Stop the feature tracker process + if(processingController->ShouldStop()){ + return; + } + std::shared_ptr f = video.GetFrame(frame_number); // Grab OpenCV Mat image cv::Mat cvimage = f->GetImageCV(); cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); TrackFrameFeatures(cvimage, frame_number); + + // Update progress + processingController->SetProgress(uint(100*frame_number/end)); } // Calculate trajectory data @@ -63,7 +77,7 @@ void CVStabilization::ProcessClip(openshot::Clip &video){ } // Track current frame features and find the relative transformation -void CVStabilization::TrackFrameFeatures(cv::Mat frame, int frameNum){ +void CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ if(prev_grey.empty()){ prev_grey = frame; return; @@ -192,10 +206,8 @@ std::map CVStabilization::GenNewCamPosition(std::map CVTracker::select_tracker(std::string trackerType){ +cv::Ptr CVTracker::selectTracker(std::string trackerType){ cv::Ptr t; if (trackerType == "BOOSTING") @@ -61,20 +60,28 @@ cv::Ptr CVTracker::select_tracker(std::string trackerType){ return t; } -// Track object in the hole clip -void CVTracker::trackClip(openshot::Clip& video){ - // Opencv display window - cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); - // Create Tracker +// Track object in the hole clip or in a given interval +void CVTracker::trackClip(openshot::Clip& video, size_t start, size_t end, bool process_interval){ bool trackerInit = false; - int videoLenght = video.Reader()->info.video_length; + size_t frame; + + if(!process_interval || end == 0 || end-start <= 0){ + // Get total number of frames in video + end = video.Reader()->info.video_length; + } // Loop through video - for (long int frame = 0; frame < videoLenght; frame++) + for (frame = start; frame < end; frame++) { - std::cout<<"frame: "<ShouldStop()){ + return; + } + + std::cout<<"Frame: "< f = video.GetFrame(frame_number); @@ -83,13 +90,9 @@ void CVTracker::trackClip(openshot::Clip& video){ // Pass the first frame to initialize the tracker if(!trackerInit){ - // Select the object to be tracked - cv::Rect2d bbox = cv::selectROI("Display Image", cvimage); // Initialize the tracker - initTracker(bbox, cvimage, frame_number); - // Draw in the frame the box containing the object - cv::rectangle(cvimage, bbox, cv::Scalar( 255, 0, 0 ), 2, 1 ); + initTracker(cvimage, frame_number); trackerInit = true; } @@ -100,26 +103,17 @@ void CVTracker::trackClip(openshot::Clip& video){ // Draw box on image FrameData fd = GetTrackedData(frame_number); - cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); - cv::rectangle(cvimage, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); } - - // Show tracking process - cv::imshow("Display Image", cvimage); - // Press ESC on keyboard to exit - char c=(char)cv::waitKey(1); - if(c==27) - break; - + // Update progress + processingController->SetProgress(uint(100*frame_number/end)); } } // Initialize the tracker -bool CVTracker::initTracker(cv::Rect2d initial_bbox, cv::Mat &frame, int frameId){ - // Set initial bounding box coords - bbox = initial_bbox; +bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){ + // Create new tracker object - tracker = select_tracker(trackerType); + tracker = selectTracker(trackerType); // Initialize tracker tracker->init(frame, bbox); @@ -131,7 +125,7 @@ bool CVTracker::initTracker(cv::Rect2d initial_bbox, cv::Mat &frame, int frameId } // Update the object tracker according to frame -bool CVTracker::trackFrame(cv::Mat &frame, int frameId){ +bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){ // Update the tracking result bool ok = tracker->update(frame, bbox); @@ -151,12 +145,12 @@ bool CVTracker::trackFrame(cv::Mat &frame, int frameId){ return ok; } -bool CVTracker::SaveTrackedData(std::string outputFilePath){ +bool CVTracker::SaveTrackedData(){ // Create tracker message libopenshottracker::Tracker trackerMessage; // Iterate over all frames data and save in protobuf message - for(std::map::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){ + for(std::map::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){ FrameData fData = it->second; libopenshottracker::Frame* pbFrameData; AddFrameDataToProto(trackerMessage.add_frame(), fData); @@ -167,7 +161,7 @@ bool CVTracker::SaveTrackedData(std::string outputFilePath){ { // Write the new message to disk. - std::fstream output(outputFilePath, ios::out | ios::trunc | ios::binary); + std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary); if (!trackerMessage.SerializeToOstream(&output)) { cerr << "Failed to write protobuf message." << endl; return false; @@ -197,13 +191,13 @@ void CVTracker::AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, Fram } // Load protobuf data file -bool CVTracker::LoadTrackedData(std::string inputFilePath){ +bool CVTracker::LoadTrackedData(){ // Create tracker message libopenshottracker::Tracker trackerMessage; { // Read the existing tracker message. - fstream input(inputFilePath, ios::in | ios::binary); + fstream input(protobuf_data_path, ios::in | ios::binary); if (!trackerMessage.ParseFromIstream(&input)) { cerr << "Failed to parse protobuf message." << endl; return false; @@ -214,11 +208,11 @@ bool CVTracker::LoadTrackedData(std::string inputFilePath){ trackedDataById.clear(); // Iterate over all frames of the saved message - for (int i = 0; i < trackerMessage.frame_size(); i++) { + for (size_t i = 0; i < trackerMessage.frame_size(); i++) { const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i); // Load frame and rotation data - int id = pbFrameData.id(); + size_t id = pbFrameData.id(); float rotation = pbFrameData.rotation(); // Load bounding box data @@ -244,7 +238,7 @@ bool CVTracker::LoadTrackedData(std::string inputFilePath){ } // Get tracker info for the desired frame -FrameData CVTracker::GetTrackedData(int frameId){ +FrameData CVTracker::GetTrackedData(size_t frameId){ // Check if the tracker info for the requested frame exists if ( trackedDataById.find(frameId) == trackedDataById.end() ) { @@ -256,3 +250,41 @@ FrameData CVTracker::GetTrackedData(int frameId){ } } + +// Load JSON string into this object +void CVTracker::SetJson(const std::string value) { + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + // throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + std::cout<<"JSON is invalid (missing keys or invalid data types)"< pEffects{"Stabilizer", "Tracker"}; - std::string effectName = existing_effect["type"].asString(); + // Create instance of effect + if ( (e = EffectInfo().CreateEffect(existing_effect["type"].asString()))) { + + // Load Json into Effect + e->SetJsonValue(existing_effect); - if(std::find(pEffects.begin(), pEffects.end(), effectName) == pEffects.end()){ - // Create instance of effect - if ( (e = EffectInfo().CreateEffect(effectName))) { - - // Load Json into Effect - e->SetJsonValue(existing_effect); - - // Add Effect to Timeline - AddEffect(e); - } - } - else{ - if ( (e = EffectInfo().CreateEffect(effectName, existing_effect["protobuf_data_path"].asString()))) { - // Load Json into Effect - e->SetJsonValue(existing_effect); - - // Add Effect to Timeline - AddEffect(e); - } + // Add Effect to Timeline + AddEffect(e); } } } diff --git a/src/ClipProcessingJobs.cpp b/src/ClipProcessingJobs.cpp index 4c74a16a..43cd288f 100644 --- a/src/ClipProcessingJobs.cpp +++ b/src/ClipProcessingJobs.cpp @@ -1,57 +1,78 @@ #include "../include/ClipProcessingJobs.h" // Constructor responsible to choose processing type and apply to clip -ClipProcessingJobs::ClipProcessingJobs(std::string processingType, Clip& videoClip){ +ClipProcessingJobs::ClipProcessingJobs(std::string processingType, std::string processInfoJson) : +processingType(processingType), processInfoJson(processInfoJson){ +} - // if(processingType == "Stabilize"){ - // std::cout<<"Stabilize"; - // stabilizeVideo(videoClip); - // } - // if(processingType == "Track"){ - // std::cout<<"Track"; - // trackVideo(videoClip); - // } +void ClipProcessingJobs::processClip(Clip& clip){ + // Process clip and save processed data + if(processingType == "Stabilizer"){ + t = std::thread(&ClipProcessingJobs::stabilizeClip, this, std::ref(clip), std::ref(this->processingController)); + } + if(processingType == "Tracker"){ + t = std::thread(&ClipProcessingJobs::trackClip, this, std::ref(clip), std::ref(this->processingController)); + } } // Apply object tracking to clip -std::string ClipProcessingJobs::trackVideo(Clip& videoClip){ - - // Opencv display window - cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); +void ClipProcessingJobs::trackClip(Clip& clip, ProcessingController& controller){ // Create CVTracker object - CVTracker tracker; + CVTracker tracker(processInfoJson, controller); // Start tracking - tracker.trackClip(videoClip); - - // Save tracking data - tracker.SaveTrackedData("/media/brenno/Data/projects/openshot/kcf_tracker.data"); + tracker.trackClip(clip); + + // Thread controller. If effect processing is done, save data + // Else, kill thread + if(controller.ShouldStop()){ + controller.SetFinished(true); + return; + } + else{ + // Save stabilization data + tracker.SaveTrackedData(); + // tells to UI that the processing finished + controller.SetFinished(true); + } - // Return path to protobuf saved data - return "/media/brenno/Data/projects/openshot/kcf_tracker.data"; } // Apply stabilization to clip -std::string ClipProcessingJobs::stabilizeVideo(Clip& videoClip){ +void ClipProcessingJobs::stabilizeClip(Clip& clip, ProcessingController& controller){ // create CVStabilization object - CVStabilization stabilizer; + CVStabilization stabilizer(processInfoJson, controller); // Start stabilization process - stabilizer.ProcessClip(videoClip); + stabilizer.stabilizeClip(clip); - // Save stabilization data - stabilizer.SaveStabilizedData("/media/brenno/Data/projects/openshot/stabilization.data"); - - // Return path to protobuf saved data - return "/media/brenno/Data/projects/openshot/stabilization.data"; + // Thread controller. If effect processing is done, save data + // Else, kill thread + if(controller.ShouldStop()){ + controller.SetFinished(true); + return; + } + else{ + // Save stabilization data + stabilizer.SaveStabilizedData(); + // tells to UI that the processing finished + controller.SetFinished(true); + } } - int ClipProcessingJobs::GetProgress(){ - return processingProgress; + + return (int)processingController.GetProgress(); } +bool ClipProcessingJobs::IsDone(){ + + if(processingController.GetFinished()){ + t.join(); + } + return processingController.GetFinished(); +} void ClipProcessingJobs::CancelProcessing(){ - stopProcessing = true; -} + processingController.CancelProcessing(); +} \ No newline at end of file diff --git a/src/EffectInfo.cpp b/src/EffectInfo.cpp index e169217f..24d8f0a2 100644 --- a/src/EffectInfo.cpp +++ b/src/EffectInfo.cpp @@ -95,14 +95,6 @@ EffectBase* EffectInfo::CreateEffect(std::string effect_type) { return NULL; } -EffectBase* EffectInfo::CreateEffect(std::string effect_type, std::string pb_data_path){ - if(effect_type == "Stabilizer") - return new Stabilizer(pb_data_path); - - else if(effect_type == "Tracker") - return new Tracker(); -} - // Generate Json::Value for this object Json::Value EffectInfo::JsonValue() { diff --git a/src/effects/Stabilizer.cpp b/src/effects/Stabilizer.cpp index 2a61b25c..874b0169 100644 --- a/src/effects/Stabilizer.cpp +++ b/src/effects/Stabilizer.cpp @@ -62,6 +62,7 @@ void Stabilizer::init_effect_details() info.description = "Stabilize video clip to remove undesired shaking and jitter."; info.has_audio = false; info.has_video = true; + protobuf_data_path = ""; } @@ -148,7 +149,6 @@ bool Stabilizer::LoadStabilizedData(std::string inputFilePath){ // Assing data to transformation map transformationData[i] = EffectTransformParam(dx,dy,da); - std::cout< f = r9.GetFrame(frame_number); +// for (long int frame = 1200; frame <= 1600; frame++) +// { +// int frame_number = frame; +// std::shared_ptr f = r9.GetFrame(frame_number); - // Grab Mat image - cv::Mat cvimage = f->GetImageCV(); +// // Grab Mat image +// cv::Mat cvimage = f->GetImageCV(); - FrameData fd = kcfTracker.GetTrackedData(frame_number); - cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); - cv::rectangle(cvimage, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); +// FrameData fd = kcfTracker.GetTrackedData(frame_number); +// cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); +// cv::rectangle(cvimage, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); - cv::imshow("Display Image", cvimage); - // Press ESC on keyboard to exit - char c=(char)cv::waitKey(25); - if(c==27) - break; - } +// cv::imshow("Display Image", cvimage); +// // Press ESC on keyboard to exit +// char c=(char)cv::waitKey(25); +// if(c==27) +// break; +// } -} +// } void displayClip(openshot::Clip &r9){ @@ -99,32 +99,85 @@ void displayClip(openshot::Clip &r9){ int main(int argc, char* argv[]) { - bool TRACK_DATA = false; - bool LOAD_TRACKED_DATA = false; - bool SMOOTH_VIDEO = true; - bool LOAD_SMOOTH_DATA = false; + // bool TRACK_DATA = true; + // bool LOAD_TRACKED_DATA = false; + // bool SMOOTH_VIDEO = false; + // bool LOAD_SMOOTH_DATA = false; + // bool OBJECT_DETECTION_DATA = false; + // bool LOAD_OBJECT_DETECTION = false; + // std::string input_filepath = TEST_MEDIA_PATH; + // input_filepath = "../../src/examples/test.avi"; - std::string input_filepath = TEST_MEDIA_PATH; - input_filepath = "../../src/examples/test.avi"; + // openshot::Clip r9(input_filepath); + // r9.Open(); - openshot::Clip r9(input_filepath); - r9.Open(); + // if(TRACK_DATA) + // // ClipProcessingJobs clipProcessing("Track", r9); + // if(LOAD_TRACKED_DATA){ + // /***/ + // } + // if(SMOOTH_VIDEO) + // // ClipProcessingJobs clipProcessing("Stabilize", r9); + // if(LOAD_SMOOTH_DATA){ + // /***/ + // } + // if(OBJECT_DETECTION_DATA){ + // // CVObjectDetection objectDetection("GPU"); + // // objectDetection.ProcessClip(r9); + // } + // displayClip(r9); - if(TRACK_DATA) - ClipProcessingJobs clipProcessing("Track", r9); - if(LOAD_TRACKED_DATA){ - /***/ - } - if(SMOOTH_VIDEO) - ClipProcessingJobs clipProcessing("Stabilize", r9); - if(LOAD_SMOOTH_DATA){ - /***/ - } - displayClip(r9); + // // Close timeline + // r9.Close(); + + // // Create a video clip + // std::stringstream path; + // path << TEST_MEDIA_PATH << "test.avi"; + + // // Open clip + // openshot::Clip c1(path.str()); + // c1.Open(); + + // // Create first tracker + // CVStabilization stabilizer("{\"protobuf_data_path\": \"\", \"tracker_type\": \"KCF\", \"bbox\": {\"x\": 294, \"y\": 102, \"w\": 180, \"h\": 166}}"); + + // // Track clip for frames 0-20 + // stabilizer.stabilizeClip(c1, 0, 20+1, true); + + // // Create empty rotation matrix + // for(long int frame_number=0; frame_number<=20; frame_number++){ + // cv::Mat frame_image = c1.GetFrame(frame_number)->GetImageCV(); + // cv::Mat T(2,3,CV_64F); + + // // Get tracked data + // TransformParam tp = stabilizer.GetTransformParamData(frame_number); + // CamTrajectory ct = stabilizer.GetCamTrajectoryTrackedData(frame_number); + + // // Set rotation matrix values + // T.at(0,0) = cos(tp.da); + // T.at(0,1) = -sin(tp.da); + // T.at(1,0) = sin(tp.da); + // T.at(1,1) = cos(tp.da); + + // T.at(0,2) = tp.dx; + // T.at(1,2) = tp.dy; + + // // Apply rotation matrix to image + // cv::Mat frame_stabilized; + // cv::warpAffine(frame_image, frame_stabilized, T, frame_image.size()); + + // // Scale up the image to remove black borders + // cv::Mat T_scale = cv::getRotationMatrix2D(cv::Point2f(frame_stabilized.cols/2, frame_stabilized.rows/2), 0, 1.04); + // cv::warpAffine(frame_stabilized, frame_stabilized, T_scale, frame_stabilized.size()); + + // std::cout< + * @author FeRD (Frank Dana) + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "UnitTest++.h" +// Prevent name clashes with juce::UnitTest +#define DONT_SET_USING_JUCE_NAMESPACE 1 +#include "../include/OpenShot.h" +#include "../include/ProcessingController.h" +#include + +using namespace openshot; + +SUITE(CVStabilizer_Tests) +{ + + // Just for the stabilizer constructor, it won't be used + ProcessingController processingController; + + TEST(Stabilize_Video) + { + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + // Create stabilizer + CVStabilization stabilizer("{\"protobuf_data_path\": \"stabilizer.data\"}", processingController); + + // Stabilize clip for frames 0-20 + stabilizer.stabilizeClip(c1, 0, 20+1, true); + + // Get stabilized data + TransformParam tp = stabilizer.GetTransformParamData(20); + CamTrajectory ct = stabilizer.GetCamTrajectoryTrackedData(20); + + // Compare if stabilized data is equal to pre-tested ones + int dx = tp.dx*10000; + int dy = tp.dy*10000; + int da = tp.da*10000; + int x = ct.x*10000; + int y = ct.y*10000; + int a = ct.a*10000; + + CHECK_EQUAL((int) (37.5902 * 10000), dx); + CHECK_EQUAL((int) (-31.8099 * 10000), dy); + CHECK_EQUAL((int) (0.00720559 * 10000), da); + CHECK_EQUAL((int) (-0.41082 * 10000), x); + CHECK_EQUAL((int) (-0.368437 * 10000), y); + CHECK_EQUAL((int) (-0.000501644 * 10000), a); + } + + + TEST(SaveLoad_Protobuf) + { + + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + // Create first stabilizer + CVStabilization stabilizer_1("{\"protobuf_data_path\": \"stabilizer.data\"}", processingController); + + // Stabilize clip for frames 0-20 + stabilizer_1.stabilizeClip(c1, 0, 20+1, true); + + // Get stabilized data + TransformParam tp_1 = stabilizer_1.GetTransformParamData(20); + CamTrajectory ct_1 = stabilizer_1.GetCamTrajectoryTrackedData(20); + + // Save stabilized data + stabilizer_1.SaveStabilizedData(); + + // Create second stabilizer + CVStabilization stabilizer_2("{\"protobuf_data_path\": \"stabilizer.data\"}", processingController); + + // Load stabilized data from first stabilizer protobuf data + stabilizer_2.LoadStabilizedData(); + + // Get stabilized data + TransformParam tp_2 = stabilizer_2.GetTransformParamData(20); + CamTrajectory ct_2 = stabilizer_2.GetCamTrajectoryTrackedData(20); + + // Compare first stabilizer data with second stabilizer data + CHECK_EQUAL((int) (tp_1.dx * 10000), (int) (tp_2.dx *10000)); + CHECK_EQUAL((int) (tp_1.dy * 10000), (int) (tp_2.dy * 10000)); + CHECK_EQUAL((int) (tp_1.da * 10000), (int) (tp_2.da * 10000)); + CHECK_EQUAL((int) (ct_1.x * 10000), (int) (ct_2.x * 10000)); + CHECK_EQUAL((int) (ct_1.y * 10000), (int) (ct_2.y * 10000)); + CHECK_EQUAL((int) (ct_1.a * 10000), (int) (ct_2.a * 10000)); + } + +} // SUITE(Frame_Tests) diff --git a/tests/CVTracker_Tests.cpp b/tests/CVTracker_Tests.cpp index 4cb55da4..0a3a28bb 100644 --- a/tests/CVTracker_Tests.cpp +++ b/tests/CVTracker_Tests.cpp @@ -33,7 +33,7 @@ // Prevent name clashes with juce::UnitTest #define DONT_SET_USING_JUCE_NAMESPACE 1 #include "../include/OpenShot.h" - +#include "../include/ProcessingController.h" #include using namespace openshot; @@ -41,93 +41,88 @@ using namespace openshot; SUITE(CVTracker_Tests) { -TEST(Track_Video) -{ - // Create a video clip - std::stringstream path; - path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; - openshot::FFmpegReader c1(path.str()); - c1.Open(); + // Just for the tracker constructor, it won't be used + ProcessingController processingController; - // Create Tracker - CVTracker kcfTracker; - bool trackerInit = false; - cv::Rect2d lastTrackedBox; - - for (long int frame = 71; frame <= 97; frame++) + TEST(Track_Video) { - int frame_number = frame; - std::shared_ptr f = c1.GetFrame(frame_number); - - // Grab Mat image - cv::Mat cvimage = f->GetImageCV(); + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; - if(!trackerInit){ - cv::Rect2d bbox(82, 194, 47, 42); - kcfTracker.initTracker(bbox, cvimage, frame_number); - trackerInit = true; - } - else{ - trackerInit = kcfTracker.trackFrame(cvimage, frame_number); - FrameData fd = kcfTracker.GetTrackedData(frame_number); - cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); - lastTrackedBox = box; - } - } + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); - CHECK_EQUAL(27, lastTrackedBox.x); - CHECK_EQUAL(233, lastTrackedBox.y); - CHECK_EQUAL(47, lastTrackedBox.width); - CHECK_EQUAL(42, lastTrackedBox.height); -} + // Create tracker + CVTracker kcfTracker("{\"protobuf_data_path\": \"\", \"tracker_type\": \"KCF\", \"bbox\": {\"x\": 294, \"y\": 102, \"w\": 180, \"h\": 166}}", processingController); + + // Track clip for frames 0-20 + kcfTracker.trackClip(c1, 0, 20+1, true); + + // Get tracked data + FrameData fd = kcfTracker.GetTrackedData(20); + + int x = fd.x1; + int y = fd.y1; + int width = fd.x2-fd.x1; + int height = fd.y2-fd.y1; + + // Compare if tracked data is equal to pre-tested ones + CHECK_EQUAL(260, x); + CHECK_EQUAL(132, y); + CHECK_EQUAL(180, width); + CHECK_EQUAL(166, height); + } -TEST(SaveLoad_Protobuf) -{ - // Create a video clip - std::stringstream path; - path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; - Clip c1(path.str()); - c1.Open(); - - // Create Tracker - CVTracker kcfTracker; - bool trackerInit = false; - cv::Rect2d lastTrackedBox; - - for (long int frame = 71; frame <= 97; frame++) + TEST(SaveLoad_Protobuf) { - int frame_number = frame; - std::shared_ptr f = c1.GetFrame(frame_number); - - // Grab Mat image - cv::Mat cvimage = f->GetImageCV(); - if(!trackerInit){ - cv::Rect2d bbox(82, 194, 47, 42); - kcfTracker.initTracker(bbox, cvimage, frame_number); - trackerInit = true; - } - else{ - trackerInit = kcfTracker.trackFrame(cvimage, frame_number); - FrameData fd = kcfTracker.GetTrackedData(frame_number); - cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); - lastTrackedBox = box; - } - } + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; - kcfTracker.SaveTrackedData("kcf_tracker.data"); + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); - // Create new tracker - CVTracker kcfTracker1; - kcfTracker1.LoadTrackedData("kcf_tracker.data"); - FrameData fd = kcfTracker.GetTrackedData(97); - cv::Rect2d loadedBox(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); + // Create first tracker + CVTracker kcfTracker_1("{\"protobuf_data_path\": \"kcf_tracker.data\", \"tracker_type\": \"KCF\", \"bbox\": {\"x\": 294, \"y\": 102, \"w\": 180, \"h\": 166}}", processingController); - CHECK_EQUAL(loadedBox.x, lastTrackedBox.x); - CHECK_EQUAL(loadedBox.y, lastTrackedBox.y); - CHECK_EQUAL(loadedBox.width, lastTrackedBox.width); - CHECK_EQUAL(loadedBox.height, lastTrackedBox.height); -} + // Track clip for frames 0-20 + kcfTracker_1.trackClip(c1, 0, 20+1, true); + + // Get tracked data + FrameData fd_1 = kcfTracker_1.GetTrackedData(20); + + int x_1 = fd_1.x1; + int y_1 = fd_1.y1; + int width_1 = fd_1.x2-fd_1.x1; + int height_1 = fd_1.y2-fd_1.y1; + + // Save tracked data + kcfTracker_1.SaveTrackedData(); + + // Create second tracker + CVTracker kcfTracker_2("{\"protobuf_data_path\": \"kcf_tracker.data\", \"tracker_type\": \"\", \"bbox\": {\"x\": -1, \"y\": -1, \"w\": -1, \"h\": -1}}", processingController); + + // Load tracked data from first tracker protobuf data + kcfTracker_2.LoadTrackedData(); + + // Get tracked data + FrameData fd_2 = kcfTracker_2.GetTrackedData(20); + + int x_2 = fd_2.x1; + int y_2 = fd_2.y1; + int width_2 = fd_2.x2-fd_2.x1; + int height_2 = fd_2.y2-fd_2.y1; + + // Compare first tracker data with second tracker data + CHECK_EQUAL(x_1, x_2); + CHECK_EQUAL(y_1, y_2); + CHECK_EQUAL(width_1, width_2); + CHECK_EQUAL(height_1, height_2); + } } // SUITE(Frame_Tests)