diff --git a/.gitignore b/.gitignore index 58848653..09a42bc3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ /.metadata/ tags *~ + +.vscode/ diff --git a/include/CVStabilization.h b/include/CVStabilization.h index aca27d00..62fd174e 100644 --- a/include/CVStabilization.h +++ b/include/CVStabilization.h @@ -74,39 +74,45 @@ struct CamTrajectory double a; // angle }; -class CVStabilization { +class CVStabilization { + private: + cv::Mat last_T; cv::Mat cur, cur_grey; cv::Mat prev, prev_grey; - - public: - const int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning - std::vector prev_to_cur_transform; // previous to current - std::vector trajectoryData; // Save camera trajectory data - std::vector transformationData; // Save transormation data - - - CVStabilization(); - - CVStabilization(int _smoothingWindow); - - void ProcessClip(openshot::Clip &video); + std::vector prev_to_cur_transform; // Previous to current // Track current frame features and find the relative transformation void TrackFrameFeatures(cv::Mat frame, int frameNum); std::vector ComputeFramesTrajectory(); - std::vector SmoothTrajectory(std::vector &trajectory); + std::map SmoothTrajectory(std::vector &trajectory); // Generate new transformations parameters for each frame to follow the smoothed trajectory - std::vector GenNewCamPosition(std::vector &smoothed_trajectory); + std::map GenNewCamPosition(std::map &smoothed_trajectory); + + public: + + const int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning + std::map trajectoryData; // Save camera trajectory data + std::map transformationData; // Save transormation data + + // Set default smoothing window value to compute stabilization + CVStabilization(); + + // Set desirable smoothing window value to compute stabilization + CVStabilization(int _smoothingWindow); + + // Process clip and store necessary stabilization data + void ProcessClip(openshot::Clip &video); - // Save protobuf file + /// Protobuf Save and Load methods + // Save stabilization data to protobuf file bool SaveStabilizedData(std::string outputFilePath); - void AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, long int frame_number); - - // Load protobuf file + // Add frame stabilization data into protobuf message + void AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number); + // Load protobuf data file bool LoadStabilizedData(std::string inputFilePath); }; diff --git a/include/CVTracker.h b/include/CVTracker.h index 0acc1cf6..796efc6e 100644 --- a/include/CVTracker.h +++ b/include/CVTracker.h @@ -17,14 +17,10 @@ #include "Frame.h" #include "trackerdata.pb.h" -// using namespace cv; using namespace std; using google::protobuf::util::TimeUtil; -// namespace openshot{ -// class Clip; -// } - +// Tracking info struct struct FrameData{ int frame_id = -1; float rotation = 0; @@ -33,14 +29,10 @@ struct FrameData{ int x2 = -1; int y2 = -1; - // Keyframe kf_x1; - // Keyframe kf_y1; - // Keyframe kf_x2; - // Keyframe kf_y2; - - // constructor + // Constructors FrameData() {} + FrameData( int _frame_id) {frame_id = _frame_id;} @@ -52,35 +44,37 @@ struct FrameData{ y1 = _y1; x2 = _x2; y2 = _y2; - - // kf_x1.AddPoint(_frame_id, _x1); - // kf_y1.AddPoint(_frame_id, _y1); - // kf_x2.AddPoint(_frame_id, _x2); - // kf_y2.AddPoint(_frame_id, _y2); } }; class CVTracker { public: - // List of tracker types in OpenCV - std::string trackerTypes[8] = {"BOOSTING", "MIL", "KCF", "TLD","MEDIANFLOW", "GOTURN", "MOSSE", "CSRT"}; - std::map trackedDataById; - std::string trackerType; - cv::Ptr tracker; - cv::Rect2d bbox; + std::map trackedDataById; // Save tracked data + std::string trackerType; // Name of the chosen tracker + cv::Ptr tracker; // Pointer of the selected tracker + cv::Rect2d bbox; // Bounding box coords + /// Class constructors + // Expects a tracker type, if none is passed, set default as KCF CVTracker(); + CVTracker(std::string trackerType); + + // Set desirable tracker method cv::Ptr select_tracker(std::string trackerType); + // Track object in the hole clip void trackClip(openshot::Clip& video); + // Initialize the tracker bool initTracker(cv::Rect2d bbox, cv::Mat &frame, int frameId); + // Update the object tracker according to frame bool trackFrame(cv::Mat &frame, int frameId); + /// Protobuf Save and Load methods // Save protobuf file bool SaveTrackedData(std::string outputFilePath); + // Add frame tracked data into protobuf message. void AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, FrameData& fData); - // Load protobuf file bool LoadTrackedData(std::string inputFilePath); diff --git a/include/ClipProcessingJobs.h b/include/ClipProcessingJobs.h index 0a833efc..d1833eba 100644 --- a/include/ClipProcessingJobs.h +++ b/include/ClipProcessingJobs.h @@ -47,15 +47,16 @@ using namespace openshot; +// Constructor responsible to choose processing type and apply to clip class ClipProcessingJobs{ private: + // Apply object tracking to clip void trackVideo(Clip& videoClip); + // Apply stabilization to clip void stabilizeVideo(Clip& video); - - public: ClipProcessingJobs(std::string processingType, Clip& videoClip); diff --git a/include/effects/Stabilizer.h b/include/effects/Stabilizer.h index 0bb4d31a..02916848 100644 --- a/include/effects/Stabilizer.h +++ b/include/effects/Stabilizer.h @@ -46,13 +46,12 @@ namespace openshot { - //TODO: fix this + /** - * @brief This class draws black bars around your video (from any side), and can be animated with - * openshot::Keyframe curves over time. + * @brief This class stabilizes video clip to remove undesired shaking and jitter. * - * Adding bars around your video can be done for cinematic reasons, and creates a fun way to frame - * in the focal point of a scene. The bars can be any color, and each side can be animated independently. + * Adding stabilization is useful to increase video quality overall, since it removes + * from subtle to harsh unexpected camera movements. */ class Stabilizer : public EffectBase { @@ -62,26 +61,14 @@ namespace openshot public: - Color color; ///< Color of bars - Keyframe left; ///< Size of left bar - Keyframe top; ///< Size of top bar - Keyframe right; ///< Size of right bar - Keyframe bottom; ///< Size of bottom bar - - std::vector trajectoryData; // Save camera trajectory data - std::vector transformationData; // Save transormation data + std::map trajectoryData; // Save camera trajectory data + std::map transformationData; // Save transormation data /// Blank constructor, useful when using Json to load the effect properties Stabilizer(std::string clipTrackerDataPath); - /// Default constructor, which takes 4 curves and a color. These curves animated the bars over time. - /// - /// @param color The curve to adjust the color of bars - /// @param left The curve to adjust the left bar size (between 0 and 1) - /// @param top The curve to adjust the top bar size (between 0 and 1) - /// @param right The curve to adjust the right bar size (between 0 and 1) - /// @param bottom The curve to adjust the bottom bar size (between 0 and 1) - Stabilizer(Color color, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom); + /// Default constructor + Stabilizer(); /// @brief This method is required for all derived classes of EffectBase, and returns a /// modified openshot::Frame object @@ -94,6 +81,7 @@ namespace openshot /// @param frame_number The frame number (starting at 1) of the effect on the timeline. std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + // Load protobuf data file bool LoadStabilizedData(std::string inputFilePath); /// Get and Set JSON methods diff --git a/include/effects/Tracker.h b/include/effects/Tracker.h index c5f39daf..a84f4e75 100644 --- a/include/effects/Tracker.h +++ b/include/effects/Tracker.h @@ -47,49 +47,10 @@ namespace openshot { -// struct FrameData{ -// int frame_id = -1; -// float rotation = 0; -// int x1 = -1; -// int y1 = -1; -// int x2 = -1; -// int y2 = -1; - -// // Keyframe kf_x1; -// // Keyframe kf_y1; -// // Keyframe kf_x2; -// // Keyframe kf_y2; - -// // constructor -// FrameData() -// {} -// FrameData( int _frame_id) -// {frame_id = _frame_id;} - -// FrameData( int _frame_id , float _rotation, int _x1, int _y1, int _x2, int _y2) -// { -// frame_id = _frame_id; -// rotation = _rotation; -// x1 = _x1; -// y1 = _y1; -// x2 = _x2; -// y2 = _y2; - -// // kf_x1.AddPoint(_frame_id, _x1); -// // kf_y1.AddPoint(_frame_id, _y1); -// // kf_x2.AddPoint(_frame_id, _x2); -// // kf_y2.AddPoint(_frame_id, _y2); -// } -// }; - - - //TODO: fix this /** - * @brief This class draws black bars around your video (from any side), and can be animated with - * openshot::Keyframe curves over time. + * @brief This class track a given object through the clip and, when called, draws a box surrounding it. * - * Adding bars around your video can be done for cinematic reasons, and creates a fun way to frame - * in the focal point of a scene. The bars can be any color, and each side can be animated independently. + * Tracking is useful to better visualize and follow the movement of an object through video. */ class Tracker : public EffectBase { @@ -99,25 +60,14 @@ namespace openshot public: - Color color; ///< Color of bars - Keyframe left; ///< Size of left bar - Keyframe top; ///< Size of top bar - Keyframe right; ///< Size of right bar - Keyframe bottom; ///< Size of bottom bar - std::map trackedDataById; + std::map trackedDataById; // Save object tracking box data /// Blank constructor, useful when using Json to load the effect properties Tracker(std::string clipTrackerDataPath); - /// Default constructor, which takes 4 curves and a color. These curves animated the bars over time. - /// - /// @param color The curve to adjust the color of bars - /// @param left The curve to adjust the left bar size (between 0 and 1) - /// @param top The curve to adjust the top bar size (between 0 and 1) - /// @param right The curve to adjust the right bar size (between 0 and 1) - /// @param bottom The curve to adjust the bottom bar size (between 0 and 1) - Tracker(Color color, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom); + /// Default constructor + Tracker(); /// @brief This method is required for all derived classes of EffectBase, and returns a /// modified openshot::Frame object @@ -130,8 +80,10 @@ namespace openshot /// @param frame_number The frame number (starting at 1) of the effect on the timeline. std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + // Load protobuf data file bool LoadTrackedData(std::string inputFilePath); + // Get tracker info for the desired frame FrameData GetTrackedData(int frameId); /// Get and Set JSON methods diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp index 32e05c85..01477988 100644 --- a/src/CVStabilization.cpp +++ b/src/CVStabilization.cpp @@ -1,12 +1,44 @@ +/** + * @file + * @brief Source file for CVStabilization class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + #include "../include/CVStabilization.h" - +// Set default smoothing window value to compute stabilization CVStabilization::CVStabilization():smoothingWindow(30) {} +// Set desirable smoothing window value to compute stabilization CVStabilization::CVStabilization(int _smoothingWindow): smoothingWindow(_smoothingWindow){} +// Process clip and store necessary stabilization data void CVStabilization::ProcessClip(openshot::Clip &video){ - // Get total number of frames + // Get total number of frames in video int videoLenght = video.Reader()->info.video_length; // Extract and track opticalflow features for each frame @@ -14,16 +46,19 @@ void CVStabilization::ProcessClip(openshot::Clip &video){ { std::shared_ptr f = video.GetFrame(frame_number); - // Grab Mat image + // Grab OpenCV Mat image cv::Mat cvimage = f->GetImageCV(); cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); TrackFrameFeatures(cvimage, frame_number); } - vector trajectory = ComputeFramesTrajectory(); + // Calculate trajectory data + std::vector trajectory = ComputeFramesTrajectory(); + // Calculate and save smoothed trajectory data trajectoryData = SmoothTrajectory(trajectory); + // Calculate and save transformation data transformationData = GenNewCamPosition(trajectoryData); } @@ -35,12 +70,12 @@ void CVStabilization::TrackFrameFeatures(cv::Mat frame, int frameNum){ } // OpticalFlow features vector - vector prev_corner, cur_corner; - vector prev_corner2, cur_corner2; - vector status; - vector err; + std::vector prev_corner, cur_corner; + std::vector prev_corner2, cur_corner2; + std::vector status; + std::vector err; - // Extract new image teatures + // Extract new image features cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30); // Track features cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err); @@ -52,30 +87,30 @@ void CVStabilization::TrackFrameFeatures(cv::Mat frame, int frameNum){ cur_corner2.push_back(cur_corner[i]); } } - // translation + rotation only + // Translation + rotation only cv::Mat T = estimateRigidTransform(prev_corner2, cur_corner2, false); // false = rigid transform, no scaling/shearing - // If no transform is found. We'll just use the last known good transform. + // If no transformation is found, just use the last known good transform. if(T.data == NULL) { last_T.copyTo(T); } T.copyTo(last_T); - // decompose T + // Decompose T double dx = T.at(0,2); double dy = T.at(1,2); double da = atan2(T.at(1,0), T.at(0,0)); prev_to_cur_transform.push_back(TransformParam(dx, dy, da)); - // out_transform << frameNum << " " << dx << " " << dy << " " << da << endl; cur.copyTo(prev); frame.copyTo(prev_grey); + // Show processing info cout << "Frame: " << frameNum << " - good optical flow: " << prev_corner2.size() << endl; } -vector CVStabilization::ComputeFramesTrajectory(){ +std::vector CVStabilization::ComputeFramesTrajectory(){ // Accumulated frame to frame transform double a = 0; @@ -90,17 +125,16 @@ vector CVStabilization::ComputeFramesTrajectory(){ y += prev_to_cur_transform[i].dy; a += prev_to_cur_transform[i].da; + // Save trajectory data to vector trajectory.push_back(CamTrajectory(x,y,a)); - - // out_trajectory << (i+1) << " " << x << " " << y << " " << a << endl; } return trajectory; } -vector CVStabilization::SmoothTrajectory(vector &trajectory){ +std::map CVStabilization::SmoothTrajectory(std::vector &trajectory){ - vector smoothed_trajectory; // trajectory at all frames + std::map smoothed_trajectory; // trajectory at all frames for(size_t i=0; i < trajectory.size(); i++) { double sum_x = 0; @@ -122,16 +156,15 @@ vector CVStabilization::SmoothTrajectory(vector double avg_x = sum_x / count; double avg_y = sum_y / count; - smoothed_trajectory.push_back(CamTrajectory(avg_x, avg_y, avg_a)); - - // out_smoothed_trajectory << (i+1) << " " << avg_x << " " << avg_y << " " << avg_a << endl; + // Add smoothed trajectory data to map + smoothed_trajectory[i] = CamTrajectory(avg_x, avg_y, avg_a); } return smoothed_trajectory; } // Generate new transformations parameters for each frame to follow the smoothed trajectory -vector CVStabilization::GenNewCamPosition(vector & smoothed_trajectory){ - vector new_prev_to_cur_transform; +std::map CVStabilization::GenNewCamPosition(std::map &smoothed_trajectory){ + std::map new_prev_to_cur_transform; // Accumulated frame to frame transform double a = 0; @@ -152,24 +185,26 @@ vector CVStabilization::GenNewCamPosition(vector ::iterator trajData = trajectoryData.begin(); - std::vector::iterator transData = transformationData.begin(); + std::map::iterator trajData = trajectoryData.begin(); + std::map::iterator transData = transformationData.begin(); - for(long int frame_count = 0; trajData != trajectoryData.end(); ++trajData, ++transData){ - AddFrameDataToProto(stabilizationMessage.add_frame(), *trajData, *transData, frame_count); + // Iterate over all frames data and save in protobuf message + for(; trajData != trajectoryData.end(); ++trajData, ++transData){ + AddFrameDataToProto(stabilizationMessage.add_frame(), trajData->second, transData->second, trajData->first); } // Add timestamp *stabilizationMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL)); @@ -187,7 +222,8 @@ bool CVStabilization::SaveStabilizedData(std::string outputFilePath){ return true; } -void CVStabilization::AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, long int frame_number){ +// Add frame stabilization data into protobuf message +void CVStabilization::AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number){ // Save frame number pbFrameData->set_id(frame_number); @@ -203,8 +239,9 @@ void CVStabilization::AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameDa pbFrameData->set_dy(transData.dy); } -// Load protobuf file +// Load protobuf data file bool CVStabilization::LoadStabilizedData(std::string inputFilePath){ + // Create stabilization message libopenshotstabilize::Stabilization stabilizationMessage; // Read the existing tracker message. @@ -214,29 +251,35 @@ bool CVStabilization::LoadStabilizedData(std::string inputFilePath){ return false; } - // Make sure the data vectors are empty + // Make sure the data maps are empty transformationData.clear(); trajectoryData.clear(); - // Iterate over all frames of the saved message - for (int i = 0; i < stabilizationMessage.frame_size(); i++) { + // Iterate over all frames of the saved message and assign to the data maps + for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) { const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i); - + + // Load frame number int id = pbFrameData.id(); + // Load camera trajectory data float x = pbFrameData.x(); float y = pbFrameData.y(); float a = pbFrameData.a(); - trajectoryData.push_back(CamTrajectory(x,y,a)); + // Assign data to trajectory map + trajectoryData[i] = CamTrajectory(x,y,a); + // Load transformation data float dx = pbFrameData.dx(); float dy = pbFrameData.dy(); float da = pbFrameData.da(); - transformationData.push_back(TransformParam(dx,dy,da)); + // Assing data to transformation map + transformationData[i] = TransformParam(dx,dy,da); } + // Show the time stamp from the last update in stabilization data file if (stabilizationMessage.has_last_updated()) { cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; } diff --git a/src/CVTracker.cpp b/src/CVTracker.cpp index 4ba3598f..42f73dcd 100644 --- a/src/CVTracker.cpp +++ b/src/CVTracker.cpp @@ -1,43 +1,67 @@ +/** + * @file + * @brief Track an object selected by the user + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + #include "../include/CVTracker.h" -CVTracker::CVTracker(){ - - // Create KCF tracker - trackerType = trackerTypes[2]; - tracker = select_tracker(trackerType); +/// Class constructors +// Expects a tracker type, if none is passed, set default as KCF +CVTracker::CVTracker(std::string trackerType) : trackerType(trackerType){ +} +CVTracker::CVTracker() : trackerType("KCF"){ } // Set desirable tracker method cv::Ptr CVTracker::select_tracker(std::string trackerType){ cv::Ptr t; - #if (CV_MINOR_VERSION < 3) - { - t = cv::Tracker::create(trackerType); - } - #else - { - if (trackerType == "BOOSTING") - t = cv::TrackerBoosting::create(); - if (trackerType == "MIL") - t = cv::TrackerMIL::create(); - if (trackerType == "KCF") - t = cv::TrackerKCF::create(); - if (trackerType == "TLD") - t = cv::TrackerTLD::create(); - if (trackerType == "MEDIANFLOW") - t = cv::TrackerMedianFlow::create(); - if (trackerType == "GOTURN") - t = cv::TrackerGOTURN::create(); - if (trackerType == "MOSSE") - t = cv::TrackerMOSSE::create(); - if (trackerType == "CSRT") - t = cv::TrackerCSRT::create(); - } - #endif + + if (trackerType == "BOOSTING") + t = cv::TrackerBoosting::create(); + if (trackerType == "MIL") + t = cv::TrackerMIL::create(); + if (trackerType == "KCF") + t = cv::TrackerKCF::create(); + if (trackerType == "TLD") + t = cv::TrackerTLD::create(); + if (trackerType == "MEDIANFLOW") + t = cv::TrackerMedianFlow::create(); + if (trackerType == "GOTURN") + t = cv::TrackerGOTURN::create(); + if (trackerType == "MOSSE") + t = cv::TrackerMOSSE::create(); + if (trackerType == "CSRT") + t = cv::TrackerCSRT::create(); return t; } +// Track object in the hole clip void CVTracker::trackClip(openshot::Clip& video){ // Opencv display window cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); @@ -54,7 +78,7 @@ void CVTracker::trackClip(openshot::Clip& video){ // Get current frame std::shared_ptr f = video.GetFrame(frame_number); - // Grab Mat image + // Grab OpenCV Mat image cv::Mat cvimage = f->GetImageCV(); // Pass the first frame to initialize the tracker @@ -70,6 +94,7 @@ void CVTracker::trackClip(openshot::Clip& video){ trackerInit = true; } else{ + // Update the object tracker according to frame trackerInit = trackFrame(cvimage, frame_number); // Draw box on image @@ -79,25 +104,24 @@ void CVTracker::trackClip(openshot::Clip& video){ cv::rectangle(cvimage, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); } + // Show tracking process cv::imshow("Display Image", cvimage); - // Press ESC on keyboard to exit + // Press ESC on keyboard to exit char c=(char)cv::waitKey(1); if(c==27) break; } - - } - +// Initialize the tracker bool CVTracker::initTracker(cv::Rect2d initial_bbox, cv::Mat &frame, int frameId){ - + // Set initial bounding box coords bbox = initial_bbox; - // rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 ); // Create new tracker object tracker = select_tracker(trackerType); + // Initialize tracker tracker->init(frame, bbox); // Add new frame data @@ -106,10 +130,13 @@ bool CVTracker::initTracker(cv::Rect2d initial_bbox, cv::Mat &frame, int frameId return true; } +// Update the object tracker according to frame bool CVTracker::trackFrame(cv::Mat &frame, int frameId){ // Update the tracking result bool ok = tracker->update(frame, bbox); + // Add frame number and box coords if tracker finds the object + // Otherwise add only frame number if (ok) { // Add new frame data @@ -128,9 +155,8 @@ bool CVTracker::SaveTrackedData(std::string outputFilePath){ // Create tracker message libopenshottracker::Tracker trackerMessage; - // Add all frames data + // Iterate over all frames data and save in protobuf message for(std::map::iterator it=trackedDataById.begin(); it!=trackedDataById.end(); ++it){ - FrameData fData = it->second; libopenshottracker::Frame* pbFrameData; AddFrameDataToProto(trackerMessage.add_frame(), fData); @@ -158,19 +184,21 @@ bool CVTracker::SaveTrackedData(std::string outputFilePath){ // Add frame tracked data into protobuf message. void CVTracker::AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, FrameData& fData) { - pbFrameData->set_id(fData.frame_id); - pbFrameData->set_rotation(fData.frame_id); - pbFrameData->set_rotation(fData.frame_id); + // Save frame number and rotation + pbFrameData->set_id(fData.frame_id); + pbFrameData->set_rotation(0); - libopenshottracker::Frame::Box* box = pbFrameData->mutable_bounding_box(); + libopenshottracker::Frame::Box* box = pbFrameData->mutable_bounding_box(); + // Save bounding box data box->set_x1(fData.x1); box->set_y1(fData.y1); box->set_x2(fData.x2); box->set_y2(fData.y2); } +// Load protobuf data file bool CVTracker::LoadTrackedData(std::string inputFilePath){ - + // Create tracker message libopenshottracker::Tracker trackerMessage; { @@ -189,18 +217,22 @@ bool CVTracker::LoadTrackedData(std::string inputFilePath){ for (int i = 0; i < trackerMessage.frame_size(); i++) { const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i); + // Load frame and rotation data int id = pbFrameData.id(); float rotation = pbFrameData.rotation(); + // Load bounding box data const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box(); int x1 = box.x1(); int y1 = box.y1(); int x2 = box.x2(); int y2 = box.y2(); + // Assign data to tracker map trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2); } + // Show the time stamp from the last update in tracker data file if (trackerMessage.has_last_updated()) { cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl; } diff --git a/src/ClipProcessingJobs.cpp b/src/ClipProcessingJobs.cpp index 8d588b5b..505a4476 100644 --- a/src/ClipProcessingJobs.cpp +++ b/src/ClipProcessingJobs.cpp @@ -1,6 +1,6 @@ #include "../include/ClipProcessingJobs.h" - +// Constructor responsible to choose processing type and apply to clip ClipProcessingJobs::ClipProcessingJobs(std::string processingType, Clip& videoClip){ if(processingType == "Stabilize"){ @@ -11,30 +11,39 @@ ClipProcessingJobs::ClipProcessingJobs(std::string processingType, Clip& videoCl } +// Apply object tracking to clip void ClipProcessingJobs::trackVideo(Clip& videoClip){ // Opencv display window cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); - // Create Tracker - CVTracker tracker; - tracker.trackClip(videoClip); + // Create CVTracker object + CVTracker tracker; + // Start tracking + tracker.trackClip(videoClip); + + // Save tracking data tracker.SaveTrackedData("kcf_tracker.data"); // Create new Tracker Effect EffectBase* trackerEffect = new Tracker("kcf_tracker.data"); + // Apply Tracker Effect to clip videoClip.AddEffect(trackerEffect); } +// Apply stabilization to clip void ClipProcessingJobs::stabilizeVideo(Clip& videoClip){ // create CVStabilization object CVStabilization stabilizer; + // Start stabilization process stabilizer.ProcessClip(videoClip); + // Save stabilization data stabilizer.SaveStabilizedData("stabilization.data"); - // Create new Tracker Effect + // Create new Stabilizer Effect EffectBase* stabilizeEffect = new Stabilizer("stabilization.data"); + // Apply Stabilizer Effect to clip videoClip.AddEffect(stabilizeEffect); diff --git a/src/effects/Stabilizer.cpp b/src/effects/Stabilizer.cpp index 6015c0eb..cb367f41 100644 --- a/src/effects/Stabilizer.cpp +++ b/src/effects/Stabilizer.cpp @@ -43,8 +43,7 @@ Stabilizer::Stabilizer(std::string clipStabilizedDataPath) } // Default constructor -Stabilizer::Stabilizer(Color color, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom) : - color(color), left(left), top(top), right(right), bottom(bottom) +Stabilizer::Stabilizer() { // Init effect properties init_effect_details(); @@ -68,9 +67,12 @@ void Stabilizer::init_effect_details() // modified openshot::Frame object std::shared_ptr Stabilizer::GetFrame(std::shared_ptr frame, int64_t frame_number) { + // Create empty rotation matrix cv::Mat T(2,3,CV_64F); - // Grab Mat image + // Grab OpenCV Mat image cv::Mat cur = frame->GetImageCV(); + + // Set rotation matrix values T.at(0,0) = cos(transformationData[frame_number].da); T.at(0,1) = -sin(transformationData[frame_number].da); T.at(1,0) = sin(transformationData[frame_number].da); @@ -78,19 +80,24 @@ std::shared_ptr Stabilizer::GetFrame(std::shared_ptr frame, int64_ T.at(0,2) = transformationData[frame_number].dx; T.at(1,2) = transformationData[frame_number].dy; - cv::Mat frame_stabilized; + // Apply rotation matrix to image + cv::Mat frame_stabilized; cv::warpAffine(cur, frame_stabilized, T, cur.size()); + // Scale up the image to remove black borders cv::Mat T_scale = cv::getRotationMatrix2D(cv::Point2f(frame_stabilized.cols/2, frame_stabilized.rows/2), 0, 1.04); - cv::warpAffine(frame_stabilized, frame_stabilized, T_scale, frame_stabilized.size()); + cv::warpAffine(frame_stabilized, frame_stabilized, T_scale, frame_stabilized.size()); + + // Set stabilized image to frame frame->SetImageCV(frame_stabilized); return frame; } -// Load protobuf file +// Load protobuf data file bool Stabilizer::LoadStabilizedData(std::string inputFilePath){ + // Create stabilization message libopenshotstabilize::Stabilization stabilizationMessage; // Read the existing tracker message. @@ -100,29 +107,36 @@ bool Stabilizer::LoadStabilizedData(std::string inputFilePath){ return false; } - // Make sure the data vectors are empty + // Make sure the data maps are empty transformationData.clear(); trajectoryData.clear(); - // Iterate over all frames of the saved message - for (int i = 0; i < stabilizationMessage.frame_size(); i++) { + // Iterate over all frames of the saved message and assign to the data maps + for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) { + // Create stabilization message const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i); - + + // Load frame number int id = pbFrameData.id(); + // Load camera trajectory data float x = pbFrameData.x(); float y = pbFrameData.y(); float a = pbFrameData.a(); - trajectoryData.push_back(CamTrajectory(x,y,a)); + // Assign data to trajectory map + trajectoryData[i] = CamTrajectory(x,y,a); + // Load transformation data float dx = pbFrameData.dx(); float dy = pbFrameData.dy(); float da = pbFrameData.da(); - transformationData.push_back(TransformParam(dx,dy,da)); + // Assing data to transformation map + transformationData[i] = TransformParam(dx,dy,da); } + // Show the time stamp from the last update in stabilization data file if (stabilizationMessage.has_last_updated()) { cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; } @@ -148,11 +162,6 @@ Json::Value Stabilizer::JsonValue() const { // Create root json object Json::Value root = EffectBase::JsonValue(); // get parent properties root["type"] = info.class_name; - root["color"] = color.JsonValue(); - root["left"] = left.JsonValue(); - root["top"] = top.JsonValue(); - root["right"] = right.JsonValue(); - root["bottom"] = bottom.JsonValue(); // return JsonValue return root; @@ -180,18 +189,6 @@ void Stabilizer::SetJsonValue(const Json::Value root) { // Set parent data EffectBase::SetJsonValue(root); - - // Set data from Json (if key is found) - if (!root["color"].isNull()) - color.SetJsonValue(root["color"]); - if (!root["left"].isNull()) - left.SetJsonValue(root["left"]); - if (!root["top"].isNull()) - top.SetJsonValue(root["top"]); - if (!root["right"].isNull()) - right.SetJsonValue(root["right"]); - if (!root["bottom"].isNull()) - bottom.SetJsonValue(root["bottom"]); } // Get all properties for a specific frame @@ -206,16 +203,6 @@ std::string Stabilizer::PropertiesJSON(int64_t requested_frame) const { root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); - // Keyframes - root["color"] = add_property_json("Bar Color", 0.0, "color", "", NULL, 0, 255, false, requested_frame); - root["color"]["red"] = add_property_json("Red", color.red.GetValue(requested_frame), "float", "", &color.red, 0, 255, false, requested_frame); - root["color"]["blue"] = add_property_json("Blue", color.blue.GetValue(requested_frame), "float", "", &color.blue, 0, 255, false, requested_frame); - root["color"]["green"] = add_property_json("Green", color.green.GetValue(requested_frame), "float", "", &color.green, 0, 255, false, requested_frame); - root["left"] = add_property_json("Left Size", left.GetValue(requested_frame), "float", "", &left, 0.0, 0.5, false, requested_frame); - root["top"] = add_property_json("Top Size", top.GetValue(requested_frame), "float", "", &top, 0.0, 0.5, false, requested_frame); - root["right"] = add_property_json("Right Size", right.GetValue(requested_frame), "float", "", &right, 0.0, 0.5, false, requested_frame); - root["bottom"] = add_property_json("Bottom Size", bottom.GetValue(requested_frame), "float", "", &bottom, 0.0, 0.5, false, requested_frame); - // Return formatted string return root.toStyledString(); } diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 87035ed7..3770f80e 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -43,8 +43,7 @@ Tracker::Tracker(std::string clipTrackerDataPath) } // Default constructor -Tracker::Tracker(Color color, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom) : - color(color), left(left), top(top), right(right), bottom(bottom) +Tracker::Tracker() { // Init effect properties init_effect_details(); @@ -71,17 +70,21 @@ std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t f { // Get the frame's image cv::Mat frame_image = frame->GetImageCV(); + // Draw box on image FrameData fd = trackedDataById[frame_number]; cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); + + // Set image with drawn box to frame frame->SetImageCV(frame_image); return frame; } +// Load protobuf data file bool Tracker::LoadTrackedData(std::string inputFilePath){ - + // Create tracker message libopenshottracker::Tracker trackerMessage; { @@ -100,18 +103,22 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ for (int i = 0; i < trackerMessage.frame_size(); i++) { const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i); + // Load frame and rotation data int id = pbFrameData.id(); float rotation = pbFrameData.rotation(); + // Load bounding box data const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box(); int x1 = box.x1(); int y1 = box.y1(); int x2 = box.x2(); int y2 = box.y2(); + // Assign data to tracker map trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2); } + // Show the time stamp from the last update in tracker data file if (trackerMessage.has_last_updated()) { cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl; } @@ -122,6 +129,7 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ return true; } +// Get tracker info for the desired frame FrameData Tracker::GetTrackedData(int frameId){ // Check if the tracker info for the requested frame exists @@ -133,10 +141,6 @@ FrameData Tracker::GetTrackedData(int frameId){ } - - - - // Generate JSON string of this object std::string Tracker::Json() const { @@ -150,11 +154,6 @@ Json::Value Tracker::JsonValue() const { // Create root json object Json::Value root = EffectBase::JsonValue(); // get parent properties root["type"] = info.class_name; - root["color"] = color.JsonValue(); - root["left"] = left.JsonValue(); - root["top"] = top.JsonValue(); - root["right"] = right.JsonValue(); - root["bottom"] = bottom.JsonValue(); // return JsonValue return root; @@ -182,18 +181,6 @@ void Tracker::SetJsonValue(const Json::Value root) { // Set parent data EffectBase::SetJsonValue(root); - - // Set data from Json (if key is found) - if (!root["color"].isNull()) - color.SetJsonValue(root["color"]); - if (!root["left"].isNull()) - left.SetJsonValue(root["left"]); - if (!root["top"].isNull()) - top.SetJsonValue(root["top"]); - if (!root["right"].isNull()) - right.SetJsonValue(root["right"]); - if (!root["bottom"].isNull()) - bottom.SetJsonValue(root["bottom"]); } // Get all properties for a specific frame @@ -208,16 +195,6 @@ std::string Tracker::PropertiesJSON(int64_t requested_frame) const { root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); - // Keyframes - root["color"] = add_property_json("Bar Color", 0.0, "color", "", NULL, 0, 255, false, requested_frame); - root["color"]["red"] = add_property_json("Red", color.red.GetValue(requested_frame), "float", "", &color.red, 0, 255, false, requested_frame); - root["color"]["blue"] = add_property_json("Blue", color.blue.GetValue(requested_frame), "float", "", &color.blue, 0, 255, false, requested_frame); - root["color"]["green"] = add_property_json("Green", color.green.GetValue(requested_frame), "float", "", &color.green, 0, 255, false, requested_frame); - root["left"] = add_property_json("Left Size", left.GetValue(requested_frame), "float", "", &left, 0.0, 0.5, false, requested_frame); - root["top"] = add_property_json("Top Size", top.GetValue(requested_frame), "float", "", &top, 0.0, 0.5, false, requested_frame); - root["right"] = add_property_json("Right Size", right.GetValue(requested_frame), "float", "", &right, 0.0, 0.5, false, requested_frame); - root["bottom"] = add_property_json("Bottom Size", bottom.GetValue(requested_frame), "float", "", &bottom, 0.0, 0.5, false, requested_frame); - // Return formatted string return root.toStyledString(); } diff --git a/src/examples/Example_opencv.cpp b/src/examples/Example_opencv.cpp index 4941b7e7..47f87179 100644 --- a/src/examples/Example_opencv.cpp +++ b/src/examples/Example_opencv.cpp @@ -31,10 +31,10 @@ #include #include #include -// #include -// #include "../../include/CVTracker.h" -// #include "../../include/CVStabilization.h" -// #include "treackerdata.pb.h" +#include +#include "../../include/CVTracker.h" +#include "../../include/CVStabilization.h" +#include "../../include/trackerdata.pb.h" #include "../../include/OpenShot.h" #include "../../include/CrashHandler.h" @@ -42,73 +42,6 @@ using namespace openshot; // using namespace cv; -void trackVideo(openshot::Clip &r9){ - // Opencv display window - cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); - // Create Tracker - - ClipProcessingJobs clipProcessing("Track", r9); - - // CVTracker kcfTracker; - - // kcfTracker.trackClip(r9); - // bool trackerInit = false; - // int videoLenght = r9.Reader()->info.video_length; - // for (long int frame = 0; frame < videoLenght; frame++) - // { - // int frame_number = frame; - // std::shared_ptr f = r9.GetFrame(frame_number); - - // // Grab Mat image - // cv::Mat cvimage = f->GetImageCV(); - - // if(!trackerInit){ - // cv::Rect2d bbox = cv::selectROI("Display Image", cvimage); - - // kcfTracker.initTracker(bbox, cvimage, frame_number); - // cv::rectangle(cvimage, bbox, cv::Scalar( 255, 0, 0 ), 2, 1 ); - - // trackerInit = true; - // } - // else{ - // trackerInit = kcfTracker.trackFrame(cvimage, frame_number); - - // // Draw box on image - // FrameData fd = kcfTracker.GetTrackedData(frame_number); - // // std::cout<< "fd: "<< fd.x1<< " "<< fd.y1 <<" "<info.video_length; - for (long int frame = 0; frame < videoLenght; frame++) - { - int frame_number = frame; - std::shared_ptr f = r9.GetFrame(frame_number); - - // Grab Mat image - cv::Mat cvimage = f->GetImageCV(); - - cv::imshow("Display Tracked result", cvimage); - // Press ESC on keyboard to exit - char c=(char)cv::waitKey(25); - if(c==27) - break; - } -} - void displayTrackedData(openshot::Clip &r9){ // Opencv display window cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); @@ -143,7 +76,7 @@ void displayTrackedData(openshot::Clip &r9){ } -void displayStabilization(openshot::Clip &r9){ +void displayClip(openshot::Clip &r9){ // Opencv display window cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); @@ -169,27 +102,26 @@ int main(int argc, char* argv[]) { bool TRACK_DATA = false; bool LOAD_TRACKED_DATA = false; bool SMOOTH_VIDEO = true; + bool LOAD_SMOOTH_DATA = false; std::string input_filepath = TEST_MEDIA_PATH; - input_filepath = "/media/brenno/Data/projects/openshot/libopenshot/src/examples/test.avi"; + input_filepath = "../../src/examples/test.avi"; openshot::Clip r9(input_filepath); r9.Open(); if(TRACK_DATA) - trackVideo(r9); - if(LOAD_TRACKED_DATA) - displayTrackedData(r9); - if(SMOOTH_VIDEO){ - ClipProcessingJobs clipProcessing("Stabilize", r9); - // CVStabilization stabilization; - // r9.stabilize_video(); - displayStabilization(r9); + ClipProcessingJobs clipProcessing("Track", r9); + if(LOAD_TRACKED_DATA){ + /***/ } - - - + if(SMOOTH_VIDEO) + ClipProcessingJobs clipProcessing("Stabilize", r9); + if(LOAD_SMOOTH_DATA){ + /***/ + } + displayClip(r9); // Close timeline r9.Close();