From c60236a1c32b19cb499fd972e2e43cf7032b7f7c Mon Sep 17 00:00:00 2001 From: Brenno Date: Wed, 8 Jul 2020 22:31:26 -0300 Subject: [PATCH] Added Stabilize Effect --- include/CVStabilization.h | 18 ++- include/Clip.h | 16 +-- include/ClipProcessingJobs.h | 1 + include/effects/Stabilizer.h | 112 ++++++++++++++++ src/CMakeLists.txt | 6 +- src/CVStabilization.cpp | 135 ++++++++++++++----- src/CVTracker.cpp | 13 +- src/Clip.cpp | 116 ++++++++--------- src/ClipProcessingJobs.cpp | 44 +------ src/effects/Stabilizer.cpp | 221 ++++++++++++++++++++++++++++++++ src/effects/Tracker.cpp | 11 +- src/examples/Example_opencv.cpp | 4 +- 12 files changed, 541 insertions(+), 156 deletions(-) create mode 100644 include/effects/Stabilizer.h create mode 100644 src/effects/Stabilizer.cpp diff --git a/include/CVStabilization.h b/include/CVStabilization.h index 303b240e..aca27d00 100644 --- a/include/CVStabilization.h +++ b/include/CVStabilization.h @@ -31,6 +31,8 @@ #ifndef OPENSHOT_STABILIZATION_H #define OPENSHOT_STABILIZATION_H +#include + #define int64 opencv_broken_int #define uint64 opencv_broken_uint #include @@ -38,8 +40,11 @@ #undef uint64 #undef int64 #include +#include "stabilizedata.pb.h" +#include "Clip.h" using namespace std; +using google::protobuf::util::TimeUtil; struct TransformParam { @@ -78,12 +83,15 @@ class CVStabilization { public: const int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning std::vector prev_to_cur_transform; // previous to current + std::vector trajectoryData; // Save camera trajectory data + std::vector transformationData; // Save transormation data + CVStabilization(); CVStabilization(int _smoothingWindow); - // void ProcessVideo(openshot::Clip &video); + void ProcessClip(openshot::Clip &video); // Track current frame features and find the relative transformation void TrackFrameFeatures(cv::Mat frame, int frameNum); @@ -94,8 +102,12 @@ class CVStabilization { // Generate new transformations parameters for each frame to follow the smoothed trajectory std::vector GenNewCamPosition(std::vector &smoothed_trajectory); - // Send smoothed camera transformation to be applyed on clip - // void ApplyNewTrajectoryToClip(openshot::Clip &video, std::vector &new_prev_to_cur_transform); + // Save protobuf file + bool SaveStabilizedData(std::string outputFilePath); + void AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, long int frame_number); + + // Load protobuf file + bool LoadStabilizedData(std::string inputFilePath); }; diff --git a/include/Clip.h b/include/Clip.h index a698c549..dc33223d 100644 --- a/include/Clip.h +++ b/include/Clip.h @@ -39,7 +39,7 @@ #undef uint64 #undef int64 - #include "CVStabilization.h" + // #include "CVStabilization.h" #endif #include @@ -161,13 +161,13 @@ namespace openshot { openshot::FrameDisplayType display; ///< The format to display the frame number (if any) openshot::VolumeMixType mixing; ///< What strategy should be followed when mixing audio with other clips - #ifdef USE_OPENCV - /// Smoothed transformation for all the clip frames - std::vector new_prev_to_cur_transform; - /// apply the smoothed transformation warp when retrieving a frame - bool hasStabilization = false; - void apply_stabilization(std::shared_ptr f, int64_t frame_number); - #endif + // #ifdef USE_OPENCV + // /// Smoothed transformation for all the clip frames + // std::vector new_prev_to_cur_transform; + // /// apply the smoothed transformation warp when retrieving a frame + // bool hasStabilization = false; + // void apply_stabilization(std::shared_ptr f, int64_t frame_number); + // #endif /// Default Constructor Clip(); diff --git a/include/ClipProcessingJobs.h b/include/ClipProcessingJobs.h index a0b323f2..0a833efc 100644 --- a/include/ClipProcessingJobs.h +++ b/include/ClipProcessingJobs.h @@ -43,6 +43,7 @@ #include "Clip.h" #include "effects/Tracker.h" +#include "effects/Stabilizer.h" using namespace openshot; diff --git a/include/effects/Stabilizer.h b/include/effects/Stabilizer.h new file mode 100644 index 00000000..0bb4d31a --- /dev/null +++ b/include/effects/Stabilizer.h @@ -0,0 +1,112 @@ +/** + * @file + * @brief Header file for Stabilizer effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_STABILIZER_EFFECT_H +#define OPENSHOT_STABILIZER_EFFECT_H + +#include "../EffectBase.h" + +#include +#include +#include +#include "../Color.h" +#include "../Json.h" +#include "../KeyFrame.h" +#include "../CVStabilization.h" +#include "../Clip.h" +#include "../stabilizedata.pb.h" + + +namespace openshot +{ + //TODO: fix this + /** + * @brief This class draws black bars around your video (from any side), and can be animated with + * openshot::Keyframe curves over time. + * + * Adding bars around your video can be done for cinematic reasons, and creates a fun way to frame + * in the focal point of a scene. The bars can be any color, and each side can be animated independently. + */ + class Stabilizer : public EffectBase + { + private: + /// Init effect settings + void init_effect_details(); + + + public: + Color color; ///< Color of bars + Keyframe left; ///< Size of left bar + Keyframe top; ///< Size of top bar + Keyframe right; ///< Size of right bar + Keyframe bottom; ///< Size of bottom bar + + std::vector trajectoryData; // Save camera trajectory data + std::vector transformationData; // Save transormation data + + /// Blank constructor, useful when using Json to load the effect properties + Stabilizer(std::string clipTrackerDataPath); + + /// Default constructor, which takes 4 curves and a color. These curves animated the bars over time. + /// + /// @param color The curve to adjust the color of bars + /// @param left The curve to adjust the left bar size (between 0 and 1) + /// @param top The curve to adjust the top bar size (between 0 and 1) + /// @param right The curve to adjust the right bar size (between 0 and 1) + /// @param bottom The curve to adjust the bottom bar size (between 0 and 1) + Stabilizer(Color color, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom); + + /// @brief This method is required for all derived classes of EffectBase, and returns a + /// modified openshot::Frame object + /// + /// The frame object is passed into this method, and a frame_number is passed in which + /// tells the effect which settings to use from its keyframes (starting at 1). + /// + /// @returns The modified openshot::Frame object + /// @param frame The frame object that needs the effect applied to it + /// @param frame_number The frame number (starting at 1) of the effect on the timeline. + std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + + bool LoadStabilizedData(std::string inputFilePath); + + /// Get and Set JSON methods + std::string Json() const override; ///< Generate JSON string of this object + void SetJson(const std::string value) override; ///< Load JSON string into this object + Json::Value JsonValue() const override; ///< Generate Json::Value for this object + void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object + + /// Get all properties for a specific frame (perfect for a UI to display the current state + /// of all properties at any time) + std::string PropertiesJSON(int64_t requested_frame) const override; + }; + +} + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6f87d058..1ad4eae2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -195,7 +195,8 @@ set(OPENSHOT_CV_SOURCES # Compiled Protobuf messages set(PROTOBUF_MESSAGES - trackerdata.pb.cc) + trackerdata.pb.cc + stabilizedata.pb.cc) # Video effects set(EFFECTS_SOURCES @@ -213,7 +214,8 @@ set(EFFECTS_SOURCES effects/Saturation.cpp effects/Shift.cpp effects/Wave.cpp - effects/Tracker.cpp) + effects/Tracker.cpp + effects/Stabilizer.cpp) # Qt video player components set(QT_PLAYER_SOURCES diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp index 32cb7f1a..32e05c85 100644 --- a/src/CVStabilization.cpp +++ b/src/CVStabilization.cpp @@ -5,37 +5,27 @@ CVStabilization::CVStabilization():smoothingWindow(30) {} CVStabilization::CVStabilization(int _smoothingWindow): smoothingWindow(_smoothingWindow){} -// void CVStabilization::ProcessVideo(openshot::Clip &video){ -// // Make sure Clip is opened -// video.Open(); -// // Get total number of frames -// int videoLenght = video.Reader()->info.video_length; +void CVStabilization::ProcessClip(openshot::Clip &video){ + // Get total number of frames + int videoLenght = video.Reader()->info.video_length; -// // Get first Opencv image -// std::shared_ptr f = video.GetFrame(0); -// cv::Mat prev = f->GetImageCV(); -// // OpticalFlow works with grayscale images -// cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); - -// // Extract and track opticalflow features for each frame -// for (long int frame_number = 1; frame_number <= videoLenght; frame_number++) -// { -// std::shared_ptr f = video.GetFrame(frame_number); + // Extract and track opticalflow features for each frame + for (long int frame_number = 0; frame_number <= videoLenght; frame_number++) + { + std::shared_ptr f = video.GetFrame(frame_number); -// // Grab Mat image -// cv::Mat cvimage = f->GetImageCV(); -// cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); -// TrackFrameFeatures(cvimage, frame_number); -// } + // Grab Mat image + cv::Mat cvimage = f->GetImageCV(); + cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); + TrackFrameFeatures(cvimage, frame_number); + } -// vector trajectory = ComputeFramesTrajectory(); + vector trajectory = ComputeFramesTrajectory(); -// vector smoothed_trajectory = SmoothTrajectory(trajectory); + trajectoryData = SmoothTrajectory(trajectory); -// vector new_prev_to_cur_transform = GenNewCamPosition(smoothed_trajectory); - -// ApplyNewTrajectoryToClip(video, new_prev_to_cur_transform); -// } + transformationData = GenNewCamPosition(trajectoryData); +} // Track current frame features and find the relative transformation void CVStabilization::TrackFrameFeatures(cv::Mat frame, int frameNum){ @@ -169,9 +159,90 @@ vector CVStabilization::GenNewCamPosition(vector &new_prev_to_cur_transform){ - -// video.new_prev_to_cur_transform = new_prev_to_cur_transform; -// video.hasStabilization = true; -// } \ No newline at end of file +// Save protobuf file +bool CVStabilization::SaveStabilizedData(std::string outputFilePath){ + // Create tracker message + libopenshotstabilize::Stabilization stabilizationMessage; + + // Add all frames data + std::vector::iterator trajData = trajectoryData.begin(); + std::vector::iterator transData = transformationData.begin(); + + for(long int frame_count = 0; trajData != trajectoryData.end(); ++trajData, ++transData){ + AddFrameDataToProto(stabilizationMessage.add_frame(), *trajData, *transData, frame_count); + } + // Add timestamp + *stabilizationMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL)); + + // Write the new message to disk. + std::fstream output(outputFilePath, ios::out | ios::trunc | ios::binary); + if (!stabilizationMessage.SerializeToOstream(&output)) { + cerr << "Failed to write protobuf message." << endl; + return false; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + +void CVStabilization::AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, long int frame_number){ + + // Save frame number + pbFrameData->set_id(frame_number); + + // Save camera trajectory data + pbFrameData->set_a(trajData.a); + pbFrameData->set_x(trajData.x); + pbFrameData->set_y(trajData.y); + + // Save transformation data + pbFrameData->set_da(transData.da); + pbFrameData->set_dx(transData.dx); + pbFrameData->set_dy(transData.dy); +} + +// Load protobuf file +bool CVStabilization::LoadStabilizedData(std::string inputFilePath){ + libopenshotstabilize::Stabilization stabilizationMessage; + + // Read the existing tracker message. + fstream input(inputFilePath, ios::in | ios::binary); + if (!stabilizationMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + + // Make sure the data vectors are empty + transformationData.clear(); + trajectoryData.clear(); + + // Iterate over all frames of the saved message + for (int i = 0; i < stabilizationMessage.frame_size(); i++) { + const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i); + + int id = pbFrameData.id(); + + float x = pbFrameData.x(); + float y = pbFrameData.y(); + float a = pbFrameData.a(); + + trajectoryData.push_back(CamTrajectory(x,y,a)); + + float dx = pbFrameData.dx(); + float dy = pbFrameData.dy(); + float da = pbFrameData.da(); + + transformationData.push_back(TransformParam(dx,dy,da)); + } + + if (stabilizationMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} \ No newline at end of file diff --git a/src/CVTracker.cpp b/src/CVTracker.cpp index 42f3bd7e..4ba3598f 100644 --- a/src/CVTracker.cpp +++ b/src/CVTracker.cpp @@ -7,6 +7,7 @@ CVTracker::CVTracker(){ tracker = select_tracker(trackerType); } +// Set desirable tracker method cv::Ptr CVTracker::select_tracker(std::string trackerType){ cv::Ptr t; #if (CV_MINOR_VERSION < 3) @@ -44,19 +45,26 @@ void CVTracker::trackClip(openshot::Clip& video){ bool trackerInit = false; int videoLenght = video.Reader()->info.video_length; + + // Loop through video for (long int frame = 0; frame < videoLenght; frame++) { std::cout<<"frame: "< f = video.GetFrame(frame_number); // Grab Mat image cv::Mat cvimage = f->GetImageCV(); + // Pass the first frame to initialize the tracker if(!trackerInit){ + // Select the object to be tracked cv::Rect2d bbox = cv::selectROI("Display Image", cvimage); - + + // Initialize the tracker initTracker(bbox, cvimage, frame_number); + // Draw in the frame the box containing the object cv::rectangle(cvimage, bbox, cv::Scalar( 255, 0, 0 ), 2, 1 ); trackerInit = true; @@ -159,7 +167,6 @@ void CVTracker::AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, Fram box->set_y1(fData.y1); box->set_x2(fData.x2); box->set_y2(fData.y2); - } bool CVTracker::LoadTrackedData(std::string inputFilePath){ @@ -204,8 +211,10 @@ bool CVTracker::LoadTrackedData(std::string inputFilePath){ return true; } +// Get tracker info for the desired frame FrameData CVTracker::GetTrackedData(int frameId){ + // Check if the tracker info for the requested frame exists if ( trackedDataById.find(frameId) == trackedDataById.end() ) { return FrameData(); diff --git a/src/Clip.cpp b/src/Clip.cpp index e621b56d..09c3586e 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -360,11 +360,11 @@ std::shared_ptr Clip::GetFrame(int64_t requested_frame) // Apply effects to the frame (if any) apply_effects(frame); -#ifdef USE_OPENCV - if(hasStabilization){ - apply_stabilization(frame, requested_frame); - } -#endif +// #ifdef USE_OPENCV +// if(hasStabilization){ +// apply_stabilization(frame, requested_frame); +// } +// #endif // Return processed 'frame' return frame; @@ -374,32 +374,32 @@ std::shared_ptr Clip::GetFrame(int64_t requested_frame) throw ReaderClosed("No Reader has been initialized for this Clip. Call Reader(*reader) before calling this method."); } -#ifdef USE_OPENCV -void Clip::apply_stabilization(std::shared_ptr f, int64_t frame_number){ - cv::Mat T(2,3,CV_64F); +// #ifdef USE_OPENCV +// void Clip::apply_stabilization(std::shared_ptr f, int64_t frame_number){ +// cv::Mat T(2,3,CV_64F); - // Grab Mat image - cv::Mat cur = f->GetImageCV(); +// // Grab Mat image +// cv::Mat cur = f->GetImageCV(); - T.at(0,0) = cos(new_prev_to_cur_transform[frame_number].da); - T.at(0,1) = -sin(new_prev_to_cur_transform[frame_number].da); - T.at(1,0) = sin(new_prev_to_cur_transform[frame_number].da); - T.at(1,1) = cos(new_prev_to_cur_transform[frame_number].da); +// T.at(0,0) = cos(new_prev_to_cur_transform[frame_number].da); +// T.at(0,1) = -sin(new_prev_to_cur_transform[frame_number].da); +// T.at(1,0) = sin(new_prev_to_cur_transform[frame_number].da); +// T.at(1,1) = cos(new_prev_to_cur_transform[frame_number].da); - T.at(0,2) = new_prev_to_cur_transform[frame_number].dx; - T.at(1,2) = new_prev_to_cur_transform[frame_number].dy; +// T.at(0,2) = new_prev_to_cur_transform[frame_number].dx; +// T.at(1,2) = new_prev_to_cur_transform[frame_number].dy; - cv::Mat frame_stabilized; +// cv::Mat frame_stabilized; - cv::warpAffine(cur, frame_stabilized, T, cur.size()); +// cv::warpAffine(cur, frame_stabilized, T, cur.size()); - // Scale up the image to remove black borders - cv::Mat T_scale = cv::getRotationMatrix2D(cv::Point2f(frame_stabilized.cols/2, frame_stabilized.rows/2), 0, 1.04); - cv::warpAffine(frame_stabilized, frame_stabilized, T_scale, frame_stabilized.size()); +// // Scale up the image to remove black borders +// cv::Mat T_scale = cv::getRotationMatrix2D(cv::Point2f(frame_stabilized.cols/2, frame_stabilized.rows/2), 0, 1.04); +// cv::warpAffine(frame_stabilized, frame_stabilized, T_scale, frame_stabilized.size()); - f->SetImageCV(frame_stabilized); -} -#endif +// f->SetImageCV(frame_stabilized); +// } +// #endif // Get file extension std::string Clip::get_file_extension(std::string path) @@ -1076,44 +1076,44 @@ std::shared_ptr Clip::apply_effects(std::shared_ptr frame) return frame; } -#ifdef USE_OPENCV -void Clip::stabilize_video(){ - // create CVStabilization object - CVStabilization stabilizer; +// #ifdef USE_OPENCV +// void Clip::stabilize_video(){ +// // create CVStabilization object +// CVStabilization stabilizer; - // Make sure Clip is opened - Open(); - // Get total number of frames - int videoLenght = Reader()->info.video_length; +// // Make sure Clip is opened +// Open(); +// // Get total number of frames +// int videoLenght = Reader()->info.video_length; - // Get first Opencv image - // std::shared_ptr f = GetFrame(0); - // cv::Mat prev = f->GetImageCV(); - // // OpticalFlow works with grayscale images - // cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); +// // Get first Opencv image +// // std::shared_ptr f = GetFrame(0); +// // cv::Mat prev = f->GetImageCV(); +// // // OpticalFlow works with grayscale images +// // cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); - // Extract and track opticalflow features for each frame - for (long int frame_number = 0; frame_number <= videoLenght; frame_number++) - { - std::shared_ptr f = GetFrame(frame_number); +// // Extract and track opticalflow features for each frame +// for (long int frame_number = 0; frame_number <= videoLenght; frame_number++) +// { +// std::shared_ptr f = GetFrame(frame_number); - // Grab Mat image - cv::Mat cvimage = f->GetImageCV(); - cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); - stabilizer.TrackFrameFeatures(cvimage, frame_number); - } +// // Grab Mat image +// cv::Mat cvimage = f->GetImageCV(); +// cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); +// stabilizer.TrackFrameFeatures(cvimage, frame_number); +// } - vector trajectory = stabilizer.ComputeFramesTrajectory(); +// vector trajectory = stabilizer.ComputeFramesTrajectory(); - vector smoothed_trajectory = stabilizer.SmoothTrajectory(trajectory); +// vector smoothed_trajectory = stabilizer.SmoothTrajectory(trajectory); - // Get the smoothed trajectory - new_prev_to_cur_transform = stabilizer.GenNewCamPosition(smoothed_trajectory); - // Will apply the smoothed transformation warp when retrieving a frame - hasStabilization = true; -} -#else -void Clip::stabilize_video(){ - throw "Please compile libopenshot with OpenCV to use this feature"; -} -#endif +// // Get the smoothed trajectory +// new_prev_to_cur_transform = stabilizer.GenNewCamPosition(smoothed_trajectory); +// // Will apply the smoothed transformation warp when retrieving a frame +// hasStabilization = true; +// } +// #else +// void Clip::stabilize_video(){ +// throw "Please compile libopenshot with OpenCV to use this feature"; +// } +// #endif diff --git a/src/ClipProcessingJobs.cpp b/src/ClipProcessingJobs.cpp index d2144648..8d588b5b 100644 --- a/src/ClipProcessingJobs.cpp +++ b/src/ClipProcessingJobs.cpp @@ -1,15 +1,6 @@ #include "../include/ClipProcessingJobs.h" -// get the clip to add a preprocessing job - -// run the preprocessing job on the clip - -// create a new effect with the processed result - -// modify the clip to include the correspondent processed effect - - ClipProcessingJobs::ClipProcessingJobs(std::string processingType, Clip& videoClip){ if(processingType == "Stabilize"){ @@ -20,7 +11,6 @@ ClipProcessingJobs::ClipProcessingJobs(std::string processingType, Clip& videoCl } - void ClipProcessingJobs::trackVideo(Clip& videoClip){ // Opencv display window @@ -34,40 +24,18 @@ void ClipProcessingJobs::trackVideo(Clip& videoClip){ // Create new Tracker Effect EffectBase* trackerEffect = new Tracker("kcf_tracker.data"); videoClip.AddEffect(trackerEffect); - - - // - } - -void ClipProcessingJobs::stabilizeVideo(Clip& video){ +void ClipProcessingJobs::stabilizeVideo(Clip& videoClip){ // create CVStabilization object CVStabilization stabilizer; + stabilizer.ProcessClip(videoClip); - // Get total number of frames - int videoLenght = video.Reader()->info.video_length; + stabilizer.SaveStabilizedData("stabilization.data"); - // Extract and track opticalflow features for each frame - for (long int frame_number = 0; frame_number <= videoLenght; frame_number++) - { - std::shared_ptr f = video.GetFrame(frame_number); - - // Grab Mat image - cv::Mat cvimage = f->GetImageCV(); - cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); - stabilizer.TrackFrameFeatures(cvimage, frame_number); - } + // Create new Tracker Effect + EffectBase* stabilizeEffect = new Stabilizer("stabilization.data"); + videoClip.AddEffect(stabilizeEffect); - vector trajectory = stabilizer.ComputeFramesTrajectory(); - - vector smoothed_trajectory = stabilizer.SmoothTrajectory(trajectory); - - // Get the smoothed trajectory - std::vector new_prev_to_cur_transform = stabilizer.GenNewCamPosition(smoothed_trajectory); - - // Will apply the smoothed transformation warp when retrieving a frame - video.hasStabilization = true; - video.new_prev_to_cur_transform = new_prev_to_cur_transform; } \ No newline at end of file diff --git a/src/effects/Stabilizer.cpp b/src/effects/Stabilizer.cpp new file mode 100644 index 00000000..6015c0eb --- /dev/null +++ b/src/effects/Stabilizer.cpp @@ -0,0 +1,221 @@ +/** + * @file + * @brief Source file for Stabilizer effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "../../include/effects/Stabilizer.h" + +using namespace openshot; + +/// Blank constructor, useful when using Json to load the effect properties +Stabilizer::Stabilizer(std::string clipStabilizedDataPath) +{ + // Init effect properties + init_effect_details(); + + // Tries to load the stabilization data from protobuf + LoadStabilizedData(clipStabilizedDataPath); +} + +// Default constructor +Stabilizer::Stabilizer(Color color, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom) : + color(color), left(left), top(top), right(right), bottom(bottom) +{ + // Init effect properties + init_effect_details(); +} + +// Init effect settings +void Stabilizer::init_effect_details() +{ + /// Initialize the values of the EffectInfo struct. + InitEffectInfo(); + + /// Set the effect info + info.class_name = "Stabilizer"; + info.name = "Stabilizer"; + info.description = "Stabilize video clip to remove undesired shaking and jitter."; + info.has_audio = false; + info.has_video = true; +} + +// This method is required for all derived classes of EffectBase, and returns a +// modified openshot::Frame object +std::shared_ptr Stabilizer::GetFrame(std::shared_ptr frame, int64_t frame_number) +{ + cv::Mat T(2,3,CV_64F); + // Grab Mat image + cv::Mat cur = frame->GetImageCV(); + T.at(0,0) = cos(transformationData[frame_number].da); + T.at(0,1) = -sin(transformationData[frame_number].da); + T.at(1,0) = sin(transformationData[frame_number].da); + T.at(1,1) = cos(transformationData[frame_number].da); + + T.at(0,2) = transformationData[frame_number].dx; + T.at(1,2) = transformationData[frame_number].dy; + cv::Mat frame_stabilized; + + cv::warpAffine(cur, frame_stabilized, T, cur.size()); + // Scale up the image to remove black borders + cv::Mat T_scale = cv::getRotationMatrix2D(cv::Point2f(frame_stabilized.cols/2, frame_stabilized.rows/2), 0, 1.04); + cv::warpAffine(frame_stabilized, frame_stabilized, T_scale, frame_stabilized.size()); + frame->SetImageCV(frame_stabilized); + + return frame; +} + +// Load protobuf file +bool Stabilizer::LoadStabilizedData(std::string inputFilePath){ + libopenshotstabilize::Stabilization stabilizationMessage; + + // Read the existing tracker message. + fstream input(inputFilePath, ios::in | ios::binary); + if (!stabilizationMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + + // Make sure the data vectors are empty + transformationData.clear(); + trajectoryData.clear(); + + // Iterate over all frames of the saved message + for (int i = 0; i < stabilizationMessage.frame_size(); i++) { + const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i); + + int id = pbFrameData.id(); + + float x = pbFrameData.x(); + float y = pbFrameData.y(); + float a = pbFrameData.a(); + + trajectoryData.push_back(CamTrajectory(x,y,a)); + + float dx = pbFrameData.dx(); + float dy = pbFrameData.dy(); + float da = pbFrameData.da(); + + transformationData.push_back(TransformParam(dx,dy,da)); + } + + if (stabilizationMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + + + +// Generate JSON string of this object +std::string Stabilizer::Json() const { + + // Return formatted string + return JsonValue().toStyledString(); +} + +// Generate Json::Value for this object +Json::Value Stabilizer::JsonValue() const { + + // Create root json object + Json::Value root = EffectBase::JsonValue(); // get parent properties + root["type"] = info.class_name; + root["color"] = color.JsonValue(); + root["left"] = left.JsonValue(); + root["top"] = top.JsonValue(); + root["right"] = right.JsonValue(); + root["bottom"] = bottom.JsonValue(); + + // return JsonValue + return root; +} + +// Load JSON string into this object +void Stabilizer::SetJson(const std::string value) { + + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +// Load Json::Value into this object +void Stabilizer::SetJsonValue(const Json::Value root) { + + // Set parent data + EffectBase::SetJsonValue(root); + + // Set data from Json (if key is found) + if (!root["color"].isNull()) + color.SetJsonValue(root["color"]); + if (!root["left"].isNull()) + left.SetJsonValue(root["left"]); + if (!root["top"].isNull()) + top.SetJsonValue(root["top"]); + if (!root["right"].isNull()) + right.SetJsonValue(root["right"]); + if (!root["bottom"].isNull()) + bottom.SetJsonValue(root["bottom"]); +} + +// Get all properties for a specific frame +std::string Stabilizer::PropertiesJSON(int64_t requested_frame) const { + + // Generate JSON properties list + Json::Value root; + root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame); + root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame); + root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); + + // Keyframes + root["color"] = add_property_json("Bar Color", 0.0, "color", "", NULL, 0, 255, false, requested_frame); + root["color"]["red"] = add_property_json("Red", color.red.GetValue(requested_frame), "float", "", &color.red, 0, 255, false, requested_frame); + root["color"]["blue"] = add_property_json("Blue", color.blue.GetValue(requested_frame), "float", "", &color.blue, 0, 255, false, requested_frame); + root["color"]["green"] = add_property_json("Green", color.green.GetValue(requested_frame), "float", "", &color.green, 0, 255, false, requested_frame); + root["left"] = add_property_json("Left Size", left.GetValue(requested_frame), "float", "", &left, 0.0, 0.5, false, requested_frame); + root["top"] = add_property_json("Top Size", top.GetValue(requested_frame), "float", "", &top, 0.0, 0.5, false, requested_frame); + root["right"] = add_property_json("Right Size", right.GetValue(requested_frame), "float", "", &right, 0.0, 0.5, false, requested_frame); + root["bottom"] = add_property_json("Bottom Size", bottom.GetValue(requested_frame), "float", "", &bottom, 0.0, 0.5, false, requested_frame); + + // Return formatted string + return root.toStyledString(); +} diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index d38c71db..87035ed7 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -40,8 +40,6 @@ Tracker::Tracker(std::string clipTrackerDataPath) // Tries to load the tracker data from protobuf LoadTrackedData(clipTrackerDataPath); - std::cout<<"passou0\n"; - } // Default constructor @@ -71,19 +69,13 @@ void Tracker::init_effect_details() // modified openshot::Frame object std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t frame_number) { - std::cout<<"passou1\n"; // Get the frame's image cv::Mat frame_image = frame->GetImageCV(); - std::cout<<"passou2\n"; // Draw box on image FrameData fd = trackedDataById[frame_number]; - std::cout<<"passou3\n"; cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); - std::cout<<"passou4\n"; cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); - std::cout<<"passou5\n"; frame->SetImageCV(frame_image); - std::cout<<"passou6\n"; return frame; } @@ -132,11 +124,10 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ FrameData Tracker::GetTrackedData(int frameId){ + // Check if the tracker info for the requested frame exists if ( trackedDataById.find(frameId) == trackedDataById.end() ) { - return FrameData(); } else { - return trackedDataById[frameId]; } diff --git a/src/examples/Example_opencv.cpp b/src/examples/Example_opencv.cpp index 54333be4..4941b7e7 100644 --- a/src/examples/Example_opencv.cpp +++ b/src/examples/Example_opencv.cpp @@ -153,10 +153,8 @@ void displayStabilization(openshot::Clip &r9){ { int frame_number = frame; std::shared_ptr f = r9.GetFrame(frame_number); - // Grab Mat image cv::Mat cvimage = f->GetImageCV(); - cv::imshow("Display Image", cvimage); // Press ESC on keyboard to exit char c=(char)cv::waitKey(25); @@ -187,7 +185,7 @@ int main(int argc, char* argv[]) { ClipProcessingJobs clipProcessing("Stabilize", r9); // CVStabilization stabilization; // r9.stabilize_video(); - // displayStabilization(r9); + displayStabilization(r9); }