diff --git a/include/CVStabilization.h b/include/CVStabilization.h index 4756e336..303b240e 100644 --- a/include/CVStabilization.h +++ b/include/CVStabilization.h @@ -1,3 +1,36 @@ +/** + * @file + * @brief Header file for CVStabilization class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_STABILIZATION_H +#define OPENSHOT_STABILIZATION_H + #define int64 opencv_broken_int #define uint64 opencv_broken_uint #include @@ -5,23 +38,22 @@ #undef uint64 #undef int64 #include -#include "Clip.h" using namespace std; -// struct TransformParam -// { -// TransformParam() {} -// TransformParam(double _dx, double _dy, double _da) { -// dx = _dx; -// dy = _dy; -// da = _da; -// } +struct TransformParam +{ + TransformParam() {} + TransformParam(double _dx, double _dy, double _da) { + dx = _dx; + dy = _dy; + da = _da; + } -// double dx; -// double dy; -// double da; // angle -// }; + double dx; + double dy; + double da; // angle +}; struct CamTrajectory { @@ -44,13 +76,14 @@ class CVStabilization { cv::Mat prev, prev_grey; public: - const int SMOOTHING_RADIUS = 30; // In frames. The larger the more stable the video, but less reactive to sudden panning - const int HORIZONTAL_BORDER_CROP = 20; // In pixels. Crops the border to reduce the black borders from stabilisation being too noticeable. + const int smoothingWindow; // In frames. The larger the more stable the video, but less reactive to sudden panning std::vector prev_to_cur_transform; // previous to current CVStabilization(); - void ProcessVideo(openshot::Clip &video); + CVStabilization(int _smoothingWindow); + + // void ProcessVideo(openshot::Clip &video); // Track current frame features and find the relative transformation void TrackFrameFeatures(cv::Mat frame, int frameNum); @@ -62,6 +95,8 @@ class CVStabilization { std::vector GenNewCamPosition(std::vector &smoothed_trajectory); // Send smoothed camera transformation to be applyed on clip - void ApplyNewTrajectoryToClip(openshot::Clip &video, std::vector &new_prev_to_cur_transform); + // void ApplyNewTrajectoryToClip(openshot::Clip &video, std::vector &new_prev_to_cur_transform); -}; \ No newline at end of file +}; + +#endif \ No newline at end of file diff --git a/include/Clip.h b/include/Clip.h index 0fd7852b..35ead2d1 100644 --- a/include/Clip.h +++ b/include/Clip.h @@ -38,6 +38,8 @@ #include #undef uint64 #undef int64 + + #include "CVStabilization.h" #endif #include @@ -56,26 +58,9 @@ #include "JuceHeader.h" - - -// TODO: move to stabilization effect -struct TransformParam -{ - TransformParam() {} - TransformParam(double _dx, double _dy, double _da) { - dx = _dx; - dy = _dy; - da = _da; - } - - double dx; - double dy; - double da; // angle -}; - - namespace openshot { + /// Comparison method for sorting effect pointers (by Position, Layer, and Order). Effects are sorted /// from lowest layer to top layer (since that is sequence clips are combined), and then by /// position, and then by effect order. @@ -140,6 +125,14 @@ namespace openshot { /// (reader member variable itself may have been replaced) openshot::ReaderBase* allocated_reader; + #ifdef USE_OPENCV + /// Smoothed transformation for all the clip frames + std::vector new_prev_to_cur_transform; + /// apply the smoothed transformation warp when retrieving a frame + bool hasStabilization = false; + void apply_stabilization(std::shared_ptr f, int64_t frame_number); + #endif + /// Adjust frame number minimum value int64_t adjust_frame_number_minimum(int64_t frame_number); @@ -167,6 +160,8 @@ namespace openshot { /// Reverse an audio buffer void reverse_buffer(juce::AudioSampleBuffer* buffer); + + public: openshot::GravityType gravity; ///< The gravity of a clip determines where it snaps to its parent openshot::ScaleType scale; ///< The scale determines how a clip should be resized to fit its parent @@ -174,14 +169,6 @@ namespace openshot { openshot::FrameDisplayType display; ///< The format to display the frame number (if any) openshot::VolumeMixType mixing; ///< What strategy should be followed when mixing audio with other clips - - #ifdef USE_OPENCV - std::vector new_prev_to_cur_transform; - bool hasStabilization = false; - void apply_stabilization(std::shared_ptr f, int64_t frame_number); - #endif - - /// Default Constructor Clip(); @@ -244,6 +231,9 @@ namespace openshot { bool Waveform() { return waveform; } ///< Get the waveform property of this clip void Waveform(bool value) { waveform = value; } ///< Set the waveform property of this clip + /// Stabilize the clip using opencv and opticalflow + void stabilize_video(); + // Scale, Location, and Alpha curves openshot::Keyframe scale_x; ///< Curve representing the horizontal scaling in percent (0 to 1) openshot::Keyframe scale_y; ///< Curve representing the vertical scaling in percent (0 to 1) diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp index e160ff2d..32cb7f1a 100644 --- a/src/CVStabilization.cpp +++ b/src/CVStabilization.cpp @@ -1,43 +1,48 @@ #include "../include/CVStabilization.h" -CVStabilization::CVStabilization(){ -} +CVStabilization::CVStabilization():smoothingWindow(30) {} -void CVStabilization::ProcessVideo(openshot::Clip &video){ - // Make sure Clip is opened - video.Open(); - // Get total number of frames - int videoLenght = video.Reader()->info.video_length; +CVStabilization::CVStabilization(int _smoothingWindow): smoothingWindow(_smoothingWindow){} - // Get first Opencv image - std::shared_ptr f = video.GetFrame(0); - cv::Mat prev = f->GetImageCV(); - // OpticalFlow works with grayscale images - cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); +// void CVStabilization::ProcessVideo(openshot::Clip &video){ +// // Make sure Clip is opened +// video.Open(); +// // Get total number of frames +// int videoLenght = video.Reader()->info.video_length; - // Extract and track opticalflow features for each frame - for (long int frame_number = 1; frame_number <= videoLenght; frame_number++) - { - std::shared_ptr f = video.GetFrame(frame_number); +// // Get first Opencv image +// std::shared_ptr f = video.GetFrame(0); +// cv::Mat prev = f->GetImageCV(); +// // OpticalFlow works with grayscale images +// cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); + +// // Extract and track opticalflow features for each frame +// for (long int frame_number = 1; frame_number <= videoLenght; frame_number++) +// { +// std::shared_ptr f = video.GetFrame(frame_number); - // Grab Mat image - cv::Mat cvimage = f->GetImageCV(); - cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); - TrackFrameFeatures(cvimage, frame_number); - } +// // Grab Mat image +// cv::Mat cvimage = f->GetImageCV(); +// cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); +// TrackFrameFeatures(cvimage, frame_number); +// } - vector trajectory = ComputeFramesTrajectory(); +// vector trajectory = ComputeFramesTrajectory(); - vector smoothed_trajectory = SmoothTrajectory(trajectory); +// vector smoothed_trajectory = SmoothTrajectory(trajectory); - vector new_prev_to_cur_transform = GenNewCamPosition(smoothed_trajectory); +// vector new_prev_to_cur_transform = GenNewCamPosition(smoothed_trajectory); - ApplyNewTrajectoryToClip(video, new_prev_to_cur_transform); -} +// ApplyNewTrajectoryToClip(video, new_prev_to_cur_transform); +// } // Track current frame features and find the relative transformation void CVStabilization::TrackFrameFeatures(cv::Mat frame, int frameNum){ + if(prev_grey.empty()){ + prev_grey = frame; + return; + } // OpticalFlow features vector vector prev_corner, cur_corner; @@ -113,7 +118,7 @@ vector CVStabilization::SmoothTrajectory(vector double sum_a = 0; int count = 0; - for(int j=-SMOOTHING_RADIUS; j <= SMOOTHING_RADIUS; j++) { + for(int j=-smoothingWindow; j <= smoothingWindow; j++) { if(i+j >= 0 && i+j < trajectory.size()) { sum_x += trajectory[i+j].x; sum_y += trajectory[i+j].y; @@ -164,9 +169,9 @@ vector CVStabilization::GenNewCamPosition(vector &new_prev_to_cur_transform){ +// // Send smoothed camera transformation to be applyed on clip +// void CVStabilization::ApplyNewTrajectoryToClip(openshot::Clip &video, vector &new_prev_to_cur_transform){ - video.new_prev_to_cur_transform = new_prev_to_cur_transform; - video.hasStabilization = true; -} \ No newline at end of file +// video.new_prev_to_cur_transform = new_prev_to_cur_transform; +// video.hasStabilization = true; +// } \ No newline at end of file diff --git a/src/Clip.cpp b/src/Clip.cpp index 01e25c95..e621b56d 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -1075,3 +1075,45 @@ std::shared_ptr Clip::apply_effects(std::shared_ptr frame) // Return modified frame return frame; } + +#ifdef USE_OPENCV +void Clip::stabilize_video(){ + // create CVStabilization object + CVStabilization stabilizer; + + // Make sure Clip is opened + Open(); + // Get total number of frames + int videoLenght = Reader()->info.video_length; + + // Get first Opencv image + // std::shared_ptr f = GetFrame(0); + // cv::Mat prev = f->GetImageCV(); + // // OpticalFlow works with grayscale images + // cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); + + // Extract and track opticalflow features for each frame + for (long int frame_number = 0; frame_number <= videoLenght; frame_number++) + { + std::shared_ptr f = GetFrame(frame_number); + + // Grab Mat image + cv::Mat cvimage = f->GetImageCV(); + cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); + stabilizer.TrackFrameFeatures(cvimage, frame_number); + } + + vector trajectory = stabilizer.ComputeFramesTrajectory(); + + vector smoothed_trajectory = stabilizer.SmoothTrajectory(trajectory); + + // Get the smoothed trajectory + new_prev_to_cur_transform = stabilizer.GenNewCamPosition(smoothed_trajectory); + // Will apply the smoothed transformation warp when retrieving a frame + hasStabilization = true; +} +#else +void Clip::stabilize_video(){ + throw "Please compile libopenshot with OpenCV to use this feature"; +} +#endif diff --git a/src/examples/Boneyard Memories.mp4 b/src/examples/Boneyard Memories.mp4 deleted file mode 100644 index 9b6e9854..00000000 Binary files a/src/examples/Boneyard Memories.mp4 and /dev/null differ diff --git a/src/examples/Example_opencv.cpp b/src/examples/Example_opencv.cpp index a232d787..c808ef23 100644 --- a/src/examples/Example_opencv.cpp +++ b/src/examples/Example_opencv.cpp @@ -164,7 +164,7 @@ int main(int argc, char* argv[]) { displayTrackedData(r9); if(SMOOTH_VIDEO){ CVStabilization stabilization; - stabilization.ProcessVideo(r9); + r9.stabilize_video(); displayStabilization(r9); }