diff --git a/include/CVStabilization.h b/include/CVStabilization.h new file mode 100644 index 00000000..4756e336 --- /dev/null +++ b/include/CVStabilization.h @@ -0,0 +1,67 @@ +#define int64 opencv_broken_int +#define uint64 opencv_broken_uint +#include +#include +#undef uint64 +#undef int64 +#include +#include "Clip.h" + +using namespace std; + +// struct TransformParam +// { +// TransformParam() {} +// TransformParam(double _dx, double _dy, double _da) { +// dx = _dx; +// dy = _dy; +// da = _da; +// } + +// double dx; +// double dy; +// double da; // angle +// }; + +struct CamTrajectory +{ + CamTrajectory() {} + CamTrajectory(double _x, double _y, double _a) { + x = _x; + y = _y; + a = _a; + } + + double x; + double y; + double a; // angle +}; + +class CVStabilization { + private: + cv::Mat last_T; + cv::Mat cur, cur_grey; + cv::Mat prev, prev_grey; + + public: + const int SMOOTHING_RADIUS = 30; // In frames. The larger the more stable the video, but less reactive to sudden panning + const int HORIZONTAL_BORDER_CROP = 20; // In pixels. Crops the border to reduce the black borders from stabilisation being too noticeable. + std::vector prev_to_cur_transform; // previous to current + + CVStabilization(); + + void ProcessVideo(openshot::Clip &video); + + // Track current frame features and find the relative transformation + void TrackFrameFeatures(cv::Mat frame, int frameNum); + + std::vector ComputeFramesTrajectory(); + std::vector SmoothTrajectory(std::vector &trajectory); + + // Generate new transformations parameters for each frame to follow the smoothed trajectory + std::vector GenNewCamPosition(std::vector &smoothed_trajectory); + + // Send smoothed camera transformation to be applyed on clip + void ApplyNewTrajectoryToClip(openshot::Clip &video, std::vector &new_prev_to_cur_transform); + +}; \ No newline at end of file diff --git a/include/CVTracker.h b/include/CVTracker.h index c00e5369..d971c2a9 100644 --- a/include/CVTracker.h +++ b/include/CVTracker.h @@ -12,7 +12,7 @@ #include "trackerdata.pb.h" -using namespace cv; +// using namespace cv; using namespace std; using google::protobuf::util::TimeUtil; @@ -48,13 +48,13 @@ class CVTracker { std::map trackedDataById; std::string trackerType; - Ptr tracker; - Rect2d bbox; + cv::Ptr tracker; + cv::Rect2d bbox; CVTracker(); - Ptr select_tracker(std::string trackerType); - bool initTracker(Rect2d bbox, Mat &frame, int frameId); - bool trackFrame(Mat &frame, int frameId); + cv::Ptr select_tracker(std::string trackerType); + bool initTracker(cv::Rect2d bbox, cv::Mat &frame, int frameId); + bool trackFrame(cv::Mat &frame, int frameId); // Save protobuf file bool SaveTrackedData(std::string outputFilePath); diff --git a/include/Clip.h b/include/Clip.h index 0fbed159..9eeeacbe 100644 --- a/include/Clip.h +++ b/include/Clip.h @@ -31,6 +31,13 @@ #ifndef OPENSHOT_CLIP_H #define OPENSHOT_CLIP_H +#define int64 opencv_broken_int +#define uint64 opencv_broken_uint +#include +#include +#undef uint64 +#undef int64 + #include #include #include @@ -46,6 +53,23 @@ #include "ReaderBase.h" #include "JuceHeader.h" + +// TODO: move to stabilization effect +struct TransformParam +{ + TransformParam() {} + TransformParam(double _dx, double _dy, double _da) { + dx = _dx; + dy = _dy; + da = _da; + } + + double dx; + double dy; + double da; // angle +}; + + namespace openshot { /// Comparison method for sorting effect pointers (by Position, Layer, and Order). Effects are sorted @@ -59,6 +83,7 @@ namespace openshot { return false; }}; + /** * @brief This class represents a clip (used to arrange readers on the timeline) * @@ -145,6 +170,13 @@ namespace openshot { openshot::FrameDisplayType display; ///< The format to display the frame number (if any) openshot::VolumeMixType mixing; ///< What strategy should be followed when mixing audio with other clips + + /// TODO: [OS-17] move it to stabilize effect + std::vector new_prev_to_cur_transform; + bool hasStabilization = false; + void apply_stabilization(std::shared_ptr f, int64_t frame_number); + + /// Default Constructor Clip(); diff --git a/include/FFmpegReader.h b/include/FFmpegReader.h index ec082965..63359326 100644 --- a/include/FFmpegReader.h +++ b/include/FFmpegReader.h @@ -271,6 +271,9 @@ namespace openshot { /// Open File - which is called by the constructor automatically void Open() override; + + /// Return true if frame can be read with GetFrame() + bool GetIsDurationKnown(); }; } diff --git a/include/Frame.h b/include/Frame.h index 9d2e881b..2ac62016 100644 --- a/include/Frame.h +++ b/include/Frame.h @@ -113,8 +113,8 @@ namespace openshot class Frame { private: - std::shared_ptr image; - cv::Mat imagecv; + std::shared_ptr image; ///< RGBA Format + cv::Mat imagecv; ///< OpenCV image. It will be always on BGR format std::shared_ptr wave_image; std::shared_ptr audio; std::shared_ptr previewApp; @@ -237,6 +237,9 @@ namespace openshot /// Get pointer to OpenCV Mat image object cv::Mat GetImageCV(); + // Set pointer to OpenCV image object + void SetImageCV(cv::Mat _image); + #ifdef USE_IMAGEMAGICK /// Get pointer to ImageMagick image object std::shared_ptr GetMagickImage(); @@ -300,6 +303,7 @@ namespace openshot /// Convert Qimage to Mat cv::Mat Qimage2mat( std::shared_ptr& qimage); + std::shared_ptr Mat2Qimage(cv::Mat img); }; } diff --git a/include/OpenShot.h b/include/OpenShot.h index 5273ff0d..40865e6b 100644 --- a/include/OpenShot.h +++ b/include/OpenShot.h @@ -140,5 +140,7 @@ #include "QtTextReader.h" #include "Timeline.h" #include "Settings.h" +#include "CVStabilization.h" +#include "CVTracker.h" #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c9c075e7..6cac7fc5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -173,7 +173,8 @@ set(OPENSHOT_SOURCES QtTextReader.cpp Settings.cpp Timeline.cpp - CVTracker.cpp) + CVTracker.cpp + CVStabilization.cpp) # Compiled Protobuf messages set(PROTOBUF_MESSAGES diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp new file mode 100644 index 00000000..517996d0 --- /dev/null +++ b/src/CVStabilization.cpp @@ -0,0 +1,168 @@ +#include "../include/CVStabilization.h" + +void CVStabilization::ProcessVideo(openshot::Clip &video){ + // Make sure Clip is opened + video.Open(); + // Get total number of frames + int videoLenght = video.Reader()->info.video_length; + + // Get first Opencv image + std::shared_ptr f = video.GetFrame(0); + cv::Mat prev = f->GetImageCV(); + // OpticalFlow works with grayscale images + cv::cvtColor(prev, prev_grey, cv::COLOR_BGR2GRAY); + + // Extract and track opticalflow features for each frame + for (long int frame_number = 1; frame_number <= videoLenght; frame_number++) + { + std::shared_ptr f = video.GetFrame(frame_number); + + // Grab Mat image + cv::Mat cvimage = f->GetImageCV(); + cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); + TrackFrameFeatures(cvimage, frame_number); + } + + vector trajectory = ComputeFramesTrajectory(); + + vector smoothed_trajectory = SmoothTrajectory(trajectory); + + vector new_prev_to_cur_transform = GenNewCamPosition(smoothed_trajectory); + + ApplyNewTrajectoryToVideo(video, new_prev_to_cur_transform); +} + +// Track current frame features and find the relative transformation +void CVStabilization::TrackFrameFeatures(cv::Mat frame, int frameNum){ + + // OpticalFlow features vector + vector prev_corner, cur_corner; + vector prev_corner2, cur_corner2; + vector status; + vector err; + + // Extract new image teatures + cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30); + // Track features + cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err); + + // Remove untracked features + for(size_t i=0; i < status.size(); i++) { + if(status[i]) { + prev_corner2.push_back(prev_corner[i]); + cur_corner2.push_back(cur_corner[i]); + } + } + // translation + rotation only + cv::Mat T = estimateRigidTransform(prev_corner2, cur_corner2, false); // false = rigid transform, no scaling/shearing + + // If no transform is found. We'll just use the last known good transform. + if(T.data == NULL) { + last_T.copyTo(T); + } + + T.copyTo(last_T); + // decompose T + double dx = T.at(0,2); + double dy = T.at(1,2); + double da = atan2(T.at(1,0), T.at(0,0)); + + prev_to_cur_transform.push_back(TransformParam(dx, dy, da)); + + // out_transform << frameNum << " " << dx << " " << dy << " " << da << endl; + cur.copyTo(prev); + frame.copyTo(prev_grey); + + cout << "Frame: " << frameNum << " - good optical flow: " << prev_corner2.size() << endl; +} + +vector CVStabilization::ComputeFramesTrajectory(){ + + // Accumulated frame to frame transform + double a = 0; + double x = 0; + double y = 0; + + vector trajectory; // trajectory at all frames + + // Compute global camera trajectory. First frame is the origin + for(size_t i=0; i < prev_to_cur_transform.size(); i++) { + x += prev_to_cur_transform[i].dx; + y += prev_to_cur_transform[i].dy; + a += prev_to_cur_transform[i].da; + + trajectory.push_back(CamTrajectory(x,y,a)); + + // out_trajectory << (i+1) << " " << x << " " << y << " " << a << endl; + } + + return trajectory; +} + +vector CVStabilization::SmoothTrajectory(vector &trajectory){ + + vector smoothed_trajectory; // trajectory at all frames + + for(size_t i=0; i < trajectory.size(); i++) { + double sum_x = 0; + double sum_y = 0; + double sum_a = 0; + int count = 0; + + for(int j=-SMOOTHING_RADIUS; j <= SMOOTHING_RADIUS; j++) { + if(i+j >= 0 && i+j < trajectory.size()) { + sum_x += trajectory[i+j].x; + sum_y += trajectory[i+j].y; + sum_a += trajectory[i+j].a; + + count++; + } + } + + double avg_a = sum_a / count; + double avg_x = sum_x / count; + double avg_y = sum_y / count; + + smoothed_trajectory.push_back(CamTrajectory(avg_x, avg_y, avg_a)); + + // out_smoothed_trajectory << (i+1) << " " << avg_x << " " << avg_y << " " << avg_a << endl; + } + return smoothed_trajectory; +} + +// Generate new transformations parameters for each frame to follow the smoothed trajectory +vector CVStabilization::GenNewCamPosition(vector & smoothed_trajectory){ + vector new_prev_to_cur_transform; + + // Accumulated frame to frame transform + double a = 0; + double x = 0; + double y = 0; + + for(size_t i=0; i < prev_to_cur_transform.size(); i++) { + x += prev_to_cur_transform[i].dx; + y += prev_to_cur_transform[i].dy; + a += prev_to_cur_transform[i].da; + + // target - current + double diff_x = smoothed_trajectory[i].x - x; + double diff_y = smoothed_trajectory[i].y - y; + double diff_a = smoothed_trajectory[i].a - a; + + double dx = prev_to_cur_transform[i].dx + diff_x; + double dy = prev_to_cur_transform[i].dy + diff_y; + double da = prev_to_cur_transform[i].da + diff_a; + + new_prev_to_cur_transform.push_back(TransformParam(dx, dy, da)); + + // out_new_transform << (i+1) << " " << dx << " " << dy << " " << da << endl; + } + return new_prev_to_cur_transform; +} + +// Send smoothed camera transformation to be applyed on clip +void CVStabilization::ApplyNewTrajectoryToClip(openshot::Clip &video, vector &new_prev_to_cur_transform){ + + video.new_prev_to_cur_transform = new_prev_to_cur_transform; + video.hasStabilization = true; +} \ No newline at end of file diff --git a/src/CVTracker.cpp b/src/CVTracker.cpp index fe97fc10..1a84edc2 100644 --- a/src/CVTracker.cpp +++ b/src/CVTracker.cpp @@ -7,37 +7,37 @@ CVTracker::CVTracker(){ tracker = select_tracker(trackerType); } -Ptr CVTracker::select_tracker(std::string trackerType){ - Ptr t; +cv::Ptr CVTracker::select_tracker(std::string trackerType){ + cv::Ptr t; #if (CV_MINOR_VERSION < 3) { - t = Tracker::create(trackerType); + t = cv::Tracker::create(trackerType); } #else { if (trackerType == "BOOSTING") - t = TrackerBoosting::create(); + t = cv::TrackerBoosting::create(); if (trackerType == "MIL") - t = TrackerMIL::create(); + t = cv::TrackerMIL::create(); if (trackerType == "KCF") - t = TrackerKCF::create(); + t = cv::TrackerKCF::create(); if (trackerType == "TLD") - t = TrackerTLD::create(); + t = cv::TrackerTLD::create(); if (trackerType == "MEDIANFLOW") - t = TrackerMedianFlow::create(); + t = cv::TrackerMedianFlow::create(); if (trackerType == "GOTURN") - t = TrackerGOTURN::create(); + t = cv::TrackerGOTURN::create(); if (trackerType == "MOSSE") - t = TrackerMOSSE::create(); + t = cv::TrackerMOSSE::create(); if (trackerType == "CSRT") - t = TrackerCSRT::create(); + t = cv::TrackerCSRT::create(); } #endif return t; } -bool CVTracker::initTracker(Rect2d initial_bbox, Mat &frame, int frameId){ +bool CVTracker::initTracker(cv::Rect2d initial_bbox, cv::Mat &frame, int frameId){ bbox = initial_bbox; // rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 ); @@ -52,7 +52,7 @@ bool CVTracker::initTracker(Rect2d initial_bbox, Mat &frame, int frameId){ return true; } -bool CVTracker::trackFrame(Mat &frame, int frameId){ +bool CVTracker::trackFrame(cv::Mat &frame, int frameId){ // Update the tracking result bool ok = tracker->update(frame, bbox); diff --git a/src/Clip.cpp b/src/Clip.cpp index d9f69440..9c20b4f8 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -360,6 +360,10 @@ std::shared_ptr Clip::GetFrame(int64_t requested_frame) // Apply effects to the frame (if any) apply_effects(frame); + if(hasStabilization){ + apply_stabilization(frame, requested_frame); + } + // Return processed 'frame' return frame; } @@ -368,6 +372,27 @@ std::shared_ptr Clip::GetFrame(int64_t requested_frame) throw ReaderClosed("No Reader has been initialized for this Clip. Call Reader(*reader) before calling this method."); } +void Clip::apply_stabilization(std::shared_ptr f, int64_t frame_number){ + cv::Mat T(2,3,CV_64F); + + // Grab Mat image + cv::Mat cur = f->GetImageCV(); + + T.at(0,0) = cos(new_prev_to_cur_transform[frame_number].da); + T.at(0,1) = -sin(new_prev_to_cur_transform[frame_number].da); + T.at(1,0) = sin(new_prev_to_cur_transform[frame_number].da); + T.at(1,1) = cos(new_prev_to_cur_transform[frame_number].da); + + T.at(0,2) = new_prev_to_cur_transform[frame_number].dx; + T.at(1,2) = new_prev_to_cur_transform[frame_number].dy; + + cv::Mat cur2; + + cv::warpAffine(cur, cur2, T, cur.size()); + + f->SetImageCV(cur2); +} + // Get file extension std::string Clip::get_file_extension(std::string path) { diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index c8ce141f..a8561863 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -813,6 +813,9 @@ void FFmpegReader::UpdateVideoInfo() { } } +bool FFmpegReader::GetIsDurationKnown() { + return this->is_duration_known; +} std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { // Check for open reader (or throw exception) diff --git a/src/Frame.cpp b/src/Frame.cpp index 2fb6d48d..289ab693 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -932,6 +932,7 @@ cv::Mat Frame::Qimage2mat( std::shared_ptr& qimage) { cv::Mat mat2 = cv::Mat(mat.rows, mat.cols, CV_8UC3 ); int from_to[] = { 0,0, 1,1, 2,2 }; cv::mixChannels( &mat, 1, &mat2, 1, from_to, 3 ); + cv::cvtColor(mat2, mat2, cv::COLOR_RGB2BGR); return mat2; }; @@ -950,6 +951,24 @@ cv::Mat Frame::GetImageCV() return imagecv; } +std::shared_ptr Frame::Mat2Qimage(cv::Mat img){ + // cv::cvtColor(img, img, cv::COLOR_BGR2RGB); + std::shared_ptr imgIn = std::shared_ptr(new QImage((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888)); + // Always convert to RGBA8888 (if different) + if (imgIn->format() != QImage::Format_RGBA8888) + *image = imgIn->convertToFormat(QImage::Format_RGBA8888); + + return imgIn; +} + +// Set pointer to OpenCV image object +void Frame::SetImageCV(cv::Mat _image) +{ + imagecv = _image; + image = Mat2Qimage(_image); +} + + #ifdef USE_IMAGEMAGICK // Get pointer to ImageMagick image object std::shared_ptr Frame::GetMagickImage() diff --git a/src/effects/Stabilize.cpp b/src/effects/Stabilize.cpp new file mode 100644 index 00000000..e69de29b diff --git a/src/examples/Example_opencv.cpp b/src/examples/Example_opencv.cpp index e0d012a6..dbb5fdaf 100644 --- a/src/examples/Example_opencv.cpp +++ b/src/examples/Example_opencv.cpp @@ -32,38 +32,36 @@ #include #include // #include -#include "../../include/CVTracker.h" +// #include "../../include/CVTracker.h" +// #include "../../include/CVStabilization.h" // #include "treackerdata.pb.h" #include "../../include/OpenShot.h" #include "../../include/CrashHandler.h" using namespace openshot; -using namespace cv; +// using namespace cv; - - -void trackVideo(openshot::FFmpegReader &r9){ +void trackVideo(openshot::Clip &r9){ // Opencv display window cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); // Create Tracker CVTracker kcfTracker; bool trackerInit = false; - for (long int frame = 1100; frame <= 1500; frame++) + for (long int frame = 1200; frame <= 1600; frame++) { int frame_number = frame; std::shared_ptr f = r9.GetFrame(frame_number); // Grab Mat image cv::Mat cvimage = f->GetImageCV(); - cvtColor(cvimage, cvimage, CV_RGB2BGR); if(!trackerInit){ - Rect2d bbox = selectROI("Display Image", cvimage); + cv::Rect2d bbox = cv::selectROI("Display Image", cvimage); kcfTracker.initTracker(bbox, cvimage, frame_number); - rectangle(cvimage, bbox, Scalar( 255, 0, 0 ), 2, 1 ); + cv::rectangle(cvimage, bbox, cv::Scalar( 255, 0, 0 ), 2, 1 ); trackerInit = true; } @@ -73,13 +71,13 @@ void trackVideo(openshot::FFmpegReader &r9){ // Draw box on image FrameData fd = kcfTracker.GetTrackedData(frame_number); // std::cout<< "fd: "<< fd.x1<< " "<< fd.y1 <<" "< f = r9.GetFrame(frame_number); // Grab Mat image cv::Mat cvimage = f->GetImageCV(); - cvtColor(cvimage, cvimage, CV_RGB2BGR); FrameData fd = kcfTracker.GetTrackedData(frame_number); - Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); - rectangle(cvimage, box, Scalar( 255, 0, 0 ), 2, 1 ); + cv::Rect2d box(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); + cv::rectangle(cvimage, box, cv::Scalar( 255, 0, 0 ), 2, 1 ); cv::imshow("Display Image", cvimage); // Press ESC on keyboard to exit - char c=(char)waitKey(25); + char c=(char)cv::waitKey(25); if(c==27) break; } @@ -125,26 +122,53 @@ void displayTrackedData(openshot::FFmpegReader &r9){ } +void displayStabilization(openshot::Clip &r9){ + // Opencv display window + cv::namedWindow("Display Image", cv::WINDOW_NORMAL ); + + int videoLenght = r9.Reader()->info.video_length; + + for (long int frame = 0; frame < videoLenght; frame++) + { + int frame_number = frame; + std::shared_ptr f = r9.GetFrame(frame_number); + + // Grab Mat image + cv::Mat cvimage = f->GetImageCV(); + + cv::imshow("Display Image1", cvimage); + // Press ESC on keyboard to exit + char c=(char)cv::waitKey(25); + if(c==27) + break; + } + +} int main(int argc, char* argv[]) { + bool TRACK_DATA = false; bool LOAD_TRACKED_DATA = false; + bool SMOOTH_VIDEO = true; - openshot::Settings *s = openshot::Settings::Instance(); - s->HARDWARE_DECODER = 2; // 1 VA-API, 2 NVDEC, 6 VDPAU - s->HW_DE_DEVICE_SET = 0; std::string input_filepath = TEST_MEDIA_PATH; - input_filepath += "Boneyard Memories.mp4"; + input_filepath += "test.avi"; - openshot::FFmpegReader r9(input_filepath); + openshot::Clip r9(input_filepath); r9.Open(); - r9.DisplayInfo(); - if(!LOAD_TRACKED_DATA) + if(TRACK_DATA) trackVideo(r9); - else + if(LOAD_TRACKED_DATA) displayTrackedData(r9); + if(SMOOTH_VIDEO){ + CVStabilization stabilization; + stabilization.ProcessVideo(r9); + displayStabilization(r9); + } + + // Close timeline diff --git a/src/examples/test.avi b/src/examples/test.avi new file mode 100644 index 00000000..4d6b8fc6 Binary files /dev/null and b/src/examples/test.avi differ diff --git a/tests/CVTracker_Tests.cpp b/tests/CVTracker_Tests.cpp index 225eeb5c..5e958071 100644 --- a/tests/CVTracker_Tests.cpp +++ b/tests/CVTracker_Tests.cpp @@ -33,7 +33,7 @@ // Prevent name clashes with juce::UnitTest #define DONT_SET_USING_JUCE_NAMESPACE 1 #include "../include/OpenShot.h" -#include "../include/CVTracker.h" +// #include "../include/CVTracker.h" #include @@ -62,7 +62,6 @@ TEST(Track_Video) // Grab Mat image cv::Mat cvimage = f->GetImageCV(); - cvtColor(cvimage, cvimage, CV_RGB2BGR); if(!trackerInit){ cv::Rect2d bbox(82, 194, 47, 42); @@ -104,7 +103,6 @@ TEST(SaveLoad_Protobuf) // Grab Mat image cv::Mat cvimage = f->GetImageCV(); - cvtColor(cvimage, cvimage, CV_RGB2BGR); if(!trackerInit){ cv::Rect2d bbox(82, 194, 47, 42); @@ -125,7 +123,7 @@ TEST(SaveLoad_Protobuf) CVTracker kcfTracker1; kcfTracker1.LoadTrackedData("kcf_tracker.data"); FrameData fd = kcfTracker.GetTrackedData(97); - Rect2d loadedBox(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); + cv::Rect2d loadedBox(fd.x1, fd.y1, fd.x2-fd.x1, fd.y2-fd.y1); CHECK_EQUAL(loadedBox.x, lastTrackedBox.x); CHECK_EQUAL(loadedBox.y, lastTrackedBox.y);