From 5db1c0f4532eb42a8eb4044ef9e897748257465a Mon Sep 17 00:00:00 2001 From: Brenno Date: Thu, 29 Oct 2020 00:34:31 -0300 Subject: [PATCH] Initial design of new keyframes specialized classes The keyframe collections would better integrate with effects that uses bounding boxes, skeletons and facial points. --- src/CMakeLists.txt | 1 + src/IKeyFrame.h | 90 +++++++++++ src/KeyFrameBBox.cpp | 341 +++++++++++++++++++++++++++++++++++++++ src/KeyFrameBBox.h | 129 +++++++++++++++ src/OpenShot.h | 1 + tests/KeyFrame_Tests.cpp | 40 +++++ 6 files changed, 602 insertions(+) create mode 100644 src/IKeyFrame.h create mode 100644 src/KeyFrameBBox.cpp create mode 100644 src/KeyFrameBBox.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac2c032f..8144127a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -86,6 +86,7 @@ set(OPENSHOT_SOURCES FrameMapper.cpp Json.cpp KeyFrame.cpp + KeyFrameBBox.cpp OpenShotVersion.cpp ZmqLogger.cpp PlayerBase.cpp diff --git a/src/IKeyFrame.h b/src/IKeyFrame.h new file mode 100644 index 00000000..3aef26cd --- /dev/null +++ b/src/IKeyFrame.h @@ -0,0 +1,90 @@ +/** + * @file + * @brief Header file for the IKeyframe class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_IKEYFRAME_H +#define OPENSHOT_IKEYFRAME_H + +#include +#include +#include +#include +#include +#include "Exceptions.h" +#include "Fraction.h" +#include "Coordinate.h" +#include "Point.h" +#include "Json.h" + + +namespace openshot { + /** + * @brief A Keyframe is a collection of Point instances, which is used to vary a number or property over time. + * + * Keyframes are used to animate and interpolate values of properties over time. For example, a single property + * can use a Keyframe instead of a constant value. Assume you want to slide an image (from left to right) over + * a video. You can create a Keyframe which will adjust the X value of the image over 100 frames (or however many + * frames the animation needs to last) from the value of 0 to 640. + * + * \endcode + */ + + class IKeyFrame { + public: + virtual void AddPoint(Point p) = 0; + virtual void AddPoint(double x, double y) = 0; + virtual bool Contains(Point p) const = 0; + virtual std::vector GetValue(int64_t frame_number) = 0; + virtual double GetDelta(int64_t index) const = 0; + virtual int64_t GetLength() const = 0; + virtual int64_t GetCount() const = 0; + + /// Get and Set JSON methods + virtual std::string Json() const = 0; ///< Generate JSON string of this object + virtual Json::Value JsonValue() const = 0; ///< Generate Json::Value for this object + virtual void SetJson(const std::string value) = 0; ///< Load JSON string into this object + virtual void SetJsonValue(const Json::Value root) = 0; ///< Load Json::Value into this object + + /// Remove a single point by matching a value + virtual void RemovePoint(Point p) = 0; + + /// Remove a points by frame_number + virtual void RemovePoint(int64_t frame_number) = 0; + + /// Replace an existing point with a new point + virtual void UpdatePoint(int64_t index, Point p) = 0; + + /// Print collection of points + virtual void PrintPoints() const = 0; + + }; + +} + +#endif \ No newline at end of file diff --git a/src/KeyFrameBBox.cpp b/src/KeyFrameBBox.cpp new file mode 100644 index 00000000..c0d3edcc --- /dev/null +++ b/src/KeyFrameBBox.cpp @@ -0,0 +1,341 @@ +/** + * @file + * @brief Source file for the Keyframe class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "KeyFrameBBox.h" +#include +#include +//#include "Point.h" +//#include + +using namespace std; +using namespace openshot; + +namespace { + bool IsPointBeforeX(Point const & p, double const x) { + return p.co.X < x; + } + + double InterpolateLinearCurve(Point const & left, Point const & right, double const target) { + double const diff_Y = right.co.Y - left.co.Y; + double const diff_X = right.co.X - left.co.X; + double const slope = diff_Y / diff_X; + return left.co.Y + slope * (target - left.co.X); + } + + double InterpolateBezierCurve(Point const & left, Point const & right, double const target, double const allowed_error) { + double const X_diff = right.co.X - left.co.X; + double const Y_diff = right.co.Y - left.co.Y; + Coordinate const p0 = left.co; + Coordinate const p1 = Coordinate(p0.X + left.handle_right.X * X_diff, p0.Y + left.handle_right.Y * Y_diff); + Coordinate const p2 = Coordinate(p0.X + right.handle_left.X * X_diff, p0.Y + right.handle_left.Y * Y_diff); + Coordinate const p3 = right.co; + + double t = 0.5; + double t_step = 0.25; + do { + // Bernstein polynoms + double B[4] = {1, 3, 3, 1}; + double oneMinTExp = 1; + double tExp = 1; + for (int i = 0; i < 4; ++i, tExp *= t) { + B[i] *= tExp; + } + for (int i = 0; i < 4; ++i, oneMinTExp *= 1 - t) { + B[4 - i - 1] *= oneMinTExp; + } + double const x = p0.X * B[0] + p1.X * B[1] + p2.X * B[2] + p3.X * B[3]; + double const y = p0.Y * B[0] + p1.Y * B[1] + p2.Y * B[2] + p3.Y * B[3]; + if (fabs(target - x) < allowed_error) { + return y; + } + if (x > target) { + t -= t_step; + } + else { + t += t_step; + } + t_step /= 2; + } while (true); + } + + + double InterpolateBetween(Point const & left, Point const & right, double target, double allowed_error) { + assert(left.co.X < target); + assert(target <= right.co.X); + switch (right.interpolation) { + case CONSTANT: return left.co.Y; + case LINEAR: return InterpolateLinearCurve(left, right, target); + case BEZIER: return InterpolateBezierCurve(left, right, target, allowed_error); + } + } + + + template + int64_t SearchBetweenPoints(Point const & left, Point const & right, int64_t const current, Check check) { + int64_t start = left.co.X; + int64_t stop = right.co.X; + while (start < stop) { + int64_t const mid = (start + stop + 1) / 2; + double const value = InterpolateBetween(left, right, mid, 0.01); + if (check(round(value), current)) { + start = mid; + } else { + stop = mid - 1; + } + } + return start; + } +} + + +KeyFrameBBox::KeyFrameBBox(): delta_x(0.0), delta_y(0.0), scale_x(0.0), scale_y(0.0) { + + return; +} + +void KeyFrameBBox::AddDisplacement(int64_t frame_num, double _delta_x, double _delta_y){ + if (!this->Contains((int64_t) frame_num)) + return; + + double time = this->FrameNToTime(frame_num); + + if (_delta_x != 0.0) + delta_x.AddPoint(time, _delta_x, openshot::InterpolationType::LINEAR); + if (_delta_y != 0.0) + delta_y.AddPoint(time, _delta_y, openshot::InterpolationType::LINEAR); + + return; +} + +void KeyFrameBBox::AddScale(int64_t frame_num, double _scale_x, double _scale_y){ + if (!this->Contains((double) frame_num)) + return; + + double time = this->FrameNToTime(frame_num); + + if (_scale_x != 0.0) + scale_x.AddPoint(time, _scale_x, openshot::InterpolationType::LINEAR); + if (_scale_y != 0.0) + scale_y.AddPoint(time, _scale_y, openshot::InterpolationType::LINEAR); + + return; +} + +void KeyFrameBBox::AddBox(int64_t _frame_num , float _cx, float _cy, float _width, float _height){ + + if (_frame_num < 0) + return; + + BBox box = BBox(_cx, _cy, _width, _height); + + double time = this->FrameNToTime(_frame_num); + + auto it = BoxVec.find(time); + if (it != BoxVec.end()) + it->second = box; + else + BoxVec.insert({time, box}); +} + +int64_t KeyFrameBBox::GetLength() const{ + + if (BoxVec.empty()) + return 0; + if (BoxVec.size() == 1) + return 1; + + return BoxVec.size(); +} + +bool KeyFrameBBox::Contains(int64_t frame_num) { + + double time = this->FrameNToTime(frame_num); + + auto it = BoxVec.find(time); + if (it != BoxVec.end()) + return true; + + return false; +} + +void KeyFrameBBox::RemovePoint(int64_t frame_number){ + + double time = this->FrameNToTime(frame_number); + + auto it = BoxVec.find(time); + if (it != BoxVec.end()){ + BoxVec.erase(frame_number); + + RemoveDelta(frame_number); + RemoveScale(frame_number); + } + return; +} + +void KeyFrameBBox::RemoveDelta(int64_t frame_number) { + + double attr_x = this->delta_x.GetValue(frame_number); + Point point_x = this->delta_x.GetClosestPoint(Point((double) frame_number, attr_x)); + if (point_x.co.X == (double) frame_number) + this->delta_x.RemovePoint(point_x); + + double attr_y = this->delta_y.GetValue(frame_number); + Point point_y = this->delta_y.GetClosestPoint(Point((double) frame_number, attr_y)); + if (point_y.co.X == (double) frame_number) + this->delta_y.RemovePoint(point_y); + + + return; +} + +void KeyFrameBBox::PrintParams() { + std::cout << "delta_x "; + this->delta_x.PrintPoints(); + + std::cout << "delta_y "; + this->delta_y.PrintPoints(); + + std::cout << "scale_x "; + this->scale_x.PrintPoints(); + + std::cout << "scale_y "; + this->scale_y.PrintPoints(); +} + + +void KeyFrameBBox::RemoveScale(int64_t frame_number) { + + double attr_x = this->scale_x.GetValue(frame_number); + Point point_x = this->scale_x.GetClosestPoint(Point((double) frame_number, attr_x)); + if (point_x.co.X == (double) frame_number) + this->scale_x.RemovePoint(point_x); + + double attr_y = this->scale_y.GetValue(frame_number); + Point point_y = this->scale_y.GetClosestPoint(Point((double) frame_number, attr_y)); + if (point_y.co.X == (double) frame_number) + this->scale_y.RemovePoint(point_y); + + + return; +} + +/*BBox KeyFrameBBox::GetValue(int64_t frame_number){ + + double time = this->FrameNToTime(frame_number); + + auto it = BoxVec.find(time); + if (it != BoxVec.end()){ + BBox res = it->second; + res.cx += this->delta_x.GetValue(time); + res.cy += this->delta_y.GetValue(time); + res.height += this->scale_y.GetValue(time); + res.width += this->scale_x.GetValue(time); + + return res; + } else { + + + } + + BBox val; + + return val; +}*/ +BBox KeyFrameBBox::GetValue(int64_t frame_number){ + double time = this->FrameNToTime(frame_number); + + auto it = BoxVec.lower_bound(time); + if (it->first == time){ + BBox res = it->second; + res.cx += this->delta_x.GetValue(time); + res.cy += this->delta_y.GetValue(time); + res.height += this->scale_y.GetValue(time); + res.width += this->scale_x.GetValue(time); + + return res; + } else { + BBox second_ref = it->second; + //advance(it, -1); + BBox first_ref = prev(it, 1)->second; + + BBox res = InterpolateBoxes(prev(it, 1)->first, it->first, first_ref, second_ref, time); + + res.cx += this->delta_x.GetValue(time); + res.cy += this->delta_y.GetValue(time); + res.height += this->scale_y.GetValue(time); + res.width += this->scale_x.GetValue(time); + + return res; + } + +} + +BBox KeyFrameBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target){ + + Point p1_left(t1, left.cx, openshot::InterpolationType::LINEAR); + Point p1_right(t2, right.cx, openshot::InterpolationType::LINEAR); + + Point p1 = InterpolateBetween(p1_left, p1_right, target, 0.01); + + Point p2_left(t1, left.cy, openshot::InterpolationType::LINEAR); + Point p2_right(t2, right.cy, openshot::InterpolationType::LINEAR); + + Point p2 = InterpolateBetween(p2_left, p2_right, target, 0.01); + + Point p3_left(t1, left.height, openshot::InterpolationType::LINEAR); + Point p3_right(t2, right.height, openshot::InterpolationType::LINEAR); + + Point p3 = InterpolateBetween(p3_left, p3_right, target, 0.01); + + Point p4_left(t1, left.width, openshot::InterpolationType::LINEAR); + Point p4_right(t2, right.width, openshot::InterpolationType::LINEAR); + + Point p4 = InterpolateBetween(p4_left, p4_right, target, 0.01); + + BBox ans(p1.co.Y, p2.co.Y, p3.co.Y, p4.co.Y); + + return ans; +} + + +void KeyFrameBBox::SetFPS(Fraction fps){ + this->fps = fps; + return; +} + +Fraction KeyFrameBBox::GetFPS(){ + return fps; +} + +double KeyFrameBBox::FrameNToTime(int64_t frame_number){ + double time = ((double) frame_number) * this->fps.Reciprocal().ToDouble(); + + return time; +} diff --git a/src/KeyFrameBBox.h b/src/KeyFrameBBox.h new file mode 100644 index 00000000..9d308318 --- /dev/null +++ b/src/KeyFrameBBox.h @@ -0,0 +1,129 @@ +/** + * @file + * @brief Header file for the IKeyframe class + * @author Jonathan Thomas + * + * @ref License + */ + + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_BBOXKEYFRAME_H +#define OPENSHOT_BBOXKEYFRAME_H + +#include +#include +#include +#include +#include +#include "Exceptions.h" +#include "Fraction.h" +#include "Coordinate.h" +#include "Point.h" +#include "Json.h" +#include "IKeyFrame.h" +#include "KeyFrame.h" + + +struct BBox{ + float cx = -1; + float cy = -1; + float width = -1; + float height = -1; + + // Constructors + BBox(){ + return; + } + + BBox(float _cx, float _cy, float _width, float _height){ + //frame_num = _frame_num; + cx = _cx; + cy = _cy; + width = _width; + height = _height; + } +}; + + +namespace openshot { + /** + * @brief A Keyframe is a collection of Point instances, which is used to vary a number or property over time. + * + * Keyframes are used to animate and interpolate values of properties over time. For example, a single property + * can use a Keyframe instead of a constant value. Assume you want to slide an image (from left to right) over + * a video. You can create a Keyframe which will adjust the X value of the image over 100 frames (or however many + * frames the animation needs to last) from the value of 0 to 640. + * + * \endcode + */ + + class KeyFrameBBox { + private: + bool visible; + Fraction fps; + std::map BoxVec; + public: + Keyframe delta_x; + Keyframe delta_y; + Keyframe scale_x; + Keyframe scale_y; + + KeyFrameBBox(); + + void AddDisplacement(int64_t _frame_num, double _delta_x, double _delta_y); + void AddScale(int64_t _frame_num, double _delta_x, double _delta_y); + void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height); + + void SetFPS(Fraction fps); + Fraction GetFPS(); + + bool Contains(int64_t frame_number); + //double GetDelta(int64_t index) const ; + int64_t GetLength() const; + + /// Get and Set JSON methods + //std::string Json() const ; ///< Generate JSON string of this object + //Json::Value JsonValue() const ; ///< Generate Json::Value for this object + //void SetJson(const std::string value) ; ///< Load JSON string into this object + //void SetJsonValue(const Json::Value root) ; ///< Load Json::Value into this object + + /// Remove a points by frame_number + void RemovePoint(int64_t frame_number); + void RemoveDelta(int64_t frame_number); + void RemoveScale(int64_t frame_number); + + BBox GetValue(int64_t frame_number); + + /// Print collection of points + void PrintParams(); + + double FrameNToTime(int64_t frame_number); + BBox InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target); + + }; + +} + +#endif \ No newline at end of file diff --git a/src/OpenShot.h b/src/OpenShot.h index e1209ac4..1cb8700a 100644 --- a/src/OpenShot.h +++ b/src/OpenShot.h @@ -132,6 +132,7 @@ #include "TextReader.h" #endif #include "KeyFrame.h" +#include "KeyFrameBBox.h" #include "PlayerBase.h" #include "Point.h" #include "Profiles.h" diff --git a/tests/KeyFrame_Tests.cpp b/tests/KeyFrame_Tests.cpp index 84025165..b38eb8c9 100644 --- a/tests/KeyFrame_Tests.cpp +++ b/tests/KeyFrame_Tests.cpp @@ -494,3 +494,43 @@ TEST(Keyframe_Handle_Large_Segment) Fraction fr = kf.GetRepeatFraction(250000); CHECK_CLOSE(0.5, (double)fr.num / fr.den, 0.01); } + + +TEST(KeyFrameBBox_init_test) { + + KeyFrameBBox kfb; + +} + +TEST(KeyFrameBBox_addBox_test) { + KeyFrameBBox kfb; + + kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0); + + CHECK_EQUAL(true, kfb.Contains(1)); + CHECK_EQUAL(1, kfb.GetLength()); + + kfb.PrintParams(); + + kfb.RemovePoint(1); + + CHECK_EQUAL(false, kfb.Contains(1)); + CHECK_EQUAL(0, kfb.GetLength()); +} + + +TEST(KeyFrameBBox_GetVal_test) { + KeyFrameBBox kfb; + + kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0); + + kfb.AddDisplacement(1, 20.0, 20.0); + kfb.AddScale(1, 30, 30); + + BBox val = kfb.GetValue(1); + + CHECK_EQUAL(30.0, val.cx); + CHECK_EQUAL(30.0, val.cy); + CHECK_EQUAL(130.0,val.width); + CHECK_EQUAL(130.0, val.height); +} \ No newline at end of file