openshot/libopenshot
0
0
Fork 0
mirror of https://github.com/OpenShot/libopenshot.git synced 2026-03-02 08:53:52 -08:00
Code Issues Packages Projects Releases Wiki Activity

Added rotation transform to the tracker bounding-box.

Browse Source 
Changed the bounding-box struct point convention to (cx, cy, width, height, angle).
The GetFrame member function from Tracker class now uses the cv::RotatedRect object to represent the bounding-box and draw it on screen.
The JSON and Protobuf communication between the backend and frontend still uses the (x1,y1)(x2,y2) point convention, the backend performs the adequate transformations.
This commit is contained in:
Brenno
2020-12-14 18:08:44 -03:00
parent 60cec478d3
commit 4fdd325ea3
4 changed files with 113 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
src/KeyFrameBBox.cpp
View File

@@ -142,14 +142,14 @@ KeyFrameBBox::KeyFrameBBox() : delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y
}
// Add a BBox to the BoxVec map
void KeyFrameBBox::AddBox(int64_t _frame_num, float _x1, float _y1, float _width, float _height)
void KeyFrameBBox::AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle)
{
// Check if the given frame number is valid
if (_frame_num < 0)
return;
// Instantiate a new bounding-box
BBox newBBox = BBox(_x1, _y1, _width, _height);
BBox newBBox = BBox(_cx, _cy, _width, _height, _angle);
// Get the time of given frame
double time = this->FrameNToTime(_frame_num, 1.0);
@@ -231,25 +231,12 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number)
BBox currentBBox = currentBBoxIterator->second;
// Adjust the BBox properties by the Keyframes values
currentBBox.x1 += this->delta_x.GetValue(frame_number);
currentBBox.y1 += this->delta_y.GetValue(frame_number);
currentBBox.cx += this->delta_x.GetValue(frame_number);
currentBBox.cy += this->delta_y.GetValue(frame_number);
currentBBox.width *= this->scale_x.GetValue(frame_number);
currentBBox.height *= this->scale_y.GetValue(frame_number);
currentBBox.angle += this->rotation.GetValue(frame_number);
/* TODO - Add rotation
(x1,y1) -> current point
(x1',y1') -> transformed point
(xc, yc) -> center of the BBox
(xc, yc) = (x1 + w/2, y1 + h/2)
rot -> rotation angle [radians]
x1' = xc + (x1 - xc)*cos(rot) - (y1-yc)*sin(rot)
y1' = yc + (x1 - xc)*sin(rot) + (y1-yc)*cos(rot)
***
x1' = (x1 + w/2) + (-w/2)*cos(rot) - (-h/2)*sin(rot)
y1' = (y1 + h/2) + (-w/2)*sin(rot) + (-h/2)*cos(rot)
currentBBox.x1 += currentBBox.width/2 - (currentBBox.width/2)*cos(rot*PI/180) + (currentBBox.height)*sin(rot*PI/180);
currentBBox.y1 += currentBBox.height/2 - (currentBBox.width/2)*sin(rot*PI/180) - (currentBBox.height)*cos(rot*PI/180);
*/
return currentBBox;
}
@@ -263,10 +250,11 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number)
previousBBox, currentBBox, time);
// Adjust the BBox properties by the Keyframes values
interpolatedBBox.x1 += this->delta_x.GetValue(frame_number);
interpolatedBBox.y1 += this->delta_y.GetValue(frame_number);
interpolatedBBox.cx += this->delta_x.GetValue(frame_number);
interpolatedBBox.cy += this->delta_y.GetValue(frame_number);
interpolatedBBox.width *= this->scale_x.GetValue(frame_number);
interpolatedBBox.height *= this->scale_y.GetValue(frame_number);
interpolatedBBox.angle += this->rotation.GetValue(frame_number);
return interpolatedBBox;
}
@@ -274,26 +262,35 @@ BBox KeyFrameBBox::GetValue(int64_t frame_number)
// Interpolate the bouding-boxes properties
BBox KeyFrameBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target)
{
// Interpolate the x-coordinate of the center point
Point cx_left(t1, left.cx, openshot::InterpolationType::LINEAR);
Point cx_right(t2, right.cx, openshot::InterpolationType::LINEAR);
Point cx = InterpolateBetween(cx_left, cx_right, target, 0.01);
Point p1_left(t1, left.x1, openshot::InterpolationType::LINEAR);
Point p1_right(t2, right.x1, openshot::InterpolationType::LINEAR);
Point p1 = InterpolateBetween(p1_left, p1_right, target, 0.01);
// Interpolate de y-coordinate of the center point
Point cy_left(t1, left.cy, openshot::InterpolationType::LINEAR);
Point cy_right(t2, right.cy, openshot::InterpolationType::LINEAR);
Point cy = InterpolateBetween(cy_left, cy_right, target, 0.01);
Point p2_left(t1, left.y1, openshot::InterpolationType::LINEAR);
Point p2_right(t2, right.y1, openshot::InterpolationType::LINEAR);
Point p2 = InterpolateBetween(p2_left, p2_right, target, 0.01);
// Interpolate the width
Point width_left(t1, left.width, openshot::InterpolationType::LINEAR);
Point width_right(t2, right.width, openshot::InterpolationType::LINEAR);
Point width = InterpolateBetween(width_left, width_right, target, 0.01);
Point p3_left(t1, left.height, openshot::InterpolationType::LINEAR);
Point p3_right(t2, right.height, openshot::InterpolationType::LINEAR);
Point p3 = InterpolateBetween(p3_left, p3_right, target, 0.01);
// Interpolate the height
Point height_left(t1, left.height, openshot::InterpolationType::LINEAR);
Point height_right(t2, right.height, openshot::InterpolationType::LINEAR);
Point height = InterpolateBetween(height_left, height_right, target, 0.01);
Point p4_left(t1, left.width, openshot::InterpolationType::LINEAR);
Point p4_right(t2, right.width, openshot::InterpolationType::LINEAR);
Point p4 = InterpolateBetween(p4_left, p4_right, target, 0.01);
// Interpolate the rotation angle
Point angle_left(t1, left.angle, openshot::InterpolationType::LINEAR);
Point angle_right(t1, right.angle, openshot::InterpolationType::LINEAR);
Point angle = InterpolateBetween(angle_left, angle_right, target, 0.01);
BBox ans(p1.co.Y, p2.co.Y, p4.co.Y, p3.co.Y);
// Create a bounding box with the interpolated points
BBox interpolatedBox(cx.co.Y, cy.co.Y, width.co.Y, height.co.Y, angle.co.Y);
return ans;
return interpolatedBox;
}
// Update object's BaseFps
@@ -348,16 +345,18 @@ bool KeyFrameBBox::LoadBoxData(std::string inputFilePath)
// Get bounding box data from current frame
const libopenshottracker::Frame::Box &box = pbFrameData.bounding_box();
float x1 = box.x1();
float y1 = box.y1();
float x2 = box.x2();
float y2 = box.y2();
if ( (x1 >= 0.0) && (y1 >= 0.0) && (x2 >= 0.0) && (y2 >= 0.0) )
float width = box.x2() - box.x1();
float height = box.y2() - box.y1();
float cx = box.x1() + width/2;
float cy = box.y1() + height/2;
float angle = 0.0;
if ( (cx >= 0.0) && (cy >= 0.0) && (width >= 0.0) && (height >= 0.0) )
{
// The bounding-box properties are valid, so add it to the BoxVec map
this->AddBox(frame_number, x1, y1, (x2 - x1), (y2 - y1));
this->AddBox(frame_number, cx, cy, width, height, angle);
}
}
@@ -464,7 +463,6 @@ void KeyFrameBBox::SetJsonValue(const Json::Value root)
// Insert BBox into the BoxVec map
BBox box;
box.SetJsonValue(existing_point["data"]);
//BoxVec.insert({existing_point["time"].asDouble(), box});
BoxVec[existing_point["time"].asDouble()] = box;
}
}
@@ -482,4 +480,4 @@ void KeyFrameBBox::SetJsonValue(const Json::Value root)
rotation.SetJsonValue(root["rotation"]);
return;
}
}

47
src/KeyFrameBBox.h
View File

@@ -60,10 +60,11 @@ namespace openshot
struct BBox
{
float x1 = -1; ///< x-coordinate of the top left corner
float y1 = -1; ///< y-coordinate of the top left corner
float cx = -1; ///< x-coordinate of the bounding box center
float cy = -1; ///< y-coordinate of the bounding box center
float width = -1; ///< bounding box width
float height = -1; ///< bounding box height
float angle = -1; ///< bounding box rotation angle [degrees]
/// Blank constructor
BBox()
@@ -72,18 +73,21 @@ namespace openshot
}
/// Default constructor, which takes the bounding box top-left corner coordinates, width and height.
/// @param _x1 X-coordinate of the top left corner
/// @param _y1 Y-coordinate of the top left corner
/// @param _cx X-coordinate of the bounding box center
/// @param _cy Y-coordinate of the bounding box center
/// @param _width Bounding box width
/// @param _height Bouding box height
BBox(float _x1, float _y1, float _width, float _height)
/// @param _height Bounding box height
/// @param _angle Bounding box rotation angle [degrees]
BBox(float _cx, float _cy, float _width, float _height, float _angle)
{
x1 = _x1;
y1 = _y1;
cx = _cx;
cy = _cy;
width = _width;
height = _height;
angle = _angle;
}
/// Generate JSON string of this object
std::string Json() const
{
@@ -94,10 +98,11 @@ namespace openshot
Json::Value JsonValue() const
{
Json::Value root;
root["x1"] = x1;
root["y1"] = y1;
root["height"] = height;
root["cx"] = cx;
root["cy"] = cy;
root["width"] = width;
root["height"] = height;
root["angle"] = angle;
return root;
}
@@ -124,15 +129,17 @@ namespace openshot
{
// Set data from Json (if key is found)
if (!root["x1"].isNull())
x1 = root["x1"].asDouble();
if (!root["y1"].isNull())
y1 = root["y1"].asDouble();
if (!root["height"].isNull())
height = root["height"].asDouble();
if (!root["cx"].isNull())
cx = root["cx"].asDouble();
if (!root["cy"].isNull())
cy = root["cy"].asDouble();
if (!root["width"].isNull())
width = root["width"].asDouble();
}
if (!root["height"].isNull())
height = root["height"].asDouble();
if (!root["angle"].isNull())
angle = root["angle"].asDouble();
}
};
/**
@@ -166,7 +173,7 @@ namespace openshot
KeyFrameBBox();
/// Add a BBox to the BoxVec map
void AddBox(int64_t _frame_num, float _x1, float _y1, float _width, float _height);
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle);
/// Update object's BaseFps
void SetBaseFPS(Fraction fps);
@@ -214,4 +221,4 @@ namespace openshot
} // namespace openshot
#endif
#endif

36
src/effects/Tracker.cpp
View File

@@ -73,11 +73,11 @@ std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t f
cv::Mat frame_image = frame->GetImageCV();
// Check if frame isn't NULL
if(!frame_image.empty()){
if(!frame_image.empty())
{
// Check if track data exists for the requested frame
if (trackedData.Contains(frame_number)) {
if (trackedData.Contains(frame_number))
{
// Get the width and height of the image
float fw = frame_image.size().width;
float fh = frame_image.size().height;
@@ -85,12 +85,18 @@ std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t f
// Get the bounding-box of given frame
BBox fd = this->trackedData.GetValue(frame_number);
// Draw the bounding-box on the image
cv::Rect2d box((int)( (fd.x1 ) * fw ),
(int)( (fd.y1 ) * fh ),
(int)( (fd.width) * fw),
(int)( (fd.height) * fh) );
cv::rectangle(frame_image, box, cv::Scalar( 255, 0, 0 ), 2, 1 );
// Create a rotated rectangle object that holds the bounding box
cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
(int) (fd.angle) );
// Get the bouding box vertices
cv::Point2f vertices[4];
box.points(vertices);
// Draw the bounding-box on the image
for (int i = 0; i < 4; i++)
{
cv::line(frame_image, vertices[i], vertices[(i+1)%4], cv::Scalar(255,0,0), 2);
}
}
}
@@ -194,10 +200,10 @@ std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
// Get the bounding-box for the given-frame
BBox fd = trackedData.GetValue(requested_frame);
// Add the data of given frame bounding-box to the JSON object
root["x1"] = add_property_json("X1", fd.x1, "float", "", NULL, 0.0, 1.0, false, requested_frame);
root["y1"] = add_property_json("Y1", fd.y1, "float", "", NULL, 0.0, 1.0, false, requested_frame);
root["x2"] = add_property_json("X2", fd.x1+fd.width, "float", "", NULL, 0.0, 1.0, false, requested_frame);
root["y2"] = add_property_json("Y2", fd.y1+fd.height, "float", "", NULL, 0.0, 1.0, false, requested_frame);
root["x1"] = add_property_json("X1", fd.cx-(fd.width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
root["y1"] = add_property_json("Y1", fd.cy-(fd.height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
root["x2"] = add_property_json("X2", fd.cx+(fd.width/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
root["y2"] = add_property_json("Y2", fd.cy+(fd.height/2), "float", "", NULL, 0.0, 1.0, false, requested_frame);
// Add the bounding-box Keyframes to the JSON object
root["delta_x"] = add_property_json("Displacement X-axis", trackedData.delta_x.GetValue(requested_frame), "float", "", &trackedData.delta_x, -1.0, 1.0, false, requested_frame);
@@ -260,4 +266,4 @@ void Tracker::SetJson(int64_t requested_frame, const std::string value)
throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
}
return;
}
}

46
tests/KeyFrame_Tests.cpp
View File

@@ -505,7 +505,7 @@ TEST(KeyFrameBBox_init_test) {
TEST(KeyFrameBBox_addBox_test) {
KeyFrameBBox kfb;
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0);
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0, 0.0);
CHECK_EQUAL(true, kfb.Contains(1));
CHECK_EQUAL(1, kfb.GetLength());
@@ -520,43 +520,44 @@ TEST(KeyFrameBBox_addBox_test) {
TEST(KeyFrameBBox_GetVal_test) {
KeyFrameBBox kfb;
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0);
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0, 0.0);
BBox val = kfb.GetValue(1);
CHECK_EQUAL(10.0, val.x1);
CHECK_EQUAL(10.0, val.y1);
CHECK_EQUAL(10.0, val.cx);
CHECK_EQUAL(10.0, val.cy);
CHECK_EQUAL(100.0,val.width);
CHECK_EQUAL(100.0,val.height);
CHECK_EQUAL(0.0, val.angle);
}
TEST(KeyFrameBBox_GetVal_Interpolation) {
KeyFrameBBox kfb;
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0);
kfb.AddBox(11, 20.0, 20.0, 100.0, 100.0);
kfb.AddBox(21, 30.0, 30.0, 100.0, 100.0);
kfb.AddBox(31, 40.0, 40.0, 100.0, 100.0);
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0, 0.0);
kfb.AddBox(11, 20.0, 20.0, 100.0, 100.0, 0.0);
kfb.AddBox(21, 30.0, 30.0, 100.0, 100.0, 0.0);
kfb.AddBox(31, 40.0, 40.0, 100.0, 100.0, 0.0);
BBox val = kfb.GetValue(5);
CHECK_EQUAL(14.0, val.x1);
CHECK_EQUAL(14.0, val.y1);
CHECK_EQUAL(14.0, val.cx);
CHECK_EQUAL(14.0, val.cy);
CHECK_EQUAL(100.0,val.width);
CHECK_EQUAL(100.0, val.height);
CHECK_EQUAL(100.0,val.height);
val = kfb.GetValue(15);
CHECK_EQUAL(24.0, val.x1);
CHECK_EQUAL(24.0, val.y1);
CHECK_EQUAL(24.0, val.cx);
CHECK_EQUAL(24.0, val.cy);
CHECK_EQUAL(100.0,val.width);
CHECK_EQUAL(100.0, val.height);
val = kfb.GetValue(25);
CHECK_EQUAL(34.0, val.x1);
CHECK_EQUAL(34.0, val.y1);
CHECK_EQUAL(34.0, val.cx);
CHECK_EQUAL(34.0, val.cy);
CHECK_EQUAL(100.0,val.width);
CHECK_EQUAL(100.0, val.height);
@@ -566,10 +567,10 @@ TEST(KeyFrameBBox_GetVal_Interpolation) {
TEST(KeyFrameBBox_Json_set) {
KeyFrameBBox kfb;
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0);
kfb.AddBox(10, 20.0, 20.0, 100.0, 100.0);
kfb.AddBox(20, 30.0, 30.0, 100.0, 100.0);
kfb.AddBox(30, 40.0, 40.0, 100.0, 100.0);
kfb.AddBox(1, 10.0, 10.0, 100.0, 100.0, 0.0);
kfb.AddBox(10, 20.0, 20.0, 100.0, 100.0, 0.0);
kfb.AddBox(20, 30.0, 30.0, 100.0, 100.0, 0.0);
kfb.AddBox(30, 40.0, 40.0, 100.0, 100.0, 0.0);
kfb.scale_x.AddPoint(1, 2.0);
kfb.scale_x.AddPoint(10, 3.0);
@@ -591,16 +592,17 @@ TEST(KeyFrameBBox_Json_set) {
BBox kfb_bbox = kfb.BoxVec[time_kfb];
BBox fromJSON_bbox = fromJSON_kfb.BoxVec[time_fromJSON_kfb];
CHECK_EQUAL(kfb_bbox.x1, fromJSON_bbox.x1);
CHECK_EQUAL(kfb_bbox.y1, fromJSON_bbox.y1);
CHECK_EQUAL(kfb_bbox.cx, fromJSON_bbox.cx);
CHECK_EQUAL(kfb_bbox.cy, fromJSON_bbox.cy);
CHECK_EQUAL(kfb_bbox.width, fromJSON_bbox.width);
CHECK_EQUAL(kfb_bbox.height, fromJSON_bbox.height);
CHECK_EQUAL(kfb_bbox.angle, fromJSON_bbox.angle);
}
TEST(KeyFrameBBox_Scale_test){
KeyFrameBBox kfb;
kfb.AddBox(1, 10.0, 10.0, 10.0, 10.0);
kfb.AddBox(1, 10.0, 10.0, 10.0, 10.0, 0.0);
kfb.scale_x.AddPoint(1.0, 2.0);
kfb.scale_y.AddPoint(1.0, 3.0);