libopenshot/src/KeyFrameBBox.cpp

/**
 * @file
 * @brief Source file for the Keyframe class
 * @author Jonathan Thomas <jonathan@openshot.org>
 *
 * @ref License
 */

/* LICENSE
 *
 * Copyright (c) 2008-2019 OpenShot Studios, LLC
 * <http://www.openshotstudios.com/>. This file is part of
 * OpenShot Library (libopenshot), an open-source project dedicated to
 * delivering high quality video editing and animation solutions to the
 * world. For more information visit <http://www.openshot.org/>.
 *
 * OpenShot Library (libopenshot) is free software: you can redistribute it
 * and/or modify it under the terms of the GNU Lesser General Public License
 * as published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * OpenShot Library (libopenshot) is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
 */

#include "KeyFrameBBox.h"
#include <algorithm>
#include <fstream>
#include <iostream>
#include <functional>

//#define PI 3.14159265

using namespace std;
using namespace openshot;

namespace openshot
{
    bool IsPointBeforeX(Point const &p, double const x)
    {
        return p.co.X < x;
    }

    double InterpolateLinearCurve(Point const &left, Point const &right, double const target)
    {
        double const diff_Y = right.co.Y - left.co.Y;
        double const diff_X = right.co.X - left.co.X;
        double const slope = diff_Y / diff_X;
        return left.co.Y + slope * (target - left.co.X);
    }

    double InterpolateBezierCurve(Point const &left, Point const &right, double const target, double const allowed_error)
    {
        double const X_diff = right.co.X - left.co.X;
        double const Y_diff = right.co.Y - left.co.Y;
        Coordinate const p0 = left.co;
        Coordinate const p1 = Coordinate(p0.X + left.handle_right.X * X_diff, p0.Y + left.handle_right.Y * Y_diff);
        Coordinate const p2 = Coordinate(p0.X + right.handle_left.X * X_diff, p0.Y + right.handle_left.Y * Y_diff);
        Coordinate const p3 = right.co;

        double t = 0.5;
        double t_step = 0.25;
        do
        {
            // Bernstein polynoms
            double B[4] = {1, 3, 3, 1};
            double oneMinTExp = 1;
            double tExp = 1;
            for (int i = 0; i < 4; ++i, tExp *= t)
            {
                B[i] *= tExp;
            }
            for (int i = 0; i < 4; ++i, oneMinTExp *= 1 - t)
            {
                B[4 - i - 1] *= oneMinTExp;
            }
            double const x = p0.X * B[0] + p1.X * B[1] + p2.X * B[2] + p3.X * B[3];
            double const y = p0.Y * B[0] + p1.Y * B[1] + p2.Y * B[2] + p3.Y * B[3];
            if (fabs(target - x) < allowed_error)
            {
                return y;
            }
            if (x > target)
            {
                t -= t_step;
            }
            else
            {
                t += t_step;
            }
            t_step /= 2;
        } while (true);
    }

    double InterpolateBetween(Point const &left, Point const &right, double target, double allowed_error)
    {
        assert(left.co.X < target);
        assert(target <= right.co.X);
        switch (right.interpolation)
        {
        case CONSTANT:
            return left.co.Y;
        case LINEAR:
            return InterpolateLinearCurve(left, right, target);
        case BEZIER:
            return InterpolateBezierCurve(left, right, target, allowed_error);
        }
    }

    template <typename Check>
    int64_t SearchBetweenPoints(Point const &left, Point const &right, int64_t const current, Check check)
    {
        int64_t start = left.co.X;
        int64_t stop = right.co.X;
        while (start < stop)
        {
            int64_t const mid = (start + stop + 1) / 2;
            double const value = InterpolateBetween(left, right, mid, 0.01);
            if (check(round(value), current))
            {
                start = mid;
            }
            else
            {
                stop = mid - 1;
            }
        }
        return start;
    }
} // namespace openshot

// Default Constructor that sets the bounding-box displacement as 0 and the scales as 1 for the first frame
KeyFrameBBox::KeyFrameBBox() : delta_x(0.0), delta_y(0.0), scale_x(1.0), scale_y(1.0), rotation(0.0)
{
    this->TimeScale = 1.0;
    return;
}

// Add a BBox to the BoxVec map
void KeyFrameBBox::AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle)
{
    // Check if the given frame number is valid
    if (_frame_num < 0)
        return;

    // Instantiate a new bounding-box
    BBox newBBox = BBox(_cx, _cy, _width, _height, _angle);

    // Get the time of given frame
    double time = this->FrameNToTime(_frame_num, 1.0);
    // Create an iterator that points to the BoxVec pair indexed by the time of given frame
    auto BBoxIterator = BoxVec.find(time);
    
    if (BBoxIterator != BoxVec.end())
    {
        // There is a bounding-box indexed by the time of given frame, update-it
        BBoxIterator->second = newBBox;
    }
    else
    {
        // There isn't a bounding-box indexed by the time of given frame, insert a new one
        BoxVec.insert({time, newBBox});
    }
}

// Get the size of BoxVec map
int64_t KeyFrameBBox::GetLength() const
{
    if (BoxVec.empty())
        return 0;
    if (BoxVec.size() == 1)
        return 1;
    return BoxVec.size();
}

// Check if there is a bounding-box in the given frame
bool KeyFrameBBox::Contains(int64_t frame_num)
{
    // Get the time of given frame
    double time = this->FrameNToTime(frame_num, 1.0);
    // Create an iterator that points to the BoxVec pair indexed by the time of given frame (or the closest time)
    auto it = BoxVec.lower_bound(time);
    if (it == BoxVec.end())
        // BoxVec pair not found
        return false;
    return true;
}

// Remove a bounding-box from the BoxVec map
void KeyFrameBBox::RemoveBox(int64_t frame_number)
{
    // Get the time of given frame
    double time = this->FrameNToTime(frame_number, 1.0);
    // Create an iterator that points to the BoxVec pair indexed by the time of given frame
    auto it = BoxVec.find(time);
    if (it != BoxVec.end())
    {
        // The BoxVec pair exists, so remove it
        BoxVec.erase(time);
    }
    return;
}

// Return a bounding-box from BoxVec with it's properties adjusted by the Keyframes
BBox KeyFrameBBox::GetValue(int64_t frame_number)
{
    // Get the time position of the given frame.
    double time = this->FrameNToTime(frame_number, this->TimeScale);

    // Return a iterator pointing to the BoxVec pair indexed by time or to the pair indexed
    // by the closest upper time value.  
    auto currentBBoxIterator = BoxVec.lower_bound(time);

    // Check if there is a pair indexed by time, returns an empty bbox if there isn't.
    if (currentBBoxIterator == BoxVec.end())
    {
        // Create and return an empty bounding-box object
        BBox emptyBBox;
        return emptyBBox;
    }

    // Check if the iterator matches a BBox indexed by time or points to the first element of BoxVec
    if ((currentBBoxIterator->first == time) || (currentBBoxIterator == BoxVec.begin()))
    {
        // Get the BBox indexed by time
        BBox currentBBox = currentBBoxIterator->second;

        // Adjust the BBox properties by the Keyframes values
        currentBBox.cx += this->delta_x.GetValue(frame_number);
        currentBBox.cy += this->delta_y.GetValue(frame_number);
        currentBBox.width *= this->scale_x.GetValue(frame_number);
        currentBBox.height *= this->scale_y.GetValue(frame_number);
        currentBBox.angle += this->rotation.GetValue(frame_number);
    
        return currentBBox;
    }

    // BBox indexed by the closest upper time
    BBox currentBBox = currentBBoxIterator->second;
    // BBox indexed by the closet lower time
    BBox previousBBox = prev(currentBBoxIterator, 1)->second;

    // Interpolate a BBox in the middle of previousBBox and currentBBox
    BBox interpolatedBBox = InterpolateBoxes(prev(currentBBoxIterator, 1)->first, currentBBoxIterator->first,
                                             previousBBox, currentBBox, time);

    // Adjust the BBox properties by the Keyframes values
    interpolatedBBox.cx += this->delta_x.GetValue(frame_number);
    interpolatedBBox.cy += this->delta_y.GetValue(frame_number);
    interpolatedBBox.width *= this->scale_x.GetValue(frame_number);
    interpolatedBBox.height *= this->scale_y.GetValue(frame_number);
    interpolatedBBox.angle += this->rotation.GetValue(frame_number);
    
    return interpolatedBBox;
}

// Interpolate the bouding-boxes properties
BBox KeyFrameBBox::InterpolateBoxes(double t1, double t2, BBox left, BBox right, double target)
{
    // Interpolate the x-coordinate of the center point
    Point cx_left(t1, left.cx, openshot::InterpolationType::LINEAR);
    Point cx_right(t2, right.cx, openshot::InterpolationType::LINEAR);
    Point cx = InterpolateBetween(cx_left, cx_right, target, 0.01);

    // Interpolate de y-coordinate of the center point
    Point cy_left(t1, left.cy, openshot::InterpolationType::LINEAR);
    Point cy_right(t2, right.cy, openshot::InterpolationType::LINEAR);
    Point cy = InterpolateBetween(cy_left, cy_right, target, 0.01);

    // Interpolate the width
    Point width_left(t1, left.width, openshot::InterpolationType::LINEAR);
    Point width_right(t2, right.width, openshot::InterpolationType::LINEAR);
    Point width = InterpolateBetween(width_left, width_right, target, 0.01);

    // Interpolate the height
    Point height_left(t1, left.height, openshot::InterpolationType::LINEAR);
    Point height_right(t2, right.height, openshot::InterpolationType::LINEAR);
    Point height = InterpolateBetween(height_left, height_right, target, 0.01);

    // Interpolate the rotation angle
    Point angle_left(t1, left.angle, openshot::InterpolationType::LINEAR);
    Point angle_right(t1, right.angle, openshot::InterpolationType::LINEAR);
    Point angle = InterpolateBetween(angle_left, angle_right, target, 0.01);

    // Create a bounding box with the interpolated points
    BBox interpolatedBox(cx.co.Y, cy.co.Y, width.co.Y, height.co.Y, angle.co.Y);

    return interpolatedBox;
}

// Update object's BaseFps
void KeyFrameBBox::SetBaseFPS(Fraction fps){
    this->BaseFps = fps;
    return;
}

// Return the object's BaseFps
Fraction KeyFrameBBox::GetBaseFPS(){
    return BaseFps;
}

// Get the time of the given frame
double KeyFrameBBox::FrameNToTime(int64_t frame_number, double time_scale){
    double time = ((double)frame_number) * this->BaseFps.Reciprocal().ToDouble() * (1.0 / time_scale);

    return time;
}

// Update the TimeScale member variable
void KeyFrameBBox::ScalePoints(double time_scale){
    this->TimeScale = time_scale;
}

// Load the bounding-boxes information from the protobuf file 
bool KeyFrameBBox::LoadBoxData(std::string inputFilePath)
{
    // Variable to hold the loaded data
    libopenshottracker::Tracker bboxMessage;

    // Read the existing tracker message.
    fstream input(inputFilePath, ios::in | ios::binary);

    //Check if it was able to read the protobuf data
    if (!bboxMessage.ParseFromIstream(&input))
    {
        cerr << "Failed to parse protobuf message." << endl;
        return false;
    }
    
    this->clear();

    // Iterate over all frames of the saved message
    for (size_t i = 0; i < bboxMessage.frame_size(); i++)
    {
        // Get data of the i-th frame
        const libopenshottracker::Frame &pbFrameData = bboxMessage.frame(i);

        // Get frame number
        size_t frame_number = pbFrameData.id();

        // Get bounding box data from current frame
        const libopenshottracker::Frame::Box &box = pbFrameData.bounding_box();

        float width = box.x2() - box.x1();
        float height = box.y2() - box.y1();
        float cx = box.x1() + width/2;
        float cy = box.y1() + height/2;
        float angle = 0.0;


        if ( (cx >= 0.0) && (cy >= 0.0) && (width >= 0.0) && (height >= 0.0) )
        {
            // The bounding-box properties are valid, so add it to the BoxVec map
            this->AddBox(frame_number, cx, cy, width, height, angle);
        }
    }
    
    // Show the time stamp from the last update in tracker data file
    if (bboxMessage.has_last_updated())
    {
        cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(bboxMessage.last_updated()) << endl;
    }

    // Delete all global objects allocated by libprotobuf.
    google::protobuf::ShutdownProtobufLibrary();

    return true;
}

// Generate JSON string of this object
std::string KeyFrameBBox::Json()
{
    // Return formatted string
    return JsonValue().toStyledString();
}

// Generate Json::Value for this object
Json::Value KeyFrameBBox::JsonValue()
{
    // Create root json object
    Json::Value root;

    root["BaseFPS"]["num"] = BaseFps.num;
    root["BaseFPS"]["den"] = BaseFps.den;
    root["TimeScale"] = this->TimeScale;
    root["Boxes"] = Json::Value(Json::arrayValue);

    // Loop through the BoxVec map and save the BBox data
    for (auto const &x : BoxVec)
    {
        Json::Value box;
        box["time"] = x.first;
        box["data"] = x.second.JsonValue();
        root["Boxes"].append(box);
    }
    // Get the Keyframe's Json strings
    root["delta_x"] = delta_x.JsonValue();
    root["delta_y"] = delta_y.JsonValue();
    root["scale_x"] = scale_x.JsonValue();
    root["scale_y"] = scale_y.JsonValue(); 
    root["rotation"] = rotation.JsonValue();

    // return JsonValue
    return root;
}

// Load JSON string into this object
void KeyFrameBBox::SetJson(const std::string value)
{
    // Parse JSON string into JSON objects
    try
    {
        const Json::Value root = openshot::stringToJson(value);
        // Set all values that match
        SetJsonValue(root);
    }
    catch (const std::exception &e)
    {
        // Error parsing JSON (or missing keys)
        throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
    }
    return;
}

// Clear the BoxVec map
void KeyFrameBBox::clear()
{
    BoxVec.clear();
}

// Load Json::Value into this object
void KeyFrameBBox::SetJsonValue(const Json::Value root)
{
    // Clear BoxVec
    this->clear();

    if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
    {
        // Set the BaseFps by the given JSON object
        if (!root["BaseFPS"]["num"].isNull())
            BaseFps.num = (int)root["BaseFPS"]["num"].asInt();
        if (!root["BaseFPS"]["den"].isNull())
            BaseFps.den = (int)root["BaseFPS"]["den"].asInt();
    }

    if (!root["TimeScale"].isNull())
    {
        // Set the TimeScale by the given JSON object
        double scale = (double)root["TimeScale"].asDouble();
        this->ScalePoints(scale);
    }

    if (!root["Boxes"].isNull())
    {
        // Loop through the BBoxes data
        for (const auto existing_point : root["Boxes"])
        {
            // Insert BBox into the BoxVec map 
            BBox box;
            box.SetJsonValue(existing_point["data"]);
            BoxVec[existing_point["time"].asDouble()] = box;
        }
    }

    // Set the Keyframes by the given JSON object
    if (!root["delta_x"].isNull())
        delta_x.SetJsonValue(root["delta_x"]);
    if (!root["delta_y"].isNull())
		delta_y.SetJsonValue(root["delta_y"]);
    if (!root["scale_x"].isNull())
        scale_x.SetJsonValue(root["scale_x"]);
    if (!root["scale_y"].isNull())
        scale_y.SetJsonValue(root["scale_y"]);
    if (!root["rotation"].isNull())
        rotation.SetJsonValue(root["rotation"]);

    return;
}