Added protobuf dependency to load and save CVtracker data

include/CVTracker.h

@@ -1,20 +1,57 @@
-
+#include <google/protobuf/util/time_util.h>
 #include <opencv2/opencv.hpp>
 #include <opencv2/tracking.hpp>
 #include <opencv2/core.hpp>
+#include <fstream>
+
+#include "trackerdata.pb.h"
 
 using namespace cv;
+using namespace std;
+using google::protobuf::util::TimeUtil;
 
+struct FrameData{
+  int frame_id = -1;
+  float rotation = 0;
+  int x1 = -1;
+  int y1 = -1;
+  int x2 = -1;
+  int y2 = -1;
+
+  // constructor
+  FrameData( int _frame_id)
+  {frame_id = _frame_id;}
+
+  FrameData( int _frame_id , float _rotation, int _x1, int _y1, int _x2, int _y2)
+  {
+      frame_id = _frame_id;
+      rotation = _rotation;
+      x1 = _x1;
+      y1 = _y1;
+      x2 = _x2;
+      y2 = _y2;
+  }
+};
 
 class CVTracker {       
-  public:             
+  public:
+    // List of tracker types in OpenCV
+    std::string trackerTypes[8] = {"BOOSTING", "MIL", "KCF", "TLD","MEDIANFLOW", "GOTURN", "MOSSE", "CSRT"};
+
+    std::vector<FrameData> trackedData;             
     std::string trackerType;
     Ptr<Tracker> tracker;  
     Rect2d bbox;    
 
     CVTracker();
     Ptr<Tracker> select_tracker(std::string trackerType);
-    bool initTracker(Rect2d bbox, Mat &frame);
-    bool trackFrame(Mat &frame);
+    bool initTracker(Rect2d bbox, Mat &frame, int frameId);
+    bool trackFrame(Mat &frame, int frameId);
 
+    // Save protobuf file
+    bool SaveTrackedData(std::string outputFilePath);
+    void AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, FrameData& fData);
+
+    // Load protobuf file
+    bool LoadTrackedData(std::string inputFilePath);
 };

src/CMakeLists.txt

@@ -81,6 +81,10 @@ endif()
 
 find_package( OpenCV 4 REQUIRED )
 
+################ PROTOBUF ##################
+
+find_package(Protobuf REQUIRED)
+include_directories(${PROTOBUF_INCLUDE_DIR})
 
 ################# LIBOPENSHOT-AUDIO ###################
 # Find JUCE-based openshot Audio libraries
@@ -171,6 +175,11 @@ set(OPENSHOT_SOURCES
   Timeline.cpp
   CVTracker.cpp)
 
+# Compiled Protobuf messages
+set(PROTOBUF_MESSAGES
+  trackerdata.pb.cc)
+
+
 # Video effects
 set(EFFECTS_SOURCES
   effects/Bars.cpp
@@ -213,6 +222,7 @@ target_sources(openshot PRIVATE
   ${EFFECTS_SOURCES}
   ${QT_PLAYER_SOURCES}
   ${OPENSHOT_QT_HEADERS}
+  ${PROTOBUF_MESSAGES}
   )
 
 # Set SONAME and other library properties
@@ -372,7 +382,8 @@ endif()
 target_link_libraries(openshot PUBLIC
 	${LIBOPENSHOT_AUDIO_LIBRARIES}
   ${PROFILER}
-  ${OpenCV_LIBS})
+  ${OpenCV_LIBS}
+  ${PROTOBUF_LIBRARY})
 
 if(ImageMagick_FOUND)
   target_link_libraries(openshot PUBLIC ${ImageMagick_LIBRARIES})

src/CVTracker.cpp

@@ -1,18 +1,10 @@
 #include "../include/CVTracker.h"
 
-using namespace cv;
-
-
 CVTracker::CVTracker(){
-    // List of tracker types in OpenCV 3.4.1
-    std::string trackerTypes[8] = {"BOOSTING", "MIL", "KCF", "TLD","MEDIANFLOW", "GOTURN", "MOSSE", "CSRT"};
-    // vector <string> trackerTypes(types, std::end(types));
 
-    // Create a tracker
+    // Create KCF tracker
     trackerType = trackerTypes[2];
-
     tracker = select_tracker(trackerType);
-
 }
 
 Ptr<Tracker> CVTracker::select_tracker(std::string trackerType){
@@ -45,24 +37,22 @@ Ptr<Tracker> CVTracker::select_tracker(std::string trackerType){
     return t;
 }
 
+bool CVTracker::initTracker(Rect2d initial_bbox, Mat &frame, int frameId){
 
-bool CVTracker::initTracker(Rect2d initial_bbox, Mat &frame){
-    // Rect2d bbox(287, 23, 86, 320); 
-    bbox = initial_bbox;
-
-    // Uncomment the line below to select a different bounding box 
-    // bbox = selectROI(frame, false); 
-    // Display bounding box. 
+    bbox = initial_bbox; 
     rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 ); 
-    
+    // Create new tracker object
     tracker = select_tracker(trackerType);
 
     tracker->init(frame, bbox);
 
+    // Add new frame data
+    trackedData.push_back(FrameData(frameId, 0, bbox.x, bbox.y, bbox.x+bbox.width, bbox.y+bbox.height));
+
     return true;
 }
 
-bool CVTracker::trackFrame(Mat &frame){
+bool CVTracker::trackFrame(Mat &frame, int frameId){
     // Update the tracking result
     bool ok = tracker->update(frame, bbox);
 
@@ -70,12 +60,109 @@ bool CVTracker::trackFrame(Mat &frame){
     {
         // Tracking success : Draw the tracked object
         rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 );
+        
+        // Add new frame data
+        trackedData.push_back(FrameData(frameId, 0, bbox.x, bbox.y, bbox.x+bbox.width, bbox.y+bbox.height));
+
     }
     else
     {
         // Tracking failure detected.
         putText(frame, "Tracking failure detected", Point(100,80), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(0,0,255),2);
+        
+        // Add new frame data
+        trackedData.push_back(FrameData(frameId));
+
     }
 
     return ok;
 }
+
+bool CVTracker::SaveTrackedData(std::string outputFilePath){
+    // Create tracker message
+    libopenshottracker::Tracker trackerMessage;
+
+    // Add all frames data
+    for(int i=0; i < trackedData.size(); i++){
+        
+        FrameData fData = trackedData[i];
+        libopenshottracker::Frame* pbFrameData;
+        AddFrameDataToProto(trackerMessage.add_frame(), fData);
+    }
+
+    // Add timestamp
+    *trackerMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
+
+    {
+        // Write the new message to disk.
+        std::fstream output(outputFilePath, ios::out | ios::trunc | ios::binary);
+        if (!trackerMessage.SerializeToOstream(&output)) {
+        cerr << "Failed to write protobuf message." << endl;
+        return false;
+        }
+    }
+
+    // Delete all global objects allocated by libprotobuf.
+    google::protobuf::ShutdownProtobufLibrary();
+
+    return true;
+
+}
+
+// Add frame tracked data into protobuf message.
+void CVTracker::AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, FrameData& fData) {
+
+  pbFrameData->set_id(fData.frame_id);
+  pbFrameData->set_rotation(fData.frame_id);
+  pbFrameData->set_rotation(fData.frame_id);
+
+  libopenshottracker::Frame::Box* box = pbFrameData->mutable_bounding_box();
+    box->set_x1(fData.x1);
+    box->set_y1(fData.y1);
+    box->set_x2(fData.x2);
+    box->set_y2(fData.y2);
+
+}
+
+bool CVTracker::LoadTrackedData(std::string inputFilePath){
+
+    libopenshottracker::Tracker trackerMessage;
+
+    {
+        // Read the existing tracker message.
+        fstream input(inputFilePath, ios::in | ios::binary);
+        if (!trackerMessage.ParseFromIstream(&input)) {
+            cerr << "Failed to parse protobuf message." << endl;
+            return false;
+        }
+    }
+
+    // Make sure the trackedData is empty
+    trackedData.clear();
+    trackedData.reserve(trackerMessage.frame_size());
+
+    // Iterate over all frames of the saved message
+    for (int i = 0; i < trackerMessage.frame_size(); i++) {
+        const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i);
+
+        int id = pbFrameData.id();
+        float rotation = pbFrameData.rotation();
+
+        const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box();
+        int x1 = box.x1();
+        int y1 = box.y1();
+        int x2 = box.x2();
+        int y2 = box.y2();
+
+        trackedData[i] = FrameData(id, rotation, x1, y1, x2, y2);
+    }
+
+    if (trackerMessage.has_last_updated()) {
+        cout << "  Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl;
+    }
+
+    // Delete all global objects allocated by libprotobuf.
+    google::protobuf::ShutdownProtobufLibrary();
+
+    return true;
+}

src/examples/Example_opencv.cpp

@@ -32,8 +32,9 @@
 #include <iostream>
 #include <memory>
 #include <opencv2/opencv.hpp>
+// #include <google/protobuf/util/time_util.h>
 #include "../../include/CVTracker.h"
-
+// #include "treackerdata.pb.h"
 
 #include "../../include/OpenShot.h"
 #include "../../include/CrashHandler.h"
@@ -94,20 +95,27 @@ int main(int argc, char* argv[]) {
         if(!trackerInit){
             Rect2d bbox = selectROI("Display Image", cvimage);
 
-            kcfTracker.initTracker(bbox, cvimage);
+            kcfTracker.initTracker(bbox, cvimage, frame_number);
             trackerInit = true;
         }
         else{
-            trackerInit = kcfTracker.trackFrame(cvimage);
+            trackerInit = kcfTracker.trackFrame(cvimage, frame_number);
         }
         
         cv::imshow("Display Image", cvimage);
-        cv::waitKey(30);
-
+        // Press  ESC on keyboard to exit
+        char c=(char)waitKey(25);
+        if(c==27)
+            break;
 
         w9.WriteFrame(f);
     }
 
+
+    // Save tracked data to file
+    std::cout << "Saving tracker data!" << std::endl;
+    kcfTracker.SaveTrackedData("kcf_tracker.data");
+
     // Close writer & reader
     w9.Close();
 

src/trackerdata.proto

@@ -0,0 +1,29 @@
+
+// [START declaration]
+syntax = "proto3";
+package libopenshottracker;
+
+import "google/protobuf/timestamp.proto";
+// [END declaration]
+
+// [START messages]
+message Frame {
+  int32 id = 1;  // Frame ID.
+  float rotation = 2;
+
+  message Box {
+    int32 x1 = 1;
+    int32 y1 = 2;
+    int32 x2 = 3;
+    int32 y2 = 4;
+  }
+
+  Box bounding_box = 3;
+}
+
+message Tracker {
+  repeated Frame frame = 1;
+  
+  google.protobuf.Timestamp last_updated = 2;
+}
+// [END messages]