From 6ca35bbf9bc19073d5844d4e7956b27efe91c5ab Mon Sep 17 00:00:00 2001 From: Brenno Date: Thu, 30 Jul 2020 21:39:20 -0300 Subject: [PATCH] Fixed Stabilizer and created Object Detector tests --- include/CVObjectDetection.h | 7 +- include/CVStabilization.h | 5 +- include/CVTracker.h | 5 +- src/CVObjectDetection.cpp | 90 ++++++++++- src/CVStabilization.cpp | 174 ++++++++++++---------- src/CVTracker.cpp | 104 +++++++------ src/ClipProcessingJobs.cpp | 2 +- src/effects/ObjectDetection.cpp | 42 +++--- src/examples/Example_opencv.cpp | 10 +- src/protobuf_messages/objdetectdata.proto | 8 +- tests/CVObjectDetection_Tests.cpp | 79 ++++++---- tests/CVStabilizer_Tests.cpp | 2 +- tests/CVTracker_Tests.cpp | 42 +++--- 13 files changed, 351 insertions(+), 219 deletions(-) diff --git a/include/CVObjectDetection.h b/include/CVObjectDetection.h index 36086fd8..f5742161 100644 --- a/include/CVObjectDetection.h +++ b/include/CVObjectDetection.h @@ -110,13 +110,14 @@ class CVObjectDetection{ /// Protobuf Save and Load methods // Save protobuf file - bool SaveTrackedData(); + bool SaveObjDetectedData(); // Add frame object detection data into protobuf message. void AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrameData, CVDetectionData& dData); - // Load protobuf file - bool LoadTrackedData(); /// Get and Set JSON methods void SetJson(const std::string value); ///< Load JSON string into this object void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object + + // Load protobuf file (ONLY FOR MAKE TEST) + bool _LoadObjDetectdData(); }; diff --git a/include/CVStabilization.h b/include/CVStabilization.h index 249f518f..ea3630d0 100644 --- a/include/CVStabilization.h +++ b/include/CVStabilization.h @@ -84,6 +84,7 @@ class CVStabilization { size_t start; size_t end; + double avr_dx, avr_dy, avr_da, max_dx, max_dy, max_da; cv::Mat last_T; cv::Mat prev_grey; @@ -120,8 +121,6 @@ class CVStabilization { bool SaveStabilizedData(); // Add frame stabilization data into protobuf message void AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number); - // Load protobuf data file - bool LoadStabilizedData(); // Return requested struct info for a given frame TransformParam GetTransformParamData(size_t frameId); @@ -131,6 +130,8 @@ class CVStabilization { void SetJson(const std::string value); ///< Load JSON string into this object void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object + // Load protobuf data file (ONLY FOR MAKE TEST) + bool _LoadStabilizedData(); }; #endif \ No newline at end of file diff --git a/include/CVTracker.h b/include/CVTracker.h index d4f42a97..c5c18aee 100644 --- a/include/CVTracker.h +++ b/include/CVTracker.h @@ -98,12 +98,13 @@ class CVTracker { bool SaveTrackedData(); // Add frame tracked data into protobuf message. void AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, FrameData& fData); - // Load protobuf file - bool LoadTrackedData(); /// Get and Set JSON methods void SetJson(const std::string value); ///< Load JSON string into this object void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object + + // Load protobuf file (ONLY FOR MAKE TEST) + bool _LoadTrackedData(); }; diff --git a/src/CVObjectDetection.cpp b/src/CVObjectDetection.cpp index 277a9cc4..13e6f0cb 100644 --- a/src/CVObjectDetection.cpp +++ b/src/CVObjectDetection.cpp @@ -118,7 +118,7 @@ void CVObjectDetection::DetectObjects(const cv::Mat &frame, size_t frameId){ // Runs the forward pass to get output of the output layers std::vector outs; net.forward(outs, getOutputsNames(net)); - + // Remove the bounding boxes with low confidence postprocess(frame.size(), outs, frameId); @@ -239,8 +239,8 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector cv::Rect_ normalized_box; normalized_box.x = (box.x)/(float)frameDims.width; normalized_box.y = (box.y)/(float)frameDims.height; - normalized_box.width = (box.x+box.width)/(float)frameDims.width; - normalized_box.height = (box.y+box.height)/(float)frameDims.height; + normalized_box.width = (box.width)/(float)frameDims.width; + normalized_box.height = (box.height)/(float)frameDims.height; normalized_boxes.push_back(normalized_box); } @@ -300,7 +300,7 @@ CVDetectionData CVObjectDetection::GetDetectionData(size_t frameId){ } } -bool CVObjectDetection::SaveTrackedData(){ +bool CVObjectDetection::SaveObjDetectedData(){ // Create tracker message libopenshotobjdetect::ObjDetect objMessage; @@ -346,10 +346,10 @@ void CVObjectDetection::AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrame libopenshotobjdetect::Frame_Box* box = pbFrameData->add_bounding_box(); // Save bounding box data - box->set_x1(dData.boxes.at(i).x); - box->set_y1(dData.boxes.at(i).y); - box->set_x2(dData.boxes.at(i).x + dData.boxes.at(i).width); - box->set_y2(dData.boxes.at(i).y + dData.boxes.at(i).height); + box->set_x(dData.boxes.at(i).x); + box->set_y(dData.boxes.at(i).y); + box->set_w(dData.boxes.at(i).width); + box->set_h(dData.boxes.at(i).height); box->set_classid(dData.classIds.at(i)); box->set_confidence(dData.confidences.at(i)); @@ -394,3 +394,77 @@ void CVObjectDetection::SetJsonValue(const Json::Value root) { classesFile = (root["classes_file"].asString()); } } + + + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||| + ONLY FOR MAKE TEST +|||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + + + +// Load protobuf data file +bool CVObjectDetection::_LoadObjDetectdData(){ + // Create tracker message + libopenshotobjdetect::ObjDetect objMessage; + + { + // Read the existing tracker message. + fstream input(protobuf_data_path, ios::in | ios::binary); + if (!objMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + } + + // Make sure classNames and detectionsData are empty + classNames.clear(); detectionsData.clear(); + + // Get all classes names and assign a color to them + for(int i = 0; i < objMessage.classnames_size(); i++){ + classNames.push_back(objMessage.classnames(i)); + } + + // Iterate over all frames of the saved message + for (size_t i = 0; i < objMessage.frame_size(); i++) { + // Create protobuf message reader + const libopenshotobjdetect::Frame& pbFrameData = objMessage.frame(i); + + // Get frame Id + size_t id = pbFrameData.id(); + + // Load bounding box data + const google::protobuf::RepeatedPtrField &pBox = pbFrameData.bounding_box(); + + // Construct data vectors related to detections in the current frame + std::vector classIds; std::vector confidences; std::vector> boxes; + + for(int i = 0; i < pbFrameData.bounding_box_size(); i++){ + // Get bounding box coordinates + float x = pBox.Get(i).x(); float y = pBox.Get(i).y(); + float w = pBox.Get(i).w(); float h = pBox.Get(i).h(); + // Create OpenCV rectangle with the bouding box info + cv::Rect_ box(x, y, w, h); + + // Get class Id (which will be assign to a class name) and prediction confidence + int classId = pBox.Get(i).classid(); float confidence = pBox.Get(i).confidence(); + + // Push back data into vectors + boxes.push_back(box); classIds.push_back(classId); confidences.push_back(confidence); + } + + // Assign data to object detector map + detectionsData[id] = CVDetectionData(classIds, confidences, boxes, id); + } + + // Show the time stamp from the last update in object detector data file + if (objMessage.has_last_updated()) + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(objMessage.last_updated()) << endl; + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} diff --git a/src/CVStabilization.cpp b/src/CVStabilization.cpp index b723b553..92c74e8e 100644 --- a/src/CVStabilization.cpp +++ b/src/CVStabilization.cpp @@ -35,13 +35,13 @@ CVStabilization::CVStabilization(std::string processInfoJson, ProcessingControll : processingController(&processingController){ SetJson(processInfoJson); } -double mediax=0, mediay=0, mediaa=0, mediastatus=0, maiora = 0, maiorx = 0, maiory = 0; -int maiorstatus=0; // Process clip and store necessary stabilization data void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){ start = _start; end = _end; + // Compute max and average transformation parameters + avr_dx=0; avr_dy=0; avr_da=0; max_dx=0; max_dy=0; max_da=0; video.Open(); @@ -66,12 +66,16 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t cv::Mat cvimage = f->GetImageCV(); cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY); - if(! TrackFrameFeatures(cvimage, frame_number)) + if(!TrackFrameFeatures(cvimage, frame_number)){ prev_to_cur_transform.push_back(TransformParam(0, 0, 0)); + } // Update progress processingController->SetProgress(uint(100*(frame_number-start)/(end-start))); } + // Show average and max transformation parameters + std::cout<<"\nAVERAGE DX: "< trajectory = ComputeFramesTrajectory(); @@ -85,16 +89,15 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t // Track current frame features and find the relative transformation bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ - std::cout<<"frame "< status; std::vector err; // Extract new image features - cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 15); + cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30); // Track features cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err); // Remove untracked features - mediastatus+=status.size(); - if(status.size() > maiorstatus) - maiorstatus = status.size(); - for(size_t i=0; i < status.size(); i++) { if(status[i]) { prev_corner2.push_back(prev_corner[i]); @@ -120,7 +119,7 @@ bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ // In case no feature was detected if(prev_corner2.empty() || cur_corner2.empty()){ last_T = cv::Mat(); - prev_grey = cv::Mat(); + // prev_grey = cv::Mat(); return false; } @@ -128,36 +127,40 @@ bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ cv::Mat T = cv::estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing double da, dx, dy; + // If T has nothing inside return (probably a segment where there is nothing to stabilize) if(T.size().width == 0 || T.size().height == 0){ return false; } else{ - // If no transformation is found, just use the last known good transform. - if(T.data == NULL && !last_T.empty()) - last_T.copyTo(T); + // If no transformation is found, just use the last known good transform + if(T.data == NULL){ + if(!last_T.empty()) + last_T.copyTo(T); + else + return false; + } // Decompose T dx = T.at(0,2); dy = T.at(1,2); da = atan2(T.at(1,0), T.at(0,0)); } - if(dx > 100 || dy > 100 || da > 0.1){ + // Filter transformations parameters, if they are higher than these: return + if(dx > 200 || dy > 200 || da > 0.1){ return false; } - mediax+=fabs(dx); - mediay+=fabs(dy); - mediaa+=fabs(da); - - if(fabs(dx) > maiorx) - maiorx = dx; - if(fabs(dy) > maiory) - maiory = dy; - if(fabs(da) > maiora) - maiora = da; - - std::cout< max_dx) + max_dx = dx; + if(fabs(dy) > max_dy) + max_dy = dy; + if(fabs(da) > max_da) + max_da = da; + T.copyTo(last_T); prev_to_cur_transform.push_back(TransformParam(dx, dy, da)); @@ -166,6 +169,7 @@ bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){ // Show processing info cout << "Frame: " << frameNum << " - good optical flow: " << prev_corner2.size() << endl; + return true; } std::vector CVStabilization::ComputeFramesTrajectory(){ @@ -295,56 +299,6 @@ void CVStabilization::AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameDa pbFrameData->set_dy(transData.dy); } -// Load protobuf data file -bool CVStabilization::LoadStabilizedData(){ - // Create stabilization message - libopenshotstabilize::Stabilization stabilizationMessage; - // Read the existing tracker message. - fstream input(protobuf_data_path, ios::in | ios::binary); - if (!stabilizationMessage.ParseFromIstream(&input)) { - cerr << "Failed to parse protobuf message." << endl; - return false; - } - - // Make sure the data maps are empty - transformationData.clear(); - trajectoryData.clear(); - - // Iterate over all frames of the saved message and assign to the data maps - for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) { - const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i); - - // Load frame number - size_t id = pbFrameData.id(); - - // Load camera trajectory data - float x = pbFrameData.x(); - float y = pbFrameData.y(); - float a = pbFrameData.a(); - - // Assign data to trajectory map - trajectoryData[id] = CamTrajectory(x,y,a); - - // Load transformation data - float dx = pbFrameData.dx(); - float dy = pbFrameData.dy(); - float da = pbFrameData.da(); - - // Assing data to transformation map - transformationData[id] = TransformParam(dx,dy,da); - } - - // Show the time stamp from the last update in stabilization data file - if (stabilizationMessage.has_last_updated()) { - cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; - } - - // Delete all global objects allocated by libprotobuf. - google::protobuf::ShutdownProtobufLibrary(); - - return true; -} - TransformParam CVStabilization::GetTransformParamData(size_t frameId){ // Check if the stabilizer info for the requested frame exists @@ -396,4 +350,64 @@ void CVStabilization::SetJsonValue(const Json::Value root) { if (!root["smoothing_window"].isNull()){ smoothingWindow = (root["smoothing_window"].asInt()); } +} + + + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||| + ONLY FOR MAKE TEST +|||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + + + +// Load protobuf data file +bool CVStabilization::_LoadStabilizedData(){ + // Create stabilization message + libopenshotstabilize::Stabilization stabilizationMessage; + // Read the existing tracker message. + fstream input(protobuf_data_path, ios::in | ios::binary); + if (!stabilizationMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + + // Make sure the data maps are empty + transformationData.clear(); + trajectoryData.clear(); + + // Iterate over all frames of the saved message and assign to the data maps + for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) { + const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i); + + // Load frame number + size_t id = pbFrameData.id(); + + // Load camera trajectory data + float x = pbFrameData.x(); + float y = pbFrameData.y(); + float a = pbFrameData.a(); + + // Assign data to trajectory map + trajectoryData[id] = CamTrajectory(x,y,a); + + // Load transformation data + float dx = pbFrameData.dx(); + float dy = pbFrameData.dy(); + float da = pbFrameData.da(); + + // Assing data to transformation map + transformationData[id] = TransformParam(dx,dy,da); + } + + // Show the time stamp from the last update in stabilization data file + if (stabilizationMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; } \ No newline at end of file diff --git a/src/CVTracker.cpp b/src/CVTracker.cpp index f2493eca..eff9acac 100644 --- a/src/CVTracker.cpp +++ b/src/CVTracker.cpp @@ -222,53 +222,6 @@ void CVTracker::AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, Fram box->set_y2(fData.y2); } -// Load protobuf data file -bool CVTracker::LoadTrackedData(){ - // Create tracker message - libopenshottracker::Tracker trackerMessage; - - { - // Read the existing tracker message. - fstream input(protobuf_data_path, ios::in | ios::binary); - if (!trackerMessage.ParseFromIstream(&input)) { - cerr << "Failed to parse protobuf message." << endl; - return false; - } - } - - // Make sure the trackedData is empty - trackedDataById.clear(); - - // Iterate over all frames of the saved message - for (size_t i = 0; i < trackerMessage.frame_size(); i++) { - const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i); - - // Load frame and rotation data - size_t id = pbFrameData.id(); - float rotation = pbFrameData.rotation(); - - // Load bounding box data - const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box(); - float x1 = box.x1(); - float y1 = box.y1(); - float x2 = box.x2(); - float y2 = box.y2(); - - // Assign data to tracker map - trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2); - } - - // Show the time stamp from the last update in tracker data file - if (trackerMessage.has_last_updated()) { - cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl; - } - - // Delete all global objects allocated by libprotobuf. - google::protobuf::ShutdownProtobufLibrary(); - - return true; -} - // Get tracker info for the desired frame FrameData CVTracker::GetTrackedData(size_t frameId){ @@ -324,3 +277,60 @@ void CVTracker::SetJsonValue(const Json::Value root) { } } + + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||| + ONLY FOR MAKE TEST +|||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + + + +// Load protobuf data file +bool CVTracker::_LoadTrackedData(){ + // Create tracker message + libopenshottracker::Tracker trackerMessage; + + { + // Read the existing tracker message. + fstream input(protobuf_data_path, ios::in | ios::binary); + if (!trackerMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + } + + // Make sure the trackedData is empty + trackedDataById.clear(); + + // Iterate over all frames of the saved message + for (size_t i = 0; i < trackerMessage.frame_size(); i++) { + const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i); + + // Load frame and rotation data + size_t id = pbFrameData.id(); + float rotation = pbFrameData.rotation(); + + // Load bounding box data + const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box(); + float x1 = box.x1(); + float y1 = box.y1(); + float x2 = box.x2(); + float y2 = box.y2(); + + // Assign data to tracker map + trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2); + } + + // Show the time stamp from the last update in tracker data file + if (trackerMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + diff --git a/src/ClipProcessingJobs.cpp b/src/ClipProcessingJobs.cpp index 80c44c0e..8745615a 100644 --- a/src/ClipProcessingJobs.cpp +++ b/src/ClipProcessingJobs.cpp @@ -57,7 +57,7 @@ void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& con } else{ // Save object detection data - objDetector.SaveTrackedData(); + objDetector.SaveObjDetectedData(); // tells to UI that the processing finished controller.SetFinished(true); } diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 00370be8..7906989f 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -102,7 +102,7 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i { // Get the frame's image cv::Mat cv_image = frame->GetImageCV(); - + std::cout<<"Frame number: "< ObjectDetection::GetFrame(std::shared_ptr frame, i cv::Rect_ bb_nrml = detections.boxes.at(i); cv::Rect2d box((int)(bb_nrml.x*fw), (int)(bb_nrml.y*fh), - (int)((bb_nrml.width - bb_nrml.x)*fw), - (int)((bb_nrml.height - bb_nrml.y)*fh)); + (int)(bb_nrml.width*fw), + (int)(bb_nrml.height*fh)); drawPred(detections.classIds.at(i), detections.confidences.at(i), box, cv_image); } @@ -154,6 +154,8 @@ void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& cv::rectangle(frame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine), classesColor[classId], cv::FILLED); putText(frame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1); + std::cout<<"X1: "< &box = pbFrameData.bounding_box(); + const google::protobuf::RepeatedPtrField &pBox = pbFrameData.bounding_box(); + // Construct data vectors related to detections in the current frame std::vector classIds; std::vector confidences; std::vector> boxes; for(int i = 0; i < pbFrameData.bounding_box_size(); i++){ - float x1 = box.Get(i).x1(); - float y1 = box.Get(i).y1(); - float x2 = box.Get(i).x2(); - float y2 = box.Get(i).y2(); - int classId = box.Get(i).classid(); - float confidence = box.Get(i).confidence(); + // Get bounding box coordinates + float x = pBox.Get(i).x(); + float y = pBox.Get(i).y(); + float w = pBox.Get(i).w(); + float h = pBox.Get(i).h(); + // Get class Id (which will be assign to a class name) + int classId = pBox.Get(i).classid(); + // Get prediction confidence + float confidence = pBox.Get(i).confidence(); - cv::Rect_ box(x1, y1, x2-x1, y2-y1); + // Create OpenCV rectangle with the bouding box info + cv::Rect_ box(x, y, w, h); + // Push back data into vectors boxes.push_back(box); classIds.push_back(classId); confidences.push_back(confidence); } - // Assign data to tracker map + // Assign data to object detector map detectionsData[id] = DetectionData(classIds, confidences, boxes, id); } - // Show the time stamp from the last update in tracker data file - if (objMessage.has_last_updated()) { + // Show the time stamp from the last update in object detector data file + if (objMessage.has_last_updated()) cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(objMessage.last_updated()) << endl; - } // Delete all global objects allocated by libprotobuf. google::protobuf::ShutdownProtobufLibrary(); diff --git a/src/examples/Example_opencv.cpp b/src/examples/Example_opencv.cpp index 7a2caf2a..e820708a 100644 --- a/src/examples/Example_opencv.cpp +++ b/src/examples/Example_opencv.cpp @@ -88,15 +88,15 @@ void displayClip(openshot::Clip &r9){ int main(int argc, char* argv[]) { // Set pre-processing effects - bool TRACK_DATA = false; + bool TRACK_DATA = true; bool SMOOTH_VIDEO = false; - bool OBJECT_DETECTION_DATA = true; + bool OBJECT_DETECTION_DATA = false; // Get media path std::stringstream path; path << TEST_MEDIA_PATH << ((OBJECT_DETECTION_DATA) ? "run.mp4" : "test.avi"); - // test_video.mp4 --> Used for object detector - // test.avi --> Used for tracker and stabilizer + // run.mp4 --> Used for object detector + // test.avi --> Used for tracker and stabilizer // Thread controller just for the pre-processing constructors, it won't be used ProcessingController processingController; @@ -164,7 +164,7 @@ int main(int argc, char* argv[]) { // Start the object detection objectDetection.detectObjectsClip(r9, 0, 100, true); // Save the object detection data - objectDetection.SaveTrackedData(); + objectDetection.SaveObjDetectedData(); // Create a object detector effect EffectBase* e = EffectInfo().CreateEffect("Object Detector"); diff --git a/src/protobuf_messages/objdetectdata.proto b/src/protobuf_messages/objdetectdata.proto index 7cd4d79f..c3f540c9 100644 --- a/src/protobuf_messages/objdetectdata.proto +++ b/src/protobuf_messages/objdetectdata.proto @@ -11,10 +11,10 @@ message Frame { int32 id = 1; // Frame ID. message Box{ - float x1 = 1; - float y1 = 2; - float x2 = 3; - float y2 = 4; + float x = 1; + float y = 2; + float w = 3; + float h = 4; int32 classId = 5; float confidence = 6; } diff --git a/tests/CVObjectDetection_Tests.cpp b/tests/CVObjectDetection_Tests.cpp index baf7cabc..9a8e09ba 100644 --- a/tests/CVObjectDetection_Tests.cpp +++ b/tests/CVObjectDetection_Tests.cpp @@ -34,10 +34,17 @@ #define DONT_SET_USING_JUCE_NAMESPACE 1 #include "../include/OpenShot.h" #include "../include/ProcessingController.h" +#include "../include/Json.h" #include using namespace openshot; +std::string effectInfo =(" {\"protobuf_data_path\": \"objdetector.data\", " + " \"processing_device\": \"GPU\", " + " \"model_configuration\": \"~/yolo/yolov3.cfg\", " + " \"model_weights\": \"~/yolo/yolov3.weights\", " + " \"classes_file\": \"~/yolo/obj.names\"} "); + SUITE(CVObjectDetection_Tests) { @@ -48,24 +55,32 @@ SUITE(CVObjectDetection_Tests) { // Create a video clip std::stringstream path; - path << TEST_MEDIA_PATH << "test_video.mp4"; + path << TEST_MEDIA_PATH << "run.mp4"; // Open clip openshot::Clip c1(path.str()); c1.Open(); - CVObjectDetection objectDetector("\"processing_device\": \"GPU\"", processingController); + //TODO remove hardcoded path + CVObjectDetection objectDetector(effectInfo, processingController); - objectDetector.detectObjectsClip(c1, 0, 100, true); + objectDetector.detectObjectsClip(c1, 0, 20, true); CVDetectionData dd = objectDetector.GetDetectionData(20); - // int x1 = dd.boxes[20].x; - // int y1 = dd.boxes[20].y; - // int x2 = x1 + dd.boxes[20].width(); - // int y2 = y2 + dd.boxes[20].height(); - // float confidence = dd.confidences[20]; - // int classId = dd.classIds[20]; + float x1 = dd.boxes.at(20).x; + float y1 = dd.boxes.at(20).y; + float x2 = x1 + dd.boxes.at(20).width; + float y2 = y1 + dd.boxes.at(20).height; + float confidence = dd.confidences.at(20); + int classId = dd.classIds.at(20); + + CHECK_EQUAL((int) (x1 * 720), 106); + CHECK_EQUAL((int) (y1 * 400), 21); + CHECK_EQUAL((int) (x2 * 720), 628); + CHECK_EQUAL((int) (y2 * 400), 429); + CHECK_EQUAL((int) (confidence * 1000), 554); + CHECK_EQUAL(classId, 0); } @@ -75,39 +90,47 @@ SUITE(CVObjectDetection_Tests) // Create a video clip std::stringstream path; - path << TEST_MEDIA_PATH << "test_video.mp4"; + path << TEST_MEDIA_PATH << "run.mp4"; // Open clip openshot::Clip c1(path.str()); c1.Open(); - CVObjectDetection objectDetector_1("{\"protobuf_data_path\": \"object_detector.data\", \"processing_device\": \"GPU\"}", processingController); + //TODO remove hardcoded path + CVObjectDetection objectDetector_1(effectInfo ,processingController); - objectDetector_1.detectObjectsClip(c1, 0, 100, true); + objectDetector_1.detectObjectsClip(c1, 0, 20, true); CVDetectionData dd_1 = objectDetector_1.GetDetectionData(20); - objectDetector_1.SaveTrackedData(); + float x1_1 = dd_1.boxes.at(20).x; + float y1_1 = dd_1.boxes.at(20).y; + float x2_1 = x1_1 + dd_1.boxes.at(20).width; + float y2_1 = y1_1 + dd_1.boxes.at(20).height; + float confidence_1 = dd_1.confidences.at(20); + int classId_1 = dd_1.classIds.at(20); - CVObjectDetection objectDetector_2("{\"protobuf_data_path\": \"object_detector.data\", \"processing_device\": \"\"}", processingController); + objectDetector_1.SaveObjDetectedData(); - // objectDetector_2.LoadTrackedData(); + CVObjectDetection objectDetector_2(effectInfo, processingController); + + objectDetector_2._LoadObjDetectdData(); CVDetectionData dd_2 = objectDetector_2.GetDetectionData(20); - - // int x1_1 = dd_1.boxes[20].x; - // int y1_1 = dd_1.boxes[20].y; - // int x2_1 = x1_1 + dd_1.boxes[20].width(); - // int y2_1 = y2_1 + dd_1.boxes[20].height(); - // float confidence_1 = dd_1.confidences[20]; - // int classId_1 = dd_1.classIds[20]; - // int x1_2 = dd_2.boxes[20].x; - // int y1_2 = dd_2.boxes[20].y; - // int x2_2 = x1_2 + dd_2.boxes[20].width(); - // int y2_2 = y2_2 + dd_2.boxes[20].height(); - // float confidence_2 = dd_2.confidences[20]; - // int classId_2 = dd_2.classIds[20]; + float x1_2 = dd_2.boxes.at(20).x; + float y1_2 = dd_2.boxes.at(20).y; + float x2_2 = x1_2 + dd_2.boxes.at(20).width; + float y2_2 = y1_2 + dd_2.boxes.at(20).height; + float confidence_2 = dd_2.confidences.at(20); + int classId_2 = dd_2.classIds.at(20); + + CHECK_EQUAL((int) (x1_1 * 720), (int) (x1_2 * 720)); + CHECK_EQUAL((int) (y1_1 * 400), (int) (y1_2 * 400)); + CHECK_EQUAL((int) (x2_1 * 720), (int) (x2_2 * 720)); + CHECK_EQUAL((int) (y2_1 * 400), (int) (y2_2 * 400)); + CHECK_EQUAL((int) (confidence_1 * 1000), (int) (confidence_2 * 1000)); + CHECK_EQUAL(classId_1, classId_2); } diff --git a/tests/CVStabilizer_Tests.cpp b/tests/CVStabilizer_Tests.cpp index 016c25b7..8680ebe1 100644 --- a/tests/CVStabilizer_Tests.cpp +++ b/tests/CVStabilizer_Tests.cpp @@ -109,7 +109,7 @@ SUITE(CVStabilizer_Tests) CVStabilization stabilizer_2("{\"protobuf_data_path\": \"stabilizer.data\", \"smoothing_window\": 30}", processingController); // Load stabilized data from first stabilizer protobuf data - stabilizer_2.LoadStabilizedData(); + stabilizer_2._LoadStabilizedData(); // Get stabilized data TransformParam tp_2 = stabilizer_2.GetTransformParamData(20); diff --git a/tests/CVTracker_Tests.cpp b/tests/CVTracker_Tests.cpp index 44a3e453..ff89550c 100644 --- a/tests/CVTracker_Tests.cpp +++ b/tests/CVTracker_Tests.cpp @@ -63,16 +63,16 @@ SUITE(CVTracker_Tests) // Get tracked data FrameData fd = kcfTracker.GetTrackedData(20); - int x = fd.x1; - int y = fd.y1; - int width = fd.x2-fd.x1; - int height = fd.y2-fd.y1; + float x = fd.x1; + float y = fd.y1; + float width = fd.x2 - x; + float height = fd.y2 - y; // Compare if tracked data is equal to pre-tested ones - CHECK_EQUAL(259, x); - CHECK_EQUAL(131, y); - CHECK_EQUAL(180, width); - CHECK_EQUAL(166, height); + CHECK_EQUAL(259, (int)(x * 640)); + CHECK_EQUAL(131, (int)(y * 360)); + CHECK_EQUAL(180, (int)(width * 640)); + CHECK_EQUAL(166, (int)(height * 360)); } @@ -96,10 +96,10 @@ SUITE(CVTracker_Tests) // Get tracked data FrameData fd_1 = kcfTracker_1.GetTrackedData(20); - int x_1 = fd_1.x1; - int y_1 = fd_1.y1; - int width_1 = fd_1.x2-fd_1.x1; - int height_1 = fd_1.y2-fd_1.y1; + float x_1 = fd_1.x1; + float y_1 = fd_1.y1; + float width_1 = fd_1.x2 - x_1; + float height_1 = fd_1.y2 - y_1; // Save tracked data kcfTracker_1.SaveTrackedData(); @@ -108,21 +108,21 @@ SUITE(CVTracker_Tests) CVTracker kcfTracker_2("{\"protobuf_data_path\": \"kcf_tracker.data\", \"tracker_type\": \"\", \"bbox\": {\"x\": -1, \"y\": -1, \"w\": -1, \"h\": -1}}", processingController); // Load tracked data from first tracker protobuf data - kcfTracker_2.LoadTrackedData(); + kcfTracker_2._LoadTrackedData(); // Get tracked data FrameData fd_2 = kcfTracker_2.GetTrackedData(20); - int x_2 = fd_2.x1; - int y_2 = fd_2.y1; - int width_2 = fd_2.x2-fd_2.x1; - int height_2 = fd_2.y2-fd_2.y1; + float x_2 = fd_2.x1; + float y_2 = fd_2.y1; + float width_2 = fd_2.x2 - x_2; + float height_2 = fd_2.y2 - y_2; // Compare first tracker data with second tracker data - CHECK_EQUAL(x_1, x_2); - CHECK_EQUAL(y_1, y_2); - CHECK_EQUAL(width_1, width_2); - CHECK_EQUAL(height_1, height_2); + CHECK_EQUAL((int)(x_1 * 640), (int)(x_2 * 640)); + CHECK_EQUAL((int)(y_1 * 360), (int)(y_2 * 360)); + CHECK_EQUAL((int)(width_1 * 640), (int)(width_2 * 640)); + CHECK_EQUAL((int)(height_1 * 360), (int)(height_2 * 360)); } } // SUITE(Frame_Tests)