openshot/libopenshot
0
0
Fork 0
mirror of https://github.com/OpenShot/libopenshot.git synced 2026-03-02 08:53:52 -08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed Stabilizer and created Object Detector tests

Browse Source
This commit is contained in:
Brenno
2020-07-30 21:39:20 -03:00
parent 58d2e8fe07
commit 6ca35bbf9b
13 changed files with 351 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/CVObjectDetection.h
View File

@@ -110,13 +110,14 @@ class CVObjectDetection{
/// Protobuf Save and Load methods
// Save protobuf file
bool SaveTrackedData();
bool SaveObjDetectedData();
// Add frame object detection data into protobuf message.
void AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrameData, CVDetectionData& dData);
// Load protobuf file
bool LoadTrackedData();
/// Get and Set JSON methods
void SetJson(const std::string value); ///< Load JSON string into this object
void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
// Load protobuf file (ONLY FOR MAKE TEST)
bool _LoadObjDetectdData();
};

5
include/CVStabilization.h
View File

@@ -84,6 +84,7 @@ class CVStabilization {
size_t start;
size_t end;
double avr_dx, avr_dy, avr_da, max_dx, max_dy, max_da;
cv::Mat last_T;
cv::Mat prev_grey;
@@ -120,8 +121,6 @@ class CVStabilization {
bool SaveStabilizedData();
// Add frame stabilization data into protobuf message
void AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameData, CamTrajectory& trajData, TransformParam& transData, size_t frame_number);
// Load protobuf data file
bool LoadStabilizedData();
// Return requested struct info for a given frame
TransformParam GetTransformParamData(size_t frameId);
@@ -131,6 +130,8 @@ class CVStabilization {
void SetJson(const std::string value); ///< Load JSON string into this object
void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
// Load protobuf data file (ONLY FOR MAKE TEST)
bool _LoadStabilizedData();
};
#endif

5
include/CVTracker.h
View File

@@ -98,12 +98,13 @@ class CVTracker {
bool SaveTrackedData();
// Add frame tracked data into protobuf message.
void AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, FrameData& fData);
// Load protobuf file
bool LoadTrackedData();
/// Get and Set JSON methods
void SetJson(const std::string value); ///< Load JSON string into this object
void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
// Load protobuf file (ONLY FOR MAKE TEST)
bool _LoadTrackedData();
};

90
src/CVObjectDetection.cpp
View File

@@ -118,7 +118,7 @@ void CVObjectDetection::DetectObjects(const cv::Mat &frame, size_t frameId){
// Runs the forward pass to get output of the output layers
std::vector<cv::Mat> outs;
net.forward(outs, getOutputsNames(net));
// Remove the bounding boxes with low confidence
postprocess(frame.size(), outs, frameId);
@@ -239,8 +239,8 @@ void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector
cv::Rect_<float> normalized_box;
normalized_box.x = (box.x)/(float)frameDims.width;
normalized_box.y = (box.y)/(float)frameDims.height;
normalized_box.width = (box.x+box.width)/(float)frameDims.width;
normalized_box.height = (box.y+box.height)/(float)frameDims.height;
normalized_box.width = (box.width)/(float)frameDims.width;
normalized_box.height = (box.height)/(float)frameDims.height;
normalized_boxes.push_back(normalized_box);
}
@@ -300,7 +300,7 @@ CVDetectionData CVObjectDetection::GetDetectionData(size_t frameId){
}
}
bool CVObjectDetection::SaveTrackedData(){
bool CVObjectDetection::SaveObjDetectedData(){
// Create tracker message
libopenshotobjdetect::ObjDetect objMessage;
@@ -346,10 +346,10 @@ void CVObjectDetection::AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrame
libopenshotobjdetect::Frame_Box* box = pbFrameData->add_bounding_box();
// Save bounding box data
box->set_x1(dData.boxes.at(i).x);
box->set_y1(dData.boxes.at(i).y);
box->set_x2(dData.boxes.at(i).x + dData.boxes.at(i).width);
box->set_y2(dData.boxes.at(i).y + dData.boxes.at(i).height);
box->set_x(dData.boxes.at(i).x);
box->set_y(dData.boxes.at(i).y);
box->set_w(dData.boxes.at(i).width);
box->set_h(dData.boxes.at(i).height);
box->set_classid(dData.classIds.at(i));
box->set_confidence(dData.confidences.at(i));
@@ -394,3 +394,77 @@ void CVObjectDetection::SetJsonValue(const Json::Value root) {
classesFile = (root["classes_file"].asString());
}
}
/*
||||||||||||||||||||||||||||||||||||||||||||||||||
ONLY FOR MAKE TEST
||||||||||||||||||||||||||||||||||||||||||||||||||
*/
// Load protobuf data file
bool CVObjectDetection::_LoadObjDetectdData(){
// Create tracker message
libopenshotobjdetect::ObjDetect objMessage;
{
// Read the existing tracker message.
fstream input(protobuf_data_path, ios::in | ios::binary);
if (!objMessage.ParseFromIstream(&input)) {
cerr << "Failed to parse protobuf message." << endl;
return false;
}
}
// Make sure classNames and detectionsData are empty
classNames.clear(); detectionsData.clear();
// Get all classes names and assign a color to them
for(int i = 0; i < objMessage.classnames_size(); i++){
classNames.push_back(objMessage.classnames(i));
}
// Iterate over all frames of the saved message
for (size_t i = 0; i < objMessage.frame_size(); i++) {
// Create protobuf message reader
const libopenshotobjdetect::Frame& pbFrameData = objMessage.frame(i);
// Get frame Id
size_t id = pbFrameData.id();
// Load bounding box data
const google::protobuf::RepeatedPtrField<libopenshotobjdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
// Construct data vectors related to detections in the current frame
std::vector<int> classIds; std::vector<float> confidences; std::vector<cv::Rect_<float>> boxes;
for(int i = 0; i < pbFrameData.bounding_box_size(); i++){
// Get bounding box coordinates
float x = pBox.Get(i).x(); float y = pBox.Get(i).y();
float w = pBox.Get(i).w(); float h = pBox.Get(i).h();
// Create OpenCV rectangle with the bouding box info
cv::Rect_<float> box(x, y, w, h);
// Get class Id (which will be assign to a class name) and prediction confidence
int classId = pBox.Get(i).classid(); float confidence = pBox.Get(i).confidence();
// Push back data into vectors
boxes.push_back(box); classIds.push_back(classId); confidences.push_back(confidence);
}
// Assign data to object detector map
detectionsData[id] = CVDetectionData(classIds, confidences, boxes, id);
}
// Show the time stamp from the last update in object detector data file
if (objMessage.has_last_updated())
cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(objMessage.last_updated()) << endl;
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();
return true;
}

174
src/CVStabilization.cpp
View File

@@ -35,13 +35,13 @@ CVStabilization::CVStabilization(std::string processInfoJson, ProcessingControll
: processingController(&processingController){
SetJson(processInfoJson);
}
double mediax=0, mediay=0, mediaa=0, mediastatus=0, maiora = 0, maiorx = 0, maiory = 0;
int maiorstatus=0;
// Process clip and store necessary stabilization data
void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){
start = _start; end = _end;
// Compute max and average transformation parameters
avr_dx=0; avr_dy=0; avr_da=0; max_dx=0; max_dy=0; max_da=0;
video.Open();
@@ -66,12 +66,16 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t
cv::Mat cvimage = f->GetImageCV();
cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2GRAY);
if(! TrackFrameFeatures(cvimage, frame_number))
if(!TrackFrameFeatures(cvimage, frame_number)){
prev_to_cur_transform.push_back(TransformParam(0, 0, 0));
}
// Update progress
processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
}
// Show average and max transformation parameters
std::cout<<"\nAVERAGE DX: "<<avr_dx/(frame_number-1)<<" AVERAGE DY: "<<avr_dy/(frame_number-1)<<" AVERAGE A: "<<avr_da/(frame_number-1)<<"\n";
std::cout<<"MAX X: "<<max_dx<<" MAX Y: "<<max_dy<<" MAX A: "<<max_da<<"\n\n";
// Calculate trajectory data
std::vector <CamTrajectory> trajectory = ComputeFramesTrajectory();
@@ -85,16 +89,15 @@ void CVStabilization::stabilizeClip(openshot::Clip& video, size_t _start, size_t
// Track current frame features and find the relative transformation
bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
std::cout<<"frame "<<frameNum<<"\n";
// Check if there are black frames
if(cv::countNonZero(frame) < 1){
// last_T = cv::Mat();
// prev_grey = cv::Mat();
return false;
}
// Initialize prev_grey if not
if(prev_grey.empty()){
prev_grey = frame;
return false;
return true;
}
// OpticalFlow features vector
@@ -103,14 +106,10 @@ bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
std::vector <uchar> status;
std::vector <float> err;
// Extract new image features
cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 15);
cv::goodFeaturesToTrack(prev_grey, prev_corner, 200, 0.01, 30);
// Track features
cv::calcOpticalFlowPyrLK(prev_grey, frame, prev_corner, cur_corner, status, err);
// Remove untracked features
mediastatus+=status.size();
if(status.size() > maiorstatus)
maiorstatus = status.size();
for(size_t i=0; i < status.size(); i++) {
if(status[i]) {
prev_corner2.push_back(prev_corner[i]);
@@ -120,7 +119,7 @@ bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
// In case no feature was detected
if(prev_corner2.empty() || cur_corner2.empty()){
last_T = cv::Mat();
prev_grey = cv::Mat();
// prev_grey = cv::Mat();
return false;
}
@@ -128,36 +127,40 @@ bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
cv::Mat T = cv::estimateAffinePartial2D(prev_corner2, cur_corner2); // false = rigid transform, no scaling/shearing
double da, dx, dy;
// If T has nothing inside return (probably a segment where there is nothing to stabilize)
if(T.size().width == 0 || T.size().height == 0){
return false;
}
else{
// If no transformation is found, just use the last known good transform.
if(T.data == NULL && !last_T.empty())
last_T.copyTo(T);
// If no transformation is found, just use the last known good transform
if(T.data == NULL){
if(!last_T.empty())
last_T.copyTo(T);
else
return false;
}
// Decompose T
dx = T.at<double>(0,2);
dy = T.at<double>(1,2);
da = atan2(T.at<double>(1,0), T.at<double>(0,0));
}
if(dx > 100 || dy > 100 || da > 0.1){
// Filter transformations parameters, if they are higher than these: return
if(dx > 200 || dy > 200 || da > 0.1){
return false;
}
mediax+=fabs(dx);
mediay+=fabs(dy);
mediaa+=fabs(da);
if(fabs(dx) > maiorx)
maiorx = dx;
if(fabs(dy) > maiory)
maiory = dy;
if(fabs(da) > maiora)
maiora = da;
std::cout<<dx<<" "<<dy<<" "<<da<<"\n";
// Keep computing average and max transformation parameters
avr_dx+=fabs(dx);
avr_dy+=fabs(dy);
avr_da+=fabs(da);
if(fabs(dx) > max_dx)
max_dx = dx;
if(fabs(dy) > max_dy)
max_dy = dy;
if(fabs(da) > max_da)
max_da = da;
T.copyTo(last_T);
prev_to_cur_transform.push_back(TransformParam(dx, dy, da));
@@ -166,6 +169,7 @@ bool CVStabilization::TrackFrameFeatures(cv::Mat frame, size_t frameNum){
// Show processing info
cout << "Frame: " << frameNum << " - good optical flow: " << prev_corner2.size() << endl;
return true;
}
std::vector<CamTrajectory> CVStabilization::ComputeFramesTrajectory(){
@@ -295,56 +299,6 @@ void CVStabilization::AddFrameDataToProto(libopenshotstabilize::Frame* pbFrameDa
pbFrameData->set_dy(transData.dy);
}
// Load protobuf data file
bool CVStabilization::LoadStabilizedData(){
// Create stabilization message
libopenshotstabilize::Stabilization stabilizationMessage;
// Read the existing tracker message.
fstream input(protobuf_data_path, ios::in | ios::binary);
if (!stabilizationMessage.ParseFromIstream(&input)) {
cerr << "Failed to parse protobuf message." << endl;
return false;
}
// Make sure the data maps are empty
transformationData.clear();
trajectoryData.clear();
// Iterate over all frames of the saved message and assign to the data maps
for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) {
const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i);
// Load frame number
size_t id = pbFrameData.id();
// Load camera trajectory data
float x = pbFrameData.x();
float y = pbFrameData.y();
float a = pbFrameData.a();
// Assign data to trajectory map
trajectoryData[id] = CamTrajectory(x,y,a);
// Load transformation data
float dx = pbFrameData.dx();
float dy = pbFrameData.dy();
float da = pbFrameData.da();
// Assing data to transformation map
transformationData[id] = TransformParam(dx,dy,da);
}
// Show the time stamp from the last update in stabilization data file
if (stabilizationMessage.has_last_updated()) {
cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl;
}
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();
return true;
}
TransformParam CVStabilization::GetTransformParamData(size_t frameId){
// Check if the stabilizer info for the requested frame exists
@@ -396,4 +350,64 @@ void CVStabilization::SetJsonValue(const Json::Value root) {
if (!root["smoothing_window"].isNull()){
smoothingWindow = (root["smoothing_window"].asInt());
}
}
/*
||||||||||||||||||||||||||||||||||||||||||||||||||
ONLY FOR MAKE TEST
||||||||||||||||||||||||||||||||||||||||||||||||||
*/
// Load protobuf data file
bool CVStabilization::_LoadStabilizedData(){
// Create stabilization message
libopenshotstabilize::Stabilization stabilizationMessage;
// Read the existing tracker message.
fstream input(protobuf_data_path, ios::in | ios::binary);
if (!stabilizationMessage.ParseFromIstream(&input)) {
cerr << "Failed to parse protobuf message." << endl;
return false;
}
// Make sure the data maps are empty
transformationData.clear();
trajectoryData.clear();
// Iterate over all frames of the saved message and assign to the data maps
for (size_t i = 0; i < stabilizationMessage.frame_size(); i++) {
const libopenshotstabilize::Frame& pbFrameData = stabilizationMessage.frame(i);
// Load frame number
size_t id = pbFrameData.id();
// Load camera trajectory data
float x = pbFrameData.x();
float y = pbFrameData.y();
float a = pbFrameData.a();
// Assign data to trajectory map
trajectoryData[id] = CamTrajectory(x,y,a);
// Load transformation data
float dx = pbFrameData.dx();
float dy = pbFrameData.dy();
float da = pbFrameData.da();
// Assing data to transformation map
transformationData[id] = TransformParam(dx,dy,da);
}
// Show the time stamp from the last update in stabilization data file
if (stabilizationMessage.has_last_updated()) {
cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(stabilizationMessage.last_updated()) << endl;
}
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();
return true;
}

104
src/CVTracker.cpp
View File

@@ -222,53 +222,6 @@ void CVTracker::AddFrameDataToProto(libopenshottracker::Frame* pbFrameData, Fram
box->set_y2(fData.y2);
}
// Load protobuf data file
bool CVTracker::LoadTrackedData(){
// Create tracker message
libopenshottracker::Tracker trackerMessage;
{
// Read the existing tracker message.
fstream input(protobuf_data_path, ios::in | ios::binary);
if (!trackerMessage.ParseFromIstream(&input)) {
cerr << "Failed to parse protobuf message." << endl;
return false;
}
}
// Make sure the trackedData is empty
trackedDataById.clear();
// Iterate over all frames of the saved message
for (size_t i = 0; i < trackerMessage.frame_size(); i++) {
const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i);
// Load frame and rotation data
size_t id = pbFrameData.id();
float rotation = pbFrameData.rotation();
// Load bounding box data
const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box();
float x1 = box.x1();
float y1 = box.y1();
float x2 = box.x2();
float y2 = box.y2();
// Assign data to tracker map
trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2);
}
// Show the time stamp from the last update in tracker data file
if (trackerMessage.has_last_updated()) {
cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl;
}
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();
return true;
}
// Get tracker info for the desired frame
FrameData CVTracker::GetTrackedData(size_t frameId){
@@ -324,3 +277,60 @@ void CVTracker::SetJsonValue(const Json::Value root) {
}
}
/*
||||||||||||||||||||||||||||||||||||||||||||||||||
ONLY FOR MAKE TEST
||||||||||||||||||||||||||||||||||||||||||||||||||
*/
// Load protobuf data file
bool CVTracker::_LoadTrackedData(){
// Create tracker message
libopenshottracker::Tracker trackerMessage;
{
// Read the existing tracker message.
fstream input(protobuf_data_path, ios::in | ios::binary);
if (!trackerMessage.ParseFromIstream(&input)) {
cerr << "Failed to parse protobuf message." << endl;
return false;
}
}
// Make sure the trackedData is empty
trackedDataById.clear();
// Iterate over all frames of the saved message
for (size_t i = 0; i < trackerMessage.frame_size(); i++) {
const libopenshottracker::Frame& pbFrameData = trackerMessage.frame(i);
// Load frame and rotation data
size_t id = pbFrameData.id();
float rotation = pbFrameData.rotation();
// Load bounding box data
const libopenshottracker::Frame::Box& box = pbFrameData.bounding_box();
float x1 = box.x1();
float y1 = box.y1();
float x2 = box.x2();
float y2 = box.y2();
// Assign data to tracker map
trackedDataById[id] = FrameData(id, rotation, x1, y1, x2, y2);
}
// Show the time stamp from the last update in tracker data file
if (trackerMessage.has_last_updated()) {
cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(trackerMessage.last_updated()) << endl;
}
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();
return true;
}

2
src/ClipProcessingJobs.cpp
View File

@@ -57,7 +57,7 @@ void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& con
}
else{
// Save object detection data
objDetector.SaveTrackedData();
objDetector.SaveObjDetectedData();
// tells to UI that the processing finished
controller.SetFinished(true);
}

42
src/effects/ObjectDetection.cpp
View File

@@ -102,7 +102,7 @@ std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, i
{
// Get the frame's image
cv::Mat cv_image = frame->GetImageCV();
std::cout<<"Frame number: "<<frame_number<<"\n\n";
// Check if frame isn't NULL
if(!cv_image.empty()){
@@ -116,8 +116,8 @@ std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, i
cv::Rect_<float> bb_nrml = detections.boxes.at(i);
cv::Rect2d box((int)(bb_nrml.x*fw),
(int)(bb_nrml.y*fh),
(int)((bb_nrml.width - bb_nrml.x)*fw),
(int)((bb_nrml.height - bb_nrml.y)*fh));
(int)(bb_nrml.width*fw),
(int)(bb_nrml.height*fh));
drawPred(detections.classIds.at(i), detections.confidences.at(i),
box, cv_image);
}
@@ -154,6 +154,8 @@ void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat&
cv::rectangle(frame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine), classesColor[classId], cv::FILLED);
putText(frame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
std::cout<<"X1: "<<box.x<<" Y1: "<<box.y<<" X2: "<<box.width + box.x<<" Y2: "<<box.height + box.y<<"\n";
std::cout<<"Class: "<<label<<"\n\n";
}
// Load protobuf data file
@@ -170,7 +172,7 @@ bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){
}
}
// Make sure the trackedData is empty
// Make sure classNames and detectionsData are empty
classNames.clear();
detectionsData.clear();
@@ -184,41 +186,47 @@ bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){
// Iterate over all frames of the saved message
for (size_t i = 0; i < objMessage.frame_size(); i++) {
// Create protobuf message reader
const libopenshotobjdetect::Frame& pbFrameData = objMessage.frame(i);
// Load frame and rotation data
// Get frame Id
size_t id = pbFrameData.id();
// Load bounding box data
const google::protobuf::RepeatedPtrField<libopenshotobjdetect::Frame_Box > &box = pbFrameData.bounding_box();
const google::protobuf::RepeatedPtrField<libopenshotobjdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
// Construct data vectors related to detections in the current frame
std::vector<int> classIds;
std::vector<float> confidences;
std::vector<cv::Rect_<float>> boxes;
for(int i = 0; i < pbFrameData.bounding_box_size(); i++){
float x1 = box.Get(i).x1();
float y1 = box.Get(i).y1();
float x2 = box.Get(i).x2();
float y2 = box.Get(i).y2();
int classId = box.Get(i).classid();
float confidence = box.Get(i).confidence();
// Get bounding box coordinates
float x = pBox.Get(i).x();
float y = pBox.Get(i).y();
float w = pBox.Get(i).w();
float h = pBox.Get(i).h();
// Get class Id (which will be assign to a class name)
int classId = pBox.Get(i).classid();
// Get prediction confidence
float confidence = pBox.Get(i).confidence();
cv::Rect_<float> box(x1, y1, x2-x1, y2-y1);
// Create OpenCV rectangle with the bouding box info
cv::Rect_<float> box(x, y, w, h);
// Push back data into vectors
boxes.push_back(box);
classIds.push_back(classId);
confidences.push_back(confidence);
}
// Assign data to tracker map
// Assign data to object detector map
detectionsData[id] = DetectionData(classIds, confidences, boxes, id);
}
// Show the time stamp from the last update in tracker data file
if (objMessage.has_last_updated()) {
// Show the time stamp from the last update in object detector data file
if (objMessage.has_last_updated())
cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(objMessage.last_updated()) << endl;
}
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();

10
src/examples/Example_opencv.cpp
View File

@@ -88,15 +88,15 @@ void displayClip(openshot::Clip &r9){
int main(int argc, char* argv[]) {
// Set pre-processing effects
bool TRACK_DATA = false;
bool TRACK_DATA = true;
bool SMOOTH_VIDEO = false;
bool OBJECT_DETECTION_DATA = true;
bool OBJECT_DETECTION_DATA = false;
// Get media path
std::stringstream path;
path << TEST_MEDIA_PATH << ((OBJECT_DETECTION_DATA) ? "run.mp4" : "test.avi");
// test_video.mp4 --> Used for object detector
// test.avi --> Used for tracker and stabilizer
// run.mp4 --> Used for object detector
// test.avi --> Used for tracker and stabilizer
// Thread controller just for the pre-processing constructors, it won't be used
ProcessingController processingController;
@@ -164,7 +164,7 @@ int main(int argc, char* argv[]) {
// Start the object detection
objectDetection.detectObjectsClip(r9, 0, 100, true);
// Save the object detection data
objectDetection.SaveTrackedData();
objectDetection.SaveObjDetectedData();
// Create a object detector effect
EffectBase* e = EffectInfo().CreateEffect("Object Detector");

8
src/protobuf_messages/objdetectdata.proto
View File

@@ -11,10 +11,10 @@ message Frame {
int32 id = 1; // Frame ID.
message Box{
float x1 = 1;
float y1 = 2;
float x2 = 3;
float y2 = 4;
float x = 1;
float y = 2;
float w = 3;
float h = 4;
int32 classId = 5;
float confidence = 6;
}

79
tests/CVObjectDetection_Tests.cpp
View File

@@ -34,10 +34,17 @@
#define DONT_SET_USING_JUCE_NAMESPACE 1
#include "../include/OpenShot.h"
#include "../include/ProcessingController.h"
#include "../include/Json.h"
#include <QImage>
using namespace openshot;
std::string effectInfo =(" {\"protobuf_data_path\": \"objdetector.data\", "
" \"processing_device\": \"GPU\", "
" \"model_configuration\": \"~/yolo/yolov3.cfg\", "
" \"model_weights\": \"~/yolo/yolov3.weights\", "
" \"classes_file\": \"~/yolo/obj.names\"} ");
SUITE(CVObjectDetection_Tests)
{
@@ -48,24 +55,32 @@ SUITE(CVObjectDetection_Tests)
{
// Create a video clip
std::stringstream path;
path << TEST_MEDIA_PATH << "test_video.mp4";
path << TEST_MEDIA_PATH << "run.mp4";
// Open clip
openshot::Clip c1(path.str());
c1.Open();
CVObjectDetection objectDetector("\"processing_device\": \"GPU\"", processingController);
//TODO remove hardcoded path
CVObjectDetection objectDetector(effectInfo, processingController);
objectDetector.detectObjectsClip(c1, 0, 100, true);
objectDetector.detectObjectsClip(c1, 0, 20, true);
CVDetectionData dd = objectDetector.GetDetectionData(20);
// int x1 = dd.boxes[20].x;
// int y1 = dd.boxes[20].y;
// int x2 = x1 + dd.boxes[20].width();
// int y2 = y2 + dd.boxes[20].height();
// float confidence = dd.confidences[20];
// int classId = dd.classIds[20];
float x1 = dd.boxes.at(20).x;
float y1 = dd.boxes.at(20).y;
float x2 = x1 + dd.boxes.at(20).width;
float y2 = y1 + dd.boxes.at(20).height;
float confidence = dd.confidences.at(20);
int classId = dd.classIds.at(20);
CHECK_EQUAL((int) (x1 * 720), 106);
CHECK_EQUAL((int) (y1 * 400), 21);
CHECK_EQUAL((int) (x2 * 720), 628);
CHECK_EQUAL((int) (y2 * 400), 429);
CHECK_EQUAL((int) (confidence * 1000), 554);
CHECK_EQUAL(classId, 0);
}
@@ -75,39 +90,47 @@ SUITE(CVObjectDetection_Tests)
// Create a video clip
std::stringstream path;
path << TEST_MEDIA_PATH << "test_video.mp4";
path << TEST_MEDIA_PATH << "run.mp4";
// Open clip
openshot::Clip c1(path.str());
c1.Open();
CVObjectDetection objectDetector_1("{\"protobuf_data_path\": \"object_detector.data\", \"processing_device\": \"GPU\"}", processingController);
//TODO remove hardcoded path
CVObjectDetection objectDetector_1(effectInfo ,processingController);
objectDetector_1.detectObjectsClip(c1, 0, 100, true);
objectDetector_1.detectObjectsClip(c1, 0, 20, true);
CVDetectionData dd_1 = objectDetector_1.GetDetectionData(20);
objectDetector_1.SaveTrackedData();
float x1_1 = dd_1.boxes.at(20).x;
float y1_1 = dd_1.boxes.at(20).y;
float x2_1 = x1_1 + dd_1.boxes.at(20).width;
float y2_1 = y1_1 + dd_1.boxes.at(20).height;
float confidence_1 = dd_1.confidences.at(20);
int classId_1 = dd_1.classIds.at(20);
CVObjectDetection objectDetector_2("{\"protobuf_data_path\": \"object_detector.data\", \"processing_device\": \"\"}", processingController);
objectDetector_1.SaveObjDetectedData();
// objectDetector_2.LoadTrackedData();
CVObjectDetection objectDetector_2(effectInfo, processingController);
objectDetector_2._LoadObjDetectdData();
CVDetectionData dd_2 = objectDetector_2.GetDetectionData(20);
// int x1_1 = dd_1.boxes[20].x;
// int y1_1 = dd_1.boxes[20].y;
// int x2_1 = x1_1 + dd_1.boxes[20].width();
// int y2_1 = y2_1 + dd_1.boxes[20].height();
// float confidence_1 = dd_1.confidences[20];
// int classId_1 = dd_1.classIds[20];
// int x1_2 = dd_2.boxes[20].x;
// int y1_2 = dd_2.boxes[20].y;
// int x2_2 = x1_2 + dd_2.boxes[20].width();
// int y2_2 = y2_2 + dd_2.boxes[20].height();
// float confidence_2 = dd_2.confidences[20];
// int classId_2 = dd_2.classIds[20];
float x1_2 = dd_2.boxes.at(20).x;
float y1_2 = dd_2.boxes.at(20).y;
float x2_2 = x1_2 + dd_2.boxes.at(20).width;
float y2_2 = y1_2 + dd_2.boxes.at(20).height;
float confidence_2 = dd_2.confidences.at(20);
int classId_2 = dd_2.classIds.at(20);
CHECK_EQUAL((int) (x1_1 * 720), (int) (x1_2 * 720));
CHECK_EQUAL((int) (y1_1 * 400), (int) (y1_2 * 400));
CHECK_EQUAL((int) (x2_1 * 720), (int) (x2_2 * 720));
CHECK_EQUAL((int) (y2_1 * 400), (int) (y2_2 * 400));
CHECK_EQUAL((int) (confidence_1 * 1000), (int) (confidence_2 * 1000));
CHECK_EQUAL(classId_1, classId_2);
}

2
tests/CVStabilizer_Tests.cpp
View File

@@ -109,7 +109,7 @@ SUITE(CVStabilizer_Tests)
CVStabilization stabilizer_2("{\"protobuf_data_path\": \"stabilizer.data\", \"smoothing_window\": 30}", processingController);
// Load stabilized data from first stabilizer protobuf data
stabilizer_2.LoadStabilizedData();
stabilizer_2._LoadStabilizedData();
// Get stabilized data
TransformParam tp_2 = stabilizer_2.GetTransformParamData(20);

42
tests/CVTracker_Tests.cpp
View File

@@ -63,16 +63,16 @@ SUITE(CVTracker_Tests)
// Get tracked data
FrameData fd = kcfTracker.GetTrackedData(20);
int x = fd.x1;
int y = fd.y1;
int width = fd.x2-fd.x1;
int height = fd.y2-fd.y1;
float x = fd.x1;
float y = fd.y1;
float width = fd.x2 - x;
float height = fd.y2 - y;
// Compare if tracked data is equal to pre-tested ones
CHECK_EQUAL(259, x);
CHECK_EQUAL(131, y);
CHECK_EQUAL(180, width);
CHECK_EQUAL(166, height);
CHECK_EQUAL(259, (int)(x * 640));
CHECK_EQUAL(131, (int)(y * 360));
CHECK_EQUAL(180, (int)(width * 640));
CHECK_EQUAL(166, (int)(height * 360));
}
@@ -96,10 +96,10 @@ SUITE(CVTracker_Tests)
// Get tracked data
FrameData fd_1 = kcfTracker_1.GetTrackedData(20);
int x_1 = fd_1.x1;
int y_1 = fd_1.y1;
int width_1 = fd_1.x2-fd_1.x1;
int height_1 = fd_1.y2-fd_1.y1;
float x_1 = fd_1.x1;
float y_1 = fd_1.y1;
float width_1 = fd_1.x2 - x_1;
float height_1 = fd_1.y2 - y_1;
// Save tracked data
kcfTracker_1.SaveTrackedData();
@@ -108,21 +108,21 @@ SUITE(CVTracker_Tests)
CVTracker kcfTracker_2("{\"protobuf_data_path\": \"kcf_tracker.data\", \"tracker_type\": \"\", \"bbox\": {\"x\": -1, \"y\": -1, \"w\": -1, \"h\": -1}}", processingController);
// Load tracked data from first tracker protobuf data
kcfTracker_2.LoadTrackedData();
kcfTracker_2._LoadTrackedData();
// Get tracked data
FrameData fd_2 = kcfTracker_2.GetTrackedData(20);
int x_2 = fd_2.x1;
int y_2 = fd_2.y1;
int width_2 = fd_2.x2-fd_2.x1;
int height_2 = fd_2.y2-fd_2.y1;
float x_2 = fd_2.x1;
float y_2 = fd_2.y1;
float width_2 = fd_2.x2 - x_2;
float height_2 = fd_2.y2 - y_2;
// Compare first tracker data with second tracker data
CHECK_EQUAL(x_1, x_2);
CHECK_EQUAL(y_1, y_2);
CHECK_EQUAL(width_1, width_2);
CHECK_EQUAL(height_1, height_2);
CHECK_EQUAL((int)(x_1 * 640), (int)(x_2 * 640));
CHECK_EQUAL((int)(y_1 * 360), (int)(y_2 * 360));
CHECK_EQUAL((int)(width_1 * 640), (int)(width_2 * 640));
CHECK_EQUAL((int)(height_1 * 360), (int)(height_2 * 360));
}
} // SUITE(Frame_Tests)