openshot/libopenshot
0
0
Fork 0
mirror of https://github.com/OpenShot/libopenshot.git synced 2026-03-02 08:53:52 -08:00
Code Issues Packages Projects Releases Wiki Activity

Added CVObjectDetection and ObjectDetection effect

Browse Source 
Also included kalman filter functions and code for tracking the output boxes from DNN model
This commit is contained in:
Brenno
2020-07-26 16:19:55 -03:00
parent d87a1260f3
commit 1a598b16df
26 changed files with 4039 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
include/CVObjectDetection.h Normal file
View File

@@ -0,0 +1,114 @@
/**
* @file
* @brief Header file for CVObjectDetection class
* @author Jonathan Thomas <jonathan@openshot.org>
*
* @ref License
*/
/* LICENSE
*
* Copyright (c) 2008-2019 OpenShot Studios, LLC
* <http://www.openshotstudios.com/>. This file is part of
* OpenShot Library (libopenshot), an open-source project dedicated to
* delivering high quality video editing and animation solutions to the
* world. For more information visit <http://www.openshot.org/>.
*
* OpenShot Library (libopenshot) is free software: you can redistribute it
* and/or modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* OpenShot Library (libopenshot) is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <google/protobuf/util/time_util.h>
#define int64 opencv_broken_int
#define uint64 opencv_broken_uint
#include <opencv2/dnn.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/core.hpp>
#undef uint64
#undef int64
#include "Json.h"
#include "ProcessingController.h"
#include "Clip.h"
#include "objdetectdata.pb.h"
using google::protobuf::util::TimeUtil;
struct CVDetectionData{
CVDetectionData(){}
CVDetectionData(std::vector<int> _classIds, std::vector<float> _confidences, std::vector<cv::Rect> _boxes, size_t _frameId){
classIds = _classIds;
confidences = _confidences;
boxes = _boxes;
frameId = _frameId;
}
size_t frameId;
std::vector<int> classIds;
std::vector<float> confidences;
std::vector<cv::Rect> boxes;
};
class CVObjectDetection{
private:
cv::dnn::Net net;
std::vector<std::string> classNames;
float confThreshold, nmsThreshold;
std::string classesFile;
std::string modelConfiguration;
std::string modelWeights;
std::string processingDevice;
std::string protobuf_data_path;
uint progress;
size_t start;
size_t end;
/// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes
ProcessingController *processingController;
void setProcessingDevice();
void DetectObjects(const cv::Mat &frame, size_t frame_number);
// Remove the bounding boxes with low confidence using non-maxima suppression
void postprocess(const cv::Size &frameDims, const std::vector<cv::Mat>& out, size_t frame_number);
// Get the names of the output layers
std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net);
public:
std::map<size_t, CVDetectionData> detectionsData;
CVObjectDetection(std::string processInfoJson, ProcessingController &processingController);
void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false);
/// Protobuf Save and Load methods
// Save protobuf file
bool SaveTrackedData();
// Add frame object detection data into protobuf message.
void AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrameData, CVDetectionData& dData);
// Load protobuf file
bool LoadTrackedData();
/// Get and Set JSON methods
void SetJson(const std::string value); ///< Load JSON string into this object
void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object
};

16
include/CVTracker.h
View File

@@ -19,6 +19,8 @@
#include "ProcessingController.h"
#include "trackerdata.pb.h"
#include "../src/sort_filter/sort.hpp"
using namespace std;
using google::protobuf::util::TimeUtil;
@@ -49,6 +51,17 @@ struct FrameData{
}
};
class RemoveJitter{
private:
std::vector<cv::Rect2d> bboxTracker;
int boxesInterval;
int boxesInVector;
public:
RemoveJitter(int boxesInterval);
void update(cv::Rect2d bbox, cv::Rect2d &out_bbox);
};
class CVTracker {
private:
std::map<size_t, FrameData> trackedDataById; // Save tracked data
@@ -67,12 +80,13 @@ class CVTracker {
bool json_interval;
size_t start;
size_t end;
// Initialize the tracker
bool initTracker(cv::Mat &frame, size_t frameId);
// Update the object tracker according to frame
bool trackFrame(cv::Mat &frame, size_t frameId);
bool trackFrame(cv::Mat &frame, size_t frameId, SortTracker &sort, RemoveJitter &removeJitter);
public:

3
include/ClipProcessingJobs.h
View File

@@ -39,6 +39,7 @@
#include "CVStabilization.h"
#include "CVTracker.h"
#include "CVObjectDetection.h"
#endif
#include <thread>
@@ -66,6 +67,8 @@ class ClipProcessingJobs{
void trackClip(Clip& clip, ProcessingController& controller);
// Apply stabilization to clip
void stabilizeClip(Clip& clip, ProcessingController& controller);
// Apply object detection to clip
void detectObjectsClip(Clip& clip, ProcessingController& controller);
public:

1
include/Effects.h
View File

@@ -42,6 +42,7 @@
#include "effects/Hue.h"
#include "effects/Mask.h"
#include "effects/Negate.h"
#include "effects/ObjectDetection.h"
#include "effects/Pixelate.h"
#include "effects/Stabilizer.h"
#include "effects/Saturation.h"

117
include/effects/ObjectDetection.h Normal file
View File

@@ -0,0 +1,117 @@
/**
* @file
* @brief Header file for Object Detection effect class
* @author Jonathan Thomas <jonathan@openshot.org>
*
* @ref License
*/
/* LICENSE
*
* Copyright (c) 2008-2019 OpenShot Studios, LLC
* <http://www.openshotstudios.com/>. This file is part of
* OpenShot Library (libopenshot), an open-source project dedicated to
* delivering high quality video editing and animation solutions to the
* world. For more information visit <http://www.openshot.org/>.
*
* OpenShot Library (libopenshot) is free software: you can redistribute it
* and/or modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* OpenShot Library (libopenshot) is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef OPENSHOT_OBJECT_DETECTION_EFFECT_H
#define OPENSHOT_OBJECT_DETECTION_EFFECT_H
#include "../EffectBase.h"
#include <cmath>
#include <stdio.h>
#include <memory>
#include "../Color.h"
#include "../Json.h"
#include "../KeyFrame.h"
#include "../objdetectdata.pb.h"
struct DetectionData{
DetectionData(){}
DetectionData(std::vector<int> _classIds, std::vector<float> _confidences, std::vector<cv::Rect> _boxes, size_t _frameId){
classIds = _classIds;
confidences = _confidences;
boxes = _boxes;
frameId = _frameId;
}
size_t frameId;
std::vector<int> classIds;
std::vector<float> confidences;
std::vector<cv::Rect> boxes;
};
namespace openshot
{
/**
* @brief This class stabilizes video clip to remove undesired shaking and jitter.
*
* Adding stabilization is useful to increase video quality overall, since it removes
* from subtle to harsh unexpected camera movements.
*/
class ObjectDetection : public EffectBase
{
private:
std::string protobuf_data_path;
std::map<size_t, DetectionData> detectionsData;
std::vector<std::string> classNames;
/// Init effect settings
void init_effect_details();
void drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame);
public:
/// Blank constructor, useful when using Json to load the effect properties
ObjectDetection(std::string clipTrackerDataPath);
/// Default constructor
ObjectDetection();
/// @brief This method is required for all derived classes of EffectBase, and returns a
/// modified openshot::Frame object
///
/// The frame object is passed into this method, and a frame_number is passed in which
/// tells the effect which settings to use from its keyframes (starting at 1).
///
/// @returns The modified openshot::Frame object
/// @param frame The frame object that needs the effect applied to it
/// @param frame_number The frame number (starting at 1) of the effect on the timeline.
std::shared_ptr<Frame> GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number) override;
// Load protobuf data file
bool LoadObjDetectdData(std::string inputFilePath);
DetectionData GetTrackedData(size_t frameId);
/// Get and Set JSON methods
std::string Json() const override; ///< Generate JSON string of this object
void SetJson(const std::string value) override; ///< Load JSON string into this object
Json::Value JsonValue() const override; ///< Generate Json::Value for this object
void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object
/// Get all properties for a specific frame (perfect for a UI to display the current state
/// of all properties at any time)
std::string PropertiesJSON(int64_t requested_frame) const override;
};
}
#endif

2
include/effects/Tracker.h
View File

@@ -116,7 +116,7 @@ namespace openshot
bool LoadTrackedData(std::string inputFilePath);
// Get tracker info for the desired frame
EffectFrameData GetTrackedData(int frameId);
EffectFrameData GetTrackedData(size_t frameId);
/// Get and Set JSON methods
std::string Json() const override; ///< Generate JSON string of this object

1020
include/objdetectdata.pb.h Normal file
View File

File diff suppressed because it is too large Load Diff

9
src/CMakeLists.txt
View File

@@ -118,7 +118,6 @@ if (OpenCV_FOUND)
execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROTO_H} ${CMAKE_CURRENT_SOURCE_DIR}/../include)
file(REMOVE ${PROTO_H})
endforeach()
endif()
@@ -214,12 +213,17 @@ set(OPENSHOT_SOURCES
set(OPENSHOT_CV_SOURCES
CVTracker.cpp
CVStabilization.cpp
ClipProcessingJobs.cpp)
ClipProcessingJobs.cpp
CVObjectDetection.cpp
./sort_filter/sort.cpp
./sort_filter/Hungarian.cpp
./sort_filter/KalmanTracker.cpp)
# Compiled Protobuf messages
set(PROTOBUF_MESSAGES
stabilizedata.pb.cc
trackerdata.pb.cc
objdetectdata.pb.cc
)
# Video effects
@@ -234,6 +238,7 @@ set(EFFECTS_SOURCES
effects/Hue.cpp
effects/Mask.cpp
effects/Negate.cpp
effects/ObjectDetection.cpp
effects/Pixelate.cpp
effects/Saturation.cpp
effects/Shift.cpp

281
src/CVObjectDetection.cpp Normal file
View File

@@ -0,0 +1,281 @@
/**
* @file
* @brief Source file for CVObjectDetection class
* @author Jonathan Thomas <jonathan@openshot.org>
*
* @ref License
*/
/* LICENSE
*
* Copyright (c) 2008-2019 OpenShot Studios, LLC
* <http://www.openshotstudios.com/>. This file is part of
* OpenShot Library (libopenshot), an open-source project dedicated to
* delivering high quality video editing and animation solutions to the
* world. For more information visit <http://www.openshot.org/>.
*
* OpenShot Library (libopenshot) is free software: you can redistribute it
* and/or modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* OpenShot Library (libopenshot) is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "../include/CVObjectDetection.h"
// // Initialize the parameters
// float confThreshold = 0.5; // Confidence threshold
// float nmsThreshold = 0.4; // Non-maximum suppression threshold
// int inpWidth = 416; // Width of network's input image
// int inpHeight = 416; // Height of network's input image
// vector<string> classes;
CVObjectDetection::CVObjectDetection(std::string processInfoJson, ProcessingController &processingController)
: processingController(&processingController), processingDevice("CPU"){
SetJson(processInfoJson);
setProcessingDevice();
}
void CVObjectDetection::setProcessingDevice(){
if(processingDevice == "GPU"){
net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA);
net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA);
}
else if(processingDevice == "CPU"){
net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
}
}
void CVObjectDetection::detectObjectsClip(openshot::Clip &video, size_t _start, size_t _end, bool process_interval)
{
start = _start; end = _end;
video.Open();
// Load names of classes
std::ifstream ifs(classesFile.c_str());
std::string line;
while (std::getline(ifs, line)) classNames.push_back(line);
confThreshold = 0.5;
nmsThreshold = 0.1;
// Load the network
net = cv::dnn::readNetFromDarknet(modelConfiguration, modelWeights);
size_t frame_number;
if(!process_interval || end == 0 || end-start <= 0){
// Get total number of frames in video
start = video.Start() * video.Reader()->info.fps.ToInt();
end = video.End() * video.Reader()->info.fps.ToInt();
}
for (frame_number = start; frame_number <= end; frame_number++)
{
// Stop the feature tracker process
if(processingController->ShouldStop()){
return;
}
std::shared_ptr<openshot::Frame> f = video.GetFrame(frame_number);
// Grab OpenCV Mat image
cv::Mat cvimage = f->GetImageCV();
DetectObjects(cvimage, frame_number);
// Update progress
processingController->SetProgress(uint(100*(frame_number-start)/(end-start)));
std::cout<<"Frame: "<<frame_number<<"\n";
}
}
void CVObjectDetection::DetectObjects(const cv::Mat &frame, size_t frameId){
// Get frame as OpenCV Mat
cv::Mat blob;
// Create a 4D blob from the frame.
int inpWidth, inpHeight;
inpWidth = inpHeight = 416;
cv::dnn::blobFromImage(frame, blob, 1/255.0, cv::Size(inpWidth, inpHeight), cv::Scalar(0,0,0), true, false);
//Sets the input to the network
net.setInput(blob);
// Runs the forward pass to get output of the output layers
std::vector<cv::Mat> outs;
net.forward(outs, getOutputsNames(net));
// Remove the bounding boxes with low confidence
postprocess(frame.size(), outs, frameId);
}
// Remove the bounding boxes with low confidence using non-maxima suppression
void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector<cv::Mat>& outs, size_t frameId)
{
std::vector<int> classIds;
std::vector<float> confidences;
std::vector<cv::Rect> boxes;
for (size_t i = 0; i < outs.size(); ++i)
{
// Scan through all the bounding boxes output from the network and keep only the
// ones with high confidence scores. Assign the box's class label as the class
// with the highest score for the box.
float* data = (float*)outs[i].data;
for (int j = 0; j < outs[i].rows; ++j, data += outs[i].cols)
{
cv::Mat scores = outs[i].row(j).colRange(5, outs[i].cols);
cv::Point classIdPoint;
double confidence;
// Get the value and location of the maximum score
cv::minMaxLoc(scores, 0, &confidence, 0, &classIdPoint);
if (confidence > confThreshold)
{
int centerX = (int)(data[0] * frameDims.width);
int centerY = (int)(data[1] * frameDims.height);
int width = (int)(data[2] * frameDims.width);
int height = (int)(data[3] * frameDims.height);
int left = centerX - width / 2;
int top = centerY - height / 2;
classIds.push_back(classIdPoint.x);
confidences.push_back((float)confidence);
boxes.push_back(cv::Rect(left, top, width, height));
}
}
}
// Perform non maximum suppression to eliminate redundant overlapping boxes with
// lower confidences
std::vector<int> indices;
cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
detectionsData[frameId] = CVDetectionData(classIds, confidences, boxes, frameId);
}
// Get the names of the output layers
std::vector<cv::String> CVObjectDetection::getOutputsNames(const cv::dnn::Net& net)
{
static std::vector<cv::String> names;
//Get the indices of the output layers, i.e. the layers with unconnected outputs
std::vector<int> outLayers = net.getUnconnectedOutLayers();
//get the names of all the layers in the network
std::vector<cv::String> layersNames = net.getLayerNames();
// Get the names of the output layers in names
names.resize(outLayers.size());
for (size_t i = 0; i < outLayers.size(); ++i)
names[i] = layersNames[outLayers[i] - 1];
return names;
}
bool CVObjectDetection::SaveTrackedData(){
// Create tracker message
libopenshotobjdetect::ObjDetect objMessage;
//Save class names in protobuf message
for(int i = 0; i<classNames.size(); i++){
std::string* className = objMessage.add_classnames();
className->assign(classNames.at(i));
}
// Iterate over all frames data and save in protobuf message
for(std::map<size_t,CVDetectionData>::iterator it=detectionsData.begin(); it!=detectionsData.end(); ++it){
CVDetectionData dData = it->second;
libopenshotobjdetect::Frame* pbFrameData;
AddFrameDataToProto(objMessage.add_frame(), dData);
}
// Add timestamp
*objMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL));
{
// Write the new message to disk.
std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary);
if (!objMessage.SerializeToOstream(&output)) {
cerr << "Failed to write protobuf message." << endl;
return false;
}
}
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();
return true;
}
// Add frame object detection into protobuf message.
void CVObjectDetection::AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrameData, CVDetectionData& dData) {
// Save frame number and rotation
pbFrameData->set_id(dData.frameId);
for(size_t i = 0; i < dData.boxes.size(); i++){
libopenshotobjdetect::Frame_Box* box = pbFrameData->add_bounding_box();
// Save bounding box data
box->set_x1(dData.boxes.at(i).x);
box->set_y1(dData.boxes.at(i).y);
box->set_x2(dData.boxes.at(i).x + dData.boxes.at(i).width);
box->set_y2(dData.boxes.at(i).y + dData.boxes.at(i).height);
box->set_classid(dData.classIds.at(i));
box->set_confidence(dData.confidences.at(i));
}
}
// Load JSON string into this object
void CVObjectDetection::SetJson(const std::string value) {
// Parse JSON string into JSON objects
try
{
const Json::Value root = openshot::stringToJson(value);
// Set all values that match
SetJsonValue(root);
}
catch (const std::exception& e)
{
// Error parsing JSON (or missing keys)
// throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
std::cout<<"JSON is invalid (missing keys or invalid data types)"<<std::endl;
}
}
// Load Json::Value into this object
void CVObjectDetection::SetJsonValue(const Json::Value root) {
// Set data from Json (if key is found)
if (!root["protobuf_data_path"].isNull()){
protobuf_data_path = (root["protobuf_data_path"].asString());
}
if (!root["processing_device"].isNull()){
processingDevice = (root["processing_device"].asString());
}
if (!root["model_configuration"].isNull()){
modelConfiguration = (root["model_configuration"].asString());
}
if (!root["model_weights"].isNull()){
modelWeights= (root["model_weights"].asString());
}
if (!root["classes_file"].isNull()){
classesFile = (root["classes_file"].asString());
}
}

58
src/CVTracker.cpp
View File

@@ -81,6 +81,9 @@ void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, boo
bool trackerInit = false;
SortTracker sort;
RemoveJitter removeJitter(0);
size_t frame;
// Loop through video
for (frame = start; frame <= end; frame++)
@@ -109,7 +112,7 @@ void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, boo
}
else{
// Update the object tracker according to frame
trackerInit = trackFrame(cvimage, frame_number);
trackerInit = trackFrame(cvimage, frame_number, sort, removeJitter);
// Draw box on image
FrameData fd = GetTrackedData(frame_number);
@@ -136,7 +139,7 @@ bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){
}
// Update the object tracker according to frame
bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId, SortTracker &sort, RemoveJitter &removeJitter){
// Update the tracking result
bool ok = tracker->update(frame, bbox);
@@ -144,6 +147,15 @@ bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){
// Otherwise add only frame number
if (ok)
{
std::vector<cv::Rect> bboxes = {bbox};
sort.update(bboxes, frameId, sqrt(pow(frame.rows, 2) + pow(frame.cols, 2)));
for(auto TBox : sort.frameTrackingResult)
bbox = TBox.box;
// removeJitter.update(bbox, bbox);
// Add new frame data
trackedDataById[frameId] = FrameData(frameId, 0, bbox.x, bbox.y, bbox.x+bbox.width, bbox.y+bbox.height);
}
@@ -301,4 +313,44 @@ void CVTracker::SetJsonValue(const Json::Value root) {
start = root["first_frame"].asInt64();
json_interval = true;
}
}
}
RemoveJitter::RemoveJitter(int boxesInterval) : boxesInterval(boxesInterval), boxesInVector(0){
}
void RemoveJitter::update(cv::Rect2d bbox, cv::Rect2d &out_bbox){
bboxTracker.push_back(bbox);
// Just to initialize the vector properly
if(boxesInVector < boxesInterval+1){
boxesInVector++;
out_bbox = bbox;
}
else{
cv::Rect2d old_bbox = bboxTracker.front();
cv::Rect2d new_bbox = bboxTracker.back();
int centerX_1 = old_bbox.x + old_bbox.width/2;
int centerY_1 = old_bbox.y + old_bbox.height/2;
int centerX_2 = new_bbox.x + new_bbox.width/2;
int centerY_2 = new_bbox.y + new_bbox.height/2;
int dif_centerXs = abs(centerX_1 - centerX_2);
int dif_centerYs = abs(centerY_1 - centerY_2);
cout<<dif_centerXs<<"\n";
cout<<dif_centerYs<<"\n\n";
if(dif_centerXs > 6 || dif_centerYs > 6){
out_bbox = new_bbox;
}
else{
cv::Rect2d mean_bbox((old_bbox.x + new_bbox.x)/2, (old_bbox.y + new_bbox.y)/2, (old_bbox.width + new_bbox.width)/2, (old_bbox.height + new_bbox.height)/2);
out_bbox = mean_bbox;
}
bboxTracker.erase(bboxTracker.begin());
}
}

25
src/ClipProcessingJobs.cpp
View File

@@ -14,6 +14,9 @@ void ClipProcessingJobs::processClip(Clip& clip){
if(processingType == "Tracker"){
t = std::thread(&ClipProcessingJobs::trackClip, this, std::ref(clip), std::ref(this->processingController));
}
if(processingType == "Object Detector"){
t = std::thread(&ClipProcessingJobs::detectObjectsClip, this, std::ref(clip), std::ref(this->processingController));
}
}
// Apply object tracking to clip
@@ -40,8 +43,28 @@ void ClipProcessingJobs::trackClip(Clip& clip, ProcessingController& controller)
}
// Apply stabilization to clip
void ClipProcessingJobs::stabilizeClip(Clip& clip, ProcessingController& controller){
void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& controller){
// create CVStabilization object
CVObjectDetection objDetector(processInfoJson, controller);
// Start stabilization process
objDetector.detectObjectsClip(clip);
// Thread controller. If effect processing is done, save data
// Else, kill thread
if(controller.ShouldStop()){
controller.SetFinished(true);
return;
}
else{
// Save stabilization data
objDetector.SaveTrackedData();
// tells to UI that the processing finished
controller.SetFinished(true);
}
}
void ClipProcessingJobs::stabilizeClip(Clip& clip, ProcessingController& controller){
// create CVStabilization object
CVStabilization stabilizer(processInfoJson, controller);
// Start stabilization process
stabilizer.stabilizeClip(clip);

4
src/EffectInfo.cpp
View File

@@ -91,6 +91,9 @@ EffectBase* EffectInfo::CreateEffect(std::string effect_type) {
else if(effect_type == "Tracker")
return new Tracker();
else if(effect_type == "Object Detector")
return new ObjectDetection();
return NULL;
}
@@ -118,6 +121,7 @@ Json::Value EffectInfo::JsonValue() {
root.append(Wave().JsonInfo());
root.append(Stabilizer().JsonInfo());
root.append(Tracker().JsonInfo());
root.append(ObjectDetection().JsonInfo());
// return JsonValue
return root;

1
src/bindings/python/openshot.i
View File

@@ -231,6 +231,7 @@
%include "effects/Wave.h"
%include "effects/Stabilizer.h"
%include "effects/Tracker.h"
%include "effects/ObjectDetection.h"
/* Wrap std templates (list, vector, etc...) */

261
src/effects/ObjectDetection.cpp Normal file
View File

@@ -0,0 +1,261 @@
/**
* @file
* @brief Source file for Object Detection effect class
* @author Jonathan Thomas <jonathan@openshot.org>
*
* @ref License
*/
/* LICENSE
*
* Copyright (c) 2008-2019 OpenShot Studios, LLC
* <http://www.openshotstudios.com/>. This file is part of
* OpenShot Library (libopenshot), an open-source project dedicated to
* delivering high quality video editing and animation solutions to the
* world. For more information visit <http://www.openshot.org/>.
*
* OpenShot Library (libopenshot) is free software: you can redistribute it
* and/or modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* OpenShot Library (libopenshot) is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "../../include/effects/ObjectDetection.h"
#include "../../include/effects/Tracker.h"
using namespace openshot;
/// Blank constructor, useful when using Json to load the effect properties
ObjectDetection::ObjectDetection(std::string clipObDetectDataPath)
{
// Init effect properties
init_effect_details();
// Tries to load the tracker data from protobuf
LoadObjDetectdData(clipObDetectDataPath);
}
// Default constructor
ObjectDetection::ObjectDetection()
{
// Init effect properties
init_effect_details();
}
// Init effect settings
void ObjectDetection::init_effect_details()
{
/// Initialize the values of the EffectInfo struct.
InitEffectInfo();
/// Set the effect info
info.class_name = "Object Detector";
info.name = "Object Detector";
info.description = "Detect objects through the video.";
info.has_audio = false;
info.has_video = true;
}
// This method is required for all derived classes of EffectBase, and returns a
// modified openshot::Frame object
std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
{
// Get the frame's image
cv::Mat cv_image = frame->GetImageCV();
// Check if frame isn't NULL
if(!cv_image.empty()){
// Check if track data exists for the requested frame
if (detectionsData.find(frame_number) != detectionsData.end()) {
DetectionData detections = detectionsData[frame_number];
for(int i = 0; i<detections.boxes.size(); i++){
drawPred(detections.classIds.at(i), detections.confidences.at(i),
detections.boxes.at(i), cv_image);
}
}
}
// Set image with drawn box to frame
// If the input image is NULL or doesn't have tracking data, it's returned as it came
frame->SetImageCV(cv_image);
return frame;
}
void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame)
{
//Draw a rectangle displaying the bounding box
cv::rectangle(frame, box, cv::Scalar(255, 178, 50), 3);
//Get the label for the class name and its confidence
std::string label = cv::format("%.2f", conf);
if (!classNames.empty())
{
CV_Assert(classId < (int)classNames.size());
label = classNames[classId] + ":" + label;
}
//Display the label at the box.y of the bounding box
int baseLine;
cv::Size labelSize = getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
box.y = std::max((int)box.y, labelSize.height);
cv::rectangle(frame, cv::Point(box.x, box.y - round(1.5*labelSize.height)), cv::Point(box.x + round(1.5*labelSize.width), box.y + baseLine), cv::Scalar(255, 255, 255), cv::FILLED);
putText(frame, label, cv::Point(box.x, box.y), cv::FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0,0,0),1);
}
// Load protobuf data file
bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){
// Create tracker message
libopenshotobjdetect::ObjDetect objMessage;
{
// Read the existing tracker message.
fstream input(inputFilePath, ios::in | ios::binary);
if (!objMessage.ParseFromIstream(&input)) {
cerr << "Failed to parse protobuf message." << endl;
return false;
}
}
// Make sure the trackedData is empty
classNames.clear();
detectionsData.clear();
for(int i = 0; i < objMessage.classnames_size(); i++){
classNames.push_back(objMessage.classnames(i));
}
// Iterate over all frames of the saved message
for (size_t i = 0; i < objMessage.frame_size(); i++) {
const libopenshotobjdetect::Frame& pbFrameData = objMessage.frame(i);
// Load frame and rotation data
size_t id = pbFrameData.id();
// Load bounding box data
const google::protobuf::RepeatedPtrField<libopenshotobjdetect::Frame_Box > &box = pbFrameData.bounding_box();
std::vector<int> classIds;
std::vector<float> confidences;
std::vector<cv::Rect> boxes;
for(int i = 0; i < pbFrameData.bounding_box_size(); i++){
int x1 = box.at(i).x1();
int y1 = box.at(i).y1();
int x2 = box.at(i).x2();
int y2 = box.at(i).y2();
int classId = box.at(i).classid();
float confidence = box.at(i).confidence();
cv::Rect2d box(x1, y1, x2-x1, y2-y1);
boxes.push_back(box);
classIds.push_back(classId);
confidences.push_back(confidence);
}
// Assign data to tracker map
detectionsData[id] = DetectionData(classIds, confidences, boxes, id);
}
// Show the time stamp from the last update in tracker data file
if (objMessage.has_last_updated()) {
cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(objMessage.last_updated()) << endl;
}
// Delete all global objects allocated by libprotobuf.
google::protobuf::ShutdownProtobufLibrary();
return true;
}
// Get tracker info for the desired frame
DetectionData ObjectDetection::GetTrackedData(size_t frameId){
// Check if the tracker info for the requested frame exists
if ( detectionsData.find(frameId) == detectionsData.end() ) {
return DetectionData();
} else {
return detectionsData[frameId];
}
}
// Generate JSON string of this object
std::string ObjectDetection::Json() const {
// Return formatted string
return JsonValue().toStyledString();
}
// Generate Json::Value for this object
Json::Value ObjectDetection::JsonValue() const {
// Create root json object
Json::Value root = EffectBase::JsonValue(); // get parent properties
root["type"] = info.class_name;
root["protobuf_data_path"] = protobuf_data_path;
// return JsonValue
return root;
}
// Load JSON string into this object
void ObjectDetection::SetJson(const std::string value) {
// Parse JSON string into JSON objects
try
{
const Json::Value root = openshot::stringToJson(value);
// Set all values that match
SetJsonValue(root);
}
catch (const std::exception& e)
{
// Error parsing JSON (or missing keys)
throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
}
}
// Load Json::Value into this object
void ObjectDetection::SetJsonValue(const Json::Value root) {
// Set parent data
EffectBase::SetJsonValue(root);
// Set data from Json (if key is found)
if (!root["protobuf_data_path"].isNull()){
protobuf_data_path = (root["protobuf_data_path"].asString());
if(!LoadObjDetectdData(protobuf_data_path)){
std::cout<<"Invalid protobuf data path";
protobuf_data_path = "";
}
}
}
// Get all properties for a specific frame
std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const {
// Generate JSON properties list
Json::Value root;
root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
// Return formatted string
return root.toStyledString();
}

2
src/effects/Tracker.cpp
View File

@@ -139,7 +139,7 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){
}
// Get tracker info for the desired frame
EffectFrameData Tracker::GetTrackedData(int frameId){
EffectFrameData Tracker::GetTrackedData(size_t frameId){
// Check if the tracker info for the requested frame exists
if ( trackedDataById.find(frameId) == trackedDataById.end() ) {

221
src/examples/Example_opencv.cpp
View File

@@ -33,6 +33,7 @@
#include <memory>
#include "../../include/CVTracker.h"
#include "../../include/CVStabilization.h"
#include "../../include/CVObjectDetection.h"
#include "../../include/OpenShot.h"
#include "../../include/CrashHandler.h"
@@ -40,6 +41,20 @@
using namespace openshot;
using namespace std;
/*
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
The following methods are just for getting JSON info to the pre-processing effects
*/
string jsonFormat(string key, string value, string type="string"); // Format variables to the needed JSON format
string trackerJson(cv::Rect2d r, bool onlyProtoPath); // Set variable values for tracker effect
string stabilizerJson(bool onlyProtoPath); // Set variable values for stabilizer effect
string objectDetectionJson(bool onlyProtoPath); // Set variable values for object detector effect
/*
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
*/
// Show the pre-processed clip on the screen
void displayClip(openshot::Clip &r9){
@@ -57,8 +72,7 @@ void displayClip(openshot::Clip &r9){
std::shared_ptr<openshot::Frame> f = r9.GetFrame(frame_number);
// Grab OpenCV::Mat image
cv::Mat cvimage = f->GetImageCV();
// Convert color scheme from RGB (QImage scheme) to BGR (OpenCV scheme)
cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2BGR);
// Display the frame
cv::imshow("Display Image", cvimage);
@@ -71,73 +85,18 @@ void displayClip(openshot::Clip &r9){
cv::destroyAllWindows();
}
/*
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
The following methods are just for getting JSON info to the pre-processing effects
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
*/
// Return JSON string for the tracker effect
string trackerJson(cv::Rect2d r, bool onlyProtoPath){
// Set the tracker
string tracker = "KCF";
// Construct all the composition of the JSON string
string trackerType = "\"tracker_type\": \"" + tracker + "\"";
string protobuf_data_path = "\"protobuf_data_path\": \"kcf_tracker.data\"";
stringstream bboxCoords;
bboxCoords << "\"bbox\": {\"x\":"<<r.x<<", \"y\": "<<r.y<<", \"w\": "<<r.width<<", \"h\": "<<r.height<<"}";
// Return only the the protobuf path in JSON format
if(onlyProtoPath)
return "{" + protobuf_data_path + "}";
// Return all the parameters for the pre-processing effect
else
return "{" + protobuf_data_path + ", " + trackerType + ", " + bboxCoords.str() + "}";
}
// Return JSON string for the stabilizer effect
string stabilizerJson(bool onlyProtoPath){
// Set smoothing window value
int smoothingWindow = 30;
// Construct all the composition of the JSON string
string protobuf_data_path = "\"protobuf_data_path\": \"example_stabilizer.data\"";
stringstream smoothing_window;
smoothing_window << "\"smoothing_window\": "<< smoothingWindow;
// Return only the the protobuf path in JSON format
if(onlyProtoPath)
return "{" + protobuf_data_path + "}";
// Return all the parameters for the pre-processing effect
else
return "{" + protobuf_data_path + ", " + smoothing_window.str() + "}";
}
string objectDetectionJson(bool onlyProtoPath){
// Construct all the composition of the JSON string
string protobuf_data_path = "\"protobuf_data_path\": \"example_object_detection.data\"";
// Return only the the protobuf path in JSON format
if(onlyProtoPath)
return "{" + protobuf_data_path + "}";
}
int main(int argc, char* argv[]) {
// Set pre-processing effects
bool TRACK_DATA = false;
bool SMOOTH_VIDEO = true;
bool OBJECT_DETECTION_DATA = false;
bool SMOOTH_VIDEO = false;
bool OBJECT_DETECTION_DATA = true;
// Get media path
std::stringstream path;
path << TEST_MEDIA_PATH << "test.avi";
path << TEST_MEDIA_PATH << ((OBJECT_DETECTION_DATA) ? "test_video.mp4" : "test.avi");
// test_video.mp4 --> Used for object detector
// test.avi --> Used for tracker and stabilizer
// Thread controller just for the pre-processing constructors, it won't be used
ProcessingController processingController;
@@ -146,7 +105,7 @@ int main(int argc, char* argv[]) {
openshot::Clip r9(path.str());
r9.Open();
// Aplly tracking effect on the clip
// Apply tracking effect on the clip
if(TRACK_DATA){
// Take the bounding box coordinates
@@ -159,7 +118,7 @@ int main(int argc, char* argv[]) {
CVTracker tracker(trackerJson(r, false), processingController);
// Start the tracking
tracker.trackClip(r9);
tracker.trackClip(r9, 0, 100, true);
// Save the tracked data
tracker.SaveTrackedData();
@@ -173,7 +132,7 @@ int main(int argc, char* argv[]) {
r9.AddEffect(e);
}
// Aplly stabilizer effect on the clip
// Apply stabilizer effect on the clip
if(SMOOTH_VIDEO){
// Create a stabilizer object by passing a JSON string and a thread controller, this last one won't be used
@@ -181,7 +140,7 @@ int main(int argc, char* argv[]) {
CVStabilization stabilizer(stabilizerJson(false), processingController);
// Start the stabilization
stabilizer.stabilizeClip(r9);
stabilizer.stabilizeClip(r9, 0, 100, true);
// Save the stabilization data
stabilizer.SaveStabilizedData();
@@ -195,9 +154,27 @@ int main(int argc, char* argv[]) {
r9.AddEffect(e);
}
// Apply object detection effect on the clip
if(OBJECT_DETECTION_DATA){
// CVObjectDetection objectDetection("GPU");
// objectDetection.ProcessClip(r9);
// Create a object detection object by passing a JSON string and a thread controller, this last one won't be used
// JSON info: path to save the detection data, processing devicee, model weights, model configuration and class names
CVObjectDetection objectDetection(objectDetectionJson(false), processingController);
// Start the object detection
objectDetection.detectObjectsClip(r9, 0, 100, true);
// Save the object detection data
objectDetection.SaveTrackedData();
// Create a object detector effect
EffectBase* e = EffectInfo().CreateEffect("Object Detector");
// Pass a JSON string with the saved detections data
// The effect will read and save the detections in a map::<frame,data_struct>
e->SetJson(objectDetectionJson(true));
// Add the effect to the clip
r9.AddEffect(e);
}
// Show the pre-processed clip on the screen
@@ -210,3 +187,111 @@ int main(int argc, char* argv[]) {
return 0;
}
/*
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
The following methods are just for getting JSON info to the pre-processing effects
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
*/
string jsonFormat(string key, string value, string type){
stringstream jsonFormatMessage;
jsonFormatMessage << ( "\"" + key + "\": " );
if(type == "string")
jsonFormatMessage << ( "\"" + value + "\"" );
if(type == "rstring")
jsonFormatMessage << value;
if(type == "int")
jsonFormatMessage << stoi(value);
if(type == "float")
jsonFormatMessage << (float)stof(value);
if(type == "double")
jsonFormatMessage << (double)stof(value);
if (type == "bool")
jsonFormatMessage << ((value == "true" || value == "1") ? "true" : "false");
return jsonFormatMessage.str();
}
// Return JSON string for the tracker effect
string trackerJson(cv::Rect2d r, bool onlyProtoPath){
// Define path to save tracked data
string protobufDataPath = "kcf_tracker.data";
// Set the tracker
string tracker = "KCF";
// Construct all the composition of the JSON string
string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath);
string trackerType = jsonFormat("tracker_type", tracker);
string bboxCoords = jsonFormat(
"bbox",
"{" + jsonFormat("x", to_string(r.x), "int") +
"," + jsonFormat("y", to_string(r.y), "int") +
"," + jsonFormat("w", to_string(r.width), "int") +
"," + jsonFormat("h", to_string(r.height), "int") +
"}",
"rstring");
// Return only the the protobuf path in JSON format
if(onlyProtoPath)
return "{" + protobuf_data_path + "}";
// Return all the parameters for the pre-processing effect
else
return "{" + protobuf_data_path + "," + trackerType + "," + bboxCoords + "}";
}
// Return JSON string for the stabilizer effect
string stabilizerJson(bool onlyProtoPath){
// Define path to save stabilized data
string protobufDataPath = "example_stabilizer.data";
// Set smoothing window value
string smoothingWindow = "30";
// Construct all the composition of the JSON string
string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath);
string smoothing_window = jsonFormat("smoothing_window", smoothingWindow, "int");
// Return only the the protobuf path in JSON format
if(onlyProtoPath)
return "{" + protobuf_data_path + "}";
// Return all the parameters for the pre-processing effect
else
return "{" + protobuf_data_path + "," + smoothing_window + "}";
}
string objectDetectionJson(bool onlyProtoPath){
// Define path to save object detection data
string protobufDataPath = "example_object_detection.data";
// Define processing device
string processingDevice = "GPU";
// Set path to model configuration file
string modelConfiguration = "yolov3.cfg";
// Set path to model weights
string modelWeights = "yolov3.weights";
// Set path to class names file
string classesFile = "obj.names";
// Construct all the composition of the JSON string
string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath);
string processing_device = jsonFormat("processing_device", processingDevice);
string model_configuration = jsonFormat("model_configuration", modelConfiguration);
string model_weights = jsonFormat("model_weights", modelWeights);
string classes_file = jsonFormat("classes_file", classesFile);
// Return only the the protobuf path in JSON format
if(onlyProtoPath)
return "{" + protobuf_data_path + "}";
else
return "{" + protobuf_data_path + "," + processing_device + "," + model_configuration + ","
+ model_weights + "," + classes_file + "}";
}

BIN
src/examples/test_video.mp4 Normal file
View File

Binary file not shown.

1011
src/objdetectdata.pb.cc Normal file
View File

File diff suppressed because it is too large Load Diff

32
src/protobuf_messages/objdetectdata.proto Normal file
View File

@@ -0,0 +1,32 @@
// [START declaration]
syntax = "proto3";
package libopenshotobjdetect;
import "google/protobuf/timestamp.proto";
// [END declaration]
// [START messages]
message Frame {
int32 id = 1; // Frame ID.
message Box{
int32 x1 = 1;
int32 y1 = 2;
int32 x2 = 3;
int32 y2 = 4;
int32 classId = 5;
float confidence = 6;
}
repeated Box bounding_box = 2;
}
message ObjDetect {
repeated Frame frame = 1;
google.protobuf.Timestamp last_updated = 2;
repeated string classNames = 3;
}
// [END messages]

456
src/sort_filter/Hungarian.cpp Normal file
View File

@@ -0,0 +1,456 @@
///////////////////////////////////////////////////////////////////////////////
// Hungarian.cpp: Implementation file for Class HungarianAlgorithm.
//
// This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren.
// The original implementation is a few mex-functions for use in MATLAB, found here:
// http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem
//
// Both this code and the orignal code are published under the BSD license.
// by Cong Ma, 2016
//
#include "Hungarian.h"
using namespace std;
HungarianAlgorithm::HungarianAlgorithm() {}
HungarianAlgorithm::~HungarianAlgorithm() {}
//********************************************************//
// A single function wrapper for solving assignment problem.
//********************************************************//
double HungarianAlgorithm::Solve(
vector<vector<double>> &DistMatrix,
vector<int> &Assignment)
{
unsigned int nRows = DistMatrix.size();
unsigned int nCols = DistMatrix[0].size();
double *distMatrixIn = new double[nRows * nCols];
int *assignment = new int[nRows];
double cost = 0.0;
// Fill in the distMatrixIn. Mind the index is "i + nRows * j".
// Here the cost matrix of size MxN is defined as a double precision array of N*M elements.
// In the solving functions matrices are seen to be saved MATLAB-internally in row-order.
// (i.e. the matrix [1 2; 3 4] will be stored as a vector [1 3 2 4], NOT [1 2 3 4]).
for (unsigned int i = 0; i < nRows; i++)
for (unsigned int j = 0; j < nCols; j++)
distMatrixIn[i + nRows * j] = DistMatrix[i][j];
// call solving function
assignmentoptimal(assignment, &cost, distMatrixIn, nRows, nCols);
Assignment.clear();
for (unsigned int r = 0; r < nRows; r++)
Assignment.push_back(assignment[r]);
delete[] distMatrixIn;
delete[] assignment;
return cost;
}
//********************************************************//
// Solve optimal solution for assignment problem using Munkres algorithm, also known as Hungarian Algorithm.
//********************************************************//
void HungarianAlgorithm::assignmentoptimal(
int *assignment,
double *cost,
double *distMatrixIn,
int nOfRows,
int nOfColumns)
{
double *distMatrix, *distMatrixTemp, *distMatrixEnd, *columnEnd, value, minValue;
bool *coveredColumns, *coveredRows, *starMatrix, *newStarMatrix, *primeMatrix;
int nOfElements, minDim, row, col;
/* initialization */
*cost = 0;
for (row = 0; row < nOfRows; row++)
assignment[row] = -1;
/* generate working copy of distance Matrix */
/* check if all matrix elements are positive */
nOfElements = nOfRows * nOfColumns;
distMatrix = (double *)malloc(nOfElements * sizeof(double));
distMatrixEnd = distMatrix + nOfElements;
for (row = 0; row < nOfElements; row++)
{
value = distMatrixIn[row];
if (value < 0)
cerr << "All matrix elements have to be non-negative." << endl;
distMatrix[row] = value;
}
/* memory allocation */
coveredColumns = (bool *)calloc(nOfColumns, sizeof(bool));
coveredRows = (bool *)calloc(nOfRows, sizeof(bool));
starMatrix = (bool *)calloc(nOfElements, sizeof(bool));
primeMatrix = (bool *)calloc(nOfElements, sizeof(bool));
newStarMatrix = (bool *)calloc(nOfElements, sizeof(bool)); /* used in step4 */
/* preliminary steps */
if (nOfRows <= nOfColumns)
{
minDim = nOfRows;
for (row = 0; row < nOfRows; row++)
{
/* find the smallest element in the row */
distMatrixTemp = distMatrix + row;
minValue = *distMatrixTemp;
distMatrixTemp += nOfRows;
while (distMatrixTemp < distMatrixEnd)
{
value = *distMatrixTemp;
if (value < minValue)
minValue = value;
distMatrixTemp += nOfRows;
}
/* subtract the smallest element from each element of the row */
distMatrixTemp = distMatrix + row;
while (distMatrixTemp < distMatrixEnd)
{
*distMatrixTemp -= minValue;
distMatrixTemp += nOfRows;
}
}
/* Steps 1 and 2a */
for (row = 0; row < nOfRows; row++)
for (col = 0; col < nOfColumns; col++)
if (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON)
if (!coveredColumns[col])
{
starMatrix[row + nOfRows * col] = true;
coveredColumns[col] = true;
break;
}
}
else /* if(nOfRows > nOfColumns) */
{
minDim = nOfColumns;
for (col = 0; col < nOfColumns; col++)
{
/* find the smallest element in the column */
distMatrixTemp = distMatrix + nOfRows * col;
columnEnd = distMatrixTemp + nOfRows;
minValue = *distMatrixTemp++;
while (distMatrixTemp < columnEnd)
{
value = *distMatrixTemp++;
if (value < minValue)
minValue = value;
}
/* subtract the smallest element from each element of the column */
distMatrixTemp = distMatrix + nOfRows * col;
while (distMatrixTemp < columnEnd)
*distMatrixTemp++ -= minValue;
}
/* Steps 1 and 2a */
for (col = 0; col < nOfColumns; col++)
for (row = 0; row < nOfRows; row++)
if (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON)
if (!coveredRows[row])
{
starMatrix[row + nOfRows * col] = true;
coveredColumns[col] = true;
coveredRows[row] = true;
break;
}
for (row = 0; row < nOfRows; row++)
coveredRows[row] = false;
}
/* move to step 2b */
step2b(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim);
/* compute cost and remove invalid assignments */
computeassignmentcost(assignment, cost, distMatrixIn, nOfRows);
/* free allocated memory */
free(distMatrix);
free(coveredColumns);
free(coveredRows);
free(starMatrix);
free(primeMatrix);
free(newStarMatrix);
return;
}
/********************************************************/
void HungarianAlgorithm::buildassignmentvector(
int *assignment,
bool *starMatrix,
int nOfRows,
int nOfColumns)
{
int row, col;
for (row = 0; row < nOfRows; row++)
for (col = 0; col < nOfColumns; col++)
if (starMatrix[row + nOfRows * col])
{
#ifdef ONE_INDEXING
assignment[row] = col + 1; /* MATLAB-Indexing */
#else
assignment[row] = col;
#endif
break;
}
}
/********************************************************/
void HungarianAlgorithm::computeassignmentcost(
int *assignment,
double *cost,
double *distMatrix,
int nOfRows)
{
int row, col;
for (row = 0; row < nOfRows; row++)
{
col = assignment[row];
if (col >= 0)
*cost += distMatrix[row + nOfRows * col];
}
}
/********************************************************/
void HungarianAlgorithm::step2a(
int *assignment,
double *distMatrix,
bool *starMatrix,
bool *newStarMatrix,
bool *primeMatrix,
bool *coveredColumns,
bool *coveredRows,
int nOfRows,
int nOfColumns,
int minDim)
{
bool *starMatrixTemp, *columnEnd;
int col;
/* cover every column containing a starred zero */
for (col = 0; col < nOfColumns; col++)
{
starMatrixTemp = starMatrix + nOfRows * col;
columnEnd = starMatrixTemp + nOfRows;
while (starMatrixTemp < columnEnd)
{
if (*starMatrixTemp++)
{
coveredColumns[col] = true;
break;
}
}
}
/* move to step 3 */
step2b(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim);
}
/********************************************************/
void HungarianAlgorithm::step2b(
int *assignment,
double *distMatrix,
bool *starMatrix,
bool *newStarMatrix,
bool *primeMatrix,
bool *coveredColumns,
bool *coveredRows,
int nOfRows,
int nOfColumns,
int minDim)
{
int col, nOfCoveredColumns;
/* count covered columns */
nOfCoveredColumns = 0;
for (col = 0; col < nOfColumns; col++)
if (coveredColumns[col])
nOfCoveredColumns++;
if (nOfCoveredColumns == minDim)
{
/* algorithm finished */
buildassignmentvector(assignment, starMatrix, nOfRows, nOfColumns);
}
else
{
/* move to step 3 */
step3(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim);
}
}
/********************************************************/
void HungarianAlgorithm::step3(
int *assignment,
double *distMatrix,
bool *starMatrix,
bool *newStarMatrix,
bool *primeMatrix,
bool *coveredColumns,
bool *coveredRows,
int nOfRows,
int nOfColumns,
int minDim)
{
bool zerosFound;
int row, col, starCol;
zerosFound = true;
while (zerosFound)
{
zerosFound = false;
for (col = 0; col < nOfColumns; col++)
if (!coveredColumns[col])
for (row = 0; row < nOfRows; row++)
if ((!coveredRows[row]) && (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON))
{
/* prime zero */
primeMatrix[row + nOfRows * col] = true;
/* find starred zero in current row */
for (starCol = 0; starCol < nOfColumns; starCol++)
if (starMatrix[row + nOfRows * starCol])
break;
if (starCol == nOfColumns) /* no starred zero found */
{
/* move to step 4 */
step4(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim, row, col);
return;
}
else
{
coveredRows[row] = true;
coveredColumns[starCol] = false;
zerosFound = true;
break;
}
}
}
/* move to step 5 */
step5(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim);
}
/********************************************************/
void HungarianAlgorithm::step4(
int *assignment,
double *distMatrix,
bool *starMatrix,
bool *newStarMatrix,
bool *primeMatrix,
bool *coveredColumns,
bool *coveredRows,
int nOfRows,
int nOfColumns,
int minDim,
int row,
int col)
{
int n, starRow, starCol, primeRow, primeCol;
int nOfElements = nOfRows * nOfColumns;
/* generate temporary copy of starMatrix */
for (n = 0; n < nOfElements; n++)
newStarMatrix[n] = starMatrix[n];
/* star current zero */
newStarMatrix[row + nOfRows * col] = true;
/* find starred zero in current column */
starCol = col;
for (starRow = 0; starRow < nOfRows; starRow++)
if (starMatrix[starRow + nOfRows * starCol])
break;
while (starRow < nOfRows)
{
/* unstar the starred zero */
newStarMatrix[starRow + nOfRows * starCol] = false;
/* find primed zero in current row */
primeRow = starRow;
for (primeCol = 0; primeCol < nOfColumns; primeCol++)
if (primeMatrix[primeRow + nOfRows * primeCol])
break;
/* star the primed zero */
newStarMatrix[primeRow + nOfRows * primeCol] = true;
/* find starred zero in current column */
starCol = primeCol;
for (starRow = 0; starRow < nOfRows; starRow++)
if (starMatrix[starRow + nOfRows * starCol])
break;
}
/* use temporary copy as new starMatrix */
/* delete all primes, uncover all rows */
for (n = 0; n < nOfElements; n++)
{
primeMatrix[n] = false;
starMatrix[n] = newStarMatrix[n];
}
for (n = 0; n < nOfRows; n++)
coveredRows[n] = false;
/* move to step 2a */
step2a(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim);
}
/********************************************************/
void HungarianAlgorithm::step5(
int *assignment,
double *distMatrix,
bool *starMatrix,
bool *newStarMatrix,
bool *primeMatrix,
bool *coveredColumns,
bool *coveredRows,
int nOfRows,
int nOfColumns,
int minDim)
{
double h, value;
int row, col;
/* find smallest uncovered element h */
h = DBL_MAX;
for (row = 0; row < nOfRows; row++)
if (!coveredRows[row])
for (col = 0; col < nOfColumns; col++)
if (!coveredColumns[col])
{
value = distMatrix[row + nOfRows * col];
if (value < h)
h = value;
}
/* add h to each covered row */
for (row = 0; row < nOfRows; row++)
if (coveredRows[row])
for (col = 0; col < nOfColumns; col++)
distMatrix[row + nOfRows * col] += h;
/* subtract h from each uncovered column */
for (col = 0; col < nOfColumns; col++)
if (!coveredColumns[col])
for (row = 0; row < nOfRows; row++)
distMatrix[row + nOfRows * col] -= h;
/* move to step 3 */
step3(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim);
}

Some files were not shown because too many files have changed in this diff Show More