diff --git a/include/CVObjectDetection.h b/include/CVObjectDetection.h new file mode 100644 index 00000000..db07a5cc --- /dev/null +++ b/include/CVObjectDetection.h @@ -0,0 +1,114 @@ +/** + * @file + * @brief Header file for CVObjectDetection class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#pragma once + +#include + +#define int64 opencv_broken_int +#define uint64 opencv_broken_uint +#include +#include +#include +#undef uint64 +#undef int64 +#include "Json.h" +#include "ProcessingController.h" +#include "Clip.h" +#include "objdetectdata.pb.h" + +using google::protobuf::util::TimeUtil; + +struct CVDetectionData{ + CVDetectionData(){} + CVDetectionData(std::vector _classIds, std::vector _confidences, std::vector _boxes, size_t _frameId){ + classIds = _classIds; + confidences = _confidences; + boxes = _boxes; + frameId = _frameId; + } + size_t frameId; + std::vector classIds; + std::vector confidences; + std::vector boxes; +}; + +class CVObjectDetection{ + + private: + + cv::dnn::Net net; + std::vector classNames; + float confThreshold, nmsThreshold; + + std::string classesFile; + std::string modelConfiguration; + std::string modelWeights; + std::string processingDevice; + std::string protobuf_data_path; + + uint progress; + + size_t start; + size_t end; + + /// Will handle a Thread safely comutication between ClipProcessingJobs and the processing effect classes + ProcessingController *processingController; + + void setProcessingDevice(); + + void DetectObjects(const cv::Mat &frame, size_t frame_number); + + // Remove the bounding boxes with low confidence using non-maxima suppression + void postprocess(const cv::Size &frameDims, const std::vector& out, size_t frame_number); + + // Get the names of the output layers + std::vector getOutputsNames(const cv::dnn::Net& net); + + public: + + std::map detectionsData; + + CVObjectDetection(std::string processInfoJson, ProcessingController &processingController); + + void detectObjectsClip(openshot::Clip &video, size_t start=0, size_t end=0, bool process_interval=false); + + /// Protobuf Save and Load methods + // Save protobuf file + bool SaveTrackedData(); + // Add frame object detection data into protobuf message. + void AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrameData, CVDetectionData& dData); + // Load protobuf file + bool LoadTrackedData(); + + /// Get and Set JSON methods + void SetJson(const std::string value); ///< Load JSON string into this object + void SetJsonValue(const Json::Value root); ///< Load Json::Value into this object +}; diff --git a/include/CVTracker.h b/include/CVTracker.h index 40fdbec6..9e52d5d2 100644 --- a/include/CVTracker.h +++ b/include/CVTracker.h @@ -19,6 +19,8 @@ #include "ProcessingController.h" #include "trackerdata.pb.h" +#include "../src/sort_filter/sort.hpp" + using namespace std; using google::protobuf::util::TimeUtil; @@ -49,6 +51,17 @@ struct FrameData{ } }; +class RemoveJitter{ + private: + std::vector bboxTracker; + int boxesInterval; + int boxesInVector; + + public: + RemoveJitter(int boxesInterval); + void update(cv::Rect2d bbox, cv::Rect2d &out_bbox); +}; + class CVTracker { private: std::map trackedDataById; // Save tracked data @@ -67,12 +80,13 @@ class CVTracker { bool json_interval; size_t start; size_t end; + // Initialize the tracker bool initTracker(cv::Mat &frame, size_t frameId); // Update the object tracker according to frame - bool trackFrame(cv::Mat &frame, size_t frameId); + bool trackFrame(cv::Mat &frame, size_t frameId, SortTracker &sort, RemoveJitter &removeJitter); public: diff --git a/include/ClipProcessingJobs.h b/include/ClipProcessingJobs.h index 55d551d4..835b2911 100644 --- a/include/ClipProcessingJobs.h +++ b/include/ClipProcessingJobs.h @@ -39,6 +39,7 @@ #include "CVStabilization.h" #include "CVTracker.h" + #include "CVObjectDetection.h" #endif #include @@ -66,6 +67,8 @@ class ClipProcessingJobs{ void trackClip(Clip& clip, ProcessingController& controller); // Apply stabilization to clip void stabilizeClip(Clip& clip, ProcessingController& controller); + // Apply object detection to clip + void detectObjectsClip(Clip& clip, ProcessingController& controller); public: diff --git a/include/Effects.h b/include/Effects.h index 5773ae07..e6b9255d 100644 --- a/include/Effects.h +++ b/include/Effects.h @@ -42,6 +42,7 @@ #include "effects/Hue.h" #include "effects/Mask.h" #include "effects/Negate.h" +#include "effects/ObjectDetection.h" #include "effects/Pixelate.h" #include "effects/Stabilizer.h" #include "effects/Saturation.h" diff --git a/include/effects/ObjectDetection.h b/include/effects/ObjectDetection.h new file mode 100644 index 00000000..772ef682 --- /dev/null +++ b/include/effects/ObjectDetection.h @@ -0,0 +1,117 @@ +/** + * @file + * @brief Header file for Object Detection effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_OBJECT_DETECTION_EFFECT_H +#define OPENSHOT_OBJECT_DETECTION_EFFECT_H + +#include "../EffectBase.h" + +#include +#include +#include +#include "../Color.h" +#include "../Json.h" +#include "../KeyFrame.h" +#include "../objdetectdata.pb.h" + +struct DetectionData{ + DetectionData(){} + DetectionData(std::vector _classIds, std::vector _confidences, std::vector _boxes, size_t _frameId){ + classIds = _classIds; + confidences = _confidences; + boxes = _boxes; + frameId = _frameId; + } + size_t frameId; + std::vector classIds; + std::vector confidences; + std::vector boxes; +}; + +namespace openshot +{ + + /** + * @brief This class stabilizes video clip to remove undesired shaking and jitter. + * + * Adding stabilization is useful to increase video quality overall, since it removes + * from subtle to harsh unexpected camera movements. + */ + class ObjectDetection : public EffectBase + { + private: + std::string protobuf_data_path; + std::map detectionsData; + std::vector classNames; + + /// Init effect settings + void init_effect_details(); + + void drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame); + + public: + + + /// Blank constructor, useful when using Json to load the effect properties + ObjectDetection(std::string clipTrackerDataPath); + + /// Default constructor + ObjectDetection(); + + /// @brief This method is required for all derived classes of EffectBase, and returns a + /// modified openshot::Frame object + /// + /// The frame object is passed into this method, and a frame_number is passed in which + /// tells the effect which settings to use from its keyframes (starting at 1). + /// + /// @returns The modified openshot::Frame object + /// @param frame The frame object that needs the effect applied to it + /// @param frame_number The frame number (starting at 1) of the effect on the timeline. + std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + + // Load protobuf data file + bool LoadObjDetectdData(std::string inputFilePath); + + DetectionData GetTrackedData(size_t frameId); + + /// Get and Set JSON methods + std::string Json() const override; ///< Generate JSON string of this object + void SetJson(const std::string value) override; ///< Load JSON string into this object + Json::Value JsonValue() const override; ///< Generate Json::Value for this object + void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object + + /// Get all properties for a specific frame (perfect for a UI to display the current state + /// of all properties at any time) + std::string PropertiesJSON(int64_t requested_frame) const override; + }; + +} + +#endif diff --git a/include/effects/Tracker.h b/include/effects/Tracker.h index c2df82b0..c237a11d 100644 --- a/include/effects/Tracker.h +++ b/include/effects/Tracker.h @@ -116,7 +116,7 @@ namespace openshot bool LoadTrackedData(std::string inputFilePath); // Get tracker info for the desired frame - EffectFrameData GetTrackedData(int frameId); + EffectFrameData GetTrackedData(size_t frameId); /// Get and Set JSON methods std::string Json() const override; ///< Generate JSON string of this object diff --git a/include/objdetectdata.pb.h b/include/objdetectdata.pb.h new file mode 100644 index 00000000..0b9966eb --- /dev/null +++ b/include/objdetectdata.pb.h @@ -0,0 +1,1020 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: objdetectdata.proto + +#ifndef GOOGLE_PROTOBUF_INCLUDED_objdetectdata_2eproto +#define GOOGLE_PROTOBUF_INCLUDED_objdetectdata_2eproto + +#include +#include + +#include +#if PROTOBUF_VERSION < 3012000 +#error This file was generated by a newer version of protoc which is +#error incompatible with your Protocol Buffer headers. Please update +#error your headers. +#endif +#if 3012003 < PROTOBUF_MIN_PROTOC_VERSION +#error This file was generated by an older version of protoc which is +#error incompatible with your Protocol Buffer headers. Please +#error regenerate this file with a newer version of protoc. +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // IWYU pragma: export +#include // IWYU pragma: export +#include +#include +// @@protoc_insertion_point(includes) +#include +#define PROTOBUF_INTERNAL_EXPORT_objdetectdata_2eproto +PROTOBUF_NAMESPACE_OPEN +namespace internal { +class AnyMetadata; +} // namespace internal +PROTOBUF_NAMESPACE_CLOSE + +// Internal implementation detail -- do not use these members. +struct TableStruct_objdetectdata_2eproto { + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[3] + PROTOBUF_SECTION_VARIABLE(protodesc_cold); + static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; + static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; + static const ::PROTOBUF_NAMESPACE_ID::uint32 offsets[]; +}; +extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_objdetectdata_2eproto; +namespace libopenshotobjdetect { +class Frame; +class FrameDefaultTypeInternal; +extern FrameDefaultTypeInternal _Frame_default_instance_; +class Frame_Box; +class Frame_BoxDefaultTypeInternal; +extern Frame_BoxDefaultTypeInternal _Frame_Box_default_instance_; +class ObjDetect; +class ObjDetectDefaultTypeInternal; +extern ObjDetectDefaultTypeInternal _ObjDetect_default_instance_; +} // namespace libopenshotobjdetect +PROTOBUF_NAMESPACE_OPEN +template<> ::libopenshotobjdetect::Frame* Arena::CreateMaybeMessage<::libopenshotobjdetect::Frame>(Arena*); +template<> ::libopenshotobjdetect::Frame_Box* Arena::CreateMaybeMessage<::libopenshotobjdetect::Frame_Box>(Arena*); +template<> ::libopenshotobjdetect::ObjDetect* Arena::CreateMaybeMessage<::libopenshotobjdetect::ObjDetect>(Arena*); +PROTOBUF_NAMESPACE_CLOSE +namespace libopenshotobjdetect { + +// =================================================================== + +class Frame_Box PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:libopenshotobjdetect.Frame.Box) */ { + public: + inline Frame_Box() : Frame_Box(nullptr) {}; + virtual ~Frame_Box(); + + Frame_Box(const Frame_Box& from); + Frame_Box(Frame_Box&& from) noexcept + : Frame_Box() { + *this = ::std::move(from); + } + + inline Frame_Box& operator=(const Frame_Box& from) { + CopyFrom(from); + return *this; + } + inline Frame_Box& operator=(Frame_Box&& from) noexcept { + if (GetArena() == from.GetArena()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const Frame_Box& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const Frame_Box* internal_default_instance() { + return reinterpret_cast( + &_Frame_Box_default_instance_); + } + static constexpr int kIndexInFileMessages = + 0; + + friend void swap(Frame_Box& a, Frame_Box& b) { + a.Swap(&b); + } + inline void Swap(Frame_Box* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(Frame_Box* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline Frame_Box* New() const final { + return CreateMaybeMessage(nullptr); + } + + Frame_Box* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const Frame_Box& from); + void MergeFrom(const Frame_Box& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(Frame_Box* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "libopenshotobjdetect.Frame.Box"; + } + protected: + explicit Frame_Box(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_objdetectdata_2eproto); + return ::descriptor_table_objdetectdata_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kX1FieldNumber = 1, + kY1FieldNumber = 2, + kX2FieldNumber = 3, + kY2FieldNumber = 4, + kClassIdFieldNumber = 5, + kConfidenceFieldNumber = 6, + }; + // int32 x1 = 1; + void clear_x1(); + ::PROTOBUF_NAMESPACE_ID::int32 x1() const; + void set_x1(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_x1() const; + void _internal_set_x1(::PROTOBUF_NAMESPACE_ID::int32 value); + public: + + // int32 y1 = 2; + void clear_y1(); + ::PROTOBUF_NAMESPACE_ID::int32 y1() const; + void set_y1(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_y1() const; + void _internal_set_y1(::PROTOBUF_NAMESPACE_ID::int32 value); + public: + + // int32 x2 = 3; + void clear_x2(); + ::PROTOBUF_NAMESPACE_ID::int32 x2() const; + void set_x2(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_x2() const; + void _internal_set_x2(::PROTOBUF_NAMESPACE_ID::int32 value); + public: + + // int32 y2 = 4; + void clear_y2(); + ::PROTOBUF_NAMESPACE_ID::int32 y2() const; + void set_y2(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_y2() const; + void _internal_set_y2(::PROTOBUF_NAMESPACE_ID::int32 value); + public: + + // int32 classId = 5; + void clear_classid(); + ::PROTOBUF_NAMESPACE_ID::int32 classid() const; + void set_classid(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_classid() const; + void _internal_set_classid(::PROTOBUF_NAMESPACE_ID::int32 value); + public: + + // float confidence = 6; + void clear_confidence(); + float confidence() const; + void set_confidence(float value); + private: + float _internal_confidence() const; + void _internal_set_confidence(float value); + public: + + // @@protoc_insertion_point(class_scope:libopenshotobjdetect.Frame.Box) + private: + class _Internal; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::int32 x1_; + ::PROTOBUF_NAMESPACE_ID::int32 y1_; + ::PROTOBUF_NAMESPACE_ID::int32 x2_; + ::PROTOBUF_NAMESPACE_ID::int32 y2_; + ::PROTOBUF_NAMESPACE_ID::int32 classid_; + float confidence_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_objdetectdata_2eproto; +}; +// ------------------------------------------------------------------- + +class Frame PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:libopenshotobjdetect.Frame) */ { + public: + inline Frame() : Frame(nullptr) {}; + virtual ~Frame(); + + Frame(const Frame& from); + Frame(Frame&& from) noexcept + : Frame() { + *this = ::std::move(from); + } + + inline Frame& operator=(const Frame& from) { + CopyFrom(from); + return *this; + } + inline Frame& operator=(Frame&& from) noexcept { + if (GetArena() == from.GetArena()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const Frame& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const Frame* internal_default_instance() { + return reinterpret_cast( + &_Frame_default_instance_); + } + static constexpr int kIndexInFileMessages = + 1; + + friend void swap(Frame& a, Frame& b) { + a.Swap(&b); + } + inline void Swap(Frame* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(Frame* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline Frame* New() const final { + return CreateMaybeMessage(nullptr); + } + + Frame* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const Frame& from); + void MergeFrom(const Frame& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(Frame* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "libopenshotobjdetect.Frame"; + } + protected: + explicit Frame(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_objdetectdata_2eproto); + return ::descriptor_table_objdetectdata_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + typedef Frame_Box Box; + + // accessors ------------------------------------------------------- + + enum : int { + kBoundingBoxFieldNumber = 2, + kIdFieldNumber = 1, + }; + // repeated .libopenshotobjdetect.Frame.Box bounding_box = 2; + int bounding_box_size() const; + private: + int _internal_bounding_box_size() const; + public: + void clear_bounding_box(); + ::libopenshotobjdetect::Frame_Box* mutable_bounding_box(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame_Box >* + mutable_bounding_box(); + private: + const ::libopenshotobjdetect::Frame_Box& _internal_bounding_box(int index) const; + ::libopenshotobjdetect::Frame_Box* _internal_add_bounding_box(); + public: + const ::libopenshotobjdetect::Frame_Box& bounding_box(int index) const; + ::libopenshotobjdetect::Frame_Box* add_bounding_box(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame_Box >& + bounding_box() const; + + // int32 id = 1; + void clear_id(); + ::PROTOBUF_NAMESPACE_ID::int32 id() const; + void set_id(::PROTOBUF_NAMESPACE_ID::int32 value); + private: + ::PROTOBUF_NAMESPACE_ID::int32 _internal_id() const; + void _internal_set_id(::PROTOBUF_NAMESPACE_ID::int32 value); + public: + + // @@protoc_insertion_point(class_scope:libopenshotobjdetect.Frame) + private: + class _Internal; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame_Box > bounding_box_; + ::PROTOBUF_NAMESPACE_ID::int32 id_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_objdetectdata_2eproto; +}; +// ------------------------------------------------------------------- + +class ObjDetect PROTOBUF_FINAL : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:libopenshotobjdetect.ObjDetect) */ { + public: + inline ObjDetect() : ObjDetect(nullptr) {}; + virtual ~ObjDetect(); + + ObjDetect(const ObjDetect& from); + ObjDetect(ObjDetect&& from) noexcept + : ObjDetect() { + *this = ::std::move(from); + } + + inline ObjDetect& operator=(const ObjDetect& from) { + CopyFrom(from); + return *this; + } + inline ObjDetect& operator=(ObjDetect&& from) noexcept { + if (GetArena() == from.GetArena()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const ObjDetect& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const ObjDetect* internal_default_instance() { + return reinterpret_cast( + &_ObjDetect_default_instance_); + } + static constexpr int kIndexInFileMessages = + 2; + + friend void swap(ObjDetect& a, ObjDetect& b) { + a.Swap(&b); + } + inline void Swap(ObjDetect* other) { + if (other == this) return; + if (GetArena() == other->GetArena()) { + InternalSwap(other); + } else { + ::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other); + } + } + void UnsafeArenaSwap(ObjDetect* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArena() == other->GetArena()); + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline ObjDetect* New() const final { + return CreateMaybeMessage(nullptr); + } + + ObjDetect* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const ObjDetect& from); + void MergeFrom(const ObjDetect& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + ::PROTOBUF_NAMESPACE_ID::uint8* _InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(ObjDetect* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "libopenshotobjdetect.ObjDetect"; + } + protected: + explicit ObjDetect(::PROTOBUF_NAMESPACE_ID::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena* arena); + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_objdetectdata_2eproto); + return ::descriptor_table_objdetectdata_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kFrameFieldNumber = 1, + kClassNamesFieldNumber = 3, + kLastUpdatedFieldNumber = 2, + }; + // repeated .libopenshotobjdetect.Frame frame = 1; + int frame_size() const; + private: + int _internal_frame_size() const; + public: + void clear_frame(); + ::libopenshotobjdetect::Frame* mutable_frame(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame >* + mutable_frame(); + private: + const ::libopenshotobjdetect::Frame& _internal_frame(int index) const; + ::libopenshotobjdetect::Frame* _internal_add_frame(); + public: + const ::libopenshotobjdetect::Frame& frame(int index) const; + ::libopenshotobjdetect::Frame* add_frame(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame >& + frame() const; + + // repeated string classNames = 3; + int classnames_size() const; + private: + int _internal_classnames_size() const; + public: + void clear_classnames(); + const std::string& classnames(int index) const; + std::string* mutable_classnames(int index); + void set_classnames(int index, const std::string& value); + void set_classnames(int index, std::string&& value); + void set_classnames(int index, const char* value); + void set_classnames(int index, const char* value, size_t size); + std::string* add_classnames(); + void add_classnames(const std::string& value); + void add_classnames(std::string&& value); + void add_classnames(const char* value); + void add_classnames(const char* value, size_t size); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& classnames() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_classnames(); + private: + const std::string& _internal_classnames(int index) const; + std::string* _internal_add_classnames(); + public: + + // .google.protobuf.Timestamp last_updated = 2; + bool has_last_updated() const; + private: + bool _internal_has_last_updated() const; + public: + void clear_last_updated(); + const PROTOBUF_NAMESPACE_ID::Timestamp& last_updated() const; + PROTOBUF_NAMESPACE_ID::Timestamp* release_last_updated(); + PROTOBUF_NAMESPACE_ID::Timestamp* mutable_last_updated(); + void set_allocated_last_updated(PROTOBUF_NAMESPACE_ID::Timestamp* last_updated); + private: + const PROTOBUF_NAMESPACE_ID::Timestamp& _internal_last_updated() const; + PROTOBUF_NAMESPACE_ID::Timestamp* _internal_mutable_last_updated(); + public: + void unsafe_arena_set_allocated_last_updated( + PROTOBUF_NAMESPACE_ID::Timestamp* last_updated); + PROTOBUF_NAMESPACE_ID::Timestamp* unsafe_arena_release_last_updated(); + + // @@protoc_insertion_point(class_scope:libopenshotobjdetect.ObjDetect) + private: + class _Internal; + + template friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame > frame_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField classnames_; + PROTOBUF_NAMESPACE_ID::Timestamp* last_updated_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_objdetectdata_2eproto; +}; +// =================================================================== + + +// =================================================================== + +#ifdef __GNUC__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstrict-aliasing" +#endif // __GNUC__ +// Frame_Box + +// int32 x1 = 1; +inline void Frame_Box::clear_x1() { + x1_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::_internal_x1() const { + return x1_; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::x1() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.Box.x1) + return _internal_x1(); +} +inline void Frame_Box::_internal_set_x1(::PROTOBUF_NAMESPACE_ID::int32 value) { + + x1_ = value; +} +inline void Frame_Box::set_x1(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_x1(value); + // @@protoc_insertion_point(field_set:libopenshotobjdetect.Frame.Box.x1) +} + +// int32 y1 = 2; +inline void Frame_Box::clear_y1() { + y1_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::_internal_y1() const { + return y1_; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::y1() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.Box.y1) + return _internal_y1(); +} +inline void Frame_Box::_internal_set_y1(::PROTOBUF_NAMESPACE_ID::int32 value) { + + y1_ = value; +} +inline void Frame_Box::set_y1(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_y1(value); + // @@protoc_insertion_point(field_set:libopenshotobjdetect.Frame.Box.y1) +} + +// int32 x2 = 3; +inline void Frame_Box::clear_x2() { + x2_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::_internal_x2() const { + return x2_; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::x2() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.Box.x2) + return _internal_x2(); +} +inline void Frame_Box::_internal_set_x2(::PROTOBUF_NAMESPACE_ID::int32 value) { + + x2_ = value; +} +inline void Frame_Box::set_x2(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_x2(value); + // @@protoc_insertion_point(field_set:libopenshotobjdetect.Frame.Box.x2) +} + +// int32 y2 = 4; +inline void Frame_Box::clear_y2() { + y2_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::_internal_y2() const { + return y2_; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::y2() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.Box.y2) + return _internal_y2(); +} +inline void Frame_Box::_internal_set_y2(::PROTOBUF_NAMESPACE_ID::int32 value) { + + y2_ = value; +} +inline void Frame_Box::set_y2(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_y2(value); + // @@protoc_insertion_point(field_set:libopenshotobjdetect.Frame.Box.y2) +} + +// int32 classId = 5; +inline void Frame_Box::clear_classid() { + classid_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::_internal_classid() const { + return classid_; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame_Box::classid() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.Box.classId) + return _internal_classid(); +} +inline void Frame_Box::_internal_set_classid(::PROTOBUF_NAMESPACE_ID::int32 value) { + + classid_ = value; +} +inline void Frame_Box::set_classid(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_classid(value); + // @@protoc_insertion_point(field_set:libopenshotobjdetect.Frame.Box.classId) +} + +// float confidence = 6; +inline void Frame_Box::clear_confidence() { + confidence_ = 0; +} +inline float Frame_Box::_internal_confidence() const { + return confidence_; +} +inline float Frame_Box::confidence() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.Box.confidence) + return _internal_confidence(); +} +inline void Frame_Box::_internal_set_confidence(float value) { + + confidence_ = value; +} +inline void Frame_Box::set_confidence(float value) { + _internal_set_confidence(value); + // @@protoc_insertion_point(field_set:libopenshotobjdetect.Frame.Box.confidence) +} + +// ------------------------------------------------------------------- + +// Frame + +// int32 id = 1; +inline void Frame::clear_id() { + id_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame::_internal_id() const { + return id_; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 Frame::id() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.id) + return _internal_id(); +} +inline void Frame::_internal_set_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + + id_ = value; +} +inline void Frame::set_id(::PROTOBUF_NAMESPACE_ID::int32 value) { + _internal_set_id(value); + // @@protoc_insertion_point(field_set:libopenshotobjdetect.Frame.id) +} + +// repeated .libopenshotobjdetect.Frame.Box bounding_box = 2; +inline int Frame::_internal_bounding_box_size() const { + return bounding_box_.size(); +} +inline int Frame::bounding_box_size() const { + return _internal_bounding_box_size(); +} +inline void Frame::clear_bounding_box() { + bounding_box_.Clear(); +} +inline ::libopenshotobjdetect::Frame_Box* Frame::mutable_bounding_box(int index) { + // @@protoc_insertion_point(field_mutable:libopenshotobjdetect.Frame.bounding_box) + return bounding_box_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame_Box >* +Frame::mutable_bounding_box() { + // @@protoc_insertion_point(field_mutable_list:libopenshotobjdetect.Frame.bounding_box) + return &bounding_box_; +} +inline const ::libopenshotobjdetect::Frame_Box& Frame::_internal_bounding_box(int index) const { + return bounding_box_.Get(index); +} +inline const ::libopenshotobjdetect::Frame_Box& Frame::bounding_box(int index) const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.Frame.bounding_box) + return _internal_bounding_box(index); +} +inline ::libopenshotobjdetect::Frame_Box* Frame::_internal_add_bounding_box() { + return bounding_box_.Add(); +} +inline ::libopenshotobjdetect::Frame_Box* Frame::add_bounding_box() { + // @@protoc_insertion_point(field_add:libopenshotobjdetect.Frame.bounding_box) + return _internal_add_bounding_box(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame_Box >& +Frame::bounding_box() const { + // @@protoc_insertion_point(field_list:libopenshotobjdetect.Frame.bounding_box) + return bounding_box_; +} + +// ------------------------------------------------------------------- + +// ObjDetect + +// repeated .libopenshotobjdetect.Frame frame = 1; +inline int ObjDetect::_internal_frame_size() const { + return frame_.size(); +} +inline int ObjDetect::frame_size() const { + return _internal_frame_size(); +} +inline void ObjDetect::clear_frame() { + frame_.Clear(); +} +inline ::libopenshotobjdetect::Frame* ObjDetect::mutable_frame(int index) { + // @@protoc_insertion_point(field_mutable:libopenshotobjdetect.ObjDetect.frame) + return frame_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame >* +ObjDetect::mutable_frame() { + // @@protoc_insertion_point(field_mutable_list:libopenshotobjdetect.ObjDetect.frame) + return &frame_; +} +inline const ::libopenshotobjdetect::Frame& ObjDetect::_internal_frame(int index) const { + return frame_.Get(index); +} +inline const ::libopenshotobjdetect::Frame& ObjDetect::frame(int index) const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.ObjDetect.frame) + return _internal_frame(index); +} +inline ::libopenshotobjdetect::Frame* ObjDetect::_internal_add_frame() { + return frame_.Add(); +} +inline ::libopenshotobjdetect::Frame* ObjDetect::add_frame() { + // @@protoc_insertion_point(field_add:libopenshotobjdetect.ObjDetect.frame) + return _internal_add_frame(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::libopenshotobjdetect::Frame >& +ObjDetect::frame() const { + // @@protoc_insertion_point(field_list:libopenshotobjdetect.ObjDetect.frame) + return frame_; +} + +// .google.protobuf.Timestamp last_updated = 2; +inline bool ObjDetect::_internal_has_last_updated() const { + return this != internal_default_instance() && last_updated_ != nullptr; +} +inline bool ObjDetect::has_last_updated() const { + return _internal_has_last_updated(); +} +inline const PROTOBUF_NAMESPACE_ID::Timestamp& ObjDetect::_internal_last_updated() const { + const PROTOBUF_NAMESPACE_ID::Timestamp* p = last_updated_; + return p != nullptr ? *p : *reinterpret_cast( + &PROTOBUF_NAMESPACE_ID::_Timestamp_default_instance_); +} +inline const PROTOBUF_NAMESPACE_ID::Timestamp& ObjDetect::last_updated() const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.ObjDetect.last_updated) + return _internal_last_updated(); +} +inline void ObjDetect::unsafe_arena_set_allocated_last_updated( + PROTOBUF_NAMESPACE_ID::Timestamp* last_updated) { + if (GetArena() == nullptr) { + delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(last_updated_); + } + last_updated_ = last_updated; + if (last_updated) { + + } else { + + } + // @@protoc_insertion_point(field_unsafe_arena_set_allocated:libopenshotobjdetect.ObjDetect.last_updated) +} +inline PROTOBUF_NAMESPACE_ID::Timestamp* ObjDetect::release_last_updated() { + auto temp = unsafe_arena_release_last_updated(); + if (GetArena() != nullptr) { + temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp); + } + return temp; +} +inline PROTOBUF_NAMESPACE_ID::Timestamp* ObjDetect::unsafe_arena_release_last_updated() { + // @@protoc_insertion_point(field_release:libopenshotobjdetect.ObjDetect.last_updated) + + PROTOBUF_NAMESPACE_ID::Timestamp* temp = last_updated_; + last_updated_ = nullptr; + return temp; +} +inline PROTOBUF_NAMESPACE_ID::Timestamp* ObjDetect::_internal_mutable_last_updated() { + + if (last_updated_ == nullptr) { + auto* p = CreateMaybeMessage(GetArena()); + last_updated_ = p; + } + return last_updated_; +} +inline PROTOBUF_NAMESPACE_ID::Timestamp* ObjDetect::mutable_last_updated() { + // @@protoc_insertion_point(field_mutable:libopenshotobjdetect.ObjDetect.last_updated) + return _internal_mutable_last_updated(); +} +inline void ObjDetect::set_allocated_last_updated(PROTOBUF_NAMESPACE_ID::Timestamp* last_updated) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArena(); + if (message_arena == nullptr) { + delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(last_updated_); + } + if (last_updated) { + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = + reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(last_updated)->GetArena(); + if (message_arena != submessage_arena) { + last_updated = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( + message_arena, last_updated, submessage_arena); + } + + } else { + + } + last_updated_ = last_updated; + // @@protoc_insertion_point(field_set_allocated:libopenshotobjdetect.ObjDetect.last_updated) +} + +// repeated string classNames = 3; +inline int ObjDetect::_internal_classnames_size() const { + return classnames_.size(); +} +inline int ObjDetect::classnames_size() const { + return _internal_classnames_size(); +} +inline void ObjDetect::clear_classnames() { + classnames_.Clear(); +} +inline std::string* ObjDetect::add_classnames() { + // @@protoc_insertion_point(field_add_mutable:libopenshotobjdetect.ObjDetect.classNames) + return _internal_add_classnames(); +} +inline const std::string& ObjDetect::_internal_classnames(int index) const { + return classnames_.Get(index); +} +inline const std::string& ObjDetect::classnames(int index) const { + // @@protoc_insertion_point(field_get:libopenshotobjdetect.ObjDetect.classNames) + return _internal_classnames(index); +} +inline std::string* ObjDetect::mutable_classnames(int index) { + // @@protoc_insertion_point(field_mutable:libopenshotobjdetect.ObjDetect.classNames) + return classnames_.Mutable(index); +} +inline void ObjDetect::set_classnames(int index, const std::string& value) { + // @@protoc_insertion_point(field_set:libopenshotobjdetect.ObjDetect.classNames) + classnames_.Mutable(index)->assign(value); +} +inline void ObjDetect::set_classnames(int index, std::string&& value) { + // @@protoc_insertion_point(field_set:libopenshotobjdetect.ObjDetect.classNames) + classnames_.Mutable(index)->assign(std::move(value)); +} +inline void ObjDetect::set_classnames(int index, const char* value) { + GOOGLE_DCHECK(value != nullptr); + classnames_.Mutable(index)->assign(value); + // @@protoc_insertion_point(field_set_char:libopenshotobjdetect.ObjDetect.classNames) +} +inline void ObjDetect::set_classnames(int index, const char* value, size_t size) { + classnames_.Mutable(index)->assign( + reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:libopenshotobjdetect.ObjDetect.classNames) +} +inline std::string* ObjDetect::_internal_add_classnames() { + return classnames_.Add(); +} +inline void ObjDetect::add_classnames(const std::string& value) { + classnames_.Add()->assign(value); + // @@protoc_insertion_point(field_add:libopenshotobjdetect.ObjDetect.classNames) +} +inline void ObjDetect::add_classnames(std::string&& value) { + classnames_.Add(std::move(value)); + // @@protoc_insertion_point(field_add:libopenshotobjdetect.ObjDetect.classNames) +} +inline void ObjDetect::add_classnames(const char* value) { + GOOGLE_DCHECK(value != nullptr); + classnames_.Add()->assign(value); + // @@protoc_insertion_point(field_add_char:libopenshotobjdetect.ObjDetect.classNames) +} +inline void ObjDetect::add_classnames(const char* value, size_t size) { + classnames_.Add()->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_add_pointer:libopenshotobjdetect.ObjDetect.classNames) +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& +ObjDetect::classnames() const { + // @@protoc_insertion_point(field_list:libopenshotobjdetect.ObjDetect.classNames) + return classnames_; +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* +ObjDetect::mutable_classnames() { + // @@protoc_insertion_point(field_mutable_list:libopenshotobjdetect.ObjDetect.classNames) + return &classnames_; +} + +#ifdef __GNUC__ + #pragma GCC diagnostic pop +#endif // __GNUC__ +// ------------------------------------------------------------------- + +// ------------------------------------------------------------------- + + +// @@protoc_insertion_point(namespace_scope) + +} // namespace libopenshotobjdetect + +// @@protoc_insertion_point(global_scope) + +#include +#endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_objdetectdata_2eproto diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd83fe13..aca39bff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -118,7 +118,6 @@ if (OpenCV_FOUND) execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${PROTO_H} ${CMAKE_CURRENT_SOURCE_DIR}/../include) file(REMOVE ${PROTO_H}) endforeach() - endif() @@ -214,12 +213,17 @@ set(OPENSHOT_SOURCES set(OPENSHOT_CV_SOURCES CVTracker.cpp CVStabilization.cpp - ClipProcessingJobs.cpp) + ClipProcessingJobs.cpp + CVObjectDetection.cpp + ./sort_filter/sort.cpp + ./sort_filter/Hungarian.cpp + ./sort_filter/KalmanTracker.cpp) # Compiled Protobuf messages set(PROTOBUF_MESSAGES stabilizedata.pb.cc trackerdata.pb.cc + objdetectdata.pb.cc ) # Video effects @@ -234,6 +238,7 @@ set(EFFECTS_SOURCES effects/Hue.cpp effects/Mask.cpp effects/Negate.cpp + effects/ObjectDetection.cpp effects/Pixelate.cpp effects/Saturation.cpp effects/Shift.cpp diff --git a/src/CVObjectDetection.cpp b/src/CVObjectDetection.cpp new file mode 100644 index 00000000..d124c3ac --- /dev/null +++ b/src/CVObjectDetection.cpp @@ -0,0 +1,281 @@ +/** + * @file + * @brief Source file for CVObjectDetection class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "../include/CVObjectDetection.h" + +// // Initialize the parameters +// float confThreshold = 0.5; // Confidence threshold +// float nmsThreshold = 0.4; // Non-maximum suppression threshold +// int inpWidth = 416; // Width of network's input image +// int inpHeight = 416; // Height of network's input image +// vector classes; + +CVObjectDetection::CVObjectDetection(std::string processInfoJson, ProcessingController &processingController) +: processingController(&processingController), processingDevice("CPU"){ + SetJson(processInfoJson); + setProcessingDevice(); +} + +void CVObjectDetection::setProcessingDevice(){ + if(processingDevice == "GPU"){ + net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA); + } + else if(processingDevice == "CPU"){ + net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); + } +} + +void CVObjectDetection::detectObjectsClip(openshot::Clip &video, size_t _start, size_t _end, bool process_interval) +{ + + start = _start; end = _end; + + video.Open(); + + // Load names of classes + std::ifstream ifs(classesFile.c_str()); + std::string line; + while (std::getline(ifs, line)) classNames.push_back(line); + + confThreshold = 0.5; + nmsThreshold = 0.1; + + // Load the network + net = cv::dnn::readNetFromDarknet(modelConfiguration, modelWeights); + + size_t frame_number; + if(!process_interval || end == 0 || end-start <= 0){ + // Get total number of frames in video + start = video.Start() * video.Reader()->info.fps.ToInt(); + end = video.End() * video.Reader()->info.fps.ToInt(); + } + + for (frame_number = start; frame_number <= end; frame_number++) + { + // Stop the feature tracker process + if(processingController->ShouldStop()){ + return; + } + + std::shared_ptr f = video.GetFrame(frame_number); + + // Grab OpenCV Mat image + cv::Mat cvimage = f->GetImageCV(); + + DetectObjects(cvimage, frame_number); + + // Update progress + processingController->SetProgress(uint(100*(frame_number-start)/(end-start))); + + std::cout<<"Frame: "< outs; + net.forward(outs, getOutputsNames(net)); + + // Remove the bounding boxes with low confidence + postprocess(frame.size(), outs, frameId); + +} + + +// Remove the bounding boxes with low confidence using non-maxima suppression +void CVObjectDetection::postprocess(const cv::Size &frameDims, const std::vector& outs, size_t frameId) +{ + std::vector classIds; + std::vector confidences; + std::vector boxes; + + for (size_t i = 0; i < outs.size(); ++i) + { + // Scan through all the bounding boxes output from the network and keep only the + // ones with high confidence scores. Assign the box's class label as the class + // with the highest score for the box. + float* data = (float*)outs[i].data; + for (int j = 0; j < outs[i].rows; ++j, data += outs[i].cols) + { + cv::Mat scores = outs[i].row(j).colRange(5, outs[i].cols); + cv::Point classIdPoint; + double confidence; + // Get the value and location of the maximum score + cv::minMaxLoc(scores, 0, &confidence, 0, &classIdPoint); + if (confidence > confThreshold) + { + int centerX = (int)(data[0] * frameDims.width); + int centerY = (int)(data[1] * frameDims.height); + int width = (int)(data[2] * frameDims.width); + int height = (int)(data[3] * frameDims.height); + int left = centerX - width / 2; + int top = centerY - height / 2; + + classIds.push_back(classIdPoint.x); + confidences.push_back((float)confidence); + boxes.push_back(cv::Rect(left, top, width, height)); + } + } + } + + // Perform non maximum suppression to eliminate redundant overlapping boxes with + // lower confidences + std::vector indices; + cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); + + detectionsData[frameId] = CVDetectionData(classIds, confidences, boxes, frameId); +} + +// Get the names of the output layers +std::vector CVObjectDetection::getOutputsNames(const cv::dnn::Net& net) +{ + static std::vector names; + + //Get the indices of the output layers, i.e. the layers with unconnected outputs + std::vector outLayers = net.getUnconnectedOutLayers(); + + //get the names of all the layers in the network + std::vector layersNames = net.getLayerNames(); + + // Get the names of the output layers in names + names.resize(outLayers.size()); + for (size_t i = 0; i < outLayers.size(); ++i) + names[i] = layersNames[outLayers[i] - 1]; + return names; +} + +bool CVObjectDetection::SaveTrackedData(){ + // Create tracker message + libopenshotobjdetect::ObjDetect objMessage; + + //Save class names in protobuf message + for(int i = 0; iassign(classNames.at(i)); + } + + // Iterate over all frames data and save in protobuf message + for(std::map::iterator it=detectionsData.begin(); it!=detectionsData.end(); ++it){ + CVDetectionData dData = it->second; + libopenshotobjdetect::Frame* pbFrameData; + AddFrameDataToProto(objMessage.add_frame(), dData); + } + + // Add timestamp + *objMessage.mutable_last_updated() = TimeUtil::SecondsToTimestamp(time(NULL)); + + { + // Write the new message to disk. + std::fstream output(protobuf_data_path, ios::out | ios::trunc | ios::binary); + if (!objMessage.SerializeToOstream(&output)) { + cerr << "Failed to write protobuf message." << endl; + return false; + } + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; + +} + +// Add frame object detection into protobuf message. +void CVObjectDetection::AddFrameDataToProto(libopenshotobjdetect::Frame* pbFrameData, CVDetectionData& dData) { + + // Save frame number and rotation + pbFrameData->set_id(dData.frameId); + + for(size_t i = 0; i < dData.boxes.size(); i++){ + libopenshotobjdetect::Frame_Box* box = pbFrameData->add_bounding_box(); + + // Save bounding box data + box->set_x1(dData.boxes.at(i).x); + box->set_y1(dData.boxes.at(i).y); + box->set_x2(dData.boxes.at(i).x + dData.boxes.at(i).width); + box->set_y2(dData.boxes.at(i).y + dData.boxes.at(i).height); + box->set_classid(dData.classIds.at(i)); + box->set_confidence(dData.confidences.at(i)); + + } +} + +// Load JSON string into this object +void CVObjectDetection::SetJson(const std::string value) { + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + // throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + std::cout<<"JSON is invalid (missing keys or invalid data types)"<update(frame, bbox); @@ -144,6 +147,15 @@ bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){ // Otherwise add only frame number if (ok) { + std::vector bboxes = {bbox}; + + sort.update(bboxes, frameId, sqrt(pow(frame.rows, 2) + pow(frame.cols, 2))); + + for(auto TBox : sort.frameTrackingResult) + bbox = TBox.box; + + // removeJitter.update(bbox, bbox); + // Add new frame data trackedDataById[frameId] = FrameData(frameId, 0, bbox.x, bbox.y, bbox.x+bbox.width, bbox.y+bbox.height); } @@ -301,4 +313,44 @@ void CVTracker::SetJsonValue(const Json::Value root) { start = root["first_frame"].asInt64(); json_interval = true; } -} \ No newline at end of file +} + +RemoveJitter::RemoveJitter(int boxesInterval) : boxesInterval(boxesInterval), boxesInVector(0){ +} + +void RemoveJitter::update(cv::Rect2d bbox, cv::Rect2d &out_bbox){ + + bboxTracker.push_back(bbox); + + // Just to initialize the vector properly + if(boxesInVector < boxesInterval+1){ + boxesInVector++; + out_bbox = bbox; + } + else{ + cv::Rect2d old_bbox = bboxTracker.front(); + cv::Rect2d new_bbox = bboxTracker.back(); + + int centerX_1 = old_bbox.x + old_bbox.width/2; + int centerY_1 = old_bbox.y + old_bbox.height/2; + int centerX_2 = new_bbox.x + new_bbox.width/2; + int centerY_2 = new_bbox.y + new_bbox.height/2; + + int dif_centerXs = abs(centerX_1 - centerX_2); + int dif_centerYs = abs(centerY_1 - centerY_2); + + cout< 6 || dif_centerYs > 6){ + out_bbox = new_bbox; + } + else{ + cv::Rect2d mean_bbox((old_bbox.x + new_bbox.x)/2, (old_bbox.y + new_bbox.y)/2, (old_bbox.width + new_bbox.width)/2, (old_bbox.height + new_bbox.height)/2); + out_bbox = mean_bbox; + } + + bboxTracker.erase(bboxTracker.begin()); + } +} + diff --git a/src/ClipProcessingJobs.cpp b/src/ClipProcessingJobs.cpp index 43cd288f..41eaa2b1 100644 --- a/src/ClipProcessingJobs.cpp +++ b/src/ClipProcessingJobs.cpp @@ -14,6 +14,9 @@ void ClipProcessingJobs::processClip(Clip& clip){ if(processingType == "Tracker"){ t = std::thread(&ClipProcessingJobs::trackClip, this, std::ref(clip), std::ref(this->processingController)); } + if(processingType == "Object Detector"){ + t = std::thread(&ClipProcessingJobs::detectObjectsClip, this, std::ref(clip), std::ref(this->processingController)); + } } // Apply object tracking to clip @@ -40,8 +43,28 @@ void ClipProcessingJobs::trackClip(Clip& clip, ProcessingController& controller) } // Apply stabilization to clip -void ClipProcessingJobs::stabilizeClip(Clip& clip, ProcessingController& controller){ +void ClipProcessingJobs::detectObjectsClip(Clip& clip, ProcessingController& controller){ // create CVStabilization object + CVObjectDetection objDetector(processInfoJson, controller); + // Start stabilization process + objDetector.detectObjectsClip(clip); + + // Thread controller. If effect processing is done, save data + // Else, kill thread + if(controller.ShouldStop()){ + controller.SetFinished(true); + return; + } + else{ + // Save stabilization data + objDetector.SaveTrackedData(); + // tells to UI that the processing finished + controller.SetFinished(true); + } +} + +void ClipProcessingJobs::stabilizeClip(Clip& clip, ProcessingController& controller){ + // create CVStabilization object CVStabilization stabilizer(processInfoJson, controller); // Start stabilization process stabilizer.stabilizeClip(clip); diff --git a/src/EffectInfo.cpp b/src/EffectInfo.cpp index 24d8f0a2..89348ec8 100644 --- a/src/EffectInfo.cpp +++ b/src/EffectInfo.cpp @@ -91,6 +91,9 @@ EffectBase* EffectInfo::CreateEffect(std::string effect_type) { else if(effect_type == "Tracker") return new Tracker(); + + else if(effect_type == "Object Detector") + return new ObjectDetection(); return NULL; } @@ -118,6 +121,7 @@ Json::Value EffectInfo::JsonValue() { root.append(Wave().JsonInfo()); root.append(Stabilizer().JsonInfo()); root.append(Tracker().JsonInfo()); + root.append(ObjectDetection().JsonInfo()); // return JsonValue return root; diff --git a/src/bindings/python/openshot.i b/src/bindings/python/openshot.i index bbe4196a..19cce668 100644 --- a/src/bindings/python/openshot.i +++ b/src/bindings/python/openshot.i @@ -231,6 +231,7 @@ %include "effects/Wave.h" %include "effects/Stabilizer.h" %include "effects/Tracker.h" +%include "effects/ObjectDetection.h" /* Wrap std templates (list, vector, etc...) */ diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp new file mode 100644 index 00000000..ba332cf7 --- /dev/null +++ b/src/effects/ObjectDetection.cpp @@ -0,0 +1,261 @@ +/** + * @file + * @brief Source file for Object Detection effect class + * @author Jonathan Thomas + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2019 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "../../include/effects/ObjectDetection.h" +#include "../../include/effects/Tracker.h" + +using namespace openshot; + +/// Blank constructor, useful when using Json to load the effect properties +ObjectDetection::ObjectDetection(std::string clipObDetectDataPath) +{ + // Init effect properties + init_effect_details(); + + // Tries to load the tracker data from protobuf + LoadObjDetectdData(clipObDetectDataPath); +} + +// Default constructor +ObjectDetection::ObjectDetection() +{ + // Init effect properties + init_effect_details(); + +} + +// Init effect settings +void ObjectDetection::init_effect_details() +{ + /// Initialize the values of the EffectInfo struct. + InitEffectInfo(); + + /// Set the effect info + info.class_name = "Object Detector"; + info.name = "Object Detector"; + info.description = "Detect objects through the video."; + info.has_audio = false; + info.has_video = true; +} + +// This method is required for all derived classes of EffectBase, and returns a +// modified openshot::Frame object +std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, int64_t frame_number) +{ + // Get the frame's image + cv::Mat cv_image = frame->GetImageCV(); + + // Check if frame isn't NULL + if(!cv_image.empty()){ + + // Check if track data exists for the requested frame + if (detectionsData.find(frame_number) != detectionsData.end()) { + + DetectionData detections = detectionsData[frame_number]; + for(int i = 0; iSetImageCV(cv_image); + + return frame; +} + +void ObjectDetection::drawPred(int classId, float conf, cv::Rect2d box, cv::Mat& frame) +{ + //Draw a rectangle displaying the bounding box + cv::rectangle(frame, box, cv::Scalar(255, 178, 50), 3); + + //Get the label for the class name and its confidence + std::string label = cv::format("%.2f", conf); + if (!classNames.empty()) + { + CV_Assert(classId < (int)classNames.size()); + label = classNames[classId] + ":" + label; + } + + //Display the label at the box.y of the bounding box + int baseLine; + cv::Size labelSize = getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + box.y = std::max((int)box.y, labelSize.height); + cv::rectangle(frame, cv::Point(box.x, box.y - round(1.5*labelSize.height)), cv::Point(box.x + round(1.5*labelSize.width), box.y + baseLine), cv::Scalar(255, 255, 255), cv::FILLED); + putText(frame, label, cv::Point(box.x, box.y), cv::FONT_HERSHEY_SIMPLEX, 0.75, cv::Scalar(0,0,0),1); +} + +// Load protobuf data file +bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){ + // Create tracker message + libopenshotobjdetect::ObjDetect objMessage; + + { + // Read the existing tracker message. + fstream input(inputFilePath, ios::in | ios::binary); + if (!objMessage.ParseFromIstream(&input)) { + cerr << "Failed to parse protobuf message." << endl; + return false; + } + } + + // Make sure the trackedData is empty + classNames.clear(); + detectionsData.clear(); + + for(int i = 0; i < objMessage.classnames_size(); i++){ + classNames.push_back(objMessage.classnames(i)); + } + + // Iterate over all frames of the saved message + for (size_t i = 0; i < objMessage.frame_size(); i++) { + const libopenshotobjdetect::Frame& pbFrameData = objMessage.frame(i); + + // Load frame and rotation data + size_t id = pbFrameData.id(); + + // Load bounding box data + const google::protobuf::RepeatedPtrField &box = pbFrameData.bounding_box(); + + std::vector classIds; + std::vector confidences; + std::vector boxes; + + for(int i = 0; i < pbFrameData.bounding_box_size(); i++){ + int x1 = box.at(i).x1(); + int y1 = box.at(i).y1(); + int x2 = box.at(i).x2(); + int y2 = box.at(i).y2(); + int classId = box.at(i).classid(); + float confidence = box.at(i).confidence(); + + cv::Rect2d box(x1, y1, x2-x1, y2-y1); + + boxes.push_back(box); + classIds.push_back(classId); + confidences.push_back(confidence); + } + + // Assign data to tracker map + detectionsData[id] = DetectionData(classIds, confidences, boxes, id); + } + + // Show the time stamp from the last update in tracker data file + if (objMessage.has_last_updated()) { + cout << " Loaded Data. Saved Time Stamp: " << TimeUtil::ToString(objMessage.last_updated()) << endl; + } + + // Delete all global objects allocated by libprotobuf. + google::protobuf::ShutdownProtobufLibrary(); + + return true; +} + +// Get tracker info for the desired frame +DetectionData ObjectDetection::GetTrackedData(size_t frameId){ + + // Check if the tracker info for the requested frame exists + if ( detectionsData.find(frameId) == detectionsData.end() ) { + return DetectionData(); + } else { + return detectionsData[frameId]; + } + +} + +// Generate JSON string of this object +std::string ObjectDetection::Json() const { + + // Return formatted string + return JsonValue().toStyledString(); +} + +// Generate Json::Value for this object +Json::Value ObjectDetection::JsonValue() const { + + // Create root json object + Json::Value root = EffectBase::JsonValue(); // get parent properties + root["type"] = info.class_name; + root["protobuf_data_path"] = protobuf_data_path; + + // return JsonValue + return root; +} + +// Load JSON string into this object +void ObjectDetection::SetJson(const std::string value) { + + // Parse JSON string into JSON objects + try + { + const Json::Value root = openshot::stringToJson(value); + // Set all values that match + SetJsonValue(root); + } + catch (const std::exception& e) + { + // Error parsing JSON (or missing keys) + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +// Load Json::Value into this object +void ObjectDetection::SetJsonValue(const Json::Value root) { + + // Set parent data + EffectBase::SetJsonValue(root); + // Set data from Json (if key is found) + if (!root["protobuf_data_path"].isNull()){ + protobuf_data_path = (root["protobuf_data_path"].asString()); + + if(!LoadObjDetectdData(protobuf_data_path)){ + std::cout<<"Invalid protobuf data path"; + protobuf_data_path = ""; + } + } +} + +// Get all properties for a specific frame +std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const { + + // Generate JSON properties list + Json::Value root; + root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame); + root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame); + root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame); + root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame); + + // Return formatted string + return root.toStyledString(); +} diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index 77bb196b..b2f479d8 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -139,7 +139,7 @@ bool Tracker::LoadTrackedData(std::string inputFilePath){ } // Get tracker info for the desired frame -EffectFrameData Tracker::GetTrackedData(int frameId){ +EffectFrameData Tracker::GetTrackedData(size_t frameId){ // Check if the tracker info for the requested frame exists if ( trackedDataById.find(frameId) == trackedDataById.end() ) { diff --git a/src/examples/Example_opencv.cpp b/src/examples/Example_opencv.cpp index 2ed03408..2d196710 100644 --- a/src/examples/Example_opencv.cpp +++ b/src/examples/Example_opencv.cpp @@ -33,6 +33,7 @@ #include #include "../../include/CVTracker.h" #include "../../include/CVStabilization.h" +#include "../../include/CVObjectDetection.h" #include "../../include/OpenShot.h" #include "../../include/CrashHandler.h" @@ -40,6 +41,20 @@ using namespace openshot; using namespace std; +/* +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +The following methods are just for getting JSON info to the pre-processing effects +*/ + +string jsonFormat(string key, string value, string type="string"); // Format variables to the needed JSON format +string trackerJson(cv::Rect2d r, bool onlyProtoPath); // Set variable values for tracker effect +string stabilizerJson(bool onlyProtoPath); // Set variable values for stabilizer effect +string objectDetectionJson(bool onlyProtoPath); // Set variable values for object detector effect + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + // Show the pre-processed clip on the screen void displayClip(openshot::Clip &r9){ @@ -57,8 +72,7 @@ void displayClip(openshot::Clip &r9){ std::shared_ptr f = r9.GetFrame(frame_number); // Grab OpenCV::Mat image cv::Mat cvimage = f->GetImageCV(); - // Convert color scheme from RGB (QImage scheme) to BGR (OpenCV scheme) - cv::cvtColor(cvimage, cvimage, cv::COLOR_RGB2BGR); + // Display the frame cv::imshow("Display Image", cvimage); @@ -71,73 +85,18 @@ void displayClip(openshot::Clip &r9){ cv::destroyAllWindows(); } - -/* -|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| - -The following methods are just for getting JSON info to the pre-processing effects - -|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| -*/ - - -// Return JSON string for the tracker effect -string trackerJson(cv::Rect2d r, bool onlyProtoPath){ - // Set the tracker - string tracker = "KCF"; - - // Construct all the composition of the JSON string - string trackerType = "\"tracker_type\": \"" + tracker + "\""; - string protobuf_data_path = "\"protobuf_data_path\": \"kcf_tracker.data\""; - stringstream bboxCoords; - bboxCoords << "\"bbox\": {\"x\":"< Used for object detector + // test.avi --> Used for tracker and stabilizer // Thread controller just for the pre-processing constructors, it won't be used ProcessingController processingController; @@ -146,7 +105,7 @@ int main(int argc, char* argv[]) { openshot::Clip r9(path.str()); r9.Open(); - // Aplly tracking effect on the clip + // Apply tracking effect on the clip if(TRACK_DATA){ // Take the bounding box coordinates @@ -159,7 +118,7 @@ int main(int argc, char* argv[]) { CVTracker tracker(trackerJson(r, false), processingController); // Start the tracking - tracker.trackClip(r9); + tracker.trackClip(r9, 0, 100, true); // Save the tracked data tracker.SaveTrackedData(); @@ -173,7 +132,7 @@ int main(int argc, char* argv[]) { r9.AddEffect(e); } - // Aplly stabilizer effect on the clip + // Apply stabilizer effect on the clip if(SMOOTH_VIDEO){ // Create a stabilizer object by passing a JSON string and a thread controller, this last one won't be used @@ -181,7 +140,7 @@ int main(int argc, char* argv[]) { CVStabilization stabilizer(stabilizerJson(false), processingController); // Start the stabilization - stabilizer.stabilizeClip(r9); + stabilizer.stabilizeClip(r9, 0, 100, true); // Save the stabilization data stabilizer.SaveStabilizedData(); @@ -195,9 +154,27 @@ int main(int argc, char* argv[]) { r9.AddEffect(e); } + // Apply object detection effect on the clip if(OBJECT_DETECTION_DATA){ - // CVObjectDetection objectDetection("GPU"); - // objectDetection.ProcessClip(r9); + + // Create a object detection object by passing a JSON string and a thread controller, this last one won't be used + // JSON info: path to save the detection data, processing devicee, model weights, model configuration and class names + CVObjectDetection objectDetection(objectDetectionJson(false), processingController); + + // Start the object detection + objectDetection.detectObjectsClip(r9, 0, 100, true); + // Save the object detection data + objectDetection.SaveTrackedData(); + + // Create a object detector effect + EffectBase* e = EffectInfo().CreateEffect("Object Detector"); + + // Pass a JSON string with the saved detections data + // The effect will read and save the detections in a map:: + + e->SetJson(objectDetectionJson(true)); + // Add the effect to the clip + r9.AddEffect(e); } // Show the pre-processed clip on the screen @@ -210,3 +187,111 @@ int main(int argc, char* argv[]) { return 0; } + + + +/* +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| + +The following methods are just for getting JSON info to the pre-processing effects + +|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +*/ + + + +string jsonFormat(string key, string value, string type){ + stringstream jsonFormatMessage; + jsonFormatMessage << ( "\"" + key + "\": " ); + + if(type == "string") + jsonFormatMessage << ( "\"" + value + "\"" ); + if(type == "rstring") + jsonFormatMessage << value; + if(type == "int") + jsonFormatMessage << stoi(value); + if(type == "float") + jsonFormatMessage << (float)stof(value); + if(type == "double") + jsonFormatMessage << (double)stof(value); + if (type == "bool") + jsonFormatMessage << ((value == "true" || value == "1") ? "true" : "false"); + + return jsonFormatMessage.str(); +} + +// Return JSON string for the tracker effect +string trackerJson(cv::Rect2d r, bool onlyProtoPath){ + + // Define path to save tracked data + string protobufDataPath = "kcf_tracker.data"; + // Set the tracker + string tracker = "KCF"; + + // Construct all the composition of the JSON string + string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath); + string trackerType = jsonFormat("tracker_type", tracker); + string bboxCoords = jsonFormat( + "bbox", + "{" + jsonFormat("x", to_string(r.x), "int") + + "," + jsonFormat("y", to_string(r.y), "int") + + "," + jsonFormat("w", to_string(r.width), "int") + + "," + jsonFormat("h", to_string(r.height), "int") + + "}", + "rstring"); + + // Return only the the protobuf path in JSON format + if(onlyProtoPath) + return "{" + protobuf_data_path + "}"; + // Return all the parameters for the pre-processing effect + else + return "{" + protobuf_data_path + "," + trackerType + "," + bboxCoords + "}"; +} + +// Return JSON string for the stabilizer effect +string stabilizerJson(bool onlyProtoPath){ + + // Define path to save stabilized data + string protobufDataPath = "example_stabilizer.data"; + // Set smoothing window value + string smoothingWindow = "30"; + + // Construct all the composition of the JSON string + string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath); + string smoothing_window = jsonFormat("smoothing_window", smoothingWindow, "int"); + + // Return only the the protobuf path in JSON format + if(onlyProtoPath) + return "{" + protobuf_data_path + "}"; + // Return all the parameters for the pre-processing effect + else + return "{" + protobuf_data_path + "," + smoothing_window + "}"; +} + +string objectDetectionJson(bool onlyProtoPath){ + + // Define path to save object detection data + string protobufDataPath = "example_object_detection.data"; + // Define processing device + string processingDevice = "GPU"; + // Set path to model configuration file + string modelConfiguration = "yolov3.cfg"; + // Set path to model weights + string modelWeights = "yolov3.weights"; + // Set path to class names file + string classesFile = "obj.names"; + + // Construct all the composition of the JSON string + string protobuf_data_path = jsonFormat("protobuf_data_path", protobufDataPath); + string processing_device = jsonFormat("processing_device", processingDevice); + string model_configuration = jsonFormat("model_configuration", modelConfiguration); + string model_weights = jsonFormat("model_weights", modelWeights); + string classes_file = jsonFormat("classes_file", classesFile); + + // Return only the the protobuf path in JSON format + if(onlyProtoPath) + return "{" + protobuf_data_path + "}"; + else + return "{" + protobuf_data_path + "," + processing_device + "," + model_configuration + "," + + model_weights + "," + classes_file + "}"; +} diff --git a/src/examples/test_video.mp4 b/src/examples/test_video.mp4 new file mode 100644 index 00000000..b648984c Binary files /dev/null and b/src/examples/test_video.mp4 differ diff --git a/src/objdetectdata.pb.cc b/src/objdetectdata.pb.cc new file mode 100644 index 00000000..2d20dba0 --- /dev/null +++ b/src/objdetectdata.pb.cc @@ -0,0 +1,1011 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: objdetectdata.proto + +#include "objdetectdata.pb.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +// @@protoc_insertion_point(includes) +#include +extern PROTOBUF_INTERNAL_EXPORT_google_2fprotobuf_2ftimestamp_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Timestamp_google_2fprotobuf_2ftimestamp_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_objdetectdata_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_Frame_objdetectdata_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_objdetectdata_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Frame_Box_objdetectdata_2eproto; +namespace libopenshotobjdetect { +class Frame_BoxDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _Frame_Box_default_instance_; +class FrameDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _Frame_default_instance_; +class ObjDetectDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _ObjDetect_default_instance_; +} // namespace libopenshotobjdetect +static void InitDefaultsscc_info_Frame_objdetectdata_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::libopenshotobjdetect::_Frame_default_instance_; + new (ptr) ::libopenshotobjdetect::Frame(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::libopenshotobjdetect::Frame::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_Frame_objdetectdata_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, 0, InitDefaultsscc_info_Frame_objdetectdata_2eproto}, { + &scc_info_Frame_Box_objdetectdata_2eproto.base,}}; + +static void InitDefaultsscc_info_Frame_Box_objdetectdata_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::libopenshotobjdetect::_Frame_Box_default_instance_; + new (ptr) ::libopenshotobjdetect::Frame_Box(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::libopenshotobjdetect::Frame_Box::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Frame_Box_objdetectdata_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_Frame_Box_objdetectdata_2eproto}, {}}; + +static void InitDefaultsscc_info_ObjDetect_objdetectdata_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::libopenshotobjdetect::_ObjDetect_default_instance_; + new (ptr) ::libopenshotobjdetect::ObjDetect(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::libopenshotobjdetect::ObjDetect::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_ObjDetect_objdetectdata_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, 0, InitDefaultsscc_info_ObjDetect_objdetectdata_2eproto}, { + &scc_info_Frame_objdetectdata_2eproto.base, + &scc_info_Timestamp_google_2fprotobuf_2ftimestamp_2eproto.base,}}; + +static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_objdetectdata_2eproto[3]; +static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum_descriptors_objdetectdata_2eproto = nullptr; +static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_objdetectdata_2eproto = nullptr; + +const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_objdetectdata_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame_Box, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame_Box, x1_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame_Box, y1_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame_Box, x2_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame_Box, y2_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame_Box, classid_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame_Box, confidence_), + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame, id_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::Frame, bounding_box_), + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::ObjDetect, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::ObjDetect, frame_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::ObjDetect, last_updated_), + PROTOBUF_FIELD_OFFSET(::libopenshotobjdetect::ObjDetect, classnames_), +}; +static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { + { 0, -1, sizeof(::libopenshotobjdetect::Frame_Box)}, + { 11, -1, sizeof(::libopenshotobjdetect::Frame)}, + { 18, -1, sizeof(::libopenshotobjdetect::ObjDetect)}, +}; + +static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { + reinterpret_cast(&::libopenshotobjdetect::_Frame_Box_default_instance_), + reinterpret_cast(&::libopenshotobjdetect::_Frame_default_instance_), + reinterpret_cast(&::libopenshotobjdetect::_ObjDetect_default_instance_), +}; + +const char descriptor_table_protodef_objdetectdata_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = + "\n\023objdetectdata.proto\022\024libopenshotobjdet" + "ect\032\037google/protobuf/timestamp.proto\"\246\001\n" + "\005Frame\022\n\n\002id\030\001 \001(\005\0225\n\014bounding_box\030\002 \003(\013" + "2\037.libopenshotobjdetect.Frame.Box\032Z\n\003Box" + "\022\n\n\002x1\030\001 \001(\005\022\n\n\002y1\030\002 \001(\005\022\n\n\002x2\030\003 \001(\005\022\n\n\002" + "y2\030\004 \001(\005\022\017\n\007classId\030\005 \001(\005\022\022\n\nconfidence\030" + "\006 \001(\002\"}\n\tObjDetect\022*\n\005frame\030\001 \003(\0132\033.libo" + "penshotobjdetect.Frame\0220\n\014last_updated\030\002" + " \001(\0132\032.google.protobuf.Timestamp\022\022\n\nclas" + "sNames\030\003 \003(\tb\006proto3" + ; +static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_objdetectdata_2eproto_deps[1] = { + &::descriptor_table_google_2fprotobuf_2ftimestamp_2eproto, +}; +static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_objdetectdata_2eproto_sccs[3] = { + &scc_info_Frame_objdetectdata_2eproto.base, + &scc_info_Frame_Box_objdetectdata_2eproto.base, + &scc_info_ObjDetect_objdetectdata_2eproto.base, +}; +static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_objdetectdata_2eproto_once; +const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_objdetectdata_2eproto = { + false, false, descriptor_table_protodef_objdetectdata_2eproto, "objdetectdata.proto", 380, + &descriptor_table_objdetectdata_2eproto_once, descriptor_table_objdetectdata_2eproto_sccs, descriptor_table_objdetectdata_2eproto_deps, 3, 1, + schemas, file_default_instances, TableStruct_objdetectdata_2eproto::offsets, + file_level_metadata_objdetectdata_2eproto, 3, file_level_enum_descriptors_objdetectdata_2eproto, file_level_service_descriptors_objdetectdata_2eproto, +}; + +// Force running AddDescriptors() at dynamic initialization time. +static bool dynamic_init_dummy_objdetectdata_2eproto = (static_cast(::PROTOBUF_NAMESPACE_ID::internal::AddDescriptors(&descriptor_table_objdetectdata_2eproto)), true); +namespace libopenshotobjdetect { + +// =================================================================== + +void Frame_Box::InitAsDefaultInstance() { +} +class Frame_Box::_Internal { + public: +}; + +Frame_Box::Frame_Box(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::Message(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:libopenshotobjdetect.Frame.Box) +} +Frame_Box::Frame_Box(const Frame_Box& from) + : ::PROTOBUF_NAMESPACE_ID::Message() { + _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); + ::memcpy(&x1_, &from.x1_, + static_cast(reinterpret_cast(&confidence_) - + reinterpret_cast(&x1_)) + sizeof(confidence_)); + // @@protoc_insertion_point(copy_constructor:libopenshotobjdetect.Frame.Box) +} + +void Frame_Box::SharedCtor() { + ::memset(&x1_, 0, static_cast( + reinterpret_cast(&confidence_) - + reinterpret_cast(&x1_)) + sizeof(confidence_)); +} + +Frame_Box::~Frame_Box() { + // @@protoc_insertion_point(destructor:libopenshotobjdetect.Frame.Box) + SharedDtor(); + _internal_metadata_.Delete<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); +} + +void Frame_Box::SharedDtor() { + GOOGLE_DCHECK(GetArena() == nullptr); +} + +void Frame_Box::ArenaDtor(void* object) { + Frame_Box* _this = reinterpret_cast< Frame_Box* >(object); + (void)_this; +} +void Frame_Box::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} +void Frame_Box::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const Frame_Box& Frame_Box::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Frame_Box_objdetectdata_2eproto.base); + return *internal_default_instance(); +} + + +void Frame_Box::Clear() { +// @@protoc_insertion_point(message_clear_start:libopenshotobjdetect.Frame.Box) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + ::memset(&x1_, 0, static_cast( + reinterpret_cast(&confidence_) - + reinterpret_cast(&x1_)) + sizeof(confidence_)); + _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); +} + +const char* Frame_Box::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // int32 x1 = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) { + x1_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // int32 y1 = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + y1_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // int32 x2 = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + x2_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // int32 y2 = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { + y2_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // int32 classId = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) { + classid_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // float confidence = 6; + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 53)) { + confidence_ = ::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad(ptr); + ptr += sizeof(float); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} + +::PROTOBUF_NAMESPACE_ID::uint8* Frame_Box::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:libopenshotobjdetect.Frame.Box) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // int32 x1 = 1; + if (this->x1() != 0) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_x1(), target); + } + + // int32 y1 = 2; + if (this->y1() != 0) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->_internal_y1(), target); + } + + // int32 x2 = 3; + if (this->x2() != 0) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->_internal_x2(), target); + } + + // int32 y2 = 4; + if (this->y2() != 0) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->_internal_y2(), target); + } + + // int32 classId = 5; + if (this->classid() != 0) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(5, this->_internal_classid(), target); + } + + // float confidence = 6; + if (!(this->confidence() <= 0 && this->confidence() >= 0)) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatToArray(6, this->_internal_confidence(), target); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); + } + // @@protoc_insertion_point(serialize_to_array_end:libopenshotobjdetect.Frame.Box) + return target; +} + +size_t Frame_Box::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:libopenshotobjdetect.Frame.Box) + size_t total_size = 0; + + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // int32 x1 = 1; + if (this->x1() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_x1()); + } + + // int32 y1 = 2; + if (this->y1() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_y1()); + } + + // int32 x2 = 3; + if (this->x2() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_x2()); + } + + // int32 y2 = 4; + if (this->y2() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_y2()); + } + + // int32 classId = 5; + if (this->classid() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_classid()); + } + + // float confidence = 6; + if (!(this->confidence() <= 0 && this->confidence() >= 0)) { + total_size += 1 + 4; + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + return ::PROTOBUF_NAMESPACE_ID::internal::ComputeUnknownFieldsSize( + _internal_metadata_, total_size, &_cached_size_); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void Frame_Box::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:libopenshotobjdetect.Frame.Box) + GOOGLE_DCHECK_NE(&from, this); + const Frame_Box* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:libopenshotobjdetect.Frame.Box) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:libopenshotobjdetect.Frame.Box) + MergeFrom(*source); + } +} + +void Frame_Box::MergeFrom(const Frame_Box& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:libopenshotobjdetect.Frame.Box) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + if (from.x1() != 0) { + _internal_set_x1(from._internal_x1()); + } + if (from.y1() != 0) { + _internal_set_y1(from._internal_y1()); + } + if (from.x2() != 0) { + _internal_set_x2(from._internal_x2()); + } + if (from.y2() != 0) { + _internal_set_y2(from._internal_y2()); + } + if (from.classid() != 0) { + _internal_set_classid(from._internal_classid()); + } + if (!(from.confidence() <= 0 && from.confidence() >= 0)) { + _internal_set_confidence(from._internal_confidence()); + } +} + +void Frame_Box::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:libopenshotobjdetect.Frame.Box) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void Frame_Box::CopyFrom(const Frame_Box& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:libopenshotobjdetect.Frame.Box) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool Frame_Box::IsInitialized() const { + return true; +} + +void Frame_Box::InternalSwap(Frame_Box* other) { + using std::swap; + _internal_metadata_.Swap<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(&other->_internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::internal::memswap< + PROTOBUF_FIELD_OFFSET(Frame_Box, confidence_) + + sizeof(Frame_Box::confidence_) + - PROTOBUF_FIELD_OFFSET(Frame_Box, x1_)>( + reinterpret_cast(&x1_), + reinterpret_cast(&other->x1_)); +} + +::PROTOBUF_NAMESPACE_ID::Metadata Frame_Box::GetMetadata() const { + return GetMetadataStatic(); +} + + +// =================================================================== + +void Frame::InitAsDefaultInstance() { +} +class Frame::_Internal { + public: +}; + +Frame::Frame(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::Message(arena), + bounding_box_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:libopenshotobjdetect.Frame) +} +Frame::Frame(const Frame& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + bounding_box_(from.bounding_box_) { + _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); + id_ = from.id_; + // @@protoc_insertion_point(copy_constructor:libopenshotobjdetect.Frame) +} + +void Frame::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_Frame_objdetectdata_2eproto.base); + id_ = 0; +} + +Frame::~Frame() { + // @@protoc_insertion_point(destructor:libopenshotobjdetect.Frame) + SharedDtor(); + _internal_metadata_.Delete<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); +} + +void Frame::SharedDtor() { + GOOGLE_DCHECK(GetArena() == nullptr); +} + +void Frame::ArenaDtor(void* object) { + Frame* _this = reinterpret_cast< Frame* >(object); + (void)_this; +} +void Frame::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} +void Frame::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const Frame& Frame::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Frame_objdetectdata_2eproto.base); + return *internal_default_instance(); +} + + +void Frame::Clear() { +// @@protoc_insertion_point(message_clear_start:libopenshotobjdetect.Frame) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + bounding_box_.Clear(); + id_ = 0; + _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); +} + +const char* Frame::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // int32 id = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) { + id_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // repeated .libopenshotobjdetect.Frame.Box bounding_box = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(_internal_add_bounding_box(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<18>(ptr)); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} + +::PROTOBUF_NAMESPACE_ID::uint8* Frame::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:libopenshotobjdetect.Frame) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // int32 id = 1; + if (this->id() != 0) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(1, this->_internal_id(), target); + } + + // repeated .libopenshotobjdetect.Frame.Box bounding_box = 2; + for (unsigned int i = 0, + n = static_cast(this->_internal_bounding_box_size()); i < n; i++) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(2, this->_internal_bounding_box(i), target, stream); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); + } + // @@protoc_insertion_point(serialize_to_array_end:libopenshotobjdetect.Frame) + return target; +} + +size_t Frame::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:libopenshotobjdetect.Frame) + size_t total_size = 0; + + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // repeated .libopenshotobjdetect.Frame.Box bounding_box = 2; + total_size += 1UL * this->_internal_bounding_box_size(); + for (const auto& msg : this->bounding_box_) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); + } + + // int32 id = 1; + if (this->id() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->_internal_id()); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + return ::PROTOBUF_NAMESPACE_ID::internal::ComputeUnknownFieldsSize( + _internal_metadata_, total_size, &_cached_size_); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void Frame::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:libopenshotobjdetect.Frame) + GOOGLE_DCHECK_NE(&from, this); + const Frame* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:libopenshotobjdetect.Frame) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:libopenshotobjdetect.Frame) + MergeFrom(*source); + } +} + +void Frame::MergeFrom(const Frame& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:libopenshotobjdetect.Frame) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + bounding_box_.MergeFrom(from.bounding_box_); + if (from.id() != 0) { + _internal_set_id(from._internal_id()); + } +} + +void Frame::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:libopenshotobjdetect.Frame) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void Frame::CopyFrom(const Frame& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:libopenshotobjdetect.Frame) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool Frame::IsInitialized() const { + return true; +} + +void Frame::InternalSwap(Frame* other) { + using std::swap; + _internal_metadata_.Swap<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(&other->_internal_metadata_); + bounding_box_.InternalSwap(&other->bounding_box_); + swap(id_, other->id_); +} + +::PROTOBUF_NAMESPACE_ID::Metadata Frame::GetMetadata() const { + return GetMetadataStatic(); +} + + +// =================================================================== + +void ObjDetect::InitAsDefaultInstance() { + ::libopenshotobjdetect::_ObjDetect_default_instance_._instance.get_mutable()->last_updated_ = const_cast< PROTOBUF_NAMESPACE_ID::Timestamp*>( + PROTOBUF_NAMESPACE_ID::Timestamp::internal_default_instance()); +} +class ObjDetect::_Internal { + public: + static const PROTOBUF_NAMESPACE_ID::Timestamp& last_updated(const ObjDetect* msg); +}; + +const PROTOBUF_NAMESPACE_ID::Timestamp& +ObjDetect::_Internal::last_updated(const ObjDetect* msg) { + return *msg->last_updated_; +} +void ObjDetect::clear_last_updated() { + if (GetArena() == nullptr && last_updated_ != nullptr) { + delete last_updated_; + } + last_updated_ = nullptr; +} +ObjDetect::ObjDetect(::PROTOBUF_NAMESPACE_ID::Arena* arena) + : ::PROTOBUF_NAMESPACE_ID::Message(arena), + frame_(arena), + classnames_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:libopenshotobjdetect.ObjDetect) +} +ObjDetect::ObjDetect(const ObjDetect& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + frame_(from.frame_), + classnames_(from.classnames_) { + _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); + if (from._internal_has_last_updated()) { + last_updated_ = new PROTOBUF_NAMESPACE_ID::Timestamp(*from.last_updated_); + } else { + last_updated_ = nullptr; + } + // @@protoc_insertion_point(copy_constructor:libopenshotobjdetect.ObjDetect) +} + +void ObjDetect::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_ObjDetect_objdetectdata_2eproto.base); + last_updated_ = nullptr; +} + +ObjDetect::~ObjDetect() { + // @@protoc_insertion_point(destructor:libopenshotobjdetect.ObjDetect) + SharedDtor(); + _internal_metadata_.Delete<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); +} + +void ObjDetect::SharedDtor() { + GOOGLE_DCHECK(GetArena() == nullptr); + if (this != internal_default_instance()) delete last_updated_; +} + +void ObjDetect::ArenaDtor(void* object) { + ObjDetect* _this = reinterpret_cast< ObjDetect* >(object); + (void)_this; +} +void ObjDetect::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) { +} +void ObjDetect::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const ObjDetect& ObjDetect::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_ObjDetect_objdetectdata_2eproto.base); + return *internal_default_instance(); +} + + +void ObjDetect::Clear() { +// @@protoc_insertion_point(message_clear_start:libopenshotobjdetect.ObjDetect) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + frame_.Clear(); + classnames_.Clear(); + if (GetArena() == nullptr && last_updated_ != nullptr) { + delete last_updated_; + } + last_updated_ = nullptr; + _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); +} + +const char* ObjDetect::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena; + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // repeated .libopenshotobjdetect.Frame frame = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(_internal_add_frame(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<10>(ptr)); + } else goto handle_unusual; + continue; + // .google.protobuf.Timestamp last_updated = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr = ctx->ParseMessage(_internal_mutable_last_updated(), ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // repeated string classNames = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { + ptr -= 1; + do { + ptr += 1; + auto str = _internal_add_classnames(); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx); + CHK_(::PROTOBUF_NAMESPACE_ID::internal::VerifyUTF8(str, "libopenshotobjdetect.ObjDetect.classNames")); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<26>(ptr)); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, + _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(), + ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} + +::PROTOBUF_NAMESPACE_ID::uint8* ObjDetect::_InternalSerialize( + ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const { + // @@protoc_insertion_point(serialize_to_array_start:libopenshotobjdetect.ObjDetect) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // repeated .libopenshotobjdetect.Frame frame = 1; + for (unsigned int i = 0, + n = static_cast(this->_internal_frame_size()); i < n; i++) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage(1, this->_internal_frame(i), target, stream); + } + + // .google.protobuf.Timestamp last_updated = 2; + if (this->has_last_updated()) { + target = stream->EnsureSpace(target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessage( + 2, _Internal::last_updated(this), target, stream); + } + + // repeated string classNames = 3; + for (int i = 0, n = this->_internal_classnames_size(); i < n; i++) { + const auto& s = this->_internal_classnames(i); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + s.data(), static_cast(s.length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "libopenshotobjdetect.ObjDetect.classNames"); + target = stream->WriteString(3, s, target); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); + } + // @@protoc_insertion_point(serialize_to_array_end:libopenshotobjdetect.ObjDetect) + return target; +} + +size_t ObjDetect::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:libopenshotobjdetect.ObjDetect) + size_t total_size = 0; + + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // repeated .libopenshotobjdetect.Frame frame = 1; + total_size += 1UL * this->_internal_frame_size(); + for (const auto& msg : this->frame_) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(msg); + } + + // repeated string classNames = 3; + total_size += 1 * + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(classnames_.size()); + for (int i = 0, n = classnames_.size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + classnames_.Get(i)); + } + + // .google.protobuf.Timestamp last_updated = 2; + if (this->has_last_updated()) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + *last_updated_); + } + + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { + return ::PROTOBUF_NAMESPACE_ID::internal::ComputeUnknownFieldsSize( + _internal_metadata_, total_size, &_cached_size_); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void ObjDetect::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:libopenshotobjdetect.ObjDetect) + GOOGLE_DCHECK_NE(&from, this); + const ObjDetect* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:libopenshotobjdetect.ObjDetect) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:libopenshotobjdetect.ObjDetect) + MergeFrom(*source); + } +} + +void ObjDetect::MergeFrom(const ObjDetect& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:libopenshotobjdetect.ObjDetect) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + frame_.MergeFrom(from.frame_); + classnames_.MergeFrom(from.classnames_); + if (from.has_last_updated()) { + _internal_mutable_last_updated()->PROTOBUF_NAMESPACE_ID::Timestamp::MergeFrom(from._internal_last_updated()); + } +} + +void ObjDetect::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:libopenshotobjdetect.ObjDetect) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void ObjDetect::CopyFrom(const ObjDetect& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:libopenshotobjdetect.ObjDetect) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool ObjDetect::IsInitialized() const { + return true; +} + +void ObjDetect::InternalSwap(ObjDetect* other) { + using std::swap; + _internal_metadata_.Swap<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(&other->_internal_metadata_); + frame_.InternalSwap(&other->frame_); + classnames_.InternalSwap(&other->classnames_); + swap(last_updated_, other->last_updated_); +} + +::PROTOBUF_NAMESPACE_ID::Metadata ObjDetect::GetMetadata() const { + return GetMetadataStatic(); +} + + +// @@protoc_insertion_point(namespace_scope) +} // namespace libopenshotobjdetect +PROTOBUF_NAMESPACE_OPEN +template<> PROTOBUF_NOINLINE ::libopenshotobjdetect::Frame_Box* Arena::CreateMaybeMessage< ::libopenshotobjdetect::Frame_Box >(Arena* arena) { + return Arena::CreateMessageInternal< ::libopenshotobjdetect::Frame_Box >(arena); +} +template<> PROTOBUF_NOINLINE ::libopenshotobjdetect::Frame* Arena::CreateMaybeMessage< ::libopenshotobjdetect::Frame >(Arena* arena) { + return Arena::CreateMessageInternal< ::libopenshotobjdetect::Frame >(arena); +} +template<> PROTOBUF_NOINLINE ::libopenshotobjdetect::ObjDetect* Arena::CreateMaybeMessage< ::libopenshotobjdetect::ObjDetect >(Arena* arena) { + return Arena::CreateMessageInternal< ::libopenshotobjdetect::ObjDetect >(arena); +} +PROTOBUF_NAMESPACE_CLOSE + +// @@protoc_insertion_point(global_scope) +#include diff --git a/src/protobuf_messages/objdetectdata.proto b/src/protobuf_messages/objdetectdata.proto new file mode 100644 index 00000000..6ff87c8c --- /dev/null +++ b/src/protobuf_messages/objdetectdata.proto @@ -0,0 +1,32 @@ + +// [START declaration] +syntax = "proto3"; +package libopenshotobjdetect; + +import "google/protobuf/timestamp.proto"; +// [END declaration] + +// [START messages] +message Frame { + int32 id = 1; // Frame ID. + + message Box{ + int32 x1 = 1; + int32 y1 = 2; + int32 x2 = 3; + int32 y2 = 4; + int32 classId = 5; + float confidence = 6; + } + + repeated Box bounding_box = 2; +} + +message ObjDetect { + repeated Frame frame = 1; + + google.protobuf.Timestamp last_updated = 2; + + repeated string classNames = 3; +} +// [END messages] \ No newline at end of file diff --git a/src/sort_filter/Hungarian.cpp b/src/sort_filter/Hungarian.cpp new file mode 100644 index 00000000..61b9c36b --- /dev/null +++ b/src/sort_filter/Hungarian.cpp @@ -0,0 +1,456 @@ +/////////////////////////////////////////////////////////////////////////////// +// Hungarian.cpp: Implementation file for Class HungarianAlgorithm. +// +// This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren. +// The original implementation is a few mex-functions for use in MATLAB, found here: +// http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem +// +// Both this code and the orignal code are published under the BSD license. +// by Cong Ma, 2016 +// + +#include "Hungarian.h" + +using namespace std; + +HungarianAlgorithm::HungarianAlgorithm() {} +HungarianAlgorithm::~HungarianAlgorithm() {} + +//********************************************************// +// A single function wrapper for solving assignment problem. +//********************************************************// +double HungarianAlgorithm::Solve( + vector> &DistMatrix, + vector &Assignment) +{ + unsigned int nRows = DistMatrix.size(); + unsigned int nCols = DistMatrix[0].size(); + + double *distMatrixIn = new double[nRows * nCols]; + int *assignment = new int[nRows]; + double cost = 0.0; + + // Fill in the distMatrixIn. Mind the index is "i + nRows * j". + // Here the cost matrix of size MxN is defined as a double precision array of N*M elements. + // In the solving functions matrices are seen to be saved MATLAB-internally in row-order. + // (i.e. the matrix [1 2; 3 4] will be stored as a vector [1 3 2 4], NOT [1 2 3 4]). + for (unsigned int i = 0; i < nRows; i++) + for (unsigned int j = 0; j < nCols; j++) + distMatrixIn[i + nRows * j] = DistMatrix[i][j]; + + // call solving function + assignmentoptimal(assignment, &cost, distMatrixIn, nRows, nCols); + + Assignment.clear(); + for (unsigned int r = 0; r < nRows; r++) + Assignment.push_back(assignment[r]); + + delete[] distMatrixIn; + delete[] assignment; + return cost; +} + +//********************************************************// +// Solve optimal solution for assignment problem using Munkres algorithm, also known as Hungarian Algorithm. +//********************************************************// +void HungarianAlgorithm::assignmentoptimal( + int *assignment, + double *cost, + double *distMatrixIn, + int nOfRows, + int nOfColumns) +{ + double *distMatrix, *distMatrixTemp, *distMatrixEnd, *columnEnd, value, minValue; + bool *coveredColumns, *coveredRows, *starMatrix, *newStarMatrix, *primeMatrix; + int nOfElements, minDim, row, col; + + /* initialization */ + *cost = 0; + for (row = 0; row < nOfRows; row++) + assignment[row] = -1; + + /* generate working copy of distance Matrix */ + /* check if all matrix elements are positive */ + nOfElements = nOfRows * nOfColumns; + distMatrix = (double *)malloc(nOfElements * sizeof(double)); + distMatrixEnd = distMatrix + nOfElements; + + for (row = 0; row < nOfElements; row++) + { + value = distMatrixIn[row]; + if (value < 0) + cerr << "All matrix elements have to be non-negative." << endl; + distMatrix[row] = value; + } + + /* memory allocation */ + coveredColumns = (bool *)calloc(nOfColumns, sizeof(bool)); + coveredRows = (bool *)calloc(nOfRows, sizeof(bool)); + starMatrix = (bool *)calloc(nOfElements, sizeof(bool)); + primeMatrix = (bool *)calloc(nOfElements, sizeof(bool)); + newStarMatrix = (bool *)calloc(nOfElements, sizeof(bool)); /* used in step4 */ + + /* preliminary steps */ + if (nOfRows <= nOfColumns) + { + minDim = nOfRows; + + for (row = 0; row < nOfRows; row++) + { + /* find the smallest element in the row */ + distMatrixTemp = distMatrix + row; + minValue = *distMatrixTemp; + distMatrixTemp += nOfRows; + while (distMatrixTemp < distMatrixEnd) + { + value = *distMatrixTemp; + if (value < minValue) + minValue = value; + distMatrixTemp += nOfRows; + } + + /* subtract the smallest element from each element of the row */ + distMatrixTemp = distMatrix + row; + while (distMatrixTemp < distMatrixEnd) + { + *distMatrixTemp -= minValue; + distMatrixTemp += nOfRows; + } + } + + /* Steps 1 and 2a */ + for (row = 0; row < nOfRows; row++) + for (col = 0; col < nOfColumns; col++) + if (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON) + if (!coveredColumns[col]) + { + starMatrix[row + nOfRows * col] = true; + coveredColumns[col] = true; + break; + } + } + else /* if(nOfRows > nOfColumns) */ + { + minDim = nOfColumns; + + for (col = 0; col < nOfColumns; col++) + { + /* find the smallest element in the column */ + distMatrixTemp = distMatrix + nOfRows * col; + columnEnd = distMatrixTemp + nOfRows; + + minValue = *distMatrixTemp++; + while (distMatrixTemp < columnEnd) + { + value = *distMatrixTemp++; + if (value < minValue) + minValue = value; + } + + /* subtract the smallest element from each element of the column */ + distMatrixTemp = distMatrix + nOfRows * col; + while (distMatrixTemp < columnEnd) + *distMatrixTemp++ -= minValue; + } + + /* Steps 1 and 2a */ + for (col = 0; col < nOfColumns; col++) + for (row = 0; row < nOfRows; row++) + if (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON) + if (!coveredRows[row]) + { + starMatrix[row + nOfRows * col] = true; + coveredColumns[col] = true; + coveredRows[row] = true; + break; + } + for (row = 0; row < nOfRows; row++) + coveredRows[row] = false; + } + + /* move to step 2b */ + step2b(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); + + /* compute cost and remove invalid assignments */ + computeassignmentcost(assignment, cost, distMatrixIn, nOfRows); + + /* free allocated memory */ + free(distMatrix); + free(coveredColumns); + free(coveredRows); + free(starMatrix); + free(primeMatrix); + free(newStarMatrix); + + return; +} + +/********************************************************/ +void HungarianAlgorithm::buildassignmentvector( + int *assignment, + bool *starMatrix, + int nOfRows, + int nOfColumns) +{ + int row, col; + + for (row = 0; row < nOfRows; row++) + for (col = 0; col < nOfColumns; col++) + if (starMatrix[row + nOfRows * col]) + { +#ifdef ONE_INDEXING + assignment[row] = col + 1; /* MATLAB-Indexing */ +#else + assignment[row] = col; +#endif + break; + } +} + +/********************************************************/ +void HungarianAlgorithm::computeassignmentcost( + int *assignment, + double *cost, + double *distMatrix, + int nOfRows) +{ + int row, col; + + for (row = 0; row < nOfRows; row++) + { + col = assignment[row]; + if (col >= 0) + *cost += distMatrix[row + nOfRows * col]; + } +} + +/********************************************************/ +void HungarianAlgorithm::step2a( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + bool *starMatrixTemp, *columnEnd; + int col; + + /* cover every column containing a starred zero */ + for (col = 0; col < nOfColumns; col++) + { + starMatrixTemp = starMatrix + nOfRows * col; + columnEnd = starMatrixTemp + nOfRows; + while (starMatrixTemp < columnEnd) + { + if (*starMatrixTemp++) + { + coveredColumns[col] = true; + break; + } + } + } + + /* move to step 3 */ + step2b(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} + +/********************************************************/ +void HungarianAlgorithm::step2b( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + int col, nOfCoveredColumns; + + /* count covered columns */ + nOfCoveredColumns = 0; + for (col = 0; col < nOfColumns; col++) + if (coveredColumns[col]) + nOfCoveredColumns++; + + if (nOfCoveredColumns == minDim) + { + /* algorithm finished */ + buildassignmentvector(assignment, starMatrix, nOfRows, nOfColumns); + } + else + { + /* move to step 3 */ + step3(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); + } +} + +/********************************************************/ +void HungarianAlgorithm::step3( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + bool zerosFound; + int row, col, starCol; + + zerosFound = true; + while (zerosFound) + { + zerosFound = false; + for (col = 0; col < nOfColumns; col++) + if (!coveredColumns[col]) + for (row = 0; row < nOfRows; row++) + if ((!coveredRows[row]) && (fabs(distMatrix[row + nOfRows * col]) < DBL_EPSILON)) + { + /* prime zero */ + primeMatrix[row + nOfRows * col] = true; + + /* find starred zero in current row */ + for (starCol = 0; starCol < nOfColumns; starCol++) + if (starMatrix[row + nOfRows * starCol]) + break; + + if (starCol == nOfColumns) /* no starred zero found */ + { + /* move to step 4 */ + step4(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim, row, col); + return; + } + else + { + coveredRows[row] = true; + coveredColumns[starCol] = false; + zerosFound = true; + break; + } + } + } + + /* move to step 5 */ + step5(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} + +/********************************************************/ +void HungarianAlgorithm::step4( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim, + int row, + int col) +{ + int n, starRow, starCol, primeRow, primeCol; + int nOfElements = nOfRows * nOfColumns; + + /* generate temporary copy of starMatrix */ + for (n = 0; n < nOfElements; n++) + newStarMatrix[n] = starMatrix[n]; + + /* star current zero */ + newStarMatrix[row + nOfRows * col] = true; + + /* find starred zero in current column */ + starCol = col; + for (starRow = 0; starRow < nOfRows; starRow++) + if (starMatrix[starRow + nOfRows * starCol]) + break; + + while (starRow < nOfRows) + { + /* unstar the starred zero */ + newStarMatrix[starRow + nOfRows * starCol] = false; + + /* find primed zero in current row */ + primeRow = starRow; + for (primeCol = 0; primeCol < nOfColumns; primeCol++) + if (primeMatrix[primeRow + nOfRows * primeCol]) + break; + + /* star the primed zero */ + newStarMatrix[primeRow + nOfRows * primeCol] = true; + + /* find starred zero in current column */ + starCol = primeCol; + for (starRow = 0; starRow < nOfRows; starRow++) + if (starMatrix[starRow + nOfRows * starCol]) + break; + } + + /* use temporary copy as new starMatrix */ + /* delete all primes, uncover all rows */ + for (n = 0; n < nOfElements; n++) + { + primeMatrix[n] = false; + starMatrix[n] = newStarMatrix[n]; + } + for (n = 0; n < nOfRows; n++) + coveredRows[n] = false; + + /* move to step 2a */ + step2a(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} + +/********************************************************/ +void HungarianAlgorithm::step5( + int *assignment, + double *distMatrix, + bool *starMatrix, + bool *newStarMatrix, + bool *primeMatrix, + bool *coveredColumns, + bool *coveredRows, + int nOfRows, + int nOfColumns, + int minDim) +{ + double h, value; + int row, col; + + /* find smallest uncovered element h */ + h = DBL_MAX; + for (row = 0; row < nOfRows; row++) + if (!coveredRows[row]) + for (col = 0; col < nOfColumns; col++) + if (!coveredColumns[col]) + { + value = distMatrix[row + nOfRows * col]; + if (value < h) + h = value; + } + + /* add h to each covered row */ + for (row = 0; row < nOfRows; row++) + if (coveredRows[row]) + for (col = 0; col < nOfColumns; col++) + distMatrix[row + nOfRows * col] += h; + + /* subtract h from each uncovered column */ + for (col = 0; col < nOfColumns; col++) + if (!coveredColumns[col]) + for (row = 0; row < nOfRows; row++) + distMatrix[row + nOfRows * col] -= h; + + /* move to step 3 */ + step3(assignment, distMatrix, starMatrix, newStarMatrix, primeMatrix, coveredColumns, coveredRows, nOfRows, nOfColumns, minDim); +} diff --git a/src/sort_filter/Hungarian.h b/src/sort_filter/Hungarian.h new file mode 100644 index 00000000..3bd834e7 --- /dev/null +++ b/src/sort_filter/Hungarian.h @@ -0,0 +1,34 @@ +/////////////////////////////////////////////////////////////////////////////// +// Hungarian.h: Header file for Class HungarianAlgorithm. +// +// This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren. +// The original implementation is a few mex-functions for use in MATLAB, found here: +// http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem +// +// Both this code and the orignal code are published under the BSD license. +// by Cong Ma, 2016 +// + +#include +#include +#include +#include +#include + +class HungarianAlgorithm +{ +public: + HungarianAlgorithm(); + ~HungarianAlgorithm(); + double Solve(std::vector> &DistMatrix, std::vector &Assignment); + +private: + void assignmentoptimal(int *assignment, double *cost, double *distMatrix, int nOfRows, int nOfColumns); + void buildassignmentvector(int *assignment, bool *starMatrix, int nOfRows, int nOfColumns); + void computeassignmentcost(int *assignment, double *cost, double *distMatrix, int nOfRows); + void step2a(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step2b(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step3(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); + void step4(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim, int row, int col); + void step5(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); +}; diff --git a/src/sort_filter/KalmanTracker.cpp b/src/sort_filter/KalmanTracker.cpp new file mode 100644 index 00000000..0762ba56 --- /dev/null +++ b/src/sort_filter/KalmanTracker.cpp @@ -0,0 +1,117 @@ +/////////////////////////////////////////////////////////////////////////////// +// KalmanTracker.cpp: KalmanTracker Class Implementation Declaration + +#include "KalmanTracker.h" +#include + +using namespace std; +using namespace cv; + +int KalmanTracker::kf_count = 0; + +// initialize Kalman filter +void KalmanTracker::init_kf( + StateType stateMat) +{ + int stateNum = 7; + int measureNum = 4; + kf = KalmanFilter(stateNum, measureNum, 0); + + measurement = Mat::zeros(measureNum, 1, CV_32F); + + kf.transitionMatrix = (Mat_(stateNum, stateNum) << 1, 0, 0, 0, 1, 0, 0, + 0, 1, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 1, + 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 1); + + setIdentity(kf.measurementMatrix); + setIdentity(kf.processNoiseCov, Scalar::all(1e-1)); + setIdentity(kf.measurementNoiseCov, Scalar::all(1e-4)); + setIdentity(kf.errorCovPost, Scalar::all(1e-2)); + + // initialize state vector with bounding box in [cx,cy,s,r] style + kf.statePost.at(0, 0) = stateMat.x + stateMat.width / 2; + kf.statePost.at(1, 0) = stateMat.y + stateMat.height / 2; + kf.statePost.at(2, 0) = stateMat.area(); + kf.statePost.at(3, 0) = stateMat.width / stateMat.height; +} + +// Predict the estimated bounding box. +StateType KalmanTracker::predict() +{ + // predict + Mat p = kf.predict(); + m_age += 1; + + if (m_time_since_update > 0) + m_hit_streak = 0; + m_time_since_update += 1; + + StateType predictBox = get_rect_xysr(p.at(0, 0), p.at(1, 0), p.at(2, 0), p.at(3, 0)); + + m_history.push_back(predictBox); + return m_history.back(); +} + +StateType KalmanTracker::predict2() +{ + // predict + Mat p = kf.predict(); + + StateType predictBox = get_rect_xysr(p.at(0, 0), p.at(1, 0), p.at(2, 0), p.at(3, 0)); + + return predictBox; +} + +// Update the state vector with observed bounding box. +void KalmanTracker::update( + StateType stateMat) +{ + m_time_since_update = 0; + m_history.clear(); + m_hits += 1; + m_hit_streak += 1; + + // measurement + measurement.at(0, 0) = stateMat.x + stateMat.width / 2; + measurement.at(1, 0) = stateMat.y + stateMat.height / 2; + measurement.at(2, 0) = stateMat.area(); + measurement.at(3, 0) = stateMat.width / stateMat.height; + + // update + kf.correct(measurement); + // time_t now = time(0); + // convert now to string form + + // detect_times.push_back(dt); +} + +// Return the current state vector +StateType KalmanTracker::get_state() +{ + Mat s = kf.statePost; + return get_rect_xysr(s.at(0, 0), s.at(1, 0), s.at(2, 0), s.at(3, 0)); +} + +// Convert bounding box from [cx,cy,s,r] to [x,y,w,h] style. +StateType KalmanTracker::get_rect_xysr( + float cx, + float cy, + float s, + float r) +{ + float w = sqrt(s * r); + float h = s / w; + float x = (cx - w / 2); + float y = (cy - h / 2); + + if (x < 0 && cx > 0) + x = 0; + if (y < 0 && cy > 0) + y = 0; + + return StateType(x, y, w, h); +} diff --git a/src/sort_filter/KalmanTracker.h b/src/sort_filter/KalmanTracker.h new file mode 100644 index 00000000..ac9b7029 --- /dev/null +++ b/src/sort_filter/KalmanTracker.h @@ -0,0 +1,67 @@ +/////////////////////////////////////////////////////////////////////////////// +// KalmanTracker.h: KalmanTracker Class Declaration + +#ifndef KALMAN_H +#define KALMAN_H 2 + +#include "opencv2/video/tracking.hpp" +#include "opencv2/highgui/highgui.hpp" + + +#define StateType cv::Rect_ + +// This class represents the internel state of individual tracked objects observed as bounding box. +class KalmanTracker +{ +public: + KalmanTracker() + { + init_kf(StateType()); + m_time_since_update = 0; + m_hits = 0; + m_hit_streak = 0; + m_age = 0; + m_id = kf_count; + //kf_count++; + } + KalmanTracker(StateType initRect) + { + init_kf(initRect); + m_time_since_update = 0; + m_hits = 0; + m_hit_streak = 0; + m_age = 0; + m_id = kf_count; + kf_count++; + } + + ~KalmanTracker() + { + m_history.clear(); + } + + StateType predict(); + StateType predict2(); + void update(StateType stateMat); + + StateType get_state(); + StateType get_rect_xysr(float cx, float cy, float s, float r); + + static int kf_count; + + int m_time_since_update; + int m_hits; + int m_hit_streak; + int m_age; + int m_id; + +private: + void init_kf(StateType stateMat); + + cv::KalmanFilter kf; + cv::Mat measurement; + + std::vector m_history; +}; + +#endif \ No newline at end of file diff --git a/src/sort_filter/sort.cpp b/src/sort_filter/sort.cpp new file mode 100644 index 00000000..c89151b1 --- /dev/null +++ b/src/sort_filter/sort.cpp @@ -0,0 +1,204 @@ +#include "sort.hpp" + +using namespace std; + +// Constructor +SortTracker::SortTracker(int max_age, int min_hits) +{ + _min_hits = min_hits; + _max_age = max_age; +} + +// Computes IOU between two bounding boxes +double SortTracker::GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt) +{ + float in = (bb_test & bb_gt).area(); + float un = bb_test.area() + bb_gt.area() - in; + + if (un < DBL_EPSILON) + return 0; + + return (double)(in / un); +} + +// Computes centroid distance between two bounding boxes +double SortTracker::GetCentroidsDistance( + cv::Rect_ bb_test, + cv::Rect_ bb_gt) +{ + float bb_test_centroid_x = (bb_test.x + bb_test.width / 2); + float bb_test_centroid_y = (bb_test.y + bb_test.height / 2); + + float bb_gt_centroid_x = (bb_gt.x + bb_gt.width / 2); + float bb_gt_centroid_y = (bb_gt.y + bb_gt.height / 2); + + double distance = (double)sqrt(pow(bb_gt_centroid_x - bb_test_centroid_x, 2) + pow(bb_gt_centroid_y - bb_test_centroid_y, 2)); + + return distance; +} + +void SortTracker::update(vector detections_cv, int frame_count, double image_diagonal) +{ + vector detections; + if (trackers.size() == 0) // the first frame met + { + alive_tracker = false; + // initialize kalman trackers using first detections. + for (unsigned int i = 0; i < detections_cv.size(); i++) + { + TrackingBox tb; + + tb.box = cv::Rect_(detections_cv[i]); + detections.push_back(tb); + + KalmanTracker trk = KalmanTracker(detections[i].box); + trackers.push_back(trk); + } + return; + } + else + { + for (unsigned int i = 0; i < detections_cv.size(); i++) + { + TrackingBox tb; + tb.box = cv::Rect_(detections_cv[i]); + detections.push_back(tb); + } + for (auto it = frameTrackingResult.begin(); it != frameTrackingResult.end(); it++) + { + int frame_age = frame_count - it->frame; + if (frame_age >= _max_age || frame_age < 0) + { + dead_trackers_id.push_back(it->id); + } + } + } + + /////////////////////////////////////// + // 3.1. get predicted locations from existing trackers. + predictedBoxes.clear(); + for (unsigned int i = 0; i < trackers.size();) + { + cv::Rect_ pBox = trackers[i].predict(); + if (pBox.x >= 0 && pBox.y >= 0) + { + predictedBoxes.push_back(pBox); + i++; + continue; + } + trackers.erase(trackers.begin() + i); + } + + trkNum = predictedBoxes.size(); + detNum = detections.size(); + + centroid_dist_matrix.clear(); + centroid_dist_matrix.resize(trkNum, vector(detNum, 0)); + + for (unsigned int i = 0; i < trkNum; i++) // compute iou matrix as a distance matrix + { + for (unsigned int j = 0; j < detNum; j++) + { + // use 1-iou because the hungarian algorithm computes a minimum-cost assignment. + // TODO: change norm hardcode to proportion of image diagonal + double distance = SortTracker::GetCentroidsDistance(predictedBoxes[i], detections[j].box) / image_diagonal; + centroid_dist_matrix[i][j] = distance; + } + } + + HungarianAlgorithm HungAlgo; + assignment.clear(); + HungAlgo.Solve(centroid_dist_matrix, assignment); + // find matches, unmatched_detections and unmatched_predictions + unmatchedTrajectories.clear(); + unmatchedDetections.clear(); + allItems.clear(); + matchedItems.clear(); + + if (detNum > trkNum) // there are unmatched detections + { + for (unsigned int n = 0; n < detNum; n++) + allItems.insert(n); + + for (unsigned int i = 0; i < trkNum; ++i) + matchedItems.insert(assignment[i]); + + set_difference(allItems.begin(), allItems.end(), + matchedItems.begin(), matchedItems.end(), + insert_iterator>(unmatchedDetections, unmatchedDetections.begin())); + } + else if (detNum < trkNum) // there are unmatched trajectory/predictions + { + for (unsigned int i = 0; i < trkNum; ++i) + if (assignment[i] == -1) // unassigned label will be set as -1 in the assignment algorithm + unmatchedTrajectories.insert(i); + } + else + ; + + // filter out matched with low IOU + matchedPairs.clear(); + for (unsigned int i = 0; i < trkNum; ++i) + { + if (assignment[i] == -1) // pass over invalid values + continue; + if (centroid_dist_matrix[i][assignment[i]] > max_centroid_dist_norm) + { + unmatchedTrajectories.insert(i); + unmatchedDetections.insert(assignment[i]); + } + else + matchedPairs.push_back(cv::Point(i, assignment[i])); + } + + int detIdx, trkIdx; + for (unsigned int i = 0; i < matchedPairs.size(); i++) + { + trkIdx = matchedPairs[i].x; + detIdx = matchedPairs[i].y; + trackers[trkIdx].update(detections[detIdx].box); + } + + // create and initialise new trackers for unmatched detections + for (auto umd : unmatchedDetections) + { + KalmanTracker tracker = KalmanTracker(detections[umd].box); + trackers.push_back(tracker); + } + + for (auto it2 = dead_trackers_id.begin(); it2 != dead_trackers_id.end(); it2++) + { + for (unsigned int i = 0; i < trackers.size();) + { + if (trackers[i].m_id == (*it2)) + { + trackers.erase(trackers.begin() + i); + continue; + } + i++; + } + } + + // get trackers' output + frameTrackingResult.clear(); + for (unsigned int i = 0; i < trackers.size();) + { + if ((trackers[i].m_time_since_update < 1 && trackers[i].m_hit_streak >= _min_hits) || frame_count <= _min_hits) + { + alive_tracker = true; + TrackingBox res; + res.box = trackers[i].get_state(); + res.id = trackers[i].m_id; + res.frame = frame_count; + frameTrackingResult.push_back(res); + } + + // remove dead tracklet + if (trackers[i].m_time_since_update >= _max_age) + { + trackers.erase(trackers.begin() + i); + continue; + } + i++; + } +} diff --git a/src/sort_filter/sort.hpp b/src/sort_filter/sort.hpp new file mode 100644 index 00000000..c21b0e52 --- /dev/null +++ b/src/sort_filter/sort.hpp @@ -0,0 +1,56 @@ + +#include "KalmanTracker.h" +#include "Hungarian.h" + +#include +#include +#include // to format image names using setw() and setfill() +#include + +#include "opencv2/video/tracking.hpp" +#include "opencv2/highgui/highgui.hpp" + +#ifndef _OPENCV_KCFTRACKER_HPP_ +#define _OPENCV_KCFTRACKER_HPP_ +#endif + +typedef struct TrackingBox +{ + int frame = 0; + int id = 0; + cv::Rect_ box = cv::Rect_(0.0, 0.0, 0.0, 0.0); +} TrackingBox; + +class SortTracker +{ +public: + // Constructor + SortTracker(int max_age = 7, int min_hits = 2); + // Initialize tracker + + // Update position based on the new frame + void update(std::vector detection, int frame_count, double image_diagonal); + double GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt); + double GetCentroidsDistance(cv::Rect_ bb_test, cv::Rect_ bb_gt); + std::vector trackers; + + double max_centroid_dist_norm = 0.15; + + std::vector> predictedBoxes; + std::vector> centroid_dist_matrix; + std::vector assignment; + std::set unmatchedDetections; + std::set unmatchedTrajectories; + std::set allItems; + std::set matchedItems; + std::vector matchedPairs; + + std::vector frameTrackingResult; + std::vector dead_trackers_id; + + unsigned int trkNum = 0; + unsigned int detNum = 0; + int _min_hits; + int _max_age; + bool alive_tracker; +}; diff --git a/tests/CVTracker_Tests.cpp b/tests/CVTracker_Tests.cpp index 0a3a28bb..44a3e453 100644 --- a/tests/CVTracker_Tests.cpp +++ b/tests/CVTracker_Tests.cpp @@ -58,7 +58,7 @@ SUITE(CVTracker_Tests) CVTracker kcfTracker("{\"protobuf_data_path\": \"\", \"tracker_type\": \"KCF\", \"bbox\": {\"x\": 294, \"y\": 102, \"w\": 180, \"h\": 166}}", processingController); // Track clip for frames 0-20 - kcfTracker.trackClip(c1, 0, 20+1, true); + kcfTracker.trackClip(c1, 0, 20, true); // Get tracked data FrameData fd = kcfTracker.GetTrackedData(20); @@ -69,8 +69,8 @@ SUITE(CVTracker_Tests) int height = fd.y2-fd.y1; // Compare if tracked data is equal to pre-tested ones - CHECK_EQUAL(260, x); - CHECK_EQUAL(132, y); + CHECK_EQUAL(259, x); + CHECK_EQUAL(131, y); CHECK_EQUAL(180, width); CHECK_EQUAL(166, height); } @@ -91,7 +91,7 @@ SUITE(CVTracker_Tests) CVTracker kcfTracker_1("{\"protobuf_data_path\": \"kcf_tracker.data\", \"tracker_type\": \"KCF\", \"bbox\": {\"x\": 294, \"y\": 102, \"w\": 180, \"h\": 166}}", processingController); // Track clip for frames 0-20 - kcfTracker_1.trackClip(c1, 0, 20+1, true); + kcfTracker_1.trackClip(c1, 0, 20, true); // Get tracked data FrameData fd_1 = kcfTracker_1.GetTrackedData(20);