diff --git a/projects/llm_framework/main_melotts/src/runner/utils/checker.h b/projects/llm_framework/include/utils/checker.h similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/checker.h rename to projects/llm_framework/include/utils/checker.h diff --git a/projects/llm_framework/main_melotts/src/runner/utils/io.hpp b/projects/llm_framework/include/utils/io.hpp similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/io.hpp rename to projects/llm_framework/include/utils/io.hpp diff --git a/projects/llm_framework/main_melotts/src/runner/utils/logger.h b/projects/llm_framework/include/utils/logger.h similarity index 100% rename from projects/llm_framework/main_melotts/src/runner/utils/logger.h rename to projects/llm_framework/include/utils/logger.h diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index 2183831..6124c3e 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -23,7 +23,7 @@ LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] REQUIREMENTS += ['onnxruntime', 'samplerate'] - +INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] STATIC_FILES += [AFile('melotts_zh-cn.json')] diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index 50258ca..7685db1 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -21,8 +21,7 @@ LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '- LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] -INCLUDE += [ADir('src/runner')] -INCLUDE += [ADir('../include/opencv4')] +INCLUDE += [ADir('../include'), ADir('../include/opencv4')] static_file = Glob('../static_lib/module-llm/libabsl_*') static_file = Glob('../static_lib/libopencv-4.6-aarch64-none/lib/lib*') diff --git a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp similarity index 52% rename from projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp rename to projects/llm_framework/main_yolo/src/EngineWrapper.cpp index f199ee0..12bbb94 100644 --- a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.cpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp @@ -7,24 +7,27 @@ * written consent of Axera Semiconductor (Ningbo) Co., Ltd. * **************************************************************************************************/ +#include "detection.hpp" +#define UNUSE_STRUCT_OBJECT #include "EngineWrapper.hpp" #include "utils/io.hpp" #include -#include "base/detection.hpp" -static const char *strAlgoModelType[AX_ENGINE_VIRTUAL_NPU_BUTT] = {"1.6T", "3.2T"}; +static const char* strAlgoModelType[AX_ENGINE_VIRTUAL_NPU_BUTT] = {"1.6T", "3.2T"}; /// @brief npu type typedef enum axNPU_TYPE_E { AX_NPU_DEFAULT = 0, /* running under default NPU according to system */ - AX_STD_VNPU_1 = (1 << 0), /* running under STD VNPU1 */ - AX_STD_VNPU_2 = (1 << 1), /* running under STD VNPU2 */ - AX_STD_VNPU_3 = (1 << 2), /* running under STD VNPU3 */ - AX_BL_VNPU_1 = (1 << 3), /* running under BIG-LITTLE VNPU1 */ - AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ + AX_STD_VNPU_1 = (1 << 0), /* running under STD VNPU1 */ + AX_STD_VNPU_2 = (1 << 1), /* running under STD VNPU2 */ + AX_STD_VNPU_3 = (1 << 2), /* running under STD VNPU3 */ + AX_BL_VNPU_1 = (1 << 3), /* running under BIG-LITTLE VNPU1 */ + AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ } AX_NPU_TYPE_E; -static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_TYPE_T &eModelType, const AX_S32 &nNpuType, AX_U32 &nNpuSet) { +static AX_S32 CheckModelVNpu(const std::string& strModel, const AX_ENGINE_MODEL_TYPE_T& eModelType, + const AX_S32& nNpuType, AX_U32& nNpuSet) +{ AX_ENGINE_NPU_ATTR_T stNpuAttr; memset(&stNpuAttr, 0x00, sizeof(stNpuAttr)); @@ -32,29 +35,27 @@ static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_ if (ret == 0) { // VNPU DISABLE if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_DISABLE) { - nNpuSet = 0x01; // NON-VNPU (0b111) + nNpuSet = 0x01; // NON-VNPU (0b111) } // STD VNPU else if (stNpuAttr.eHardMode == AX_ENGINE_VIRTUAL_NPU_BUTT) { // 7.2T & 10.8T no allow - if (eModelType == AX_ENGINE_MODEL_TYPE1 - || eModelType == AX_ENGINE_MODEL_TYPE1) { + if (eModelType == AX_ENGINE_MODEL_TYPE1 || eModelType == AX_ENGINE_MODEL_TYPE1) { return -1; } // default STD VNPU2 if (nNpuType == 0) { - nNpuSet = 0x02; // VNPU2 (0b010) - } - else { + nNpuSet = 0x02; // VNPU2 (0b010) + } else { if (nNpuType & AX_STD_VNPU_1) { - nNpuSet |= 0x01; // VNPU1 (0b001) + nNpuSet |= 0x01; // VNPU1 (0b001) } if (nNpuType & AX_STD_VNPU_2) { - nNpuSet |= 0x02; // VNPU2 (0b010) + nNpuSet |= 0x02; // VNPU2 (0b010) } if (nNpuType & AX_STD_VNPU_3) { - nNpuSet |= 0x04; // VNPU3 (0b100) + nNpuSet |= 0x04; // VNPU3 (0b100) } } } @@ -69,14 +70,13 @@ static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_ if (nNpuType == 0) { // 7.2T default BL VNPU1 if (eModelType == AX_ENGINE_MODEL_TYPE1) { - nNpuSet = 0x01; // VNPU1 (0b001) + nNpuSet = 0x01; // VNPU1 (0b001) } // 3.6T default BL VNPU2 else { - nNpuSet = 0x02; // VNPU2 (0b010) + nNpuSet = 0x02; // VNPU2 (0b010) } - } - else { + } else { // 7.2T if (eModelType == AX_ENGINE_MODEL_TYPE1) { // no allow set to BL VNPU2 @@ -84,22 +84,21 @@ static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_ return -1; } if (nNpuType & AX_BL_VNPU_1) { - nNpuSet |= 0x01; // VNPU1 (0b001) + nNpuSet |= 0x01; // VNPU1 (0b001) } } // 3.6T else { if (nNpuType & AX_BL_VNPU_1) { - nNpuSet |= 0x01; // VNPU1 (0b001) + nNpuSet |= 0x01; // VNPU1 (0b001) } if (nNpuType & AX_BL_VNPU_2) { - nNpuSet |= 0x02; // VNPU2 (0b010) + nNpuSet |= 0x02; // VNPU2 (0b010) } } } } - } - else { + } else { printf("AX_ENGINE_GetVNPUAttr fail ret = %x\n", ret); } @@ -111,27 +110,26 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) AX_S32 ret = 0; // 1. load model - AX_BOOL bLoadModelUseCmm = AX_FALSE; - AX_CHAR *pModelBufferVirAddr = nullptr; + AX_BOOL bLoadModelUseCmm = AX_FALSE; + AX_CHAR* pModelBufferVirAddr = nullptr; AX_U64 u64ModelBufferPhyAddr = 0; - AX_U32 nModelBufferSize = 0; + AX_U32 nModelBufferSize = 0; std::vector model_buffer; if (bLoadModelUseCmm) { - if (!utils::read_file(strModelPath, (AX_VOID **)&pModelBufferVirAddr, u64ModelBufferPhyAddr, nModelBufferSize)) { + if (!utils::read_file(strModelPath, (AX_VOID**)&pModelBufferVirAddr, u64ModelBufferPhyAddr, nModelBufferSize)) { printf("ALGO read model(%s) fail\n", strModelPath); return -1; } - } - else { + } else { if (!utils::read_file(strModelPath, model_buffer)) { printf("ALGO read model(%s) fail\n", strModelPath); return -1; } pModelBufferVirAddr = model_buffer.data(); - nModelBufferSize = model_buffer.size(); + nModelBufferSize = model_buffer.size(); } auto freeModelBuffer = [&]() { @@ -139,35 +137,34 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) if (u64ModelBufferPhyAddr != 0) { AX_SYS_MemFree(u64ModelBufferPhyAddr, &pModelBufferVirAddr); } - } - else { + } else { std::vector().swap(model_buffer); } return; }; // 1.1 Get Model Type - AX_ENGINE_MODEL_TYPE_T eModelType = AX_ENGINE_MODEL_TYPE0; - ret = AX_ENGINE_GetModelType(pModelBufferVirAddr, nModelBufferSize, &eModelType); - if (0 != ret || eModelType >= AX_ENGINE_MODEL_TYPE_BUTT) { - printf("%s AX_ENGINE_GetModelType fail ret=%x, eModelType=%d\n", strModelPath, eModelType); - freeModelBuffer(); - return -1; - } + AX_ENGINE_MODEL_TYPE_T eModelType = AX_ENGINE_MODEL_TYPE0; + ret = AX_ENGINE_GetModelType(pModelBufferVirAddr, nModelBufferSize, &eModelType); + if (0 != ret || eModelType >= AX_ENGINE_MODEL_TYPE_BUTT) { + printf("%s AX_ENGINE_GetModelType fail ret=%x, eModelType=%d\n", strModelPath, eModelType); + freeModelBuffer(); + return -1; + } // 1.2 Check VNPU - AX_ENGINE_NPU_SET_T nNpuSet = 0; - ret = CheckModelVNpu(strModelPath, eModelType, nNpuType, nNpuSet); - if (0 != ret) { - printf("ALGO CheckModelVNpu fail\n"); - freeModelBuffer(); - return -1; - } + AX_ENGINE_NPU_SET_T nNpuSet = 0; + ret = CheckModelVNpu(strModelPath, eModelType, nNpuType, nNpuSet); + if (0 != ret) { + printf("ALGO CheckModelVNpu fail\n"); + freeModelBuffer(); + return -1; + } // 2. create handle AX_ENGINE_HANDLE handle = nullptr; - ret = AX_ENGINE_CreateHandle(&handle, pModelBufferVirAddr, nModelBufferSize); - auto deinit_handle = [&handle]() { + ret = AX_ENGINE_CreateHandle(&handle, pModelBufferVirAddr, nModelBufferSize); + auto deinit_handle = [&handle]() { if (handle) { AX_ENGINE_DestroyHandle(handle); } @@ -190,26 +187,26 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) // 4. set io m_io_info = nullptr; - ret = AX_ENGINE_GetIOInfo(handle, &m_io_info); + ret = AX_ENGINE_GetIOInfo(handle, &m_io_info); if (0 != ret) { return deinit_handle(); } - m_input_num = m_io_info->nInputSize; + m_input_num = m_io_info->nInputSize; m_output_num = m_io_info->nOutputSize; // 4.1 query io -// AX_IMG_FORMAT_E eDtype; -// ret = utils::query_model_input_size(m_io_info, m_input_size, eDtype);//FIXME. -// if (0 != ret) { -// printf("model(%s) query model input size fail\n", strModelPath.c_str()); -// return deinit_handle(); -// } + // AX_IMG_FORMAT_E eDtype; + // ret = utils::query_model_input_size(m_io_info, m_input_size, eDtype);//FIXME. + // if (0 != ret) { + // printf("model(%s) query model input size fail\n", strModelPath.c_str()); + // return deinit_handle(); + // } -// if (!(eDtype == AX_FORMAT_YUV420_SEMIPLANAR || eDtype == AX_FORMAT_YUV420_SEMIPLANAR_VU || -// eDtype == AX_FORMAT_RGB888 || eDtype == AX_FORMAT_BGR888)) { -// printf("model(%s) data type is: 0x%02X, unsupport\n", strModelPath, eDtype); -// return deinit_handle(); -// } + // if (!(eDtype == AX_FORMAT_YUV420_SEMIPLANAR || eDtype == AX_FORMAT_YUV420_SEMIPLANAR_VU || + // eDtype == AX_FORMAT_RGB888 || eDtype == AX_FORMAT_BGR888)) { + // printf("model(%s) data type is: 0x%02X, unsupport\n", strModelPath, eDtype); + // return deinit_handle(); + // } // 4.2 brief io #ifdef __DEBUG__ @@ -217,9 +214,9 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) utils::brief_io_info(strModelPath, m_io_info); #endif - //5. Config VNPU - // printf("model(%s) nNpuSet: 0x%08X\n", strModelPath.c_str(), nNpuSet); - // will do nothing for using create handle v2 api + // 5. Config VNPU + // printf("model(%s) nNpuSet: 0x%08X\n", strModelPath.c_str(), nNpuSet); + // will do nothing for using create handle v2 api // 6. prepare io // AX_U32 nIoDepth = (stCtx.vecOutputBufferFlag.size() == 0) ? 1 : stCtx.vecOutputBufferFlag.size(); @@ -230,20 +227,20 @@ int EngineWrapper::Init(const char* strModelPath, uint32_t nNpuType) return deinit_handle(); } - m_handle = handle; + m_handle = handle; m_hasInit = true; return 0; } -int EngineWrapper::SetInput(void* pInput, int index) { +int EngineWrapper::SetInput(void* pInput, int index) +{ return utils::push_io_input(pInput, index, m_io); } int EngineWrapper::RunSync() { - if (!m_hasInit) - return -1; + if (!m_hasInit) return -1; // 7.3 run & benchmark auto ret = AX_ENGINE_RunSync(m_handle, &m_io); @@ -256,25 +253,29 @@ int EngineWrapper::RunSync() } const char* CLASS_NAMES[] = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", - "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", - "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", - "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", - "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", - "hair drier", "toothbrush"}; + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", + "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", + "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", + "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", + "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", + "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", + "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", + "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", + "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", + "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", + "teddy bear", "hair drier", "toothbrush"}; -void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int& input_w, int& input_h, int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects) +void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int& input_w, int& input_h, + int& cls_num, float& prob_threshold, float& nms_threshold, std::vector& objects) { // std::vector objects; std::vector proposals; - for (int i = 0; i < 3; ++i) - { - auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; + for (int i = 0; i < 3; ++i) { + auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr; int32_t stride = (1 << i) * 8; - detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, cls_num); + detection::generate_proposals_yolov8_native(stride, feat_ptr, prob_threshold, proposals, input_w, input_h, + cls_num); } detection::get_out_bbox(proposals, objects, nms_threshold, input_h, input_w, mat.rows, mat.cols); fprintf(stdout, "detection num: %zu\n", objects.size()); @@ -282,21 +283,25 @@ void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const c detection::draw_objects(mat, objects, CLASS_NAMES, "yolo11_out"); } -int EngineWrapper::Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, std::vector& objects) +int EngineWrapper::Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, + float& nms_threshold, std::vector& objects) { post_process(m_io_info, &m_io, mat, input_w, input_, cls_num, pron_threshold, nms_threshold, objects); return 0; } -int EngineWrapper::GetOutput(void* pOutput, int index) { +int EngineWrapper::GetOutput(void* pOutput, int index) +{ return utils::push_io_output(pOutput, index, m_io); } -int EngineWrapper::GetInputSize(int index) { +int EngineWrapper::GetInputSize(int index) +{ return m_io.pInputs[index].nSize; } -int EngineWrapper::GetOutputSize(int index) { +int EngineWrapper::GetOutputSize(int index) +{ return m_io.pOutputs[index].nSize; } diff --git a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp similarity index 64% rename from projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp rename to projects/llm_framework/main_yolo/src/EngineWrapper.hpp index 7da4549..4c018f4 100644 --- a/projects/llm_framework/main_yolo/src/runner/EngineWrapper.hpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.hpp @@ -12,16 +12,34 @@ #include #include -#include #include "ax_engine_api.h" +#ifndef UNUSE_STRUCT_OBJECT +namespace detection { +typedef struct Object { + cv::Rect_ rect; + int label; + float prob; + cv::Point2f landmark[5]; + /* for yolov5-seg */ + cv::Mat mask; + std::vector mask_feat; + std::vector kps_feat; + /* for yolov8-obb */ + float angle; +} Object; + +} // namespace detection +#endif + class EngineWrapper { public: - EngineWrapper() : - m_hasInit(false), - m_handle(nullptr) {} + EngineWrapper() : m_hasInit(false), m_handle(nullptr) + { + } - ~EngineWrapper() { + ~EngineWrapper() + { Release(); } @@ -31,7 +49,8 @@ public: int RunSync(); - int Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, std::vector& objects); + int Post_Process(cv::Mat& mat, int& input_w, int& input_, int& cls_num, float& pron_threshold, float& nms_threshold, + std::vector& objects); int GetOutput(void* pOutput, int index); @@ -43,7 +62,7 @@ public: protected: bool m_hasInit; AX_ENGINE_HANDLE m_handle; - AX_ENGINE_IO_INFO_T *m_io_info{}; + AX_ENGINE_IO_INFO_T* m_io_info{}; AX_ENGINE_IO_T m_io{}; int m_input_num{}, m_output_num{}; -}; \ No newline at end of file +}; diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index ab9e3b3..ef6fd1c 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -5,8 +5,7 @@ */ #include "StackFlow.h" #include "EngineWrapper.hpp" -#include "base/common.hpp" -#include "base/detection.h" +#include "common.hpp" #include #include #include @@ -153,7 +152,7 @@ public: } std::vector objects; yolo_->Post_Process(src, mode_config_.img_w, mode_config_.img_h, mode_config_.cls_num, - mode_config_.pron_threshold, mode_config_.nms_threshold, objects); + mode_config_.pron_threshold, mode_config_.nms_threshold, objects); std::vector yolo_output; for (size_t i = 0; i < objects.size(); i++) { const detection::Object &obj = objects[i]; diff --git a/projects/llm_framework/main_yolo/src/runner/base/common.hpp b/projects/llm_framework/main_yolo/src/runner/base/common.hpp deleted file mode 100644 index cc5ab2a..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/common.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace common -{ - // opencv mat(h, w) - // resize cv::Size(dstw, dsth) - void get_input_data_no_letterbox(const cv::Mat& mat, std::vector& image, int model_h, int model_w, bool bgr2rgb = false) - { - cv::Mat img_new(model_h, model_w, CV_8UC3, image.data()); - cv::resize(mat, img_new, cv::Size(model_w, model_h)); - if (bgr2rgb) - { - cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); - } - } - - void get_input_data_letterbox(cv::Mat mat, std::vector& image, int letterbox_rows, int letterbox_cols, bool bgr2rgb = false) - { - /* letterbox process to support different letterbox size */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / mat.rows) < (letterbox_cols * 1.0 / mat.cols)) - { - scale_letterbox = (float)letterbox_rows * 1.0f / (float)mat.rows; - } - else - { - scale_letterbox = (float)letterbox_cols * 1.0f / (float)mat.cols; - } - resize_cols = int(scale_letterbox * (float)mat.cols); - resize_rows = int(scale_letterbox * (float)mat.rows); - - cv::Mat img_new(letterbox_rows, letterbox_cols, CV_8UC3, image.data()); - - cv::resize(mat, mat, cv::Size(resize_cols, resize_rows)); - - int top = (letterbox_rows - resize_rows) / 2; - int bot = (letterbox_rows - resize_rows + 1) / 2; - int left = (letterbox_cols - resize_cols) / 2; - int right = (letterbox_cols - resize_cols + 1) / 2; - - // Letterbox filling - cv::copyMakeBorder(mat, img_new, top, bot, left, right, cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0)); - if (bgr2rgb) - { - cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); - } - } - - void get_input_data_centercrop(cv::Mat mat, std::vector& image, int model_h, int model_w, bool bgr2rgb = false) - { - /* letterbox process to support different letterbox size */ - - /* C2C BGR */ - if (mat.channels() == 4) - { - cv::cvtColor(mat, mat, cv::COLOR_BGRA2BGR); - } - - if (mat.channels() == 1) - { - cv::cvtColor(mat, mat, cv::COLOR_GRAY2BGR); - } - - /* Center */ - int h0; - int w0; - if (mat.rows < mat.cols) - { - h0 = 256; - w0 = int(mat.cols * (256.0 / mat.rows)); - } - else - { - h0 = int(mat.rows * (256.0 / mat.cols)); - w0 = 256; - } - int center_h = int(h0 / 2); - int center_w = int(w0 / 2); - - cv::resize(mat, mat, cv::Size(w0, h0)); - - // cv::imwrite("center.jpg", mat); - - /* Crop */ - cv::Rect crop_box(center_w - int(model_w / 2), center_h - int(model_h / 2), model_w, model_h); - cv::Mat img_new(model_h, model_w, CV_8UC3, image.data()); - - cv::Mat mat_crop = mat(crop_box).clone(); - - // cv::imwrite("mat_crop.jpg", mat_crop); - mat_crop.copyTo(img_new); - - // cv::imwrite("img_new.jpg", img_new); - - /* SwapRB*/ - if (bgr2rgb) - { - cv::cvtColor(img_new, img_new, cv::COLOR_BGR2RGB); - } - } - - bool read_file(const char* fn, std::vector& data) - { - FILE* fp = fopen(fn, "r"); - if (fp != nullptr) - { - fseek(fp, 0L, SEEK_END); - auto len = ftell(fp); - fseek(fp, 0, SEEK_SET); - data.clear(); - size_t read_size = 0; - if (len > 0) - { - data.resize(len); - read_size = fread(data.data(), 1, len, fp); - } - fclose(fp); - return read_size == (size_t)len; - } - return false; - } -} // namespace common \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/runner/base/detection.h b/projects/llm_framework/main_yolo/src/runner/base/detection.h deleted file mode 100644 index cebeba1..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/detection.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include - -namespace detection -{ - - typedef struct Object - { - cv::Rect_ rect; - int label; - float prob; - cv::Point2f landmark[5]; - /* for yolov5-seg */ - cv::Mat mask; - std::vector mask_feat; - std::vector kps_feat; - /* for yolov8-obb */ - float angle; - } Object; - -} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/runner/base/detection.hpp b/projects/llm_framework/main_yolo/src/runner/base/detection.hpp deleted file mode 100644 index 775eed0..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/detection.hpp +++ /dev/null @@ -1,3122 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -namespace detection -{ - typedef struct - { - int grid0; - int grid1; - int stride; - } GridAndStride; - - // typedef struct Object - // { - // cv::Rect_ rect; - // int label; - // float prob; - // cv::Point2f landmark[5]; - // /* for yolov5-seg */ - // cv::Mat mask; - // std::vector mask_feat; - // std::vector kps_feat; - // /* for yolov8-obb */ - // float angle; - // } Object; - - /* for palm detection */ - typedef struct PalmObject - { - cv::Rect_ rect; - float prob; - cv::Point2f vertices[4]; - cv::Point2f landmarks[7]; - cv::Mat affine_trans_mat; - cv::Mat affine_trans_mat_inv; - } PalmObject; - - static inline float sigmoid(float x) - { - return static_cast(1.f / (1.f + exp(-x))); - } - - static float softmax(const float* src, float* dst, int length) - { - const float alpha = *std::max_element(src, src + length); - float denominator = 0; - float dis_sum = 0; - for (int i = 0; i < length; ++i) - { - dst[i] = exp(src[i] - alpha); - denominator += dst[i]; - } - for (int i = 0; i < length; ++i) - { - dst[i] /= denominator; - dis_sum += i * dst[i]; - } - return dis_sum; - } - - template - static inline float intersection_area(const T& a, const T& b) - { - cv::Rect_ inter = a.rect & b.rect; - return inter.area(); - } - - template - static void qsort_descent_inplace(std::vector& faceobjects, int left, int right) - { - int i = left; - int j = right; - float p = faceobjects[(left + right) / 2].prob; - - while (i <= j) - { - while (faceobjects[i].prob > p) - i++; - - while (faceobjects[j].prob < p) - j--; - - if (i <= j) - { - // swap - std::swap(faceobjects[i], faceobjects[j]); - - i++; - j--; - } - } -#pragma omp parallel sections - { -#pragma omp section - { - if (left < j) qsort_descent_inplace(faceobjects, left, j); - } -#pragma omp section - { - if (i < right) qsort_descent_inplace(faceobjects, i, right); - } - } - } - - template - static void qsort_descent_inplace(std::vector& faceobjects) - { - if (faceobjects.empty()) - return; - - qsort_descent_inplace(faceobjects, 0, faceobjects.size() - 1); - } - - template - static void nms_sorted_bboxes(const std::vector& faceobjects, std::vector& picked, float nms_threshold) - { - picked.clear(); - - const int n = faceobjects.size(); - - std::vector areas(n); - for (int i = 0; i < n; i++) - { - areas[i] = faceobjects[i].rect.area(); - } - - for (int i = 0; i < n; i++) - { - const T& a = faceobjects[i]; - - int keep = 1; - for (int j = 0; j < (int)picked.size(); j++) - { - const T& b = faceobjects[picked[j]]; - - // intersection over union - float inter_area = intersection_area(a, b); - float union_area = areas[i] + areas[picked[j]] - inter_area; - // float IoU = inter_area / union_area - if (inter_area / union_area > nms_threshold) - keep = 0; - } - - if (keep) - picked.push_back(i); - } - } - - static void generate_grids_and_stride(const int target_w, const int target_h, std::vector& strides, std::vector& grid_strides) - { - for (auto stride : strides) - { - int num_grid_w = target_w / stride; - int num_grid_h = target_h / stride; - for (int g1 = 0; g1 < num_grid_h; g1++) - { - for (int g0 = 0; g0 < num_grid_w; g0++) - { - GridAndStride gs; - gs.grid0 = g0; - gs.grid1 = g1; - gs.stride = stride; - grid_strides.push_back(gs); - } - } - } - } - - static void generate_proposals_scrfd(int feat_stride, const float* score_blob, - const float* bbox_blob, const float* kps_blob, - float prob_threshold, std::vector& faceobjects, int letterbox_cols, int letterbox_rows) - { - static float anchors[] = {-8.f, -8.f, 8.f, 8.f, -16.f, -16.f, 16.f, 16.f, -32.f, -32.f, 32.f, 32.f, -64.f, -64.f, 64.f, 64.f, -128.f, -128.f, 128.f, 128.f, -256.f, -256.f, 256.f, 256.f}; - int feat_w = letterbox_cols / feat_stride; - int feat_h = letterbox_rows / feat_stride; - int feat_size = feat_w * feat_h; - int anchor_group = 1; - if (feat_stride == 8) - anchor_group = 1; - if (feat_stride == 16) - anchor_group = 2; - if (feat_stride == 32) - anchor_group = 3; - - // generate face proposal from bbox deltas and shifted anchors - const int num_anchors = 2; - - for (int q = 0; q < num_anchors; q++) - { - // shifted anchor - float anchor_y = anchors[(anchor_group - 1) * 8 + q * 4 + 1]; - - float anchor_w = anchors[(anchor_group - 1) * 8 + q * 4 + 2] - anchors[(anchor_group - 1) * 8 + q * 4 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 8 + q * 4 + 3] - anchors[(anchor_group - 1) * 8 + q * 4 + 1]; - - for (int i = 0; i < feat_h; i++) - { - float anchor_x = anchors[(anchor_group - 1) * 8 + q * 4 + 0]; - - for (int j = 0; j < feat_w; j++) - { - int index = i * feat_w + j; - - float prob = sigmoid(score_blob[q * feat_size + index]); - - if (prob >= prob_threshold) - { - // insightface/detection/scrfd/mmdet/models/dense_heads/scrfd_head.py _get_bboxes_single() - float dx = bbox_blob[(q * 4 + 0) * feat_size + index] * feat_stride; - float dy = bbox_blob[(q * 4 + 1) * feat_size + index] * feat_stride; - float dw = bbox_blob[(q * 4 + 2) * feat_size + index] * feat_stride; - float dh = bbox_blob[(q * 4 + 3) * feat_size + index] * feat_stride; - // insightface/detection/scrfd/mmdet/core/bbox/transforms.py distance2bbox() - float cx = anchor_x + anchor_w * 0.5f; - float cy = anchor_y + anchor_h * 0.5f; - - float x0 = cx - dx; - float y0 = cy - dy; - float x1 = cx + dw; - float y1 = cy + dh; - - Object obj; - obj.label = 0; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0 + 1; - obj.rect.height = y1 - y0 + 1; - obj.prob = prob; - - if (kps_blob != 0) - { - obj.landmark[0].x = cx + kps_blob[index] * feat_stride; - obj.landmark[0].y = cy + kps_blob[1 * feat_h * feat_w + index] * feat_stride; - obj.landmark[1].x = cx + kps_blob[2 * feat_h * feat_w + index] * feat_stride; - obj.landmark[1].y = cy + kps_blob[3 * feat_h * feat_w + index] * feat_stride; - obj.landmark[2].x = cx + kps_blob[4 * feat_h * feat_w + index] * feat_stride; - obj.landmark[2].y = cy + kps_blob[5 * feat_h * feat_w + index] * feat_stride; - obj.landmark[3].x = cx + kps_blob[6 * feat_h * feat_w + index] * feat_stride; - obj.landmark[3].y = cy + kps_blob[7 * feat_h * feat_w + index] * feat_stride; - obj.landmark[4].x = cx + kps_blob[8 * feat_h * feat_w + index] * feat_stride; - obj.landmark[4].y = cy + kps_blob[9 * feat_h * feat_w + index] * feat_stride; - } - - faceobjects.push_back(obj); - } - - anchor_x += feat_stride; - } - - anchor_y += feat_stride; - } - } - } - - static void generate_proposals_mobilenet_ssd(const float* score, const float* boxes, const int head_count, const int* feature_map_size, const int* anchor_size, const int cls_num, - float prob_threshold, const float* strides, const float center_val, const float scale_val, const float* anchor_info, std::vector& objects) - { - auto ptr_score = score; - auto ptr_boxes = boxes; - auto ptr_anchor_info = anchor_info; - for (int head = 0; head < head_count; ++head) - { - for (int fea_h = 0; fea_h < feature_map_size[head]; ++fea_h) - { - for (int fea_w = 0; fea_w < feature_map_size[head]; ++fea_w) - { - for (int anchor_i = 0; anchor_i < anchor_size[head]; ++anchor_i) - { - float softmax_sum = 0; - float class_score = -FLT_MAX; - int class_index = 0; - for (int s = 0; s < cls_num + 1; s++) - { - softmax_sum += std::exp(ptr_score[s]); - } - for (int i = 0; i < cls_num + 1; ++i) - { - float temp = std::exp(ptr_score[i]) / softmax_sum; - // if (temp > class_score) - // { - // class_index = i; - // class_score = temp; - // } - - class_index = i; - class_score = temp; - - if (temp >= prob_threshold and class_index != 0) - { - // fprintf(stderr, "class_score: %f %d \n", class_score, i); - - float pred_x = (((float)fea_w + 0.5f) / (300.0f / strides[head]) + ptr_boxes[0] * center_val * ptr_anchor_info[anchor_i * 2] / 300.0f); - float pred_y = (((float)fea_h + 0.5f) / (300.0f / strides[head]) + ptr_boxes[1] * center_val * ptr_anchor_info[anchor_i * 2 + 1] / 300.0f); - float pred_w = std::exp(ptr_boxes[2] * scale_val) * ptr_anchor_info[anchor_i * 2] / 300.0f; - float pred_h = std::exp(ptr_boxes[3] * scale_val) * ptr_anchor_info[anchor_i * 2 + 1] / 300.0f; - - float x0 = (pred_x - pred_w * 0.5f) * 300.0f; - float y0 = (pred_y - pred_h * 0.5f) * 300.0f; - float x1 = (pred_x + pred_w * 0.5f) * 300.0f; - float y1 = (pred_y + pred_h * 0.5f) * 300.0f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = class_score; - - objects.push_back(obj); - } - } - - ptr_score += cls_num + 1; - ptr_boxes += 4; - } - } - } - ptr_anchor_info += anchor_size[head] * 2; - } - } - - static void generate_proposals_yolox(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - - auto feat_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - float box_objectness = feat_ptr[4]; - if (box_objectness < prob_threshold) - { - feat_ptr += cls_num + 5; - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = box_objectness * class_score; - - if (box_prob > prob_threshold) - { - float x_center = (feat_ptr[0] + w) * stride; - float y_center = (feat_ptr[1] + h) * stride; - float w = exp(feat_ptr[2]) * stride; - float h = exp(feat_ptr[3]) * stride; - float x0 = x_center - w * 0.5f; - float y0 = y_center - h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = w; - obj.rect.height = h; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += cls_num + 5; - } - } - } - - static void generate_proposals_yolov7(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - - auto feat_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a_index = 0; a_index < 3; ++a_index) - { - float box_objectness = feat_ptr[4]; - if (box_objectness < prob_threshold) - { - feat_ptr += cls_num + 5; - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = box_objectness * class_score; - - if (box_prob > prob_threshold) - { - float x_center = (feat_ptr[0] * 2 - 0.5f + (float)w) * (float)stride; - float y_center = (feat_ptr[1] * 2 - 0.5f + (float)h) * (float)stride; - float box_w = (feat_ptr[2] * 2) * (feat_ptr[2] * 2) * anchors[a_index * 2]; - float box_h = (feat_ptr[3] * 2) * (feat_ptr[3] * 2) * anchors[a_index * 2 + 1]; - float x0 = x_center - box_w * 0.5f; - float y0 = y_center - box_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = box_w; - obj.rect.height = box_h; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += cls_num + 5; - } - } - } - } - static void generate_proposals_yolov5_face(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 10); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 10]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 5; - for (int l = 0; l < 5; l++) - { - float lx = landmark_ptr[l * 2 + 0]; - float ly = landmark_ptr[l * 2 + 1]; - lx = lx * anchor_w + w * stride; - ly = ly * anchor_h + h * stride; - obj.landmark[l] = cv::Point2f(lx, ly); - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 10); - } - } - } - } - - static void generate_proposals_yolov5_license_plate(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 8); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 8]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 5; - for (int l = 0; l < 4; l++) - { - float lx = landmark_ptr[l * 2 + 0]; - float ly = landmark_ptr[l * 2 + 1]; - lx = lx * anchor_w + w * stride; - ly = ly * anchor_h + h * stride; - obj.landmark[l] = cv::Point2f(lx, ly); - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 8); - } - } - } - } - - static void generate_proposals_yolov5(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 80) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5); - } - } - } - } - - static void generate_proposals_yolov5_seg(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 80, int mask_proto_dim = 32) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + mask_proto_dim); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - obj.mask_feat.resize(mask_proto_dim); - for (int k = 0; k < mask_proto_dim; k++) - { - obj.mask_feat[k] = feature_ptr[cls_num + 5 + k]; - } - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + mask_proto_dim); - } - } - } - } - - static void generate_proposals_yolov5_visdrone(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid, int cls_num = 10) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5); - } - } - } - } - - static void generate_proposals_yolov6(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - - auto feat_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat_ptr[s + 4]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = class_score; - - if (box_prob > prob_threshold) - { - float x0 = (w + 0.5f - feat_ptr[0]) * stride; - float y0 = (h + 0.5f - feat_ptr[1]) * stride; - float x1 = (w + 0.5f + feat_ptr[2]) * stride; - float y1 = (h + 0.5f + feat_ptr[3]) * stride; - - float w = x1 - x0; - float h = y1 - y0; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = w; - obj.rect.height = h; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += 84; - } - } - } - - static void generate_proposals_yolov7_face(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 15); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 15]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 6; - for (int l = 0; l < 5; l++) - { - float lx = (landmark_ptr[3 * l] * 2.0f - 0.5f + w) * stride; - float ly = (landmark_ptr[3 * l + 1] * 2.0f - 0.5f + h) * stride; - //float score = sigmoid(landmark_ptr[3 * l + 2]); - obj.landmark[l] = cv::Point2f(lx, ly); - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 15); - } - } - } - } - - static void generate_proposals_yolov7_palm(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, float prob_threshold_unsigmoid) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int cls_num = 1; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - const int landmark_sort[7] = {0, 3, 4, 5, 6, 1, 2}; - auto feature_ptr = feat; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - if (feature_ptr[4] < prob_threshold_unsigmoid) - { - feature_ptr += (cls_num + 5 + 21); - continue; - } - - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feature_ptr[s + 5 + 21]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feature_ptr[4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - float dx = sigmoid(feature_ptr[0]); - float dy = sigmoid(feature_ptr[1]); - float dw = sigmoid(feature_ptr[2]); - float dh = sigmoid(feature_ptr[3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - PalmObject obj; - obj.rect.x = x0 / (float)letterbox_cols; - obj.rect.y = y0 / (float)letterbox_rows; - obj.rect.width = (x1 - x0) / (float)letterbox_cols; - obj.rect.height = (y1 - y0) / (float)letterbox_rows; - obj.prob = final_score; - - const float* landmark_ptr = feature_ptr + 6; - std::vector tmp(7); - float min_x = FLT_MAX, min_y = FLT_MAX, max_x = 0, max_y = 0; - for (int l = 0; l < 7; l++) - { - float lx = (landmark_ptr[3 * l] * 2.0f - 0.5f + w) * stride; - float ly = (landmark_ptr[3 * l + 1] * 2.0f - 0.5f + h) * stride; - lx /= (float)letterbox_cols; - ly /= (float)letterbox_rows; - - tmp[l] = cv::Point2f(lx, ly); - min_x = lx < min_x ? lx : min_x; - min_y = ly < min_y ? ly : min_y; - max_x = lx > max_x ? lx : max_x; - max_y = ly > max_y ? ly : max_y; - } - float w = max_x - min_x; - float h = max_y - min_y; - float long_side = h > w ? h : w; - long_side *= 1.1f; - obj.rect.x = min_x + w * 0.5f - long_side * 0.5f; - obj.rect.y = min_y + h * 0.5f - long_side * 0.5f; - obj.rect.width = long_side; - obj.rect.height = long_side; - for (int l = 0; l < 7; l++) - { - obj.landmarks[l] = tmp[landmark_sort[l]]; - } - - objects.push_back(obj); - } - - feature_ptr += (cls_num + 5 + 21); - } - } - } - } - - static void generate_proposals_yolov8(int stride, const float* dfl_feat, const float* cls_feat, const float* cls_idx, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto dfl_ptr = dfl_feat; - auto cls_ptr = cls_feat; - auto cls_idx_ptr = cls_idx; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - int class_index = static_cast(cls_idx_ptr[h * feat_w + w]); - float class_score = cls_ptr[h * feat_w * cls_num + w * cls_num + class_index]; - - float box_prob = sigmoid(class_score); - - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(dfl_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - dfl_ptr += (4 * reg_max); - } - } - } - - static void generate_proposals_yolov8_seg(int stride, const float* dfl_feat, const float* cls_feat, const float* cls_idx, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80, int mask_proto_dim = 32) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto dfl_ptr = dfl_feat; - auto cls_ptr = cls_feat; - auto cls_idx_ptr = cls_idx; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - int class_index = static_cast(cls_idx_ptr[h * feat_w + w]); - float class_score = cls_ptr[h * feat_w * cls_num + w * cls_num + class_index]; - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(dfl_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - obj.mask_feat.resize(mask_proto_dim); - for (int k = 0; k < mask_proto_dim; k++) - { - obj.mask_feat[k] = dfl_ptr[4 * reg_max + k]; - } - objects.push_back(obj); - } - - dfl_ptr += (4 * reg_max + mask_proto_dim); - } - } - } - - static void generate_proposals_yolov8_pose(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const int num_point = 17) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - //process cls score - auto scores = feat; - auto bboxes = feat + 1; - auto kps = feat + 1 + 4 * reg_max; - - float box_prob = sigmoid(*scores); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(bboxes + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = 0; - obj.prob = box_prob; - obj.kps_feat.clear(); - for (int k = 0; k < num_point; k++) - { - float kps_x = (kps[k * 3] * 2.f + w) * stride; - float kps_y = (kps[k * 3 + 1] * 2.f + h) * stride; - float kps_s = sigmoid(kps[k * 3 + 2]); - obj.kps_feat.push_back(kps_x); - obj.kps_feat.push_back(kps_y); - obj.kps_feat.push_back(kps_s); - } - objects.push_back(obj); - } - feat += (1 + 4 * reg_max + 3 * num_point); - } - } - } - - static void generate_proposals_yolov9(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += (cls_num + 4 * reg_max); - } - } - } - - static void generate_proposals_yolov8_native(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_ptr += (cls_num + 4 * reg_max); - } - } - } - - static void generate_proposals_yolov8_seg_native(int stride, const float* feat, const float* feat_seg, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80, int mask_proto_dim = 32) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - auto feat_seg_ptr = feat_seg; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - obj.mask_feat.resize(mask_proto_dim); - memcpy(obj.mask_feat.data(), feat_seg_ptr, sizeof(float) * mask_proto_dim); - // for (int k = 0; k < mask_proto_dim; k++) - // { - // obj.mask_feat[k] = feat_seg_ptr[k]; - // } - objects.push_back(obj); - } - - feat_ptr += cls_num + 4 * reg_max; - feat_seg_ptr += mask_proto_dim; - } - } - } - - static void generate_proposals_yolov8_pose_native(int stride, const float* feat, const float* feat_kps, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const int num_point = 17, int cls_num = 1) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_ptr = feat; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - obj.kps_feat.clear(); - for (int k = 0; k < num_point; k++) - { - float kps_x = (feat_kps[k * 3] * 2.f + w) * stride; - float kps_y = (feat_kps[k * 3 + 1] * 2.f + h) * stride; - float kps_s = sigmoid(feat_kps[k * 3 + 2]); - obj.kps_feat.push_back(kps_x); - obj.kps_feat.push_back(kps_y); - obj.kps_feat.push_back(kps_s); - } - objects.push_back(obj); - } - feat_ptr += (cls_num + 4 * reg_max); - feat_kps += 3 * num_point; - } - } - } - - static void generate_proposals_yolo_world(int stride, const float* feat_cls, const float* feat_reg, float exp, float bias, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int reg_max = 16; - - auto feat_cls_ptr = feat_cls; - auto feat_reg_ptr = feat_reg; - - std::vector dis_after_sm(reg_max, 0.f); - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_cls_ptr[s]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - class_score = class_score * exp + bias; - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_reg_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * stride; - } - - float pb_cx = (w + 0.5f) * stride; - float pb_cy = (h + 0.5f) * stride; - - float x0 = pb_cx - pred_ltrb[0]; - float y0 = pb_cy - pred_ltrb[1]; - float x1 = pb_cx + pred_ltrb[2]; - float y1 = pb_cy + pred_ltrb[3]; - - x0 = std::max(std::min(x0, (float)(letterbox_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(letterbox_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(letterbox_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(letterbox_rows - 1)), 0.f); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = box_prob; - - objects.push_back(obj); - } - - feat_cls_ptr += cls_num; - feat_reg_ptr += 4 * reg_max; - } - } - } - - static void generate_proposals(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, const float* anchors, int cls_num = 80) - { - int anchor_num = 3; - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - int anchor_group; - if (stride == 8) - anchor_group = 1; - if (stride == 16) - anchor_group = 2; - if (stride == 32) - anchor_group = 3; - - int w_stride = (cls_num + 5); - int h_stride = feat_w * w_stride; - int a_stride = feat_h * h_stride; - - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - for (int a = 0; a <= anchor_num - 1; a++) - { - //process cls score - int class_index = 0; - float class_score = -FLT_MAX; - int offset = a * a_stride + h * h_stride + w * w_stride; - for (int s = 0; s <= cls_num - 1; s++) - { - float score = feat[offset + s + 5]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - //process box score - float box_score = feat[offset + 4]; - float final_score = sigmoid(box_score) * sigmoid(class_score); - - if (final_score >= prob_threshold) - { - int loc_idx = offset; - float dx = sigmoid(feat[loc_idx + 0]); - float dy = sigmoid(feat[loc_idx + 1]); - float dw = sigmoid(feat[loc_idx + 2]); - float dh = sigmoid(feat[loc_idx + 3]); - float pred_cx = (dx * 2.0f - 0.5f + w) * stride; - float pred_cy = (dy * 2.0f - 0.5f + h) * stride; - float anchor_w = anchors[(anchor_group - 1) * 6 + a * 2 + 0]; - float anchor_h = anchors[(anchor_group - 1) * 6 + a * 2 + 1]; - float pred_w = dw * dw * 4.0f * anchor_w; - float pred_h = dh * dh * 4.0f * anchor_h; - float x0 = pred_cx - pred_w * 0.5f; - float y0 = pred_cy - pred_h * 0.5f; - float x1 = pred_cx + pred_w * 0.5f; - float y1 = pred_cy + pred_h * 0.5f; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = class_index; - obj.prob = final_score; - objects.push_back(obj); - } - } - } - } - } - - static void generate_proposals_palm(std::vector& region_list, float score_thresh, int input_img_w, int input_img_h, float* scores_ptr, float* bboxes_ptr, int head_count, const int* strides, const int* anchor_size, const float* anchor_offset, const int* feature_map_size, float prob_threshold_unsigmoid) - { - int idx = 0; - for (int i = 0; i < head_count; i++) - { - for (int y = 0; y < feature_map_size[i]; y++) - { - for (int x = 0; x < feature_map_size[i]; x++) - { - for (int k = 0; k < anchor_size[i]; k++) - { - if (scores_ptr[idx] < prob_threshold_unsigmoid) - { - idx++; - continue; - } - - const float x_center = (x + anchor_offset[i]) * 1.0f / feature_map_size[i]; - const float y_center = (y + anchor_offset[i]) * 1.0f / feature_map_size[i]; - float score = sigmoid(scores_ptr[idx]); - - if (score > score_thresh) - { - float* p = bboxes_ptr + (idx * 18); - - float cx = p[0] / input_img_w + x_center; - float cy = p[1] / input_img_h + y_center; - float w = p[2] / input_img_w; - float h = p[3] / input_img_h; - - float x0 = cx - w * 0.5f; - float y0 = cy - h * 0.5f; - float x1 = cx + w * 0.5f; - float y1 = cy + h * 0.5f; - - PalmObject region; - region.prob = score; - region.rect.x = x0; - region.rect.y = y0; - region.rect.width = x1 - x0; - region.rect.height = y1 - y0; - - for (int j = 0; j < 7; j++) - { - float lx = p[4 + (2 * j) + 0]; - float ly = p[4 + (2 * j) + 1]; - lx += x_center * input_img_w; - ly += y_center * input_img_h; - lx /= (float)input_img_w; - ly /= (float)input_img_h; - - region.landmarks[j].x = lx; - region.landmarks[j].y = ly; - } - region_list.push_back(region); - } - idx++; - } - } - } - } - } - - static void draw_objects(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const char* output_name,double fontScale = 0.5, int thickness = 1) - { - static const std::vector COCO_COLORS = { - {128, 56, 0, 255}, {128, 226, 255, 0}, {128, 0, 94, 255}, {128, 0, 37, 255}, {128, 0, 255, 94}, {128, 255, 226, 0}, {128, 0, 18, 255}, {128, 255, 151, 0}, {128, 170, 0, 255}, {128, 0, 255, 56}, {128, 255, 0, 75}, {128, 0, 75, 255}, {128, 0, 255, 169}, {128, 255, 0, 207}, {128, 75, 255, 0}, {128, 207, 0, 255}, {128, 37, 0, 255}, {128, 0, 207, 255}, {128, 94, 0, 255}, {128, 0, 255, 113}, {128, 255, 18, 0}, {128, 255, 0, 56}, {128, 18, 0, 255}, {128, 0, 255, 226}, {128, 170, 255, 0}, {128, 255, 0, 245}, {128, 151, 255, 0}, {128, 132, 255, 0}, {128, 75, 0, 255}, {128, 151, 0, 255}, {128, 0, 151, 255}, {128, 132, 0, 255}, {128, 0, 255, 245}, {128, 255, 132, 0}, {128, 226, 0, 255}, {128, 255, 37, 0}, {128, 207, 255, 0}, {128, 0, 255, 207}, {128, 94, 255, 0}, {128, 0, 226, 255}, {128, 56, 255, 0}, {128, 255, 94, 0}, {128, 255, 113, 0}, {128, 0, 132, 255}, {128, 255, 0, 132}, {128, 255, 170, 0}, {128, 255, 0, 188}, {128, 113, 255, 0}, {128, 245, 0, 255}, {128, 113, 0, 255}, {128, 255, 188, 0}, {128, 0, 113, 255}, {128, 255, 0, 0}, {128, 0, 56, 255}, {128, 255, 0, 113}, {128, 0, 255, 188}, {128, 255, 0, 94}, {128, 255, 0, 18}, {128, 18, 255, 0}, {128, 0, 255, 132}, {128, 0, 188, 255}, {128, 0, 245, 255}, {128, 0, 169, 255}, {128, 37, 255, 0}, {128, 255, 0, 151}, {128, 188, 0, 255}, {128, 0, 255, 37}, {128, 0, 255, 0}, {128, 255, 0, 170}, {128, 255, 0, 37}, {128, 255, 75, 0}, {128, 0, 0, 255}, {128, 255, 207, 0}, {128, 255, 0, 226}, {128, 255, 245, 0}, {128, 188, 255, 0}, {128, 0, 255, 18}, {128, 0, 255, 75}, {128, 0, 255, 151}, {128, 255, 56, 0}, {128, 245, 255, 0}}; - cv::Mat image = bgr.clone(); - - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); - - cv::rectangle(image, obj.rect, COCO_COLORS[obj.label], thickness); - - char text[256]; - sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, fontScale, thickness, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > image.cols) - x = image.cols - label_size.width; - - cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(0, 0, 0), -1); - - cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, fontScale, - cv::Scalar(255, 255, 255), thickness); - } - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_keypoints(const cv::Mat& bgr, const std::vector& objects, - const std::vector >& kps_colors, - const std::vector >& limb_colors, - const std::vector >& skeleton, - const char* output_name) - { - cv::Mat image = bgr.clone(); - - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], person\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height); - - cv::rectangle(image, obj.rect, cv::Scalar(255, 0, 0)); - - char text[256]; - sprintf(text, "person %.1f%%", obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > image.cols) - x = image.cols - label_size.width; - - cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(255, 255, 255), -1); - - cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, - cv::Scalar(0, 0, 0)); - - const int num_point = obj.kps_feat.size() / 3; - for (int j = 0; j < num_point + 2; j++) - { - // draw circle - if (j < num_point) - { - int kps_x = std::round(obj.kps_feat[j * 3]); - int kps_y = std::round(obj.kps_feat[j * 3 + 1]); - float kps_s = obj.kps_feat[j * 3 + 2]; - if (kps_s > 0.5f) - { - auto kps_color = cv::Scalar(kps_colors[j][0], kps_colors[j][1], kps_colors[j][2]); - cv::circle(image, {kps_x, kps_y}, 5, kps_color, -1); - } - } - // draw line - auto& ske = skeleton[j]; - int pos1_x = obj.kps_feat[(ske[0] - 1) * 3]; - int pos1_y = obj.kps_feat[(ske[0] - 1) * 3 + 1]; - - int pos2_x = obj.kps_feat[(ske[1] - 1) * 3]; - int pos2_y = obj.kps_feat[(ske[1] - 1) * 3 + 1]; - - float pos1_s = obj.kps_feat[(ske[0] - 1) * 3 + 2]; - float pos2_s = obj.kps_feat[(ske[1] - 1) * 3 + 2]; - - if (pos1_s > 0.5f && pos2_s > 0.5f) - { - auto limb_color = cv::Scalar(limb_colors[j][0], limb_colors[j][1], limb_colors[j][2]); - cv::line(image, {pos1_x, pos1_y}, {pos2_x, pos2_y}, limb_color, 2); - } - } - } - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_objects_mask(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const std::vector >& colors, const char* output_name) - { - cv::Mat image = bgr.clone(); - cv::Mat mask = bgr.clone(); - int color_index = 0; - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - - const auto& color = colors[color_index % 80]; - color_index++; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); - - mask(cv::Rect((int)obj.rect.x, (int)obj.rect.y, (int)objects[i].rect.width, (int)objects[i].rect.height)).setTo(color, objects[i].mask); - - cv::rectangle(image, obj.rect, cv::Scalar(255, 0, 0)); - - char text[256]; - sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > image.cols) - x = image.cols - label_size.width; - - cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(255, 255, 255), -1); - - cv::putText(image, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, - cv::Scalar(0, 0, 0)); - } - float blended_alpha = 0.5; - image = (1 - blended_alpha) * mask + blended_alpha * image; - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_objects_palm(const cv::Mat& bgr, const std::vector& objects, const char* output_name) - { - cv::Mat image = bgr.clone(); - for (size_t i = 0; i < objects.size(); i++) - { - const PalmObject& obj = objects[i]; - //fprintf(stdout, "prob:%.2f, x0:%.2f, y0:%.2f, x1:%.2f, y1:%.2f, x2:%.2f, y2:%.2f, x3:%.2f, y3:%.2f\n", obj.prob, - // obj.vertices[0].x, obj.vertices[0].y, obj.vertices[1].x, obj.vertices[1].y, obj.vertices[2].x, - // obj.vertices[2].y, obj.vertices[3].x, obj.vertices[3].y); - cv::line(image, obj.vertices[0], obj.vertices[1], cv::Scalar(0, 0, 255), 2, 8, 0); - cv::line(image, obj.vertices[1], obj.vertices[2], cv::Scalar(0, 0, 255), 2, 8, 0); - cv::line(image, obj.vertices[2], obj.vertices[3], cv::Scalar(0, 0, 255), 2, 8, 0); - cv::line(image, obj.vertices[3], obj.vertices[0], cv::Scalar(0, 0, 255), 2, 8, 0); - for (auto ld : obj.landmarks) - { - cv::circle(image, ld, 2, cv::Scalar(0, 255, 0), -1, 8); - } - } - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - static void draw_objects_yolopv2(const cv::Mat& bgr, const std::vector& objects, const cv::Mat& da_seg_mask, const cv::Mat& ll_seg_mask, const char* output_name) - { - cv::Mat image = bgr.clone(); - cv::Mat mask = bgr.clone(); - for (size_t i = 0; i < objects.size(); i++) - { - const Object& obj = objects[i]; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f]\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height); - - cv::rectangle(image, obj.rect, cv::Scalar(0, 255, 255), 2, 8, 0); - } - - mask.setTo(cv::Scalar(0, 255, 0), da_seg_mask); - mask.setTo(cv::Scalar(0, 0, 255), ll_seg_mask); - float blended_alpha = 0.5; - image = (1 - blended_alpha) * mask + blended_alpha * image; - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - - void reverse_letterbox(std::vector& proposal, std::vector& objects, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = proposal.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposal[i]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox_no_letterbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int model_h, int model_w, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float ratio_x = (float)src_cols / (float)model_w; - float ratio_y = (float)src_rows / (float)model_h; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0)*ratio_x; - y0 = (y0)*ratio_y; - x1 = (x1)*ratio_x; - y1 = (y1)*ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox(std::vector& objects, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = objects.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - for (int l = 0; l < 5; l++) - { - auto lx = objects[i].landmark[l].x; - auto ly = objects[i].landmark[l].y; - objects[i].landmark[l] = cv::Point2f((lx - tmp_w) * ratio_x, (ly - tmp_h) * ratio_y); - } - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - for (int l = 0; l < 5; l++) - { - auto lx = objects[i].landmark[l].x; - auto ly = objects[i].landmark[l].y; - objects[i].landmark[l] = cv::Point2f((lx - tmp_w) * ratio_x, (ly - tmp_h) * ratio_y); - } - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - } - - void get_out_bbox_mask(std::vector& proposals, std::vector& objects, const float* mask_proto, int mask_proto_dim, int mask_stride, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int mask_proto_h = int(letterbox_rows / mask_stride); - int mask_proto_w = int(letterbox_cols / mask_stride); - - int count = picked.size(); - objects.resize(count); - - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - /* naive RoiAlign by opencv */ - int hstart = std::floor(objects[i].rect.y / mask_stride); - int hend = std::ceil(objects[i].rect.y / mask_stride + objects[i].rect.height / mask_stride); - int wstart = std::floor(objects[i].rect.x / mask_stride); - int wend = std::ceil(objects[i].rect.x / mask_stride + objects[i].rect.width / mask_stride); - - hstart = std::min(std::max(hstart, 0), mask_proto_h); - wstart = std::min(std::max(wstart, 0), mask_proto_w); - hend = std::min(std::max(hend, 0), mask_proto_h); - wend = std::min(std::max(wend, 0), mask_proto_w); - - int mask_w = wend - wstart; - int mask_h = hend - hstart; - - cv::Mat mask = cv::Mat(mask_h, mask_w, CV_32FC1); - if (mask_w > 0 && mask_h > 0) - { - std::vector roi_ranges; - roi_ranges.push_back(cv::Range(0, 1)); - roi_ranges.push_back(cv::Range::all()); - roi_ranges.push_back(cv::Range(hstart, hend)); - roi_ranges.push_back(cv::Range(wstart, wend)); - - cv::Mat mask_protos = cv::Mat(mask_proto_dim, mask_proto_h * mask_proto_w, CV_32FC1, (float*)mask_proto); - int sz[] = {1, mask_proto_dim, mask_proto_h, mask_proto_w}; - cv::Mat mask_protos_reshape = mask_protos.reshape(1, 4, sz); - cv::Mat protos = mask_protos_reshape(roi_ranges).clone().reshape(0, {mask_proto_dim, mask_w * mask_h}); - cv::Mat mask_proposals = cv::Mat(1, mask_proto_dim, CV_32FC1, (float*)objects[i].mask_feat.data()); - cv::Mat masks_feature = (mask_proposals * protos); - /* sigmoid */ - cv::exp(-masks_feature.reshape(1, {mask_h, mask_w}), mask); - mask = 1.0 / (1.0 + mask); - } - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - cv::resize(mask, mask, cv::Size((int)objects[i].rect.width, (int)objects[i].rect.height)); - objects[i].mask = mask > 0.5; - } - } - - void get_out_bbox_kps(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov8 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - - for (int j = 0; j < objects[i].kps_feat.size() / 3; j++) - { - objects[i].kps_feat[j * 3] = std::max( - std::min((objects[i].kps_feat[j * 3] - tmp_w) * ratio_x, (float)(src_cols - 1)), 0.f); - objects[i].kps_feat[j * 3 + 1] = std::max( - std::min((objects[i].kps_feat[j * 3 + 1] - tmp_h) * ratio_y, (float)(src_rows - 1)), 0.f); - } - } - } - - static void transform_rects_palm(PalmObject& object) - { - float x0 = object.landmarks[0].x; - float y0 = object.landmarks[0].y; - float x1 = object.landmarks[2].x; - float y1 = object.landmarks[2].y; - float rotation = M_PI * 0.5f - std::atan2(-(y1 - y0), x1 - x0); - - float hand_cx; - float hand_cy; - float shift_x = 0.0f; - float shift_y = -0.5f; - if (rotation == 0) - { - hand_cx = object.rect.x + object.rect.width * 0.5f + (object.rect.width * shift_x); - hand_cy = object.rect.y + object.rect.height * 0.5f + (object.rect.height * shift_y); - } - else - { - float dx = (object.rect.width * shift_x) * std::cos(rotation) - (object.rect.height * shift_y) * std::sin(rotation); - float dy = (object.rect.width * shift_x) * std::sin(rotation) + (object.rect.height * shift_y) * std::cos(rotation); - hand_cx = object.rect.x + object.rect.width * 0.5f + dx; - hand_cy = object.rect.y + object.rect.height * 0.5f + dy; - } - - float long_side = (std::max)(object.rect.width, object.rect.height); - float dx = long_side * 1.3f; - float dy = long_side * 1.3f; - - object.vertices[0].x = -dx; - object.vertices[0].y = -dy; - object.vertices[1].x = +dx; - object.vertices[1].y = -dy; - object.vertices[2].x = +dx; - object.vertices[2].y = +dy; - object.vertices[3].x = -dx; - object.vertices[3].y = +dy; - - for (int i = 0; i < 4; i++) - { - float sx = object.vertices[i].x; - float sy = object.vertices[i].y; - object.vertices[i].x = sx * std::cos(rotation) - sy * std::sin(rotation); - object.vertices[i].y = sx * std::sin(rotation) + sy * std::cos(rotation); - object.vertices[i].x += hand_cx; - object.vertices[i].y += hand_cy; - } - } - - static void get_out_bbox_palm(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - int count = picked.size(); - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - transform_rects_palm(objects[i]); - } - - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_cols / resize_cols; - float ratio_y = (float)src_rows / resize_rows; - - for (auto& object : objects) - { - for (auto& vertice : object.vertices) - { - vertice.x = (vertice.x * letterbox_cols - tmp_w) * ratio_x; - vertice.y = (vertice.y * letterbox_rows - tmp_h) * ratio_y; - } - - for (auto& ld : object.landmarks) - { - ld.x = (ld.x * letterbox_cols - tmp_w) * ratio_x; - ld.y = (ld.y * letterbox_rows - tmp_h) * ratio_y; - } - // get warpaffine transform mat to landmark detect - cv::Point2f src_pts[4]; - src_pts[0] = object.vertices[0]; - src_pts[1] = object.vertices[1]; - src_pts[2] = object.vertices[2]; - src_pts[3] = object.vertices[3]; - - cv::Point2f dst_pts[4]; - dst_pts[0] = cv::Point2f(0, 0); - dst_pts[1] = cv::Point2f(224, 0); - dst_pts[2] = cv::Point2f(224, 224); - dst_pts[3] = cv::Point2f(0, 224); - - object.affine_trans_mat = cv::getAffineTransform(src_pts, dst_pts); - cv::invertAffineTransform(object.affine_trans_mat, object.affine_trans_mat_inv); - } - } - - void get_out_bbox_yolopv2(std::vector& proposals, std::vector& objects, const float* da_ptr, const float* ll_ptr, cv::Mat& ll_seg_mask, cv::Mat& da_seg_mask, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - nms_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - - objects.resize(count); - for (int i = 0; i < count; i++) - { - objects[i] = proposals[picked[i]]; - float x0 = (objects[i].rect.x); - float y0 = (objects[i].rect.y); - float x1 = (objects[i].rect.x + objects[i].rect.width); - float y1 = (objects[i].rect.y + objects[i].rect.height); - - x0 = (x0 - tmp_w) * ratio_x; - y0 = (y0 - tmp_h) * ratio_y; - x1 = (x1 - tmp_w) * ratio_x; - y1 = (y1 - tmp_h) * ratio_y; - - x0 = std::max(std::min(x0, (float)(src_cols - 1)), 0.f); - y0 = std::max(std::min(y0, (float)(src_rows - 1)), 0.f); - x1 = std::max(std::min(x1, (float)(src_cols - 1)), 0.f); - y1 = std::max(std::min(y1, (float)(src_rows - 1)), 0.f); - - objects[i].rect.x = x0; - objects[i].rect.y = y0; - objects[i].rect.width = x1 - x0; - objects[i].rect.height = y1 - y0; - } - - cv::Mat ll = cv::Mat(cv::Size(letterbox_cols, letterbox_rows), CV_32FC1, (float*)ll_ptr); - ll = ll > 0.5; - cv::resize(ll(cv::Rect(tmp_w, tmp_h, resize_cols, resize_rows)), ll_seg_mask, cv::Size(src_cols, src_rows), 0, 0, cv::INTER_LINEAR); - - cv::Mat da = cv::Mat(cv::Size(letterbox_cols, letterbox_rows), CV_32FC1, (float*)da_ptr); - da = da > 0; - cv::resize(da(cv::Rect(tmp_w, tmp_h, resize_cols, resize_rows)), da_seg_mask, cv::Size(src_cols, src_rows), 0, 0, cv::INTER_NEAREST); - } - - namespace mmyolo - { - inline static float clamp( - float val, - float min = 0.f, - float max = 1536.f) - { - return val > min ? (val < max ? val : max) : min; - } - - inline float fast_exp(const float& x) - { - union - { - uint32_t i; - float f; - } v{}; - v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); - return v.f; - } - - inline float fast_sigmoid(const float& x) - { - return 1.0f / (1.0f + fast_exp(-x)); - } - - inline static float fast_softmax( - const float* src, - float* dst, - int length) - { - const float alpha = *std::max_element(src, src + length); - float denominator = 0; - float dis_sum = 0; - for (int i = 0; i < length; ++i) - { - dst[i] = fast_exp(src[i] - alpha); - denominator += dst[i]; - } - for (int i = 0; i < length; ++i) - { - dst[i] /= denominator; - dis_sum += i * dst[i]; - } - return dis_sum; - } - - static void generate_proposals_ppyoloeplus( - int stride, - const float* cls_feat, - const float* box_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - int reg_max = 17; - float dis_after_sm[reg_max]; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max); - - if (box_prob > prob_threshold) - { - float x0 = w + 0.5f - fast_softmax(boxes_ptr, dis_after_sm, reg_max); - float y0 = h + 0.5f - fast_softmax(boxes_ptr + reg_max, dis_after_sm, reg_max); - float x1 = w + 0.5f + fast_softmax(boxes_ptr + 2 * reg_max, dis_after_sm, reg_max); - float y1 = h + 0.5f + fast_softmax(boxes_ptr + 3 * reg_max, dis_after_sm, reg_max); - - x0 *= stride; - y0 *= stride; - x1 *= stride; - y1 *= stride; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - x1 = clamp(x1, 0.f, letterbox_cols - 1); - y1 = clamp(y1, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4 * reg_max; - } - } - } - - static void generate_proposals_yolox( - int stride, - const float* cls_feat, - const float* box_feat, - const float* conf_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - auto conf_ptr = conf_feat; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - //process cls score - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max) * fast_sigmoid(*conf_ptr); - - if (box_prob > prob_threshold) - { - float x = (w + boxes_ptr[0]) * stride; - float y = (h + boxes_ptr[1]) * stride; - - float width = fast_exp(boxes_ptr[2]) * stride; - float height = fast_exp(boxes_ptr[3]) * stride; - - float x0 = x - width * 0.5f; - float y0 = y - height * 0.5f; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = width; - obj.rect.height = height; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4; - conf_ptr++; - } - } - } - - static void generate_proposals_yolov6( - int stride, - const float* cls_feat, - const float* box_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - //process cls score - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max); - - if (box_prob > prob_threshold) - { - float x0 = (w + 0.5f - boxes_ptr[0]) * stride; - float y0 = (h + 0.5f - boxes_ptr[1]) * stride; - float x1 = (w + 0.5f + boxes_ptr[2]) * stride; - float y1 = (h + 0.5f + boxes_ptr[3]) * stride; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - x1 = clamp(x1, 0.f, letterbox_cols - 1); - y1 = clamp(y1, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4; - } - } - } - - static void generate_proposals_yolov8( - int stride, - const float* cls_feat, - const float* box_feat, - float prob_threshold, - std::vector& objects, - int letterbox_cols, - int letterbox_rows, - int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto cls_ptr = cls_feat; - auto boxes_ptr = box_feat; - int reg_max = 16; - float dis_after_sm[reg_max]; - - for (int h = 0; h < feat_h; h++) - { - for (int w = 0; w < feat_w; w++) - { - auto max = std::max_element(cls_ptr, cls_ptr + cls_num); - float box_prob = fast_sigmoid(*max); - - if (box_prob > prob_threshold) - { - float x0 = w + 0.5f - fast_softmax(boxes_ptr, dis_after_sm, reg_max); - float y0 = h + 0.5f - fast_softmax(boxes_ptr + reg_max, dis_after_sm, reg_max); - float x1 = w + 0.5f + fast_softmax(boxes_ptr + 2 * reg_max, dis_after_sm, reg_max); - float y1 = h + 0.5f + fast_softmax(boxes_ptr + 3 * reg_max, dis_after_sm, reg_max); - - x0 *= stride; - y0 *= stride; - x1 *= stride; - y1 *= stride; - - x0 = clamp(x0, 0.f, letterbox_cols - 1); - y0 = clamp(y0, 0.f, letterbox_rows - 1); - x1 = clamp(x1, 0.f, letterbox_cols - 1); - y1 = clamp(y1, 0.f, letterbox_rows - 1); - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0; - obj.rect.height = y1 - y0; - obj.label = max - cls_ptr; - obj.prob = box_prob; - - objects.push_back(obj); - } - cls_ptr += cls_num; - boxes_ptr += 4 * reg_max; - } - } - } - } // namespace mmyolo - - namespace obb - { - static inline float fast_exp(const float& x) - { - union - { - uint32_t i; - float f; - } v; - v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); - return v.f; - } - - static inline float clamp_(float v, float min = 0.f, float max = std::numeric_limits::infinity()) - { - return std::max(std::min(v, max), min); - } - static inline void get_covariance_matrix( - const std::vector& objects, - std::vector& covar_maxtirx) - { - int n = objects.size(); - for (size_t i = 0; i < n; ++i) { - - float a = objects[i].rect.width * objects[i].rect.width * 0.0833333333f; - float b = objects[i].rect.height * objects[i].rect.height * 0.0833333333f; - float c = objects[i].angle; - - float c_sin = std::sin(c); - float c_cos = std::cos(c); - - float x = a * c_cos * c_cos + b * c_sin * c_sin; - float y = a * c_sin * c_sin + b * c_cos * c_cos; - float z = a * c_cos * c_sin - b * c_sin * c_cos; - covar_maxtirx.push_back(cv::Point3f(x, y, z)); - } - } - - static inline float fast_log2 (float x) - { - union { - float f; - int i; - } v; - v.f = x; - int* const exp_ptr = &v.i; - int m = *exp_ptr; - const int log_2 = ((m >> 23) & 255) - 128; - m &= ~(255 << 23); - m += 127 << 23; - *exp_ptr = m; - return (((-1.f / 3) * v.f + 2) * v.f - 2.f / 3 + log_2); - } - - static inline float fast_log (const float x) - { - return 0.69314718f * fast_log2 (x); - } - - static inline float probiou( - const Object& obj1, - const Object& obj2, - const cv::Point3f& covar1, - const cv::Point3f& covar2) - { - - float v_x1_x2 = covar1.x + covar2.x; - float v_y1_y2 = covar1.y + covar2.y; - float x1_x2 = obj1.rect.x - obj2.rect.x; - float y1_y2 = obj1.rect.y - obj2.rect.y; - float dem1 = clamp_(covar1.x * covar1.y - covar1.z * covar1.z); - float dem2 = clamp_(covar2.x * covar2.y - covar2.z * covar2.z); - float dem_num = v_x1_x2 * v_y1_y2 - (covar1.z + covar2.z) * (covar1.z + covar2.z); - - float t1 = (v_x1_x2 * y1_y2 * y1_y2 + v_y1_y2 * x1_x2 * x1_x2) / (dem_num + 1e-7) * 0.25f; - float t2 = ((covar1.z + covar2.z) * (-x1_x2) * y1_y2) / (dem_num + 1e-7) * 0.5f; - float t3 = fast_log(0.25f * dem_num / (std::sqrt(dem1 * dem2) + 1e-7) + 1e-7) * 0.5f; - - float bd = t1 + t2 + t3; - float iou = fast_exp(-clamp_(bd, 1e-7, 100.f)); - return iou; - } - - static inline void nms_rotated_sorted_bboxes( - const std::vector& objects, - std::vector& picked, - float nms_threshold) - { - std::vector covar_maxtrix; - get_covariance_matrix(objects, covar_maxtrix); - - float nms_threshold_ = (1.f - nms_threshold) * (1.f - nms_threshold); - int n = objects.size(); - - for (size_t i = 0; i < n; ++i) { - float max_iou = 0.f; - const Object& obj1 = objects[i]; - - for (size_t j = 0; j < i; ++j) { - const Object& obj2 = objects[j]; - - float iou = probiou(obj1, obj2, covar_maxtrix[i], covar_maxtrix[j]); - if (iou > max_iou){ - max_iou = iou; - } - - } - if (max_iou < nms_threshold_) - picked.push_back(i); - } - } - static void get_out_obb_bbox(std::vector& proposals, std::vector& objects, const float nms_threshold, int letterbox_rows, int letterbox_cols, int src_rows, int src_cols) - { - qsort_descent_inplace(proposals); - std::vector picked; - obb::nms_rotated_sorted_bboxes(proposals, picked, nms_threshold); - - /* yolov5 draw the result */ - float scale_letterbox; - int resize_rows; - int resize_cols; - if ((letterbox_rows * 1.0 / src_rows) < (letterbox_cols * 1.0 / src_cols)) - { - scale_letterbox = letterbox_rows * 1.0 / src_rows; - } - else - { - scale_letterbox = letterbox_cols * 1.0 / src_cols; - } - resize_cols = int(scale_letterbox * src_cols); - resize_rows = int(scale_letterbox * src_rows); - - int tmp_h = (letterbox_rows - resize_rows) / 2; - int tmp_w = (letterbox_cols - resize_cols) / 2; - - float ratio_x = (float)src_rows / resize_rows; - float ratio_y = (float)src_cols / resize_cols; - - int count = picked.size(); - double pi = M_PI; - double pi_2 = M_PI_2; - objects.resize(count); - for (int i = 0; i < count; i++) { - objects[i] = proposals[picked[i]]; - - float w_ = objects[i].rect.width > objects[i].rect.height ? objects[i].rect.width : objects[i].rect.height; - float h_ = objects[i].rect.width > objects[i].rect.height ? objects[i].rect.height : objects[i].rect.width; - float a_ = (float)std::fmod((objects[i].rect.width > objects[i].rect.height ? objects[i].angle : objects[i].angle + pi_2), pi); - - float xc = (objects[i].rect.x - tmp_w) * ratio_x; - float yc = (objects[i].rect.y - tmp_h) * ratio_y; - float w = w_ * ratio_x; - float h = h_ * ratio_y; - - // clip - xc = std::max(std::min(xc, (float)(src_cols - 1)), 0.f); - yc = std::max(std::min(yc, (float)(src_rows - 1)), 0.f); - w = std::max(std::min(w, (float)(src_cols - 1)), 0.f); - h = std::max(std::min(h, (float)(src_rows - 1)), 0.f); - - - objects[i].rect.x = xc; - objects[i].rect.y = yc; - objects[i].rect.width = w; - objects[i].rect.height = h; - objects[i].angle = a_; - } - } - static void generate_proposals_yolov8_obb_native(const std::vector& grid_strides, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 15) - { - const int num_points = grid_strides.size(); - int reg_max = 16; - auto feat_ptr = feat; - std::vector dis_after_sm(reg_max, 0.f); - for (int i = 0; i < num_points; i++) - { - // process cls score - int class_index = 0; - float class_score = -FLT_MAX; - for (int s = 0; s < cls_num; s++) - { - float score = feat_ptr[s + 4 * reg_max]; - if (score > class_score) - { - class_index = s; - class_score = score; - } - } - - float box_prob = sigmoid(class_score); - if (box_prob > prob_threshold) - { - float pred_ltrb[4]; - for (int k = 0; k < 4; k++) - { - float dis = softmax(feat_ptr + k * reg_max, dis_after_sm.data(), reg_max); - pred_ltrb[k] = dis * grid_strides[i].stride; - } - - float angle = feat_ptr[4 * reg_max + cls_num]; - - float pb_cx = (grid_strides[i].grid0 + 0.5f) * grid_strides[i].stride; - float pb_cy = (grid_strides[i].grid1 + 0.5f) * grid_strides[i].stride; - - float cos = std::cos(angle); - float sin = std::sin(angle); - - float x = (pred_ltrb[2] - pred_ltrb[0]) * 0.5f; - float y = (pred_ltrb[3] - pred_ltrb[1]) * 0.5f; - float xc = x * cos - y * sin + pb_cx; - float yc = x * sin + y * cos + pb_cy; - float w = pred_ltrb[2] + pred_ltrb[0]; - float h = pred_ltrb[3] + pred_ltrb[1]; - - Object obj; - obj.rect.x = xc; //center x - obj.rect.y = yc; //center y - obj.rect.width = w; - obj.rect.height = h; - obj.label = class_index; - obj.prob = box_prob; - obj.angle = angle; - - objects.push_back(obj); - } - feat_ptr += (cls_num + 4 * reg_max + 1); - } - } - static void draw_objects_obb(const cv::Mat& bgr, const std::vector& objects, const char** class_names, const char* output_name, int thickness = 1) - { - static const std::vector COCO_COLORS = { - {128, 56, 0, 255}, {128, 226, 255, 0}, {128, 0, 94, 255}, {128, 0, 37, 255}, {128, 0, 255, 94}, {128, 255, 226, 0}, {128, 0, 18, 255}, {128, 255, 151, 0}, {128, 170, 0, 255}, {128, 0, 255, 56}, {128, 255, 0, 75}, {128, 0, 75, 255}, {128, 0, 255, 169}, {128, 255, 0, 207}, {128, 75, 255, 0}, {128, 207, 0, 255}, {128, 37, 0, 255}, {128, 0, 207, 255}, {128, 94, 0, 255}, {128, 0, 255, 113}, {128, 255, 18, 0}, {128, 255, 0, 56}, {128, 18, 0, 255}, {128, 0, 255, 226}, {128, 170, 255, 0}, {128, 255, 0, 245}, {128, 151, 255, 0}, {128, 132, 255, 0}, {128, 75, 0, 255}, {128, 151, 0, 255}, {128, 0, 151, 255}, {128, 132, 0, 255}, {128, 0, 255, 245}, {128, 255, 132, 0}, {128, 226, 0, 255}, {128, 255, 37, 0}, {128, 207, 255, 0}, {128, 0, 255, 207}, {128, 94, 255, 0}, {128, 0, 226, 255}, {128, 56, 255, 0}, {128, 255, 94, 0}, {128, 255, 113, 0}, {128, 0, 132, 255}, {128, 255, 0, 132}, {128, 255, 170, 0}, {128, 255, 0, 188}, {128, 113, 255, 0}, {128, 245, 0, 255}, {128, 113, 0, 255}, {128, 255, 188, 0}, {128, 0, 113, 255}, {128, 255, 0, 0}, {128, 0, 56, 255}, {128, 255, 0, 113}, {128, 0, 255, 188}, {128, 255, 0, 94}, {128, 255, 0, 18}, {128, 18, 255, 0}, {128, 0, 255, 132}, {128, 0, 188, 255}, {128, 0, 245, 255}, {128, 0, 169, 255}, {128, 37, 255, 0}, {128, 255, 0, 151}, {128, 188, 0, 255}, {128, 0, 255, 37}, {128, 0, 255, 0}, {128, 255, 0, 170}, {128, 255, 0, 37}, {128, 255, 75, 0}, {128, 0, 0, 255}, {128, 255, 207, 0}, {128, 255, 0, 226}, {128, 255, 245, 0}, {128, 188, 255, 0}, {128, 0, 255, 18}, {128, 0, 255, 75}, {128, 0, 255, 151}, {128, 255, 56, 0}, {128, 245, 255, 0}}; - cv::Mat image = bgr.clone(); - - for (size_t i = 0; i < objects.size(); i++){ - const Object& obj = objects[i]; - - fprintf(stdout, "%2d: %3.0f%%, [%4.0f, %4.0f, %4.0f, %4.0f], %s\n", obj.label, obj.prob * 100, obj.rect.x, - obj.rect.y, obj.rect.x + obj.rect.width, obj.rect.y + obj.rect.height, class_names[obj.label]); - { - float xc = obj.rect.x; - float yc = obj.rect.y; - float w = obj.rect.width; - float h = obj.rect.height; - float ag = obj.angle; - float wx = w / 2 * std::cos(ag); - float wy = w / 2 * std::sin(ag); - float hx = -h / 2 * std::sin(ag); - float hy = h / 2 * std::cos(ag); - cv::Point2f p1{ xc - wx - hx, yc - wy - hy }; - cv::Point2f p2{ xc + wx - hx, yc + wy - hy }; - cv::Point2f p3{ xc + wx + hx, yc + wy + hy }; - cv::Point2f p4{ xc - wx + hx, yc - wy + hy }; - std::vector points = { p1, p2, p3, p4, p1 }; - std::vector> contours= { points }; - cv::polylines(image, contours, true, COCO_COLORS[obj.label], thickness, cv::LINE_AA); - } - } - - cv::imwrite(std::string(output_name) + ".jpg", image); - } - } // namespace obb - - static void generate_proposals_yolov10(int stride, const float* feat, float prob_threshold, std::vector& objects, - int letterbox_cols, int letterbox_rows, int cls_num = 80) - { - int feat_w = letterbox_cols / stride; - int feat_h = letterbox_rows / stride; - auto feat_ptr = feat; - float dis[16]; - for (int h = 0; h <= feat_h - 1; h++) - { - for (int w = 0; w <= feat_w - 1; w++) - { - int c_index = 0; - float c_score = 0; - - for (int c = 0; c < cls_num; c++) - { - float score = feat_ptr[c]; - if (score > c_score) - { - c_index = c; - c_score = score; - } - } - - if(c_score >= prob_threshold) - { - float x0 = w + 0.5f - mmyolo::fast_softmax(feat_ptr + cls_num + 0 * 16,dis, 16); - float y0 = h + 0.5f - mmyolo::fast_softmax(feat_ptr + cls_num + 1 * 16,dis, 16); - float x1 = w + 0.5f + mmyolo::fast_softmax(feat_ptr + cls_num + 2 * 16,dis, 16); - float y1 = h + 0.5f + mmyolo::fast_softmax(feat_ptr + cls_num + 3 * 16,dis, 16); - - x0 *= stride; - y0 *= stride; - x1 *= stride; - y1 *= stride; - - Object obj; - obj.rect.x = x0; - obj.rect.y = y0; - obj.rect.width = x1 - x0;; - obj.rect.height = y1 - y0; - obj.label = c_index; - obj.prob = c_score; - objects.push_back(obj); - } - feat_ptr += (cls_num + 4 * 16); - } - } - } - -} // namespace detection diff --git a/projects/llm_framework/main_yolo/src/runner/base/pose.hpp b/projects/llm_framework/main_yolo/src/runner/base/pose.hpp deleted file mode 100644 index b06badf..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/pose.hpp +++ /dev/null @@ -1,432 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: hebing - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace pose -{ - typedef struct - { - float x; - float y; - float score; - } ai_point_t; - - struct skeleton - { - int connection[2]; - int left_right_neutral; - }; - - std::vector pairs = {{15, 13, 0}, - {13, 11, 0}, - {16, 14, 0}, - {14, 12, 0}, - {11, 12, 0}, - {5, 11, 0}, - {6, 12, 0}, - {5, 6, 0}, - {5, 7, 0}, - {6, 8, 0}, - {7, 9, 0}, - {8, 10, 0}, - {1, 2, 0}, - {0, 1, 0}, - {0, 2, 0}, - {1, 3, 0}, - {2, 4, 0}, - {0, 5, 0}, - {0, 6, 0}}; - std::vector hand_pairs = {{0, 1, 0}, - {1, 2, 0}, - {2, 3, 0}, - {3, 4, 0}, - {0, 5, 1}, - {5, 6, 1}, - {6, 7, 1}, - {7, 8, 1}, - {0, 9, 2}, - {9, 10, 2}, - {10, 11, 2}, - {11, 12, 2}, - {0, 13, 3}, - {13, 14, 3}, - {14, 15, 3}, - {15, 16, 3}, - {0, 17, 4}, - {17, 18, 4}, - {18, 19, 4}, - {19, 20, 4}}; - std::vector animal_pairs = {{19, 15, 0}, - {18, 14, 0}, - {17, 13, 0}, - {16, 12, 0}, - {15, 11, 0}, - {14, 10, 0}, - {13, 9, 0}, - {12, 8, 0}, - {11, 6, 0}, - {10, 6, 0}, - {9, 7, 0}, - {8, 7, 0}, - {6, 7, 0}, - {7, 5, 0}, - {5, 4, 0}, - {0, 2, 0}, - {1, 3, 0}, - {0, 1, 0}, - {0, 4, 0}, - {1, 4, 0}}; - - typedef struct ai_body_parts_s - { - std::vector keypoints; - int32_t img_width = 0; - int32_t img_heigh = 0; - uint64_t timestamp = 0; - } ai_body_parts_s; - - typedef struct ai_hand_parts_s - { - std::vector keypoints; - int32_t hand_side = 0; //0-left hand,1-right hand - int32_t img_width = 0; - int32_t img_heigh = 0; - uint64_t timestamp = 0; - } ai_hand_parts_s; - - typedef struct ai_animal_parts_s - { - std::vector keypoints; - int32_t img_width = 0; - int32_t img_heigh = 0; - uint64_t timestamp = 0; - } ai_animal_parts_s; - - static inline void find_max_2d(float* buf, int width, int height, int* max_idx_width, int* max_idx_height, float* max_value, int c) - { - float* ptr = buf; - *max_value = -10.f; - *max_idx_width = 0; - *max_idx_height = 0; - for (int h = 0; h < height; h++) - { - for (int w = 0; w < width; w++) - { - float score = ptr[c * height * width + h * width + w]; - if (score > *max_value) - { - *max_value = score; - *max_idx_height = h; - *max_idx_width = w; - } - } - } - } - - static inline void draw_result(cv::Mat img, ai_body_parts_s& pose, int joints_num, int model_w, int model_h) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x * img.cols); - int y = (int)(pose.keypoints[i].y * img.rows); - - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(255, 0, 0); - break; - case 1: - color = cv::Scalar(0, 0, 255); - break; - default: - color = cv::Scalar(0, 255, 0); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); - int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); - int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); - int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - } - - static inline void draw_animal_result(cv::Mat img, ai_animal_parts_s& pose, int joints_num, int model_w, int model_h) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x * img.cols); - int y = (int)(pose.keypoints[i].y * img.rows); - - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : animal_pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(255, 0, 0); - break; - case 1: - color = cv::Scalar(0, 0, 255); - break; - default: - color = cv::Scalar(0, 255, 0); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); - int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); - int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); - int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - } - - static inline void draw_result(cv::Mat img, ai_body_parts_s& pose, int joints_num, int model_w, int model_h, const detection::Object& obj) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x); - int y = (int)(pose.keypoints[i].y); - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 255, 0), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(255, 0, 0); - break; - case 1: - color = cv::Scalar(0, 0, 255); - break; - default: - color = cv::Scalar(0, 255, 0); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x); - int y1 = (int)(pose.keypoints[element.connection[0]].y); - int x2 = (int)(pose.keypoints[element.connection[1]].x); - int y2 = (int)(pose.keypoints[element.connection[1]].y); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - // 画框 - cv::rectangle(img, obj.rect, cv::Scalar(255, 0, 0)); - char text[256]; - sprintf(text, "%s %.1f%%", "person", obj.prob * 100); - - int baseLine = 0; - cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - int x = obj.rect.x; - int y = obj.rect.y - label_size.height - baseLine; - if (y < 0) - y = 0; - if (x + label_size.width > img.cols) - x = img.cols - label_size.width; - - cv::rectangle(img, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), - cv::Scalar(255, 255, 255), -1); - - cv::putText(img, text, cv::Point(x, y + label_size.height), cv::FONT_HERSHEY_SIMPLEX, 0.5, - cv::Scalar(0, 0, 0)); - cv::imwrite("./pose_ppl_out.png", img); - } - - static inline void draw_result_hand(cv::Mat img, ai_hand_parts_s& pose, int joints_num) - { - for (int i = 0; i < joints_num; i++) - { - int x = (int)(pose.keypoints[i].x * img.cols); - int y = (int)(pose.keypoints[i].y * img.rows); - - x = std::max(std::min(x, (img.cols - 1)), 0); - y = std::max(std::min(y, (img.rows - 1)), 0); - - cv::circle(img, cv::Point(x, y), 4, cv::Scalar(0, 0, 255), cv::FILLED); - } - - cv::Scalar color; - cv::Point pt1; - cv::Point pt2; - for (auto& element : hand_pairs) - { - switch (element.left_right_neutral) - { - case 0: - color = cv::Scalar(10, 215, 255); - break; - case 1: - color = cv::Scalar(255, 115, 55); - break; - case 2: - color = cv::Scalar(5, 255, 55); - break; - case 3: - color = cv::Scalar(25, 15, 255); - break; - default: - color = cv::Scalar(225, 15, 55); - } - - int x1 = (int)(pose.keypoints[element.connection[0]].x * img.cols); - int y1 = (int)(pose.keypoints[element.connection[0]].y * img.rows); - int x2 = (int)(pose.keypoints[element.connection[1]].x * img.cols); - int y2 = (int)(pose.keypoints[element.connection[1]].y * img.rows); - - x1 = std::max(std::min(x1, (img.cols - 1)), 0); - y1 = std::max(std::min(y1, (img.rows - 1)), 0); - x2 = std::max(std::min(x2, (img.cols - 1)), 0); - y2 = std::max(std::min(y2, (img.rows - 1)), 0); - - pt1 = cv::Point(x1, y1); - pt2 = cv::Point(x2, y2); - cv::line(img, pt1, pt2, color, 2); - } - - cv::imwrite("./hand_pose_out.png", img); - } - - static inline void post_process(float* data, ai_body_parts_s& pose, int joint_num, int img_h, int img_w) - { - int heatmap_width = img_w / 4; - int heatmap_height = img_h / 4; - int max_idx_width, max_idx_height; - float max_score; - - ai_point_t kp; - for (int c = 0; c < joint_num; ++c) - { - find_max_2d(data, heatmap_width, heatmap_height, &max_idx_width, &max_idx_height, &max_score, c); - kp.x = (float)max_idx_width / (float)heatmap_width; - kp.y = (float)max_idx_height / (float)heatmap_height; - kp.score = max_score; - pose.keypoints.push_back(kp); - - // std::cout << "x: " << pose.keypoints[c].x << ", y: " << pose.keypoints[c].y << ", score: " - // << pose.keypoints[c].score << std::endl; - } - } - - static inline void animal_post_process(float* data, ai_animal_parts_s& pose, int joint_num, int img_h, int img_w) - { - int heatmap_width = img_w / 4; - int heatmap_height = img_h / 4; - int max_idx_width, max_idx_height; - float max_score; - - ai_point_t kp; - for (int c = 0; c < joint_num; ++c) - { - find_max_2d(data, heatmap_width, heatmap_height, &max_idx_width, &max_idx_height, &max_score, c); - kp.x = (float)max_idx_width / (float)heatmap_width; - kp.y = (float)max_idx_height / (float)heatmap_height; - kp.score = max_score; - pose.keypoints.push_back(kp); - - // std::cout << "x: " << pose.keypoints[c].x << ", y: " << pose.keypoints[c].y << ", score: " - // << pose.keypoints[c].score << std::endl; - } - } - - static inline void ppl_pose_post_process(float* data1, float* data2, ai_body_parts_s& pose, int joint_num, int img_h, int img_w, int offset_top, int offset_left, int offset_x, int offset_y, float ratio) - { - ai_point_t kp; - - for (int c = 0; c < joint_num; ++c) - { - kp.x = (data1[c] / 2 - offset_left) / ratio + offset_x; - kp.y = (data2[c] / 2 - offset_top) / ratio + offset_y; - std::cout << "x1: " << kp.x << ", y1: " << kp.y << std::endl; - pose.keypoints.push_back(kp); - } - } - - static inline void post_process_hand(float* point_data, float* score_data, ai_hand_parts_s& pose, int joint_num, int img_h, int img_w) - { - ai_point_t kp; - for (int c = 0; c < joint_num; ++c) - { - kp.x = (float)point_data[c * 3] / img_w; - kp.y = (float)point_data[c * 3 + 1] / img_h; - pose.keypoints.push_back(kp); - } - if (score_data[0] > 0.5) - pose.hand_side = 1; - } - -} // namespace pose \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/runner/base/score.hpp b/projects/llm_framework/main_yolo/src/runner/base/score.hpp deleted file mode 100644 index 080815d..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/score.hpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include - - -namespace classification -{ - typedef struct - { - uint32_t id; - float score; - } score; -} diff --git a/projects/llm_framework/main_yolo/src/runner/base/topk.hpp b/projects/llm_framework/main_yolo/src/runner/base/topk.hpp deleted file mode 100644 index 50d4314..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/topk.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include -#include -#include - -#include "base/score.hpp" - - -namespace classification -{ - void sort_score(std::vector& array, bool reverse = false) - { - auto compare_func = [](const score& a, const score& b) -> bool - { - return a.score > b.score; - }; - - std::sort(array.begin(), array.end(), compare_func); - - if (reverse) std::reverse(array.begin(), array.end()); - } - - - void print_score(const std::vector& array, const size_t& n) - { - for (size_t i = 0; i < n; i++) - { - fprintf(stdout, "%.4f, %d\n", array[i].score, array[i].id); - } - } -} diff --git a/projects/llm_framework/main_yolo/src/runner/base/transform.hpp b/projects/llm_framework/main_yolo/src/runner/base/transform.hpp deleted file mode 100644 index edee13a..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/transform.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: hebing - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -namespace transform -{ - static void nhwc2nchw(const float* input, float* output, int h, int w, int c) - { - int output_index = 0; - for (int i = 0; i < c; ++i) - { - for (int j = 0; j < h; ++j) - { - for (int k = 0; k < w; ++k) - { - int input_index = j * w * c + k * c + i; - output[output_index++] = input[input_index]; - } - } - } - } - -} // namespace transform diff --git a/projects/llm_framework/main_yolo/src/runner/base/yolo.hpp b/projects/llm_framework/main_yolo/src/runner/base/yolo.hpp deleted file mode 100644 index 023cd54..0000000 --- a/projects/llm_framework/main_yolo/src/runner/base/yolo.hpp +++ /dev/null @@ -1,578 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: hebing - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace yolo -{ - enum - { - YOLOV3 = 0, - YOLOV3_TINY = 1, - YOLOV4 = 2, - YOLOV4_TINY = 3, - YOLO_FASTEST = 4, - YOLO_FASTEST_XL = 5, - YOLO_FASTEST_BODY = 6, - YOLOV4_TINY_3L = 7 - }; - - struct BBoxRect - { - float score; - float xmin; - float ymin; - float xmax; - float ymax; - float area; - int label; - }; - - static inline float sigmoid(float x) - { - return (float)(1.f / (1.f + std::exp(-x))); - } - static inline float intersection_area(const BBoxRect& a, const BBoxRect& b) - { - if (a.xmin > b.xmax || a.xmax < b.xmin || a.ymin > b.ymax || a.ymax < b.ymin) - { - // no intersection - return 0.f; - } - - float inter_width = std::min(a.xmax, b.xmax) - std::max(a.xmin, b.xmin); - float inter_height = std::min(a.ymax, b.ymax) - std::max(a.ymin, b.ymin); - - return inter_width * inter_height; - } - static void qsort_descent_inplace(std::vector& datas, int left, int right) - { - int i = left; - int j = right; - float p = datas[(left + right) / 2].score; - - while (i <= j) - { - while (datas[i].score > p) - i++; - - while (datas[j].score < p) - j--; - - if (i <= j) - { - // swap - std::swap(datas[i], datas[j]); - - i++; - j--; - } - } - - if (left < j) - qsort_descent_inplace(datas, left, j); - - if (i < right) - qsort_descent_inplace(datas, i, right); - } - - static void qsort_descent_inplace(std::vector& datas) - { - if (datas.empty()) - return; - - qsort_descent_inplace(datas, 0, (int)(datas.size() - 1)); - } - - static void nms_sorted_bboxes(std::vector& bboxes, std::vector& picked, float nms_threshold) - { - picked.clear(); - - const size_t n = bboxes.size(); - - for (size_t i = 0; i < n; i++) - { - const BBoxRect& a = bboxes[i]; - - int keep = 1; - for (unsigned int j : picked) - { - const BBoxRect& b = bboxes[j]; - - // intersection over union - float inter_area = intersection_area(a, b); - float union_area = a.area + b.area - inter_area; - // float IoU = inter_area / union_area - if (inter_area > nms_threshold * union_area) - { - keep = 0; - break; - } - } - - if (keep) - picked.push_back(i); - } - } - - struct TMat - { - operator const float*() const - { - return (const float*)data; - } - - float* row(int row) const - { - return (float*)data + w * row; - } - - TMat channel_range(int start, int chn_num) const - { - TMat mat = {0}; - - mat.batch = 1; - mat.c = chn_num; - mat.h = h; - mat.w = w; - mat.data = (float*)data + start * h * w; - - return mat; - } - - TMat channel(int channel) const - { - return channel_range(channel, 1); - } - - int batch, c, h, w; - void* data; - }; - - class YoloDetectionOutput - { - public: - int init(int version, float nms_threshold = 0.45f, float confidence_threshold = 0.48f, int class_num = 80); - int forward(const std::vector& bottom_blobs, std::vector& top_blobs); - int forward_nhwc(const std::vector& bottom_blobs, std::vector& top_blobs); - - private: - int m_num_box; - int m_num_class; - int m_anchors_scale[32]; - float m_biases[32]; - int m_mask[32]; - float m_confidence_threshold; - float m_confidence_threshold_unsigmoid; - float m_nms_threshold; - }; - - int YoloDetectionOutput::init(int version, float nms_threshold, float confidence_threshold, int class_num) - { - memset(this, 0, sizeof(*this)); - m_num_box = 3; - m_num_class = class_num; - fprintf(stderr, "YoloDetectionOutput init param[%d]\n", version); - - if (version == YOLOV3) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - m_anchors_scale[2] = 8; - - float bias[] = {10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 373, 326}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 6; - m_mask[1] = 7; - m_mask[2] = 8; - - m_mask[3] = 3; - m_mask[4] = 4; - m_mask[5] = 5; - - m_mask[6] = 0; - m_mask[7] = 1; - m_mask[8] = 2; - } - else if (version == YOLOV3_TINY || version == YOLOV4_TINY) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - - float bias[] = {10, 14, 23, 27, 37, 58, 81, 82, 135, 169, 344, 319}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 3; - m_mask[1] = 4; - m_mask[2] = 5; - - m_mask[3] = 0; - m_mask[4] = 1; - m_mask[5] = 2; - } - else if (version == YOLOV4) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - m_anchors_scale[2] = 8; - - float bias[] = {12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 6; - m_mask[1] = 7; - m_mask[2] = 8; - - m_mask[3] = 3; - m_mask[4] = 4; - m_mask[5] = 5; - - m_mask[6] = 0; - m_mask[7] = 1; - m_mask[8] = 2; - } - else if (version == YOLO_FASTEST || version == YOLO_FASTEST_XL) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - - float bias[] = {12, 18, 37, 49, 52, 132, 115, 73, 119, 199, 242, 238}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 3; - m_mask[1] = 4; - m_mask[2] = 5; - - m_mask[3] = 0; - m_mask[4] = 1; - m_mask[5] = 2; - } - else if (version == YOLO_FASTEST_BODY) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - - float bias[] = {7, 17, 20, 50, 45, 99, 64, 187, 123, 211, 227, 264}; - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 3; - m_mask[1] = 4; - m_mask[2] = 5; - - m_mask[3] = 0; - m_mask[4] = 1; - m_mask[5] = 2; - } - else if (version == YOLOV4_TINY_3L) - { - m_anchors_scale[0] = 32; - m_anchors_scale[1] = 16; - m_anchors_scale[2] = 8; - - //float bias_official[] = {12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401}; - float bias[] = {10, 14, 23, 27, 37, 58, 36, 75, 76, 55, 72, 146, 81, 82, 135, 169, 344, 319}; - - memcpy(m_biases, bias, sizeof(bias)); - - m_mask[0] = 6; - m_mask[1] = 7; - m_mask[2] = 8; - - m_mask[3] = 3; - m_mask[4] = 4; - m_mask[5] = 5; - - m_mask[6] = 0; - m_mask[7] = 1; - m_mask[8] = 2; - } - - m_confidence_threshold = confidence_threshold; - m_nms_threshold = nms_threshold; - m_confidence_threshold_unsigmoid = -1.0f * (float)std::log((1.0f / m_confidence_threshold) - 1.0f); - - return 0; - } - - int YoloDetectionOutput::forward_nhwc(const std::vector& bottom_blobs, std::vector& top_blobs) - { - // gather all box - std::vector all_bbox_rects; - for (size_t b = 0; b < bottom_blobs.size(); b++) - { - const TMat& bottom_top_blobs = bottom_blobs[b]; - - int w = bottom_top_blobs.w; - int h = bottom_top_blobs.h; - size_t mask_offset = b * m_num_box; - int net_w = (int)(m_anchors_scale[b] * w); - int net_h = (int)(m_anchors_scale[b] * h); - - auto feature_ptr = (float*)bottom_top_blobs.data; - for (int i = 0; i < h; i++) - { - for (int j = 0; j < w; j++) - { - for (int box = 0; box < m_num_box; ++box) - { - if (feature_ptr[4] < m_confidence_threshold_unsigmoid) - { - feature_ptr += (m_num_class + 5); - continue; - } - - int class_index = 0; - float class_score = -FLT_MAX; - for (int k = 5; k < m_num_class + 5; ++k) - { - if (class_score < feature_ptr[k]) - { - class_score = feature_ptr[k]; - class_index = k - 5; - } - } - - //sigmoid(box_score) * sigmoid(class_score) - float confidence_1 = 1.0f / ((1.f + std::exp(-feature_ptr[4])) * (1.f + std::exp(-class_score))); - if (confidence_1 >= m_confidence_threshold) - { - int biases_index = (int)(m_mask[box + mask_offset]); - const float bias_w = m_biases[biases_index * 2]; - const float bias_h = m_biases[biases_index * 2 + 1]; - - // region box - // fprintf(stderr, "%f %f %d \n", class_score, feature_ptr[4], class_index); - float bbox_cx = ((float)j + sigmoid(feature_ptr[0])) / (float)w; - float bbox_cy = ((float)i + sigmoid(feature_ptr[1])) / (float)h; - auto bbox_w = (float)(std::exp(feature_ptr[2]) * bias_w / (float)net_w); - auto bbox_h = (float)(std::exp(feature_ptr[3]) * bias_h / (float)net_h); - - float bbox_xmin = bbox_cx - bbox_w * 0.5f; - float bbox_ymin = bbox_cy - bbox_h * 0.5f; - float bbox_xmax = bbox_cx + bbox_w * 0.5f; - float bbox_ymax = bbox_cy + bbox_h * 0.5f; - - float area = bbox_w * bbox_h; - - BBoxRect c = {confidence_1, bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, area, class_index}; - all_bbox_rects.push_back(c); - } - - feature_ptr += (m_num_class + 5); - } - } - } - } - - // global sort inplace - qsort_descent_inplace(all_bbox_rects); - - // apply nms - std::vector picked; - nms_sorted_bboxes(all_bbox_rects, picked, m_nms_threshold); - - // select - std::vector bbox_rects; - - for (unsigned int z : picked) - { - bbox_rects.push_back(all_bbox_rects[z]); - } - - // fill result - int num_detected = (int)(bbox_rects.size()); - if (num_detected == 0) - { - top_blobs[0].h = 0; - return 0; - } - - TMat& top_blob = top_blobs[0]; - - for (int i = 0; i < num_detected; i++) - { - const BBoxRect& r = bbox_rects[i]; - float score = r.score; - float* outptr = top_blob.row(i); - - outptr[0] = (float)r.label; // +1 for prepend background class - outptr[1] = score; - outptr[2] = r.xmin; - outptr[3] = r.ymin; - outptr[4] = r.xmax; - outptr[5] = r.ymax; - } - top_blob.h = num_detected; - return 0; - } - - int YoloDetectionOutput::forward(const std::vector& bottom_blobs, std::vector& top_blobs) - { - // gather all box - std::vector all_bbox_rects; - - for (size_t b = 0; b < bottom_blobs.size(); b++) - { - std::vector > all_box_bbox_rects; - all_box_bbox_rects.resize(m_num_box); - const TMat& bottom_top_blobs = bottom_blobs[b]; - - int w = bottom_top_blobs.w; - int h = bottom_top_blobs.h; - int channels = bottom_top_blobs.c; - //printf("%d %d %d\n", w, h, channels); - const int channels_per_box = channels / m_num_box; - - // anchor coord + box score + num_class - if (channels_per_box != 4 + 1 + m_num_class) - return -1; - size_t mask_offset = b * m_num_box; - int net_w = (int)(m_anchors_scale[b] * w); - int net_h = (int)(m_anchors_scale[b] * h); - //printf("%d %d\n", net_w, net_h); - - //printf("%d %d %d\n", w, h, channels); - for (int pp = 0; pp < m_num_box; pp++) - { - int p = pp * channels_per_box; - int biases_index = (int)(m_mask[pp + mask_offset]); - //printf("%d\n", biases_index); - const float bias_w = m_biases[biases_index * 2]; - const float bias_h = m_biases[biases_index * 2 + 1]; - //printf("%f %f\n", bias_w, bias_h); - const float* xptr = bottom_top_blobs.channel(p); - const float* yptr = bottom_top_blobs.channel(p + 1); - const float* wptr = bottom_top_blobs.channel(p + 2); - const float* hptr = bottom_top_blobs.channel(p + 3); - - const float* box_score_ptr = bottom_top_blobs.channel(p + 4); - - // softmax class scores - TMat scores = bottom_top_blobs.channel_range(p + 5, m_num_class); - //softmax->forward_inplace(scores, opt); - - for (int i = 0; i < h; i++) - { - for (int j = 0; j < w; j++) - { - // find class index with max class score - int class_index = 0; - float class_score = -FLT_MAX; - for (int q = 0; q < m_num_class; q++) - { - float score = scores.channel(q).row(i)[j]; - if (score > class_score) - { - class_index = q; - class_score = score; - } - } - - //sigmoid(box_score) * sigmoid(class_score) - float confidence = (float)1.f / ((1.f + std::exp(-box_score_ptr[0]) * (1.f + std::exp(-class_score)))); - if (confidence >= m_confidence_threshold) - { - // fprintf(stderr, "%f %d \n", class_score, class_index); - // region box - float bbox_cx = ((float)j + sigmoid(xptr[0])) / (float)w; - float bbox_cy = ((float)i + sigmoid(yptr[0])) / (float)h; - auto bbox_w = (float)(std::exp(wptr[0]) * bias_w / (float)net_w); - auto bbox_h = (float)(std::exp(hptr[0]) * bias_h / (float)net_h); - - float bbox_xmin = bbox_cx - bbox_w * 0.5f; - float bbox_ymin = bbox_cy - bbox_h * 0.5f; - float bbox_xmax = bbox_cx + bbox_w * 0.5f; - float bbox_ymax = bbox_cy + bbox_h * 0.5f; - - float area = bbox_w * bbox_h; - - BBoxRect c = {confidence, bbox_xmin, bbox_ymin, bbox_xmax, bbox_ymax, area, class_index}; - all_box_bbox_rects[pp].push_back(c); - } - - xptr++; - yptr++; - wptr++; - hptr++; - - box_score_ptr++; - } - } - } - - for (int i = 0; i < m_num_box; i++) - { - const std::vector& box_bbox_rects = all_box_bbox_rects[i]; - - all_bbox_rects.insert(all_bbox_rects.end(), box_bbox_rects.begin(), box_bbox_rects.end()); - } - } - - // global sort inplace - qsort_descent_inplace(all_bbox_rects); - - // apply nms - std::vector picked; - nms_sorted_bboxes(all_bbox_rects, picked, m_nms_threshold); - - // select - std::vector bbox_rects; - - for (unsigned int z : picked) - { - bbox_rects.push_back(all_bbox_rects[z]); - } - - // fill result - int num_detected = (int)(bbox_rects.size()); - if (num_detected == 0) - { - top_blobs[0].h = 0; - return 0; - } - - TMat& top_blob = top_blobs[0]; - - for (int i = 0; i < num_detected; i++) - { - const BBoxRect& r = bbox_rects[i]; - float score = r.score; - float* outptr = top_blob.row(i); - - outptr[0] = (float)r.label; // +1 for prepend background class - outptr[1] = score; - outptr[2] = r.xmin; - outptr[3] = r.ymin; - outptr[4] = r.xmax; - outptr[5] = r.ymax; - } - top_blob.h = num_detected; - - return 0; - } - -} // namespace yolo diff --git a/projects/llm_framework/main_yolo/src/runner/utils/checker.h b/projects/llm_framework/main_yolo/src/runner/utils/checker.h deleted file mode 100644 index 9b7e288..0000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/checker.h +++ /dev/null @@ -1,32 +0,0 @@ -/************************************************************************************************** - * - * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. - * - * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and - * may not be copied or distributed in any isomorphic form without the prior - * written consent of Axera Semiconductor (Ningbo) Co., Ltd. - * - **************************************************************************************************/ - -#ifndef AX_CHECKER_H -#define AX_CHECKER_H - -#include "utils/logger.h" - -#define CHECK_PTR(p) \ - do { \ - if (!p) { \ - ALOGE("%s nil pointer\n", #p); \ - return -1; \ - } \ - } while (0) - -#define CHECK_INITED(p) \ - do { \ - if (!p->HasInit()) { \ - ALOGE("%s has not init\n", #p); \ - return -1; \ - } \ - } while (0) - -#endif //AX_CHECKER_H diff --git a/projects/llm_framework/main_yolo/src/runner/utils/io.hpp b/projects/llm_framework/main_yolo/src/runner/utils/io.hpp deleted file mode 100644 index 64db991..0000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/io.hpp +++ /dev/null @@ -1,601 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/checker.h" -#include "ax_sys_api.h" -#include "ax_engine_type.h" - - -#define IO_CMM_ALIGN_SIZE 128 - -namespace utils { - typedef enum { - IO_BUFFER_STRATEGY_DEFAULT, - IO_BUFFER_STRATEGY_CACHED - } IO_BUFFER_STRATEGY_T; - - static inline AX_S32 query_model_input_size(const AX_ENGINE_IO_INFO_T* io_info, std::array &input_size, - AX_IMG_FORMAT_E &eDtype) { - int height = 0; - int width = 0; - int size = 0; - int channel = 0; - int data_type_size = 0; - auto& input = io_info->pInputs[0]; - - switch (input.eLayout) { - case AX_ENGINE_TENSOR_LAYOUT_NHWC: - height = input.pShape[1]; - width = input.pShape[2]; - channel = input.pShape[3]; - size = input.nSize; - break; - case AX_ENGINE_TENSOR_LAYOUT_NCHW: - channel = input.pShape[1]; - height = input.pShape[2]; - width = input.pShape[3]; - size = input.nSize; - break; - default: // NHWC - height = input.pShape[1]; - width = input.pShape[2]; - channel = input.pShape[3]; - size = input.nSize; - break; - } - - switch (input.eDataType) { - case AX_ENGINE_DT_UINT8: - case AX_ENGINE_DT_SINT8: - data_type_size = 1; - break; - case AX_ENGINE_DT_UINT16: - case AX_ENGINE_DT_SINT16: - data_type_size = 2; - break; - case AX_ENGINE_DT_FLOAT32: - data_type_size = 4; - break; - case AX_ENGINE_DT_SINT32: - case AX_ENGINE_DT_UINT32: - data_type_size = 4; - break; - case AX_ENGINE_DT_FLOAT64: - data_type_size = 8; - break; - default: - data_type_size = 1; - break; - } - - if (channel == 0 || height == 0 || width == 0 || size == 0 || data_type_size == 0) { - return -1; - } - - if (input.pExtraMeta) { - switch (input.pExtraMeta->eColorSpace) { - case AX_ENGINE_CS_BGR: - input_size[0] = height; - input_size[1] = width; - eDtype = AX_FORMAT_BGR888; - break; - case AX_ENGINE_CS_RGB: - input_size[0] = height; - input_size[1] = width; - eDtype = AX_FORMAT_RGB888; - break; - case AX_ENGINE_CS_NV12: - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR; - break; - case AX_ENGINE_CS_NV21: - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR_VU; - break; - default: // AX_ENGINE_CS_NV12 - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR; - break; - } - } - else { - input_size[0] = height * 2 / 3; - input_size[1] = width; - eDtype = AX_FORMAT_YUV420_SEMIPLANAR; - } - - ALOGD("eLayout:%d, eDataType:%d, channel:%d, height:%d, width:%d, size:%d, data_type_size:%d", - input.eLayout, input.eDataType, channel, input_size[0], input_size[1], size, data_type_size); - - return 0; - } - - static inline void brief_io_info(std::string strModel, const AX_ENGINE_IO_INFO_T* io_info) { - auto describe_shape_type = [](AX_ENGINE_TENSOR_LAYOUT_T type) -> const char* { - switch (type) { - case AX_ENGINE_TENSOR_LAYOUT_NHWC: - return "NHWC"; - case AX_ENGINE_TENSOR_LAYOUT_NCHW: - return "NCHW"; - default: - return "unknown"; - } - }; - auto describe_data_type = [](AX_ENGINE_DATA_TYPE_T type) -> const char* { - switch (type) { - case AX_ENGINE_DT_UINT8: - return "uint8"; - case AX_ENGINE_DT_UINT16: - return "uint16"; - case AX_ENGINE_DT_FLOAT32: - return "float32"; - case AX_ENGINE_DT_SINT16: - return "sint16"; - case AX_ENGINE_DT_SINT8: - return "sint8"; - case AX_ENGINE_DT_SINT32: - return "sint32"; - case AX_ENGINE_DT_UINT32: - return "uint32"; - case AX_ENGINE_DT_FLOAT64: - return "float64"; - case AX_ENGINE_DT_UINT10_PACKED: - return "uint10_packed"; - case AX_ENGINE_DT_UINT12_PACKED: - return "uint12_packed"; - case AX_ENGINE_DT_UINT14_PACKED: - return "uint14_packed"; - case AX_ENGINE_DT_UINT16_PACKED: - return "uint16_packed"; - default: - return "unknown"; - } - }; - auto describe_memory_type = [](AX_ENGINE_MEMORY_TYPE_T type) -> const char* { - switch (type) { - case AX_ENGINE_MT_PHYSICAL: - return "Physical"; - case AX_ENGINE_MT_VIRTUAL: - return "Virtual"; - default: - return "unknown"; - } - }; - auto describe_color_space = [](AX_ENGINE_COLOR_SPACE_T cs) -> const char* { - switch (cs) { - case AX_ENGINE_CS_FEATUREMAP: - return "FeatureMap"; - case AX_ENGINE_CS_BGR: - return "BGR"; - case AX_ENGINE_CS_RGB: - return "RGB"; - case AX_ENGINE_CS_RGBA: - return "RGBA"; - case AX_ENGINE_CS_GRAY: - return "GRAY"; - case AX_ENGINE_CS_NV12: - return "NV12"; - case AX_ENGINE_CS_NV21: - return "NV21"; - case AX_ENGINE_CS_YUV444: - return "YUV444"; - case AX_ENGINE_CS_RAW8: - return "RAW8"; - case AX_ENGINE_CS_RAW10: - return "RAW10"; - case AX_ENGINE_CS_RAW12: - return "RAW12"; - case AX_ENGINE_CS_RAW14: - return "RAW14"; - case AX_ENGINE_CS_RAW16: - return "RAW16"; - default: - return "unknown"; - } - }; - printf("Model Name: %s\n", strModel.c_str()); - printf("Max Batch Size %d\n", io_info->nMaxBatchSize); - printf("Support Dynamic Batch? %s\n", io_info->bDynamicBatchSize == AX_TRUE ? "Yes" : "No"); - - for (uint32_t i = 0; i < io_info->nInputSize; ++i) { - auto& input = io_info->pInputs[i]; - printf("Input[%d]: %s\n", i, input.pName); - printf(" Shape ["); - for (uint32_t j = 0; j < input.nShapeSize; ++j) { - printf("%d", (int)input.pShape[j]); - if (j + 1 < input.nShapeSize) printf(", "); - } - printf("] %s %s %s %s\n", describe_shape_type(input.eLayout), describe_data_type(input.eDataType), - input.pExtraMeta ? describe_color_space(input.pExtraMeta->eColorSpace) : "", - input.nQuantizationValue > 0 ? ("Q=" + std::to_string(input.nQuantizationValue)).c_str() : ""); - printf(" Memory %s\n", describe_memory_type(input.eMemoryType)); - printf(" Size %u\n", input.nSize); - } - for (uint32_t i = 0; i < io_info->nOutputSize; ++i) { - auto& output = io_info->pOutputs[i]; - printf("Output[%d]: %s\n", i, output.pName); - printf(" Shape ["); - for (uint32_t j = 0; j < output.nShapeSize; ++j) { - printf("%d", (int)output.pShape[j]); - if (j + 1 < output.nShapeSize) printf(", "); - } - printf("] %s %s %s\n", describe_shape_type(output.eLayout), describe_data_type(output.eDataType), - output.nQuantizationValue > 0 ? ("Q=" + std::to_string(output.nQuantizationValue)).c_str() : ""); - printf(" Memory %s\n", describe_memory_type(output.eMemoryType)); - printf(" Size %u\n", output.nSize); - } - } - - static inline AX_S32 alloc_engine_buffer(const std::string& token, const std::string& appendix, size_t index, const AX_ENGINE_IOMETA_T* pMeta, AX_ENGINE_IO_BUFFER_T* pBuf, IO_BUFFER_STRATEGY_T eStrategy = IO_BUFFER_STRATEGY_DEFAULT) { - AX_S32 ret = -1; - if (eStrategy != IO_BUFFER_STRATEGY_DEFAULT && eStrategy != IO_BUFFER_STRATEGY_CACHED) { - fprintf(stderr, "strategy %d not supported\n", (int)eStrategy); - return -1; - } - memset(pBuf, 0, sizeof(AX_ENGINE_IO_BUFFER_T)); - pBuf->nSize = pMeta->nSize; - - const std::string token_name = "skel_" + token + appendix + std::to_string(index); - - if (eStrategy == IO_BUFFER_STRATEGY_CACHED) { - ret = AX_SYS_MemAllocCached((AX_U64*)&pBuf->phyAddr, &pBuf->pVirAddr, pBuf->nSize, IO_CMM_ALIGN_SIZE, (const AX_S8*)token_name.c_str()); - } - else { - ret = AX_SYS_MemAlloc((AX_U64*)&pBuf->phyAddr, &pBuf->pVirAddr, pBuf->nSize, IO_CMM_ALIGN_SIZE, (const AX_S8*)token_name.c_str()); - } - - return ret; - } - - static inline AX_S32 free_engine_buffer(AX_ENGINE_IO_BUFFER_T* pBuf) { - if (pBuf->phyAddr == 0) { - delete[] reinterpret_cast(pBuf->pVirAddr); - } - else { - AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); - } - pBuf->phyAddr = 0; - pBuf->pVirAddr = nullptr; - - return 0; - } - - static inline void free_io_index(AX_ENGINE_IO_BUFFER_T* io_buf, size_t index) { - AX_ENGINE_IO_BUFFER_T* pBuf = io_buf + index; - free_engine_buffer(pBuf); - } - - static inline void free_io(AX_ENGINE_IO_T &io) { - for (size_t j = 0; j < io.nInputSize; ++j) - { - AX_ENGINE_IO_BUFFER_T *pBuf = io.pInputs + j; - AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); - } - for (size_t j = 0; j < io.nOutputSize; ++j) - { - AX_ENGINE_IO_BUFFER_T *pBuf = io.pOutputs + j; - AX_SYS_MemFree(pBuf->phyAddr, pBuf->pVirAddr); - } - delete[] io.pInputs; - delete[] io.pOutputs; - } - - static inline void free_io(AX_ENGINE_IO_T &io, std::vector> &vecOutputBuffer) { - if (io.pInputs) { - delete[] io.pInputs; - io.pInputs = nullptr; - } - - if (io.pOutputs) { - for (size_t index = 0; index < vecOutputBuffer.size(); ++index) { - AX_ENGINE_IO_BUFFER_T *pOutputs = &vecOutputBuffer[index][0]; - for (size_t j = 0; j < io.nOutputSize; ++j) { - free_io_index(pOutputs, j); - } - } - - delete[] io.pOutputs; - io.pOutputs = nullptr; - } - } - - static inline int prepare_io(const std::string& token, const AX_ENGINE_IO_INFO_T* info, AX_ENGINE_IO_T &io, IO_BUFFER_STRATEGY_T strategy) { - auto ret = 0; - - memset(&io, 0, sizeof(io)); - - io.pInputs = new AX_ENGINE_IO_BUFFER_T[info->nInputSize]; - - if (!io.pInputs) { - goto EXIT; - } - - memset(io.pInputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nInputSize); - io.nInputSize = info->nInputSize; - for (AX_U32 i = 0; i < info->nInputSize; ++i) - { - auto meta = info->pInputs[i]; - auto buffer = &io.pInputs[i]; - ret = alloc_engine_buffer(token, "_input_", i, &meta, buffer, strategy); - if (ret != 0) - { - free_io_index(io.pInputs, i); - return ret; - } - } - - io.pOutputs = new AX_ENGINE_IO_BUFFER_T[info->nOutputSize]; - - if (!io.pOutputs) { - goto EXIT; - } - - memset(io.pOutputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nOutputSize); - io.nOutputSize = info->nOutputSize; - - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto meta = info->pOutputs[i]; - auto buffer = &io.pOutputs[i]; - ret = alloc_engine_buffer(token, "_output_", i, &meta, buffer, strategy); - if (ret != 0) { - goto EXIT; - } - } - - EXIT: - if (ret != 0) { - free_io(io); - return -1; - } - - return 0; - } - - static inline int prepare_io(const std::string& token, - const AX_ENGINE_IO_INFO_T* info, AX_ENGINE_IO_T &io, - std::vector &vecOutputBuffer, - const IO_BUFFER_STRATEGY_T &strategy) { - AX_S32 ret = 0; - memset(&io, 0, sizeof(io)); - - std::vector outputBuffer; - - if (1 != info->nInputSize) { - fprintf(stderr, "[ERR]: Only single input was accepted(got %u).\n", info->nInputSize); - return -1; - } - - io.pInputs = new AX_ENGINE_IO_BUFFER_T[info->nInputSize]; - - if (!io.pInputs) { - goto EXIT; - } - - memset(io.pInputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nInputSize); - io.nInputSize = info->nInputSize; - - for (AX_U32 i = 0; i < info->nInputSize; ++i) - { - auto meta = info->pInputs[i]; - auto buffer = &io.pInputs[i]; - ret = alloc_engine_buffer(token, "_input_", i, &meta, buffer, strategy); - if (ret != 0) - { - free_io_index(io.pInputs, i); - return ret; - } - } - - io.pOutputs = new AX_ENGINE_IO_BUFFER_T[info->nOutputSize]; - - if (!io.pOutputs) { - goto EXIT; - } - - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto meta = info->pOutputs[i]; - auto buffer = &io.pOutputs[i]; - ret = alloc_engine_buffer(token, "_output_", i, &meta, buffer, strategy); - - if (ret != 0) { - goto EXIT; - } - - vecOutputBuffer.push_back(*buffer); - } - - memset(io.pOutputs, 0x00, sizeof(AX_ENGINE_IO_BUFFER_T) * info->nOutputSize); - io.nOutputSize = info->nOutputSize; - - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto buffer = &io.pOutputs[i]; - *buffer = vecOutputBuffer[i]; - } - - EXIT: - if (ret != 0) { - free_io(io); - return -1; - } - - return 0; - } - - static inline AX_S32 push_io_output(const AX_ENGINE_IO_INFO_T* info, - AX_ENGINE_IO_T& io, - std::vector &outputBuffer) { - for (size_t i = 0; i < info->nOutputSize; ++i) { - auto buffer = &io.pOutputs[i]; - *buffer = outputBuffer[i]; - } - - return 0; - } - - static inline AX_S32 push_io_input(void* input, int index, AX_ENGINE_IO_T& io) { - // img ranks_depth ranks_feat ranks_bev, n_points - AX_ENGINE_IO_BUFFER_T* pImg = &io.pInputs[index]; - memcpy(pImg->pVirAddr, input, pImg->nSize); - return 0; - } - - static inline AX_S32 cache_io_flush(const AX_ENGINE_IO_BUFFER_T *io_buf) { - if (io_buf->phyAddr != 0) { - AX_SYS_MflushCache(io_buf->phyAddr, io_buf->pVirAddr, io_buf->nSize); - } - - return 0; - } - - static inline AX_S32 push_io_output(void* output, - int index, - AX_ENGINE_IO_T& io) { - AX_ENGINE_IO_BUFFER_T* pImg = &io.pOutputs[index]; - cache_io_flush(pImg); - memcpy(output, pImg->pVirAddr, pImg->nSize); - return 0; - } - - static inline AX_S32 cpu_copy(AX_U64 nPhyAddrSrc, AX_U64 nPhyAddrDst, AX_U32 nLen) { - if (nPhyAddrSrc != 0 && nPhyAddrDst != 0 && nLen > 0) { - AX_VOID* pSrcVirAddr = AX_SYS_MmapCache(nPhyAddrSrc, nLen); - AX_VOID* pDstVirAddr = AX_SYS_MmapCache(nPhyAddrDst, nLen); - - memcpy((AX_VOID*)pDstVirAddr, (AX_VOID*)pSrcVirAddr, nLen); - - AX_SYS_Munmap(pSrcVirAddr, nLen); - AX_SYS_Munmap(pDstVirAddr, nLen); - - return 0; - } - - return -1; - } - - static inline AX_S32 inc_io_ref_cnt(const AX_VIDEO_FRAME_T &stFrame) { - if (stFrame.u32BlkId[0] > 0) { - AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[0]); - } - if (stFrame.u32BlkId[1] > 0) { - AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[1]); - } - if (stFrame.u32BlkId[2] > 0) { - AX_POOL_IncreaseRefCnt(stFrame.u32BlkId[2]); - } - - return 0; - } - - static inline AX_S32 dec_io_ref_cnt(const AX_VIDEO_FRAME_T &stFrame) { - if (stFrame.u32BlkId[0] > 0) { - AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[0]); - } - if (stFrame.u32BlkId[1] > 0) { - AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[1]); - } - if (stFrame.u32BlkId[2] > 0) { - AX_POOL_DecreaseRefCnt(stFrame.u32BlkId[2]); - } - - return 0; - } - - static inline bool read_file(const char* path, std::vector& data) { - std::fstream fs(path, std::ios::in | std::ios::binary); - - if (!fs.is_open()) { - return false; - } - - fs.seekg(std::ios::end); - auto fs_end = fs.tellg(); - fs.seekg(std::ios::beg); - auto fs_beg = fs.tellg(); - - auto file_size = static_cast(fs_end - fs_beg); - auto vector_size = data.size(); - - data.reserve(vector_size + file_size); - data.insert(data.end(), std::istreambuf_iterator(fs), std::istreambuf_iterator()); - - fs.close(); - - return true; - } - - static inline bool read_file(const char* path, AX_VOID **pModelBufferVirAddr, - AX_U64 &u64ModelBufferPhyAddr, AX_U32 &nModelBufferSize) { - std::fstream fs(path, std::ios::in | std::ios::binary); - - if (!fs.is_open()) { - return false; - } - - fs.seekg(0, std::ios::end); - int file_size = fs.tellg(); - fs.seekg(0, std::ios::beg); - - nModelBufferSize = (AX_U32)file_size; - - AX_SYS_MemAlloc(&u64ModelBufferPhyAddr, pModelBufferVirAddr, nModelBufferSize, 0x100, (AX_S8 *)"SKEL-CV"); - - if (!pModelBufferVirAddr || (u64ModelBufferPhyAddr == 0)) { - return false; - } - - fs.read((AX_CHAR *)*pModelBufferVirAddr, nModelBufferSize); - - fs.close(); - - return true; - } - - static inline void dequant(float** pptrOutput, const AX_ENGINE_IOMETA_T& ptrIoInfo, const AX_ENGINE_IO_BUFFER_T& ioBuf, float zp, float scale) - { - if (ptrIoInfo.eDataType == AX_ENGINE_DT_FLOAT32) - { - *pptrOutput = (float*)ioBuf.pVirAddr; - return; - } - - *pptrOutput = (float*)malloc(ptrIoInfo.nSize * sizeof(float)); - uint8_t *pBuf = (uint8_t*)ioBuf.pVirAddr; - float* pOutput = *pptrOutput; - // float inv_scale = 1.0f / scale; - for (int i = 0; i < ptrIoInfo.nSize; i++) - { - pOutput[i] = ((float)pBuf[i] - zp) * scale; - } - } -} - diff --git a/projects/llm_framework/main_yolo/src/runner/utils/logger.h b/projects/llm_framework/main_yolo/src/runner/utils/logger.h deleted file mode 100644 index 83a2e69..0000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/logger.h +++ /dev/null @@ -1,82 +0,0 @@ -/************************************************************************************************** - * - * Copyright (c) 2019-2023 Axera Semiconductor (Ningbo) Co., Ltd. All Rights Reserved. - * - * This source file is the property of Axera Semiconductor (Ningbo) Co., Ltd. and - * may not be copied or distributed in any isomorphic form without the prior - * written consent of Axera Semiconductor (Ningbo) Co., Ltd. - * - **************************************************************************************************/ - -#ifndef SKEL_LOGGER_H -#define SKEL_LOGGER_H - -#include "ax_global_type.h" -#include "ax_sys_log.h" - -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - -//#define SKEL_LOG_TAG "SKEL" -// -//#define ALOGE(fmt, ...) AX_LOG_ERR_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGW(fmt, ...) AX_LOG_WARN_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGI(fmt, ...) AX_LOG_INFO_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGD(fmt, ...) AX_LOG_DBG_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) -//#define ALOGN(fmt, ...) AX_LOG_NOTICE_EX(SKEL_LOG_TAG, AX_ID_SKEL, fmt, ##__VA_ARGS__) - - -typedef enum { - SKEL_LOG_MIN = -1, - SKEL_LOG_EMERGENCY = 0, - SKEL_LOG_ALERT = 1, - SKEL_LOG_CRITICAL = 2, - SKEL_LOG_ERROR = 3, - SKEL_LOG_WARN = 4, - SKEL_LOG_NOTICE = 5, - SKEL_LOG_INFO = 6, - SKEL_LOG_DEBUG = 7, - SKEL_LOG_MAX -} SKEL_LOG_LEVEL_E; - -static SKEL_LOG_LEVEL_E log_level = SKEL_LOG_DEBUG; - -#if 1 -#define MACRO_BLACK "\033[1;30;30m" -#define MACRO_RED "\033[1;30;31m" -#define MACRO_GREEN "\033[1;30;32m" -#define MACRO_YELLOW "\033[1;30;33m" -#define MACRO_BLUE "\033[1;30;34m" -#define MACRO_PURPLE "\033[1;30;35m" -#define MACRO_WHITE "\033[1;30;37m" -#define MACRO_END "\033[0m" -#else -#define MACRO_BLACK -#define MACRO_RED -#define MACRO_GREEN -#define MACRO_YELLOW -#define MACRO_BLUE -#define MACRO_PURPLE -#define MACRO_WHITE -#define MACRO_END -#endif - -#define ALOGE(fmt, ...) printf(MACRO_RED "[E][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGW(fmt, ...) if (log_level >= SKEL_LOG_WARN) \ - printf(MACRO_YELLOW "[W][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGI(fmt, ...) if (log_level >= SKEL_LOG_INFO) \ - printf(MACRO_GREEN "[I][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGD(fmt, ...) if (log_level >= SKEL_LOG_DEBUG) \ - printf(MACRO_WHITE "[D][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) -#define ALOGN(fmt, ...) if (log_level >= SKEL_LOG_NOTICE) \ - printf(MACRO_PURPLE "[N][%32s][%4d]: " fmt MACRO_END "\n", __FUNCTION__, __LINE__, ##__VA_ARGS__) - -#ifdef __cplusplus -} -#endif - -#endif //SKEL_LOGGER_H diff --git a/projects/llm_framework/main_yolo/src/runner/utils/timer.hpp b/projects/llm_framework/main_yolo/src/runner/utils/timer.hpp deleted file mode 100644 index e753192..0000000 --- a/projects/llm_framework/main_yolo/src/runner/utils/timer.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * AXERA is pleased to support the open source community by making ax-samples available. - * - * Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved. - * - * Licensed under the BSD 3-Clause License (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * https://opensource.org/licenses/BSD-3-Clause - * - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -/* - * Author: ls.wang - */ - -#pragma once - -#include - -class timer -{ -private: - std::chrono::system_clock::time_point start_time, end_time; - -public: - timer() - { - start(); - } - - void start() - { - stop(); - this->start_time = this->end_time; - } - - void stop() - { -#ifdef _MSC_VER - this->end_time = std::chrono::system_clock::now(); -#else - this->end_time = std::chrono::high_resolution_clock::now(); -#endif - } - - float cost() - { - if (this->end_time <= this->start_time) - { - this->stop(); - } - - auto ms = std::chrono::duration_cast(this->end_time - this->start_time).count(); - return static_cast(ms) / 1000.f; - } -};