From 7ced4d8a7259af6725ae060bc02f3449eda6b1dc Mon Sep 17 00:00:00 2001 From: dianjixz <18637716021@163.com> Date: Fri, 21 Mar 2025 10:42:02 +0800 Subject: [PATCH] [update] llm Tokenizer --- .../StackFlow/stackflow/StackFlowUtil.cpp | 121 +++++++- .../StackFlow/stackflow/StackFlowUtil.h | 3 +- projects/llm_framework/main_llm/SConstruct | 15 +- ...izer_auto.py => llm-llm_tokenizer_auto.py} | 0 projects/llm_framework/main_llm/src/main.cpp | 42 ++- .../llm_framework/main_llm/src/runner/LLM.hpp | 3 + .../src/runner/Tokenizer/Tokenizer.cpp | 267 ++++++++---------- projects/llm_framework/main_yolo/src/main.cpp | 63 +++-- 8 files changed, 316 insertions(+), 198 deletions(-) rename projects/llm_framework/main_llm/scripts/{tokenizer_auto.py => llm-llm_tokenizer_auto.py} (100%) diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp index 24ee290..a5cebea 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -8,6 +8,11 @@ #include #include #include "pzmq.hpp" +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif + +// #include std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) { @@ -126,22 +131,112 @@ std::string StackFlows::sample_escapeString(const std::string &input) return escaped; } -std::string StackFlows::sample_unescapeString(const std::string &input) +void StackFlows::unicode_to_utf8(unsigned int codepoint, char *output, int *length) { + if (codepoint <= 0x7F) { + output[0] = codepoint & 0x7F; + *length = 1; + } else if (codepoint <= 0x7FF) { + output[0] = 0xC0 | ((codepoint >> 6) & 0x1F); + output[1] = 0x80 | (codepoint & 0x3F); + *length = 2; + } else if (codepoint <= 0xFFFF) { + output[0] = 0xE0 | ((codepoint >> 12) & 0x0F); + output[1] = 0x80 | ((codepoint >> 6) & 0x3F); + output[2] = 0x80 | (codepoint & 0x3F); + *length = 3; + } else if (codepoint <= 0x10FFFF) { + output[0] = 0xF0 | ((codepoint >> 18) & 0x07); + output[1] = 0x80 | ((codepoint >> 12) & 0x3F); + output[2] = 0x80 | ((codepoint >> 6) & 0x3F); + output[3] = 0x80 | (codepoint & 0x3F); + *length = 4; + } else { + *length = 0; + } +} + +std::string StackFlows::sample_unescapeString(const std::string &input, bool ucs2) { std::string unescaped; - for (size_t i = 0; i < input.length(); ++i) { - if (input[i] == '\\' && i + 1 < input.length()) { - switch (input[i + 1]) { - case 'n' :unescaped += '\n';++i;break; - case 't' :unescaped += '\t';++i;break; - case '\\':unescaped += '\\';++i;break; - case '\"':unescaped += '\"';++i;break; - case 'r' :unescaped += '\r';++i;break; - case 'b' :unescaped += '\b';++i;break; - default :unescaped += input[i];break; + unescaped.reserve(input.length()); + const char *itc = input.c_str(); + const char *itd = itc + input.length(); + while (itc != itd){ +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + if ((itd - itc) >= 16) { + const uint8_t *srcstr = (const uint8_t *)itc; + uint8x16_t target_open = vdupq_n_u8('\\'); + uint8x16_t input_vector = vld1q_u8(srcstr); + uint8x16_t result_open = vceqq_u8(input_vector, target_open); + __uint128_t jflage; + vst1q_u8((uint8_t *)&jflage, result_open); + if (jflage == 0) { + int pos = unescaped.size(); + unescaped.resize(pos + 16); + memcpy((void*)(unescaped.c_str() + pos), srcstr, 16); + itc += 16; + continue; } - } else { - unescaped += input[i]; + } +#endif + if((*itc == '\\') && (++itc != itd)){ + switch (*itc) + { + case 'n' :unescaped += '\n';itc++;break; + case 't' :unescaped += '\t';itc++;break; + case '\\':unescaped += '\\';itc++;break; + case '\"':unescaped += '\"';itc++;break; + case 'r' :unescaped += '\r';itc++;break; + case 'b' :unescaped += '\b';itc++;break; + case 'u' :{ + itc++; + if(ucs2) { + unescaped += "\\u"; + break; + } + unsigned int codepoint = 0; + auto itcb = itc; + do { + if((itd - itc) < 4) { + codepoint = -1; + break; + } + for (int i = 0; i < 4; i++) { + char c = *itc++; + codepoint <<= 4; + switch (c) { + case '0' ... '9' : codepoint |= (c - '0'); break; + case 'A' ... 'F' : codepoint |= (c - 'A' + 10); break; + case 'a' ... 'f' : codepoint |= (c - 'a' + 10); break; + default: codepoint = -1; i += 4 ; break; + } + } + } while (codepoint >= 0xD800 && codepoint <= 0xDBFF); + if(codepoint == -1) { + itc = itcb; + unescaped += "\\u"; + break; + } + char buff[4]; + int len = 0; + StackFlows::unicode_to_utf8(codepoint, buff, &len); + if(len != 0) { + for (int i = 0; i < len; i++) { + unescaped += buff[i]; + } + } else { + itc = itcb; + unescaped += "\\u"; + } + } + break; + default: + unescaped += *itc++; + break; + } + } + else { + unescaped += *itc++; } } return unescaped; diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.h b/ext_components/StackFlow/stackflow/StackFlowUtil.h index fa1d125..9b3df86 100644 --- a/ext_components/StackFlow/stackflow/StackFlowUtil.h +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.h @@ -28,7 +28,7 @@ int sample_get_work_id_num(const std::string &work_id); std::string sample_get_work_id_name(const std::string &work_id); std::string sample_get_work_id(int work_id_num, const std::string &unit_name); std::string sample_escapeString(const std::string &input); -std::string sample_unescapeString(const std::string &input); +std::string sample_unescapeString(const std::string &input, bool ucs2 = false); bool decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff); int decode_base64(const std::string &in, std::string &out); int encode_base64(const std::string &in, std::string &out); @@ -36,4 +36,5 @@ std::string unit_call(const std::string &unit_name, const std::string &unit_acti std::list get_config_file_paths(std::string &base_model_path, std::string &base_model_config_path, const std::string &mode_name); std::vector glob_files(const std::vector &patterns); bool file_exists(const std::string& filePath); +void unicode_to_utf8(unsigned int codepoint, char *output, int *length); }; // namespace StackFlows diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index 0381705..2170851 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -17,13 +17,13 @@ LDFLAGS = [] LINK_SEARCH_PATH = [] STATIC_FILES = [] -REQUIREMENTS += ['Backward_cpp'] -DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'), - AFile('../static_lib/libelf.so.1'), - AFile('../static_lib/libz.so.1'), - AFile('../static_lib/liblzma.so.5'), - AFile('../static_lib/libbz2.so.1.0')] -DEFINITIONS += ["-DENABLE_BACKWARD"] +# REQUIREMENTS += ['Backward_cpp'] +# DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'), +# AFile('../static_lib/libelf.so.1'), +# AFile('../static_lib/libz.so.1'), +# AFile('../static_lib/liblzma.so.5'), +# AFile('../static_lib/libbz2.so.1.0')] +# DEFINITIONS += ["-DENABLE_BACKWARD"] DEFINITIONS += ['-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] @@ -43,6 +43,7 @@ STATIC_LIB += static_file * 4 STATIC_FILES += Glob('scripts/tokenizer_*.py') STATIC_FILES += Glob('models/mode_*.json') +STATIC_FILES += [AFile('scripts/llm-llm_tokenizer_auto.py')] env['COMPONENTS'].append({'target':'llm_llm', 'SRCS':SRCS, diff --git a/projects/llm_framework/main_llm/scripts/tokenizer_auto.py b/projects/llm_framework/main_llm/scripts/llm-llm_tokenizer_auto.py similarity index 100% rename from projects/llm_framework/main_llm/scripts/tokenizer_auto.py rename to projects/llm_framework/main_llm/scripts/llm-llm_tokenizer_auto.py diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 34fee4a..d45fc5b 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "../../../../SDK/components/utilities/include/sample_log.h" using namespace StackFlows; #ifdef ENABLE_BACKWARD @@ -52,6 +53,10 @@ public: bool enstream_; std::atomic_bool tokenizer_server_flage_; unsigned int port_ = 8080; + sem_t inference_semaphore; + std::unique_ptr inference_run_; + std::atomic_bool is_running_; + std::string _inference_msg; void set_output(task_callback_t out_callback) { @@ -209,6 +214,30 @@ public: return oss_prompt.str(); } + void run() + { + sem_wait(&inference_semaphore); + while (is_running_) { + { + sem_wait(&inference_semaphore); + inference(_inference_msg); + sem_wait(&inference_semaphore); + } + } + } + + int inference_async(const std::string &msg) + { + int count = 0; + sem_getvalue(&inference_semaphore, &count); + if (count == 0) { + _inference_msg = msg; + sem_post(&inference_semaphore); + sem_post(&inference_semaphore); + } + return count; + } + void inference(const std::string &msg) { #if 1 @@ -265,10 +294,16 @@ public: llm_task(const std::string &workid) { + sem_init(&inference_semaphore, 0, 0); + is_running_ = true; + inference_run_ = std::make_unique(std::bind(&llm_task::run, this)); } ~llm_task() { + is_running_ = false; + sem_post(&inference_semaphore); + if (inference_run_) inference_run_->join(); if (lLaMa_) { lLaMa_->Deinit(); } @@ -385,7 +420,7 @@ public: } next_data = &tmp_msg2; } - llm_task_obj->inference((*next_data)); + llm_task_obj->inference_async(sample_unescapeString(*next_data)); } void task_asr_data(const std::weak_ptr llm_task_obj_weak, @@ -399,10 +434,10 @@ public: } if (object.find("stream") != std::string::npos) { if (sample_json_str_get(data, "finish") == "true") { - llm_task_obj->inference(sample_json_str_get(data, "delta")); + llm_task_obj->inference_async(sample_json_str_get(data, "delta")); } } else { - llm_task_obj->inference(data); + llm_task_obj->inference_async(data); } } @@ -586,6 +621,7 @@ public: } auto llm_channel = get_channel(work_id_num); llm_channel->stop_subscriber(""); + llm_task_[work_id_num]->lLaMa_->Stop(); llm_task_.erase(work_id_num); send("None", "None", LLM_NO_ERROR, work_id); return 0; diff --git a/projects/llm_framework/main_llm/src/runner/LLM.hpp b/projects/llm_framework/main_llm/src/runner/LLM.hpp index 73a6d91..988495c 100644 --- a/projects/llm_framework/main_llm/src/runner/LLM.hpp +++ b/projects/llm_framework/main_llm/src/runner/LLM.hpp @@ -318,6 +318,9 @@ public: { std::vector test_embed; Encode(test_embed, input_str); + if(test_embed.size() == 0) { + return ""; + } return Run(test_embed); } diff --git a/projects/llm_framework/main_llm/src/runner/Tokenizer/Tokenizer.cpp b/projects/llm_framework/main_llm/src/runner/Tokenizer/Tokenizer.cpp index 3ac09b3..5d4c71c 100644 --- a/projects/llm_framework/main_llm/src/runner/Tokenizer/Tokenizer.cpp +++ b/projects/llm_framework/main_llm/src/runner/Tokenizer/Tokenizer.cpp @@ -628,28 +628,26 @@ public: } }; - -class ResultObj -{ +class ResultObj { private: /* data */ public: bool success; nlohmann::json result; - ResultObj(bool _success, const nlohmann::json &_result){ + ResultObj(bool _success, const nlohmann::json &_result) + { success = _success; - result = _result; + result = _result; } }; -class Tokenizer_Auto : public BaseTokenizer -{ +class Tokenizer_Auto : public BaseTokenizer { bool _b_bos, _b_eos; std::string base_url; int bos_id, eos_id; - pid_t pid = 0; - int pipe_c2p[2] = {0}; // C++ -> Python - int pipe_p2c[2] = {0}; // Python -> C++ + pid_t pid = 0; + int pipe_c2p[2] = {0}; // C++ -> Python + int pipe_p2c[2] = {0}; // Python -> C++ private: ResultObj readPython(int id, int timeout = 10000) { @@ -659,41 +657,47 @@ private: while (timeout > 0) { ssize_t bytesRead = read(pipe_p2c[0], buffer, sizeof(buffer) - 1); if (bytesRead > 0) { - for(int i = 0; i < bytesRead; i++) - { + for (int i = 0; i < bytesRead; i++) { message += buffer[i]; - if(buffer[i] == '\n') - { - ALOGE("readPython:%s", message.c_str()); + if (buffer[i] == '\n') { + ALOGI("readPython:%s", message.c_str()); int pos = message.size() - 12; - if((message.size() > 12)&&(message.substr(pos) == "rpccontinue\n")) - { + if ((message.size() > 12) && (message.substr(pos) == "rpccontinue\n")) { message.erase(pos); continue; } nlohmann::json j2; - try - { + try { j2 = nlohmann::json::parse(message); - if(j2["id"].is_number_integer() && j2["id"] == id) - { + if (j2["id"].is_number_integer() && j2["id"] == id) { return ResultObj(true, j2["result"]); - } - else if(j2["id"].is_null()) - { + } else if (j2["id"].is_null()) { + std::string errormesg; + if (j2["result"].is_string()) { + errormesg = j2["result"]; + } + ALOGE("python runtime error: %s", errormesg.c_str()); return ResultObj(false, j2); } - } - catch (...) - { - message = ""; - continue; + } catch (const nlohmann::json::parse_error &e) { + ALOGE("JSON parse error: %s", e.what()); + return ResultObj(false, ""); + } catch (const nlohmann::json::type_error &e) { + ALOGE("JSON type error: %s", e.what()); + return ResultObj(false, ""); + } catch (const nlohmann::json::out_of_range &e) { + ALOGE("JSON out of range error: %s", e.what()); + return ResultObj(false, ""); + } catch (const std::exception &e) { + ALOGE("Standard exception: %s", e.what()); + return ResultObj(false, ""); + } catch (...) { + ALOGE("Unknown error occurred while parsing JSON"); + return ResultObj(false, ""); } } } - } - else - { + } else { break; } usleep(10000); @@ -704,73 +708,67 @@ private: ResultObj callPython(nlohmann::json &rpcobj) { - static int id = 0; - rpcobj["jsonrpc"] = "2.0"; - rpcobj["id"] = ++id; - std::string _rawdata = rpcobj.dump(); + static int id = 0; + rpcobj["jsonrpc"] = "2.0"; + rpcobj["id"] = ++id; + std::string _rawdata = rpcobj.dump(-1, ' ', true); _rawdata += "\n"; std::string rawdata; rawdata.reserve(_rawdata.size()); - for (int i = 0; i < _rawdata.length(); i++) - { - if((_rawdata.length() - i > 4)&&(_rawdata[i] == '\\')&&(_rawdata[i + 1] == '\\')&&(_rawdata[i + 2] == 'u')) - { + for (int i = 0; i < _rawdata.length(); i++) { + if ((_rawdata.length() - i > 4) && (_rawdata[i] == '\\') && (_rawdata[i + 1] == '\\') && + (_rawdata[i + 2] == 'u')) { rawdata += '\\'; rawdata += 'u'; i += 2; - } - else - { + } else { rawdata += _rawdata[i]; } } - const char* delstr = "\"DELSELF\":0"; - size_t pos = rawdata.find(delstr); - if(pos != std::string::npos) - { + const char *delstr = "\"DELSELF\":0"; + size_t pos = rawdata.find(delstr); + if (pos != std::string::npos) { rawdata.erase(pos, strlen(delstr)); } - for(;;) - { - if(rawdata.size() > 1024) - { - auto part_data = rawdata.substr(0, 1024); - part_data += "rpccontinue\n"; - ALOGE("callPython write:%s", part_data.c_str()); - write(pipe_c2p[1], part_data.c_str(), part_data.size()); - rawdata.erase(0, 1024); + int start_pos = 0; + for (;;) { + if (rawdata.length() - start_pos > 1024) { + // ALOGE("callPython write:%.1024s", rawdata.c_str() + start_pos); + write(pipe_c2p[1], rawdata.c_str() + start_pos, 1024); + write(pipe_c2p[1], "rpccontinue\n", 12); + start_pos += 1024; continue; } - ALOGE("callPython write:%s", rawdata.c_str()); - write(pipe_c2p[1], rawdata.c_str(), rawdata.size()); + // ALOGE("callPython write:%.*s", rawdata.length() - start_pos, rawdata.c_str() + start_pos); + write(pipe_c2p[1], rawdata.c_str() + start_pos, rawdata.length() - start_pos); break; } + ALOGI("callPython write:%s", rawdata.c_str()); return readPython(id); } - void setNonBlocking(int fd) { - int flags = fcntl(fd, F_GETFL, 0); - if (flags == -1) { - perror("fcntl F_GETFL"); - exit(EXIT_FAILURE); - } - if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) { - perror("fcntl F_SETFL"); - exit(EXIT_FAILURE); - } - } public: ~Tokenizer_Auto() { - if(pid > 0) - { + if (pid > 0) { kill(pid, SIGTERM); waitpid(pid, nullptr, 0); } } + bool Init(std::string model_path = "tokenizer", bool b_bos = true, bool b_eos = false) override { - ALOGE("Tokenizer_Auto model_path:%s", model_path.c_str()); + // ALOGE("Tokenizer_Auto model_path:%s", model_path.c_str()); + std::string tokenizer_script = model_path; + std::string model_id = model_path.substr(0, model_path.find("_")); + struct stat st; + if (stat(tokenizer_script.c_str(), &st) == 0) { + if (S_ISDIR(st.st_mode)) { + tokenizer_script = "/opt/m5stack/scripts/llm-llm_tokenizer_auto.py"; + } + } else { + return false; + } if (pipe(pipe_c2p) == -1 || pipe(pipe_p2c) == -1) { perror("pipe"); return false; @@ -781,30 +779,22 @@ public: return false; } if (pid == 0) { - // 子进程（Python） - // 子进程关闭不必要的管道端 - close(pipe_c2p[1]); // 关闭写端 - close(pipe_p2c[0]); // 关闭读端 - // 将管道端重定向到标准输入输出 - dup2(pipe_c2p[0], STDIN_FILENO); // 将C++写端重定向为Python的标准输入 - dup2(pipe_p2c[1], STDOUT_FILENO); // 将Python的标准输出重定向为C++的读端 - // 关闭重定向后的管道端 + close(pipe_c2p[1]); + close(pipe_p2c[0]); + dup2(pipe_c2p[0], STDIN_FILENO); + dup2(pipe_p2c[1], STDOUT_FILENO); close(pipe_c2p[0]); close(pipe_p2c[1]); - // 执行Python脚本 - execlp("/usr/bin/python3", "python3", "/opt/m5stack/share/tokenizer_auto.py", "--model_id", model_path.c_str(), nullptr); + execlp("/usr/bin/python3", "python3", tokenizer_script.c_str(), "--model_id", model_id.c_str(), nullptr); perror("execlp"); exit(EXIT_FAILURE); - } - // 父进程（C++） - // 父进程关闭不必要的管道端 - close(pipe_c2p[0]); // 关闭读端 - close(pipe_p2c[1]); // 关闭写端 + } + close(pipe_c2p[0]); + close(pipe_p2c[1]); auto ret = readPython(0, 15000); - if(ret.success) - { - bos_id = ret.result["bos_id"].is_number_integer() ? (int)ret.result["bos_id"] : 0; - eos_id = ret.result["eos_id"].is_number_integer() ? (int)ret.result["eos_id"] : 0; + if (ret.success) { + bos_id = ret.result["bos_id"].is_number_integer() ? (int)ret.result["bos_id"] : 0; + eos_id = ret.result["eos_id"].is_number_integer() ? (int)ret.result["eos_id"] : 0; this->_b_bos = b_bos; this->_b_eos = b_eos; } @@ -813,17 +803,13 @@ public: bool Encode(std::string input, std::vector &output, bool b_img_prompt = false) override { - nlohmann::json rpcobj; rpcobj["method"] = "encode"; rpcobj["params"] = nlohmann::json::array({nlohmann::json::array({input}), {{"DELSELF", 0}}}); - auto ret = callPython(rpcobj); - if(ret.success) - { + auto ret = callPython(rpcobj); + if (ret.success) { if (ret.result.is_array()) { output = ret.result.get>(); - } else { - std::cout << "result 键不存在或不是一个数组！" << std::endl; } } return ret.success; @@ -841,14 +827,13 @@ public: nlohmann::json rpcobj; rpcobj["method"] = "decode"; rpcobj["params"] = nlohmann::json::array({nlohmann::json::array({input}), {{"DELSELF", 0}}}); - auto ret = callPython(rpcobj); - if(ret.success) - { + auto ret = callPython(rpcobj); + if (ret.success) { return ret.result.get(); } return ""; } - + int GetBosID() override { return bos_id; @@ -860,53 +845,40 @@ public: } std::string apply_chat_template() override { - // std::list> messages; - // auto ret = callPython("apply_chat_template", R"([[[{"role":"user","content":"hello"}]],{"tokenize":false,"add_generation_prompt":true}])"); - std::list messages_list; - // nlohmann::json messages_list = nlohmann::json::array(); - for(auto &message : messages_) - { + nlohmann::json messages_list = nlohmann::json::array(); + for (auto &message : messages_) { nlohmann::json msg; - switch (message.first) - { - case ROLE_USER: - { - // {"role": "user", "content": prompt} - msg["role"] = "user"; - msg["content"] = (std::string)message.second; - messages_list.push_back(msg); + switch (message.first) { + case ROLE_USER: { + // {"role": "user", "content": prompt} + msg["role"] = "user"; + msg["content"] = (std::string)message.second; + messages_list.push_back(msg); + } break; + case ROLE_SYSTEM: { + msg["role"] = "system"; + msg["content"] = message.second; + messages_list.push_back(msg); + } break; + case ROLE_ASSISTANT: { + msg["role"] = "assistant"; + msg["content"] = message.second; + messages_list.push_back(msg); + } break; + case ROLE_ASSISTANT_HELP: { + } break; + default: + break; } - break; - case ROLE_SYSTEM: - { - msg["role"] = "system"; - msg["content"] = message.second; - messages_list.push_back(msg); - } - break; - case ROLE_ASSISTANT: - { - msg["role"] = "assistant"; - msg["content"] = message.second; - messages_list.push_back(msg); - } - break; - case ROLE_ASSISTANT_HELP: - { - } - break; - default: - break; - } - } + } nlohmann::json rpcobj; rpcobj["method"] = "apply_chat_template"; - rpcobj["params"] = nlohmann::json::array({nlohmann::json::array({messages_list}), {{"tokenize", false}, {"add_generation_prompt", true}}}); + rpcobj["params"] = nlohmann::json::array( + {nlohmann::json::array({messages_list}), {{"tokenize", false}, {"add_generation_prompt", true}}}); auto ret = callPython(rpcobj); - if(ret.success) - { + if (ret.success) { std::string out_str = (std::string)ret.result; - ALOGE("out_str:%s", out_str.c_str()); + // ALOGE("out_str:%s", out_str.c_str()); return out_str; } return ""; @@ -934,14 +906,3 @@ std::shared_ptr CreateTokenizer(TokenizerType type) } } - - - - - -// {"id":32,"jsonrpc":"2.0","method":"decode","params":[[[15469,1,13352,369,330,471,16488,11229,497,892,374,264,8870,315,6366,8038,429,23497,389,6825,24514,12645,429,646,2736,9079,429,11136,1373,3738,11229,11,1741,438,6832,504,821,323,3259,11181,13,15235,17601,279,4401,315,25185,11,4119,11,323,5942,429,646,23643,821,11,3960,504,3139,11,323,1281,11181,3118,389,429,6540,13,15235,702,1657,8357,304,5257,5043,1741,438,18478,11,17017,11,17903,11,323,803,13]],{}]} - - - -// {"id":404,"jsonrpc":"2.0","method":"decode","params":[[[131070,796,15371,379,655,10919,78,198,6582,40887,1594,11825,20172,1531,650,33900,29040,1187,64466,409,1187,308,3760,1395,92433,655,65484,379,511,5165,64,662,5093,42838,645,409,18299,78,11,1709,1320,55611,264,49352,1594,10919,78,13,38676,18299,78,511,93676,64466,379,15466,662,733,4942,88026,11,1395,269,5039,78,655,65484,28526,74783,264,5141,24743,1603,1594,11825,20172,92001,399,78,13,55708,64466,1395,92433,1187,82309,409,5093,7473,3655,409,15045,1794,67116,11,6368,4883,4154,655,10919,78,379,32502,4883,264,5141,24743,1603,1594,11825,20172,13,4929,64466,1395,92433,913,83444,409,43892,733,4942,379,1531,1187,140100,4154,1187,1709,655,11825,20172,92001,399,78,20673,23191,3828,264,465,65251,11,13526,1709,902,41598,1531,9434,13,17632,47602,11,17669,11825,52430,24743,1603,92001,399,300,1709,40504,1420,10918,2098,20410,379,511,40504,82911,261,13,19801,57200,11,5141,24743,1603,1594,11825,20172,92001,399,78,409,48976,297,655,326,6304,409,619,6654,79,2015,4438,22506,2098,20410,379,511,40504,82911,261,662,1187,64466,1594,11825,20172,92001,399,78,13,87483,11,5141,24743,1603,409,48976,379,619,6654,79,2015,4438,22506,264,465,65251,300,90892,264,913,64466,24245,64,1709,20673,29570,391,277,1187,39036,409,2478,9282,18244,13,2308,1508,4706,68,11,58366,5141,24743,1603,92001,399,300,40504,1420,10918,2098,20410,379,264,465,65251,300,11,902,30101,4438,14197,24245,300,7953,5141,24743,1603,409,48976,297,619,6654,79,2015,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,35797,409,1187,24949,662,655,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187]],{}]} -// {"id":404,"jsonrpc":"2.0","method":"decode","params":[[[131070,796,15371,379,655,10919,78,198,6582,40887,1594,11825,20172,1531,650,33900,29040,1187,64466,409,1187,308,3760,1395,92433,655,65484,379,511,5165,64,662,5093,42838,645,409,18299,78,11,1709,1320,55611,264,49352,1594,10919,78,13,38676,18299,78,511,93676,64466,379,15466,662,733,4942,88026,11,1395,269,5039,78,655,65484,28526,74783,264,5141,24743,1603,1594,11825,20172,92001,399,78,13,55708,64466,1395,92433,1187,82309,409,5093,7473,3655,409,15045,1794,67116,11,6368,4883,4154,655,10919,78,379,32502,4883,264,5141,24743,1603,1594,11825,20172,13,4929,64466,1395,92433,913,83444,409,43892,733,4942,379,1531,1187,140100,4154,1187,1709,655,11825,20172,92001,399,78,20673,23191,3828,264,465,65251,11,13526,1709,902,41598,1531,9434,13,17632,47602,11,17669,11825,52430,24743,1603,92001,399,300,1709,40504,1420,10918,2098,20410,379,511,40504,82911,261,13,19801,57200,11,5141,24743,1603,1594,11825,20172,92001,399,78,409,48976,297,655,326,6304,409,619,6654,79,2015,4438,22506,2098,20410,379,511,40504,82911,261,662,1187,64466,1594,11825,20172,92001,399,78,13,87483,11,5141,24743,1603,409,48976,379,619,6654,79,2015,4438,22506,264,465,65251,300,90892,264,913,64466,24245,64,1709,20673,29570,391,277,1187,39036,409,2478,9282,18244,13,2308,1508,4706,68,11,58366,5141,24743,1603,92001,399,300,40504,1420,10918,2098,20410,379,264,465,65251,300,11,902,30101,4438,14197,24245,300,7953,5141,24743,1603,409,48976,297,619,6654,79,2015,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,35797,409,1187,24949,662,655,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950,6743,650,69157,16587,662,54896,2478,11763,3630,16282,6654,276,390,1187,64466,1594,11825,20172,92001,399,78,13,4929,64466,409,1187,308,3760,1395,92433,655,65484,379,1320,55611,264,49352,1594,10919,78,11,6368,4883,4154,655,11825,20172,92001,399,78,28526,74783,264,5141,24743,1603,409,1187,22325,13886,379,38405,56295,966,6494,267,288,13,2925,592,27899,11,1187,64466,409,1187,308,3760,1531,5093,19538,15811,1594,10919,78,379,46950]],{}]} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/main.cpp b/projects/llm_framework/main_yolo/src/main.cpp index 50cbaa6..02f49f5 100644 --- a/projects/llm_framework/main_yolo/src/main.cpp +++ b/projects/llm_framework/main_yolo/src/main.cpp @@ -9,7 +9,7 @@ #include #include #include - +#include #include "../../../../SDK/components/utilities/include/sample_log.h" using namespace StackFlows; @@ -59,6 +59,11 @@ public: task_callback_t out_callback_; std::atomic_bool camera_flage_; std::mutex inference_mtx_; + sem_t inference_semaphore; + std::unique_ptr inference_run_; + std::atomic_bool is_running_; + cv::Mat _inference_src; + bool _inference_bgr2rgb; bool parse_config(const nlohmann::json &config_body) { @@ -145,54 +150,63 @@ public: bool inference_decode(const std::string &msg) { - if (inference_mtx_.try_lock()) - std::lock_guard guard(inference_mtx_, std::adopt_lock); - else - return true; cv::Mat src = cv::imdecode(std::vector(msg.begin(), msg.end()), cv::IMREAD_COLOR); if (src.empty()) return true; - return inference(src); + return inference_async(src) ? false : true; } bool inference_raw_yuv(const std::string &msg) { - if (inference_mtx_.try_lock()) - std::lock_guard guard(inference_mtx_, std::adopt_lock); - else - return true; if (msg.size() != mode_config_.img_w * mode_config_.img_h * 2) { throw std::string("img size error"); } cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC2, (void *)msg.data()); cv::Mat rgb; cv::cvtColor(camera_data, rgb, cv::COLOR_YUV2RGB_YUYV); - return inference(rgb, true); + return inference_async(rgb, true) ? false : true; } bool inference_raw_rgb(const std::string &msg) { - if (inference_mtx_.try_lock()) - std::lock_guard guard(inference_mtx_, std::adopt_lock); - else - return true; if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { throw std::string("img size error"); } cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); - return inference(camera_data, false); + return inference_async(camera_data, false) ? false : true; } bool inference_raw_bgr(const std::string &msg) { - if (inference_mtx_.try_lock()) - std::lock_guard guard(inference_mtx_, std::adopt_lock); - else - return true; if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) { throw std::string("img size error"); } cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data()); - return inference(camera_data); + return inference_async(camera_data) ? false : true; + } + + void run() + { + sem_wait(&inference_semaphore); + while (is_running_) { + { + sem_wait(&inference_semaphore); + inference(_inference_src, _inference_bgr2rgb); + sem_wait(&inference_semaphore); + } + } + } + + int inference_async(cv::Mat &src, bool bgr2rgb = true) + { + int count = 0; + sem_getvalue(&inference_semaphore, &count); + if (count == 0) { + _inference_src = src; + _inference_bgr2rgb = bgr2rgb; + sem_post(&inference_semaphore); + sem_post(&inference_semaphore); + } + return count; } bool inference(cv::Mat &src, bool bgr2rgb = true) @@ -278,12 +292,19 @@ public: llm_task(const std::string &workid) { + sem_init(&inference_semaphore, 0, 0); _ax_init(); + is_running_ = true; + inference_run_ = std::make_unique(std::bind(&llm_task::run, this)); } ~llm_task() { + is_running_ = false; + sem_post(&inference_semaphore); + if (inference_run_) inference_run_->join(); _ax_deinit(); + sem_destroy(&inference_semaphore); } }; int llm_task::ax_init_flage_ = 0;