m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] llm Tokenizer

Browse Source
This commit is contained in:
dianjixz
2025-03-21 10:42:02 +08:00
parent 09fa34804e
commit 7ced4d8a72
8 changed files with 316 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ext_components/StackFlow/stackflow/StackFlowUtil.cpp
+108 -13
View File
@@ -8,6 +8,11 @@
#include <glob.h>
#include <fstream>
#include "pzmq.hpp"
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
#include <arm_neon.h>
#endif
// #include <iconv.h>
std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key)
{
@@ -126,22 +131,112 @@ std::string StackFlows::sample_escapeString(const std::string &input)
return escaped;
}
std::string StackFlows::sample_unescapeString(const std::string &input)
void StackFlows::unicode_to_utf8(unsigned int codepoint, char *output, int *length) {
if (codepoint <= 0x7F) {
output[0] = codepoint & 0x7F;
*length = 1;
} else if (codepoint <= 0x7FF) {
output[0] = 0xC0 | ((codepoint >> 6) & 0x1F);
output[1] = 0x80 | (codepoint & 0x3F);
*length = 2;
} else if (codepoint <= 0xFFFF) {
output[0] = 0xE0 | ((codepoint >> 12) & 0x0F);
output[1] = 0x80 | ((codepoint >> 6) & 0x3F);
output[2] = 0x80 | (codepoint & 0x3F);
*length = 3;
} else if (codepoint <= 0x10FFFF) {
output[0] = 0xF0 | ((codepoint >> 18) & 0x07);
output[1] = 0x80 | ((codepoint >> 12) & 0x3F);
output[2] = 0x80 | ((codepoint >> 6) & 0x3F);
output[3] = 0x80 | (codepoint & 0x3F);
*length = 4;
} else {
*length = 0;
}
}
std::string StackFlows::sample_unescapeString(const std::string &input, bool ucs2)
{
std::string unescaped;
for (size_t i = 0; i < input.length(); ++i) {
if (input[i] == '\\' && i + 1 < input.length()) {
switch (input[i + 1]) {
case 'n' :unescaped += '\n';++i;break;
case 't' :unescaped += '\t';++i;break;
case '\\':unescaped += '\\';++i;break;
case '\"':unescaped += '\"';++i;break;
case 'r' :unescaped += '\r';++i;break;
case 'b' :unescaped += '\b';++i;break;
default :unescaped += input[i];break;
unescaped.reserve(input.length());
const char *itc = input.c_str();
const char *itd = itc + input.length();
while (itc != itd){
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
if ((itd - itc) >= 16) {
const uint8_t *srcstr = (const uint8_t *)itc;
uint8x16_t target_open = vdupq_n_u8('\\');
uint8x16_t input_vector = vld1q_u8(srcstr);
uint8x16_t result_open = vceqq_u8(input_vector, target_open);
__uint128_t jflage;
vst1q_u8((uint8_t *)&jflage, result_open);
if (jflage == 0) {
int pos = unescaped.size();
unescaped.resize(pos + 16);
memcpy((void*)(unescaped.c_str() + pos), srcstr, 16);
itc += 16;
continue;
}
} else {
unescaped += input[i];
}
#endif
if((*itc == '\\') && (++itc != itd)){
switch (*itc)
{
case 'n' :unescaped += '\n';itc++;break;
case 't' :unescaped += '\t';itc++;break;
case '\\':unescaped += '\\';itc++;break;
case '\"':unescaped += '\"';itc++;break;
case 'r' :unescaped += '\r';itc++;break;
case 'b' :unescaped += '\b';itc++;break;
case 'u' :{
itc++;
if(ucs2) {
unescaped += "\\u";
break;
}
unsigned int codepoint = 0;
auto itcb = itc;
do {
if((itd - itc) < 4) {
codepoint = -1;
break;
}
for (int i = 0; i < 4; i++) {
char c = *itc++;
codepoint <<= 4;
switch (c) {
case '0' ... '9' : codepoint |= (c - '0'); break;
case 'A' ... 'F' : codepoint |= (c - 'A' + 10); break;
case 'a' ... 'f' : codepoint |= (c - 'a' + 10); break;
default: codepoint = -1; i += 4 ; break;
}
}
} while (codepoint >= 0xD800 && codepoint <= 0xDBFF);
if(codepoint == -1) {
itc = itcb;
unescaped += "\\u";
break;
}
char buff[4];
int len = 0;
StackFlows::unicode_to_utf8(codepoint, buff, &len);
if(len != 0) {
for (int i = 0; i < len; i++) {
unescaped += buff[i];
}
} else {
itc = itcb;
unescaped += "\\u";
}
}
break;
default:
unescaped += *itc++;
break;
}
}
else {
unescaped += *itc++;
}
}
return unescaped;
ext_components/StackFlow/stackflow/StackFlowUtil.h
+2 -1
View File
@@ -28,7 +28,7 @@ int sample_get_work_id_num(const std::string &work_id);
std::string sample_get_work_id_name(const std::string &work_id);
std::string sample_get_work_id(int work_id_num, const std::string &unit_name);
std::string sample_escapeString(const std::string &input);
std::string sample_unescapeString(const std::string &input);
std::string sample_unescapeString(const std::string &input, bool ucs2 = false);
bool decode_stream(const std::string &in, std::string &out, std::unordered_map<int, std::string> &stream_buff);
int decode_base64(const std::string &in, std::string &out);
int encode_base64(const std::string &in, std::string &out);
@@ -36,4 +36,5 @@ std::string unit_call(const std::string &unit_name, const std::string &unit_acti
std::list<std::string> get_config_file_paths(std::string &base_model_path, std::string &base_model_config_path, const std::string &mode_name);
std::vector<std::string> glob_files(const std::vector<std::string> &patterns);
bool file_exists(const std::string& filePath);
void unicode_to_utf8(unsigned int codepoint, char *output, int *length);
}; // namespace StackFlows
projects/llm_framework/main_llm/SConstruct
+8 -7
View File
@@ -17,13 +17,13 @@ LDFLAGS = []
LINK_SEARCH_PATH = []
STATIC_FILES = []
REQUIREMENTS += ['Backward_cpp']
DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'),
AFile('../static_lib/libelf.so.1'),
AFile('../static_lib/libz.so.1'),
AFile('../static_lib/liblzma.so.5'),
AFile('../static_lib/libbz2.so.1.0')]
DEFINITIONS += ["-DENABLE_BACKWARD"]
# REQUIREMENTS += ['Backward_cpp']
# DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'),
# AFile('../static_lib/libelf.so.1'),
# AFile('../static_lib/libz.so.1'),
# AFile('../static_lib/liblzma.so.5'),
# AFile('../static_lib/libbz2.so.1.0')]
# DEFINITIONS += ["-DENABLE_BACKWARD"]
DEFINITIONS += ['-std=c++17']
LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
@@ -43,6 +43,7 @@ STATIC_LIB += static_file * 4
STATIC_FILES += Glob('scripts/tokenizer_*.py')
STATIC_FILES += Glob('models/mode_*.json')
STATIC_FILES += [AFile('scripts/llm-llm_tokenizer_auto.py')]
env['COMPONENTS'].append({'target':'llm_llm',
'SRCS':SRCS,
projects/llm_framework/main_llm/scripts/tokenizer_auto.py → projects/llm_framework/main_llm/scripts/llm-llm_tokenizer_auto.py
View File
projects/llm_framework/main_llm/src/main.cpp
+39 -3
View File
@@ -13,6 +13,7 @@
#include <base64.h>
#include <fstream>
#include <stdexcept>
#include <semaphore.h>
#include "../../../../SDK/components/utilities/include/sample_log.h"
using namespace StackFlows;
#ifdef ENABLE_BACKWARD
@@ -52,6 +53,10 @@ public:
bool enstream_;
std::atomic_bool tokenizer_server_flage_;
unsigned int port_ = 8080;
sem_t inference_semaphore;
std::unique_ptr<std::thread> inference_run_;
std::atomic_bool is_running_;
std::string _inference_msg;
void set_output(task_callback_t out_callback)
{
@@ -209,6 +214,30 @@ public:
return oss_prompt.str();
}
void run()
{
sem_wait(&inference_semaphore);
while (is_running_) {
{
sem_wait(&inference_semaphore);
inference(_inference_msg);
sem_wait(&inference_semaphore);
}
}
}
int inference_async(const std::string &msg)
{
int count = 0;
sem_getvalue(&inference_semaphore, &count);
if (count == 0) {
_inference_msg = msg;
sem_post(&inference_semaphore);
sem_post(&inference_semaphore);
}
return count;
}
void inference(const std::string &msg)
{
#if 1
@@ -265,10 +294,16 @@ public:
llm_task(const std::string &workid)
{
sem_init(&inference_semaphore, 0, 0);
is_running_ = true;
inference_run_ = std::make_unique<std::thread>(std::bind(&llm_task::run, this));
}
~llm_task()
{
is_running_ = false;
sem_post(&inference_semaphore);
if (inference_run_) inference_run_->join();
if (lLaMa_) {
lLaMa_->Deinit();
}
@@ -385,7 +420,7 @@ public:
}
next_data = &tmp_msg2;
}
llm_task_obj->inference((*next_data));
llm_task_obj->inference_async(sample_unescapeString(*next_data));
}
void task_asr_data(const std::weak_ptr<llm_task> llm_task_obj_weak,
@@ -399,10 +434,10 @@ public:
}
if (object.find("stream") != std::string::npos) {
if (sample_json_str_get(data, "finish") == "true") {
llm_task_obj->inference(sample_json_str_get(data, "delta"));
llm_task_obj->inference_async(sample_json_str_get(data, "delta"));
}
} else {
llm_task_obj->inference(data);
llm_task_obj->inference_async(data);
}
}
@@ -586,6 +621,7 @@ public:
}
auto llm_channel = get_channel(work_id_num);
llm_channel->stop_subscriber("");
llm_task_[work_id_num]->lLaMa_->Stop();
llm_task_.erase(work_id_num);
send("None", "None", LLM_NO_ERROR, work_id);
return 0;
projects/llm_framework/main_llm/src/runner/LLM.hpp
+3
View File
@@ -318,6 +318,9 @@ public:
{
std::vector<unsigned short> test_embed;
Encode(test_embed, input_str);
if(test_embed.size() == 0) {
return "";
}
return Run(test_embed);
}
projects/llm_framework/main_llm/src/runner/Tokenizer/Tokenizer.cpp
+114 -153
View File
File diff suppressed because one or more lines are too long
projects/llm_framework/main_yolo/src/main.cpp
+42 -21
View File
@@ -9,7 +9,7 @@
#include <ax_sys_api.h>
#include <sys/stat.h>
#include <fstream>
#include <semaphore.h>
#include "../../../../SDK/components/utilities/include/sample_log.h"
using namespace StackFlows;
@@ -59,6 +59,11 @@ public:
task_callback_t out_callback_;
std::atomic_bool camera_flage_;
std::mutex inference_mtx_;
sem_t inference_semaphore;
std::unique_ptr<std::thread> inference_run_;
std::atomic_bool is_running_;
cv::Mat _inference_src;
bool _inference_bgr2rgb;
bool parse_config(const nlohmann::json &config_body)
{
@@ -145,54 +150,63 @@ public:
bool inference_decode(const std::string &msg)
{
if (inference_mtx_.try_lock())
std::lock_guard<std::mutex> guard(inference_mtx_, std::adopt_lock);
else
return true;
cv::Mat src = cv::imdecode(std::vector<uint8_t>(msg.begin(), msg.end()), cv::IMREAD_COLOR);
if (src.empty()) return true;
return inference(src);
return inference_async(src) ? false : true;
}
bool inference_raw_yuv(const std::string &msg)
{
if (inference_mtx_.try_lock())
std::lock_guard<std::mutex> guard(inference_mtx_, std::adopt_lock);
else
return true;
if (msg.size() != mode_config_.img_w * mode_config_.img_h * 2) {
throw std::string("img size error");
}
cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC2, (void *)msg.data());
cv::Mat rgb;
cv::cvtColor(camera_data, rgb, cv::COLOR_YUV2RGB_YUYV);
return inference(rgb, true);
return inference_async(rgb, true) ? false : true;
}
bool inference_raw_rgb(const std::string &msg)
{
if (inference_mtx_.try_lock())
std::lock_guard<std::mutex> guard(inference_mtx_, std::adopt_lock);
else
return true;
if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) {
throw std::string("img size error");
}
cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data());
return inference(camera_data, false);
return inference_async(camera_data, false) ? false : true;
}
bool inference_raw_bgr(const std::string &msg)
{
if (inference_mtx_.try_lock())
std::lock_guard<std::mutex> guard(inference_mtx_, std::adopt_lock);
else
return true;
if (msg.size() != mode_config_.img_w * mode_config_.img_h * 3) {
throw std::string("img size error");
}
cv::Mat camera_data(mode_config_.img_h, mode_config_.img_w, CV_8UC3, (void *)msg.data());
return inference(camera_data);
return inference_async(camera_data) ? false : true;
}
void run()
{
sem_wait(&inference_semaphore);
while (is_running_) {
{
sem_wait(&inference_semaphore);
inference(_inference_src, _inference_bgr2rgb);
sem_wait(&inference_semaphore);
}
}
}
int inference_async(cv::Mat &src, bool bgr2rgb = true)
{
int count = 0;
sem_getvalue(&inference_semaphore, &count);
if (count == 0) {
_inference_src = src;
_inference_bgr2rgb = bgr2rgb;
sem_post(&inference_semaphore);
sem_post(&inference_semaphore);
}
return count;
}
bool inference(cv::Mat &src, bool bgr2rgb = true)
@@ -278,12 +292,19 @@ public:
llm_task(const std::string &workid)
{
sem_init(&inference_semaphore, 0, 0);
_ax_init();
is_running_ = true;
inference_run_ = std::make_unique<std::thread>(std::bind(&llm_task::run, this));
}
~llm_task()
{
is_running_ = false;
sem_post(&inference_semaphore);
if (inference_run_) inference_run_->join();
_ax_deinit();
sem_destroy(&inference_semaphore);
}
};
int llm_task::ax_init_flage_ = 0;