diff --git a/doc/projects_llm_framework_doc/llm_asr_en.md b/doc/projects_llm_framework_doc/llm_asr_en.md index adf681a..8e96f25 100644 --- a/doc/projects_llm_framework_doc/llm_asr_en.md +++ b/doc/projects_llm_framework_doc/llm_asr_en.md @@ -16,7 +16,7 @@ Send JSON: "action": "setup", "object": "asr.setup", "data": { - "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "model": "sherpa-ncnn-streaming-zipformer-20M-2023-02-17", "response_format": "asr.utf-8.stream", "input": "sys.pcm", "enoutput": true, @@ -34,7 +34,7 @@ Send JSON: - work_id: For configuration units, it is `asr`. - action: The method to be called is `setup`. - object: The type of data being transmitted is `asr.setup`. -- model: The model used is the Chinese model `sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23`. +- model: The model used is the Chinese model `sherpa-ncnn-streaming-zipformer-20M-2023-02-17`. - response_format: The result format is `asr.utf-8.stream`, a UTF-8 stream output. - input: The input is `sys.pcm`, representing system audio. - enoutput: Whether to enable user result output. @@ -109,7 +109,7 @@ Example: "action": "setup", "object": "asr.setup", "data": { - "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "model": "sherpa-ncnn-streaming-zipformer-20M-2023-02-17", "response_format": "asr.utf-8.stream", "input": [ "sys.pcm", @@ -310,7 +310,7 @@ Response JSON: "inputs_": [ "sys.pcm" ], - "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "model": "sherpa-ncnn-streaming-zipformer-20M-2023-02-17", "response_format": "asr.utf-8-stream" }, "error": { diff --git a/doc/projects_llm_framework_doc/llm_kws_en.md b/doc/projects_llm_framework_doc/llm_kws_en.md index 7504e26..89a9cad 100644 --- a/doc/projects_llm_framework_doc/llm_kws_en.md +++ b/doc/projects_llm_framework_doc/llm_kws_en.md @@ -16,11 +16,11 @@ Send JSON: "action": "setup", "object": "kws.setup", "data": { - "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "model": "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01", "response_format": "kws.bool", "input": "sys.pcm", "enoutput": true, - "kws": "你好你好" + "kws": "HELLO" } } ``` @@ -29,7 +29,7 @@ Send JSON: - work_id: When configuring the unit, it is `kws`. - action: The method called is `setup`. - object: The type of data being transmitted is `kws.setup`. -- model: The model used is the Chinese model `sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01`. +- model: The model used is the Chinese model `sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01`. - response_format: The result returned is in `kws.bool` format. - input: The input is `sys.pcm`, representing system audio. - enoutput: Whether to enable user result output. @@ -204,7 +204,7 @@ Response JSON: "inputs_": [ "sys.pcm" ], - "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "model": "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01", "response_format": "kws.bool" }, "error": { diff --git a/projects/llm_framework/main/SConstruct b/projects/llm_framework/main/SConstruct index 3284163..79a6012 100644 --- a/projects/llm_framework/main/SConstruct +++ b/projects/llm_framework/main/SConstruct @@ -24,7 +24,6 @@ STATIC_FILES += [AFile('../static_lib/sherpa/ncnn/libsherpa-ncnn-core.so'), AFile('../static_lib/sherpa/ncnn/libncnn.so'), AFile('../static_lib/libtts.so'), AFile('../static_lib/sherpa/ncnn/libkaldi-native-fbank-core.so'), - AFile('../static_lib/libonnxruntime.so.1.14.0') ] env['COMPONENTS'].append({'target':'static_file', diff --git a/projects/llm_framework/main_kws/SConstruct b/projects/llm_framework/main_kws/SConstruct index 5a48407..f82c7a5 100644 --- a/projects/llm_framework/main_kws/SConstruct +++ b/projects/llm_framework/main_kws/SConstruct @@ -16,6 +16,8 @@ LDFLAGS = [] LINK_SEARCH_PATH = [] STATIC_FILES = [] +python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-kws-python-venv_v1.6.tar.gz", 'm5stack_llm-kws-python-venv_v1.6.tar.gz') + DEFINITIONS += ['-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] LINK_SEARCH_PATH += [ADir('../static_lib')] @@ -31,9 +33,28 @@ LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a', '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a'] +STATIC_FILES += [os.path.join(python_venv, 'sherpa-onnx')] STATIC_FILES += Glob('llm-kws_text2token.py') STATIC_FILES += Glob('mode_*.json') +IGNORE_FILES = [] +IGNORE_FILES += ['sherpa-onnx'] + +import json +if not os.path.exists('../dist'): + os.makedirs('../dist') +ignore = {'ignore':[]} +try: + with open('../dist/fileignore', 'a+') as f: + f.seek(0) + ignore = json.load(f) +except: + pass +ignore['ignore'] += IGNORE_FILES +ignore['ignore'] = list(set(ignore['ignore'])) +with open('../dist/fileignore', 'w') as f: + json.dump(ignore, f, indent=4) + env['COMPONENTS'].append({'target':'llm_kws', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 4a96fc1..91d51ed 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -177,9 +177,9 @@ public: temp_awake_key.close(); std::ostringstream awake_key_compile_cmd; if (file_exists("/opt/m5stack/scripts/text2token.py")) - awake_key_compile_cmd << "/usr/bin/python3 /opt/m5stack/scripts/text2token.py "; + awake_key_compile_cmd << "PYTHONPATH=/opt/m5stack/lib/sherpa-onnx/site-packages /usr/bin/python3 /opt/m5stack/scripts/text2token.py "; else if (file_exists("/opt/m5stack/scripts/llm-kws_text2token.py")) - awake_key_compile_cmd << "/usr/bin/python3 /opt/m5stack/scripts/llm-kws_text2token.py "; + awake_key_compile_cmd << "PYTHONPATH=/opt/m5stack/lib/sherpa-onnx/site-packages /usr/bin/python3 /opt/m5stack/scripts/llm-kws_text2token.py "; else { SLOGE("text2token.py or llm-kws_text2token.py not found!"); } diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index e744507..ad02ce8 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -17,6 +17,8 @@ LDFLAGS = [] LINK_SEARCH_PATH = [] STATIC_FILES = [] +python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-llm-python-venv_v1.7.tar.gz", 'm5stack_llm-llm-python-venv_v1.7.tar.gz') + # REQUIREMENTS += ['Backward_cpp'] # DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'), # AFile('../static_lib/libelf.so.1'), @@ -41,10 +43,29 @@ static_file = Glob('../static_lib/module-llm/libabsl_*') static_file += [AFile('../static_lib/module-llm/libre2.a'), AFile('../static_lib/module-llm/libsentencepiece.a'), AFile('../static_lib/module-llm/libsentencepiece_train.a')] STATIC_LIB += static_file * 4 +STATIC_FILES += [os.path.join(python_venv, 'llm')] STATIC_FILES += Glob('scripts/tokenizer_*.py') STATIC_FILES += Glob('models/mode_*.json') STATIC_FILES += [AFile('scripts/llm-llm_tokenizer_auto.py')] +IGNORE_FILES = [] +IGNORE_FILES += ['llm'] + +import json +if not os.path.exists('../dist'): + os.makedirs('../dist') +ignore = {'ignore':[]} +try: + with open('../dist/fileignore', 'a+') as f: + f.seek(0) + ignore = json.load(f) +except: + pass +ignore['ignore'] += IGNORE_FILES +ignore['ignore'] = list(set(ignore['ignore'])) +with open('../dist/fileignore', 'w') as f: + json.dump(ignore, f, indent=4) + env['COMPONENTS'].append({'target':'llm_llm', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_llm/src/main.cpp b/projects/llm_framework/main_llm/src/main.cpp index 98a8aba..7cbb9fb 100644 --- a/projects/llm_framework/main_llm/src/main.cpp +++ b/projects/llm_framework/main_llm/src/main.cpp @@ -159,6 +159,7 @@ public: if (!tokenizer_server_flage_.load()) { tokenizer_pid_ = fork(); if (tokenizer_pid_ == 0) { + setenv("PYTHONPATH", "/opt/m5stack/lib/llm/site-packages", 1); execl("/usr/bin/python3", "python3", tokenizer_file.c_str(), "--host", "localhost", "--port", std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(), "--content", ("'" + prompt_ + "'").c_str(), nullptr); diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index e54608b..358ddb2 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -21,7 +21,7 @@ DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] -REQUIREMENTS += ['onnxruntime', 'samplerate'] +REQUIREMENTS += ['samplerate'] INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] diff --git a/projects/llm_framework/main_openai_api/SConstruct b/projects/llm_framework/main_openai_api/SConstruct index ac778df..35cbcee 100644 --- a/projects/llm_framework/main_openai_api/SConstruct +++ b/projects/llm_framework/main_openai_api/SConstruct @@ -19,7 +19,7 @@ STATIC_FILES = [] ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/Abandon-ht/ModuleLLM-OpenAI-Plugin.git', '1077efbe201ea3f29517f5ce4a0cfc3b04c25d1d', True) -python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.5.tar.gz", 'm5stack_llm-llm-openai-api-python-venv_v1.5.tar.gz') +python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.5.tar.gz", 'm5stack_llm-openai-api-python-venv_v1.5.tar.gz') DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17'] diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index 3153957..4d9e16e 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -17,7 +17,9 @@ LDFLAGS = [] LINK_SEARCH_PATH = [] STATIC_FILES = [] +python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-vlm-python-venv_v1.6.tar.gz", 'm5stack_llm-vlm-python-venv_v1.6.tar.gz') +DEFINITIONS += ['-O2'] DEFINITIONS += ['-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] @@ -49,9 +51,28 @@ static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')] STATIC_LIB += static_file * 4 +STATIC_FILES += [os.path.join(python_venv, 'vlm')] STATIC_FILES += Glob('scripts/tokenizer_*.py') STATIC_FILES += Glob('models/mode_*.json') +IGNORE_FILES = [] +IGNORE_FILES += ['vlm'] + +import json +if not os.path.exists('../dist'): + os.makedirs('../dist') +ignore = {'ignore':[]} +try: + with open('../dist/fileignore', 'a+') as f: + f.seek(0) + ignore = json.load(f) +except: + pass +ignore['ignore'] += IGNORE_FILES +ignore['ignore'] = list(set(ignore['ignore'])) +with open('../dist/fileignore', 'w') as f: + json.dump(ignore, f, indent=4) + env['COMPONENTS'].append({'target':'llm_vlm', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_vlm/models/mode_internvl2.5-1B-364-ax630c.json b/projects/llm_framework/main_vlm/models/mode_internvl2.5-1B-364-ax630c.json new file mode 100644 index 0000000..8f6a396 --- /dev/null +++ b/projects/llm_framework/main_vlm/models/mode_internvl2.5-1B-364-ax630c.json @@ -0,0 +1,35 @@ +{ + "mode":"internvl2.5-1B-364-ax630c", + "type":"vlm", + "homepage":"https://huggingface.co/AXERA-TECH/InternVL2_5-1B", + "capabilities":[ + "text_generation", + "chat" + ], + "input_type":[ + "vlm.chat_completion", + "vlm.chat_completion.stream" + ], + "output_type":[ + "vlm.utf-8", + "vlm.utf-8.stream" + ], + "mode_param":{ + "tokenizer_type":2, + "filename_tokenizer_model":"http://localhost:8080", + "filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin", + "filename_post_axmodel":"qwen2_post.axmodel", + "template_filename_axmodel":"qwen2_p256_l%d_together.axmodel", + "filename_vpm_resampler_axmodedl":"vit_intern_sim_space2depth.axmodel", + "b_use_topk":false, + "b_bos":false, + "b_eos":false, + "axmodel_num":24, + "tokens_embed_num":151674, + "img_token_id":151667, + "tokens_embed_size":896, + "b_use_mmap_load_embed":true, + "b_dynamic_load_axmodel_layer":false, + "ext_scripts":["tokenizer_internvl2.5-1B-364-ax630c.py"] + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/models/mode_smolvlm-256M-ax630c.json b/projects/llm_framework/main_vlm/models/mode_smolvlm-256M-ax630c.json new file mode 100644 index 0000000..1d3a293 --- /dev/null +++ b/projects/llm_framework/main_vlm/models/mode_smolvlm-256M-ax630c.json @@ -0,0 +1,35 @@ +{ + "mode":"smolvlm-256M-ax630c", + "type":"vlm", + "homepage":"https://huggingface.co/HuggingFaceTB/SmolVLM-256M-Instruct", + "capabilities":[ + "text_generation", + "chat" + ], + "input_type":[ + "vlm.chat_completion", + "vlm.chat_completion.stream" + ], + "output_type":[ + "vlm.utf-8", + "vlm.utf-8.stream" + ], + "mode_param":{ + "tokenizer_type":2, + "filename_tokenizer_model":"http://localhost:8080", + "filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin", + "filename_post_axmodel":"llama_post.axmodel", + "template_filename_axmodel":"llama_p128_l%d_together.axmodel", + "filename_vpm_resampler_axmodedl":"SmolVLM-256M-Instruct_vision_nhwc.axmodel", + "b_use_topk":false, + "b_bos":false, + "b_eos":false, + "axmodel_num":30, + "tokens_embed_num":49280, + "img_token_id":49190, + "tokens_embed_size":576, + "b_use_mmap_load_embed":true, + "b_dynamic_load_axmodel_layer":false, + "ext_scripts":["tokenizer_smolvlm-256M-ax630c.py"] + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/models/mode_smolvlm-500M-ax630c.json b/projects/llm_framework/main_vlm/models/mode_smolvlm-500M-ax630c.json new file mode 100644 index 0000000..4c07e36 --- /dev/null +++ b/projects/llm_framework/main_vlm/models/mode_smolvlm-500M-ax630c.json @@ -0,0 +1,35 @@ +{ + "mode":"smolvlm-500M-ax630c", + "type":"vlm", + "homepage":"https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Instruct", + "capabilities":[ + "text_generation", + "chat" + ], + "input_type":[ + "vlm.chat_completion", + "vlm.chat_completion.stream" + ], + "output_type":[ + "vlm.utf-8", + "vlm.utf-8.stream" + ], + "mode_param":{ + "tokenizer_type":2, + "filename_tokenizer_model":"http://localhost:8080", + "filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin", + "filename_post_axmodel":"llama_post.axmodel", + "template_filename_axmodel":"llama_p128_l%d_together.axmodel", + "filename_vpm_resampler_axmodedl":"SmolVLM-500M-Instruct_vision.axmodel", + "b_use_topk":false, + "b_bos":false, + "b_eos":false, + "axmodel_num":32, + "tokens_embed_num":49280, + "img_token_id":49190, + "tokens_embed_size":960, + "b_use_mmap_load_embed":true, + "b_dynamic_load_axmodel_layer":false, + "ext_scripts":["tokenizer_smolvlm-500M-ax630c.py"] + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/scripts/tokenizer_internvl2.5-1B-364-ax630c.py b/projects/llm_framework/main_vlm/scripts/tokenizer_internvl2.5-1B-364-ax630c.py new file mode 100644 index 0000000..569c5da --- /dev/null +++ b/projects/llm_framework/main_vlm/scripts/tokenizer_internvl2.5-1B-364-ax630c.py @@ -0,0 +1,138 @@ +from transformers import AutoTokenizer, PreTrainedTokenizerFast +from http.server import HTTPServer, BaseHTTPRequestHandler +import json +import argparse + + +class Tokenizer_Http: + + def __init__(self, model_id): + self.tokenizer = AutoTokenizer.from_pretrained( + model_id, trust_remote_code=True, use_fast=False + ) + + def encode(self, prompt, content): + prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n{prompt}<|im_end|><|im_start|>assistant\n" + input_ids = self.tokenizer.encode(prompt) + return input_ids + + def encode_vpm(self, prompt, content="Please describe the image shortly."): + prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n" + "" * 169 + f"\n{prompt}<|im_end|><|im_start|>assistant\n" + input_ids = self.tokenizer.encode(prompt) + return input_ids + + def decode(self, token_ids): + return self.tokenizer.decode(token_ids, clean_up_tokenization_spaces=False) + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def bos_token(self): + return self.tokenizer.bos_token + + @property + def eos_token(self): + return self.tokenizer.eos_token + +class Request(BaseHTTPRequestHandler): + # 通过类继承，新定义类 + timeout = 5 + server_version = "Apache" + + def do_GET(self): + print(self.path) + # 在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行） + self.send_response(200) + self.send_header("type", "get") # 设置响应头，可省略或设置多个 + self.end_headers() + + if self.path == "/bos_id": + bos_id = tokenizer.bos_id + # print(bos_id) + # to json + if bos_id is None: + msg = json.dumps({"bos_id": -1}) + else: + msg = json.dumps({"bos_id": bos_id}) + elif self.path == "/eos_id": + eos_id = tokenizer.eos_id + if eos_id is None: + msg = json.dumps({"eos_id": -1}) + else: + msg = json.dumps({"eos_id": eos_id}) + else: + msg = "error" + + print(msg) + msg = str(msg).encode() # 转为str再转为byte格式 + + self.wfile.write(msg) # 将byte格式的信息返回给客户端 + + def do_POST(self): + # 在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行） + data = self.rfile.read( + int(self.headers["content-length"]) + ) # 获取从客户端传入的参数（byte格式） + data = data.decode() # 将byte格式转为str格式 + + self.send_response(200) + self.send_header("type", "post") # 设置响应头，可省略或设置多个 + self.end_headers() + + if self.path == "/encode": + req = json.loads(data) + print(req) + prompt = req["text"] + b_img_prompt = False + if "img_prompt" in req: + b_img_prompt = req["img_prompt"] + if b_img_prompt: + token_ids = tokenizer.encode_vpm(prompt) + else: + token_ids = tokenizer.encode(prompt, args.content) + if token_ids is None: + msg = json.dumps({"token_ids": -1}) + else: + msg = json.dumps({"token_ids": token_ids}) + + elif self.path == "/decode": + req = json.loads(data) + token_ids = req["token_ids"] + text = tokenizer.decode(token_ids) + if text is None: + msg = json.dumps({"text": ""}) + else: + msg = json.dumps({"text": text}) + else: + msg = "error" + print(msg) + msg = str(msg).encode() # 转为str再转为byte格式 + + self.wfile.write(msg) # 将byte格式的信息返回给客户端 + + +if __name__ == "__main__": + + args = argparse.ArgumentParser() + args.add_argument("--host", type=str, default="localhost") + args.add_argument("--port", type=int, default=8080) + args.add_argument('--model_id', type=str, default='internvl2_tokenizer') + args.add_argument('--content', type=str, default='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。') + args = args.parse_args() + + tokenizer = Tokenizer_Http(args.model_id) + + + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) + + host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1' + print("http://%s:%s" % host) + server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例 + server.serve_forever() # 开启服务 diff --git a/projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-256M-ax630c.py b/projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-256M-ax630c.py new file mode 100644 index 0000000..560a71f --- /dev/null +++ b/projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-256M-ax630c.py @@ -0,0 +1,248 @@ +from transformers import AutoTokenizer, PreTrainedTokenizerFast +from transformers.tokenization_utils_base import AddedToken +from http.server import HTTPServer, BaseHTTPRequestHandler +import json +import argparse + +def _prompt_split_image( + image_seq_len, + image_rows, + image_cols, + fake_token_around_image, + image_token, + global_img_token, +): + """Prompt with expanded image tokens for when the image is split into patches.""" + text_split_images = "" + for n_h in range(image_rows): + for n_w in range(image_cols): + text_split_images += ( + f"{fake_token_around_image}" + + f"" + + f"{image_token}" * image_seq_len + ) + text_split_images += "\n" + + text_split_images += ( + f"\n{fake_token_around_image}" + + f"{global_img_token}" + + f"{image_token}" * image_seq_len + + f"{fake_token_around_image}" + ) + return text_split_images + + +def _prompt_single_image( + image_seq_len, fake_token_around_image, image_token, global_img_token +): + """Prompt with expanded image tokens for a single image.""" + return ( + f"{fake_token_around_image}" + + f"{global_img_token}" + + f"{image_token}" * image_seq_len + + f"{fake_token_around_image}" + ) + + +def get_image_prompt_string( + image_rows, + image_cols, + image_seq_len, + fake_token_around_image, + image_token, + global_img_token, +): + if image_rows == 0 and image_cols == 0: + return _prompt_single_image( + image_seq_len, + fake_token_around_image=fake_token_around_image, + image_token=image_token, + global_img_token=global_img_token, + ) + return _prompt_split_image( + image_seq_len, + image_rows, + image_cols, + fake_token_around_image, + image_token, + global_img_token, + ) + +class Tokenizer_Http: + + def __init__(self, model_id): + self.tokenizer = AutoTokenizer.from_pretrained( + model_id, trust_remote_code=True, use_fast=False + ) + + def encode(self, prompt, content): + prompt = f"<|im_start|>User:{content}\nAssistant:" + input_ids = self.tokenizer(prompt) + return input_ids["input_ids"] + + def encode_vpm(self, prompt, content="Please describe the image shortly."): + prompt = f"<|im_start|>User:{prompt}\nAssistant:" + text = [prompt] + image_rows = [[0]] + image_cols = [[0]] + image_seq_len = 64 + image_token = "" + fake_image_token = "" + global_img_token = "" + prompt_strings = [] + for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols): + # Replace the image token with fake tokens around the expanded image token sequence of length `image_seq_len` + image_prompt_strings = [] + for n_rows, n_cols in zip(sample_rows, sample_cols): + image_prompt_string = get_image_prompt_string( + n_rows, + n_cols, + image_seq_len, + image_token=image_token, + fake_token_around_image=fake_image_token, + global_img_token=global_img_token, + ) + image_prompt_strings.append(image_prompt_string) + + split_sample = sample.split(image_token) + if len(split_sample) == 0: + raise ValueError("The image token should be present in the text.") + + # Place in the image prompt strings where the image tokens are + sample = split_sample[0] + for i, image_prompt_string in enumerate(image_prompt_strings): + sample += image_prompt_string + split_sample[i + 1] + prompt_strings.append(sample) + + fake_image_token = AddedToken(fake_image_token, normalized=False, special=True) + image_token = AddedToken(image_token, normalized=False, special=True) + end_of_utterance_token = AddedToken( + "", normalized=False, special=True + ) + tokens_to_add = { + "additional_special_tokens": [ + fake_image_token, + image_token, + end_of_utterance_token, + ] + } + self.tokenizer.add_special_tokens(tokens_to_add) + + input_ids = self.tokenizer(prompt_strings)["input_ids"][0] + return input_ids + + def decode(self, token_ids): + return self.tokenizer.decode(token_ids, clean_up_tokenization_spaces=False) + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def bos_token(self): + return self.tokenizer.bos_token + + @property + def eos_token(self): + return self.tokenizer.eos_token + +class Request(BaseHTTPRequestHandler): + # 通过类继承，新定义类 + timeout = 5 + server_version = "Apache" + + def do_GET(self): + print(self.path) + # 在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行） + self.send_response(200) + self.send_header("type", "get") # 设置响应头，可省略或设置多个 + self.end_headers() + + if self.path == "/bos_id": + bos_id = tokenizer.bos_id + # print(bos_id) + # to json + if bos_id is None: + msg = json.dumps({"bos_id": -1}) + else: + msg = json.dumps({"bos_id": bos_id}) + elif self.path == "/eos_id": + eos_id = tokenizer.eos_id + if eos_id is None: + msg = json.dumps({"eos_id": -1}) + else: + msg = json.dumps({"eos_id": eos_id}) + else: + msg = "error" + + print(msg) + msg = str(msg).encode() # 转为str再转为byte格式 + + self.wfile.write(msg) # 将byte格式的信息返回给客户端 + + def do_POST(self): + # 在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行） + data = self.rfile.read( + int(self.headers["content-length"]) + ) # 获取从客户端传入的参数（byte格式） + data = data.decode() # 将byte格式转为str格式 + + self.send_response(200) + self.send_header("type", "post") # 设置响应头，可省略或设置多个 + self.end_headers() + + if self.path == "/encode": + req = json.loads(data) + print(req) + prompt = req["text"] + b_img_prompt = False + if "img_prompt" in req: + b_img_prompt = req["img_prompt"] + if b_img_prompt: + token_ids = tokenizer.encode_vpm(prompt) + else: + token_ids = tokenizer.encode(prompt, args.content) + if token_ids is None: + msg = json.dumps({"token_ids": -1}) + else: + msg = json.dumps({"token_ids": token_ids}) + + elif self.path == "/decode": + req = json.loads(data) + token_ids = req["token_ids"] + text = tokenizer.decode(token_ids) + if text is None: + msg = json.dumps({"text": ""}) + else: + msg = json.dumps({"text": text}) + else: + msg = "error" + print(msg) + msg = str(msg).encode() # 转为str再转为byte格式 + + self.wfile.write(msg) # 将byte格式的信息返回给客户端 + + +if __name__ == "__main__": + + args = argparse.ArgumentParser() + args.add_argument("--host", type=str, default="localhost") + args.add_argument("--port", type=int, default=8080) + args.add_argument('--model_id', type=str, default='internvl2_tokenizer') + args.add_argument('--content', type=str, default='') + args = args.parse_args() + + tokenizer = Tokenizer_Http(args.model_id) + + + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) + + host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1' + print("http://%s:%s" % host) + server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例 + server.serve_forever() # 开启服务 diff --git a/projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-500M-ax630c.py b/projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-500M-ax630c.py new file mode 100644 index 0000000..560a71f --- /dev/null +++ b/projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-500M-ax630c.py @@ -0,0 +1,248 @@ +from transformers import AutoTokenizer, PreTrainedTokenizerFast +from transformers.tokenization_utils_base import AddedToken +from http.server import HTTPServer, BaseHTTPRequestHandler +import json +import argparse + +def _prompt_split_image( + image_seq_len, + image_rows, + image_cols, + fake_token_around_image, + image_token, + global_img_token, +): + """Prompt with expanded image tokens for when the image is split into patches.""" + text_split_images = "" + for n_h in range(image_rows): + for n_w in range(image_cols): + text_split_images += ( + f"{fake_token_around_image}" + + f"" + + f"{image_token}" * image_seq_len + ) + text_split_images += "\n" + + text_split_images += ( + f"\n{fake_token_around_image}" + + f"{global_img_token}" + + f"{image_token}" * image_seq_len + + f"{fake_token_around_image}" + ) + return text_split_images + + +def _prompt_single_image( + image_seq_len, fake_token_around_image, image_token, global_img_token +): + """Prompt with expanded image tokens for a single image.""" + return ( + f"{fake_token_around_image}" + + f"{global_img_token}" + + f"{image_token}" * image_seq_len + + f"{fake_token_around_image}" + ) + + +def get_image_prompt_string( + image_rows, + image_cols, + image_seq_len, + fake_token_around_image, + image_token, + global_img_token, +): + if image_rows == 0 and image_cols == 0: + return _prompt_single_image( + image_seq_len, + fake_token_around_image=fake_token_around_image, + image_token=image_token, + global_img_token=global_img_token, + ) + return _prompt_split_image( + image_seq_len, + image_rows, + image_cols, + fake_token_around_image, + image_token, + global_img_token, + ) + +class Tokenizer_Http: + + def __init__(self, model_id): + self.tokenizer = AutoTokenizer.from_pretrained( + model_id, trust_remote_code=True, use_fast=False + ) + + def encode(self, prompt, content): + prompt = f"<|im_start|>User:{content}\nAssistant:" + input_ids = self.tokenizer(prompt) + return input_ids["input_ids"] + + def encode_vpm(self, prompt, content="Please describe the image shortly."): + prompt = f"<|im_start|>User:{prompt}\nAssistant:" + text = [prompt] + image_rows = [[0]] + image_cols = [[0]] + image_seq_len = 64 + image_token = "" + fake_image_token = "" + global_img_token = "" + prompt_strings = [] + for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols): + # Replace the image token with fake tokens around the expanded image token sequence of length `image_seq_len` + image_prompt_strings = [] + for n_rows, n_cols in zip(sample_rows, sample_cols): + image_prompt_string = get_image_prompt_string( + n_rows, + n_cols, + image_seq_len, + image_token=image_token, + fake_token_around_image=fake_image_token, + global_img_token=global_img_token, + ) + image_prompt_strings.append(image_prompt_string) + + split_sample = sample.split(image_token) + if len(split_sample) == 0: + raise ValueError("The image token should be present in the text.") + + # Place in the image prompt strings where the image tokens are + sample = split_sample[0] + for i, image_prompt_string in enumerate(image_prompt_strings): + sample += image_prompt_string + split_sample[i + 1] + prompt_strings.append(sample) + + fake_image_token = AddedToken(fake_image_token, normalized=False, special=True) + image_token = AddedToken(image_token, normalized=False, special=True) + end_of_utterance_token = AddedToken( + "", normalized=False, special=True + ) + tokens_to_add = { + "additional_special_tokens": [ + fake_image_token, + image_token, + end_of_utterance_token, + ] + } + self.tokenizer.add_special_tokens(tokens_to_add) + + input_ids = self.tokenizer(prompt_strings)["input_ids"][0] + return input_ids + + def decode(self, token_ids): + return self.tokenizer.decode(token_ids, clean_up_tokenization_spaces=False) + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def bos_token(self): + return self.tokenizer.bos_token + + @property + def eos_token(self): + return self.tokenizer.eos_token + +class Request(BaseHTTPRequestHandler): + # 通过类继承，新定义类 + timeout = 5 + server_version = "Apache" + + def do_GET(self): + print(self.path) + # 在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行） + self.send_response(200) + self.send_header("type", "get") # 设置响应头，可省略或设置多个 + self.end_headers() + + if self.path == "/bos_id": + bos_id = tokenizer.bos_id + # print(bos_id) + # to json + if bos_id is None: + msg = json.dumps({"bos_id": -1}) + else: + msg = json.dumps({"bos_id": bos_id}) + elif self.path == "/eos_id": + eos_id = tokenizer.eos_id + if eos_id is None: + msg = json.dumps({"eos_id": -1}) + else: + msg = json.dumps({"eos_id": eos_id}) + else: + msg = "error" + + print(msg) + msg = str(msg).encode() # 转为str再转为byte格式 + + self.wfile.write(msg) # 将byte格式的信息返回给客户端 + + def do_POST(self): + # 在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行） + data = self.rfile.read( + int(self.headers["content-length"]) + ) # 获取从客户端传入的参数（byte格式） + data = data.decode() # 将byte格式转为str格式 + + self.send_response(200) + self.send_header("type", "post") # 设置响应头，可省略或设置多个 + self.end_headers() + + if self.path == "/encode": + req = json.loads(data) + print(req) + prompt = req["text"] + b_img_prompt = False + if "img_prompt" in req: + b_img_prompt = req["img_prompt"] + if b_img_prompt: + token_ids = tokenizer.encode_vpm(prompt) + else: + token_ids = tokenizer.encode(prompt, args.content) + if token_ids is None: + msg = json.dumps({"token_ids": -1}) + else: + msg = json.dumps({"token_ids": token_ids}) + + elif self.path == "/decode": + req = json.loads(data) + token_ids = req["token_ids"] + text = tokenizer.decode(token_ids) + if text is None: + msg = json.dumps({"text": ""}) + else: + msg = json.dumps({"text": text}) + else: + msg = "error" + print(msg) + msg = str(msg).encode() # 转为str再转为byte格式 + + self.wfile.write(msg) # 将byte格式的信息返回给客户端 + + +if __name__ == "__main__": + + args = argparse.ArgumentParser() + args.add_argument("--host", type=str, default="localhost") + args.add_argument("--port", type=int, default=8080) + args.add_argument('--model_id', type=str, default='internvl2_tokenizer') + args.add_argument('--content', type=str, default='') + args = args.parse_args() + + tokenizer = Tokenizer_Http(args.model_id) + + + # print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token) + # print(tokenizer.encode("hello world", args.content)) + + host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1' + print("http://%s:%s" % host) + server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例 + server.serve_forever() # 开启服务 diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index 3ea904c..b625b5f 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -50,8 +50,8 @@ public: std::string response_format_; std::vector inputs_; std::vector prompt_data_; - std::vector> image_datas_; - std::vector> img_embeds; + std::vector image_data_; + std::vector img_embed; std::string prompt_; task_callback_t out_callback_; bool enoutput_; @@ -126,6 +126,7 @@ public: CONFIG_AUTO_SET(file_body["mode_param"], b_eos); CONFIG_AUTO_SET(file_body["mode_param"], axmodel_num); CONFIG_AUTO_SET(file_body["mode_param"], tokens_embed_num); + CONFIG_AUTO_SET(file_body["mode_param"], img_token_id); CONFIG_AUTO_SET(file_body["mode_param"], tokens_embed_size); CONFIG_AUTO_SET(file_body["mode_param"], b_use_mmap_load_embed); CONFIG_AUTO_SET(file_body["mode_param"], b_dynamic_load_axmodel_layer); @@ -154,6 +155,7 @@ public: if (!tokenizer_server_flage_.load()) { tokenizer_pid_ = fork(); if (tokenizer_pid_ == 0) { + setenv("PYTHONPATH", "/opt/m5stack/lib/vlm/site-packages", 1); execl("/usr/bin/python3", "python3", tokenizer_file.c_str(), "--host", "localhost", "--port", std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(), "--content", ("'" + prompt_ + "'").c_str(), nullptr); @@ -216,32 +218,25 @@ public: oss_prompt << input; break; } - SLOGI("prompt_complete:%s", oss_prompt.str().c_str()); + // SLOGI("prompt_complete:%s", oss_prompt.str().c_str()); return oss_prompt.str(); } void inference(const std::string &msg) { try { - if (image_datas_.empty()) { + if (image_data_.empty()) { lLaMa_->Encode(prompt_data_, prompt_complete(msg)); std::string out = lLaMa_->Run(prompt_data_); if (out_callback_) out_callback_(out, true); } else { - img_embeds.clear(); - for (auto &img_data : image_datas_) { - cv::Mat src = cv::imdecode(img_data, cv::IMREAD_COLOR); - if (src.empty()) continue; - std::vector embed; - lLaMa_->Encode(src, embed); - img_embeds.push_back(embed); - } - image_datas_.clear(); - if (!img_embeds.empty()) { - lLaMa_->Encode(img_embeds, prompt_data_, prompt_complete(msg)); - std::string out = lLaMa_->Run(prompt_data_); - if (out_callback_) out_callback_(out, true); - } + cv::Mat src = cv::imdecode(image_data_, cv::IMREAD_COLOR); + if (src.empty()) return; + image_data_.clear(); + lLaMa_->Encode(src, img_embed); + lLaMa_->Encode(img_embed, prompt_data_, prompt_complete(msg)); + std::string out = lLaMa_->Run(prompt_data_); + if (out_callback_) out_callback_(out, true); } } catch (...) { SLOGW("lLaMa_->Run have error!"); @@ -416,7 +411,7 @@ public: next_data = &tmp_msg2; } if (object.find("jpeg") != std::string::npos) { - llm_task_obj->image_datas_.emplace_back(next_data->begin(), next_data->end()); + llm_task_obj->image_data_.assign(next_data->begin(), next_data->end()); return; } llm_task_obj->inference((*next_data)); diff --git a/projects/llm_framework/main_vlm/src/runner/LLM.hpp b/projects/llm_framework/main_vlm/src/runner/LLM.hpp index a1d34ed..2cbbf38 100644 --- a/projects/llm_framework/main_vlm/src/runner/LLM.hpp +++ b/projects/llm_framework/main_vlm/src/runner/LLM.hpp @@ -26,8 +26,6 @@ struct LLMAttrType { std::string filename_post_axmodel = "tinyllama-int8/tinyllama_post.axmodel"; - bool b_use_topk = false; - std::string filename_vpm_encoder_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel"; std::string filename_vpm_resampler_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel"; int vpm_width = 280; @@ -39,6 +37,7 @@ struct LLMAttrType { bool b_bos = true, b_eos = false; std::string filename_tokens_embed = "tinyllama.model.embed_tokens.weight.bfloat16.bin"; int tokens_embed_num = 32000; + int img_token_id = 151667; // InternVL2.5 int tokens_embed_size = 2048; int max_token_len = 127; // auto calc @@ -53,6 +52,9 @@ struct LLMAttrType { bool b_use_mmap_load_layer = true; + bool b_use_topk = false; + std::string post_config_path = "post_config.json"; + // bool b_live_print = true; LLMRuningCallback runing_callback = nullptr; void *reserve = nullptr; @@ -84,36 +86,17 @@ private: bool b_stop = false; - int post_process(unsigned short *p, int n, std::vector &history, float *val = 0) + LLMPostprocess postprocess; + static int post_process(LLMPostprocess &postprocess, unsigned short *p, int n, std::vector &history, + float *val = 0) { std::vector logits(n); for (int i = 0; i < n; i++) { unsigned int proc = p[i] << 16; logits[i] = *reinterpret_cast(&proc); } - LLMPostprocess postprocess; - postprocess.set_temperature(true, _attr.temperature); - postprocess.set_repetition_penalty(true, 1.2f); - // postprocess.set_top_k_sampling(true, 40); - postprocess.set_top_p_sampling(true, _attr.top_p); return postprocess.apply(logits, history); - - // float max_val = -MAXFLOAT; - // int max_index = 0; - // for (int i = 0; i < n; i++) - // { - // unsigned int proc = p[i] << 16; - // float tmp = *reinterpret_cast(&proc); - // if (tmp > max_val) - // { - // max_val = tmp; - // max_index = i; - // } - // } - // if (val) - // *val = max_val; - // return max_index; } public: @@ -308,18 +291,24 @@ public: vpm_encoder.inference(); AX_SYS_MinvalidateCache(vpm_encoder.get_output(0).phyAddr, vpm_encoder.get_output(0).pVirAddr, vpm_encoder.get_output(0).nSize); - memcpy(vpm_resampler.get_input("input").pVirAddr, vpm_encoder.get_output(0).pVirAddr, + memcpy(vpm_resampler.get_input(0).pVirAddr, vpm_encoder.get_output(0).pVirAddr, vpm_encoder.get_output(0).nSize); } else { - void *data = vpm_resampler.get_input("input").pVirAddr; + void *data = vpm_resampler.get_input(0).pVirAddr; memcpy(data, dst.data, dst.rows * dst.cols * 3); } vpm_resampler.inference(); - out_embed.resize(vpm_resampler.get_output("output").nSize / sizeof(unsigned short)); - AX_SYS_MinvalidateCache(vpm_resampler.get_output("output").phyAddr, vpm_resampler.get_output("output").pVirAddr, - vpm_resampler.get_output("output").nSize); - memcpy(out_embed.data(), vpm_resampler.get_output("output").pVirAddr, vpm_resampler.get_output("output").nSize); + out_embed.resize(vpm_resampler.get_output(0).nSize / sizeof(float)); + AX_SYS_MinvalidateCache(vpm_resampler.get_output(0).phyAddr, vpm_resampler.get_output(0).pVirAddr, + vpm_resampler.get_output(0).nSize); + + float *output_data = (float *)vpm_resampler.get_output(0).pVirAddr; + for (size_t i = 0; i < out_embed.size(); i++) { + out_embed[i] = bfloat16(output_data[i]).data; + } + + // memcpy(out_embed.data(), vpm_resampler.get_output(0).pVirAddr, vpm_resampler.get_output(0).nSize); ALOGI("image encode time : %f ms, size : %d", t.cost(), out_embed.size()); return 0; } @@ -337,27 +326,49 @@ public: embed_selector.getByIndex(input_ids[i], out_embed.data() + i * _attr.tokens_embed_size); } - // memcpy(out_embed.data() + 5 * _attr.tokens_embed_size, vpm_resampler.get_output("output").pVirAddr, - // vpm_resampler.get_output("output").nSize); + // memcpy(out_embed.data() + 5 * _attr.tokens_embed_size, vpm_resampler.get_output(0).pVirAddr, + // vpm_resampler.get_output(0).nSize); return 0; } - int Encode(std::vector> &img_embeds, std::vector &out_embed, - std::string prompt = "What is in the images?") + int Encode(std::vector &img_embed, std::vector &out_embed, + std::string prompt = "What is in the image?") { std::vector input_ids = tokenizer->Encode(prompt, true); - constexpr int IMG_CONTEXT = 151667; // InternVL2.5 - std::vector img_positions; + // constexpr int img_token_id = 49190; // smolvlm + // constexpr int img_token_id = 151667; // InternVL2.5 + int offset = 0; + int img_context_count = 0; for (size_t i = 0; i < input_ids.size(); i++) { - if (input_ids[i] == IMG_CONTEXT) { - img_positions.push_back(i); + if (input_ids[i] == _attr.img_token_id) { + img_context_count++; + if (img_context_count == 1) { + offset = i; + } } } - if (img_positions.size() > _attr.prefill_token_num) { + if (offset == 0) { + ALOGE("offset == 0"); + return -1; + } + + if (img_context_count != img_embed.size() / _attr.tokens_embed_size) { + ALOGE("img_context_count(%d) != img_embed.size() / tokens_embed_size(%d)", img_context_count, + img_embed.size() / _attr.tokens_embed_size); + return -1; + } + + // for (size_t i = 0; i < input_ids.size(); i++) + // { + // printf("%d ", input_ids[i]); + // } + // printf("\n"); + + if (input_ids.size() > _attr.prefill_token_num) { ALOGE("input_ids(%d) > prefill_token_num(%d)", input_ids.size(), _attr.prefill_token_num); return -1; } @@ -366,11 +377,8 @@ public: for (size_t i = 0; i < input_ids.size(); i++) { embed_selector.getByIndex(input_ids[i], out_embed.data() + i * _attr.tokens_embed_size); } - for (size_t img_idx = 0; img_idx < img_embeds.size(); img_idx++) { - // int pos = img_positions[img_idx]; - memcpy(out_embed.data() + (14 + img_idx * 64) * _attr.tokens_embed_size, img_embeds[img_idx].data(), - img_embeds[img_idx].size() * sizeof(unsigned short)); - } + memcpy(out_embed.data() + offset * _attr.tokens_embed_size, img_embed.data(), + img_embed.size() * sizeof(unsigned short)); return 0; } @@ -504,7 +512,7 @@ public: AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize); unsigned short *post_out = (unsigned short *)output_post.pVirAddr; float max_val = -MAXFLOAT; - max_index = post_process(post_out, _attr.tokens_embed_num, token_ids, &max_val); + max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val); } next_token = max_index; @@ -599,7 +607,7 @@ public: AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize); unsigned short *post_out = (unsigned short *)output_post.pVirAddr; float max_val = -MAXFLOAT; - max_index = post_process(post_out, _attr.tokens_embed_num, token_ids, &max_val); + max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val); } next_token = max_index; diff --git a/projects/llm_framework/main_whisper/SConstruct b/projects/llm_framework/main_whisper/SConstruct index 4dee5cf..c14cf6b 100644 --- a/projects/llm_framework/main_whisper/SConstruct +++ b/projects/llm_framework/main_whisper/SConstruct @@ -21,7 +21,7 @@ DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] -REQUIREMENTS += ['onnxruntime', 'samplerate'] +# REQUIREMENTS += ['onnxruntime', 'samplerate'] INCLUDE += [ADir('../include')] INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')] diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 09d0967..05dea8d 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -68,23 +68,23 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): # if os.path.exists(zip_file_extrpath): # shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/scripts')) - zip_file = 'm5stack_dist-packages.tar.gz' - down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz' - zip_file_extrpath = 'm5stack_dist-packages' - if not os.path.exists(zip_file_extrpath): - # Downloading via HTTP (more common) - if not os.path.exists(zip_file): - response = requests.get(down_url) - if response.status_code == 200: - with open(zip_file, 'wb') as file: - file.write(response.content) - else: - print("{} down failed".format(down_url)) - with tarfile.open(zip_file, 'r:gz') as tar: - tar.extractall(path=zip_file_extrpath) - print("The {} download successful.".format(down_url)) - if os.path.exists(zip_file_extrpath): - shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages')) + # zip_file = 'm5stack_dist-packages.tar.gz' + # down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz' + # zip_file_extrpath = 'm5stack_dist-packages' + # if not os.path.exists(zip_file_extrpath): + # # Downloading via HTTP (more common) + # if not os.path.exists(zip_file): + # response = requests.get(down_url) + # if response.status_code == 200: + # with open(zip_file, 'wb') as file: + # file.write(response.content) + # else: + # print("{} down failed".format(down_url)) + # with tarfile.open(zip_file, 'r:gz') as tar: + # tar.extractall(path=zip_file_extrpath) + # print("The {} download successful.".format(down_url)) + # if os.path.exists(zip_file_extrpath): + # shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages')) os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True) with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f: @@ -238,6 +238,18 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): openai_api_dir = os.path.join(src_folder, 'openai-api') if os.path.exists(openai_api_dir): shutil.copytree(openai_api_dir, os.path.join(deb_folder, 'opt/m5stack/lib/openai-api')) + if package_name == 'llm-kws': + sherpa_dir = os.path.join(src_folder, 'sherpa-onnx') + if os.path.exists(sherpa_dir): + shutil.copytree(sherpa_dir, os.path.join(deb_folder, 'opt/m5stack/lib/sherpa-onnx')) + if package_name == 'llm-llm': + llm_dir = os.path.join(src_folder, 'llm') + if os.path.exists(llm_dir): + shutil.copytree(llm_dir, os.path.join(deb_folder, 'opt/m5stack/lib/llm')) + if package_name == 'llm-vlm': + vlm_dir = os.path.join(src_folder, 'vlm') + if os.path.exists(vlm_dir): + shutil.copytree(vlm_dir, os.path.join(deb_folder, 'opt/m5stack/lib/vlm')) shutil.copy2(os.path.join(src_folder, package_name.replace("-", "_")), os.path.join(deb_folder, 'opt/m5stack/bin', package_name.replace("-", "_"))) ext_scripts_files = glob.glob(os.path.join(src_folder, package_name + "_*")) if ext_scripts_files: @@ -253,7 +265,8 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'): f.write(f'Original-Maintainer: m5stack \n') f.write(f'Section: llm-module\n') f.write(f'Priority: optional\n') - f.write(f'Depends: lib-llm\n') + # f.write(f'Depends: lib-llm\n') + f.write(f'Depends: lib-llm (>= 1.7)\n') f.write(f'Homepage: https://www.m5stack.com\n') f.write(f'Description: llm-module\n') f.write(f' bsp.\n') @@ -363,22 +376,22 @@ if __name__ == "__main__": #################################################注意################################################ #################################################注意################################################ Tasks = { - 'lib-llm':[create_lib_deb,'lib-llm', 1.6, src_folder, revision], + 'lib-llm':[create_lib_deb,'lib-llm', 1.7, src_folder, revision], 'llm-sys':[create_bin_deb,'llm-sys', version, src_folder, revision], 'llm-audio':[create_bin_deb,'llm-audio', version, src_folder, revision], - 'llm-kws':[create_bin_deb,'llm-kws', version, src_folder, revision], + 'llm-kws':[create_bin_deb,'llm-kws', '1.6', src_folder, revision], 'llm-asr':[create_bin_deb,'llm-asr', version, src_folder, revision], - 'llm-llm':[create_bin_deb,'llm-llm', '1.6', src_folder, revision], + 'llm-llm':[create_bin_deb,'llm-llm', '1.7', src_folder, revision], 'llm-tts':[create_bin_deb,'llm-tts', version, src_folder, revision], 'llm-melotts':[create_bin_deb,'llm-melotts', version, src_folder, revision], 'llm-camera':[create_bin_deb,'llm-camera', '1.6', src_folder, revision], - 'llm-vlm':[create_bin_deb,'llm-vlm', version, src_folder, revision], + 'llm-vlm':[create_bin_deb,'llm-vlm', '1.6', src_folder, revision], 'llm-yolo':[create_bin_deb,'llm-yolo', '1.6', src_folder, revision], 'llm-skel':[create_bin_deb,'llm-skel', version, src_folder, revision], 'llm-depth-anything':[create_bin_deb,'llm-depth-anything', version, src_folder, revision], 'llm-vad':[create_bin_deb,'llm-vad', version, src_folder, revision], 'llm-whisper':[create_bin_deb,'llm-whisper', version, src_folder, revision], - 'llm-openai-api':[create_bin_deb,'llm-openai-api', version, src_folder, revision], + 'llm-openai-api':[create_bin_deb,'llm-openai-api', '1.6', src_folder, revision], 'llm-model-audio-en-us':[create_data_deb,'llm-model-audio-en-us', data_version, src_folder, revision], 'llm-model-audio-zh-cn':[create_data_deb,'llm-model-audio-zh-cn', data_version, src_folder, revision], 'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17', data_version, src_folder, revision], @@ -407,8 +420,11 @@ if __name__ == "__main__": 'llm-model-llama3.2-1B-p256-ax630c':[create_data_deb,'llm-model-llama3.2-1B-p256-ax630c', '0.4', src_folder, revision], 'llm-model-openbuddy-llama3.2-1B-ax630c':[create_data_deb,'llm-model-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision], 'llm-model-internvl2.5-1B-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-ax630c', '0.4', src_folder, revision], + 'llm-model-internvl2.5-1B-364-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-364-ax630c', '0.4', src_folder, revision], 'llm-model-deepseek-r1-1.5B-ax630c':[create_data_deb,'llm-model-deepseek-r1-1.5B-ax630c', '0.3', src_folder, revision], 'llm-model-deepseek-r1-1.5B-p256-ax630c':[create_data_deb,'llm-model-deepseek-r1-1.5B-p256-ax630c', '0.4', src_folder, revision], + 'llm-model-smolvlm-256M-ax630c':[create_data_deb,'llm-model-smolvlm-256M-ax630c', '0.4', src_folder, revision], + 'llm-model-smolvlm-500M-ax630c':[create_data_deb,'llm-model-smolvlm-500M-ax630c', '0.4', src_folder, revision], # 'llm-model-qwen2-0.5B-prefill-20e':[create_data_deb,'llm-model-qwen2-0.5B-prefill-20e', data_version, src_folder, revision], # 'llm-model-qwen2-1.5B-prefill-20e':[create_data_deb,'llm-model-qwen2-1.5B-prefill-20e', data_version, src_folder, revision] }