m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of github.com:m5stack/StackFlow into dev

Browse Source
This commit is contained in:
dianjixz
2025-04-09 18:10:25 +08:00
parent 9f257bca57 c39e691357
commit ce0d5c2881
20 changed files with 923 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/projects_llm_framework_doc/llm_asr_en.md
+4 -4
View File
@@ -16,7 +16,7 @@ Send JSON:
"action": "setup",
"object": "asr.setup",
"data": {
"model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23",
"model": "sherpa-ncnn-streaming-zipformer-20M-2023-02-17",
"response_format": "asr.utf-8.stream",
"input": "sys.pcm",
"enoutput": true,
@@ -34,7 +34,7 @@ Send JSON:
- work_id: For configuration units, it is `asr`.
- action: The method to be called is `setup`.
- object: The type of data being transmitted is `asr.setup`.
- model: The model used is the Chinese model `sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23`.
- model: The model used is the Chinese model `sherpa-ncnn-streaming-zipformer-20M-2023-02-17`.
- response_format: The result format is `asr.utf-8.stream`, a UTF-8 stream output.
- input: The input is `sys.pcm`, representing system audio.
- enoutput: Whether to enable user result output.
@@ -109,7 +109,7 @@ Example:
"action": "setup",
"object": "asr.setup",
"data": {
"model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23",
"model": "sherpa-ncnn-streaming-zipformer-20M-2023-02-17",
"response_format": "asr.utf-8.stream",
"input": [
"sys.pcm",
@@ -310,7 +310,7 @@ Response JSON:
"inputs_": [
"sys.pcm"
],
"model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23",
"model": "sherpa-ncnn-streaming-zipformer-20M-2023-02-17",
"response_format": "asr.utf-8-stream"
},
"error": {
doc/projects_llm_framework_doc/llm_kws_en.md
+4 -4
View File
@@ -16,11 +16,11 @@ Send JSON:
"action": "setup",
"object": "kws.setup",
"data": {
"model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01",
"model": "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01",
"response_format": "kws.bool",
"input": "sys.pcm",
"enoutput": true,
"kws": "你好你好"
"kws": "HELLO"
}
}
```
@@ -29,7 +29,7 @@ Send JSON:
- work_id: When configuring the unit, it is `kws`.
- action: The method called is `setup`.
- object: The type of data being transmitted is `kws.setup`.
- model: The model used is the Chinese model `sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01`.
- model: The model used is the Chinese model `sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01`.
- response_format: The result returned is in `kws.bool` format.
- input: The input is `sys.pcm`, representing system audio.
- enoutput: Whether to enable user result output.
@@ -204,7 +204,7 @@ Response JSON:
"inputs_": [
"sys.pcm"
],
"model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01",
"model": "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01",
"response_format": "kws.bool"
},
"error": {
projects/llm_framework/main/SConstruct
-1
View File
@@ -24,7 +24,6 @@ STATIC_FILES += [AFile('../static_lib/sherpa/ncnn/libsherpa-ncnn-core.so'),
AFile('../static_lib/sherpa/ncnn/libncnn.so'),
AFile('../static_lib/libtts.so'),
AFile('../static_lib/sherpa/ncnn/libkaldi-native-fbank-core.so'),
AFile('../static_lib/libonnxruntime.so.1.14.0')
]
env['COMPONENTS'].append({'target':'static_file',
projects/llm_framework/main_kws/SConstruct
+21
View File
@@ -16,6 +16,8 @@ LDFLAGS = []
LINK_SEARCH_PATH = []
STATIC_FILES = []
python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-kws-python-venv_v1.6.tar.gz", 'm5stack_llm-kws-python-venv_v1.6.tar.gz')
DEFINITIONS += ['-std=c++17']
LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
LINK_SEARCH_PATH += [ADir('../static_lib')]
@@ -31,9 +33,28 @@ LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a',
'-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a',
'-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a']
STATIC_FILES += [os.path.join(python_venv, 'sherpa-onnx')]
STATIC_FILES += Glob('llm-kws_text2token.py')
STATIC_FILES += Glob('mode_*.json')
IGNORE_FILES = []
IGNORE_FILES += ['sherpa-onnx']
import json
if not os.path.exists('../dist'):
os.makedirs('../dist')
ignore = {'ignore':[]}
try:
with open('../dist/fileignore', 'a+') as f:
f.seek(0)
ignore = json.load(f)
except:
pass
ignore['ignore'] += IGNORE_FILES
ignore['ignore'] = list(set(ignore['ignore']))
with open('../dist/fileignore', 'w') as f:
json.dump(ignore, f, indent=4)
env['COMPONENTS'].append({'target':'llm_kws',
'SRCS':SRCS,
'INCLUDE':INCLUDE,
projects/llm_framework/main_kws/src/main.cpp
+2 -2
View File
@@ -177,9 +177,9 @@ public:
temp_awake_key.close();
std::ostringstream awake_key_compile_cmd;
if (file_exists("/opt/m5stack/scripts/text2token.py"))
awake_key_compile_cmd << "/usr/bin/python3 /opt/m5stack/scripts/text2token.py ";
awake_key_compile_cmd << "PYTHONPATH=/opt/m5stack/lib/sherpa-onnx/site-packages /usr/bin/python3 /opt/m5stack/scripts/text2token.py ";
else if (file_exists("/opt/m5stack/scripts/llm-kws_text2token.py"))
awake_key_compile_cmd << "/usr/bin/python3 /opt/m5stack/scripts/llm-kws_text2token.py ";
awake_key_compile_cmd << "PYTHONPATH=/opt/m5stack/lib/sherpa-onnx/site-packages /usr/bin/python3 /opt/m5stack/scripts/llm-kws_text2token.py ";
else {
SLOGE("text2token.py or llm-kws_text2token.py not found!");
}
projects/llm_framework/main_llm/SConstruct
+21
View File
@@ -17,6 +17,8 @@ LDFLAGS = []
LINK_SEARCH_PATH = []
STATIC_FILES = []
python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-llm-python-venv_v1.7.tar.gz", 'm5stack_llm-llm-python-venv_v1.7.tar.gz')
# REQUIREMENTS += ['Backward_cpp']
# DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'),
# AFile('../static_lib/libelf.so.1'),
@@ -41,10 +43,29 @@ static_file = Glob('../static_lib/module-llm/libabsl_*')
static_file += [AFile('../static_lib/module-llm/libre2.a'), AFile('../static_lib/module-llm/libsentencepiece.a'), AFile('../static_lib/module-llm/libsentencepiece_train.a')]
STATIC_LIB += static_file * 4
STATIC_FILES += [os.path.join(python_venv, 'llm')]
STATIC_FILES += Glob('scripts/tokenizer_*.py')
STATIC_FILES += Glob('models/mode_*.json')
STATIC_FILES += [AFile('scripts/llm-llm_tokenizer_auto.py')]
IGNORE_FILES = []
IGNORE_FILES += ['llm']
import json
if not os.path.exists('../dist'):
os.makedirs('../dist')
ignore = {'ignore':[]}
try:
with open('../dist/fileignore', 'a+') as f:
f.seek(0)
ignore = json.load(f)
except:
pass
ignore['ignore'] += IGNORE_FILES
ignore['ignore'] = list(set(ignore['ignore']))
with open('../dist/fileignore', 'w') as f:
json.dump(ignore, f, indent=4)
env['COMPONENTS'].append({'target':'llm_llm',
'SRCS':SRCS,
'INCLUDE':INCLUDE,
projects/llm_framework/main_llm/src/main.cpp
+1
View File
@@ -159,6 +159,7 @@ public:
if (!tokenizer_server_flage_.load()) {
tokenizer_pid_ = fork();
if (tokenizer_pid_ == 0) {
setenv("PYTHONPATH", "/opt/m5stack/lib/llm/site-packages", 1);
execl("/usr/bin/python3", "python3", tokenizer_file.c_str(), "--host", "localhost", "--port",
std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(),
"--content", ("'" + prompt_ + "'").c_str(), nullptr);
projects/llm_framework/main_melotts/SConstruct
+1 -1
View File
@@ -21,7 +21,7 @@ DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17']
LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
LINK_SEARCH_PATH += [ADir('../static_lib')]
REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
REQUIREMENTS += ['onnxruntime', 'samplerate']
REQUIREMENTS += ['samplerate']
INCLUDE += [ADir('../include')]
INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')]
projects/llm_framework/main_openai_api/SConstruct
+1 -1
View File
@@ -19,7 +19,7 @@ STATIC_FILES = []
ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/Abandon-ht/ModuleLLM-OpenAI-Plugin.git', '1077efbe201ea3f29517f5ce4a0cfc3b04c25d1d', True)
python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.5.tar.gz", 'm5stack_llm-llm-openai-api-python-venv_v1.5.tar.gz')
python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.5.tar.gz", 'm5stack_llm-openai-api-python-venv_v1.5.tar.gz')
DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17']
projects/llm_framework/main_vlm/SConstruct
+21
View File
@@ -17,7 +17,9 @@ LDFLAGS = []
LINK_SEARCH_PATH = []
STATIC_FILES = []
python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-vlm-python-venv_v1.6.tar.gz", 'm5stack_llm-vlm-python-venv_v1.6.tar.gz')
DEFINITIONS += ['-O2']
DEFINITIONS += ['-std=c++17']
LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
@@ -49,9 +51,28 @@ static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libtegra_hal
static_file += [AFile('../static_lib/libopencv-4.6-aarch64-none/lib/libzlib.a')]
STATIC_LIB += static_file * 4
STATIC_FILES += [os.path.join(python_venv, 'vlm')]
STATIC_FILES += Glob('scripts/tokenizer_*.py')
STATIC_FILES += Glob('models/mode_*.json')
IGNORE_FILES = []
IGNORE_FILES += ['vlm']
import json
if not os.path.exists('../dist'):
os.makedirs('../dist')
ignore = {'ignore':[]}
try:
with open('../dist/fileignore', 'a+') as f:
f.seek(0)
ignore = json.load(f)
except:
pass
ignore['ignore'] += IGNORE_FILES
ignore['ignore'] = list(set(ignore['ignore']))
with open('../dist/fileignore', 'w') as f:
json.dump(ignore, f, indent=4)
env['COMPONENTS'].append({'target':'llm_vlm',
'SRCS':SRCS,
'INCLUDE':INCLUDE,
projects/llm_framework/main_vlm/models/mode_internvl2.5-1B-364-ax630c.json
+35
View File
@@ -0,0 +1,35 @@
{
"mode":"internvl2.5-1B-364-ax630c",
"type":"vlm",
"homepage":"https://huggingface.co/AXERA-TECH/InternVL2_5-1B",
"capabilities":[
"text_generation",
"chat"
],
"input_type":[
"vlm.chat_completion",
"vlm.chat_completion.stream"
],
"output_type":[
"vlm.utf-8",
"vlm.utf-8.stream"
],
"mode_param":{
"tokenizer_type":2,
"filename_tokenizer_model":"http://localhost:8080",
"filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
"filename_post_axmodel":"qwen2_post.axmodel",
"template_filename_axmodel":"qwen2_p256_l%d_together.axmodel",
"filename_vpm_resampler_axmodedl":"vit_intern_sim_space2depth.axmodel",
"b_use_topk":false,
"b_bos":false,
"b_eos":false,
"axmodel_num":24,
"tokens_embed_num":151674,
"img_token_id":151667,
"tokens_embed_size":896,
"b_use_mmap_load_embed":true,
"b_dynamic_load_axmodel_layer":false,
"ext_scripts":["tokenizer_internvl2.5-1B-364-ax630c.py"]
}
}
projects/llm_framework/main_vlm/models/mode_smolvlm-256M-ax630c.json
+35
View File
@@ -0,0 +1,35 @@
{
"mode":"smolvlm-256M-ax630c",
"type":"vlm",
"homepage":"https://huggingface.co/HuggingFaceTB/SmolVLM-256M-Instruct",
"capabilities":[
"text_generation",
"chat"
],
"input_type":[
"vlm.chat_completion",
"vlm.chat_completion.stream"
],
"output_type":[
"vlm.utf-8",
"vlm.utf-8.stream"
],
"mode_param":{
"tokenizer_type":2,
"filename_tokenizer_model":"http://localhost:8080",
"filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
"filename_post_axmodel":"llama_post.axmodel",
"template_filename_axmodel":"llama_p128_l%d_together.axmodel",
"filename_vpm_resampler_axmodedl":"SmolVLM-256M-Instruct_vision_nhwc.axmodel",
"b_use_topk":false,
"b_bos":false,
"b_eos":false,
"axmodel_num":30,
"tokens_embed_num":49280,
"img_token_id":49190,
"tokens_embed_size":576,
"b_use_mmap_load_embed":true,
"b_dynamic_load_axmodel_layer":false,
"ext_scripts":["tokenizer_smolvlm-256M-ax630c.py"]
}
}
projects/llm_framework/main_vlm/models/mode_smolvlm-500M-ax630c.json
+35
View File
@@ -0,0 +1,35 @@
{
"mode":"smolvlm-500M-ax630c",
"type":"vlm",
"homepage":"https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Instruct",
"capabilities":[
"text_generation",
"chat"
],
"input_type":[
"vlm.chat_completion",
"vlm.chat_completion.stream"
],
"output_type":[
"vlm.utf-8",
"vlm.utf-8.stream"
],
"mode_param":{
"tokenizer_type":2,
"filename_tokenizer_model":"http://localhost:8080",
"filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
"filename_post_axmodel":"llama_post.axmodel",
"template_filename_axmodel":"llama_p128_l%d_together.axmodel",
"filename_vpm_resampler_axmodedl":"SmolVLM-500M-Instruct_vision.axmodel",
"b_use_topk":false,
"b_bos":false,
"b_eos":false,
"axmodel_num":32,
"tokens_embed_num":49280,
"img_token_id":49190,
"tokens_embed_size":960,
"b_use_mmap_load_embed":true,
"b_dynamic_load_axmodel_layer":false,
"ext_scripts":["tokenizer_smolvlm-500M-ax630c.py"]
}
}
projects/llm_framework/main_vlm/scripts/tokenizer_internvl2.5-1B-364-ax630c.py
+138
View File
@@ -0,0 +1,138 @@
from transformers import AutoTokenizer, PreTrainedTokenizerFast
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import argparse
class Tokenizer_Http:
def __init__(self, model_id):
self.tokenizer = AutoTokenizer.from_pretrained(
model_id, trust_remote_code=True, use_fast=False
)
def encode(self, prompt, content):
prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n{prompt}<|im_end|><|im_start|>assistant\n"
input_ids = self.tokenizer.encode(prompt)
return input_ids
def encode_vpm(self, prompt, content="Please describe the image shortly."):
prompt = f"<|im_start|>system\n{content}<|im_end|><|im_start|>user\n<img>" + "<IMG_CONTEXT>" * 169 + f"</img>\n{prompt}<|im_end|><|im_start|>assistant\n"
input_ids = self.tokenizer.encode(prompt)
return input_ids
def decode(self, token_ids):
return self.tokenizer.decode(token_ids, clean_up_tokenization_spaces=False)
@property
def bos_id(self):
return self.tokenizer.bos_token_id
@property
def eos_id(self):
return self.tokenizer.eos_token_id
@property
def bos_token(self):
return self.tokenizer.bos_token
@property
def eos_token(self):
return self.tokenizer.eos_token
class Request(BaseHTTPRequestHandler):
# 通过类继承，新定义类
timeout = 5
server_version = "Apache"
def do_GET(self):
print(self.path)
# 在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行）
self.send_response(200)
self.send_header("type", "get") # 设置响应头，可省略或设置多个
self.end_headers()
if self.path == "/bos_id":
bos_id = tokenizer.bos_id
# print(bos_id)
# to json
if bos_id is None:
msg = json.dumps({"bos_id": -1})
else:
msg = json.dumps({"bos_id": bos_id})
elif self.path == "/eos_id":
eos_id = tokenizer.eos_id
if eos_id is None:
msg = json.dumps({"eos_id": -1})
else:
msg = json.dumps({"eos_id": eos_id})
else:
msg = "error"
print(msg)
msg = str(msg).encode() # 转为str再转为byte格式
self.wfile.write(msg) # 将byte格式的信息返回给客户端
def do_POST(self):
# 在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行）
data = self.rfile.read(
int(self.headers["content-length"])
) # 获取从客户端传入的参数（byte格式）
data = data.decode() # 将byte格式转为str格式
self.send_response(200)
self.send_header("type", "post") # 设置响应头，可省略或设置多个
self.end_headers()
if self.path == "/encode":
req = json.loads(data)
print(req)
prompt = req["text"]
b_img_prompt = False
if "img_prompt" in req:
b_img_prompt = req["img_prompt"]
if b_img_prompt:
token_ids = tokenizer.encode_vpm(prompt)
else:
token_ids = tokenizer.encode(prompt, args.content)
if token_ids is None:
msg = json.dumps({"token_ids": -1})
else:
msg = json.dumps({"token_ids": token_ids})
elif self.path == "/decode":
req = json.loads(data)
token_ids = req["token_ids"]
text = tokenizer.decode(token_ids)
if text is None:
msg = json.dumps({"text": ""})
else:
msg = json.dumps({"text": text})
else:
msg = "error"
print(msg)
msg = str(msg).encode() # 转为str再转为byte格式
self.wfile.write(msg) # 将byte格式的信息返回给客户端
if __name__ == "__main__":
args = argparse.ArgumentParser()
args.add_argument("--host", type=str, default="localhost")
args.add_argument("--port", type=int, default=8080)
args.add_argument('--model_id', type=str, default='internvl2_tokenizer')
args.add_argument('--content', type=str, default='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。')
args = args.parse_args()
tokenizer = Tokenizer_Http(args.model_id)
# print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token)
# print(tokenizer.encode("hello world", args.content))
host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1'
print("http://%s:%s" % host)
server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例
server.serve_forever() # 开启服务
projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-256M-ax630c.py
+248
View File
@@ -0,0 +1,248 @@
from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.tokenization_utils_base import AddedToken
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import argparse
def _prompt_split_image(
image_seq_len,
image_rows,
image_cols,
fake_token_around_image,
image_token,
global_img_token,
):
"""Prompt with expanded image tokens for when the image is split into patches."""
text_split_images = ""
for n_h in range(image_rows):
for n_w in range(image_cols):
text_split_images += (
f"{fake_token_around_image}"
+ f"<row_{n_h + 1}_col_{n_w + 1}>"
+ f"{image_token}" * image_seq_len
)
text_split_images += "\n"
text_split_images += (
f"\n{fake_token_around_image}"
+ f"{global_img_token}"
+ f"{image_token}" * image_seq_len
+ f"{fake_token_around_image}"
)
return text_split_images
def _prompt_single_image(
image_seq_len, fake_token_around_image, image_token, global_img_token
):
"""Prompt with expanded image tokens for a single image."""
return (
f"{fake_token_around_image}"
+ f"{global_img_token}"
+ f"{image_token}" * image_seq_len
+ f"{fake_token_around_image}"
)
def get_image_prompt_string(
image_rows,
image_cols,
image_seq_len,
fake_token_around_image,
image_token,
global_img_token,
):
if image_rows == 0 and image_cols == 0:
return _prompt_single_image(
image_seq_len,
fake_token_around_image=fake_token_around_image,
image_token=image_token,
global_img_token=global_img_token,
)
return _prompt_split_image(
image_seq_len,
image_rows,
image_cols,
fake_token_around_image,
image_token,
global_img_token,
)
class Tokenizer_Http:
def __init__(self, model_id):
self.tokenizer = AutoTokenizer.from_pretrained(
model_id, trust_remote_code=True, use_fast=False
)
def encode(self, prompt, content):
prompt = f"<|im_start|>User:{content}<end_of_utterance>\nAssistant:"
input_ids = self.tokenizer(prompt)
return input_ids["input_ids"]
def encode_vpm(self, prompt, content="Please describe the image shortly."):
prompt = f"<|im_start|>User:<image>{prompt}<end_of_utterance>\nAssistant:"
text = [prompt]
image_rows = [[0]]
image_cols = [[0]]
image_seq_len = 64
image_token = "<image>"
fake_image_token = "<fake_token_around_image>"
global_img_token = "<global-img>"
prompt_strings = []
for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):
# Replace the image token with fake tokens around the expanded image token sequence of length `image_seq_len`
image_prompt_strings = []
for n_rows, n_cols in zip(sample_rows, sample_cols):
image_prompt_string = get_image_prompt_string(
n_rows,
n_cols,
image_seq_len,
image_token=image_token,
fake_token_around_image=fake_image_token,
global_img_token=global_img_token,
)
image_prompt_strings.append(image_prompt_string)
split_sample = sample.split(image_token)
if len(split_sample) == 0:
raise ValueError("The image token should be present in the text.")
# Place in the image prompt strings where the image tokens are
sample = split_sample[0]
for i, image_prompt_string in enumerate(image_prompt_strings):
sample += image_prompt_string + split_sample[i + 1]
prompt_strings.append(sample)
fake_image_token = AddedToken(fake_image_token, normalized=False, special=True)
image_token = AddedToken(image_token, normalized=False, special=True)
end_of_utterance_token = AddedToken(
"<end_of_utterance>", normalized=False, special=True
)
tokens_to_add = {
"additional_special_tokens": [
fake_image_token,
image_token,
end_of_utterance_token,
]
}
self.tokenizer.add_special_tokens(tokens_to_add)
input_ids = self.tokenizer(prompt_strings)["input_ids"][0]
return input_ids
def decode(self, token_ids):
return self.tokenizer.decode(token_ids, clean_up_tokenization_spaces=False)
@property
def bos_id(self):
return self.tokenizer.bos_token_id
@property
def eos_id(self):
return self.tokenizer.eos_token_id
@property
def bos_token(self):
return self.tokenizer.bos_token
@property
def eos_token(self):
return self.tokenizer.eos_token
class Request(BaseHTTPRequestHandler):
# 通过类继承，新定义类
timeout = 5
server_version = "Apache"
def do_GET(self):
print(self.path)
# 在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行）
self.send_response(200)
self.send_header("type", "get") # 设置响应头，可省略或设置多个
self.end_headers()
if self.path == "/bos_id":
bos_id = tokenizer.bos_id
# print(bos_id)
# to json
if bos_id is None:
msg = json.dumps({"bos_id": -1})
else:
msg = json.dumps({"bos_id": bos_id})
elif self.path == "/eos_id":
eos_id = tokenizer.eos_id
if eos_id is None:
msg = json.dumps({"eos_id": -1})
else:
msg = json.dumps({"eos_id": eos_id})
else:
msg = "error"
print(msg)
msg = str(msg).encode() # 转为str再转为byte格式
self.wfile.write(msg) # 将byte格式的信息返回给客户端
def do_POST(self):
# 在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行）
data = self.rfile.read(
int(self.headers["content-length"])
) # 获取从客户端传入的参数（byte格式）
data = data.decode() # 将byte格式转为str格式
self.send_response(200)
self.send_header("type", "post") # 设置响应头，可省略或设置多个
self.end_headers()
if self.path == "/encode":
req = json.loads(data)
print(req)
prompt = req["text"]
b_img_prompt = False
if "img_prompt" in req:
b_img_prompt = req["img_prompt"]
if b_img_prompt:
token_ids = tokenizer.encode_vpm(prompt)
else:
token_ids = tokenizer.encode(prompt, args.content)
if token_ids is None:
msg = json.dumps({"token_ids": -1})
else:
msg = json.dumps({"token_ids": token_ids})
elif self.path == "/decode":
req = json.loads(data)
token_ids = req["token_ids"]
text = tokenizer.decode(token_ids)
if text is None:
msg = json.dumps({"text": ""})
else:
msg = json.dumps({"text": text})
else:
msg = "error"
print(msg)
msg = str(msg).encode() # 转为str再转为byte格式
self.wfile.write(msg) # 将byte格式的信息返回给客户端
if __name__ == "__main__":
args = argparse.ArgumentParser()
args.add_argument("--host", type=str, default="localhost")
args.add_argument("--port", type=int, default=8080)
args.add_argument('--model_id', type=str, default='internvl2_tokenizer')
args.add_argument('--content', type=str, default='')
args = args.parse_args()
tokenizer = Tokenizer_Http(args.model_id)
# print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token)
# print(tokenizer.encode("hello world", args.content))
host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1'
print("http://%s:%s" % host)
server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例
server.serve_forever() # 开启服务
projects/llm_framework/main_vlm/scripts/tokenizer_smolvlm-500M-ax630c.py
+248
View File
@@ -0,0 +1,248 @@
from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.tokenization_utils_base import AddedToken
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import argparse
def _prompt_split_image(
image_seq_len,
image_rows,
image_cols,
fake_token_around_image,
image_token,
global_img_token,
):
"""Prompt with expanded image tokens for when the image is split into patches."""
text_split_images = ""
for n_h in range(image_rows):
for n_w in range(image_cols):
text_split_images += (
f"{fake_token_around_image}"
+ f"<row_{n_h + 1}_col_{n_w + 1}>"
+ f"{image_token}" * image_seq_len
)
text_split_images += "\n"
text_split_images += (
f"\n{fake_token_around_image}"
+ f"{global_img_token}"
+ f"{image_token}" * image_seq_len
+ f"{fake_token_around_image}"
)
return text_split_images
def _prompt_single_image(
image_seq_len, fake_token_around_image, image_token, global_img_token
):
"""Prompt with expanded image tokens for a single image."""
return (
f"{fake_token_around_image}"
+ f"{global_img_token}"
+ f"{image_token}" * image_seq_len
+ f"{fake_token_around_image}"
)
def get_image_prompt_string(
image_rows,
image_cols,
image_seq_len,
fake_token_around_image,
image_token,
global_img_token,
):
if image_rows == 0 and image_cols == 0:
return _prompt_single_image(
image_seq_len,
fake_token_around_image=fake_token_around_image,
image_token=image_token,
global_img_token=global_img_token,
)
return _prompt_split_image(
image_seq_len,
image_rows,
image_cols,
fake_token_around_image,
image_token,
global_img_token,
)
class Tokenizer_Http:
def __init__(self, model_id):
self.tokenizer = AutoTokenizer.from_pretrained(
model_id, trust_remote_code=True, use_fast=False
)
def encode(self, prompt, content):
prompt = f"<|im_start|>User:{content}<end_of_utterance>\nAssistant:"
input_ids = self.tokenizer(prompt)
return input_ids["input_ids"]
def encode_vpm(self, prompt, content="Please describe the image shortly."):
prompt = f"<|im_start|>User:<image>{prompt}<end_of_utterance>\nAssistant:"
text = [prompt]
image_rows = [[0]]
image_cols = [[0]]
image_seq_len = 64
image_token = "<image>"
fake_image_token = "<fake_token_around_image>"
global_img_token = "<global-img>"
prompt_strings = []
for sample, sample_rows, sample_cols in zip(text, image_rows, image_cols):
# Replace the image token with fake tokens around the expanded image token sequence of length `image_seq_len`
image_prompt_strings = []
for n_rows, n_cols in zip(sample_rows, sample_cols):
image_prompt_string = get_image_prompt_string(
n_rows,
n_cols,
image_seq_len,
image_token=image_token,
fake_token_around_image=fake_image_token,
global_img_token=global_img_token,
)
image_prompt_strings.append(image_prompt_string)
split_sample = sample.split(image_token)
if len(split_sample) == 0:
raise ValueError("The image token should be present in the text.")
# Place in the image prompt strings where the image tokens are
sample = split_sample[0]
for i, image_prompt_string in enumerate(image_prompt_strings):
sample += image_prompt_string + split_sample[i + 1]
prompt_strings.append(sample)
fake_image_token = AddedToken(fake_image_token, normalized=False, special=True)
image_token = AddedToken(image_token, normalized=False, special=True)
end_of_utterance_token = AddedToken(
"<end_of_utterance>", normalized=False, special=True
)
tokens_to_add = {
"additional_special_tokens": [
fake_image_token,
image_token,
end_of_utterance_token,
]
}
self.tokenizer.add_special_tokens(tokens_to_add)
input_ids = self.tokenizer(prompt_strings)["input_ids"][0]
return input_ids
def decode(self, token_ids):
return self.tokenizer.decode(token_ids, clean_up_tokenization_spaces=False)
@property
def bos_id(self):
return self.tokenizer.bos_token_id
@property
def eos_id(self):
return self.tokenizer.eos_token_id
@property
def bos_token(self):
return self.tokenizer.bos_token
@property
def eos_token(self):
return self.tokenizer.eos_token
class Request(BaseHTTPRequestHandler):
# 通过类继承，新定义类
timeout = 5
server_version = "Apache"
def do_GET(self):
print(self.path)
# 在新类中定义get的内容（当客户端向该服务端使用get请求时，本服务端将如下运行）
self.send_response(200)
self.send_header("type", "get") # 设置响应头，可省略或设置多个
self.end_headers()
if self.path == "/bos_id":
bos_id = tokenizer.bos_id
# print(bos_id)
# to json
if bos_id is None:
msg = json.dumps({"bos_id": -1})
else:
msg = json.dumps({"bos_id": bos_id})
elif self.path == "/eos_id":
eos_id = tokenizer.eos_id
if eos_id is None:
msg = json.dumps({"eos_id": -1})
else:
msg = json.dumps({"eos_id": eos_id})
else:
msg = "error"
print(msg)
msg = str(msg).encode() # 转为str再转为byte格式
self.wfile.write(msg) # 将byte格式的信息返回给客户端
def do_POST(self):
# 在新类中定义post的内容（当客户端向该服务端使用post请求时，本服务端将如下运行）
data = self.rfile.read(
int(self.headers["content-length"])
) # 获取从客户端传入的参数（byte格式）
data = data.decode() # 将byte格式转为str格式
self.send_response(200)
self.send_header("type", "post") # 设置响应头，可省略或设置多个
self.end_headers()
if self.path == "/encode":
req = json.loads(data)
print(req)
prompt = req["text"]
b_img_prompt = False
if "img_prompt" in req:
b_img_prompt = req["img_prompt"]
if b_img_prompt:
token_ids = tokenizer.encode_vpm(prompt)
else:
token_ids = tokenizer.encode(prompt, args.content)
if token_ids is None:
msg = json.dumps({"token_ids": -1})
else:
msg = json.dumps({"token_ids": token_ids})
elif self.path == "/decode":
req = json.loads(data)
token_ids = req["token_ids"]
text = tokenizer.decode(token_ids)
if text is None:
msg = json.dumps({"text": ""})
else:
msg = json.dumps({"text": text})
else:
msg = "error"
print(msg)
msg = str(msg).encode() # 转为str再转为byte格式
self.wfile.write(msg) # 将byte格式的信息返回给客户端
if __name__ == "__main__":
args = argparse.ArgumentParser()
args.add_argument("--host", type=str, default="localhost")
args.add_argument("--port", type=int, default=8080)
args.add_argument('--model_id', type=str, default='internvl2_tokenizer')
args.add_argument('--content', type=str, default='')
args = args.parse_args()
tokenizer = Tokenizer_Http(args.model_id)
# print(tokenizer.bos_id, tokenizer.bos_token, tokenizer.eos_id, tokenizer.eos_token)
# print(tokenizer.encode("hello world", args.content))
host = (args.host, args.port) # 设定地址与端口号，'localhost'等价于'127.0.0.1'
print("http://%s:%s" % host)
server = HTTPServer(host, Request) # 根据地址端口号和新定义的类，创建服务器实例
server.serve_forever() # 开启服务
projects/llm_framework/main_vlm/src/main.cpp
+14 -19
View File
@@ -50,8 +50,8 @@ public:
std::string response_format_;
std::vector<std::string> inputs_;
std::vector<unsigned short> prompt_data_;
std::vector<std::vector<unsigned char>> image_datas_;
std::vector<std::vector<unsigned short>> img_embeds;
std::vector<unsigned char> image_data_;
std::vector<unsigned short> img_embed;
std::string prompt_;
task_callback_t out_callback_;
bool enoutput_;
@@ -126,6 +126,7 @@ public:
CONFIG_AUTO_SET(file_body["mode_param"], b_eos);
CONFIG_AUTO_SET(file_body["mode_param"], axmodel_num);
CONFIG_AUTO_SET(file_body["mode_param"], tokens_embed_num);
CONFIG_AUTO_SET(file_body["mode_param"], img_token_id);
CONFIG_AUTO_SET(file_body["mode_param"], tokens_embed_size);
CONFIG_AUTO_SET(file_body["mode_param"], b_use_mmap_load_embed);
CONFIG_AUTO_SET(file_body["mode_param"], b_dynamic_load_axmodel_layer);
@@ -154,6 +155,7 @@ public:
if (!tokenizer_server_flage_.load()) {
tokenizer_pid_ = fork();
if (tokenizer_pid_ == 0) {
setenv("PYTHONPATH", "/opt/m5stack/lib/vlm/site-packages", 1);
execl("/usr/bin/python3", "python3", tokenizer_file.c_str(), "--host", "localhost", "--port",
std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(),
"--content", ("'" + prompt_ + "'").c_str(), nullptr);
@@ -216,32 +218,25 @@ public:
oss_prompt << input;
break;
}
SLOGI("prompt_complete:%s", oss_prompt.str().c_str());
// SLOGI("prompt_complete:%s", oss_prompt.str().c_str());
return oss_prompt.str();
}
void inference(const std::string &msg)
{
try {
if (image_datas_.empty()) {
if (image_data_.empty()) {
lLaMa_->Encode(prompt_data_, prompt_complete(msg));
std::string out = lLaMa_->Run(prompt_data_);
if (out_callback_) out_callback_(out, true);
} else {
img_embeds.clear();
for (auto &img_data : image_datas_) {
cv::Mat src = cv::imdecode(img_data, cv::IMREAD_COLOR);
if (src.empty()) continue;
std::vector<unsigned short> embed;
lLaMa_->Encode(src, embed);
img_embeds.push_back(embed);
}
image_datas_.clear();
if (!img_embeds.empty()) {
lLaMa_->Encode(img_embeds, prompt_data_, prompt_complete(msg));
std::string out = lLaMa_->Run(prompt_data_);
if (out_callback_) out_callback_(out, true);
}
cv::Mat src = cv::imdecode(image_data_, cv::IMREAD_COLOR);
if (src.empty()) return;
image_data_.clear();
lLaMa_->Encode(src, img_embed);
lLaMa_->Encode(img_embed, prompt_data_, prompt_complete(msg));
std::string out = lLaMa_->Run(prompt_data_);
if (out_callback_) out_callback_(out, true);
}
} catch (...) {
SLOGW("lLaMa_->Run have error!");
@@ -416,7 +411,7 @@ public:
next_data = &tmp_msg2;
}
if (object.find("jpeg") != std::string::npos) {
llm_task_obj->image_datas_.emplace_back(next_data->begin(), next_data->end());
llm_task_obj->image_data_.assign(next_data->begin(), next_data->end());
return;
}
llm_task_obj->inference((*next_data));
projects/llm_framework/main_vlm/src/runner/LLM.hpp
+54 -46
View File
@@ -26,8 +26,6 @@ struct LLMAttrType {
std::string filename_post_axmodel = "tinyllama-int8/tinyllama_post.axmodel";
bool b_use_topk = false;
std::string filename_vpm_encoder_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel";
std::string filename_vpm_resampler_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel";
int vpm_width = 280;
@@ -39,6 +37,7 @@ struct LLMAttrType {
bool b_bos = true, b_eos = false;
std::string filename_tokens_embed = "tinyllama.model.embed_tokens.weight.bfloat16.bin";
int tokens_embed_num = 32000;
int img_token_id = 151667; // InternVL2.5
int tokens_embed_size = 2048;
int max_token_len = 127; // auto calc
@@ -53,6 +52,9 @@ struct LLMAttrType {
bool b_use_mmap_load_layer = true;
bool b_use_topk = false;
std::string post_config_path = "post_config.json";
// bool b_live_print = true;
LLMRuningCallback runing_callback = nullptr;
void *reserve = nullptr;
@@ -84,36 +86,17 @@ private:
bool b_stop = false;
int post_process(unsigned short *p, int n, std::vector<int> &history, float *val = 0)
LLMPostprocess postprocess;
static int post_process(LLMPostprocess &postprocess, unsigned short *p, int n, std::vector<int> &history,
float *val = 0)
{
std::vector<float> logits(n);
for (int i = 0; i < n; i++) {
unsigned int proc = p[i] << 16;
logits[i] = *reinterpret_cast<float *>(&proc);
}
LLMPostprocess postprocess;
postprocess.set_temperature(true, _attr.temperature);
postprocess.set_repetition_penalty(true, 1.2f);
// postprocess.set_top_k_sampling(true, 40);
postprocess.set_top_p_sampling(true, _attr.top_p);
return postprocess.apply(logits, history);
// float max_val = -MAXFLOAT;
// int max_index = 0;
// for (int i = 0; i < n; i++)
// {
// unsigned int proc = p[i] << 16;
// float tmp = *reinterpret_cast<float *>(&proc);
// if (tmp > max_val)
// {
// max_val = tmp;
// max_index = i;
// }
// }
// if (val)
// *val = max_val;
// return max_index;
}
public:
@@ -308,18 +291,24 @@ public:
vpm_encoder.inference();
AX_SYS_MinvalidateCache(vpm_encoder.get_output(0).phyAddr, vpm_encoder.get_output(0).pVirAddr,
vpm_encoder.get_output(0).nSize);
memcpy(vpm_resampler.get_input("input").pVirAddr, vpm_encoder.get_output(0).pVirAddr,
memcpy(vpm_resampler.get_input(0).pVirAddr, vpm_encoder.get_output(0).pVirAddr,
vpm_encoder.get_output(0).nSize);
} else {
void *data = vpm_resampler.get_input("input").pVirAddr;
void *data = vpm_resampler.get_input(0).pVirAddr;
memcpy(data, dst.data, dst.rows * dst.cols * 3);
}
vpm_resampler.inference();
out_embed.resize(vpm_resampler.get_output("output").nSize / sizeof(unsigned short));
AX_SYS_MinvalidateCache(vpm_resampler.get_output("output").phyAddr, vpm_resampler.get_output("output").pVirAddr,
vpm_resampler.get_output("output").nSize);
memcpy(out_embed.data(), vpm_resampler.get_output("output").pVirAddr, vpm_resampler.get_output("output").nSize);
out_embed.resize(vpm_resampler.get_output(0).nSize / sizeof(float));
AX_SYS_MinvalidateCache(vpm_resampler.get_output(0).phyAddr, vpm_resampler.get_output(0).pVirAddr,
vpm_resampler.get_output(0).nSize);
float *output_data = (float *)vpm_resampler.get_output(0).pVirAddr;
for (size_t i = 0; i < out_embed.size(); i++) {
out_embed[i] = bfloat16(output_data[i]).data;
}
// memcpy(out_embed.data(), vpm_resampler.get_output(0).pVirAddr, vpm_resampler.get_output(0).nSize);
ALOGI("image encode time : %f ms, size : %d", t.cost(), out_embed.size());
return 0;
}
@@ -337,27 +326,49 @@ public:
embed_selector.getByIndex(input_ids[i], out_embed.data() + i * _attr.tokens_embed_size);
}
// memcpy(out_embed.data() + 5 * _attr.tokens_embed_size, vpm_resampler.get_output("output").pVirAddr,
// vpm_resampler.get_output("output").nSize);
// memcpy(out_embed.data() + 5 * _attr.tokens_embed_size, vpm_resampler.get_output(0).pVirAddr,
// vpm_resampler.get_output(0).nSize);
return 0;
}
int Encode(std::vector<std::vector<unsigned short>> &img_embeds, std::vector<unsigned short> &out_embed,
std::string prompt = "What is in the images?")
int Encode(std::vector<unsigned short> &img_embed, std::vector<unsigned short> &out_embed,
std::string prompt = "What is in the image?")
{
std::vector<int> input_ids = tokenizer->Encode(prompt, true);
constexpr int IMG_CONTEXT = 151667; // InternVL2.5
std::vector<int> img_positions;
// constexpr int img_token_id = 49190; // smolvlm
// constexpr int img_token_id = 151667; // InternVL2.5
int offset = 0;
int img_context_count = 0;
for (size_t i = 0; i < input_ids.size(); i++) {
if (input_ids[i] == IMG_CONTEXT) {
img_positions.push_back(i);
if (input_ids[i] == _attr.img_token_id) {
img_context_count++;
if (img_context_count == 1) {
offset = i;
}
}
}
if (img_positions.size() > _attr.prefill_token_num) {
if (offset == 0) {
ALOGE("offset == 0");
return -1;
}
if (img_context_count != img_embed.size() / _attr.tokens_embed_size) {
ALOGE("img_context_count(%d) != img_embed.size() / tokens_embed_size(%d)", img_context_count,
img_embed.size() / _attr.tokens_embed_size);
return -1;
}
// for (size_t i = 0; i < input_ids.size(); i++)
// {
// printf("%d ", input_ids[i]);
// }
// printf("\n");
if (input_ids.size() > _attr.prefill_token_num) {
ALOGE("input_ids(%d) > prefill_token_num(%d)", input_ids.size(), _attr.prefill_token_num);
return -1;
}
@@ -366,11 +377,8 @@ public:
for (size_t i = 0; i < input_ids.size(); i++) {
embed_selector.getByIndex(input_ids[i], out_embed.data() + i * _attr.tokens_embed_size);
}
for (size_t img_idx = 0; img_idx < img_embeds.size(); img_idx++) {
// int pos = img_positions[img_idx];
memcpy(out_embed.data() + (14 + img_idx * 64) * _attr.tokens_embed_size, img_embeds[img_idx].data(),
img_embeds[img_idx].size() * sizeof(unsigned short));
}
memcpy(out_embed.data() + offset * _attr.tokens_embed_size, img_embed.data(),
img_embed.size() * sizeof(unsigned short));
return 0;
}
@@ -504,7 +512,7 @@ public:
AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize);
unsigned short *post_out = (unsigned short *)output_post.pVirAddr;
float max_val = -MAXFLOAT;
max_index = post_process(post_out, _attr.tokens_embed_num, token_ids, &max_val);
max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val);
}
next_token = max_index;
@@ -599,7 +607,7 @@ public:
AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize);
unsigned short *post_out = (unsigned short *)output_post.pVirAddr;
float max_val = -MAXFLOAT;
max_index = post_process(post_out, _attr.tokens_embed_num, token_ids, &max_val);
max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val);
}
next_token = max_index;
projects/llm_framework/main_whisper/SConstruct
+1 -1
View File
@@ -21,7 +21,7 @@ DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17']
LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
LINK_SEARCH_PATH += [ADir('../static_lib')]
REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
REQUIREMENTS += ['onnxruntime', 'samplerate']
# REQUIREMENTS += ['onnxruntime', 'samplerate']
INCLUDE += [ADir('../include')]
INCLUDE += [ADir('src/runner'), ADir('../include/onnxruntime/core/session')]
projects/llm_framework/tools/llm_pack.py
+39 -23
View File
@@ -68,23 +68,23 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'):
# if os.path.exists(zip_file_extrpath):
# shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/scripts'))
zip_file = 'm5stack_dist-packages.tar.gz'
down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz'
zip_file_extrpath = 'm5stack_dist-packages'
if not os.path.exists(zip_file_extrpath):
# Downloading via HTTP (more common)
if not os.path.exists(zip_file):
response = requests.get(down_url)
if response.status_code == 200:
with open(zip_file, 'wb') as file:
file.write(response.content)
else:
print("{} down failed".format(down_url))
with tarfile.open(zip_file, 'r:gz') as tar:
tar.extractall(path=zip_file_extrpath)
print("The {} download successful.".format(down_url))
if os.path.exists(zip_file_extrpath):
shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages'))
# zip_file = 'm5stack_dist-packages.tar.gz'
# down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz'
# zip_file_extrpath = 'm5stack_dist-packages'
# if not os.path.exists(zip_file_extrpath):
# # Downloading via HTTP (more common)
# if not os.path.exists(zip_file):
# response = requests.get(down_url)
# if response.status_code == 200:
# with open(zip_file, 'wb') as file:
# file.write(response.content)
# else:
# print("{} down failed".format(down_url))
# with tarfile.open(zip_file, 'r:gz') as tar:
# tar.extractall(path=zip_file_extrpath)
# print("The {} download successful.".format(down_url))
# if os.path.exists(zip_file_extrpath):
# shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages'))
os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True)
with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f:
@@ -238,6 +238,18 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'):
openai_api_dir = os.path.join(src_folder, 'openai-api')
if os.path.exists(openai_api_dir):
shutil.copytree(openai_api_dir, os.path.join(deb_folder, 'opt/m5stack/lib/openai-api'))
if package_name == 'llm-kws':
sherpa_dir = os.path.join(src_folder, 'sherpa-onnx')
if os.path.exists(sherpa_dir):
shutil.copytree(sherpa_dir, os.path.join(deb_folder, 'opt/m5stack/lib/sherpa-onnx'))
if package_name == 'llm-llm':
llm_dir = os.path.join(src_folder, 'llm')
if os.path.exists(llm_dir):
shutil.copytree(llm_dir, os.path.join(deb_folder, 'opt/m5stack/lib/llm'))
if package_name == 'llm-vlm':
vlm_dir = os.path.join(src_folder, 'vlm')
if os.path.exists(vlm_dir):
shutil.copytree(vlm_dir, os.path.join(deb_folder, 'opt/m5stack/lib/vlm'))
shutil.copy2(os.path.join(src_folder, package_name.replace("-", "_")), os.path.join(deb_folder, 'opt/m5stack/bin', package_name.replace("-", "_")))
ext_scripts_files = glob.glob(os.path.join(src_folder, package_name + "_*"))
if ext_scripts_files:
@@ -253,7 +265,8 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1'):
f.write(f'Original-Maintainer: m5stack <m5stack@m5stack.com>\n')
f.write(f'Section: llm-module\n')
f.write(f'Priority: optional\n')
f.write(f'Depends: lib-llm\n')
# f.write(f'Depends: lib-llm\n')
f.write(f'Depends: lib-llm (>= 1.7)\n')
f.write(f'Homepage: https://www.m5stack.com\n')
f.write(f'Description: llm-module\n')
f.write(f' bsp.\n')
@@ -363,22 +376,22 @@ if __name__ == "__main__":
#################################################注意################################################
#################################################注意################################################
Tasks = {
'lib-llm':[create_lib_deb,'lib-llm', 1.6, src_folder, revision],
'lib-llm':[create_lib_deb,'lib-llm', 1.7, src_folder, revision],
'llm-sys':[create_bin_deb,'llm-sys', version, src_folder, revision],
'llm-audio':[create_bin_deb,'llm-audio', version, src_folder, revision],
'llm-kws':[create_bin_deb,'llm-kws', version, src_folder, revision],
'llm-kws':[create_bin_deb,'llm-kws', '1.6', src_folder, revision],
'llm-asr':[create_bin_deb,'llm-asr', version, src_folder, revision],
'llm-llm':[create_bin_deb,'llm-llm', '1.6', src_folder, revision],
'llm-llm':[create_bin_deb,'llm-llm', '1.7', src_folder, revision],
'llm-tts':[create_bin_deb,'llm-tts', version, src_folder, revision],
'llm-melotts':[create_bin_deb,'llm-melotts', version, src_folder, revision],
'llm-camera':[create_bin_deb,'llm-camera', '1.6', src_folder, revision],
'llm-vlm':[create_bin_deb,'llm-vlm', version, src_folder, revision],
'llm-vlm':[create_bin_deb,'llm-vlm', '1.6', src_folder, revision],
'llm-yolo':[create_bin_deb,'llm-yolo', '1.6', src_folder, revision],
'llm-skel':[create_bin_deb,'llm-skel', version, src_folder, revision],
'llm-depth-anything':[create_bin_deb,'llm-depth-anything', version, src_folder, revision],
'llm-vad':[create_bin_deb,'llm-vad', version, src_folder, revision],
'llm-whisper':[create_bin_deb,'llm-whisper', version, src_folder, revision],
'llm-openai-api':[create_bin_deb,'llm-openai-api', version, src_folder, revision],
'llm-openai-api':[create_bin_deb,'llm-openai-api', '1.6', src_folder, revision],
'llm-model-audio-en-us':[create_data_deb,'llm-model-audio-en-us', data_version, src_folder, revision],
'llm-model-audio-zh-cn':[create_data_deb,'llm-model-audio-zh-cn', data_version, src_folder, revision],
'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17', data_version, src_folder, revision],
@@ -407,8 +420,11 @@ if __name__ == "__main__":
'llm-model-llama3.2-1B-p256-ax630c':[create_data_deb,'llm-model-llama3.2-1B-p256-ax630c', '0.4', src_folder, revision],
'llm-model-openbuddy-llama3.2-1B-ax630c':[create_data_deb,'llm-model-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision],
'llm-model-internvl2.5-1B-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-ax630c', '0.4', src_folder, revision],
'llm-model-internvl2.5-1B-364-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-364-ax630c', '0.4', src_folder, revision],
'llm-model-deepseek-r1-1.5B-ax630c':[create_data_deb,'llm-model-deepseek-r1-1.5B-ax630c', '0.3', src_folder, revision],
'llm-model-deepseek-r1-1.5B-p256-ax630c':[create_data_deb,'llm-model-deepseek-r1-1.5B-p256-ax630c', '0.4', src_folder, revision],
'llm-model-smolvlm-256M-ax630c':[create_data_deb,'llm-model-smolvlm-256M-ax630c', '0.4', src_folder, revision],
'llm-model-smolvlm-500M-ax630c':[create_data_deb,'llm-model-smolvlm-500M-ax630c', '0.4', src_folder, revision],
# 'llm-model-qwen2-0.5B-prefill-20e':[create_data_deb,'llm-model-qwen2-0.5B-prefill-20e', data_version, src_folder, revision],
# 'llm-model-qwen2-1.5B-prefill-20e':[create_data_deb,'llm-model-qwen2-1.5B-prefill-20e', data_version, src_folder, revision]
}