m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] update model list

Browse Source
This commit is contained in:
LittleMouse
2025-09-04 14:52:54 +08:00
parent b6d6e95eef
commit 1df8ab9d7b
9 changed files with 313 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax630c.json
+32 -24
View File
@@ -1,38 +1,46 @@
{
"mode":"qwen2.5-HA-0.5B-ctx-ax630c",
"type":"llm",
"homepage":"https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha",
"compile_flage":"pulsar2 llm_build --input_path Qwen/qwen2.5-0.5b-ha --output_path Qwen/qwen2.5-0.5B-p1024-ha-ax630c --hidden_state_type bf16 --prefill_len 128 --kv_cache_len 1280 --last_kv_cache_len 128 --last_kv_cache_len 512 --last_kv_cache_len 1024 --chip AX620E --parallel 24",
"pulsar_version":"4.1-patch1-c37957c7",
"capabilities":[
"mode": "qwen2.5-HA-0.5B-ctx-ax630c",
"type": "llm",
"homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha",
"compile_flage": "pulsar2 llm_build --input_path Qwen/qwen2.5-0.5b-ha --output_path Qwen/qwen2.5-0.5B-p1024-ha-ax630c --hidden_state_type bf16 --prefill_len 128 --kv_cache_len 1280 --last_kv_cache_len 128 --last_kv_cache_len 512 --last_kv_cache_len 1024 --chip AX620E --parallel 24",
"pulsar_version": "4.1-patch1-c37957c7",
"capabilities": [
"text_generation",
"chat"
],
"input_type":[
"input_type": [
"llm.utf-8",
"llm.utf-8.stream",
"llm.chat_completion",
"llm.chat_completion.stream"
],
"output_type":[
"output_type": [
"llm.utf-8",
"llm.utf-8.stream"
],
"mode_param":{
"tokenizer_type":2,
"url_tokenizer_model":"http://localhost:8080",
"filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
"filename_post_axmodel":"qwen2_post.axmodel",
"template_filename_axmodel":"qwen2_p128_l%d_together.axmodel",
"b_use_topk":false,
"b_bos":false,
"b_eos":false,
"axmodel_num":24,
"tokens_embed_num":151936,
"tokens_embed_size":896,
"b_use_mmap_load_embed":true,
"b_dynamic_load_axmodel_layer":false,
"precompute_len":1202,
"ext_scripts":["tokenizer_qwen2.5-HA-0.5B-ctx-ax630c.py"]
"mode_param": {
"tokenizer_type": 2,
"url_tokenizer_model": "http://localhost:8080",
"filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin",
"filename_post_axmodel": "qwen2_post.axmodel",
"template_filename_axmodel": "qwen2_p128_l%d_together.axmodel",
"enable_temperature": true,
"temperature": 0.7,
"enable_top_p_sampling": false,
"top_p": 0.9,
"enable_top_k_sampling": true,
"top_k": 40,
"enable_repetition_penalty": false,
"repetition_penalty": 1.1,
"penalty_window": 50,
"axmodel_num": 24,
"tokens_embed_num": 151936,
"tokens_embed_size": 896,
"b_use_mmap_load_embed": true,
"precompute_len": 1024,
"cmm_size": 1840108,
"ext_scripts": [
"tokenizer_qwen2.5-HA-0.5B-ctx-ax630c.py"
]
}
}
projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax650.json
+46
View File
@@ -0,0 +1,46 @@
{
"mode": "qwen2.5-HA-0.5B-ctx-ax650",
"type": "llm",
"homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha",
"compile_flage": "pulsar2 llm_build --input_path Qwen/qwen2.5-0.5b-ha --output_path Qwen/qwen2.5-0.5B-p1024-ha-ax650 --hidden_state_type bf16 --prefill_len 128 --kv_cache_len 1280 --last_kv_cache_len 128 --last_kv_cache_len 512 --last_kv_cache_len 1024 --chip AX650 --parallel 24",
"pulsar_version": "4.1-patch1-c37957c7",
"capabilities": [
"text_generation",
"chat"
],
"input_type": [
"llm.utf-8",
"llm.utf-8.stream",
"llm.chat_completion",
"llm.chat_completion.stream"
],
"output_type": [
"llm.utf-8",
"llm.utf-8.stream"
],
"mode_param": {
"tokenizer_type": 2,
"url_tokenizer_model": "http://localhost:8080",
"filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin",
"filename_post_axmodel": "qwen2_post.axmodel",
"template_filename_axmodel": "qwen2_p128_l%d_together.axmodel",
"enable_temperature": true,
"temperature": 0.7,
"enable_top_p_sampling": false,
"top_p": 0.9,
"enable_top_k_sampling": true,
"top_k": 40,
"enable_repetition_penalty": false,
"repetition_penalty": 1.1,
"penalty_window": 50,
"axmodel_num": 24,
"tokens_embed_num": 151936,
"tokens_embed_size": 896,
"b_use_mmap_load_embed": true,
"precompute_len": 1024,
"cmm_size": 730656,
"ext_scripts": [
"tokenizer_qwen2.5-HA-0.5B-ctx-ax650.py"
]
}
}
projects/llm_framework/main_llm/scripts/tokenizer_qwen2.5-HA-0.5B-ctx-ax650.py
+203
View File
@@ -0,0 +1,203 @@
from transformers import AutoTokenizer, PreTrainedTokenizerFast
from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import argparse
import uuid
# 全局字典：存储 uid 到 Tokenizer_Http 实例的映射
tokenizers = {}
class Tokenizer_Http():
def __init__(self, model_id):
self.tokenizer = AutoTokenizer.from_pretrained(model_id)
self.messages = [
{"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
]
self.token_ids = []
self.token_ids_cache = []
def encode(self, prompt, last_reply=None):
if last_reply is not None:
self.messages.append({"role": "assistant", "content": last_reply})
text = self.tokenizer.apply_chat_template(
self.messages,
tokenize=False,
add_generation_prompt=True
)
# print("生成的文本:\n============\n", text, "============\n")
self.token_ids = self.tokenizer.encode(text)[:-3]
self.messages.append({"role": "user", "content": prompt})
text = self.tokenizer.apply_chat_template(
self.messages,
tokenize=False,
add_generation_prompt=True
)
print("生成的文本:\n============\n", text, "============\n")
token_ids = self.tokenizer.encode(text)
# 找出新增部分
diff = token_ids[len(self.token_ids):]
self.token_ids = token_ids
print(self.decode(diff))
return token_ids, diff
def decode(self, token_ids):
self.token_ids_cache += token_ids
text = self.tokenizer.decode(self.token_ids_cache)
if "\ufffd" in text:
print("text 中包含非法字符")
return ""
else:
self.token_ids_cache.clear()
return text
@property
def bos_id(self):
return self.tokenizer.bos_token_id
@property
def eos_id(self):
return self.tokenizer.eos_token_id
@property
def bos_token(self):
return self.tokenizer.bos_token
@property
def eos_token(self):
return self.tokenizer.eos_token
def reset(self, system_prompt=None):
if system_prompt is None:
system_prompt = args.content
self.messages = [
{"role": "system", "content": system_prompt},
]
text = self.tokenizer.apply_chat_template(
self.messages,
tokenize=False,
add_generation_prompt=True
)
token_ids = self.tokenizer.encode(text)[:-3]
self.token_ids = token_ids
print(self.decode(token_ids))
return token_ids
class Request(BaseHTTPRequestHandler):
timeout = 5
server_version = 'Apache'
def do_GET(self):
print("GET 请求路径:", self.path)
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
# 新增接口：获取 uid
if '/get_uid' in self.path:
new_uid = str(uuid.uuid4())
print("新 uid:", new_uid)
# 为该 uid 创建一个新的 Tokenizer_Http 实例
tokenizers[new_uid] = Tokenizer_Http(args.model_id)
msg = json.dumps({'uid': new_uid})
elif '/bos_id' in self.path:
# 获取 uid 参数（例如 ?uid=xxx）
uid = self.get_query_param("uid")
instance: Tokenizer_Http = tokenizers.get(uid)
if instance is None:
msg = json.dumps({'error': 'Invalid uid'})
else:
bos_id = instance.bos_id
msg = json.dumps({'bos_id': bos_id if bos_id is not None else -1})
elif '/eos_id' in self.path:
uid = self.get_query_param("uid")
instance: Tokenizer_Http = tokenizers.get(uid)
if instance is None:
msg = json.dumps({'error': 'Invalid uid'})
else:
eos_id = instance.eos_id
msg = json.dumps({'eos_id': eos_id if eos_id is not None else -1})
else:
msg = json.dumps({'error': 'Invalid GET endpoint'})
print("响应消息:", msg)
self.wfile.write(msg.encode())
def do_POST(self):
content_length = int(self.headers.get('content-length', 0))
data = self.rfile.read(content_length).decode()
print("POST 请求路径:", self.path)
print("接收到的数据:", data)
req = json.loads(data)
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
if '/encode' in self.path:
# 请求数据中必须包含 uid, text, 和可选的 last_reply
uid = req.get('uid')
prompt = req.get('text')
last_reply = req.get('last_reply')
instance: Tokenizer_Http = tokenizers.get(uid)
if instance is None:
msg = json.dumps({'error': 'Invalid uid'})
else:
token_ids, diff = instance.encode(prompt, last_reply)
msg = json.dumps({'token_ids': token_ids, 'diff': diff})
elif '/decode' in self.path:
uid = req.get('uid')
token_ids = req.get('token_ids')
instance: Tokenizer_Http = tokenizers.get(uid)
if instance is None:
msg = json.dumps({'error': 'Invalid uid'})
else:
text = instance.decode(token_ids)
msg = json.dumps({'text': text})
elif '/reset' in self.path:
uid = req.get("uid")
system_prompt = req.get("system_prompt")
instance: Tokenizer_Http = tokenizers.get(uid)
if instance is None:
msg = json.dumps({'error': 'Invalid uid'})
else:
if system_prompt is not None:
print("system_prompt:", system_prompt)
token_ids = instance.reset(system_prompt)
msg = json.dumps({'token_ids': token_ids})
else:
token_ids = instance.reset()
msg = json.dumps({'token_ids': token_ids})
else:
msg = json.dumps({'error': 'Invalid POST endpoint'})
print("响应消息:", msg)
self.wfile.write(msg.encode())
def get_query_param(self, key):
"""
辅助函数：从 GET 请求的 URL 中获取查询参数的值
例如：/bos_id?uid=xxx
"""
from urllib.parse import urlparse, parse_qs
query = urlparse(self.path).query
params = parse_qs(query)
values = params.get(key)
return values[0] if values else None
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--host', type=str, default='0.0.0.0')
parser.add_argument('--port', type=int, default=12345)
parser.add_argument('--model_id', type=str, default='qwen3_1.7B_tokenizer')
parser.add_argument('--content', type=str, default='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.')
args = parser.parse_args()
host = (args.host, args.port)
print('Server running at http://%s:%s' % host)
server = HTTPServer(host, Request)
server.serve_forever()
projects/llm_framework/main_vlm/models/mode_internvl3-1B-ax630c.json → projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax630c.json
+13 -6
View File
@@ -1,5 +1,5 @@
{
"mode": "internvl3-1B-ax630c",
"mode": "internvl3-1B-448-ax630c",
"type": "vlm",
"homepage": "https://huggingface.co/AXERA-TECH/InternVL3-1B",
"capabilities": [
@@ -20,18 +20,25 @@
"filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin",
"filename_post_axmodel": "qwen2_post.axmodel",
"template_filename_axmodel": "qwen2_p128_l%d_together.axmodel",
"filename_image_encoder_axmodedl": "internvl3_1b_vit.axmodel",
"b_bos": false,
"b_eos": false,
"filename_image_encoder_axmodel": "internvl3_1b_vit.axmodel",
"enable_temperature": true,
"temperature": 0.7,
"enable_top_p_sampling": false,
"top_p": 0.9,
"enable_top_k_sampling": true,
"top_k": 40,
"enable_repetition_penalty": false,
"repetition_penalty": 1.1,
"penalty_window": 50,
"axmodel_num": 24,
"tokens_embed_num": 151674,
"img_token_id": 151667,
"tokens_embed_size": 896,
"b_use_mmap_load_embed": true,
"b_dynamic_load_axmodel_layer": false,
"precompute_len": 1024,
"cmm_size": 1674992,
"ext_scripts": [
"tokenizer_internvl3-1B-ax630c.py"
"tokenizer_internvl3-1B-448-ax630c.py"
]
}
}
projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax650.json
+9 -4
View File
@@ -21,15 +21,20 @@
"filename_post_axmodel": "qwen2_post.axmodel",
"template_filename_axmodel": "qwen2_p128_l%d_together.axmodel",
"filename_image_encoder_axmodel": "internvl3_1b_vit.axmodel",
"b_use_topk": false,
"b_bos": false,
"b_eos": false,
"enable_temperature": true,
"temperature": 0.7,
"enable_top_p_sampling": false,
"top_p": 0.9,
"enable_top_k_sampling": true,
"top_k": 40,
"enable_repetition_penalty": false,
"repetition_penalty": 1.1,
"penalty_window": 50,
"axmodel_num": 24,
"tokens_embed_num": 151674,
"img_token_id": 151667,
"tokens_embed_size": 896,
"b_use_mmap_load_embed": true,
"b_dynamic_load_axmodel_layer": false,
"precompute_len": 2048,
"cmm_size": 1919044,
"ext_scripts": [
projects/llm_framework/main_vlm/scripts/tokenizer_internvl3-1B-ax630c.py → projects/llm_framework/main_vlm/scripts/tokenizer_internvl3-1B-448-ax630c.py
View File
projects/llm_framework/main_vlm/src/main.cpp
+2 -2
View File
@@ -154,7 +154,7 @@ public:
CONFIG_AUTO_SET(file_body["mode_param"], url_tokenizer_model);
CONFIG_AUTO_SET(file_body["mode_param"], filename_tokens_embed);
CONFIG_AUTO_SET(file_body["mode_param"], filename_post_axmodel);
CONFIG_AUTO_SET(file_body["mode_param"], filename_vpm_resampler_axmodel);
CONFIG_AUTO_SET(file_body["mode_param"], filename_vpm_resampler_axmodedl);
CONFIG_AUTO_SET(file_body["mode_param"], filename_image_encoder_axmodel);
CONFIG_AUTO_SET(file_body["mode_param"], template_filename_axmodel);
CONFIG_AUTO_SET(file_body["mode_param"], b_use_topk);
@@ -238,7 +238,7 @@ public:
mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed;
mode_config_.filename_post_axmodel = base_model + mode_config_.filename_post_axmodel;
mode_config_.template_filename_axmodel = base_model + mode_config_.template_filename_axmodel;
mode_config_.filename_vpm_resampler_axmodel = base_model + mode_config_.filename_vpm_resampler_axmodel;
mode_config_.filename_vpm_resampler_axmodedl = base_model + mode_config_.filename_vpm_resampler_axmodedl;
mode_config_.filename_image_encoder_axmodel = base_model + mode_config_.filename_image_encoder_axmodel;
mode_config_.runing_callback = [this](int *p_token, int n_token, const char *p_str, float token_per_sec,
void *reserve) {
projects/llm_framework/main_vlm/src/runner/LLM.hpp
+5 -5
View File
@@ -26,7 +26,7 @@ struct LLMAttrType {
std::string filename_post_axmodel = "tinyllama-int8/tinyllama_post.axmodel";
std::string filename_image_encoder_axmodel = "minicpmv/vpm_resampler_version0_fp16.axmodel";
std::string filename_vpm_encoder_axmodel = "minicpmv/vpm_resampler_version0_fp16.axmodel";
std::string filename_vpm_resampler_axmodel = "minicpmv/vpm_resampler_version0_fp16.axmodel";
std::string filename_vpm_resampler_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel";
int image_encoder_width = 448;
int image_encoder_height = 448;
@@ -190,18 +190,18 @@ public:
return false;
}
ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodel.c_str(), false);
ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodedl.c_str(), false);
if (ret != 0) {
ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodel.c_str());
ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodedl.c_str());
return false;
}
_attr.vpm_height = vpm_encoder.get_input(0).vShape[1];
_attr.vpm_width = vpm_encoder.get_input(0).vShape[2];
} else {
ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodel.c_str(), false);
ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodedl.c_str(), false);
if (ret != 0) {
ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodel.c_str());
ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodedl.c_str());
return false;
}
_attr.vpm_height = vpm_resampler.get_input(0).vShape[1];
projects/llm_framework/tools/llm_pack.py
+3
View File
@@ -453,8 +453,11 @@ if __name__ == "__main__":
'llm-model-llama3.2-1B-p256-ax630c':[create_data_deb,'llm-model-llama3.2-1B-p256-ax630c', '0.4', src_folder, revision],
'llm-model-openbuddy-llama3.2-1B-ax630c':[create_data_deb,'llm-model-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision],
# InternVL model
## AX630C
'llm-model-internvl2.5-1B-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-ax630c', '0.4', src_folder, revision],
'llm-model-internvl2.5-1B-364-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-364-ax630c', '0.4', src_folder, revision],
'llm-model-internvl3-1B-448-ax630c':[create_data_deb,'llm-model-internvl3-1B-448-ax630c', '0.6', src_folder, revision],
## AX650
'llm-model-internvl3-1B-448-ax650':[create_data_deb,'llm-model-internvl3-1B-448-ax650', '0.6', src_folder, revision],
# DeepSeek model
## AX630C