diff --git a/projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax630c.json b/projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax630c.json index 1481d45..83d7cb7 100644 --- a/projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax630c.json +++ b/projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax630c.json @@ -1,38 +1,46 @@ { - "mode":"qwen2.5-HA-0.5B-ctx-ax630c", - "type":"llm", - "homepage":"https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha", - "compile_flage":"pulsar2 llm_build --input_path Qwen/qwen2.5-0.5b-ha --output_path Qwen/qwen2.5-0.5B-p1024-ha-ax630c --hidden_state_type bf16 --prefill_len 128 --kv_cache_len 1280 --last_kv_cache_len 128 --last_kv_cache_len 512 --last_kv_cache_len 1024 --chip AX620E --parallel 24", - "pulsar_version":"4.1-patch1-c37957c7", - "capabilities":[ + "mode": "qwen2.5-HA-0.5B-ctx-ax630c", + "type": "llm", + "homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha", + "compile_flage": "pulsar2 llm_build --input_path Qwen/qwen2.5-0.5b-ha --output_path Qwen/qwen2.5-0.5B-p1024-ha-ax630c --hidden_state_type bf16 --prefill_len 128 --kv_cache_len 1280 --last_kv_cache_len 128 --last_kv_cache_len 512 --last_kv_cache_len 1024 --chip AX620E --parallel 24", + "pulsar_version": "4.1-patch1-c37957c7", + "capabilities": [ "text_generation", "chat" ], - "input_type":[ + "input_type": [ "llm.utf-8", "llm.utf-8.stream", "llm.chat_completion", "llm.chat_completion.stream" ], - "output_type":[ + "output_type": [ "llm.utf-8", "llm.utf-8.stream" ], - "mode_param":{ - "tokenizer_type":2, - "url_tokenizer_model":"http://localhost:8080", - "filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin", - "filename_post_axmodel":"qwen2_post.axmodel", - "template_filename_axmodel":"qwen2_p128_l%d_together.axmodel", - "b_use_topk":false, - "b_bos":false, - "b_eos":false, - "axmodel_num":24, - "tokens_embed_num":151936, - "tokens_embed_size":896, - "b_use_mmap_load_embed":true, - "b_dynamic_load_axmodel_layer":false, - "precompute_len":1202, - "ext_scripts":["tokenizer_qwen2.5-HA-0.5B-ctx-ax630c.py"] + "mode_param": { + "tokenizer_type": 2, + "url_tokenizer_model": "http://localhost:8080", + "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin", + "filename_post_axmodel": "qwen2_post.axmodel", + "template_filename_axmodel": "qwen2_p128_l%d_together.axmodel", + "enable_temperature": true, + "temperature": 0.7, + "enable_top_p_sampling": false, + "top_p": 0.9, + "enable_top_k_sampling": true, + "top_k": 40, + "enable_repetition_penalty": false, + "repetition_penalty": 1.1, + "penalty_window": 50, + "axmodel_num": 24, + "tokens_embed_num": 151936, + "tokens_embed_size": 896, + "b_use_mmap_load_embed": true, + "precompute_len": 1024, + "cmm_size": 1840108, + "ext_scripts": [ + "tokenizer_qwen2.5-HA-0.5B-ctx-ax630c.py" + ] } } \ No newline at end of file diff --git a/projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax650.json b/projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax650.json new file mode 100644 index 0000000..c54669f --- /dev/null +++ b/projects/llm_framework/main_llm/models/mode_qwen2.5-HA-0.5B-ctx-ax650.json @@ -0,0 +1,46 @@ +{ + "mode": "qwen2.5-HA-0.5B-ctx-ax650", + "type": "llm", + "homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha", + "compile_flage": "pulsar2 llm_build --input_path Qwen/qwen2.5-0.5b-ha --output_path Qwen/qwen2.5-0.5B-p1024-ha-ax650 --hidden_state_type bf16 --prefill_len 128 --kv_cache_len 1280 --last_kv_cache_len 128 --last_kv_cache_len 512 --last_kv_cache_len 1024 --chip AX650 --parallel 24", + "pulsar_version": "4.1-patch1-c37957c7", + "capabilities": [ + "text_generation", + "chat" + ], + "input_type": [ + "llm.utf-8", + "llm.utf-8.stream", + "llm.chat_completion", + "llm.chat_completion.stream" + ], + "output_type": [ + "llm.utf-8", + "llm.utf-8.stream" + ], + "mode_param": { + "tokenizer_type": 2, + "url_tokenizer_model": "http://localhost:8080", + "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin", + "filename_post_axmodel": "qwen2_post.axmodel", + "template_filename_axmodel": "qwen2_p128_l%d_together.axmodel", + "enable_temperature": true, + "temperature": 0.7, + "enable_top_p_sampling": false, + "top_p": 0.9, + "enable_top_k_sampling": true, + "top_k": 40, + "enable_repetition_penalty": false, + "repetition_penalty": 1.1, + "penalty_window": 50, + "axmodel_num": 24, + "tokens_embed_num": 151936, + "tokens_embed_size": 896, + "b_use_mmap_load_embed": true, + "precompute_len": 1024, + "cmm_size": 730656, + "ext_scripts": [ + "tokenizer_qwen2.5-HA-0.5B-ctx-ax650.py" + ] + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_llm/scripts/tokenizer_qwen2.5-HA-0.5B-ctx-ax650.py b/projects/llm_framework/main_llm/scripts/tokenizer_qwen2.5-HA-0.5B-ctx-ax650.py new file mode 100644 index 0000000..4a63e88 --- /dev/null +++ b/projects/llm_framework/main_llm/scripts/tokenizer_qwen2.5-HA-0.5B-ctx-ax650.py @@ -0,0 +1,203 @@ +from transformers import AutoTokenizer, PreTrainedTokenizerFast +from http.server import HTTPServer, BaseHTTPRequestHandler +import json +import argparse +import uuid + +# 全局字典：存储 uid 到 Tokenizer_Http 实例的映射 +tokenizers = {} + +class Tokenizer_Http(): + def __init__(self, model_id): + self.tokenizer = AutoTokenizer.from_pretrained(model_id) + self.messages = [ + {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."}, + ] + self.token_ids = [] + + self.token_ids_cache = [] + + def encode(self, prompt, last_reply=None): + if last_reply is not None: + self.messages.append({"role": "assistant", "content": last_reply}) + text = self.tokenizer.apply_chat_template( + self.messages, + tokenize=False, + add_generation_prompt=True + ) + # print("生成的文本:\n============\n", text, "============\n") + self.token_ids = self.tokenizer.encode(text)[:-3] + self.messages.append({"role": "user", "content": prompt}) + + text = self.tokenizer.apply_chat_template( + self.messages, + tokenize=False, + add_generation_prompt=True + ) + print("生成的文本:\n============\n", text, "============\n") + token_ids = self.tokenizer.encode(text) + # 找出新增部分 + diff = token_ids[len(self.token_ids):] + self.token_ids = token_ids + print(self.decode(diff)) + return token_ids, diff + + def decode(self, token_ids): + self.token_ids_cache += token_ids + text = self.tokenizer.decode(self.token_ids_cache) + if "\ufffd" in text: + print("text 中包含非法字符") + return "" + else: + self.token_ids_cache.clear() + return text + + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def bos_token(self): + return self.tokenizer.bos_token + + @property + def eos_token(self): + return self.tokenizer.eos_token + + def reset(self, system_prompt=None): + if system_prompt is None: + system_prompt = args.content + self.messages = [ + {"role": "system", "content": system_prompt}, + ] + text = self.tokenizer.apply_chat_template( + self.messages, + tokenize=False, + add_generation_prompt=True + ) + token_ids = self.tokenizer.encode(text)[:-3] + self.token_ids = token_ids + print(self.decode(token_ids)) + return token_ids + + +class Request(BaseHTTPRequestHandler): + timeout = 5 + server_version = 'Apache' + + def do_GET(self): + print("GET 请求路径:", self.path) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + + # 新增接口：获取 uid + if '/get_uid' in self.path: + new_uid = str(uuid.uuid4()) + print("新 uid:", new_uid) + # 为该 uid 创建一个新的 Tokenizer_Http 实例 + tokenizers[new_uid] = Tokenizer_Http(args.model_id) + msg = json.dumps({'uid': new_uid}) + elif '/bos_id' in self.path: + # 获取 uid 参数（例如 ?uid=xxx） + uid = self.get_query_param("uid") + instance: Tokenizer_Http = tokenizers.get(uid) + if instance is None: + msg = json.dumps({'error': 'Invalid uid'}) + else: + bos_id = instance.bos_id + msg = json.dumps({'bos_id': bos_id if bos_id is not None else -1}) + elif '/eos_id' in self.path: + uid = self.get_query_param("uid") + instance: Tokenizer_Http = tokenizers.get(uid) + if instance is None: + msg = json.dumps({'error': 'Invalid uid'}) + else: + eos_id = instance.eos_id + msg = json.dumps({'eos_id': eos_id if eos_id is not None else -1}) + else: + msg = json.dumps({'error': 'Invalid GET endpoint'}) + + print("响应消息:", msg) + self.wfile.write(msg.encode()) + + def do_POST(self): + content_length = int(self.headers.get('content-length', 0)) + data = self.rfile.read(content_length).decode() + print("POST 请求路径:", self.path) + print("接收到的数据:", data) + req = json.loads(data) + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + + if '/encode' in self.path: + # 请求数据中必须包含 uid, text, 和可选的 last_reply + uid = req.get('uid') + prompt = req.get('text') + last_reply = req.get('last_reply') + instance: Tokenizer_Http = tokenizers.get(uid) + if instance is None: + msg = json.dumps({'error': 'Invalid uid'}) + else: + token_ids, diff = instance.encode(prompt, last_reply) + msg = json.dumps({'token_ids': token_ids, 'diff': diff}) + elif '/decode' in self.path: + uid = req.get('uid') + token_ids = req.get('token_ids') + instance: Tokenizer_Http = tokenizers.get(uid) + if instance is None: + msg = json.dumps({'error': 'Invalid uid'}) + else: + text = instance.decode(token_ids) + msg = json.dumps({'text': text}) + elif '/reset' in self.path: + uid = req.get("uid") + system_prompt = req.get("system_prompt") + instance: Tokenizer_Http = tokenizers.get(uid) + if instance is None: + msg = json.dumps({'error': 'Invalid uid'}) + else: + if system_prompt is not None: + print("system_prompt:", system_prompt) + token_ids = instance.reset(system_prompt) + msg = json.dumps({'token_ids': token_ids}) + else: + token_ids = instance.reset() + msg = json.dumps({'token_ids': token_ids}) + else: + msg = json.dumps({'error': 'Invalid POST endpoint'}) + + print("响应消息:", msg) + self.wfile.write(msg.encode()) + + def get_query_param(self, key): + """ + 辅助函数：从 GET 请求的 URL 中获取查询参数的值 + 例如：/bos_id?uid=xxx + """ + from urllib.parse import urlparse, parse_qs + query = urlparse(self.path).query + params = parse_qs(query) + values = params.get(key) + return values[0] if values else None + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--host', type=str, default='0.0.0.0') + parser.add_argument('--port', type=int, default=12345) + parser.add_argument('--model_id', type=str, default='qwen3_1.7B_tokenizer') + parser.add_argument('--content', type=str, default='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.') + + args = parser.parse_args() + + host = (args.host, args.port) + print('Server running at http://%s:%s' % host) + server = HTTPServer(host, Request) + server.serve_forever() diff --git a/projects/llm_framework/main_vlm/models/mode_internvl3-1B-ax630c.json b/projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax630c.json similarity index 64% rename from projects/llm_framework/main_vlm/models/mode_internvl3-1B-ax630c.json rename to projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax630c.json index 50848e2..84d7611 100644 --- a/projects/llm_framework/main_vlm/models/mode_internvl3-1B-ax630c.json +++ b/projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax630c.json @@ -1,5 +1,5 @@ { - "mode": "internvl3-1B-ax630c", + "mode": "internvl3-1B-448-ax630c", "type": "vlm", "homepage": "https://huggingface.co/AXERA-TECH/InternVL3-1B", "capabilities": [ @@ -20,18 +20,25 @@ "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin", "filename_post_axmodel": "qwen2_post.axmodel", "template_filename_axmodel": "qwen2_p128_l%d_together.axmodel", - "filename_image_encoder_axmodedl": "internvl3_1b_vit.axmodel", - "b_bos": false, - "b_eos": false, + "filename_image_encoder_axmodel": "internvl3_1b_vit.axmodel", + "enable_temperature": true, + "temperature": 0.7, + "enable_top_p_sampling": false, + "top_p": 0.9, + "enable_top_k_sampling": true, + "top_k": 40, + "enable_repetition_penalty": false, + "repetition_penalty": 1.1, + "penalty_window": 50, "axmodel_num": 24, "tokens_embed_num": 151674, "img_token_id": 151667, "tokens_embed_size": 896, "b_use_mmap_load_embed": true, - "b_dynamic_load_axmodel_layer": false, "precompute_len": 1024, + "cmm_size": 1674992, "ext_scripts": [ - "tokenizer_internvl3-1B-ax630c.py" + "tokenizer_internvl3-1B-448-ax630c.py" ] } } \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax650.json b/projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax650.json index 4da7a27..9151975 100644 --- a/projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax650.json +++ b/projects/llm_framework/main_vlm/models/mode_internvl3-1B-448-ax650.json @@ -21,15 +21,20 @@ "filename_post_axmodel": "qwen2_post.axmodel", "template_filename_axmodel": "qwen2_p128_l%d_together.axmodel", "filename_image_encoder_axmodel": "internvl3_1b_vit.axmodel", - "b_use_topk": false, - "b_bos": false, - "b_eos": false, + "enable_temperature": true, + "temperature": 0.7, + "enable_top_p_sampling": false, + "top_p": 0.9, + "enable_top_k_sampling": true, + "top_k": 40, + "enable_repetition_penalty": false, + "repetition_penalty": 1.1, + "penalty_window": 50, "axmodel_num": 24, "tokens_embed_num": 151674, "img_token_id": 151667, "tokens_embed_size": 896, "b_use_mmap_load_embed": true, - "b_dynamic_load_axmodel_layer": false, "precompute_len": 2048, "cmm_size": 1919044, "ext_scripts": [ diff --git a/projects/llm_framework/main_vlm/scripts/tokenizer_internvl3-1B-ax630c.py b/projects/llm_framework/main_vlm/scripts/tokenizer_internvl3-1B-448-ax630c.py similarity index 100% rename from projects/llm_framework/main_vlm/scripts/tokenizer_internvl3-1B-ax630c.py rename to projects/llm_framework/main_vlm/scripts/tokenizer_internvl3-1B-448-ax630c.py diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index 36f278a..926976d 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -154,7 +154,7 @@ public: CONFIG_AUTO_SET(file_body["mode_param"], url_tokenizer_model); CONFIG_AUTO_SET(file_body["mode_param"], filename_tokens_embed); CONFIG_AUTO_SET(file_body["mode_param"], filename_post_axmodel); - CONFIG_AUTO_SET(file_body["mode_param"], filename_vpm_resampler_axmodel); + CONFIG_AUTO_SET(file_body["mode_param"], filename_vpm_resampler_axmodedl); CONFIG_AUTO_SET(file_body["mode_param"], filename_image_encoder_axmodel); CONFIG_AUTO_SET(file_body["mode_param"], template_filename_axmodel); CONFIG_AUTO_SET(file_body["mode_param"], b_use_topk); @@ -238,7 +238,7 @@ public: mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed; mode_config_.filename_post_axmodel = base_model + mode_config_.filename_post_axmodel; mode_config_.template_filename_axmodel = base_model + mode_config_.template_filename_axmodel; - mode_config_.filename_vpm_resampler_axmodel = base_model + mode_config_.filename_vpm_resampler_axmodel; + mode_config_.filename_vpm_resampler_axmodedl = base_model + mode_config_.filename_vpm_resampler_axmodedl; mode_config_.filename_image_encoder_axmodel = base_model + mode_config_.filename_image_encoder_axmodel; mode_config_.runing_callback = [this](int *p_token, int n_token, const char *p_str, float token_per_sec, void *reserve) { diff --git a/projects/llm_framework/main_vlm/src/runner/LLM.hpp b/projects/llm_framework/main_vlm/src/runner/LLM.hpp index ff23247..ecb2196 100644 --- a/projects/llm_framework/main_vlm/src/runner/LLM.hpp +++ b/projects/llm_framework/main_vlm/src/runner/LLM.hpp @@ -26,7 +26,7 @@ struct LLMAttrType { std::string filename_post_axmodel = "tinyllama-int8/tinyllama_post.axmodel"; std::string filename_image_encoder_axmodel = "minicpmv/vpm_resampler_version0_fp16.axmodel"; std::string filename_vpm_encoder_axmodel = "minicpmv/vpm_resampler_version0_fp16.axmodel"; - std::string filename_vpm_resampler_axmodel = "minicpmv/vpm_resampler_version0_fp16.axmodel"; + std::string filename_vpm_resampler_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel"; int image_encoder_width = 448; int image_encoder_height = 448; @@ -190,18 +190,18 @@ public: return false; } - ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodel.c_str(), false); + ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodedl.c_str(), false); if (ret != 0) { - ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodel.c_str()); + ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodedl.c_str()); return false; } _attr.vpm_height = vpm_encoder.get_input(0).vShape[1]; _attr.vpm_width = vpm_encoder.get_input(0).vShape[2]; } else { - ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodel.c_str(), false); + ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodedl.c_str(), false); if (ret != 0) { - ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodel.c_str()); + ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodedl.c_str()); return false; } _attr.vpm_height = vpm_resampler.get_input(0).vShape[1]; diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index fc0097d..5f8ab57 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -453,8 +453,11 @@ if __name__ == "__main__": 'llm-model-llama3.2-1B-p256-ax630c':[create_data_deb,'llm-model-llama3.2-1B-p256-ax630c', '0.4', src_folder, revision], 'llm-model-openbuddy-llama3.2-1B-ax630c':[create_data_deb,'llm-model-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision], # InternVL model + ## AX630C 'llm-model-internvl2.5-1B-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-ax630c', '0.4', src_folder, revision], 'llm-model-internvl2.5-1B-364-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-364-ax630c', '0.4', src_folder, revision], + 'llm-model-internvl3-1B-448-ax630c':[create_data_deb,'llm-model-internvl3-1B-448-ax630c', '0.6', src_folder, revision], + ## AX650 'llm-model-internvl3-1B-448-ax650':[create_data_deb,'llm-model-internvl3-1B-448-ax650', '0.6', src_folder, revision], # DeepSeek model ## AX630C