[update] ModuleLLM support ctx model, add HomeAssistant model, add model post process config.

2026-05-20 11:32:11 -07:00 · 2025-08-22 17:38:39 +08:00
parent a916ca04d6
commit 73c4a49b05
10 changed files with 1826 additions and 489 deletions
@@ -0,0 +1,38 @@
+{
+    "mode":"qwen2.5-HA-0.5B-ctx-ax630c",
+    "type":"llm",
+    "homepage":"https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha",
+    "compile_flage":"pulsar2 llm_build --input_path Qwen/qwen2.5-0.5b-ha --output_path Qwen/qwen2.5-0.5B-p1024-ha-ax630c --hidden_state_type bf16 --prefill_len 128 --kv_cache_len 1280 --last_kv_cache_len 128 --last_kv_cache_len 512 --last_kv_cache_len 1024 --chip AX620E --parallel 24",
+    "pulsar_version":"4.1-patch1-c37957c7",
+    "capabilities":[
+        "text_generation",
+        "chat"
+    ],
+    "input_type":[
+        "llm.utf-8",
+        "llm.utf-8.stream",
+        "llm.chat_completion",
+        "llm.chat_completion.stream"
+    ],
+    "output_type":[
+        "llm.utf-8",
+        "llm.utf-8.stream"
+    ],
+    "mode_param":{
+        "tokenizer_type":2,
+        "url_tokenizer_model":"http://localhost:8080",
+        "filename_tokens_embed":"model.embed_tokens.weight.bfloat16.bin",
+        "filename_post_axmodel":"qwen2_post.axmodel",
+        "template_filename_axmodel":"qwen2_p128_l%d_together.axmodel",
+        "b_use_topk":false,
+        "b_bos":false,
+        "b_eos":false,
+        "axmodel_num":24,
+        "tokens_embed_num":151936,
+        "tokens_embed_size":896,
+        "b_use_mmap_load_embed":true,
+        "b_dynamic_load_axmodel_layer":false,
+        "precompute_len":1202,
+        "ext_scripts":["tokenizer_qwen2.5-HA-0.5B-ctx-ax630c.py"]
+    }
+}
@@ -0,0 +1,203 @@
+from transformers import AutoTokenizer, PreTrainedTokenizerFast
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import json
+import argparse
+import uuid
+
+# 全局字典：存储 uid 到 Tokenizer_Http 实例的映射
+tokenizers = {}
+
+class Tokenizer_Http():
+    def __init__(self, model_id):
+        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+        self.messages = [
+            {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
+        ]
+        self.token_ids = []
+        
+        self.token_ids_cache = []
+
+    def encode(self, prompt, last_reply=None):
+        if last_reply is not None:
+            self.messages.append({"role": "assistant", "content": last_reply})
+            text = self.tokenizer.apply_chat_template(
+                self.messages,
+                tokenize=False,
+                add_generation_prompt=True
+            )
+            # print("生成的文本:\n============\n", text, "============\n")
+            self.token_ids = self.tokenizer.encode(text)[:-3]
+        self.messages.append({"role": "user", "content": prompt})
+        
+        text = self.tokenizer.apply_chat_template(
+            self.messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+        print("生成的文本:\n============\n", text, "============\n")
+        token_ids = self.tokenizer.encode(text)
+        # 找出新增部分
+        diff = token_ids[len(self.token_ids):]
+        self.token_ids = token_ids
+        print(self.decode(diff))
+        return token_ids, diff
+
+    def decode(self, token_ids):
+        self.token_ids_cache += token_ids
+        text = self.tokenizer.decode(self.token_ids_cache)
+        if "\ufffd" in text:
+            print("text 中包含非法字符")
+            return ""
+        else:
+            self.token_ids_cache.clear()
+            return text
+        
+
+    @property
+    def bos_id(self):
+        return self.tokenizer.bos_token_id
+
+    @property
+    def eos_id(self):
+        return self.tokenizer.eos_token_id
+    
+    @property
+    def bos_token(self):
+        return self.tokenizer.bos_token
+
+    @property
+    def eos_token(self):
+        return self.tokenizer.eos_token
+    
+    def reset(self, system_prompt=None):
+        if system_prompt is None:
+            system_prompt = args.content
+        self.messages = [
+            {"role": "system", "content": system_prompt},
+        ]
+        text = self.tokenizer.apply_chat_template(
+            self.messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+        token_ids = self.tokenizer.encode(text)[:-3]
+        self.token_ids = token_ids
+        print(self.decode(token_ids))
+        return token_ids
+        
+
+class Request(BaseHTTPRequestHandler):
+    timeout = 5
+    server_version = 'Apache'
+
+    def do_GET(self):
+        print("GET 请求路径:", self.path)
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+
+        # 新增接口：获取 uid
+        if '/get_uid' in self.path:
+            new_uid = str(uuid.uuid4())
+            print("新 uid:", new_uid)
+            # 为该 uid 创建一个新的 Tokenizer_Http 实例
+            tokenizers[new_uid] = Tokenizer_Http(args.model_id)
+            msg = json.dumps({'uid': new_uid})
+        elif '/bos_id' in self.path:
+            # 获取 uid 参数（例如 ?uid=xxx）
+            uid = self.get_query_param("uid")
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                bos_id = instance.bos_id
+                msg = json.dumps({'bos_id': bos_id if bos_id is not None else -1})
+        elif '/eos_id' in self.path:
+            uid = self.get_query_param("uid")
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                eos_id = instance.eos_id
+                msg = json.dumps({'eos_id': eos_id if eos_id is not None else -1})
+        else:
+            msg = json.dumps({'error': 'Invalid GET endpoint'})
+
+        print("响应消息:", msg)
+        self.wfile.write(msg.encode())
+
+    def do_POST(self):
+        content_length = int(self.headers.get('content-length', 0))
+        data = self.rfile.read(content_length).decode()
+        print("POST 请求路径:", self.path)
+        print("接收到的数据:", data)
+        req = json.loads(data)
+
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+
+        if '/encode' in self.path:
+            # 请求数据中必须包含 uid, text, 和可选的 last_reply
+            uid = req.get('uid')
+            prompt = req.get('text')
+            last_reply = req.get('last_reply')
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                token_ids, diff = instance.encode(prompt, last_reply)
+                msg = json.dumps({'token_ids': token_ids, 'diff': diff})
+        elif '/decode' in self.path:
+            uid = req.get('uid')
+            token_ids = req.get('token_ids')
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                text = instance.decode(token_ids)
+                msg = json.dumps({'text': text})
+        elif '/reset' in self.path:
+            uid = req.get("uid")
+            system_prompt = req.get("system_prompt")
+            instance: Tokenizer_Http = tokenizers.get(uid)
+            if instance is None:
+                msg = json.dumps({'error': 'Invalid uid'})
+            else:
+                if system_prompt is not None:
+                    print("system_prompt:", system_prompt)
+                    token_ids = instance.reset(system_prompt)
+                    msg = json.dumps({'token_ids': token_ids})
+                else:
+                    token_ids = instance.reset()
+                    msg = json.dumps({'token_ids': token_ids})
+        else:
+            msg = json.dumps({'error': 'Invalid POST endpoint'})
+
+        print("响应消息:", msg)
+        self.wfile.write(msg.encode())
+
+    def get_query_param(self, key):
+        """
+        辅助函数：从 GET 请求的 URL 中获取查询参数的值
+        例如：/bos_id?uid=xxx
+        """
+        from urllib.parse import urlparse, parse_qs
+        query = urlparse(self.path).query
+        params = parse_qs(query)
+        values = params.get(key)
+        return values[0] if values else None
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--host', type=str, default='0.0.0.0')
+    parser.add_argument('--port', type=int, default=12345)
+    parser.add_argument('--model_id', type=str, default='qwen3_1.7B_tokenizer')
+    parser.add_argument('--content', type=str, default='You are Qwen, created by Alibaba Cloud. You are a helpful assistant.')
+
+    args = parser.parse_args()
+
+    host = (args.host, args.port)
+    print('Server running at http://%s:%s' % host)
+    server = HTTPServer(host, Request)
+    server.serve_forever()
@@ -55,10 +55,17 @@ public:
    enum inference_status { INFERENCE_NONE = 0, INFERENCE_RUNNING };
    LLMAttrType mode_config_;
    std::unique_ptr<LLM> lLaMa_;
+    std::unique_ptr<LLM_CTX> lLaMa_ctx_;
    std::string model_;
    std::string response_format_;
    std::vector<std::string> inputs_;
    std::string prompt_;
+    std::string last_reply;
+    std::vector<unsigned short> prompt_data;
+    std::vector<int> tokens_ids, tokens_diff;
+    std::vector<std::vector<unsigned short>> k_caches, v_caches;
+    int precompute_len = 0;
+    std::vector<int> _token_ids;
    task_callback_t out_callback_;
    bool enoutput_;
    bool enstream_;
@@ -125,10 +132,10 @@ public:

            CONFIG_AUTO_SET(file_body["mode_param"], tokenizer_type);
            CONFIG_AUTO_SET(file_body["mode_param"], filename_tokenizer_model);
+            CONFIG_AUTO_SET(file_body["mode_param"], url_tokenizer_model);
            CONFIG_AUTO_SET(file_body["mode_param"], filename_tokens_embed);
            CONFIG_AUTO_SET(file_body["mode_param"], filename_post_axmodel);
            CONFIG_AUTO_SET(file_body["mode_param"], template_filename_axmodel);
-            CONFIG_AUTO_SET(file_body["mode_param"], b_use_topk);
            CONFIG_AUTO_SET(file_body["mode_param"], b_bos);
            CONFIG_AUTO_SET(file_body["mode_param"], b_eos);
            CONFIG_AUTO_SET(file_body["mode_param"], axmodel_num);
@@ -137,61 +144,119 @@ public:
            CONFIG_AUTO_SET(file_body["mode_param"], b_use_mmap_load_embed);
            CONFIG_AUTO_SET(file_body["mode_param"], b_dynamic_load_axmodel_layer);
            CONFIG_AUTO_SET(file_body["mode_param"], max_token_len);
+            CONFIG_AUTO_SET(file_body["mode_param"], enable_temperature);
            CONFIG_AUTO_SET(file_body["mode_param"], temperature);
+            CONFIG_AUTO_SET(file_body["mode_param"], enable_top_p_sampling);
            CONFIG_AUTO_SET(file_body["mode_param"], top_p);
+            CONFIG_AUTO_SET(file_body["mode_param"], enable_top_k_sampling);
+            CONFIG_AUTO_SET(file_body["mode_param"], top_k);
+            CONFIG_AUTO_SET(file_body["mode_param"], enable_repetition_penalty);
+            CONFIG_AUTO_SET(file_body["mode_param"], repetition_penalty);
+            CONFIG_AUTO_SET(file_body["mode_param"], penalty_window);
+            CONFIG_AUTO_SET(file_body["mode_param"], precompute_len);
+            {
+                auto has_http = [](const std::string &s) { return s.find("http") != std::string::npos; };
+
+                auto find_tokenizer_file = [this]() -> std::string {
+                    const std::string base = "/opt/m5stack/scripts/";
+                    const std::string a    = base + model_ + "_tokenizer.py";
+                    if (file_exists(a)) return a;
+                    const std::string b = base + "tokenizer_" + model_ + ".py";
+                    if (file_exists(b)) return b;
+                    SLOGE("%s or %s not found!", a.c_str(), b.c_str());
+                    return {};
+                };
+
+                auto start_tokenizer_server = [&](const std::string &tokenizer_file) {
+                    if (tokenizer_file.empty()) return;
+                    if (tokenizer_server_flage_.load()) return;

-            if (mode_config_.filename_tokenizer_model.find("http:") != std::string::npos) {
-                mode_config_.filename_tokenizer_model = "http://localhost:" + std::to_string(port_);
-                std::string tokenizer_file;
-                if (file_exists(std::string("/opt/m5stack/scripts/") + model_ + std::string("_tokenizer.py"))) {
-                    tokenizer_file = std::string("/opt/m5stack/scripts/") + model_ + std::string("_tokenizer.py");
-                } else if (file_exists(std::string("/opt/m5stack/scripts/") + std::string("tokenizer_") + model_ +
-                                       std::string(".py"))) {
-                    tokenizer_file =
-                        std::string("/opt/m5stack/scripts/") + std::string("tokenizer_") + model_ + std::string(".py");
-                } else {
-                    std::string __log = model_ + std::string("_tokenizer.py");
-                    __log += " or ";
-                    __log += std::string("tokenizer_") + model_ + std::string(".py");
-                    __log += " not found!";
-                    SLOGE("%s", __log.c_str());
-                }
-                if (!tokenizer_server_flage_.load()) {
                    tokenizer_pid_ = fork();
                    if (tokenizer_pid_ == 0) {
                        setenv("PYTHONPATH", "/opt/m5stack/lib/llm/site-packages", 1);
+                        const std::string port_str = std::to_string(port_);
+                        const std::string model_id = base_model + "tokenizer";
+
                        execl("/usr/bin/python3", "python3", tokenizer_file.c_str(), "--host", "localhost", "--port",
-                              std::to_string(port_).c_str(), "--model_id", (base_model + "tokenizer").c_str(),
-                              "--content", ("'" + prompt_ + "'").c_str(), nullptr);
+                              port_str.c_str(), "--model_id", model_id.c_str(), "--content", prompt_.c_str(),
+                              (char *)nullptr);
+
                        perror("execl failed");
-                        exit(1);
+                        _exit(1);
                    }
+
                    tokenizer_server_flage_.store(true);
                    SLOGI("port_=%s model_id=%s content=%s", std::to_string(port_).c_str(),
-                          (base_model + "tokenizer").c_str(), ("'" + prompt_ + "'").c_str());
+                          (base_model + std::string("tokenizer")).c_str(), prompt_.c_str());
+
                    std::this_thread::sleep_for(std::chrono::seconds(15));
+                };
+
+                auto process_field = [&](std::string &field, const char *name_for_log) -> bool {
+                    if (!has_http(field)) return false;
+
+                    field                            = "http://localhost:" + std::to_string(port_);
+                    const std::string tokenizer_file = find_tokenizer_file();
+                    start_tokenizer_server(tokenizer_file);
+                    SLOGI("%s: %s", name_for_log, field.c_str());
+                    return true;
+                };
+
+                if (!process_field(mode_config_.filename_tokenizer_model, "filename_tokenizer_model") &&
+                    !process_field(mode_config_.url_tokenizer_model, "url_tokenizer_model")) {
+                    mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model;
+                    SLOGE("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str());
                }
-            } else {
-                mode_config_.filename_tokenizer_model = base_model + mode_config_.filename_tokenizer_model;
            }
-            SLOGI("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str());
            mode_config_.filename_tokens_embed     = base_model + mode_config_.filename_tokens_embed;
            mode_config_.filename_post_axmodel     = base_model + mode_config_.filename_post_axmodel;
            mode_config_.template_filename_axmodel = base_model + mode_config_.template_filename_axmodel;
-
            mode_config_.runing_callback = [this](int *p_token, int n_token, const char *p_str, float token_per_sec,
                                                  void *reserve) {
                if (this->out_callback_) {
                    this->out_callback_(std::string(p_str), false);
                }
            };
-            lLaMa_ = std::make_unique<LLM>();
-            if (!lLaMa_->Init(mode_config_)) {
-                lLaMa_->Deinit();
-                lLaMa_.reset();
-                return -2;
+
+            if (mode_config_.precompute_len > 0) {
+                lLaMa_ctx_ = std::make_unique<LLM_CTX>();
+                if (!lLaMa_ctx_->Init(mode_config_)) {
+                    lLaMa_ctx_->Deinit();
+                    lLaMa_ctx_.reset();
+                    return -2;
+                }
+            } else {
+                lLaMa_ = std::make_unique<LLM>();
+                if (!lLaMa_->Init(mode_config_)) {
+                    lLaMa_->Deinit();
+                    lLaMa_.reset();
+                    return -2;
+                }
            }

+            if (lLaMa_ctx_) {
+                lLaMa_ctx_->SetSystemPrompt(mode_config_.system_prompt, _token_ids);
+                std::string kvcache_path = "/tmp/.llm/";
+                if (!kvcache_path.empty() && kvcache_path != "") {
+                    if (lLaMa_ctx_->load_kvcache(kvcache_path, mode_config_.axmodel_num, k_caches, v_caches,
+                                                 mode_config_.system_prompt, precompute_len)) {
+                        ALOGI("load kvcache from path: %s success,precompute_len: %d", kvcache_path.c_str(),
+                              precompute_len);
+                    } else {
+                        ALOGW("load kvcache from path: %s failed,generate kvcache", kvcache_path.c_str());
+                        lLaMa_ctx_->GenerateKVCachePrefill(_token_ids, k_caches, v_caches, precompute_len);
+                        if (!lLaMa_ctx_->save_kvcache(kvcache_path, mode_config_.system_prompt, precompute_len,
+                                                      k_caches, v_caches)) {
+                            ALOGE("save kvcache failed");
+                        }
+                        ALOGI("generate kvcache to path: %s", kvcache_path.c_str());
+                    }
+                } else {
+                    lLaMa_ctx_->GenerateKVCachePrefill(_token_ids, k_caches, v_caches, precompute_len);
+                }
+                ALOGI("precompute_len: %d", precompute_len);
+                ALOGI("system_prompt: %s", mode_config_.system_prompt.c_str());
+            }
        } catch (...) {
            SLOGE("config false");
            return -3;
@@ -253,8 +318,25 @@ public:
    {
 #if 1
        try {
-            std::string out = lLaMa_->Run(prompt_complete(msg));
-            if (out_callback_) out_callback_(out, true);
+            if (lLaMa_) {
+                std::string out = lLaMa_->Run(prompt_complete(msg));
+                if (out_callback_) out_callback_(out, true);
+            }
+
+            if (lLaMa_ctx_) {
+                lLaMa_ctx_->Encode(prompt_data, prompt_complete(msg), last_reply, tokens_ids, tokens_diff);
+                if (auto ret = lLaMa_ctx_->SetKVCache(k_caches, v_caches, precompute_len, tokens_diff.size());
+                    ret != 0) {
+                    ALOGE("SetKVCache failed: %d,the context may be full,input \"reset\" to reset context", ret);
+                    // raise;
+                    lLaMa_ctx_->SetSystemPrompt(mode_config_.system_prompt, _token_ids);
+                    lLaMa_ctx_->GenerateKVCachePrefill(_token_ids, k_caches, v_caches, precompute_len);
+                    lLaMa_ctx_->SetKVCache(k_caches, v_caches, precompute_len, tokens_diff.size());
+                }
+                last_reply = lLaMa_ctx_->Run(prompt_data);
+                lLaMa_ctx_->GetKVCache(k_caches, v_caches, precompute_len);
+                if (out_callback_) out_callback_(last_reply, true);
+            }
        } catch (...) {
            SLOGW("lLaMa_->Run have error!");
        }
@@ -242,10 +242,11 @@ public:
        this->temperature = temperature;
    }

-    void set_repetition_penalty(bool enable, float penalty)
+    void set_repetition_penalty(bool enable, float penalty, int penalty_window)
    {
        enable_repetition_penalty = enable;
        this->repetition_penalty = penalty;
+        this->penalty_window = penalty_window;
    }

    void set_diversity_penalty(bool enable, const std::vector<int> &common_phrases, float penalty)
@@ -295,6 +296,49 @@ public:
        return true;
    }

+    bool load_config(const nlohmann::json& config)
+    {
+        if (config.is_null()) {
+            ALOGE("config is null or invalid");
+            return false;
+        }
+
+        ALOGI("load config: \n%s\n", config.dump(4).c_str());
+
+        if (config.contains("enable_temperature")) {
+            enable_temperature = config["enable_temperature"].get<bool>();
+        }
+        if (config.contains("temperature")) {
+            temperature = config["temperature"].get<float>();
+        }
+
+        if (config.contains("enable_repetition_penalty")) {
+            enable_repetition_penalty = config["enable_repetition_penalty"].get<bool>();
+        }
+        if (config.contains("repetition_penalty")) {
+            repetition_penalty = config["repetition_penalty"].get<float>();
+        }
+        if (config.contains("penalty_window")) {
+            penalty_window = config["penalty_window"].get<int>();
+        }
+
+        if (config.contains("enable_top_p_sampling")) {
+            enable_top_p_sampling = config["enable_top_p_sampling"].get<bool>();
+        }
+        if (config.contains("top_p")) {
+            top_p = config["top_p"].get<float>();
+        }
+
+        if (config.contains("enable_top_k_sampling")) {
+            enable_top_k_sampling = config["enable_top_k_sampling"].get<bool>();
+        }
+        if (config.contains("top_k")) {
+            top_k = config["top_k"].get<int>();
+        }
+
+        return true;
+    }
+
    int apply(std::vector<float> &logits, const std::vector<int> &history)
    {
        if (enable_temperature)
@@ -5,48 +5,76 @@
 #include <list>
 #include <utility>
 #include <iostream>
-enum TokenizerType
-{
-    TKT_LLaMa,
-    TKT_Qwen,
-    TKT_HTTP,
-    TKT_Phi3,
-    TKT_MINICPM,
-    TKT_AUTO,
-    TKT_END
+enum TokenizerType { TKT_LLaMa, TKT_Qwen, TKT_HTTP, TKT_Phi3, TKT_MINICPM, TKT_AUTO, TKT_END };
+
+enum TokenizeRole {
+    ROLE_USER,           // 用户输入
+    ROLE_SYSTEM,         // 提示词
+    ROLE_TOOL,           // 工具
+    ROLE_IPYTHON,        // 工具
+    ROLE_ASSISTANT,      // 助手回复
+    ROLE_ASSISTANT_HELP  // 询问句
 };

-enum TokenizeRole{
-    ROLE_USER,//用户输入
-    ROLE_SYSTEM,//提示词
-    ROLE_TOOL,  //工具
-    ROLE_IPYTHON,  //工具
-    ROLE_ASSISTANT,//助手回复
-    ROLE_ASSISTANT_HELP// 询问句
+struct ImageInfo {
+    int imgsz       = 448;
+    int num_img     = 1;
+    bool img_prompt = false;
 };

-class BaseTokenizer
-{
+class BaseTokenizer {
 public:
-    std::list<std::pair<enum TokenizeRole, std::string>> messages_;
-    void messages_clean() {messages_.clear();};
-public:
-    virtual bool Init(std::string model_path, bool b_bos = true, bool b_eos = false) = 0;
-    virtual bool Encode(std::string input, std::vector<int> &output, bool b_img_prompt = false) = 0;
-    virtual std::vector<int> Encode(std::string input, bool b_img_prompt = false) = 0;
-    virtual std::string Decode(const std::vector<int> input) = 0;
-    virtual int GetBosID() = 0;
-    virtual int GetEosID() = 0;
+    std::list<std::pair<TokenizeRole, std::string>> messages_;
+
+    void messages_clean()
+    {
+        messages_.clear();
+    }
+
+    virtual ~BaseTokenizer() = default;
+
+    virtual bool Init(std::string model_path)
+    {
+        return false;
+    };
+
+    virtual bool Init(std::string model_path, bool b_bos, bool b_eos)
+    {
+        return false;
+    };
+
+    virtual bool Reset(std::string system_prompt, std::vector<int> &tokens)
+    {
+        return false;
+    };
+
+    virtual bool Encode(std::string input, std::string last_reply, std::vector<int> &tokens,
+                        std::vector<int> &tokens_diff, ImageInfo img_info)
+    {
+        return false;
+    };
+
+    virtual bool Encode(std::string input, std::vector<int> &output, ImageInfo img_info) = 0;
+    virtual std::vector<int> Encode(std::string input, ImageInfo img_info)               = 0;
+    virtual std::string Decode(const std::vector<int> &input)                            = 0;
+    virtual int GetBosID()                                                               = 0;
+    virtual int GetEosID()                                                               = 0;
+
    virtual std::string apply_chat_template() = 0;
-    virtual std::string messages_complete(enum TokenizeRole role, const std::string &content = ""){
+
+    virtual std::string messages_complete(TokenizeRole role, const std::string &content = "")
+    {
        messages_.push_back(std::make_pair(role, content));
-        // std::cout << "messages_complete role:" << role << "content:" << content << std::endl;
-        if(ROLE_ASSISTANT_HELP == role)
+        if (ROLE_ASSISTANT_HELP == role)
            return apply_chat_template();
        else
            return "";
    }
-    virtual bool isEnd(int id) { return id == GetEosID(); }
+
+    virtual bool isEnd(int id)
+    {
+        return id == GetEosID();
+    }
 };

 std::shared_ptr<BaseTokenizer> CreateTokenizer(TokenizerType type);
@@ -61,6 +61,9 @@ public:
    int get_num_inputs() { return minput_tensors.size(); };
    int get_num_outputs() { return moutput_tensors.size(); };

+    int get_num_input_groups() { return mgroup_input_tensors.size(); };
+    int get_num_output_groups() { return mgroup_output_tensors.size(); };
+
    const ax_runner_tensor_t &get_input(int idx) { return minput_tensors[idx]; }
    const ax_runner_tensor_t *get_inputs_ptr() { return minput_tensors.data(); }
    const ax_runner_tensor_t &get_input(std::string name)
@@ -0,0 +1,102 @@
+#pragma once
+#include <iostream>
+#include <string>
+#include <cstring>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <chrono>
+#include <thread>
+
+/**
+ * @brief Attempts to establish a TCP connection to a specified host and port.
+ *
+ * This function creates a socket and tries to connect to a server specified by
+ * the host and port parameters. It returns true if the connection is successful,
+ * otherwise it returns false and outputs an error message to standard error.
+ *
+ * @param host The IP address of the server to connect to.
+ * @param port The port number of the server to connect to.
+ *
+ * @return true if the connection is successfully established, false otherwise.
+ */
+static bool test_connect(const std::string &host, int port)
+{
+    int sock = socket(AF_INET, SOCK_STREAM, 0);
+    if (sock < 0)
+    {
+        // std::cerr << "Socket creation failed\n";
+        return false;
+    }
+
+    sockaddr_in server_addr{};
+    server_addr.sin_family = AF_INET;
+    server_addr.sin_port = htons(port);
+
+    if (inet_pton(AF_INET, host.c_str(), &server_addr.sin_addr) <= 0)
+    {
+        // std::cerr << "IP address conversion failed\n";
+        close(sock);
+        return false;
+    }
+
+    if (connect(sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0)
+    {
+        // std::cerr << "Connection failed\n";
+        close(sock);
+        return false;
+    }
+
+    close(sock);
+    return true;
+}
+
+/**
+ * @brief Attempts to establish an HTTP connection to a specified URL with a timeout.
+ *
+ * This function parses the provided HTTP URL to extract the host and port information,
+ * and attempts to establish a TCP connection using the `test_connect` function.
+ * It retries the connection until the specified timeout is reached.
+ *
+ * @param http_url The HTTP URL of the server to connect to.
+ * @param timeout The maximum number of seconds to keep attempting the connection.
+ *
+ * @return true if the connection is successfully established within the timeout period,
+ * false otherwise.
+ */
+static bool test_connect_http(const std::string &http_url, int timeout)
+{
+    size_t pos = http_url.find("://");
+    if (pos == std::string::npos)
+        return false;
+    std::string host = http_url.substr(pos + 3);
+    pos = host.find('/');
+    if (pos != std::string::npos)
+        host = host.substr(0, pos);
+    pos = host.find(':');
+    int port = 80;
+    if (pos != std::string::npos)
+    {
+        port = std::stoi(host.substr(pos + 1));
+        host = host.substr(0, pos);
+    }
+    else
+    {
+        return false;
+    }
+
+    if (host == "localhost")
+        host = "127.0.0.1";
+    int tmp = timeout;
+    while (timeout--)
+    {
+        if (test_connect(host, port))
+            return true;
+        std::this_thread::sleep_for(std::chrono::seconds(1));
+        printf("\033[1;30;31m"
+               "connect failed %s, try again in %2d/%2d \n"
+               "\033[0m",
+               http_url.c_str(), timeout, tmp);
+    }
+    return false;
+}
@@ -401,6 +401,7 @@ if __name__ == "__main__":
        'llm-model-qwen2.5-0.5B-prefill-20e':[create_data_deb,'llm-model-qwen2.5-0.5B-prefill-20e', data_version, src_folder, revision],
        'llm-model-qwen2.5-0.5B-p256-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-p256-ax630c', '0.4', src_folder, revision],
        'llm-model-qwen2.5-0.5B-Int4-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-Int4-ax630c', '0.4', src_folder, revision],
+        'llm-model-qwen2.5-HA-0.5B-ctx-ax630c':[create_data_deb,'llm-model-qwen2.5-HA-0.5B-ctx-ax630c', '0.5', src_folder, revision],
        'llm-model-qwen2.5-1.5B-ax630c':[create_data_deb,'llm-model-qwen2.5-1.5B-ax630c', '0.3', src_folder, revision],
        'llm-model-qwen2.5-1.5B-p256-ax630c':[create_data_deb,'llm-model-qwen2.5-1.5B-p256-ax630c', '0.4', src_folder, revision],
        'llm-model-qwen2.5-1.5B-Int4-ax630c':[create_data_deb,'llm-model-qwen2.5-1.5B-Int4-ax630c', '0.4', src_folder, revision],