[update] vlm support qwen3-vl model, add qwen3-vl-2b model. update pzmq close timeout to 1s.

2026-05-20 11:32:11 -07:00 · 2025-11-14 14:18:10 +08:00
parent 6d7ae904f4
commit 0d3e36fb49
14 changed files with 1834 additions and 205 deletions
@@ -394,7 +394,7 @@ public:
    }
    void close_zmq()
    {
-        int linger = 0;
+        int linger = 1000;
        zmq_setsockopt(zmq_socket_, ZMQ_LINGER, &linger, sizeof(linger));
        zmq_close(zmq_socket_);
        zmq_ctx_destroy(zmq_ctx_);
@@ -0,0 +1,54 @@
+{
+    "mode": "qwen3-vl-2B-Int4-ax650",
+    "type": "vlm",
+    "homepage": "https://huggingface.co/AXERA-TECH/Qwen3-VL-2B-Instruct",
+    "capabilities": [
+        "text_generation",
+        "chat"
+    ],
+    "input_type": [
+        "vlm.chat_completion",
+        "vlm.chat_completion.stream"
+    ],
+    "output_type": [
+        "vlm.utf-8",
+        "vlm.utf-8.stream"
+    ],
+    "mode_param": {
+        "tokenizer_type": 2,
+        "url_tokenizer_model": "http://localhost:8080",
+        "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin",
+        "filename_post_axmodel": "qwen3_vl_text_post.axmodel",
+        "template_filename_axmodel": "qwen3_vl_text_p128_l%d_together.axmodel",
+        "filename_image_encoder_axmodel": "Qwen3-VL-2B-Instruct_vision.axmodel",
+        "enable_temperature": true,
+        "temperature": 0.7,
+        "enable_top_p_sampling": false,
+        "top_p": 0.9,
+        "enable_top_k_sampling": true,
+        "top_k": 40,
+        "enable_repetition_penalty": false,
+        "repetition_penalty": 1.1,
+        "penalty_window": 50,
+        "axmodel_num": 28,
+        "tokens_embed_num": 151936,
+        "tokens_embed_size": 2048,
+        "b_use_mmap_load_embed": true,
+        "b_video": false,
+        "vision_config.temporal_patch_size": 2,
+        "vision_config.tokens_per_second": 2,
+        "vision_config.spatial_merge_size": 2,
+        "vision_config.patch_size": 16,
+        "vision_config.height": 384,
+        "vision_config.width": 384,
+        "vision_config.fps": 1,
+        "image_token_id": 151655,
+        "video_token_id": 151656,
+        "vision_start_token_id": 151652,
+        "precompute_len": 0,
+        "cmm_size": 1919044,
+        "ext_scripts": [
+            "tokenizer_qwen3-vl-2B-Int4-ax650.py"
+        ]
+    }
+}
@@ -0,0 +1,195 @@
+from transformers import AutoTokenizer, PreTrainedTokenizerFast
+from transformers.tokenization_utils_base import AddedToken
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import json
+import argparse
+
+
+class Tokenizer_Http:
+    def __init__(self, model_id, system_content="You are a helpful assistant."):
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_id,
+            trust_remote_code=True,
+            use_fast=False
+        )
+        self.token_ids_cache = []
+        self.system_content = system_content
+
+    def encode(self, content):
+        text = [
+            f'<|im_start|>system\n{self.system_content}<|im_end|>\n'
+            f'<|im_start|>user\n{content}<|im_end|>\n'
+            f'<|im_start|>assistant\n'
+        ]
+        input_ids = self.tokenizer(text)
+        return input_ids["input_ids"][0]
+
+    def encode_vpm_image(self, content="Describe this image.", num_img=1, img_token_num=256):
+        imgs_token = (
+            '<|vision_start|>'
+            + '<|image_pad|>' * img_token_num
+            + '<|vision_end|>'
+        )
+        imgs_token *= num_img
+        text = (
+            f'<|im_start|>system\n{self.system_content}<|im_end|>\n'
+            f'<|im_start|>user\n{imgs_token}{content}<|im_end|>\n'
+            f'<|im_start|>assistant\n'
+        )
+
+        output_kwargs = {
+            'text_kwargs': {'padding': True, 'return_tensors': 'pt'},
+            'images_kwargs': {'return_tensors': 'pt'},
+            'audio_kwargs': {'padding': True, 'return_tensors': 'pt'},
+            'videos_kwargs': {'fps': 2.0, 'return_tensors': 'pt'},
+            'common_kwargs': {'return_tensors': 'pt'},
+        }
+
+        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+        return text_inputs["input_ids"].tolist()[0]
+
+
+    def encode_vpm_video(self, content="Describe this image.", num_img=1, img_token_num=256):
+        imgs_token = (
+            '<|vision_start|>'
+            + '<|video_pad|>' * img_token_num * num_img
+            + '<|vision_end|>'
+        )
+
+        text = (
+            f'<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n'
+            f'<|im_start|>user\n{imgs_token}{content}<|im_end|>\n'
+            f'<|im_start|>assistant\n'
+        )
+
+        output_kwargs = {
+            'text_kwargs': {'padding': True, 'return_tensors': 'pt'},
+            'images_kwargs': {'return_tensors': 'pt'},
+            'audio_kwargs': {'padding': True, 'return_tensors': 'pt'},
+            'videos_kwargs': {'fps': 2.0, 'return_tensors': 'pt'},
+            'common_kwargs': {'return_tensors': 'pt'},
+        }
+
+        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+        return text_inputs["input_ids"].tolist()[0]
+    
+    def decode(self, token_ids):
+        self.token_ids_cache += token_ids
+        text = self.tokenizer.decode(self.token_ids_cache)
+        if "\ufffd" in text:
+            print("text 中包含非法字符")
+            return ""
+        else:
+            self.token_ids_cache.clear()
+            return text
+
+    @property
+    def bos_id(self):
+        return self.tokenizer.bos_token_id
+
+    @property
+    def eos_id(self):
+        return self.tokenizer.eos_token_id
+
+    @property
+    def bos_token(self):
+        return self.tokenizer.bos_token
+
+    @property
+    def eos_token(self):
+        return self.tokenizer.eos_token
+
+    @property
+    def img_start_token(self):
+        return self.tokenizer.encode("<|vision_start|>")[0]
+
+    @property
+    def img_context_token(self):
+        return self.tokenizer.encode("<|image_pad|>")[0]
+
+
+class Request(BaseHTTPRequestHandler):
+    timeout = 5
+    server_version = 'Apache'
+
+    def do_GET(self):
+        print(self.path)
+        self.send_response(200)
+        self.send_header("type", "get")
+        self.end_headers()
+
+        if self.path == '/bos_id':
+            bos_id = tokenizer.bos_id
+            msg = json.dumps({'bos_id': -1 if bos_id is None else bos_id})
+        elif self.path == '/eos_id':
+            eos_id = tokenizer.eos_id
+            msg = json.dumps({'eos_id': -1 if eos_id is None else eos_id})
+        elif self.path == '/img_start_token':
+            img_start_token = tokenizer.img_start_token
+            msg = json.dumps({'img_start_token': -1 if img_start_token is None else img_start_token})
+        elif self.path == '/img_context_token':
+            img_context_token = tokenizer.img_context_token
+            msg = json.dumps({'img_context_token': -1 if img_context_token is None else img_context_token})
+        else:
+            msg = 'error'
+
+        print(msg)
+        msg = str(msg).encode()
+        self.wfile.write(msg)
+
+    def do_POST(self):
+        data = self.rfile.read(int(self.headers['content-length']))
+        req = json.loads(data.decode())
+
+        if self.path == "/encode":
+            prompt = req['text']
+            b_img_prompt = req.get('img_prompt', False)
+            img_type = req.get('img_type', 'image')  # 默认 image
+
+            if b_img_prompt:
+                if img_type == 'image':
+                    token_ids = tokenizer.encode_vpm_image(
+                        prompt,
+                        req.get("num_img", 1),
+                        req.get("img_token_num", 256)
+                    )
+                elif img_type == 'video':
+                    token_ids = tokenizer.encode_vpm_video(
+                        prompt,
+                        req.get("num_img", 1),
+                        req.get("img_token_num", 256)
+                    )
+                else:
+                    token_ids = tokenizer.encode(prompt)  # fallback
+            else:
+                token_ids = tokenizer.encode(prompt)
+
+            msg = json.dumps({'token_ids': -1 if token_ids is None else token_ids})
+
+        elif self.path == "/decode":
+            req = json.loads(data.decode())
+            token_ids = req['token_ids']
+            text = tokenizer.decode(token_ids)
+            msg = json.dumps({'text': "" if text is None else text})
+        else:
+            msg = 'error'
+
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(str(msg).encode())
+
+
+if __name__ == "__main__":
+    args = argparse.ArgumentParser()
+    args.add_argument('--host', type=str, default='localhost')
+    args.add_argument('--port', type=int, default=8080)
+    args.add_argument('--model_id', type=str, default='tokenizer')
+    args.add_argument('--content', type=str, default='You are a helpful assistant.')
+    args = args.parse_args()
+
+    tokenizer = Tokenizer_Http(args.model_id, system_content=args.content)
+    host = (args.host, args.port)
+
+    print(f"http://{args.host}:{args.port}")
+    server = HTTPServer(host, Request)
+    server.serve_forever()
@@ -42,17 +42,27 @@ typedef std::function<void(const std::string &data, bool finish)> task_callback_
    else if (obj.contains(#key))              \
        mode_config_.key = obj[#key];

+#define QWEN_CONFIG_AUTO_SET(obj, key)             \
+    if (config_body.contains(#key))                \
+        qwen_mode_config_.key = config_body[#key]; \
+    else if (obj.contains(#key))                   \
+        qwen_mode_config_.key = obj[#key];
+
 class llm_task {
 private:
    static std::atomic<unsigned int> next_port_;
    std::atomic_bool tokenizer_server_flage_;
    unsigned int port_;
    pid_t tokenizer_pid_ = -1;
+    enum class ModelType { Unknown = 0, Qwen, InternVL, InternVL_CTX };
+    ModelType model_type_ = ModelType::Unknown;

 public:
    LLMAttrType mode_config_;
+    Config qwen_mode_config_;
    std::unique_ptr<LLM> lLaMa_;
    std::unique_ptr<LLM_CTX> lLaMa_ctx_;
+    std::unique_ptr<LLM_Qwen> qwen_;
    std::string model_;
    std::string response_format_;
    std::vector<std::string> inputs_;
@@ -61,6 +71,10 @@ public:
    std::vector<std::vector<unsigned char>> images_data;
    std::vector<cv::Mat> mats;
    std::vector<unsigned short> img_embed;
+    std::vector<std::vector<unsigned short>> imgs_embed;
+    std::vector<std::vector<float>> deepstack_features;
+    std::vector<int> visual_pos_mask;
+    std::vector<std::vector<int>> position_ids;
    std::string prompt_;
    std::string last_reply;
    std::vector<int> tokens_ids, tokens_diff;
@@ -121,6 +135,28 @@ public:
        }
    }

+    void vlm_reset()
+    {
+        std::vector<unsigned short>().swap(prompt_data_);
+
+        for (auto &inner_vec : imgs_embed) {
+            std::vector<unsigned short>().swap(inner_vec);
+        }
+        std::vector<std::vector<unsigned short>>().swap(imgs_embed);
+
+        for (auto &inner_vec : deepstack_features) {
+            std::vector<float>().swap(inner_vec);
+        }
+        std::vector<std::vector<float>>().swap(deepstack_features);
+
+        std::vector<int>().swap(visual_pos_mask);
+
+        for (auto &inner_vec : position_ids) {
+            std::vector<int>().swap(inner_vec);
+        }
+        std::vector<std::vector<int>>().swap(position_ids);
+    }
+
    int load_model(const nlohmann::json &config_body)
    {
        if (parse_config(config_body)) {
@@ -179,7 +215,19 @@ public:
            CONFIG_AUTO_SET(file_body["mode_param"], vpm_width);
            CONFIG_AUTO_SET(file_body["mode_param"], vpm_height);
            CONFIG_AUTO_SET(file_body["mode_param"], precompute_len);
+            CONFIG_AUTO_SET(file_body["mode_param"], b_video);

+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.temporal_patch_size);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.tokens_per_second);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.spatial_merge_size);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.patch_size);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.width);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.height);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.fps);
+
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], image_token_id);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], video_token_id);
+            QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_start_token_id);
            {
                auto has_http = [](const std::string &s) { return s.find("http") != std::string::npos; };

@@ -196,10 +244,9 @@ public:
                auto start_tokenizer_server = [&](const std::string &tokenizer_file) {
                    if (tokenizer_file.empty()) return;
                    if (tokenizer_server_flage_.load()) return;
-
                    tokenizer_pid_ = fork();
                    if (tokenizer_pid_ == 0) {
-                        setenv("PYTHONPATH", "/opt/m5stack/lib/llm/site-packages", 1);
+                        setenv("PYTHONPATH", "/opt/m5stack/lib/vlm/site-packages", 1);
                        const std::string port_str = std::to_string(port_);
                        const std::string model_id = base_model + "tokenizer";

@@ -234,11 +281,25 @@ public:
                    SLOGI("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str());
                }
            }
-            mode_config_.filename_tokens_embed          = base_model + mode_config_.filename_tokens_embed;
-            mode_config_.filename_post_axmodel          = base_model + mode_config_.filename_post_axmodel;
-            mode_config_.template_filename_axmodel      = base_model + mode_config_.template_filename_axmodel;
+
+            {
+                std::string encoder_name = mode_config_.filename_image_encoder_axmodel;
+                std::transform(encoder_name.begin(), encoder_name.end(), encoder_name.begin(), ::tolower);
+
+                if (encoder_name.find("qwen3") != std::string::npos)
+                    model_type_ = ModelType::Qwen;
+                else if (encoder_name.find("internvl3") != std::string::npos && mode_config_.precompute_len > 0)
+                    model_type_ = ModelType::InternVL_CTX;
+                else if (encoder_name.find("internvl3") != std::string::npos)
+                    model_type_ = ModelType::InternVL;
+                else
+                    model_type_ = ModelType::Unknown;
+            }
+            mode_config_.filename_tokens_embed           = base_model + mode_config_.filename_tokens_embed;
+            mode_config_.filename_post_axmodel           = base_model + mode_config_.filename_post_axmodel;
+            mode_config_.filename_image_encoder_axmodel  = base_model + mode_config_.filename_image_encoder_axmodel;
+            mode_config_.template_filename_axmodel       = base_model + mode_config_.template_filename_axmodel;
            mode_config_.filename_vpm_resampler_axmodedl = base_model + mode_config_.filename_vpm_resampler_axmodedl;
-            mode_config_.filename_image_encoder_axmodel = base_model + mode_config_.filename_image_encoder_axmodel;
            mode_config_.runing_callback = [this](int *p_token, int n_token, const char *p_str, float token_per_sec,
                                                  void *reserve) {
                if (this->out_callback_) {
@@ -246,20 +307,40 @@ public:
                }
            };

-            if (mode_config_.precompute_len > 0) {
-                lLaMa_ctx_ = std::make_unique<LLM_CTX>();
-                if (!lLaMa_ctx_->Init(mode_config_)) {
-                    lLaMa_ctx_->Deinit();
-                    lLaMa_ctx_.reset();
-                    return -2;
+            switch (model_type_) {
+                case ModelType::InternVL: {
+                    lLaMa_ = std::make_unique<LLM>();
+                    if (!lLaMa_->Init(mode_config_)) {
+                        lLaMa_->Deinit();
+                        lLaMa_.reset();
+                        return -2;
+                    }
+                    break;
                }
-            } else {
-                lLaMa_ = std::make_unique<LLM>();
-                if (!lLaMa_->Init(mode_config_)) {
-                    lLaMa_->Deinit();
-                    lLaMa_.reset();
-                    return -2;
+
+                case ModelType::InternVL_CTX: {
+                    lLaMa_ctx_ = std::make_unique<LLM_CTX>();
+                    if (!lLaMa_ctx_->Init(mode_config_)) {
+                        lLaMa_ctx_->Deinit();
+                        lLaMa_ctx_.reset();
+                        return -2;
+                    }
+                    break;
                }
+
+                case ModelType::Qwen: {
+                    qwen_ = std::make_unique<LLM_Qwen>();
+                    if (!qwen_->Init(mode_config_)) {
+                        qwen_->Deinit();
+                        qwen_.reset();
+                        return -2;
+                    }
+                    break;
+                }
+                default:
+                    ALOGE("Unknown model type in filename_image_encoder_axmodel: %s",
+                          mode_config_.filename_image_encoder_axmodel.c_str());
+                    return -3;
            }

            if (lLaMa_ctx_) {
@@ -421,6 +502,33 @@ public:
                    if (out_callback_) out_callback_(last_reply, true);
                }
            }
+
+            if (qwen_) {
+                if (images_data.empty()) {
+                    qwen_->Encode(prompt_data_, position_ids, qwen_mode_config_, prompt_complete(msg));
+                    last_reply = qwen_->Run(prompt_data_, position_ids, deepstack_features, visual_pos_mask);
+                    if (out_callback_) out_callback_(last_reply, true);
+                } else {
+                    for (const auto &img_buf : images_data) {
+                        cv::Mat src = cv::imdecode(img_buf, cv::IMREAD_COLOR);
+                        if (src.empty()) {
+                            std::cerr << "Decode failed!" << std::endl;
+                            continue;
+                        }
+                        mats.push_back(src);
+                    }
+                    images_data.clear();
+                    if (mats.empty()) return;
+                    std::vector<std::vector<unsigned short>> all_embeds;
+                    qwen_->EncodeImage(mats, mode_config_.b_video, qwen_mode_config_, all_embeds, deepstack_features);
+                    mats.clear();
+                    qwen_->Encode(all_embeds, mode_config_.b_video, prompt_data_, position_ids, visual_pos_mask,
+                                  qwen_mode_config_, prompt_complete(msg));
+                    last_reply = qwen_->Run(prompt_data_, position_ids, deepstack_features, visual_pos_mask);
+                    if (out_callback_) out_callback_(last_reply, true);
+                    vlm_reset();
+                }
+            }
        } catch (...) {
            SLOGW("lLaMa_->Run have error!");
        }
@@ -428,7 +536,9 @@ public:

    bool pause()
    {
-        lLaMa_->Stop();
+        if (lLaMa_) lLaMa_->Stop();
+        if (lLaMa_ctx_) lLaMa_ctx_->Stop();
+        if (qwen_) qwen_->Stop();
        return true;
    }

@@ -439,8 +549,10 @@ public:
            waitpid(tokenizer_pid_, nullptr, 0);
            tokenizer_pid_ = -1;
        }
-        lLaMa_->Deinit();
-        lLaMa_.reset();
+        if (lLaMa_) lLaMa_->Deinit();
+        if (lLaMa_) lLaMa_.reset();
+        if (qwen_) qwen_->Deinit();
+        if (qwen_) qwen_.reset();
        return true;
    }

@@ -476,6 +588,12 @@ public:
        if (lLaMa_) {
            lLaMa_->Deinit();
        }
+        if (lLaMa_ctx_) {
+            lLaMa_ctx_->Deinit();
+        }
+        if (qwen_) {
+            qwen_->Deinit();
+        }
    }
 };

@@ -530,12 +648,14 @@ public:
        if (!(llm_task_obj && llm_channel)) {
            return;
        }
-        llm_task_obj->lLaMa_->Stop();
+        if (llm_task_obj->lLaMa_) llm_task_obj->lLaMa_->Stop();
+        if (llm_task_obj->lLaMa_ctx_) llm_task_obj->lLaMa_ctx_->Stop();
+        if (llm_task_obj->qwen_) llm_task_obj->qwen_->Stop();
    }

    void pause(const std::string &work_id, const std::string &object, const std::string &data) override
    {
-        SLOGI("llm_asr::work:%s", data.c_str());
+        SLOGI("llm_vlm::work:%s", data.c_str());

        nlohmann::json error_body;
        int work_id_num = sample_get_work_id_num(work_id);
@@ -626,7 +746,9 @@ public:
        if (!(llm_task_obj && llm_channel)) {
            return;
        }
-        llm_task_obj->lLaMa_->Stop();
+        if (llm_task_obj->lLaMa_) llm_task_obj->lLaMa_->Stop();
+        if (llm_task_obj->lLaMa_ctx_) llm_task_obj->lLaMa_ctx_->Stop();
+        if (llm_task_obj->qwen_) llm_task_obj->qwen_->Stop();
    }

    void task_camera_data(const std::weak_ptr<llm_task> llm_task_obj_weak,
@@ -3,40 +3,40 @@
 #include <vector>
 #include <memory>

-enum TokenizerType
-{
-    TKT_LLaMa,
-    TKT_Qwen,
-    TKT_HTTP,
-    TKT_Phi3,
-    TKT_END
+enum TokenizerType { TKT_LLaMa, TKT_Qwen, TKT_HTTP, TKT_Phi3, TKT_END };
+
+enum class ImgType { None, Image, Video };
+
+struct ImageInfo {
+    int imgsz         = 448;
+    int num_img       = 1;
+    bool img_prompt   = false;
+    int img_token_num = -1;
+    ImgType type      = ImgType::None;
 };

-struct ImageInfo
-{
-    int imgsz = 448;
-    int num_img = 1;
-    bool img_prompt = false;
-};
-
-class BaseTokenizer
-{
+class BaseTokenizer {
 public:
-    virtual bool Init(std::string model_path) = 0;
-    virtual bool Init(std::string model_path, bool b_bos, bool b_eos) = 0;
-    virtual bool Init_new(std::string model_path, bool b_bos, bool b_eos) = 0;
-    virtual bool Reset(std::string system_prompt, std::vector<int> &tokens) = 0;
-    virtual bool Encode(std::string input, std::string last_reply, std::vector<int> &tokens, std::vector<int> &tokens_diff, ImageInfo img_info) = 0;
+    virtual bool Init(std::string model_path)                                            = 0;
+    virtual bool Init(std::string model_path, bool b_bos, bool b_eos)                    = 0;
+    virtual bool Init_new(std::string model_path, bool b_bos, bool b_eos)                = 0;
+    virtual bool Reset(std::string system_prompt, std::vector<int> &tokens)              = 0;
+    virtual bool Encode(std::string input, std::string last_reply, std::vector<int> &tokens,
+                        std::vector<int> &tokens_diff, ImageInfo img_info)               = 0;
    virtual bool Encode(std::string input, std::vector<int> &output, ImageInfo img_info) = 0;
-    virtual std::vector<int> Encode(std::string input, ImageInfo img_info) = 0;
-    virtual std::vector<int> Encode_ctx(std::string input, ImageInfo img_info, std::vector<int> &tokens_ids, std::vector<int> &tokens_diff) = 0;
-    virtual std::string Decode(const std::vector<int> input) = 0;
-    virtual int GetBosID() = 0;
-    virtual int GetEosID() = 0;
-    virtual int GetImgStartID() = 0;
-    virtual int GetImgContextID() = 0;
+    virtual std::vector<int> Encode(std::string input, ImageInfo img_info)               = 0;
+    virtual std::vector<int> Encode_ctx(std::string input, ImageInfo img_info, std::vector<int> &tokens_ids,
+                                        std::vector<int> &tokens_diff)                   = 0;
+    virtual std::string Decode(const std::vector<int> input)                             = 0;
+    virtual int GetBosID()                                                               = 0;
+    virtual int GetEosID()                                                               = 0;
+    virtual int GetImgStartID()                                                          = 0;
+    virtual int GetImgContextID()                                                        = 0;

-    virtual bool isEnd(int id) { return id == GetEosID(); }
+    virtual bool isEnd(int id)
+    {
+        return id == GetEosID();
+    }
 };

 std::shared_ptr<BaseTokenizer> CreateTokenizer(TokenizerType type);
@@ -0,0 +1,46 @@
+#include <sys/stat.h>  
+#include <dirent.h>
+#include <unistd.h>
+#include <string>
+#include <vector>
+#include <iostream>
+#include <algorithm>
+
+bool is_directory(const std::string& path) {
+    struct stat st;
+    if (stat(path.c_str(), &st) != 0) return false;  
+    return S_ISDIR(st.st_mode);  
+}
+
+bool is_file(const std::string& path) {
+    struct stat st;
+    if (stat(path.c_str(), &st) != 0) return false;
+    return S_ISREG(st.st_mode);  
+}
+
+std::vector<std::string> list_files(const std::string& directory) {
+    std::vector<std::string> files;
+    DIR* dir = opendir(directory.c_str());
+    if (!dir) {
+        std::cerr << "无法打开目录: " << directory << std::endl;
+        return files;
+    }
+
+    struct dirent* entry;
+    while ((entry = readdir(dir)) != nullptr) {
+        std::string name = entry->d_name;
+        if (name == "." || name == "..") continue;
+
+        // 拼接完整路径并检查是否为普通文件
+        std::string full_path = directory + "/" + name;
+        struct stat st;
+        if (stat(full_path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) {
+            files.push_back(full_path);
+        }
+    }
+    closedir(dir);
+
+    // 按文件名升序排序
+    std::sort(files.begin(), files.end());
+    return files;
+}
@@ -0,0 +1,17 @@
+#ifndef _FILES_H_
+#define _FILES_H_
+
+#include <sys/stat.h>  
+#include <dirent.h>
+#include <unistd.h>
+#include <string>
+#include <vector>
+#include <iostream>
+#include <algorithm>
+
+bool is_directory(const std::string& path);
+bool is_file(const std::string& path);
+std::vector<std::string> list_files(const std::string& directory);
+
+
+#endif
@@ -0,0 +1,162 @@
+#include <vector>
+#include <math.h>
+#include <opencv2/opencv.hpp>
+#include "files.hpp"
+#include "image_processor.hpp"
+#include <iostream>
+
+std::vector<cv::Mat> ReadImages(std::string path){
+    std::vector<cv::Mat> src;
+
+    if(is_file(path)){
+        cv::Mat img = cv::imread(path, cv::IMREAD_COLOR);
+        src.push_back(img);
+    }
+    else if(is_directory(path)){
+        auto paths = list_files(path);
+        
+        for(auto &p : paths){
+            std::cout<<p<<std::endl;
+            cv::Mat img = cv::imread(p, cv::IMREAD_COLOR);
+            src.push_back(img);
+        }
+    }
+    else{
+        std::cerr << "错误的路径: " << path << std::endl;
+    }
+
+    return src;
+}
+
+std::pair<int, int> SmartResize(int height, int width, int factor){
+    int h_bar = height/factor;
+    int w_bar = width/factor;
+
+    h_bar *= factor;
+    w_bar *= factor;
+    return {h_bar, w_bar};
+}
+
+void normalizeMeanStd(cv::Mat& image) {
+    // 确保输入图像是浮点类型（避免整数溢出）
+    cv::Mat floatImage;
+    image.convertTo(floatImage, CV_32F);  // 转换为32位浮点格式 <button class="citation-flag" data-index="1">
+
+    // 计算均值和标准差
+    cv::Scalar mean, stddev;
+    cv::meanStdDev(floatImage, mean, stddev);  // 计算均值和标准差 <button class="citation-flag" data-index="2">
+
+    // 避免除以零：如果标准差为0，设置为一个小值（如1e-6）
+    for (int i = 0; i < floatImage.channels(); ++i) {
+        if (stddev[i] < 1e-6) {
+            stddev[i] = 1e-6;
+        }
+    }
+
+    // 归一化：减去均值并除以标准差
+    floatImage -= mean;  // 减去均值 <button class="citation-flag" data-index="4">
+    floatImage /= stddev;  // 除以标准差 <button class="citation-flag" data-index="5">
+
+    // 将结果转换回原始数据类型（如8位无符号整数）
+    floatImage.convertTo(image, image.type());  // 转换回原始格式 <button class="citation-flag" data-index="6">
+}
+
+int Qwen2VideoProcessor( std::vector<cv::Mat>& src, std::vector<std::vector<unsigned char>>& output, 
+                            int tgt_h, int tgt_w,
+                            int temporal_patch_size, int merge_size, int patch_size){
+
+    if(src.empty()){
+        return 0;
+    }
+
+    int height = src[0].rows;
+    int width = src[0].cols;
+
+    // auto [tgt_h, tgt_w] = SmartResize(height, width, 28);
+
+    cv::Size size(tgt_w, tgt_h);
+    std::vector<cv::Mat> imgs_resized;
+    
+    for(auto& img: src){
+        cv::Mat img_rs;
+        if(img.cols!=tgt_w || img.rows!=tgt_h){
+            cv::resize(img, img_rs, size, 0, 0, cv::INTER_CUBIC);
+        }else{
+            img_rs = img;
+        }
+        
+        cv::cvtColor(img_rs, img_rs, cv::COLOR_BGR2RGB);
+        imgs_resized.push_back(img_rs);
+    }
+    
+    if(imgs_resized.empty()){
+        return 0;
+    }
+
+    if(imgs_resized.size()%2!=0){
+        imgs_resized.push_back(imgs_resized.back());
+    }
+
+    std::vector<unsigned char> patches;
+    patches.resize( imgs_resized.size()* tgt_w*tgt_h* 3);
+    for(size_t i=0; i<imgs_resized.size(); ++i){
+        memcpy(patches.data()+i*tgt_w*tgt_h*3, imgs_resized[i].data, tgt_w*tgt_h* 3);
+    }
+
+    int grid_t = imgs_resized.size() / temporal_patch_size;
+    int channel = imgs_resized[0].channels();
+    int grid_h = tgt_h/patch_size;
+    int grid_w = tgt_w/patch_size;
+
+    // channel = patches.shape[3]
+    // patches = patches.reshape(
+    //     grid_t,                     # 0
+    //     self.temporal_patch_size,   # 1
+    //     grid_h // self.merge_size,  # 2
+    //     self.merge_size,            # 3
+    //     self.patch_size,            # 4
+    //     grid_w // self.merge_size,  # 5
+    //     self.merge_size,            # 6
+    //     self.patch_size,            # 7
+    //     channel                     # 8
+    // )   
+    // patches = patches.transpose(0, 2, 5, 3, 6, 1, 4, 7, 8 )
+
+    for(size_t d0=0; d0<grid_t; d0++){
+        std::vector<unsigned char> out_t;
+        for(size_t d2=0; d2<grid_h/merge_size; d2++){
+            for(size_t d5=0; d5<grid_w/merge_size; d5++){
+                for(size_t d3=0; d3<merge_size; d3++ ){
+                    for(size_t d6=0; d6<merge_size; d6++){
+                        for(size_t d1=0; d1<temporal_patch_size; d1++){
+                            for(size_t d4=0; d4<patch_size; d4++){
+                                for(size_t d7=0; d7<patch_size; d7++){
+                                    for(size_t d8=0; d8<channel; d8++){
+                                        size_t idx = d0*temporal_patch_size*grid_h*patch_size*grid_w*patch_size*channel;
+                                        idx += d1*grid_h*patch_size*grid_w*patch_size*channel;
+                                        idx += d2*merge_size*patch_size*grid_w*patch_size*channel;
+                                        idx += d3*patch_size*grid_w*patch_size*channel;
+                                        idx += d4*grid_w*patch_size*channel;
+                                        idx += d5*merge_size*patch_size*channel;
+                                        idx += d6*patch_size*channel;
+                                        idx += d7*channel;
+                                        idx += d8;
+
+                                        out_t.push_back(patches[idx]);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        output.push_back(out_t);
+    }
+
+    // std::vector<size_t> ret={grid_t, grid_h*grid_w, temporal_patch_size*patch_size*patch_size, channel};
+    // return ret;
+    return 0;
+
+}
+
@@ -0,0 +1,14 @@
+
+#ifndef _IMAGE_PROCESSOR_H_
+#define _IMAGE_PROCESSOR_H_
+
+#include <string>
+#include <vector>
+#include <opencv2/opencv.hpp>
+
+std::vector<cv::Mat> ReadImages(std::string path);
+int Qwen2VideoProcessor( std::vector<cv::Mat>& src, std::vector<std::vector<unsigned char>>& output,
+                        int tgt_h, int tgt_w,
+                        int temporal_patch_size=2, int merge_size=2, int patch_size=14);
+
+#endif
@@ -0,0 +1,272 @@
+#include <vector>
+#include <algorithm>
+#include <optional>
+#include <cassert>
+#include "mrope.hpp"
+
+#include <iostream>
+#include <vector>
+#include <numeric>  // std::iota
+
+#include <vector>
+#include <algorithm>
+#include <limits>  // 用于std::numeric_limits
+#include <stdexcept>  // 用于异常处理
+
+int findMaxIn2DVector(const std::vector<std::vector<int>>& vec) {
+    if (vec.empty()) {
+        throw std::invalid_argument("输入二维vector为空");
+    }
+
+    int max_value = std::numeric_limits<int>::min();  // 初始化为最小值
+    bool has_elements = false;
+
+    for (const auto& subvec : vec) {
+        if (!subvec.empty()) {
+            has_elements = true;
+            // 使用std::max_element获取子vector的最大值
+            int sub_max = *std::max_element(subvec.begin(), subvec.end());
+            if (sub_max > max_value) {
+                max_value = sub_max;
+            }
+        }
+    }
+
+    if (!has_elements) {
+        throw std::invalid_argument("所有子vector均为空");  // 处理全空子vector
+    }
+
+    return max_value;
+}
+
+// 生成范围序列 [0, text_len-1]
+std::vector<int> generateRange(int text_len, int start) {
+    std::vector<int> range(text_len);
+    std::iota(range.begin(), range.end(), start);  // 填充从0开始的序列
+    return range;
+}
+
+// 扩展为多行矩阵
+std::vector<std::vector<int>> expandToMatrix(const std::vector<int>& range, int rows) {
+    std::vector<std::vector<int>> matrix(rows, range);  // 每一行都是range的副本
+    return matrix;
+}
+
+// 生成多维索引 (unused in Qwen3, but kept for completeness)
+std::vector<std::vector<int>> generateIndices(int grid_t, int grid_h, int grid_w) {
+    std::vector<std::vector<int>> indices(3, std::vector<int>(grid_t * grid_h * grid_w));
+
+    int idx = 0;
+    for (int t = 0; t < grid_t; ++t) {
+        for (int h = 0; h < grid_h; ++h) {
+            for (int w = 0; w < grid_w; ++w) {
+                indices[0][idx] = t;  // 时间索引
+                indices[1][idx] = h;  // 高度索引
+                indices[2][idx] = w;  // 宽度索引
+                ++idx;
+            }
+        }
+    }
+
+    return indices;
+}
+
+// Qwen3-VL specific: Preprocess video_grid_thw by repeating each row t times and setting t=1
+std::vector<std::vector<int>> preprocessVideoGrid(const std::vector<std::vector<int>>& video_grid_thw) {
+    std::vector<std::vector<int>> processed;
+    for (const auto& grid : video_grid_thw) {
+        if (grid.size() != 3) {
+            throw std::invalid_argument("Invalid grid format");
+        }
+        int t = grid[0];
+        // Repeat the row t times
+        for (int i = 0; i < t; ++i) {
+            std::vector<int> repeated_grid = {1, grid[1], grid[2]};  // Set t=1
+            processed.push_back(repeated_grid);
+        }
+    }
+    return processed;
+}
+
+std::vector<std::vector<int>> get_rope_index(
+    const Config& config,
+    const std::vector<int>& input_ids,
+    const std::vector<std::vector<int>>& image_grid_thw,
+    const std::vector<std::vector<int>>& video_grid_thw
+) {
+    const int spatial_merge_size = config.vision_config.spatial_merge_size;
+    const int image_token_id = config.image_token_id;
+    const int video_token_id = config.video_token_id;
+    const int vision_start_token_id = config.vision_start_token_id;
+
+    std::vector<std::vector<int>> position_ids(3);
+
+    // Preprocess video_grid_thw for Qwen3-VL (split into single-frame segments with timestamps)
+    auto processed_video_grid = preprocessVideoGrid(video_grid_thw);
+
+    // Handle pure text case
+    if (input_ids.empty() || (image_grid_thw.empty() && video_grid_thw.empty())) {
+        int b = 0;
+        for (int i = 0; i < 3; ++i) {
+            std::vector<int> seq(input_ids.size());
+            // 手动实现递增序列
+            for (size_t j = 0; j < seq.size(); ++j) {
+                seq[j] = j;
+            }
+            position_ids[i].insert(position_ids[i].end(), seq.begin(), seq.end());
+        }
+        return position_ids;
+    }
+
+    // Multimodal case (batch_size=1, single sequence)
+    const auto& ids = input_ids;
+    // Assume full mask for simplicity (as in Qwen3 fallback)
+    const auto mask = std::vector<int>(ids.size(), 1);
+
+    // Filter valid tokens (masked)
+    std::vector<int> filtered_ids;
+    for (size_t i = 0; i < ids.size(); ++i) {
+        if (mask[i]) filtered_ids.push_back(ids[i]);
+    }
+
+    int image_nums = 0, video_nums = 0;
+
+    for (size_t i = 0; i < filtered_ids.size() - 1; ++i) {
+        if (filtered_ids[i] == vision_start_token_id) {
+            if (filtered_ids[i + 1] == config.image_token_id) {
+                image_nums++;
+            }
+            if (filtered_ids[i + 1] == config.video_token_id) {
+                video_nums++;
+            }
+        }
+    }
+
+    int image_index = 0, video_index = 0;
+    std::vector<std::vector<int>> batch_pos(3);
+    int st = 0;
+    int remain_images = image_nums;
+    int remain_videos = video_nums;
+    std::vector<std::vector<std::vector<int>>> llm_pos_ids_list;
+
+    // Loop over vision blocks (images + videos, now with processed_video_grid)
+    for (size_t i_ = 0; i_ < static_cast<size_t>(image_nums + video_nums); ++i_) {
+
+        int ed_image = filtered_ids.size() + 1;
+        int ed_video = filtered_ids.size() + 1;
+
+        if (remain_images > 0) {
+            for (size_t j = st; j < filtered_ids.size(); ++j) {
+                if (filtered_ids[j] == config.image_token_id) {
+                    ed_image = static_cast<int>(j);
+                    break;
+                }
+            }
+        }
+
+        if (remain_videos > 0) {
+            for (size_t j = st; j < filtered_ids.size(); ++j) {
+                if (filtered_ids[j] == config.video_token_id) {
+                    ed_video = static_cast<int>(j);
+                    break;
+                }
+            }
+        }
+
+        int t, h, w;
+        int ed;
+
+        if (ed_image < ed_video) {
+            // Image
+            t = image_grid_thw[image_index][0];
+            h = image_grid_thw[image_index][1];
+            w = image_grid_thw[image_index][2];
+            image_index += 1;
+            remain_images -= 1;
+            ed = ed_image;
+        } else {
+            // Video (using processed grid, each entry has t=1)
+            t = processed_video_grid[video_index][0];  // 1
+            h = processed_video_grid[video_index][1];
+            w = processed_video_grid[video_index][2];
+            video_index += 1;
+            remain_videos -= 1;
+            ed = ed_video;
+        }
+
+        int llm_grid_t = t;  // For videos, t=1
+        int llm_grid_h = h / spatial_merge_size;
+        int llm_grid_w = w / spatial_merge_size;
+
+        int text_len = ed - st;
+
+        int st_idx;
+        if (llm_pos_ids_list.empty()) {
+            st_idx = 0;
+        } else {
+            st_idx = findMaxIn2DVector(llm_pos_ids_list.back()) + 1;
+        }
+        auto range = generateRange(text_len, st_idx);
+        auto expanded_matrix = expandToMatrix(range, 3);
+
+        llm_pos_ids_list.push_back(expanded_matrix);
+
+        // For Qwen3: t_index always starts from 0 (no scaling, timestamps handle time)
+        std::vector<int> t_index;
+        for (int ti = 0; ti < llm_grid_t; ++ti) {  // llm_grid_t=1 for videos/images typically
+            for (int hw = 0; hw < llm_grid_h * llm_grid_w; ++hw) {
+                t_index.push_back(ti + text_len + st_idx);  // No second_per_grid_t scaling; ti is 0 for t=1
+            }
+        }
+
+        std::vector<int> h_index;
+        for (int ti = 0; ti < llm_grid_t; ++ti) {
+            for (int hi = 0; hi < llm_grid_h; ++hi) {
+                for (int wi = 0; wi < llm_grid_w; ++wi) {
+                    h_index.push_back(hi + text_len + st_idx);
+                }
+            }
+        }
+
+        std::vector<int> w_index;
+        for (int ti = 0; ti < llm_grid_t; ++ti) {
+            for (int hi = 0; hi < llm_grid_h; ++hi) {
+                for (int wi = 0; wi < llm_grid_w; ++wi) {
+                    w_index.push_back(wi + text_len + st_idx);
+                }
+            }
+        }
+
+        std::vector<std::vector<int>> thw_idx;
+        thw_idx.push_back(t_index);
+        thw_idx.push_back(h_index);
+        thw_idx.push_back(w_index);
+        llm_pos_ids_list.push_back(thw_idx);
+
+        st = ed + llm_grid_t * llm_grid_h * llm_grid_w;
+
+        // Append remaining text if any
+        if (st < static_cast<int>(filtered_ids.size())) {
+            if (llm_pos_ids_list.empty()) {
+                st_idx = 0;
+            } else {
+                st_idx = findMaxIn2DVector(llm_pos_ids_list.back()) + 1;
+            }
+
+            text_len = static_cast<int>(filtered_ids.size()) - st;
+
+            range = generateRange(text_len, st_idx);
+            expanded_matrix = expandToMatrix(range, 3);
+            llm_pos_ids_list.push_back(expanded_matrix);
+        }
+    }
+
+    // Concatenate all position lists
+    for (const auto& item : llm_pos_ids_list) {
+        for (size_t pi = 0; pi < position_ids.size(); ++pi) {
+            position_ids[pi].insert(position_ids[pi].end(), item[pi].begin(), item[pi].end());
+        }
+    }
+
+    return position_ids;
+}
@@ -0,0 +1,33 @@
+#ifndef MROPE_QWEN3_H
+#define MROPE_QWEN3_H
+
+#include <vector>
+
+// Forward declaration of Config (assume defined in mrope.hpp or utils.hpp)
+struct Config {
+    struct VisionConfig {
+        int temporal_patch_size;
+        int tokens_per_second;
+        int spatial_merge_size;
+        int patch_size;
+        int width;
+        int height;
+        int fps;
+    };
+    VisionConfig vision_config;
+    int image_token_id;
+    int video_token_id;
+    int vision_start_token_id;
+
+    std::vector<std::vector<int>> image_grid_thw;   // auto calc
+    std::vector<std::vector<int>> video_grid_thw;   // auto calc
+};
+
+std::vector<std::vector<int>> get_rope_index(
+    const Config& config,
+    const std::vector<int>& input_ids,
+    const std::vector<std::vector<int>>& image_grid_thw,
+    const std::vector<std::vector<int>>& video_grid_thw
+);
+
+#endif // MROPE_QWEN3_H
@@ -471,6 +471,7 @@ if __name__ == "__main__":
        'llm-model-qwen2.5-1.5B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-1.5B-Int4-ax650', '0.4', src_folder, revision],
        'llm-model-qwen2.5-3B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-3B-Int4-ax650', '0.4', src_folder, revision],
        'llm-model-qwen2.5-7B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-7B-Int4-ax650', '0.4', src_folder, revision],
+        'llm-model-qwen3-vl-2B-Int4-ax650':[create_data_deb,'llm-model-qwen3-vl-2B-Int4-ax650', '0.5', src_folder, revision],
        # Llama model
        'llm-model-llama3.2-1B-prefill-ax630c':[create_data_deb,'llm-model-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision],
        'llm-model-llama3.2-1B-p256-ax630c':[create_data_deb,'llm-model-llama3.2-1B-p256-ax630c', '0.4', src_folder, revision],