From 0d3e36fb49f415ea47ca336cdd8d6f430b268b6e Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Fri, 14 Nov 2025 14:18:10 +0800 Subject: [PATCH] [update] vlm support qwen3-vl model, add qwen3-vl-2b model. update pzmq close timeout to 1s. --- ext_components/StackFlow/stackflow/pzmq.hpp | 2 +- .../models/mode_qwen3-vl-2B-Int4-ax650.json | 54 ++ .../tokenizer_qwen3-vl-2B-Int4-ax650.py | 195 +++++ projects/llm_framework/main_vlm/src/main.cpp | 170 +++- .../llm_framework/main_vlm/src/runner/LLM.hpp | 761 ++++++++++++++++++ .../src/runner/Tokenizer/Tokenizer.cpp | 254 +++--- .../src/runner/Tokenizer/Tokenizer.hpp | 58 +- .../main_vlm/src/runner/utils/files.cpp | 46 ++ .../main_vlm/src/runner/utils/files.hpp | 17 + .../src/runner/utils/image_processor.cpp | 162 ++++ .../src/runner/utils/image_processor.hpp | 14 + .../main_vlm/src/runner/utils/mrope.cpp | 272 +++++++ .../main_vlm/src/runner/utils/mrope.hpp | 33 + projects/llm_framework/tools/llm_pack.py | 1 + 14 files changed, 1834 insertions(+), 205 deletions(-) create mode 100644 projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax650.json create mode 100644 projects/llm_framework/main_vlm/scripts/tokenizer_qwen3-vl-2B-Int4-ax650.py create mode 100644 projects/llm_framework/main_vlm/src/runner/utils/files.cpp create mode 100644 projects/llm_framework/main_vlm/src/runner/utils/files.hpp create mode 100644 projects/llm_framework/main_vlm/src/runner/utils/image_processor.cpp create mode 100644 projects/llm_framework/main_vlm/src/runner/utils/image_processor.hpp create mode 100644 projects/llm_framework/main_vlm/src/runner/utils/mrope.cpp create mode 100644 projects/llm_framework/main_vlm/src/runner/utils/mrope.hpp diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp index 8a29bd1..3adbab5 100644 --- a/ext_components/StackFlow/stackflow/pzmq.hpp +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -394,7 +394,7 @@ public: } void close_zmq() { - int linger = 0; + int linger = 1000; zmq_setsockopt(zmq_socket_, ZMQ_LINGER, &linger, sizeof(linger)); zmq_close(zmq_socket_); zmq_ctx_destroy(zmq_ctx_); diff --git a/projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax650.json b/projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax650.json new file mode 100644 index 0000000..7747056 --- /dev/null +++ b/projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax650.json @@ -0,0 +1,54 @@ +{ + "mode": "qwen3-vl-2B-Int4-ax650", + "type": "vlm", + "homepage": "https://huggingface.co/AXERA-TECH/Qwen3-VL-2B-Instruct", + "capabilities": [ + "text_generation", + "chat" + ], + "input_type": [ + "vlm.chat_completion", + "vlm.chat_completion.stream" + ], + "output_type": [ + "vlm.utf-8", + "vlm.utf-8.stream" + ], + "mode_param": { + "tokenizer_type": 2, + "url_tokenizer_model": "http://localhost:8080", + "filename_tokens_embed": "model.embed_tokens.weight.bfloat16.bin", + "filename_post_axmodel": "qwen3_vl_text_post.axmodel", + "template_filename_axmodel": "qwen3_vl_text_p128_l%d_together.axmodel", + "filename_image_encoder_axmodel": "Qwen3-VL-2B-Instruct_vision.axmodel", + "enable_temperature": true, + "temperature": 0.7, + "enable_top_p_sampling": false, + "top_p": 0.9, + "enable_top_k_sampling": true, + "top_k": 40, + "enable_repetition_penalty": false, + "repetition_penalty": 1.1, + "penalty_window": 50, + "axmodel_num": 28, + "tokens_embed_num": 151936, + "tokens_embed_size": 2048, + "b_use_mmap_load_embed": true, + "b_video": false, + "vision_config.temporal_patch_size": 2, + "vision_config.tokens_per_second": 2, + "vision_config.spatial_merge_size": 2, + "vision_config.patch_size": 16, + "vision_config.height": 384, + "vision_config.width": 384, + "vision_config.fps": 1, + "image_token_id": 151655, + "video_token_id": 151656, + "vision_start_token_id": 151652, + "precompute_len": 0, + "cmm_size": 1919044, + "ext_scripts": [ + "tokenizer_qwen3-vl-2B-Int4-ax650.py" + ] + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/scripts/tokenizer_qwen3-vl-2B-Int4-ax650.py b/projects/llm_framework/main_vlm/scripts/tokenizer_qwen3-vl-2B-Int4-ax650.py new file mode 100644 index 0000000..ed6f1aa --- /dev/null +++ b/projects/llm_framework/main_vlm/scripts/tokenizer_qwen3-vl-2B-Int4-ax650.py @@ -0,0 +1,195 @@ +from transformers import AutoTokenizer, PreTrainedTokenizerFast +from transformers.tokenization_utils_base import AddedToken +from http.server import HTTPServer, BaseHTTPRequestHandler +import json +import argparse + + +class Tokenizer_Http: + def __init__(self, model_id, system_content="You are a helpful assistant."): + self.tokenizer = AutoTokenizer.from_pretrained( + model_id, + trust_remote_code=True, + use_fast=False + ) + self.token_ids_cache = [] + self.system_content = system_content + + def encode(self, content): + text = [ + f'<|im_start|>system\n{self.system_content}<|im_end|>\n' + f'<|im_start|>user\n{content}<|im_end|>\n' + f'<|im_start|>assistant\n' + ] + input_ids = self.tokenizer(text) + return input_ids["input_ids"][0] + + def encode_vpm_image(self, content="Describe this image.", num_img=1, img_token_num=256): + imgs_token = ( + '<|vision_start|>' + + '<|image_pad|>' * img_token_num + + '<|vision_end|>' + ) + imgs_token *= num_img + text = ( + f'<|im_start|>system\n{self.system_content}<|im_end|>\n' + f'<|im_start|>user\n{imgs_token}{content}<|im_end|>\n' + f'<|im_start|>assistant\n' + ) + + output_kwargs = { + 'text_kwargs': {'padding': True, 'return_tensors': 'pt'}, + 'images_kwargs': {'return_tensors': 'pt'}, + 'audio_kwargs': {'padding': True, 'return_tensors': 'pt'}, + 'videos_kwargs': {'fps': 2.0, 'return_tensors': 'pt'}, + 'common_kwargs': {'return_tensors': 'pt'}, + } + + text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) + return text_inputs["input_ids"].tolist()[0] + + + def encode_vpm_video(self, content="Describe this image.", num_img=1, img_token_num=256): + imgs_token = ( + '<|vision_start|>' + + '<|video_pad|>' * img_token_num * num_img + + '<|vision_end|>' + ) + + text = ( + f'<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' + f'<|im_start|>user\n{imgs_token}{content}<|im_end|>\n' + f'<|im_start|>assistant\n' + ) + + output_kwargs = { + 'text_kwargs': {'padding': True, 'return_tensors': 'pt'}, + 'images_kwargs': {'return_tensors': 'pt'}, + 'audio_kwargs': {'padding': True, 'return_tensors': 'pt'}, + 'videos_kwargs': {'fps': 2.0, 'return_tensors': 'pt'}, + 'common_kwargs': {'return_tensors': 'pt'}, + } + + text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) + return text_inputs["input_ids"].tolist()[0] + + def decode(self, token_ids): + self.token_ids_cache += token_ids + text = self.tokenizer.decode(self.token_ids_cache) + if "\ufffd" in text: + print("text 中包含非法字符") + return "" + else: + self.token_ids_cache.clear() + return text + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def bos_token(self): + return self.tokenizer.bos_token + + @property + def eos_token(self): + return self.tokenizer.eos_token + + @property + def img_start_token(self): + return self.tokenizer.encode("<|vision_start|>")[0] + + @property + def img_context_token(self): + return self.tokenizer.encode("<|image_pad|>")[0] + + +class Request(BaseHTTPRequestHandler): + timeout = 5 + server_version = 'Apache' + + def do_GET(self): + print(self.path) + self.send_response(200) + self.send_header("type", "get") + self.end_headers() + + if self.path == '/bos_id': + bos_id = tokenizer.bos_id + msg = json.dumps({'bos_id': -1 if bos_id is None else bos_id}) + elif self.path == '/eos_id': + eos_id = tokenizer.eos_id + msg = json.dumps({'eos_id': -1 if eos_id is None else eos_id}) + elif self.path == '/img_start_token': + img_start_token = tokenizer.img_start_token + msg = json.dumps({'img_start_token': -1 if img_start_token is None else img_start_token}) + elif self.path == '/img_context_token': + img_context_token = tokenizer.img_context_token + msg = json.dumps({'img_context_token': -1 if img_context_token is None else img_context_token}) + else: + msg = 'error' + + print(msg) + msg = str(msg).encode() + self.wfile.write(msg) + + def do_POST(self): + data = self.rfile.read(int(self.headers['content-length'])) + req = json.loads(data.decode()) + + if self.path == "/encode": + prompt = req['text'] + b_img_prompt = req.get('img_prompt', False) + img_type = req.get('img_type', 'image') # 默认 image + + if b_img_prompt: + if img_type == 'image': + token_ids = tokenizer.encode_vpm_image( + prompt, + req.get("num_img", 1), + req.get("img_token_num", 256) + ) + elif img_type == 'video': + token_ids = tokenizer.encode_vpm_video( + prompt, + req.get("num_img", 1), + req.get("img_token_num", 256) + ) + else: + token_ids = tokenizer.encode(prompt) # fallback + else: + token_ids = tokenizer.encode(prompt) + + msg = json.dumps({'token_ids': -1 if token_ids is None else token_ids}) + + elif self.path == "/decode": + req = json.loads(data.decode()) + token_ids = req['token_ids'] + text = tokenizer.decode(token_ids) + msg = json.dumps({'text': "" if text is None else text}) + else: + msg = 'error' + + self.send_response(200) + self.end_headers() + self.wfile.write(str(msg).encode()) + + +if __name__ == "__main__": + args = argparse.ArgumentParser() + args.add_argument('--host', type=str, default='localhost') + args.add_argument('--port', type=int, default=8080) + args.add_argument('--model_id', type=str, default='tokenizer') + args.add_argument('--content', type=str, default='You are a helpful assistant.') + args = args.parse_args() + + tokenizer = Tokenizer_Http(args.model_id, system_content=args.content) + host = (args.host, args.port) + + print(f"http://{args.host}:{args.port}") + server = HTTPServer(host, Request) + server.serve_forever() \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index 7b5d5ea..7397ff6 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -42,17 +42,27 @@ typedef std::function task_callback_ else if (obj.contains(#key)) \ mode_config_.key = obj[#key]; +#define QWEN_CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + qwen_mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + qwen_mode_config_.key = obj[#key]; + class llm_task { private: static std::atomic next_port_; std::atomic_bool tokenizer_server_flage_; unsigned int port_; pid_t tokenizer_pid_ = -1; + enum class ModelType { Unknown = 0, Qwen, InternVL, InternVL_CTX }; + ModelType model_type_ = ModelType::Unknown; public: LLMAttrType mode_config_; + Config qwen_mode_config_; std::unique_ptr lLaMa_; std::unique_ptr lLaMa_ctx_; + std::unique_ptr qwen_; std::string model_; std::string response_format_; std::vector inputs_; @@ -61,6 +71,10 @@ public: std::vector> images_data; std::vector mats; std::vector img_embed; + std::vector> imgs_embed; + std::vector> deepstack_features; + std::vector visual_pos_mask; + std::vector> position_ids; std::string prompt_; std::string last_reply; std::vector tokens_ids, tokens_diff; @@ -121,6 +135,28 @@ public: } } + void vlm_reset() + { + std::vector().swap(prompt_data_); + + for (auto &inner_vec : imgs_embed) { + std::vector().swap(inner_vec); + } + std::vector>().swap(imgs_embed); + + for (auto &inner_vec : deepstack_features) { + std::vector().swap(inner_vec); + } + std::vector>().swap(deepstack_features); + + std::vector().swap(visual_pos_mask); + + for (auto &inner_vec : position_ids) { + std::vector().swap(inner_vec); + } + std::vector>().swap(position_ids); + } + int load_model(const nlohmann::json &config_body) { if (parse_config(config_body)) { @@ -179,7 +215,19 @@ public: CONFIG_AUTO_SET(file_body["mode_param"], vpm_width); CONFIG_AUTO_SET(file_body["mode_param"], vpm_height); CONFIG_AUTO_SET(file_body["mode_param"], precompute_len); + CONFIG_AUTO_SET(file_body["mode_param"], b_video); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.temporal_patch_size); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.tokens_per_second); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.spatial_merge_size); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.patch_size); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.width); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.height); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_config.fps); + + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], image_token_id); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], video_token_id); + QWEN_CONFIG_AUTO_SET(file_body["mode_param"], vision_start_token_id); { auto has_http = [](const std::string &s) { return s.find("http") != std::string::npos; }; @@ -196,10 +244,9 @@ public: auto start_tokenizer_server = [&](const std::string &tokenizer_file) { if (tokenizer_file.empty()) return; if (tokenizer_server_flage_.load()) return; - tokenizer_pid_ = fork(); if (tokenizer_pid_ == 0) { - setenv("PYTHONPATH", "/opt/m5stack/lib/llm/site-packages", 1); + setenv("PYTHONPATH", "/opt/m5stack/lib/vlm/site-packages", 1); const std::string port_str = std::to_string(port_); const std::string model_id = base_model + "tokenizer"; @@ -234,11 +281,25 @@ public: SLOGI("filename_tokenizer_model: %s", mode_config_.filename_tokenizer_model.c_str()); } } - mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed; - mode_config_.filename_post_axmodel = base_model + mode_config_.filename_post_axmodel; - mode_config_.template_filename_axmodel = base_model + mode_config_.template_filename_axmodel; + + { + std::string encoder_name = mode_config_.filename_image_encoder_axmodel; + std::transform(encoder_name.begin(), encoder_name.end(), encoder_name.begin(), ::tolower); + + if (encoder_name.find("qwen3") != std::string::npos) + model_type_ = ModelType::Qwen; + else if (encoder_name.find("internvl3") != std::string::npos && mode_config_.precompute_len > 0) + model_type_ = ModelType::InternVL_CTX; + else if (encoder_name.find("internvl3") != std::string::npos) + model_type_ = ModelType::InternVL; + else + model_type_ = ModelType::Unknown; + } + mode_config_.filename_tokens_embed = base_model + mode_config_.filename_tokens_embed; + mode_config_.filename_post_axmodel = base_model + mode_config_.filename_post_axmodel; + mode_config_.filename_image_encoder_axmodel = base_model + mode_config_.filename_image_encoder_axmodel; + mode_config_.template_filename_axmodel = base_model + mode_config_.template_filename_axmodel; mode_config_.filename_vpm_resampler_axmodedl = base_model + mode_config_.filename_vpm_resampler_axmodedl; - mode_config_.filename_image_encoder_axmodel = base_model + mode_config_.filename_image_encoder_axmodel; mode_config_.runing_callback = [this](int *p_token, int n_token, const char *p_str, float token_per_sec, void *reserve) { if (this->out_callback_) { @@ -246,20 +307,40 @@ public: } }; - if (mode_config_.precompute_len > 0) { - lLaMa_ctx_ = std::make_unique(); - if (!lLaMa_ctx_->Init(mode_config_)) { - lLaMa_ctx_->Deinit(); - lLaMa_ctx_.reset(); - return -2; + switch (model_type_) { + case ModelType::InternVL: { + lLaMa_ = std::make_unique(); + if (!lLaMa_->Init(mode_config_)) { + lLaMa_->Deinit(); + lLaMa_.reset(); + return -2; + } + break; } - } else { - lLaMa_ = std::make_unique(); - if (!lLaMa_->Init(mode_config_)) { - lLaMa_->Deinit(); - lLaMa_.reset(); - return -2; + + case ModelType::InternVL_CTX: { + lLaMa_ctx_ = std::make_unique(); + if (!lLaMa_ctx_->Init(mode_config_)) { + lLaMa_ctx_->Deinit(); + lLaMa_ctx_.reset(); + return -2; + } + break; } + + case ModelType::Qwen: { + qwen_ = std::make_unique(); + if (!qwen_->Init(mode_config_)) { + qwen_->Deinit(); + qwen_.reset(); + return -2; + } + break; + } + default: + ALOGE("Unknown model type in filename_image_encoder_axmodel: %s", + mode_config_.filename_image_encoder_axmodel.c_str()); + return -3; } if (lLaMa_ctx_) { @@ -421,6 +502,33 @@ public: if (out_callback_) out_callback_(last_reply, true); } } + + if (qwen_) { + if (images_data.empty()) { + qwen_->Encode(prompt_data_, position_ids, qwen_mode_config_, prompt_complete(msg)); + last_reply = qwen_->Run(prompt_data_, position_ids, deepstack_features, visual_pos_mask); + if (out_callback_) out_callback_(last_reply, true); + } else { + for (const auto &img_buf : images_data) { + cv::Mat src = cv::imdecode(img_buf, cv::IMREAD_COLOR); + if (src.empty()) { + std::cerr << "Decode failed!" << std::endl; + continue; + } + mats.push_back(src); + } + images_data.clear(); + if (mats.empty()) return; + std::vector> all_embeds; + qwen_->EncodeImage(mats, mode_config_.b_video, qwen_mode_config_, all_embeds, deepstack_features); + mats.clear(); + qwen_->Encode(all_embeds, mode_config_.b_video, prompt_data_, position_ids, visual_pos_mask, + qwen_mode_config_, prompt_complete(msg)); + last_reply = qwen_->Run(prompt_data_, position_ids, deepstack_features, visual_pos_mask); + if (out_callback_) out_callback_(last_reply, true); + vlm_reset(); + } + } } catch (...) { SLOGW("lLaMa_->Run have error!"); } @@ -428,7 +536,9 @@ public: bool pause() { - lLaMa_->Stop(); + if (lLaMa_) lLaMa_->Stop(); + if (lLaMa_ctx_) lLaMa_ctx_->Stop(); + if (qwen_) qwen_->Stop(); return true; } @@ -439,8 +549,10 @@ public: waitpid(tokenizer_pid_, nullptr, 0); tokenizer_pid_ = -1; } - lLaMa_->Deinit(); - lLaMa_.reset(); + if (lLaMa_) lLaMa_->Deinit(); + if (lLaMa_) lLaMa_.reset(); + if (qwen_) qwen_->Deinit(); + if (qwen_) qwen_.reset(); return true; } @@ -476,6 +588,12 @@ public: if (lLaMa_) { lLaMa_->Deinit(); } + if (lLaMa_ctx_) { + lLaMa_ctx_->Deinit(); + } + if (qwen_) { + qwen_->Deinit(); + } } }; @@ -530,12 +648,14 @@ public: if (!(llm_task_obj && llm_channel)) { return; } - llm_task_obj->lLaMa_->Stop(); + if (llm_task_obj->lLaMa_) llm_task_obj->lLaMa_->Stop(); + if (llm_task_obj->lLaMa_ctx_) llm_task_obj->lLaMa_ctx_->Stop(); + if (llm_task_obj->qwen_) llm_task_obj->qwen_->Stop(); } void pause(const std::string &work_id, const std::string &object, const std::string &data) override { - SLOGI("llm_asr::work:%s", data.c_str()); + SLOGI("llm_vlm::work:%s", data.c_str()); nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); @@ -626,7 +746,9 @@ public: if (!(llm_task_obj && llm_channel)) { return; } - llm_task_obj->lLaMa_->Stop(); + if (llm_task_obj->lLaMa_) llm_task_obj->lLaMa_->Stop(); + if (llm_task_obj->lLaMa_ctx_) llm_task_obj->lLaMa_ctx_->Stop(); + if (llm_task_obj->qwen_) llm_task_obj->qwen_->Stop(); } void task_camera_data(const std::weak_ptr llm_task_obj_weak, diff --git a/projects/llm_framework/main_vlm/src/runner/LLM.hpp b/projects/llm_framework/main_vlm/src/runner/LLM.hpp index 0caf464..6a3b90f 100644 --- a/projects/llm_framework/main_vlm/src/runner/LLM.hpp +++ b/projects/llm_framework/main_vlm/src/runner/LLM.hpp @@ -4,6 +4,8 @@ #include #include #include "bfloat16.hpp" +#include "image_processor.hpp" +#include "mrope.hpp" #include "Tokenizer/Tokenizer.hpp" #include "LLMEmbedSelector.hpp" #include "ax_model_runner/ax_model_runner_ax650.hpp" @@ -78,6 +80,7 @@ struct LLMAttrType { bool b_use_mmap_load_embed = false; bool b_dynamic_load_axmodel_layer = false; bool b_use_mmap_load_layer = true; + bool b_video = false; LLMRuningCallback runing_callback = nullptr; void *reserve = nullptr; @@ -1710,6 +1713,764 @@ public: final_out = tokenizer->Decode(token_ids); + return final_out; + } +}; + +class LLM_Qwen { +private: + std::shared_ptr tokenizer; + LLaMaEmbedSelector embed_selector; + + LLMAttrType _attr; + + struct LLMLayer { + ax_runner_ax650 layer; + std::string filename; + MMap layer_buffer; + std::vector layer_buffer_vec; + }; + + std::vector llama_layers; + ax_runner_ax650 llama_post; + ax_runner_ax650 image_encoder; + int deepstack_features_num = 3; + // int prefill_grpid = 1; + int decode_grpid = 0; + + bool b_stop = false; + + LLMPostprocess postprocess; + static int post_process(LLMPostprocess &postprocess, unsigned short *p, int n, std::vector &history, + float *val = 0) + { + std::vector logits(n); + for (int i = 0; i < n; i++) { + unsigned int proc = p[i] << 16; + logits[i] = *reinterpret_cast(&proc); + } + + return postprocess.apply(logits, history); + } + +public: + bool Init(LLMAttrType attr) + { + ALOGI("LLM init start"); + int remain_cmm = get_remaining_cmm_size(); + ALOGI("Total CMM:%d MB", remain_cmm); + + t_cqdm cqdm = create_cqdm(attr.axmodel_num + 3, 32); + this->_attr = attr; + tokenizer = CreateTokenizer(attr.tokenizer_type); + if (!tokenizer->Init(attr.url_tokenizer_model, attr.b_bos, attr.b_eos)) { + ALOGE("tokenizer.Init(%s, %d, %d) failed", attr.url_tokenizer_model.c_str(), attr.b_bos, attr.b_eos); + return false; + } + update_cqdm(&cqdm, 0, "count", "tokenizer init ok"); + + if (!embed_selector.Init(attr.filename_tokens_embed, attr.tokens_embed_num, attr.tokens_embed_size, + attr.b_use_mmap_load_embed)) { + ALOGE("embed_selector.Init(%s, %d, %d) failed", attr.filename_tokens_embed.c_str(), attr.tokens_embed_num, + attr.tokens_embed_size); + return false; + } + update_cqdm(&cqdm, 1, "count", "embed_selector init ok"); + + llama_layers.resize(attr.axmodel_num); + + ALOGI("attr.axmodel_num:%d", attr.axmodel_num); + char axmodel_path[1024]; + for (int i = 0; i < attr.axmodel_num; i++) { + sprintf(axmodel_path, attr.template_filename_axmodel.c_str(), i); + llama_layers[i].filename = axmodel_path; + + if (!attr.b_dynamic_load_axmodel_layer) { + int ret = llama_layers[i].layer.init(llama_layers[i].filename.c_str(), false); + if (ret != 0) { + ALOGE("init axmodel(%s) failed", llama_layers[i].filename.c_str()); + return false; + } + int remain_cmm = get_remaining_cmm_size(); + sprintf(axmodel_path, "init %d axmodel ok,remain_cmm(%d MB)", i, remain_cmm); + update_cqdm(&cqdm, i + 2, "count", axmodel_path); + } else { + if (!attr.b_use_mmap_load_layer) { + if (!read_file(llama_layers[i].filename, llama_layers[i].layer_buffer_vec)) { + ALOGE("read_file(%s) failed", llama_layers[i].filename.c_str()); + return false; + } + } else { + llama_layers[i].layer_buffer.open_file(llama_layers[i].filename.c_str()); + } + + sprintf(axmodel_path, "read_file %s ok", llama_layers[i].filename.c_str()); + update_cqdm(&cqdm, i + 2, "count", axmodel_path); + } + } + + int ret = llama_post.init(attr.filename_post_axmodel.c_str(), false); + if (ret != 0) { + ALOGE("init post axmodel(%s) failed", attr.filename_post_axmodel.c_str()); + return false; + } + remain_cmm = get_remaining_cmm_size(); + sprintf(axmodel_path, "init post axmodel ok,remain_cmm(%d MB)", remain_cmm); + update_cqdm(&cqdm, attr.axmodel_num + 2, "count", axmodel_path); + + ret = image_encoder.init(attr.filename_image_encoder_axmodel.c_str(), false); + if (ret != 0) { + ALOGE("init image_encoder axmodel(%s) failed", attr.filename_image_encoder_axmodel.c_str()); + return false; + } + + remain_cmm = get_remaining_cmm_size(); + sprintf(axmodel_path, "init vpm axmodel ok,remain_cmm(%d MB)", remain_cmm); + update_cqdm(&cqdm, attr.axmodel_num + 3, "count", axmodel_path); + + _attr.IMAGE_CONTEXT_TOKEN = tokenizer->GetImgContextID(); + _attr.IMAGE_START_TOKEN = tokenizer->GetImgStartID(); + + ALOGI("IMAGE_CONTEXT_TOKEN: %d, IMAGE_START_TOKEN: %d", _attr.IMAGE_CONTEXT_TOKEN, _attr.IMAGE_START_TOKEN); + + _attr.IMAGE_ENCODER_INPUT_NCHW = -1; + for (size_t i = 1; i < image_encoder.get_input(0).vShape.size(); i++) { + if (image_encoder.get_input(0).vShape[i] == 3) { + if (i == 1) { + _attr.IMAGE_ENCODER_INPUT_NCHW = 1; + } else if (i == 3) { + _attr.IMAGE_ENCODER_INPUT_NCHW = 0; + } + } + } + if (_attr.IMAGE_ENCODER_INPUT_NCHW == -1) { + ALOGE("image encoder input nchw or nhwc not found"); + return false; + } + + if (_attr.IMAGE_ENCODER_INPUT_NCHW == 1) { + ALOGE("Qwen2.5_VL Image Encoder just support NHWC"); + return false; + } + + int output_elem_size = 1; + for (int i = 0; i < image_encoder.get_output(0).vShape.size(); i++) { + output_elem_size *= image_encoder.get_output(0).vShape[i]; + } + + if (output_elem_size * 2 == image_encoder.get_output(0).nSize) { + _attr.IMAGE_ENCODER_OUTPUT_BF16 = 1; + ALOGI("image encoder output bf16"); + } else if (output_elem_size * 4 == image_encoder.get_output(0).nSize) { + _attr.IMAGE_ENCODER_OUTPUT_BF16 = 0; + ALOGI("image encoder output float32"); + } else { + ALOGE("image encoder output not support"); + return false; + } + + if (attr.b_dynamic_load_axmodel_layer) { + auto &layer = llama_layers[0]; + int ret; + if (_attr.b_use_mmap_load_layer) { + ret = layer.layer.init((char *)layer.layer_buffer.data(), layer.layer_buffer.size()); + } else { + ret = layer.layer.init(layer.layer_buffer_vec.data(), layer.layer_buffer_vec.size()); + } + if (ret != 0) { + ALOGE("init axmodel(%s) failed", layer.filename.c_str()); + } + } + + { + _attr.max_token_len = llama_layers[0].layer.get_input("mask").nSize / sizeof(unsigned short) - 1; + printf("\n"); + ALOGI("max_token_len : %d", _attr.max_token_len); + _attr.kv_cache_size = llama_layers[0].layer.get_output("K_cache_out").nSize / sizeof(unsigned short); + _attr.kv_cache_num = + llama_layers[0].layer.get_input("K_cache").nSize / _attr.kv_cache_size / sizeof(unsigned short); + ALOGI("kv_cache_size : %d, kv_cache_num: %d", _attr.kv_cache_size, _attr.kv_cache_num); + if (_attr.max_token_len > _attr.kv_cache_num) { + ALOGE("max_token_len(%d) > kv_cache_num(%d)", _attr.max_token_len, _attr.kv_cache_num); + return false; + } + + _attr.prefill_token_num = llama_layers[0].layer.get_input(1, "indices").vShape[1]; + ALOGI("prefill_token_num : %d", _attr.prefill_token_num); + for (size_t i = 0; i < llama_layers[0].layer.get_num_input_groups() - 1; i++) { + int prefill_max_kv_cache_num = llama_layers[0].layer.get_input(i + 1, "K_cache").vShape[1]; + ALOGI("grp: %ld, prefill_max_token_num : %d", i + 1, prefill_max_kv_cache_num); + _attr.prefill_max_kv_cache_num_grp.push_back(prefill_max_kv_cache_num); + } + _attr.prefill_max_token_num = + _attr.prefill_max_kv_cache_num_grp[_attr.prefill_max_kv_cache_num_grp.size() - 1]; + ALOGI("prefill_max_token_num : %d", _attr.prefill_max_token_num); + } + if (attr.b_dynamic_load_axmodel_layer) { + for (int i = 0; i < attr.axmodel_num; i++) { + auto &layer = llama_layers[i]; + layer.layer.deinit(); + } + } + + nlohmann::json dynamic_config; + + dynamic_config["enable_temperature"] = _attr.enable_temperature; + dynamic_config["temperature"] = _attr.temperature; + + dynamic_config["enable_repetition_penalty"] = _attr.enable_repetition_penalty; + dynamic_config["repetition_penalty"] = _attr.repetition_penalty; + dynamic_config["penalty_window"] = _attr.penalty_window; + + dynamic_config["enable_top_p_sampling"] = _attr.enable_top_p_sampling; + dynamic_config["top_p"] = _attr.top_p; + + dynamic_config["enable_top_k_sampling"] = _attr.enable_top_k_sampling; + dynamic_config["top_k"] = _attr.top_k; + + if (!postprocess.load_config(attr.post_config_path)) { + ALOGW("load postprocess config(%s) failed", attr.post_config_path.c_str()); + } + + if (!postprocess.load_config(dynamic_config)) { + ALOGW("load postprocess config(%s) failed", dynamic_config.dump(4).c_str()); + } + + ALOGI("LLM init ok"); + remain_cmm = get_remaining_cmm_size(); + ALOGI("Left CMM:%d MB", remain_cmm); + return true; + } + + LLMAttrType *getAttr() + { + return &_attr; + } + + LLMPostprocess *getPostprocess() + { + return &postprocess; + } + + void Deinit() + { + for (int i = 0; i < _attr.axmodel_num; i++) { + llama_layers[i].layer.release(); + } + llama_post.release(); + image_encoder.release(); + embed_selector.Deinit(); + } + + void Stop() + { + b_stop = true; + } + + int EncodeImage(std::vector &src, bool b_video, Config &cfg, + std::vector> &out_embed, + std::vector> &deepstack_features) + { + int temporal_patch_size = cfg.vision_config.temporal_patch_size; + int merge_size = cfg.vision_config.spatial_merge_size; + int patch_size = cfg.vision_config.patch_size; + int ret; + timer t; + t.start(); + + unsigned int grid_h = cfg.vision_config.height / cfg.vision_config.patch_size; + unsigned int grid_w = cfg.vision_config.width / cfg.vision_config.patch_size; + std::vector> pixel_values; + int w = cfg.vision_config.width, h = cfg.vision_config.height; + int channel = src[0].channels(); + int hwc = grid_h * grid_w * temporal_patch_size * patch_size * patch_size * channel; + + if (!b_video) { + for (int i = 0; i < src.size(); i++) { + std::vector> img_values; + std::vector si{src[i]}; + Qwen2VideoProcessor(si, img_values, h, w, temporal_patch_size, merge_size, patch_size); + pixel_values.push_back(img_values[0]); + } + for (size_t i = 0; i < pixel_values.size(); i++) { + cfg.image_grid_thw.push_back({1, static_cast(grid_h), static_cast(grid_w)}); + } + } else { + Qwen2VideoProcessor(src, pixel_values, h, w, temporal_patch_size, merge_size, patch_size); + cfg.video_grid_thw = { + {static_cast(pixel_values.size()), static_cast(grid_h), static_cast(grid_w)}}; + } + + ALOGI("pixel_values size %d", pixel_values.size()); + ALOGI("grid_h %d grid_w %d", grid_h, grid_w); + int cnt = 0; + if (out_embed.empty()) { + out_embed.resize(pixel_values.size()); + } + + deepstack_features.clear(); + for (int i = 0; i < pixel_values.size(); i++) { + void *data = image_encoder.get_input(0).pVirAddr; + memcpy(data, pixel_values[i].data(), hwc); + image_encoder.inference(); + + size_t size = image_encoder.get_output(0).nSize / sizeof(float); + if (out_embed[i].empty()) { + out_embed[i].resize(size); + } + + float *output_data = (float *)image_encoder.get_output(0).pVirAddr; + for (size_t j = 0; j < size; j++) { + out_embed[i][j] = bfloat16(output_data[j]).data; + } + + for (int j = 0; j < deepstack_features_num; j++) { + size_t size = image_encoder.get_output(j + 1).nSize / sizeof(float); + std::vector feature(size); + + float *output_data = (float *)image_encoder.get_output(j + 1).pVirAddr; + for (size_t k = 0; k < size; k++) { + feature[k] = output_data[k]; + } + + // 将不同image 的 deepstack feature 拼接到一起 + if (i == 0) { + deepstack_features.push_back(feature); + } else { + deepstack_features[j].insert(deepstack_features[j].end(), feature.begin(), feature.end()); + } + } + } + + ALOGI("image encode time : %f ms, size : %d", t.cost(), out_embed.size()); + return 0; + } + + int GetPositionIds(std::vector &input_ids, std::vector> &position_ids, Config &cfg) + { + position_ids = get_rope_index(cfg, input_ids, cfg.image_grid_thw, cfg.video_grid_thw); + return 0; + } + + int Encode(std::vector &out_embed, std::vector> &position_ids, Config &cfg, + std::string prompt = "What is in the image?") + { + ImageInfo img_info; + img_info.img_prompt = false; + std::vector input_ids = tokenizer->Encode(prompt, img_info); + if (input_ids.size() > _attr.prefill_max_token_num) { + ALOGE("input_ids(%d) > prefill_max_token_num(%d)", input_ids.size(), _attr.prefill_max_token_num); + return -1; + } + out_embed.resize(input_ids.size() * _attr.tokens_embed_size); + + for (size_t i = 0; i < input_ids.size(); i++) { + embed_selector.getByIndex(input_ids[i], out_embed.data() + i * _attr.tokens_embed_size); + } + + cfg.image_grid_thw.clear(); + cfg.video_grid_thw.clear(); + GetPositionIds(input_ids, position_ids, cfg); + return 0; + } + + int Encode(std::vector> &img_embed, bool b_video, + std::vector &out_embed, std::vector> &position_ids, + std::vector &visual_pos_mask, Config &cfg, std::string prompt = "What is in the image?") + { + ImageInfo img_info; + img_info.img_prompt = true; + img_info.type = b_video ? ImgType::Video : ImgType::Image; + img_info.img_token_num = img_embed[0].size() / _attr.tokens_embed_size; + img_info.num_img = img_embed.size(); + std::vector input_ids = tokenizer->Encode(prompt, img_info); + ALOGI("input_ids size:%d", input_ids.size()); + std::vector offsets; + int vision_start_token_id = cfg.vision_start_token_id; + for (size_t i = 0; i < input_ids.size() - 1; i++) { + if (input_ids[i] == vision_start_token_id) { + int offset = i + 1; + ALOGI("offset %d", offset); + offsets.push_back(offset); + } + } + + visual_pos_mask.resize(input_ids.size()); + for (size_t i = 0; i < input_ids.size(); i++) { + if (input_ids[i] == cfg.image_token_id || input_ids[i] == cfg.video_token_id) { + visual_pos_mask[i] = 1; + } else { + visual_pos_mask[i] = 0; + } + } + + if (input_ids.size() > _attr.prefill_max_token_num) { + ALOGE("input_ids(%ld) > prefill_max_token_num(%d)", input_ids.size(), _attr.prefill_max_token_num); + return -1; + } + out_embed.resize(input_ids.size() * _attr.tokens_embed_size); + + for (size_t i = 0; i < input_ids.size(); i++) { + embed_selector.getByIndex(input_ids[i], out_embed.data() + i * _attr.tokens_embed_size); + } + ALOGI("img_embed.size:%d, %d", img_embed.size(), img_embed[0].size()); + + if (offsets.size() == 1 && img_embed.size() > 1) { + for (int i = 1; i < img_embed.size(); i++) { + offsets.push_back(offsets[i - 1] + img_embed[i - 1].size() / _attr.tokens_embed_size); + ALOGI("offset:%d", offsets[i - 1] + img_embed[i - 1].size() / _attr.tokens_embed_size); + } + } + + for (int i = 0; i < img_embed.size(); i++) { + memcpy(out_embed.data() + offsets[i] * _attr.tokens_embed_size, img_embed[i].data(), + img_embed[i].size() * sizeof(unsigned short)); + } + + ALOGI("out_embed size:%d", out_embed.size()); + ALOGI("input_ids size %d", input_ids.size()); + GetPositionIds(input_ids, position_ids, cfg); + ALOGI("position_ids size:%d", position_ids[0].size()); + return 0; + } + + std::string Run(std::vector &test_embed, std::vector> &position_ids, + std::vector> &deepstack_features, std::vector &visual_pos_mask) + { + b_stop = false; + std::string final_out; + + bfloat16 bf16 = -65536.f; + std::vector mask(_attr.kv_cache_num + 1, bf16.data); + std::vector embed(_attr.tokens_embed_size, 0); + + std::vector cached_token; + std::vector token_ids; + + int input_embed_num = test_embed.size() / _attr.tokens_embed_size; + int prefill_split_num = ceil((double)input_embed_num / _attr.prefill_token_num); + ALOGI("input token num : %d, prefill_split_num : %d", input_embed_num, prefill_split_num); + if (input_embed_num > _attr.prefill_max_token_num) { + ALOGE("input token num(%d) > prefill_max_token_num(%d)", input_embed_num, _attr.prefill_max_token_num); + return ""; + } + + int kv_cache_num; + mask[_attr.kv_cache_num] = 0; + for (size_t i = 0; i < input_embed_num; i++) { + mask[i] = 0; + } + timer t_cost; + timer ttft_timer; + ttft_timer.start(); + + int max_pos_id = 0; + for (size_t p = 0; p < prefill_split_num; p++) { + if (b_stop) { + break; + } + _attr.prefill_grpid = p + 1; + kv_cache_num = p * _attr.prefill_token_num; + std::vector mask_tmp; + mask_tmp.resize(1 * _attr.prefill_token_num * (kv_cache_num + _attr.prefill_token_num), bf16.data); + int input_num_token = _attr.prefill_token_num; + if (p == prefill_split_num - 1) { + input_num_token = input_embed_num - p * _attr.prefill_token_num; + } + + ALOGI("input_num_token:%d", input_num_token); + for (size_t i = 0; i < _attr.prefill_token_num; i++) { + if (i < input_num_token) { + int mask_current_start = kv_cache_num; + auto mask_ptr = mask_tmp.data() + i * (kv_cache_num + _attr.prefill_token_num); + + for (int j = 0; j < _attr.precompute_len + p * _attr.prefill_token_num; j++) { + mask_ptr[j] = 0; + } + + for (int j = mask_current_start; j < mask_current_start + i + 1; j++) { + mask_ptr[j] = 0; + } + } + } + + std::vector embed_tmp(_attr.prefill_token_num * _attr.tokens_embed_size, 0); + int start, offset; + + start = p * _attr.prefill_token_num; + if (p == (prefill_split_num - 1)) { + offset = (input_embed_num - p * _attr.prefill_token_num); + memcpy(embed_tmp.data(), test_embed.data() + start * _attr.tokens_embed_size, + offset * _attr.tokens_embed_size * sizeof(unsigned short)); + } else { + offset = _attr.prefill_token_num; + memcpy(embed_tmp.data(), test_embed.data() + start * _attr.tokens_embed_size, + offset * _attr.tokens_embed_size * sizeof(unsigned short)); + } + + int start_deepstack_feat = 0, offset_deepstack_feat = 0; + if (!visual_pos_mask.empty()) { + for (int j = 0; j < start; j++) { + start_deepstack_feat += visual_pos_mask[j]; + } + for (int j = start; j < start + offset; j++) { + offset_deepstack_feat += visual_pos_mask[j]; + } + } + + for (unsigned int m = 0; m < _attr.axmodel_num; m++) { + if (b_stop) { + break; + } + + auto &layer = llama_layers[m]; + + if (_attr.b_dynamic_load_axmodel_layer) { + int ret; + if (_attr.b_use_mmap_load_layer) { + ret = layer.layer.init((char *)layer.layer_buffer.data(), layer.layer_buffer.size()); + } else { + ret = layer.layer.init(layer.layer_buffer_vec.data(), layer.layer_buffer_vec.size()); + } + if (ret != 0) { + ALOGE("init axmodel(%s) failed", layer.filename.c_str()); + } + } + + // set indices + auto &input_indices = layer.layer.get_input(_attr.prefill_grpid, "indices"); + unsigned int *input_indices_ptr = (unsigned int *)input_indices.pVirAddr; + memset(input_indices_ptr, 0, input_indices.nSize); + for (unsigned int i = 0; i < position_ids.size(); i++) { + for (unsigned int j = _attr.precompute_len + p * _attr.prefill_token_num, jj = 0; + j < _attr.precompute_len + (p + 1) * _attr.prefill_token_num; j++, jj++) { + if (j < position_ids[i].size()) { + input_indices_ptr[i * _attr.prefill_token_num + jj] = position_ids[i][j]; + if (position_ids[i][j] > max_pos_id) { + max_pos_id = position_ids[i][j]; + } + } + } + } + + // set mask + auto &input_mask = layer.layer.get_input(_attr.prefill_grpid, "mask"); + memcpy((void *)input_mask.pVirAddr, (void *)mask_tmp.data(), mask_tmp.size() * sizeof(unsigned short)); + // set input + auto &input_input = layer.layer.get_input(_attr.prefill_grpid, "input"); + memcpy((void *)input_input.pVirAddr, embed_tmp.data(), embed_tmp.size() * sizeof(unsigned short)); + + layer.layer.inference(_attr.prefill_grpid); + + auto &input_decoder_k_cache = layer.layer.get_input(decode_grpid, "K_cache"); + auto &input_decoder_v_cache = layer.layer.get_input(decode_grpid, "V_cache"); + + auto &output_k_cache = layer.layer.get_output(_attr.prefill_grpid, "K_cache_out"); + auto &output_v_cache = layer.layer.get_output(_attr.prefill_grpid, "V_cache_out"); + + int kv_offset = (_attr.precompute_len + p * _attr.prefill_token_num) * _attr.kv_cache_size; + + memcpy((unsigned short *)input_decoder_k_cache.pVirAddr + kv_offset, (void *)output_k_cache.pVirAddr, + sizeof(unsigned short) * input_num_token * _attr.kv_cache_size); + + memcpy((unsigned short *)input_decoder_v_cache.pVirAddr + kv_offset, (void *)output_v_cache.pVirAddr, + sizeof(unsigned short) * input_num_token * _attr.kv_cache_size); + + for (int gid = _attr.prefill_grpid + 1; gid < prefill_split_num + 1; gid++) { + auto &input_prefill_k_cache = layer.layer.get_input(gid, "K_cache"); + + memcpy((unsigned short *)input_prefill_k_cache.pVirAddr + kv_offset, + (void *)output_k_cache.pVirAddr, + sizeof(unsigned short) * input_num_token * _attr.kv_cache_size); + } + + for (int gid = _attr.prefill_grpid + 1; gid < prefill_split_num + 1; gid++) { + auto &input_prefill_v_cache = layer.layer.get_input(gid, "V_cache"); + + memcpy((unsigned short *)input_prefill_v_cache.pVirAddr + kv_offset, + (void *)output_v_cache.pVirAddr, + sizeof(unsigned short) * input_num_token * _attr.kv_cache_size); + } + + auto &output = layer.layer.get_output(_attr.prefill_grpid, "output"); + + memcpy(embed_tmp.data(), (void *)output.pVirAddr, embed_tmp.size() * sizeof(unsigned short)); + + if (!visual_pos_mask.empty() && m < deepstack_features.size()) { + int k = 0; + for (int j = start, k = start_deepstack_feat; j < start + offset; j++) { + if (visual_pos_mask[j] == 0) { + continue; + } + for (int di = 0; di < _attr.tokens_embed_size; di++) { + // bfloat16 to float32 + unsigned int tmp_bf16_1 = embed_tmp[(j - start) * _attr.tokens_embed_size + di] << 16; + float tmp_fp32_1 = *reinterpret_cast(&tmp_bf16_1); + + float tmp_fp32_2 = deepstack_features[m][k * _attr.tokens_embed_size + di]; + // float32 to bfloat16 + embed_tmp[(j - start) * _attr.tokens_embed_size + di] = + bfloat16(tmp_fp32_1 + tmp_fp32_2).data; + } + k++; + } + } + + if (_attr.b_dynamic_load_axmodel_layer) { + layer.layer.deinit(); + } + } + if (p == (prefill_split_num - 1)) { + memcpy(embed.data(), + embed_tmp.data() + (input_embed_num - p * _attr.prefill_token_num - 1) * _attr.tokens_embed_size, + _attr.tokens_embed_size * sizeof(unsigned short)); + } + } + + int next_token = -1; + t_cqdm cqdm = create_cqdm(_attr.max_token_len, 32); + + { + auto &input = llama_post.get_input(0); + memcpy((void *)input.pVirAddr, embed.data(), embed.size() * sizeof(unsigned short)); + llama_post.inference(); + + int max_index; + + auto &output_post = llama_post.get_output(0); + memcpy(output_post.pVirAddr, (void *)output_post.pVirAddr, output_post.nSize); + unsigned short *post_out = (unsigned short *)output_post.pVirAddr; + float max_val = -MAXFLOAT; + max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val); + + next_token = max_index; + + token_ids.push_back(max_index); + cached_token.push_back(max_index); + ALOGI("ttft: %.2f ms", ttft_timer.cost()); + } + t_cost.start(); + + bool b_hit_eos = false; + + for (unsigned int indices = max_pos_id + 1; indices < _attr.max_token_len; indices++) { + if (b_stop) { + break; + } + + embed_selector.getByIndex(next_token, embed); + + memcpy((void *)llama_layers[0].layer.get_input(decode_grpid, "input").pVirAddr, embed.data(), + llama_layers[0].layer.get_input(decode_grpid, "input").nSize); + + for (int m = 0; m < _attr.axmodel_num; m++) { + if (b_stop) { + break; + } + + auto &layer = llama_layers[m]; + + if (_attr.b_dynamic_load_axmodel_layer) { + int ret; + if (_attr.b_use_mmap_load_layer) { + ret = layer.layer.init((char *)layer.layer_buffer.data(), layer.layer_buffer.size()); + } else { + ret = layer.layer.init(layer.layer_buffer_vec.data(), layer.layer_buffer_vec.size()); + } + if (ret != 0) { + ALOGE("init axmodel(%s) failed", layer.filename.c_str()); + } + } + + auto &input_k_cache = layer.layer.get_input(decode_grpid, "K_cache"); + auto &input_v_cache = layer.layer.get_input(decode_grpid, "V_cache"); + + auto &input_indices = layer.layer.get_input(decode_grpid, "indices"); + memcpy((void *)input_indices.pVirAddr, &indices, sizeof(indices)); + + auto &input_mask = layer.layer.get_input(decode_grpid, "mask"); + memcpy((void *)input_mask.pVirAddr, mask.data(), mask.size() * sizeof(unsigned short)); + + layer.layer.inference(decode_grpid); + + auto &output_k_cache = layer.layer.get_output(decode_grpid, "K_cache_out"); + memcpy((unsigned short *)input_k_cache.pVirAddr + indices * _attr.kv_cache_size, + (void *)output_k_cache.pVirAddr, output_k_cache.nSize); + + auto &output_v_cache = layer.layer.get_output(decode_grpid, "V_cache_out"); + memcpy((unsigned short *)input_v_cache.pVirAddr + indices * _attr.kv_cache_size, + (void *)output_v_cache.pVirAddr, output_v_cache.nSize); + + if (m == _attr.axmodel_num - 1) { + memcpy((void *)llama_post.get_input(0).pVirAddr, + (void *)layer.layer.get_output(decode_grpid, "output").pVirAddr, + llama_post.get_input(0).nSize); + } else if (m < _attr.axmodel_num - 1) { + memcpy((void *)llama_layers[m + 1].layer.get_input(decode_grpid, "input").pVirAddr, + (void *)layer.layer.get_output(decode_grpid, "output").pVirAddr, + layer.layer.get_input(decode_grpid, "input").nSize); + } + } + mask[indices] = 0; + { + llama_post.inference(); + + auto &output_post = llama_post.get_output(0); + memcpy(output_post.pVirAddr, (void *)output_post.pVirAddr, output_post.nSize); + unsigned short *post_out = (unsigned short *)output_post.pVirAddr; + float max_val = -MAXFLOAT; + auto max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, nullptr); + + next_token = max_index; + + if (tokenizer->isEnd(max_index)) { + if (cached_token.size() && _attr.runing_callback) { + float t_cost_ms = t_cost.cost(); + float token_per_sec = token_ids.size() / (t_cost_ms / 1000); + auto tmp_out = tokenizer->Decode(cached_token); + _attr.runing_callback(cached_token.data(), cached_token.size(), tmp_out.c_str(), token_per_sec, + _attr.reserve); + cached_token.clear(); + } + b_hit_eos = true; + break; + } + token_ids.push_back(max_index); + + if (_attr.runing_callback) { + cached_token.push_back(max_index); + if (cached_token.size() >= 3) { + float t_cost_ms = t_cost.cost(); + float token_per_sec = token_ids.size() / (t_cost_ms / 1000); + auto tmp_out = tokenizer->Decode(cached_token); + _attr.runing_callback(cached_token.data(), cached_token.size(), tmp_out.c_str(), token_per_sec, + _attr.reserve); + cached_token.clear(); + } + } + } + + if (_attr.runing_callback == nullptr) update_cqdm(&cqdm, indices, "token", ""); + if (b_hit_eos) { + break; + } + } + printf("\n\n"); + fflush(stdout); + float t_cost_ms = t_cost.cost(); + ALOGN("hit eos,avg %.2f token/s\n", token_ids.size() / (t_cost_ms / 1000)); + + final_out = tokenizer->Decode(token_ids); + + for (size_t i = 0; i < _attr.axmodel_num; i++) { + for (size_t j = 0; j < llama_layers[i].layer.get_num_input_groups(); j++) { + memset((void *)llama_layers[i].layer.get_input(j, "K_cache").pVirAddr, 0, + llama_layers[i].layer.get_input(j, "K_cache").nSize); + memset((void *)llama_layers[i].layer.get_input(j, "V_cache").pVirAddr, 0, + llama_layers[i].layer.get_input(j, "V_cache").nSize); + } + } + return final_out; } }; \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.cpp b/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.cpp index 38974cb..918951b 100644 --- a/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.cpp +++ b/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.cpp @@ -8,8 +8,7 @@ #include "string_utility.hpp" #include "memory_utils.hpp" -class Tokenizer_Http : public BaseTokenizer -{ +class Tokenizer_Http : public BaseTokenizer { std::shared_ptr cli; bool _b_bos, _b_eos; @@ -26,13 +25,10 @@ public: bool Init(std::string model_path) override { base_url = model_path; - if (!test_connect_http(base_url, 10)) - { + if (!test_connect_http(base_url, 10)) { ALOGE("connect %s failed", base_url.c_str()); return false; - } - else - { + } else { ALOGI("connect %s ok", base_url.c_str()); } @@ -42,25 +38,20 @@ public: cli->set_write_timeout(10); int try_count = 10; - int count = try_count; - while (count-- > 0) - { - try - { + int count = try_count; + while (count-- > 0) { + try { auto ret = cli->Get("/get_uid"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get uid failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - uid = j["uid"]; + uid = j["uid"]; ALOGI("uid: %s", uid.c_str()); break; - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << e.what() << '\n'; } std::this_thread::sleep_for(std::chrono::seconds(1)); @@ -68,23 +59,18 @@ public: } count = 10; - while (count-- > 0) - { - try - { + while (count-- > 0) { + try { auto ret = cli->Get("/bos_id?uid=" + uid); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get bos_id failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - bos_id = j["bos_id"]; + bos_id = j["bos_id"]; break; - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << e.what() << '\n'; } std::this_thread::sleep_for(std::chrono::seconds(1)); @@ -92,23 +78,18 @@ public: } count = 10; - while (count-- > 0) - { - try - { + while (count-- > 0) { + try { auto ret = cli->Get("/eos_id?uid=" + uid); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get eos_id failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - eos_id = j["eos_id"]; + eos_id = j["eos_id"]; break; - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << e.what() << '\n'; } std::this_thread::sleep_for(std::chrono::seconds(1)); @@ -116,28 +97,23 @@ public: } count = 10; - while (count-- > 0) - { - try - { + while (count-- > 0) { + try { auto ret = cli->Get("/img_start_token?uid=" + uid); if (!ret) { ALOGE("get img_start_token failed, no response"); continue; } auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get img_start_token failed, status: %d", rep.status); continue; } nlohmann::json j = nlohmann::json::parse(rep.body); - img_start_token = j["img_start_token"]; + img_start_token = j["img_start_token"]; ALOGI("img_start_token: %d", img_start_token); break; - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << "Exception: " << e.what() << '\n'; } std::this_thread::sleep_for(std::chrono::seconds(1)); @@ -145,28 +121,23 @@ public: } count = 10; - while (count-- > 0) - { - try - { + while (count-- > 0) { + try { auto ret = cli->Get("/img_context_token?uid=" + uid); if (!ret) { ALOGE("get img_context_token failed, no response"); continue; } auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get img_context_token failed, status: %d", rep.status); continue; } - nlohmann::json j = nlohmann::json::parse(rep.body); + nlohmann::json j = nlohmann::json::parse(rep.body); img_context_token = j["img_context_token"]; ALOGI("img_context_token: %d", img_context_token); break; - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << "Exception: " << e.what() << '\n'; } std::this_thread::sleep_for(std::chrono::seconds(1)); @@ -182,8 +153,14 @@ public: bool Init(std::string model_path = "http://localhost:8080", bool b_bos = true, bool b_eos = false) override { base_url = model_path; - try - { + if (!test_connect_http(base_url, 10)) { + ALOGE("connect %s failed", base_url.c_str()); + return false; + } else { + ALOGI("connect %s ok", base_url.c_str()); + } + + try { cli = std::make_shared(base_url); cli->set_connection_timeout(1); cli->set_read_timeout(1); @@ -191,30 +168,26 @@ public: { auto ret = cli->Get("/bos_id"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get bos_id failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - bos_id = j["bos_id"]; + bos_id = j["bos_id"]; } { auto ret = cli->Get("/eos_id"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get eos_id failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - eos_id = j["eos_id"]; + eos_id = j["eos_id"]; } printf("bos_id: %d, eos_id: %d\n", bos_id, eos_id); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << e.what() << '\n'; return false; } @@ -227,18 +200,14 @@ public: bool Init_new(std::string model_path, bool b_bos, bool b_eos) override { base_url = model_path; - if (!test_connect_http(base_url, 10)) - { + if (!test_connect_http(base_url, 10)) { ALOGE("connect %s failed", base_url.c_str()); return false; - } - else - { + } else { ALOGI("connect %s ok", base_url.c_str()); } - try - { + try { cli = std::make_shared(base_url); cli->set_connection_timeout(1); cli->set_read_timeout(1); @@ -246,56 +215,50 @@ public: { auto ret = cli->Get("/bos_id"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get bos_id failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - bos_id = j["bos_id"]; + bos_id = j["bos_id"]; } { auto ret = cli->Get("/eos_id"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get eos_id failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - eos_id = j["eos_id"]; + eos_id = j["eos_id"]; } ALOGI("bos_id: %d, eos_id: %d", bos_id, eos_id); { auto ret = cli->Get("/img_start_token"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get img_start_token failed, status: %d", rep.status); return false; } nlohmann::json j = nlohmann::json::parse(rep.body); - img_start_token = j["img_start_token"]; + img_start_token = j["img_start_token"]; } ALOGI("img_start_token: %d", img_start_token); { auto ret = cli->Get("/img_context_token"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("get img_context_token failed, status: %d", rep.status); return false; } - nlohmann::json j = nlohmann::json::parse(rep.body); + nlohmann::json j = nlohmann::json::parse(rep.body); img_context_token = j["img_context_token"]; } ALOGI("img_context_token: %d", img_context_token); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { std::cerr << e.what() << '\n'; return false; } @@ -309,59 +272,53 @@ public: { nlohmann::json j; j["uid"] = uid; - if (!system_prompt.empty() and system_prompt != "") - { + if (!system_prompt.empty() and system_prompt != "") { j["system_prompt"] = system_prompt; } auto ret = cli->Post("/reset", j.dump(), "application/json"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("reset failed, status: %d", rep.status); return false; } - nlohmann::json j_rep = nlohmann::json::parse(rep.body); + nlohmann::json j_rep = nlohmann::json::parse(rep.body); std::vector _token_ids = j_rep["token_ids"]; - tokens = _token_ids; + tokens = _token_ids; return true; } - bool Encode(std::string input, std::string last_reply, std::vector &tokens, std::vector &tokens_diff, ImageInfo img_info) override + bool Encode(std::string input, std::string last_reply, std::vector &tokens, std::vector &tokens_diff, + ImageInfo img_info) override { nlohmann::json j; - j["uid"] = uid; - j["text"] = input; + j["uid"] = uid; + j["text"] = input; j["img_prompt"] = img_info.img_prompt; - j["imgsz"] = img_info.imgsz; - j["num_img"] = img_info.num_img; - if (!last_reply.empty() and last_reply != "") - { + j["imgsz"] = img_info.imgsz; + j["num_img"] = img_info.num_img; + if (!last_reply.empty() and last_reply != "") { j["last_reply"] = last_reply; } auto ret = cli->Post("/encode", j.dump(), "application/json"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("encode failed, status: %d", rep.status); return false; } nlohmann::json j2; - try - { + try { j2 = nlohmann::json::parse(rep.body); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { ALOGE("json parse failed: %s", e.what()); ALOGE("%s", rep.body.c_str()); return false; } - std::vector _token_ids = j2["token_ids"]; + std::vector _token_ids = j2["token_ids"]; std::vector _tokens_diff = j2["diff"]; - tokens = _token_ids; + tokens = _token_ids; tokens_diff = _tokens_diff; return true; @@ -370,38 +327,38 @@ public: bool Encode(std::string input, std::vector &output, ImageInfo img_info) override { nlohmann::json j; - j["text"] = input; - j["img_prompt"] = img_info.img_prompt; - j["imgsz"] = img_info.imgsz; - j["num_img"] = img_info.num_img; + j["text"] = input; + j["img_prompt"] = img_info.img_prompt; + j["imgsz"] = img_info.imgsz; + j["num_img"] = img_info.num_img; + j["img_token_num"] = img_info.img_token_num; + if (img_info.type == ImgType::Image) + j["img_type"] = "image"; + else if (img_info.type == ImgType::Video) + j["img_type"] = "video"; + auto ret = cli->Post("/encode", j.dump(), "application/json"); auto rep = ret.value(); - if (rep.status != 200) - { + if (rep.status != 200) { ALOGE("encode failed, status: %d", rep.status); return false; } nlohmann::json j2; - try - { + try { j2 = nlohmann::json::parse(rep.body); - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { ALOGE("json parse failed: %s", e.what()); ALOGE("%s", rep.body.c_str()); return false; } std::vector out = j2["token_ids"]; - output = out; + output = out; // output = sp->encode(input, 1024); - if (_b_bos) - { + if (_b_bos) { output.insert(output.begin(), bos_id); } - if (_b_eos) - { + if (_b_eos) { output.push_back(eos_id); } @@ -415,7 +372,8 @@ public: return output; } - std::vector Encode_ctx(std::string input, ImageInfo img_info, std::vector &tokens_ids, std::vector &tokens_diff) override + std::vector Encode_ctx(std::string input, ImageInfo img_info, std::vector &tokens_ids, + std::vector &tokens_diff) override { std::vector output; Encode(input, "", output, tokens_diff, img_info); @@ -424,30 +382,25 @@ public: std::string Decode(const std::vector input) override { - int cnt = 2; + int cnt = 2; std::string out_str = ""; - while (cnt--) - { + while (cnt--) { nlohmann::json j; j["token_ids"] = input; - j["uid"] = uid; - auto ret = cli->Post("/decode", j.dump(), "application/json"); - auto rep = ret.value(); - if (rep.status != 200) - { + j["uid"] = uid; + auto ret = cli->Post("/decode", j.dump(), "application/json"); + auto rep = ret.value(); + if (rep.status != 200) { ALOGE("decode failed, status: %d, try again", rep.status); ALOGE("%s", rep.body.c_str()); usleep(1000 * 1000); continue; } - try - { + try { nlohmann::json j2 = nlohmann::json::parse(rep.body); - out_str = j2["text"]; + out_str = j2["text"]; break; - } - catch (const std::exception &e) - { + } catch (const std::exception &e) { ALOGE("json parse failed: %s, try again", e.what()); ALOGE("%s", rep.body.c_str()); usleep(1000 * 1000); @@ -480,12 +433,11 @@ public: std::shared_ptr CreateTokenizer(TokenizerType type) { - switch (type) - { - case TKT_HTTP: - return std::make_shared(); - default: - ALOGE("unknown tokenizer type: %d", type); - return nullptr; + switch (type) { + case TKT_HTTP: + return std::make_shared(); + default: + ALOGE("unknown tokenizer type: %d", type); + return nullptr; } } \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.hpp b/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.hpp index e82bdfd..df30e61 100644 --- a/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.hpp +++ b/projects/llm_framework/main_vlm/src/runner/Tokenizer/Tokenizer.hpp @@ -3,40 +3,40 @@ #include #include -enum TokenizerType -{ - TKT_LLaMa, - TKT_Qwen, - TKT_HTTP, - TKT_Phi3, - TKT_END +enum TokenizerType { TKT_LLaMa, TKT_Qwen, TKT_HTTP, TKT_Phi3, TKT_END }; + +enum class ImgType { None, Image, Video }; + +struct ImageInfo { + int imgsz = 448; + int num_img = 1; + bool img_prompt = false; + int img_token_num = -1; + ImgType type = ImgType::None; }; -struct ImageInfo -{ - int imgsz = 448; - int num_img = 1; - bool img_prompt = false; -}; - -class BaseTokenizer -{ +class BaseTokenizer { public: - virtual bool Init(std::string model_path) = 0; - virtual bool Init(std::string model_path, bool b_bos, bool b_eos) = 0; - virtual bool Init_new(std::string model_path, bool b_bos, bool b_eos) = 0; - virtual bool Reset(std::string system_prompt, std::vector &tokens) = 0; - virtual bool Encode(std::string input, std::string last_reply, std::vector &tokens, std::vector &tokens_diff, ImageInfo img_info) = 0; + virtual bool Init(std::string model_path) = 0; + virtual bool Init(std::string model_path, bool b_bos, bool b_eos) = 0; + virtual bool Init_new(std::string model_path, bool b_bos, bool b_eos) = 0; + virtual bool Reset(std::string system_prompt, std::vector &tokens) = 0; + virtual bool Encode(std::string input, std::string last_reply, std::vector &tokens, + std::vector &tokens_diff, ImageInfo img_info) = 0; virtual bool Encode(std::string input, std::vector &output, ImageInfo img_info) = 0; - virtual std::vector Encode(std::string input, ImageInfo img_info) = 0; - virtual std::vector Encode_ctx(std::string input, ImageInfo img_info, std::vector &tokens_ids, std::vector &tokens_diff) = 0; - virtual std::string Decode(const std::vector input) = 0; - virtual int GetBosID() = 0; - virtual int GetEosID() = 0; - virtual int GetImgStartID() = 0; - virtual int GetImgContextID() = 0; + virtual std::vector Encode(std::string input, ImageInfo img_info) = 0; + virtual std::vector Encode_ctx(std::string input, ImageInfo img_info, std::vector &tokens_ids, + std::vector &tokens_diff) = 0; + virtual std::string Decode(const std::vector input) = 0; + virtual int GetBosID() = 0; + virtual int GetEosID() = 0; + virtual int GetImgStartID() = 0; + virtual int GetImgContextID() = 0; - virtual bool isEnd(int id) { return id == GetEosID(); } + virtual bool isEnd(int id) + { + return id == GetEosID(); + } }; std::shared_ptr CreateTokenizer(TokenizerType type); \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/src/runner/utils/files.cpp b/projects/llm_framework/main_vlm/src/runner/utils/files.cpp new file mode 100644 index 0000000..0222cea --- /dev/null +++ b/projects/llm_framework/main_vlm/src/runner/utils/files.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include +#include +#include +#include + +bool is_directory(const std::string& path) { + struct stat st; + if (stat(path.c_str(), &st) != 0) return false; + return S_ISDIR(st.st_mode); +} + +bool is_file(const std::string& path) { + struct stat st; + if (stat(path.c_str(), &st) != 0) return false; + return S_ISREG(st.st_mode); +} + +std::vector list_files(const std::string& directory) { + std::vector files; + DIR* dir = opendir(directory.c_str()); + if (!dir) { + std::cerr << "无法打开目录: " << directory << std::endl; + return files; + } + + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + std::string name = entry->d_name; + if (name == "." || name == "..") continue; + + // 拼接完整路径并检查是否为普通文件 + std::string full_path = directory + "/" + name; + struct stat st; + if (stat(full_path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) { + files.push_back(full_path); + } + } + closedir(dir); + + // 按文件名升序排序 + std::sort(files.begin(), files.end()); + return files; +} \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/src/runner/utils/files.hpp b/projects/llm_framework/main_vlm/src/runner/utils/files.hpp new file mode 100644 index 0000000..79f7060 --- /dev/null +++ b/projects/llm_framework/main_vlm/src/runner/utils/files.hpp @@ -0,0 +1,17 @@ +#ifndef _FILES_H_ +#define _FILES_H_ + +#include +#include +#include +#include +#include +#include +#include + +bool is_directory(const std::string& path); +bool is_file(const std::string& path); +std::vector list_files(const std::string& directory); + + +#endif \ No newline at end of file diff --git a/projects/llm_framework/main_vlm/src/runner/utils/image_processor.cpp b/projects/llm_framework/main_vlm/src/runner/utils/image_processor.cpp new file mode 100644 index 0000000..c9efb09 --- /dev/null +++ b/projects/llm_framework/main_vlm/src/runner/utils/image_processor.cpp @@ -0,0 +1,162 @@ +#include +#include +#include +#include "files.hpp" +#include "image_processor.hpp" +#include + +std::vector ReadImages(std::string path){ + std::vector src; + + if(is_file(path)){ + cv::Mat img = cv::imread(path, cv::IMREAD_COLOR); + src.push_back(img); + } + else if(is_directory(path)){ + auto paths = list_files(path); + + for(auto &p : paths){ + std::cout< SmartResize(int height, int width, int factor){ + int h_bar = height/factor; + int w_bar = width/factor; + + h_bar *= factor; + w_bar *= factor; + return {h_bar, w_bar}; +} + +void normalizeMeanStd(cv::Mat& image) { + // 确保输入图像是浮点类型（避免整数溢出） + cv::Mat floatImage; + image.convertTo(floatImage, CV_32F); // 转换为32位浮点格式