From cc1087facacaf16ba61e268e41041c45c80fa65a Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Thu, 27 Nov 2025 17:29:20 +0800 Subject: [PATCH] [update] fix postprocess Div zero bug, update llm-openai-api, update lllm-vlm --- .../main_llm/src/runner/LLMPostprocess.hpp | 165 +++++++----------- .../llm_framework/main_openai_api/SConstruct | 2 +- projects/llm_framework/main_vlm/SConstruct | 2 +- .../models/mode_qwen3-vl-2B-Int4-ax630c.json | 2 +- projects/llm_framework/main_vlm/src/main.cpp | 2 + .../llm_framework/main_vlm/src/runner/LLM.hpp | 1 + .../main_vlm/src/runner/LLMPostprocess.hpp | 165 +++++++----------- 7 files changed, 126 insertions(+), 213 deletions(-) diff --git a/projects/llm_framework/main_llm/src/runner/LLMPostprocess.hpp b/projects/llm_framework/main_llm/src/runner/LLMPostprocess.hpp index b7d5015..08d6b4e 100644 --- a/projects/llm_framework/main_llm/src/runner/LLMPostprocess.hpp +++ b/projects/llm_framework/main_llm/src/runner/LLMPostprocess.hpp @@ -9,66 +9,50 @@ #include "utils/json.hpp" #include "utils/sample_log.h" -class LLMPostprocess -{ +class LLMPostprocess { private: - // 控制随机性 void apply_temperature(std::vector &logits, float temperature) { - for (float &logit : logits) - { + if (temperature == 0.0f) temperature = 0.01f; + for (float &logit : logits) { logit /= temperature; } } - // 防止重复 void apply_repetition_penalty(std::vector &logits, const std::vector &history, float penalty) { - for (int token : history) - { - if (token < logits.size()) - { + for (int token : history) { + if (token < logits.size()) { logits[token] = logits[token] < 0 ? logits[token] * penalty : logits[token] / penalty; } } } - void apply_repetition_penalty(std::vector &logits, - const std::vector &generated_tokens, - float repetition_penalty, - int penalty_window) + void apply_repetition_penalty(std::vector &logits, const std::vector &generated_tokens, + float repetition_penalty, int penalty_window) { - if (repetition_penalty == 1.0f || generated_tokens.empty()) - { - return; // 如果 penalty = 1.0 或者没有生成 token，则不进行修改 + if (repetition_penalty == 1.0f || generated_tokens.empty()) { + return; } int start_idx = std::max(0, (int)generated_tokens.size() - penalty_window); std::unordered_set recent_tokens(generated_tokens.begin() + start_idx, generated_tokens.end()); - for (int token : recent_tokens) - { - if (token < 0 || token >= logits.size()) - continue; + for (int token : recent_tokens) { + if (token < 0 || token >= logits.size()) continue; - if (logits[token] > 0) - { + if (logits[token] > 0) { logits[token] /= std::sqrt(repetition_penalty); - } - else - { + } else { logits[token] *= std::sqrt(repetition_penalty); } } } - // 增强多样性 void apply_diversity_penalty(std::vector &logits, const std::vector &common_phrases, float penalty) { - for (int token : common_phrases) - { - if (token < logits.size()) - { + for (int token : common_phrases) { + if (token < logits.size()) { logits[token] *= penalty; } } @@ -79,46 +63,37 @@ private: { std::vector probs(logits.size()); float max_logit = *std::max_element(logits.begin(), logits.end()); - float sum = 0.0f; + float sum = 0.0f; - for (size_t i = 0; i < logits.size(); ++i) - { + for (size_t i = 0; i < logits.size(); ++i) { probs[i] = std::exp(logits[i] - max_logit); sum += probs[i]; } - for (float &p : probs) - { + for (float &p : probs) { p /= sum; } return probs; } - // 动态裁剪低概率 token int faster_top_p_sampling(const std::vector &logits, float top_p) { - // 计算softmax std::vector probs = softmax(logits); - // 构建最大堆（概率和索引的配对） std::vector> prob_index; prob_index.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); ++i) - { + for (size_t i = 0; i < logits.size(); ++i) { prob_index.emplace_back(probs[i], i); } - auto cmp = [](const auto &a, const auto &b) - { return a.first < b.first; }; + auto cmp = [](const auto &a, const auto &b) { return a.first < b.first; }; std::make_heap(prob_index.begin(), prob_index.end(), cmp); - // 提取top-p元素 std::vector filtered_indices; std::vector filtered_probs; float cumulative_prob = 0.0f; - while (!prob_index.empty() && cumulative_prob < top_p) - { + while (!prob_index.empty() && cumulative_prob < top_p) { std::pop_heap(prob_index.begin(), prob_index.end(), cmp); auto [prob, index] = prob_index.back(); prob_index.pop_back(); @@ -127,15 +102,11 @@ private: filtered_indices.push_back(index); filtered_probs.push_back(prob); - if (cumulative_prob >= top_p) - break; + if (cumulative_prob >= top_p) break; } - // 处理边缘情况（概率全零时返回第一个元素） - if (filtered_indices.empty()) - return 0; + if (filtered_indices.empty()) return 0; - // 使用thread_local随机数生成器（线程安全） static thread_local std::mt19937 gen(std::random_device{}()); std::discrete_distribution dist(filtered_probs.begin(), filtered_probs.end()); return filtered_indices[dist(gen)]; @@ -147,31 +118,26 @@ private: // Sort indices by probability in descending order std::vector indices(logits.size()); std::iota(indices.begin(), indices.end(), 0); - std::sort(indices.begin(), indices.end(), [&](size_t i, size_t j) - { return probs[i] > probs[j]; }); + std::sort(indices.begin(), indices.end(), [&](size_t i, size_t j) { return probs[i] > probs[j]; }); // Compute cumulative probabilities float cumulative_prob = 0.0f; - size_t cut_off = 0; - for (; cut_off < indices.size(); ++cut_off) - { + size_t cut_off = 0; + for (; cut_off < indices.size(); ++cut_off) { cumulative_prob += probs[indices[cut_off]]; - if (cumulative_prob >= top_p) - break; + if (cumulative_prob >= top_p) break; } // Keep only the top-p probabilities std::vector filtered_indices(indices.begin(), indices.begin() + cut_off + 1); std::vector filtered_probs(filtered_indices.size()); - for (size_t i = 0; i < filtered_indices.size(); ++i) - { + for (size_t i = 0; i < filtered_indices.size(); ++i) { filtered_probs[i] = probs[filtered_indices[i]]; } // Normalize the probabilities float filtered_sum = std::accumulate(filtered_probs.begin(), filtered_probs.end(), 0.0f); - for (float &p : filtered_probs) - { + for (float &p : filtered_probs) { p /= filtered_sum; } @@ -182,34 +148,27 @@ private: return filtered_indices[dist(gen)]; } - // 限制候选 token 数 int top_k_sampling(const std::vector &logits, int k) { // std::vector probs = softmax(logits); - // 获取 top-k 索引 std::vector indices(logits.size()); std::iota(indices.begin(), indices.end(), 0); - std::partial_sort(indices.begin(), indices.begin() + k, indices.end(), [&](size_t i, size_t j) - { return logits[i] > logits[j]; }); + std::partial_sort(indices.begin(), indices.begin() + k, indices.end(), + [&](size_t i, size_t j) { return logits[i] > logits[j]; }); - // 仅保留 top-k 概率 std::vector filtered_indices(indices.begin(), indices.begin() + k); std::vector filtered_probs(k); - for (size_t i = 0; i < k; ++i) - { + for (size_t i = 0; i < k; ++i) { filtered_probs[i] = logits[filtered_indices[i]]; } filtered_probs = softmax(filtered_probs); - // 归一化 float sum = std::accumulate(filtered_probs.begin(), filtered_probs.end(), 0.0f); - for (float &p : filtered_probs) - { + for (float &p : filtered_probs) { p /= sum; } - // 采样 std::random_device rd; std::mt19937 gen(rd()); std::discrete_distribution dist(filtered_probs.begin(), filtered_probs.end()); @@ -217,64 +176,65 @@ private: } bool enable_temperature = false; - float temperature = 1.0f; + float temperature = 1.0f; bool enable_repetition_penalty = false; - float repetition_penalty = 1.0f; - int penalty_window = 20; + float repetition_penalty = 1.0f; + int penalty_window = 20; bool enable_diversity_penalty = false; std::vector common_phrases; float diversity_penalty = 1.0f; bool enable_top_p_sampling = false; - float top_p = 1.0f; + float top_p = 1.0f; bool enable_top_k_sampling = false; - int top_k = 1; + int top_k = 1; public: - LLMPostprocess() {} + LLMPostprocess() + { + } void set_temperature(bool enable, float temperature) { enable_temperature = enable; - this->temperature = temperature; + this->temperature = temperature; } void set_repetition_penalty(bool enable, float penalty, int penalty_window) { enable_repetition_penalty = enable; - this->repetition_penalty = penalty; - this->penalty_window = penalty_window; + this->repetition_penalty = penalty; + this->penalty_window = penalty_window; } void set_diversity_penalty(bool enable, const std::vector &common_phrases, float penalty) { enable_diversity_penalty = enable; - this->common_phrases = common_phrases; - this->diversity_penalty = penalty; + this->common_phrases = common_phrases; + this->diversity_penalty = penalty; } void set_top_p_sampling(bool enable, float top_p) { enable_top_k_sampling = false; enable_top_p_sampling = enable; - this->top_p = top_p; + this->top_p = top_p; } void set_top_k_sampling(bool enable, int top_k) { enable_top_p_sampling = false; enable_top_k_sampling = enable; - this->top_k = top_k; + this->top_k = top_k; } bool load_config(std::string config_path) { std::ifstream config_file(config_path); - if (!config_file.is_open()) - { + if (!config_file.is_open()) { ALOGE("config file(%s) open failed", config_path.c_str()); return false; } @@ -282,21 +242,21 @@ public: ALOGI("load config: \n%s\n", config.dump(4).c_str()); enable_temperature = config["enable_temperature"]; - temperature = config["temperature"]; + temperature = config["temperature"]; enable_repetition_penalty = config["enable_repetition_penalty"]; - repetition_penalty = config["repetition_penalty"]; - penalty_window = config["penalty_window"]; + repetition_penalty = config["repetition_penalty"]; + penalty_window = config["penalty_window"]; enable_top_p_sampling = config["enable_top_p_sampling"]; - top_p = config["top_p"]; + top_p = config["top_p"]; enable_top_k_sampling = config["enable_top_k_sampling"]; - top_k = config["top_k"]; + top_k = config["top_k"]; return true; } - bool load_config(const nlohmann::json& config) + bool load_config(const nlohmann::json &config) { if (config.is_null()) { ALOGE("config is null or invalid"); @@ -341,22 +301,17 @@ public: int apply(std::vector &logits, const std::vector &history) { - if (enable_temperature) - apply_temperature(logits, temperature); - if (enable_repetition_penalty) - apply_repetition_penalty(logits, history, repetition_penalty, penalty_window); - if (enable_diversity_penalty) - apply_diversity_penalty(logits, common_phrases, diversity_penalty); + if (enable_temperature) apply_temperature(logits, temperature); + if (enable_repetition_penalty) apply_repetition_penalty(logits, history, repetition_penalty, penalty_window); + if (enable_diversity_penalty) apply_diversity_penalty(logits, common_phrases, diversity_penalty); if (enable_top_p_sampling) return faster_top_p_sampling(logits, top_p); else if (enable_top_k_sampling) return top_k_sampling(logits, top_k); - else - { - // 最大值 + else { float max_logit = *std::max_element(logits.begin(), logits.end()); - int max_index = std::distance(logits.begin(), std::max_element(logits.begin(), logits.end())); + int max_index = std::distance(logits.begin(), std::max_element(logits.begin(), logits.end())); return max_index; } } diff --git a/projects/llm_framework/main_openai_api/SConstruct b/projects/llm_framework/main_openai_api/SConstruct index 0bde348..f6e45d1 100644 --- a/projects/llm_framework/main_openai_api/SConstruct +++ b/projects/llm_framework/main_openai_api/SConstruct @@ -18,7 +18,7 @@ LINK_SEARCH_PATH = [] STATIC_FILES = [] -ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/m5stack/ModuleLLM-OpenAI-Plugin.git', 'a8f54b0430c478896b45828f612d0d8b0a6f2fa1', True) +ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/m5stack/ModuleLLM-OpenAI-Plugin.git', '50efecaf566a04c1786b7eda2208a0cdefd53dde', True) python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.6.tar.gz", 'm5stack_llm-openai-api-python-venv_v1.6.tar.gz') diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index 1923997..2d05a3b 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -18,7 +18,7 @@ LINK_SEARCH_PATH = [] STATIC_FILES = [] if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: - python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-vlm-python-venv_v1.6.tar.gz", 'm5stack_llm-vlm-python-venv_v1.6.tar.gz') + python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-vlm-python-venv_v1.8.tar.gz", 'm5stack_llm-vlm-python-venv_v1.8.tar.gz') else: python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-vlm-python-venv_v1.7.tar.gz", 'm5stack_llm-vlm-python-venv_v1.7.tar.gz') diff --git a/projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax630c.json b/projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax630c.json index fce6c7c..72f1d24 100644 --- a/projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax630c.json +++ b/projects/llm_framework/main_vlm/models/mode_qwen3-vl-2B-Int4-ax630c.json @@ -46,7 +46,7 @@ "video_token_id": 151656, "vision_start_token_id": 151652, "precompute_len": 0, - "cmm_size": 3582336, + "cmm_size": 3098624, "ext_scripts": [ "tokenizer_qwen3-vl-2B-Int4-ax630c.py" ] diff --git a/projects/llm_framework/main_vlm/src/main.cpp b/projects/llm_framework/main_vlm/src/main.cpp index d4694e6..9b552d3 100644 --- a/projects/llm_framework/main_vlm/src/main.cpp +++ b/projects/llm_framework/main_vlm/src/main.cpp @@ -553,6 +553,8 @@ public: } if (lLaMa_) lLaMa_->Deinit(); if (lLaMa_) lLaMa_.reset(); + if (lLaMa_ctx_) lLaMa_ctx_->Deinit(); + if (lLaMa_ctx_) lLaMa_ctx_.reset(); if (qwen_) qwen_->Deinit(); if (qwen_) qwen_.reset(); return true; diff --git a/projects/llm_framework/main_vlm/src/runner/LLM.hpp b/projects/llm_framework/main_vlm/src/runner/LLM.hpp index bf56bf8..59dd87e 100644 --- a/projects/llm_framework/main_vlm/src/runner/LLM.hpp +++ b/projects/llm_framework/main_vlm/src/runner/LLM.hpp @@ -859,6 +859,7 @@ public: llama_layers[i].layer.release(); } llama_post.release(); + image_encoder.release(); embed_selector.Deinit(); } diff --git a/projects/llm_framework/main_vlm/src/runner/LLMPostprocess.hpp b/projects/llm_framework/main_vlm/src/runner/LLMPostprocess.hpp index b7d5015..08d6b4e 100644 --- a/projects/llm_framework/main_vlm/src/runner/LLMPostprocess.hpp +++ b/projects/llm_framework/main_vlm/src/runner/LLMPostprocess.hpp @@ -9,66 +9,50 @@ #include "utils/json.hpp" #include "utils/sample_log.h" -class LLMPostprocess -{ +class LLMPostprocess { private: - // 控制随机性 void apply_temperature(std::vector &logits, float temperature) { - for (float &logit : logits) - { + if (temperature == 0.0f) temperature = 0.01f; + for (float &logit : logits) { logit /= temperature; } } - // 防止重复 void apply_repetition_penalty(std::vector &logits, const std::vector &history, float penalty) { - for (int token : history) - { - if (token < logits.size()) - { + for (int token : history) { + if (token < logits.size()) { logits[token] = logits[token] < 0 ? logits[token] * penalty : logits[token] / penalty; } } } - void apply_repetition_penalty(std::vector &logits, - const std::vector &generated_tokens, - float repetition_penalty, - int penalty_window) + void apply_repetition_penalty(std::vector &logits, const std::vector &generated_tokens, + float repetition_penalty, int penalty_window) { - if (repetition_penalty == 1.0f || generated_tokens.empty()) - { - return; // 如果 penalty = 1.0 或者没有生成 token，则不进行修改 + if (repetition_penalty == 1.0f || generated_tokens.empty()) { + return; } int start_idx = std::max(0, (int)generated_tokens.size() - penalty_window); std::unordered_set recent_tokens(generated_tokens.begin() + start_idx, generated_tokens.end()); - for (int token : recent_tokens) - { - if (token < 0 || token >= logits.size()) - continue; + for (int token : recent_tokens) { + if (token < 0 || token >= logits.size()) continue; - if (logits[token] > 0) - { + if (logits[token] > 0) { logits[token] /= std::sqrt(repetition_penalty); - } - else - { + } else { logits[token] *= std::sqrt(repetition_penalty); } } } - // 增强多样性 void apply_diversity_penalty(std::vector &logits, const std::vector &common_phrases, float penalty) { - for (int token : common_phrases) - { - if (token < logits.size()) - { + for (int token : common_phrases) { + if (token < logits.size()) { logits[token] *= penalty; } } @@ -79,46 +63,37 @@ private: { std::vector probs(logits.size()); float max_logit = *std::max_element(logits.begin(), logits.end()); - float sum = 0.0f; + float sum = 0.0f; - for (size_t i = 0; i < logits.size(); ++i) - { + for (size_t i = 0; i < logits.size(); ++i) { probs[i] = std::exp(logits[i] - max_logit); sum += probs[i]; } - for (float &p : probs) - { + for (float &p : probs) { p /= sum; } return probs; } - // 动态裁剪低概率 token int faster_top_p_sampling(const std::vector &logits, float top_p) { - // 计算softmax std::vector probs = softmax(logits); - // 构建最大堆（概率和索引的配对） std::vector> prob_index; prob_index.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); ++i) - { + for (size_t i = 0; i < logits.size(); ++i) { prob_index.emplace_back(probs[i], i); } - auto cmp = [](const auto &a, const auto &b) - { return a.first < b.first; }; + auto cmp = [](const auto &a, const auto &b) { return a.first < b.first; }; std::make_heap(prob_index.begin(), prob_index.end(), cmp); - // 提取top-p元素 std::vector filtered_indices; std::vector filtered_probs; float cumulative_prob = 0.0f; - while (!prob_index.empty() && cumulative_prob < top_p) - { + while (!prob_index.empty() && cumulative_prob < top_p) { std::pop_heap(prob_index.begin(), prob_index.end(), cmp); auto [prob, index] = prob_index.back(); prob_index.pop_back(); @@ -127,15 +102,11 @@ private: filtered_indices.push_back(index); filtered_probs.push_back(prob); - if (cumulative_prob >= top_p) - break; + if (cumulative_prob >= top_p) break; } - // 处理边缘情况（概率全零时返回第一个元素） - if (filtered_indices.empty()) - return 0; + if (filtered_indices.empty()) return 0; - // 使用thread_local随机数生成器（线程安全） static thread_local std::mt19937 gen(std::random_device{}()); std::discrete_distribution dist(filtered_probs.begin(), filtered_probs.end()); return filtered_indices[dist(gen)]; @@ -147,31 +118,26 @@ private: // Sort indices by probability in descending order std::vector indices(logits.size()); std::iota(indices.begin(), indices.end(), 0); - std::sort(indices.begin(), indices.end(), [&](size_t i, size_t j) - { return probs[i] > probs[j]; }); + std::sort(indices.begin(), indices.end(), [&](size_t i, size_t j) { return probs[i] > probs[j]; }); // Compute cumulative probabilities float cumulative_prob = 0.0f; - size_t cut_off = 0; - for (; cut_off < indices.size(); ++cut_off) - { + size_t cut_off = 0; + for (; cut_off < indices.size(); ++cut_off) { cumulative_prob += probs[indices[cut_off]]; - if (cumulative_prob >= top_p) - break; + if (cumulative_prob >= top_p) break; } // Keep only the top-p probabilities std::vector filtered_indices(indices.begin(), indices.begin() + cut_off + 1); std::vector filtered_probs(filtered_indices.size()); - for (size_t i = 0; i < filtered_indices.size(); ++i) - { + for (size_t i = 0; i < filtered_indices.size(); ++i) { filtered_probs[i] = probs[filtered_indices[i]]; } // Normalize the probabilities float filtered_sum = std::accumulate(filtered_probs.begin(), filtered_probs.end(), 0.0f); - for (float &p : filtered_probs) - { + for (float &p : filtered_probs) { p /= filtered_sum; } @@ -182,34 +148,27 @@ private: return filtered_indices[dist(gen)]; } - // 限制候选 token 数 int top_k_sampling(const std::vector &logits, int k) { // std::vector probs = softmax(logits); - // 获取 top-k 索引 std::vector indices(logits.size()); std::iota(indices.begin(), indices.end(), 0); - std::partial_sort(indices.begin(), indices.begin() + k, indices.end(), [&](size_t i, size_t j) - { return logits[i] > logits[j]; }); + std::partial_sort(indices.begin(), indices.begin() + k, indices.end(), + [&](size_t i, size_t j) { return logits[i] > logits[j]; }); - // 仅保留 top-k 概率 std::vector filtered_indices(indices.begin(), indices.begin() + k); std::vector filtered_probs(k); - for (size_t i = 0; i < k; ++i) - { + for (size_t i = 0; i < k; ++i) { filtered_probs[i] = logits[filtered_indices[i]]; } filtered_probs = softmax(filtered_probs); - // 归一化 float sum = std::accumulate(filtered_probs.begin(), filtered_probs.end(), 0.0f); - for (float &p : filtered_probs) - { + for (float &p : filtered_probs) { p /= sum; } - // 采样 std::random_device rd; std::mt19937 gen(rd()); std::discrete_distribution dist(filtered_probs.begin(), filtered_probs.end()); @@ -217,64 +176,65 @@ private: } bool enable_temperature = false; - float temperature = 1.0f; + float temperature = 1.0f; bool enable_repetition_penalty = false; - float repetition_penalty = 1.0f; - int penalty_window = 20; + float repetition_penalty = 1.0f; + int penalty_window = 20; bool enable_diversity_penalty = false; std::vector common_phrases; float diversity_penalty = 1.0f; bool enable_top_p_sampling = false; - float top_p = 1.0f; + float top_p = 1.0f; bool enable_top_k_sampling = false; - int top_k = 1; + int top_k = 1; public: - LLMPostprocess() {} + LLMPostprocess() + { + } void set_temperature(bool enable, float temperature) { enable_temperature = enable; - this->temperature = temperature; + this->temperature = temperature; } void set_repetition_penalty(bool enable, float penalty, int penalty_window) { enable_repetition_penalty = enable; - this->repetition_penalty = penalty; - this->penalty_window = penalty_window; + this->repetition_penalty = penalty; + this->penalty_window = penalty_window; } void set_diversity_penalty(bool enable, const std::vector &common_phrases, float penalty) { enable_diversity_penalty = enable; - this->common_phrases = common_phrases; - this->diversity_penalty = penalty; + this->common_phrases = common_phrases; + this->diversity_penalty = penalty; } void set_top_p_sampling(bool enable, float top_p) { enable_top_k_sampling = false; enable_top_p_sampling = enable; - this->top_p = top_p; + this->top_p = top_p; } void set_top_k_sampling(bool enable, int top_k) { enable_top_p_sampling = false; enable_top_k_sampling = enable; - this->top_k = top_k; + this->top_k = top_k; } bool load_config(std::string config_path) { std::ifstream config_file(config_path); - if (!config_file.is_open()) - { + if (!config_file.is_open()) { ALOGE("config file(%s) open failed", config_path.c_str()); return false; } @@ -282,21 +242,21 @@ public: ALOGI("load config: \n%s\n", config.dump(4).c_str()); enable_temperature = config["enable_temperature"]; - temperature = config["temperature"]; + temperature = config["temperature"]; enable_repetition_penalty = config["enable_repetition_penalty"]; - repetition_penalty = config["repetition_penalty"]; - penalty_window = config["penalty_window"]; + repetition_penalty = config["repetition_penalty"]; + penalty_window = config["penalty_window"]; enable_top_p_sampling = config["enable_top_p_sampling"]; - top_p = config["top_p"]; + top_p = config["top_p"]; enable_top_k_sampling = config["enable_top_k_sampling"]; - top_k = config["top_k"]; + top_k = config["top_k"]; return true; } - bool load_config(const nlohmann::json& config) + bool load_config(const nlohmann::json &config) { if (config.is_null()) { ALOGE("config is null or invalid"); @@ -341,22 +301,17 @@ public: int apply(std::vector &logits, const std::vector &history) { - if (enable_temperature) - apply_temperature(logits, temperature); - if (enable_repetition_penalty) - apply_repetition_penalty(logits, history, repetition_penalty, penalty_window); - if (enable_diversity_penalty) - apply_diversity_penalty(logits, common_phrases, diversity_penalty); + if (enable_temperature) apply_temperature(logits, temperature); + if (enable_repetition_penalty) apply_repetition_penalty(logits, history, repetition_penalty, penalty_window); + if (enable_diversity_penalty) apply_diversity_penalty(logits, common_phrases, diversity_penalty); if (enable_top_p_sampling) return faster_top_p_sampling(logits, top_p); else if (enable_top_k_sampling) return top_k_sampling(logits, top_k); - else - { - // 最大值 + else { float max_logit = *std::max_element(logits.begin(), logits.end()); - int max_index = std::distance(logits.begin(), std::max_element(logits.begin(), logits.end())); + int max_index = std::distance(logits.begin(), std::max_element(logits.begin(), logits.end())); return max_index; } }