From eff3a479280ca07323f6b0ee039617abd163473d Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Fri, 19 Dec 2025 17:31:29 +0800 Subject: [PATCH] [update] perf llm-asr, kws add buttons control. --- projects/llm_framework/main_asr/src/main.cpp | 74 ++++++++- projects/llm_framework/main_kws/src/main.cpp | 162 ++++++++++++++++++- projects/llm_framework/tools/llm_pack.py | 2 +- 3 files changed, 225 insertions(+), 13 deletions(-) diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 59242c9..78457cf 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -85,6 +85,23 @@ private: ENGINE_ONLINE = 3, } engine_type_ = ENGINE_NCNN; + static constexpr int kSampleRate = 16000; + static constexpr int kFrameSamples = 160; + int pre_roll_frames_ = 30; + std::deque pre_roll_pcm_; + bool prev_vad_detected_ = false; + +private: + void PushPreRollPcm(const int16_t *pcm, size_t n) + { + pre_roll_pcm_.insert(pre_roll_pcm_.end(), pcm, pcm + n); + + const size_t max_samples = (size_t)pre_roll_frames_ * kFrameSamples; + while (pre_roll_pcm_.size() > max_samples) { + pre_roll_pcm_.pop_front(); + } + } + public: std::string model_; std::string response_format_; @@ -529,48 +546,86 @@ public: void sys_pcm_on_data_onnx(const std::string &raw) { + if (raw.size() >= sizeof(int16_t)) { + const int16_t *pcm16 = reinterpret_cast(raw.data()); + size_t n16 = raw.size() / sizeof(int16_t); + PushPreRollPcm(pcm16, n16); + } + static int count = 0; if (count < delay_audio_frame_) { buffer_write_char(pcmdata, raw.data(), raw.length()); count++; return; } + buffer_write_char(pcmdata, raw.data(), raw.length()); buffer_position_set(pcmdata, 0); std::vector floatSamples; + floatSamples.reserve((delay_audio_frame_ + 1) * kFrameSamples); + int16_t audio_val; while (buffer_read_i16(pcmdata, &audio_val, 1)) { - float normalizedSample = static_cast(audio_val) / INT16_MAX; - floatSamples.push_back(normalizedSample); + floatSamples.push_back(static_cast(audio_val) / 32768.0f); } + buffer_resize(pcmdata, 0); count = 0; vad_->AcceptWaveform(floatSamples.data(), floatSamples.size()); + + bool detected = vad_->IsSpeechDetected(); + bool speech_start = (!prev_vad_detected_ && detected); + prev_vad_detected_ = detected; + while (!vad_->Empty()) { const auto &segment = vad_->Front(); - if (!offline_stream_) offline_stream_ = onnx_recognizer_->CreateStream(); - offline_stream_->AcceptWaveform(onnx_asr_config_.feat_config.sampling_rate, segment.samples.data(), - segment.samples.size()); + + if (!offline_stream_) { + offline_stream_ = onnx_recognizer_->CreateStream(); + } + + if (speech_start && !pre_roll_pcm_.empty()) { + std::vector pre; + pre.reserve(pre_roll_pcm_.size()); + for (int16_t s : pre_roll_pcm_) { + pre.push_back(static_cast(s) / 32768.0f); + } + + std::vector merged; + merged.reserve(pre.size() + segment.samples.size()); + merged.insert(merged.end(), pre.begin(), pre.end()); + merged.insert(merged.end(), segment.samples.begin(), segment.samples.end()); + + offline_stream_->AcceptWaveform(kSampleRate, merged.data(), merged.size()); + + pre_roll_pcm_.clear(); + speech_start = false; + } else { + offline_stream_->AcceptWaveform(kSampleRate, segment.samples.data(), segment.samples.size()); + } + onnx_recognizer_->DecodeStream(offline_stream_.get()); + const auto &result = offline_stream_->GetResult(); if (!result.text.empty() && out_callback_) { out_callback_(result.text, true); } + vad_->Pop(); + offline_stream_.reset(); } { - bool detected = vad_->IsSpeechDetected(); float chunk_ms = (delay_audio_frame_ + 1) * 10.0f; - if (detected) { silence_ms_accum_ = 0.0f; } else { silence_ms_accum_ += chunk_ms; } + if (silence_ms_accum_ >= silence_timeout) { if (ensleep_) { if (pause) pause(); @@ -1034,6 +1089,11 @@ public: ++it; } } + + if (data.find("sys") != std::string::npos) { + llm_task_obj->audio_flage_ = false; + } + send("None", "None", LLM_NO_ERROR, work_id); } diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index d14437f..4883478 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -81,6 +81,7 @@ private: int count_frames_ = 0; long long last_trigger_time_ms_ = -1e9; long long frame_index_global_ = 0; + int last_btn_204_state = -1; public: inline const std::string &model() const @@ -294,9 +295,9 @@ public: #undef CONFIG_AUTO_SET_SHERPA #define CONFIG_AUTO_SET_AXERA(obj, key) \ - if (config_body.contains(#key)) \ + if (config_body.contains(#key)) \ axera_config_.key = config_body[#key]; \ - else if (obj.contains(#key)) \ + else if (obj.contains(#key)) \ axera_config_.key = obj[#key]; #define OPTS_AUTO_SET(obj, key) \ @@ -537,9 +538,27 @@ public: } } - void trigger() + void trigger_wakeup() { - if (out_callback_) out_callback_("", true); + if (enwake_audio_ && (!wake_wav_file_.empty()) && play_awake_wav) { + play_awake_wav(wake_wav_file_); + } + if (out_callback_) { + if (enoutput_json_) + out_callback_("{\"reason\":\"button_204\"}", true); + else + out_callback_("", true); + } + } + + void set_btn_204_state(int state) + { + last_btn_204_state = state; + } + + int get_btn_204_state() + { + return last_btn_204_state; } bool delete_model() @@ -790,6 +809,40 @@ public: llm_task_obj->sys_pcm_on_data((*next_data)); } + void task_buttons_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if (data.empty() || (data == "None")) return; + + try { + std::string user_msg = sample_unescapeString(data); + nlohmann::json btn_json = nlohmann::json::parse(user_msg); + + if (btn_json.contains("code") && btn_json.contains("vale")) { + int current_code = btn_json["code"]; + int current_vale = btn_json["vale"]; + + if (current_vale == 204) { + int last_code = llm_task_obj->get_btn_204_state(); + + if (last_code == 0 && current_code == 1) { + llm_task_obj->trigger_wakeup(); + } + + llm_task_obj->set_btn_204_state(current_code); + } + } + } catch (const std::exception &e) { + SLOGE("Button data JSON parse error: %s", e.what()); + } + } + int setup(const std::string &work_id, const std::string &object, const std::string &data) override { nlohmann::json error_body; @@ -836,6 +889,17 @@ public: llm_channel->subscriber_work_id("", std::bind(&llm_kws::task_user_data, this, _llm_task_obj, std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); + } else if (input.find("buttons_thread") != std::string::npos) { + std::string socket_url = "ipc:///tmp/llm/ec_prox.event.socket"; + auto business_logic = std::bind( + &llm_kws::task_buttons_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2); + + llm_channel->subscriber( + socket_url, [llm_channel, business_logic](StackFlows::pzmq *p, + const std::shared_ptr &d) { + llm_channel->subscriber_event_call(business_logic, p, d); + }); } } llm_task_[work_id_num] = llm_task_obj; @@ -851,6 +915,94 @@ public: } } + void link(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_kws::link:%s", data.c_str()); + int ret = 0; + nlohmann::json error_body; + + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + + auto llm_channel = get_channel(work_id); + auto llm_task_obj = llm_task_[work_id_num]; + + if (data.find("sys") != std::string::npos) { + if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data); + + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::shared_ptr &raw) { + if (auto p = _llm_task_obj.lock()) p->sys_pcm_on_data(raw->string()); + }); + + llm_task_obj->audio_flage_ = true; + llm_task_obj->inputs_.push_back(data); + } else if (data.find("buttons_thread") != std::string::npos) { + std::string socket_url = "ipc:///tmp/llm/ec_prox.event.socket"; + auto business_logic = + std::bind(&llm_kws::task_buttons_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2); + + llm_channel->subscriber( + socket_url, + [llm_channel, business_logic](StackFlows::pzmq *p, const std::shared_ptr &d) { + llm_channel->subscriber_event_call(business_logic, p, d); + }); + + llm_task_obj->inputs_.push_back(data); + } else { + error_body["code"] = -22; + error_body["message"] = "unsupported link target"; + send("None", "None", error_body, work_id); + return; + } + + if (ret) { + error_body["code"] = -20; + error_body["message"] = "link false"; + send("None", "None", error_body, work_id); + return; + } + send("None", "None", LLM_NO_ERROR, work_id); + } + + void unlink(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_kws::unlink:%s", data.c_str()); + nlohmann::json error_body; + + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + + auto llm_channel = get_channel(work_id); + auto llm_task_obj = llm_task_[work_id_num]; + + llm_channel->stop_subscriber_work_id(data); + + for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) { + if (*it == data) + it = llm_task_obj->inputs_.erase(it); + else + ++it; + } + + if (data.find("sys") != std::string::npos) { + llm_task_obj->audio_flage_ = false; + } + + send("None", "None", LLM_NO_ERROR, work_id); + } + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { SLOGI("llm_kws::taskinfo:%s", data.c_str()); @@ -937,7 +1089,7 @@ public: _zmq.send_data(out); return LLM_NONE; } - llm_task_[work_id_num]->trigger(); + llm_task_[work_id_num]->trigger_wakeup(); return LLM_NONE; } diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 80f62e0..36a7f74 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -480,7 +480,7 @@ if __name__ == "__main__": 'llm-model-qwen3-vl-2B-Int4-ax630c':[create_data_deb,'llm-model-qwen3-vl-2B-Int4-ax630c', '0.5', src_folder, revision], ## AX650 'llm-model-qwen2.5-0.5B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-0.5B-Int4-ax650', '0.4', src_folder, revision], - 'llm-model-qwen2.5-HA-0.5B-ctx-ax650':[create_data_deb,'llm-model-qwen2.5-HA-0.5B-ctx-ax650', '0.5', src_folder, revision], + 'llm-model-qwen2.5-HA-0.5B-ctx-ax650':[create_data_deb,'llm-model-qwen2.5-HA-0.5B-ctx-ax650', '0.6', src_folder, revision], 'llm-model-qwen2.5-1.5B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-1.5B-Int4-ax650', '0.4', src_folder, revision], 'llm-model-qwen2.5-3B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-3B-Int4-ax650', '0.4', src_folder, revision], 'llm-model-qwen2.5-7B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-7B-Int4-ax650', '0.4', src_folder, revision],