[update] perf llm-asr, kws add buttons control.

2026-05-20 11:32:11 -07:00 · 2025-12-19 17:31:29 +08:00
parent 3f608d2d60
commit eff3a47928
3 changed files with 225 additions and 13 deletions
@@ -85,6 +85,23 @@ private:
        ENGINE_ONLINE = 3,
    } engine_type_ = ENGINE_NCNN;

+    static constexpr int kSampleRate   = 16000;
+    static constexpr int kFrameSamples = 160;
+    int pre_roll_frames_               = 30;
+    std::deque<int16_t> pre_roll_pcm_;
+    bool prev_vad_detected_ = false;
+
+private:
+    void PushPreRollPcm(const int16_t *pcm, size_t n)
+    {
+        pre_roll_pcm_.insert(pre_roll_pcm_.end(), pcm, pcm + n);
+
+        const size_t max_samples = (size_t)pre_roll_frames_ * kFrameSamples;
+        while (pre_roll_pcm_.size() > max_samples) {
+            pre_roll_pcm_.pop_front();
+        }
+    }
+
 public:
    std::string model_;
    std::string response_format_;
@@ -529,48 +546,86 @@ public:

    void sys_pcm_on_data_onnx(const std::string &raw)
    {
+        if (raw.size() >= sizeof(int16_t)) {
+            const int16_t *pcm16 = reinterpret_cast<const int16_t *>(raw.data());
+            size_t n16           = raw.size() / sizeof(int16_t);
+            PushPreRollPcm(pcm16, n16);
+        }
+
        static int count = 0;
        if (count < delay_audio_frame_) {
            buffer_write_char(pcmdata, raw.data(), raw.length());
            count++;
            return;
        }
+
        buffer_write_char(pcmdata, raw.data(), raw.length());
        buffer_position_set(pcmdata, 0);

        std::vector<float> floatSamples;
+        floatSamples.reserve((delay_audio_frame_ + 1) * kFrameSamples);
+
        int16_t audio_val;
        while (buffer_read_i16(pcmdata, &audio_val, 1)) {
-            float normalizedSample = static_cast<float>(audio_val) / INT16_MAX;
-            floatSamples.push_back(normalizedSample);
+            floatSamples.push_back(static_cast<float>(audio_val) / 32768.0f);
        }
+
        buffer_resize(pcmdata, 0);
        count = 0;

        vad_->AcceptWaveform(floatSamples.data(), floatSamples.size());
+
+        bool detected      = vad_->IsSpeechDetected();
+        bool speech_start  = (!prev_vad_detected_ && detected);
+        prev_vad_detected_ = detected;
+
        while (!vad_->Empty()) {
            const auto &segment = vad_->Front();
-            if (!offline_stream_) offline_stream_ = onnx_recognizer_->CreateStream();
-            offline_stream_->AcceptWaveform(onnx_asr_config_.feat_config.sampling_rate, segment.samples.data(),
-                                            segment.samples.size());
+
+            if (!offline_stream_) {
+                offline_stream_ = onnx_recognizer_->CreateStream();
+            }
+
+            if (speech_start && !pre_roll_pcm_.empty()) {
+                std::vector<float> pre;
+                pre.reserve(pre_roll_pcm_.size());
+                for (int16_t s : pre_roll_pcm_) {
+                    pre.push_back(static_cast<float>(s) / 32768.0f);
+                }
+
+                std::vector<float> merged;
+                merged.reserve(pre.size() + segment.samples.size());
+                merged.insert(merged.end(), pre.begin(), pre.end());
+                merged.insert(merged.end(), segment.samples.begin(), segment.samples.end());
+
+                offline_stream_->AcceptWaveform(kSampleRate, merged.data(), merged.size());
+
+                pre_roll_pcm_.clear();
+                speech_start = false;
+            } else {
+                offline_stream_->AcceptWaveform(kSampleRate, segment.samples.data(), segment.samples.size());
+            }
+
            onnx_recognizer_->DecodeStream(offline_stream_.get());
+
            const auto &result = offline_stream_->GetResult();
            if (!result.text.empty() && out_callback_) {
                out_callback_(result.text, true);
            }
+
            vad_->Pop();
+
            offline_stream_.reset();
        }

        {
-            bool detected  = vad_->IsSpeechDetected();
            float chunk_ms = (delay_audio_frame_ + 1) * 10.0f;
-
            if (detected) {
                silence_ms_accum_ = 0.0f;
            } else {
                silence_ms_accum_ += chunk_ms;
            }
+
            if (silence_ms_accum_ >= silence_timeout) {
                if (ensleep_) {
                    if (pause) pause();
@@ -1034,6 +1089,11 @@ public:
                ++it;
            }
        }
+
+        if (data.find("sys") != std::string::npos) {
+            llm_task_obj->audio_flage_ = false;
+        }
+
        send("None", "None", LLM_NO_ERROR, work_id);
    }

@@ -81,6 +81,7 @@ private:
    int count_frames_               = 0;
    long long last_trigger_time_ms_ = -1e9;
    long long frame_index_global_   = 0;
+    int last_btn_204_state          = -1;

 public:
    inline const std::string &model() const
@@ -294,9 +295,9 @@ public:
 #undef CONFIG_AUTO_SET_SHERPA

 #define CONFIG_AUTO_SET_AXERA(obj, key)        \
-    if (config_body.contains(#key))           \
+    if (config_body.contains(#key))            \
        axera_config_.key = config_body[#key]; \
-    else if (obj.contains(#key))              \
+    else if (obj.contains(#key))               \
        axera_config_.key = obj[#key];

 #define OPTS_AUTO_SET(obj, key)              \
@@ -537,9 +538,27 @@ public:
        }
    }

-    void trigger()
+    void trigger_wakeup()
    {
-        if (out_callback_) out_callback_("", true);
+        if (enwake_audio_ && (!wake_wav_file_.empty()) && play_awake_wav) {
+            play_awake_wav(wake_wav_file_);
+        }
+        if (out_callback_) {
+            if (enoutput_json_)
+                out_callback_("{\"reason\":\"button_204\"}", true);
+            else
+                out_callback_("", true);
+        }
+    }
+
+    void set_btn_204_state(int state)
+    {
+        last_btn_204_state = state;
+    }
+
+    int get_btn_204_state()
+    {
+        return last_btn_204_state;
    }

    bool delete_model()
@@ -790,6 +809,40 @@ public:
        llm_task_obj->sys_pcm_on_data((*next_data));
    }

+    void task_buttons_data(const std::weak_ptr<llm_task> llm_task_obj_weak,
+                           const std::weak_ptr<llm_channel_obj> llm_channel_weak, const std::string &object,
+                           const std::string &data)
+    {
+        auto llm_task_obj = llm_task_obj_weak.lock();
+        auto llm_channel  = llm_channel_weak.lock();
+        if (!(llm_task_obj && llm_channel)) {
+            return;
+        }
+        if (data.empty() || (data == "None")) return;
+
+        try {
+            std::string user_msg    = sample_unescapeString(data);
+            nlohmann::json btn_json = nlohmann::json::parse(user_msg);
+
+            if (btn_json.contains("code") && btn_json.contains("vale")) {
+                int current_code = btn_json["code"];
+                int current_vale = btn_json["vale"];
+
+                if (current_vale == 204) {
+                    int last_code = llm_task_obj->get_btn_204_state();
+
+                    if (last_code == 0 && current_code == 1) {
+                        llm_task_obj->trigger_wakeup();
+                    }
+
+                    llm_task_obj->set_btn_204_state(current_code);
+                }
+            }
+        } catch (const std::exception &e) {
+            SLOGE("Button data JSON parse error: %s", e.what());
+        }
+    }
+
    int setup(const std::string &work_id, const std::string &object, const std::string &data) override
    {
        nlohmann::json error_body;
@@ -836,6 +889,17 @@ public:
                    llm_channel->subscriber_work_id("", std::bind(&llm_kws::task_user_data, this, _llm_task_obj,
                                                                  std::weak_ptr<llm_channel_obj>(llm_channel),
                                                                  std::placeholders::_1, std::placeholders::_2));
+                } else if (input.find("buttons_thread") != std::string::npos) {
+                    std::string socket_url = "ipc:///tmp/llm/ec_prox.event.socket";
+                    auto business_logic    = std::bind(
+                        &llm_kws::task_buttons_data, this, std::weak_ptr<llm_task>(llm_task_obj),
+                        std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1, std::placeholders::_2);
+
+                    llm_channel->subscriber(
+                        socket_url, [llm_channel, business_logic](StackFlows::pzmq *p,
+                                                                  const std::shared_ptr<StackFlows::pzmq_data> &d) {
+                            llm_channel->subscriber_event_call(business_logic, p, d);
+                        });
                }
            }
            llm_task_[work_id_num] = llm_task_obj;
@@ -851,6 +915,94 @@ public:
        }
    }

+    void link(const std::string &work_id, const std::string &object, const std::string &data) override
+    {
+        SLOGI("llm_kws::link:%s", data.c_str());
+        int ret = 0;
+        nlohmann::json error_body;
+
+        int work_id_num = sample_get_work_id_num(work_id);
+        if (llm_task_.find(work_id_num) == llm_task_.end()) {
+            error_body["code"]    = -6;
+            error_body["message"] = "Unit Does Not Exist";
+            send("None", "None", error_body, work_id);
+            return;
+        }
+
+        auto llm_channel  = get_channel(work_id);
+        auto llm_task_obj = llm_task_[work_id_num];
+
+        if (data.find("sys") != std::string::npos) {
+            if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data);
+
+            std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
+            llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::shared_ptr<pzmq_data> &raw) {
+                if (auto p = _llm_task_obj.lock()) p->sys_pcm_on_data(raw->string());
+            });
+
+            llm_task_obj->audio_flage_ = true;
+            llm_task_obj->inputs_.push_back(data);
+        } else if (data.find("buttons_thread") != std::string::npos) {
+            std::string socket_url = "ipc:///tmp/llm/ec_prox.event.socket";
+            auto business_logic =
+                std::bind(&llm_kws::task_buttons_data, this, std::weak_ptr<llm_task>(llm_task_obj),
+                          std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1, std::placeholders::_2);
+
+            llm_channel->subscriber(
+                socket_url,
+                [llm_channel, business_logic](StackFlows::pzmq *p, const std::shared_ptr<StackFlows::pzmq_data> &d) {
+                    llm_channel->subscriber_event_call(business_logic, p, d);
+                });
+
+            llm_task_obj->inputs_.push_back(data);
+        } else {
+            error_body["code"]    = -22;
+            error_body["message"] = "unsupported link target";
+            send("None", "None", error_body, work_id);
+            return;
+        }
+
+        if (ret) {
+            error_body["code"]    = -20;
+            error_body["message"] = "link false";
+            send("None", "None", error_body, work_id);
+            return;
+        }
+        send("None", "None", LLM_NO_ERROR, work_id);
+    }
+
+    void unlink(const std::string &work_id, const std::string &object, const std::string &data) override
+    {
+        SLOGI("llm_kws::unlink:%s", data.c_str());
+        nlohmann::json error_body;
+
+        int work_id_num = sample_get_work_id_num(work_id);
+        if (llm_task_.find(work_id_num) == llm_task_.end()) {
+            error_body["code"]    = -6;
+            error_body["message"] = "Unit Does Not Exist";
+            send("None", "None", error_body, work_id);
+            return;
+        }
+
+        auto llm_channel  = get_channel(work_id);
+        auto llm_task_obj = llm_task_[work_id_num];
+
+        llm_channel->stop_subscriber_work_id(data);
+
+        for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) {
+            if (*it == data)
+                it = llm_task_obj->inputs_.erase(it);
+            else
+                ++it;
+        }
+
+        if (data.find("sys") != std::string::npos) {
+            llm_task_obj->audio_flage_ = false;
+        }
+
+        send("None", "None", LLM_NO_ERROR, work_id);
+    }
+
    void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override
    {
        SLOGI("llm_kws::taskinfo:%s", data.c_str());
@@ -937,7 +1089,7 @@ public:
            _zmq.send_data(out);
            return LLM_NONE;
        }
-        llm_task_[work_id_num]->trigger();
+        llm_task_[work_id_num]->trigger_wakeup();
        return LLM_NONE;
    }

@@ -480,7 +480,7 @@ if __name__ == "__main__":
        'llm-model-qwen3-vl-2B-Int4-ax630c':[create_data_deb,'llm-model-qwen3-vl-2B-Int4-ax630c', '0.5', src_folder, revision],
        ## AX650
        'llm-model-qwen2.5-0.5B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-0.5B-Int4-ax650', '0.4', src_folder, revision],
-        'llm-model-qwen2.5-HA-0.5B-ctx-ax650':[create_data_deb,'llm-model-qwen2.5-HA-0.5B-ctx-ax650', '0.5', src_folder, revision],
+        'llm-model-qwen2.5-HA-0.5B-ctx-ax650':[create_data_deb,'llm-model-qwen2.5-HA-0.5B-ctx-ax650', '0.6', src_folder, revision],
        'llm-model-qwen2.5-1.5B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-1.5B-Int4-ax650', '0.4', src_folder, revision],
        'llm-model-qwen2.5-3B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-3B-Int4-ax650', '0.4', src_folder, revision],
        'llm-model-qwen2.5-7B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-7B-Int4-ax650', '0.4', src_folder, revision],