[update] llm_asr suooported sensevoice, update llm_audio supported alsa cap & play, update static lib version

projects/llm_framework/SConstruct

@@ -5,7 +5,7 @@ import shutil
 os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK'))
 os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components'))
 
-version = 'v0.1.2'
+version = 'v0.1.3'
 static_lib = 'static_lib'
 update = False
 

projects/llm_framework/main_asr/SConstruct

@@ -7,7 +7,7 @@ with open(env['PROJECT_TOOL_S']) as f:
 SRCS = Glob('src/*.c*')
 INCLUDE = [ADir('include'), ADir('.')]
 PRIVATE_INCLUDE = []
-REQUIREMENTS = ['pthread', 'utilities', 'eventpp', 'StackFlow', 'single_header_libs']
+REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs']
 STATIC_LIB = []
 DYNAMIC_LIB = []
 DEFINITIONS = []
@@ -17,12 +17,22 @@ LINK_SEARCH_PATH = []
 STATIC_FILES = []
 
 LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
-DEFINITIONS += ['-std=c++17']
+DEFINITIONS += ['-std=c++17', '-fopenmp']
 LINK_SEARCH_PATH += [ADir('../static_lib')]
+REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
 
 INCLUDE += [ADir('../static_lib/include/sherpa'), ADir('../static_lib/include/sherpa/sherpa-ncnn')]
 LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/ncnn')]
-REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core']
+LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')]
+REQUIREMENTS += ['ncnn', '', 'onnxruntime']
+LDFLAGS += [
+            '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a','-l:libkissfft-float.a',
+            '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a', '-l:libsherpa-onnx-fst.a',
+            '-l:libsherpa-onnx-kaldifst-core.a', '-l:libsherpa-onnx-fstfar.a',
+
+            '-l:libsherpa-ncnn-core.a', '-l:libsherpa-ncnn-fst.a', '-l:libsherpa-ncnn-kaldifst-core.a',
+            '-lgomp',
+            ]
 
 STATIC_FILES += Glob('mode_*.json')
 

projects/llm_framework/main_asr/mode_sense-voice-small-10s-ax650.json

@@ -0,0 +1,30 @@
+{
+    "mode": "sense-voice-small-10s-ax650",
+    "type": "asr",
+    "homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha",
+    "compile_flage": "pulsar2 build --input model-10-seconds.onnx --config config_sensevoice_main_u16.json --output_dir sensevoice-axmodel --output_name model-10-seconds.axmodel --target_hardware AX650 --compiler.check 0",
+    "pulsar_version": "5.0-patch1-fd447d0d",
+    "capabilities": [
+        "Chinese",
+        "English",
+        "Cantonese",
+        "Japanese",
+        "Korean"
+    ],
+    "input_type": [
+        "sys.pcm",
+        "sys.cap.0_0"
+    ],
+    "output_type": [
+        "asr.utf-8"
+    ],
+    "mode_param": {
+        "model_config.sense_voice.model": "model.axmodel",
+        "model_config.tokens": "tokens.txt",
+        "silero_vad.model": "silero_vad.onnx",
+        "model_config.provider": "axera",
+        "silence_timeout": 2000,
+        "awake_delay": 50
+    },
+    "mode_param_bak": {}
+}

projects/llm_framework/main_audio/SConstruct

@@ -5,7 +5,11 @@ with open(env['PROJECT_TOOL_S']) as f:
     exec(f.read())
 
 # SRCS = append_srcs_dir(ADir('src'))
-SRCS = Glob('src/*.c*')
+if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ:
+    SRCS = [AFile('src/sample_audio.c'), AFile('src/main.cpp')]
+else:
+    SRCS = [AFile('src/alsa_audio.c'), AFile('src/main.cpp')]
+
 INCLUDE = [ADir('include'), ADir('.')]
 PRIVATE_INCLUDE = []
 REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs']

projects/llm_framework/main_audio/src/alsa_audio.c

@@ -6,7 +6,11 @@
 #include <errno.h>
 #include <string.h>
 
-static int gcapLoopExit = 0;
+static struct pcm *g_play_pcm = NULL;
+static int gplayLoopExit      = 1;
+static int gcapLoopExit       = 1;
+AlsaConfig cap_config;
+AlsaConfig play_config;
 
 void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit,
                     AUDIOCallback callback)
@@ -23,7 +27,7 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
 
     memset(&config, 0, sizeof(config));
     config.channels          = channel;
-    config.rate              = 48000;  // TODO: 部分USB MIC仅支持48k，暂时固定采集为48k
+    config.rate              = 48000;
     config.period_size       = 120;
     config.period_count      = 4;
     config.format            = PCM_FORMAT_S16_LE;
@@ -57,14 +61,16 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
 
     SRC_STATE *src_state = NULL;
     float *in_float = NULL, *out_float = NULL;
+
     int in_frames  = pcm_get_buffer_size(pcm);
     int out_frames = (int)((float)in_frames * ((float)rate / 48000.0f) + 1);
-    int out_bytes  = out_frames * channel * sizeof(short);
+
+    int src_channels = 1;
 
     if (rate != 48000) {
-        src_state = src_new(SRC_SINC_FASTEST, channel, NULL);
-        in_float  = malloc(in_frames * channel * sizeof(float));
-        out_float = malloc(out_frames * channel * sizeof(float));
+        src_state = src_new(SRC_SINC_FASTEST, src_channels, NULL);
+        in_float  = malloc(in_frames * src_channels * sizeof(float));
+        out_float = malloc(out_frames * src_channels * sizeof(float));
         if (!src_state || !in_float || !out_float) {
             fprintf(stderr, "Unable to allocate resample buffers\n");
             free(buffer);
@@ -82,16 +88,36 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
             fprintf(stderr, "Error capturing samples - %d (%s)\n", errno, strerror(errno));
             break;
         }
+
         frames_read = ret;
         total_frames_read += frames_read;
 
         if (rate == 48000) {
-            callback(buffer, frames_read * bytes_per_frame);
-        } else {
-            short *in_short = (short *)buffer;
-            for (int i = 0; i < frames_read * channel; ++i) {
-                in_float[i] = in_short[i] / 32768.0f;
+            int in_channels = channel;  // 比如 4
+            int16_t *in     = (int16_t *)buffer;
+
+            int16_t *ch0 = malloc(frames_read * sizeof(int16_t));
+            if (!ch0) {
+                fprintf(stderr, "Unable to allocate ch0 buffer\n");
+                break;
             }
+
+            for (unsigned int i = 0; i < frames_read; ++i) {
+                ch0[i] = in[i * in_channels + 0];
+            }
+
+            callback((const char *)ch0, frames_read * sizeof(int16_t));
+            free(ch0);
+
+        } else {
+            int in_channels = channel;  // 比如 4
+            short *in_short = (short *)buffer;
+
+            for (int i = 0; i < frames_read; ++i) {
+                short s     = in_short[i * in_channels + 0];
+                in_float[i] = s / 32768.0f;
+            }
+
             SRC_DATA src_data;
             src_data.data_in       = in_float;
             src_data.input_frames  = frames_read;
@@ -99,20 +125,25 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
             src_data.output_frames = out_frames;
             src_data.src_ratio     = (double)rate / 48000.0;
             src_data.end_of_input  = 0;
-            int error              = src_process(src_state, &src_data);
+
+            int error = src_process(src_state, &src_data);
             if (error) {
                 fprintf(stderr, "SRC error: %s\n", src_strerror(error));
                 break;
             }
-            // float转short
-            short *out_short = malloc(src_data.output_frames_gen * channel * sizeof(short));
-            for (int i = 0; i < src_data.output_frames_gen * channel; ++i) {
+            int out_samples  = src_data.output_frames_gen;
+            short *out_short = malloc(out_samples * sizeof(short));
+            if (!out_short) {
+                fprintf(stderr, "Unable to allocate out_short buffer\n");
+                break;
+            }
+            for (int i = 0; i < out_samples; ++i) {
                 float sample = out_float[i];
                 if (sample > 1.0f) sample = 1.0f;
                 if (sample < -1.0f) sample = -1.0f;
                 out_short[i] = (short)(sample * 32767.0f);
             }
-            callback((const char *)out_short, src_data.output_frames_gen * channel * sizeof(short));
+            callback((const char *)out_short, out_samples * sizeof(short));
             free(out_short);
         }
     }
@@ -124,7 +155,6 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
     }
     free(buffer);
     pcm_close(pcm);
-    printf("Total frames captured: %u\n", total_frames_read);
 }
 
 void alsa_close_cap()
@@ -135,4 +165,52 @@ void alsa_close_cap()
 int alsa_cap_status()
 {
     return gcapLoopExit;
+}
+
+void alsa_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data,
+               int size)
+{
+    gplayLoopExit = 0;
+
+    struct pcm_config config;
+    memset(&config, 0, sizeof(config));
+    config.channels          = channel;
+    config.rate              = rate;
+    config.period_size       = 1024;
+    config.period_count      = 2;
+    config.format            = PCM_FORMAT_S16_LE;
+    config.silence_threshold = config.period_size * config.period_count;
+    config.stop_threshold    = config.period_size * config.period_count;
+    config.start_threshold   = config.period_size;
+
+    unsigned int pcm_open_flags = PCM_OUT;
+
+    struct pcm *pcm = pcm_open(card, device, pcm_open_flags, &config);
+    if (!pcm || !pcm_is_ready(pcm)) {
+        fprintf(stderr, "Unable to open PCM playback device (%s)\n", pcm ? pcm_get_error(pcm) : "invalid pcm");
+        if (pcm) {
+            pcm_close(pcm);
+        }
+        gplayLoopExit = 2;
+        return;
+    }
+
+    int frames         = pcm_bytes_to_frames(pcm, size);
+    int written_frames = pcm_writei(pcm, data, frames);
+    if (written_frames < 0) {
+        fprintf(stderr, "PCM playback error %s\n", pcm_get_error(pcm));
+    }
+    printf("Played %d frames\n", written_frames);
+    pcm_close(pcm);
+    gplayLoopExit = 2;
+}
+
+void alsa_close_play()
+{
+    gplayLoopExit = 1;
+}
+
+int alsa_play_status()
+{
+    return gplayLoopExit;
 }

projects/llm_framework/main_audio/src/alsa_audio.h

@@ -1,5 +1,18 @@
 #pragma once
 
+typedef struct AlsaConfig
+{
+    unsigned int card;
+    unsigned int device;
+    float volume;
+    int channel;
+    int rate;
+    int bit;
+} AlsaConfig;
+
+extern AlsaConfig cap_config;
+extern AlsaConfig play_config;
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -10,6 +23,13 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch
                     AUDIOCallback callback);
 void alsa_close_cap();
 
+void alsa_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data,
+             int size);
+
+void alsa_close_play();
+int alsa_cap_status();
+int alsa_play_status();
+
 #ifdef __cplusplus
 }
 #endif

projects/llm_framework/main_audio/src/main.cpp

@@ -13,6 +13,7 @@
 #include <iostream>
 #include <stdexcept>
 #include "../../../../SDK/components/utilities/include/sample_log.h"
+#include <global_config.h>
 
 int main_exit_flage = 0;
 static void __sigint(int iSigNo)
@@ -21,8 +22,11 @@ static void __sigint(int iSigNo)
     main_exit_flage = 1;
 }
 
+#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
 #include "sample_audio.h"
+#else
 #include "alsa_audio.h"
+#endif
 
 #define CONFIG_AUTO_SET(obj, key)             \
     if (config_body.contains(#key))           \
@@ -80,8 +84,13 @@ private:
         }
 
         std::lock_guard<std::mutex> guard(ax_play_mtx);
+#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
         ax_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate,
-                play_config.bit, final_data.c_str(), final_data.length());
+                play_config.bit, audio_data.c_str(), audio_data.length());
+#else
+        alsa_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate,
+                  play_config.bit, final_data.c_str(), final_data.length());
+#endif
     }
 
     void hw_play(const std::string &audio_data)
@@ -92,8 +101,13 @@ private:
             final_data = mono_to_stereo_s16le(audio_data);
         }
         std::lock_guard<std::mutex> guard(ax_play_mtx);
+#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
         ax_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate,
                 play_config.bit, audio_data.c_str(), audio_data.length());
+#else
+        alsa_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate,
+                  play_config.bit, final_data.c_str(), final_data.length());
+#endif
     }
 
     void hw_cap()
@@ -117,7 +131,11 @@ private:
 
     void _play_stop()
     {
+#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
         ax_close_play();
+#else
+        alsa_close_play();
+#endif
         if (audio_play_thread_) {
             audio_play_thread_->join();
             audio_play_thread_.reset();
@@ -134,7 +152,7 @@ private:
 
     void _cap_stop()
     {
-#if (defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)) && !defined(CONFIG_AXCL_ENABLED)
+#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
         ax_close_cap();
 
 #else
@@ -219,8 +237,10 @@ public:
             send("None", "None", error_body, "audio");
             return -2;
         }
-        AX_AUDIO_SAMPLE_CONFIG_t mode_config_;
+
         try {
+#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
+            AX_AUDIO_SAMPLE_CONFIG_t mode_config_;
             memset(&mode_config_, 0, sizeof(AX_AUDIO_SAMPLE_CONFIG_t));
             if (object == "audio.play") {
                 CONFIG_AUTO_SET(file_body["play_param"], stPoolConfig.MetaSize);
@@ -244,22 +264,7 @@ public:
                 CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.enAgcMode);
                 CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.s16TargetLevel);
                 CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.s16Gain);
-#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
-                CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.bEnable);
-                CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.s32GainDb);
-                CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.s32Freq);
-                CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.bEnable);
-                CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.s32GainDb);
-                CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.s32Samplerate);
-                CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.s32Freq);
-                CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.bEnable);
-                CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[0]);
-                CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[1]);
-                CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[2]);
-                CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[3]);
-                CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[4]);
-                CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32Samplerate);
-#endif
+
                 CONFIG_AUTO_SET(file_body["play_param"], gResample);
                 CONFIG_AUTO_SET(file_body["play_param"], enInSampleRate);
                 CONFIG_AUTO_SET(file_body["play_param"], gInstant);
@@ -272,12 +277,12 @@ public:
                 CONFIG_AUTO_SET(file_body["play_param"], bit);
                 if (config_body.contains("stPoolConfig.PartitionName")) {
                     std::string PartitionName = config_body["stPoolConfig.PartitionName"];
-                    for (int i = 0; i < PartitionName.length(); i++) {
+                    for (int i = 0; i < (int)PartitionName.length(); i++) {
                         mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i];
                     }
-                } else if (file_body["cap_param"].contains("stPoolConfig.PartitionName")) {
-                    std::string PartitionName = file_body["cap_param"]["stPoolConfig.PartitionName"];
-                    for (int i = 0; i < PartitionName.length(); i++) {
+                } else if (file_body["play_param"].contains("stPoolConfig.PartitionName")) {
+                    std::string PartitionName = file_body["play_param"]["stPoolConfig.PartitionName"];
+                    for (int i = 0; i < (int)PartitionName.length(); i++) {
                         mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i];
                     }
                 }
@@ -292,7 +297,7 @@ public:
                 CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enBitwidth);
                 CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enLinkMode);
                 CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enSamplerate);
-                // CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enLayoutMode);
+
                 if (config_body.contains("aistAttr.enLayoutMode"))
                     mode_config_.aistAttr.enLayoutMode = config_body["aistAttr.enLayoutMode"];
                 else if (file_body["cap_param"].contains("aistAttr.enLayoutMode")) {
@@ -316,23 +321,7 @@ public:
                 CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAgcCfg.s16TargetLevel);
                 CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAgcCfg.s16Gain);
                 CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAecCfg.enAecMode);
-#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
-                CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.bEnable);
-                CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32GainDb);
-                CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32Samplerate);
-                CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32Freq);
-                CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.bEnable);
-                CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.s32GainDb);
-                CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.s32Samplerate);
-                CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.s32Freq);
-                CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.bEnable);
-                CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[0]);
-                CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[1]);
-                CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[2]);
-                CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[3]);
-                CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[4]);
-                CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32Samplerate);
-#endif
+
                 CONFIG_AUTO_SET(file_body["cap_param"], gResample);
                 CONFIG_AUTO_SET(file_body["cap_param"], enOutSampleRate);
                 CONFIG_AUTO_SET(file_body["cap_param"], gDbDetection);
@@ -345,12 +334,12 @@ public:
 
                 if (config_body.contains("stPoolConfig.PartitionName")) {
                     std::string PartitionName = config_body["stPoolConfig.PartitionName"];
-                    for (int i = 0; i < PartitionName.length(); i++) {
+                    for (int i = 0; i < (int)PartitionName.length(); i++) {
                         mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i];
                     }
                 } else if (file_body["cap_param"].contains("stPoolConfig.PartitionName")) {
                     std::string PartitionName = file_body["cap_param"]["stPoolConfig.PartitionName"];
-                    for (int i = 0; i < PartitionName.length(); i++) {
+                    for (int i = 0; i < (int)PartitionName.length(); i++) {
                         mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i];
                     }
                 }
@@ -361,6 +350,31 @@ public:
                 }
                 memcpy(&cap_config, &mode_config_, sizeof(AX_AUDIO_SAMPLE_CONFIG_t));
             }
+
+#else
+            AlsaConfig mode_config_;
+            memset(&mode_config_, 0, sizeof(AlsaConfig));
+
+            if (object == "audio.play") {
+                CONFIG_AUTO_SET(file_body["play_param"], card);
+                CONFIG_AUTO_SET(file_body["play_param"], device);
+                CONFIG_AUTO_SET(file_body["play_param"], volume);
+                CONFIG_AUTO_SET(file_body["play_param"], channel);
+                CONFIG_AUTO_SET(file_body["play_param"], rate);
+                CONFIG_AUTO_SET(file_body["play_param"], bit);
+                memcpy(&play_config, &mode_config_, sizeof(AlsaConfig));
+            }
+
+            if (object == "audio.cap") {
+                CONFIG_AUTO_SET(file_body["cap_param"], card);
+                CONFIG_AUTO_SET(file_body["cap_param"], device);
+                CONFIG_AUTO_SET(file_body["cap_param"], volume);
+                CONFIG_AUTO_SET(file_body["cap_param"], channel);
+                CONFIG_AUTO_SET(file_body["cap_param"], rate);
+                CONFIG_AUTO_SET(file_body["cap_param"], bit);
+                memcpy(&cap_config, &mode_config_, sizeof(AlsaConfig));
+            }
+#endif
         } catch (...) {
             error_body["code"]    = -22;
             error_body["message"] = "Parameter format error.";
@@ -471,33 +485,23 @@ public:
     std::string audio_status(pzmq *_pzmq, const std::shared_ptr<pzmq_data> &rawdata)
     {
         std::string _rawdata = rawdata->string();
+
+#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)
+        auto play_state = ax_play_status();
+        auto cap_state  = ax_cap_status();
+#else
+        auto play_state = alsa_play_status();
+        auto cap_state  = alsa_cap_status();
+#endif
+
         if (_rawdata == "play") {
-            if (ax_play_status()) {
-                return std::string("None");
-            } else {
-                return std::string("Runing");
-            }
+            return play_state ? "None" : "Runing";
         } else if (_rawdata == "cap") {
-            if (ax_cap_status()) {
-                return std::string("None");
-            } else {
-                return std::string("Runing");
-            }
+            return cap_state ? "None" : "Runing";
         } else {
             std::ostringstream return_val;
-            return_val << "{\"play\":";
-            if (ax_play_status()) {
-                return_val << "\"None\"";
-            } else {
-                return_val << "\"Runing\"";
-            }
-            return_val << "\"cap\":";
-            if (ax_cap_status()) {
-                return_val << "\"None\"";
-            } else {
-                return_val << "\"Runing\"";
-            }
-            return_val << "\"}";
+            return_val << "{\"play\":" << (play_state ? "\"None\"" : "\"Runing\"")
+                       << ",\"cap\":" << (cap_state ? "\"None\"" : "\"Runing\"") << "}";
             return return_val.str();
         }
     }

projects/llm_framework/main_kws/SConstruct

@@ -29,11 +29,11 @@ INCLUDE += [ADir('../static_lib/include/sherpa'),
             ]
 
 LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')]
-LDFLAGS += ['-l:libcargs.a',
-            '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a',
+REQUIREMENTS += ['onnxruntime', 'cargs']
+
+LDFLAGS += ['-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a','-l:libkissfft-float.a',
             '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a']
 
-REQUIREMENTS += ['onnxruntime']
 
 STATIC_FILES += [os.path.join(python_venv, 'sherpa-onnx')]
 STATIC_FILES += Glob('llm-kws_text2token.py')

projects/llm_framework/main_kws_new/SConstruct

@@ -57,6 +57,8 @@ ignore['ignore'] = list(set(ignore['ignore']))
 with open('../dist/fileignore', 'w') as f:
     json.dump(ignore, f, indent=4)
 
+STATIC_FILES += Glob('mode_*.json')
+
 env['COMPONENTS'].append({'target':'llm_kws_new-1.9',
                           'SRCS':SRCS,
                           'INCLUDE':INCLUDE,

projects/llm_framework/main_kws_new/mode_kws.json

@@ -0,0 +1,41 @@
+{
+    "mode": "kws",
+    "type": "kws",
+    "homepage": "",
+    "capabilities": [
+        "Keyword_spotting",
+        "English"
+    ],
+    "input_type": [
+        "sys.pcm",
+        "sys.cap.0_0"
+    ],
+    "output_type": [
+        "kws.bool"
+    ],
+    "mode_param": {
+        "model": "kws.onnx",
+        "wake_wav_file": "/opt/m5stack/data/audio/wakeup_zh_cn.wav",
+        "chunk_size": 32,
+        "threshold": 0.9,
+        "min_continuous_frames": 5,
+        "REFRACTORY_TIME_MS": 2000,
+        "RESAMPLE_RATE": 16000,
+        "FEAT_DIM": 80,
+        "frame_opts.samp_freq": 16000,
+        "frame_opts.frame_length_ms": 25.0,
+        "frame_opts.frame_shift_ms": 10.0,
+        "frame_opts.snip_edges": false,
+        "frame_opts.dither": 0.0,
+        "frame_opts.preemph_coeff": 0.97,
+        "frame_opts.remove_dc_offset": true,
+        "frame_opts.window_type": "povey",
+        "mel_opts.num_bins": 80,
+        "mel_opts.low_freq": 20,
+        "mel_opts.high_freq": 0,
+        "energy_floor": 0.0,
+        "use_energy": false,
+        "raw_energy": true
+    },
+    "mode_param_bak": {}
+}

projects/llm_framework/main_kws_new/src/main.cpp

@@ -191,9 +191,7 @@ public:
     bool detect_wakeup(const std::vector<float> &scores)
     {
         bool triggered = false;
-        SLOGE("%d", scores.size());
         for (auto score : scores) {
-            printf("%f ", score);
             if (score > threshold) {
                 count_frames++;
                 if (count_frames >= min_continuous_frames) {
@@ -208,7 +206,6 @@ public:
             }
             frame_index_global++;
         }
-        SLOGE("\n");
         return triggered;
     }
 

projects/llm_framework/main_vad/SConstruct

@@ -23,7 +23,8 @@ LINK_SEARCH_PATH += [ADir('../static_lib')]
 INCLUDE += [ADir('../static_lib/include/sherpa')]
 
 LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')]
-LDFLAGS += ['-l:libsherpa-onnx-core.a']
+
+LDFLAGS += ['-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', '-l:libkissfft-float.a']
 
 REQUIREMENTS += ['onnxruntime']