diff --git a/projects/llm_framework/SConstruct b/projects/llm_framework/SConstruct index b6c85da..731210d 100644 --- a/projects/llm_framework/SConstruct +++ b/projects/llm_framework/SConstruct @@ -5,7 +5,7 @@ import shutil os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK')) os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components')) -version = 'v0.1.2' +version = 'v0.1.3' static_lib = 'static_lib' update = False diff --git a/projects/llm_framework/main_asr/SConstruct b/projects/llm_framework/main_asr/SConstruct index cbc98e4..9bdac46 100644 --- a/projects/llm_framework/main_asr/SConstruct +++ b/projects/llm_framework/main_asr/SConstruct @@ -7,7 +7,7 @@ with open(env['PROJECT_TOOL_S']) as f: SRCS = Glob('src/*.c*') INCLUDE = [ADir('include'), ADir('.')] PRIVATE_INCLUDE = [] -REQUIREMENTS = ['pthread', 'utilities', 'eventpp', 'StackFlow', 'single_header_libs'] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs'] STATIC_LIB = [] DYNAMIC_LIB = [] DEFINITIONS = [] @@ -17,12 +17,22 @@ LINK_SEARCH_PATH = [] STATIC_FILES = [] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] -DEFINITIONS += ['-std=c++17'] +DEFINITIONS += ['-std=c++17', '-fopenmp'] LINK_SEARCH_PATH += [ADir('../static_lib')] +REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] INCLUDE += [ADir('../static_lib/include/sherpa'), ADir('../static_lib/include/sherpa/sherpa-ncnn')] LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/ncnn')] -REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core'] +LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')] +REQUIREMENTS += ['ncnn', '', 'onnxruntime'] +LDFLAGS += [ + '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a','-l:libkissfft-float.a', + '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a', '-l:libsherpa-onnx-fst.a', + '-l:libsherpa-onnx-kaldifst-core.a', '-l:libsherpa-onnx-fstfar.a', + + '-l:libsherpa-ncnn-core.a', '-l:libsherpa-ncnn-fst.a', '-l:libsherpa-ncnn-kaldifst-core.a', + '-lgomp', + ] STATIC_FILES += Glob('mode_*.json') diff --git a/projects/llm_framework/main_asr/mode_sense-voice-small-10s-ax650.json b/projects/llm_framework/main_asr/mode_sense-voice-small-10s-ax650.json new file mode 100644 index 0000000..370c834 --- /dev/null +++ b/projects/llm_framework/main_asr/mode_sense-voice-small-10s-ax650.json @@ -0,0 +1,30 @@ +{ + "mode": "sense-voice-small-10s-ax650", + "type": "asr", + "homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha", + "compile_flage": "pulsar2 build --input model-10-seconds.onnx --config config_sensevoice_main_u16.json --output_dir sensevoice-axmodel --output_name model-10-seconds.axmodel --target_hardware AX650 --compiler.check 0", + "pulsar_version": "5.0-patch1-fd447d0d", + "capabilities": [ + "Chinese", + "English", + "Cantonese", + "Japanese", + "Korean" + ], + "input_type": [ + "sys.pcm", + "sys.cap.0_0" + ], + "output_type": [ + "asr.utf-8" + ], + "mode_param": { + "model_config.sense_voice.model": "model.axmodel", + "model_config.tokens": "tokens.txt", + "silero_vad.model": "silero_vad.onnx", + "model_config.provider": "axera", + "silence_timeout": 2000, + "awake_delay": 50 + }, + "mode_param_bak": {} +} \ No newline at end of file diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index d2dff9b..19fb336 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -5,6 +5,8 @@ */ #include "StackFlow.h" #include "sherpa-ncnn/csrc/recognizer.h" +#include "sherpa-onnx/csrc/offline-recognizer.h" +#include "sherpa-onnx/csrc/voice-activity-detector.h" #include #include @@ -24,6 +26,7 @@ using namespace StackFlows; int main_exit_flage = 0; + static void __sigint(int iSigNo) { SLOGW("llm_asr will be exit!"); @@ -35,17 +38,39 @@ static std::string base_model_config_path_; typedef std::function task_callback_t; -#define CONFIG_AUTO_SET(obj, key) \ +#define NCNN_ASR_CONFIG_AUTO_SET(obj, key) \ if (config_body.contains(#key)) \ - mode_config_.key = config_body[#key]; \ + ncnn_config_.key = config_body[#key]; \ else if (obj.contains(#key)) \ - mode_config_.key = obj[#key]; + ncnn_config_.key = obj[#key]; + +#define ONNX_ASR_CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + onnx_asr_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + onnx_asr_config_.key = obj[#key]; + +#define ONNX_VAD_CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + vad_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + vad_config_.key = obj[#key]; class llm_task { private: - sherpa_ncnn::RecognizerConfig mode_config_; - std::unique_ptr recognizer_; - std::unique_ptr recognizer_stream_; + sherpa_ncnn::RecognizerConfig ncnn_config_; + std::unique_ptr ncnn_recognizer_; + std::unique_ptr ncnn_stream_; + + sherpa_onnx::OfflineRecognizerConfig onnx_asr_config_; + sherpa_onnx::VadModelConfig vad_config_; + std::unique_ptr onnx_recognizer_; + std::unique_ptr vad_; + + enum EngineType { + ENGINE_NCNN = 0, + ENGINE_ONNX = 1, + } engine_type_ = ENGINE_NCNN; public: std::string model_; @@ -57,10 +82,12 @@ public: task_callback_t out_callback_; std::atomic_bool audio_flage_; std::atomic_bool awake_flage_; - int awake_delay_ = 50; - int delay_audio_frame_ = 10; - buffer_t *pcmdata; + int awake_delay_ = 50; + int delay_audio_frame_ = 10; + float silence_ms_accum_ = 0.0f; + float silence_timeout = 1000.0f; + buffer_t *pcmdata; std::function pause; bool parse_config(const nlohmann::json &config_body) @@ -82,18 +109,208 @@ public: SLOGE("setup config_body error"); return true; } + + if (model_.rfind("sherpa-ncnn", 0) == 0) { + engine_type_ = ENGINE_NCNN; + } else { + engine_type_ = ENGINE_ONNX; + } + enstream_ = response_format_.find("stream") == std::string::npos ? false : true; return false; } + int load_ncnn_model(const nlohmann::json &config_body, const nlohmann::json &file_body) + { + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model (ncnn) %s", base_model.c_str()); + + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.sampling_rate); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.feature_dim); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.encoder_param); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.encoder_bin); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.decoder_param); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.decoder_bin); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.joiner_param); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.joiner_bin); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.tokens); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.encoder_opt.num_threads); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.decoder_opt.num_threads); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.joiner_opt.num_threads); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], decoder_config.method); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], decoder_config.num_active_paths); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.must_contain_nonsilence); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_trailing_silence); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_utterance_length); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.must_contain_nonsilence); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_trailing_silence); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_utterance_length); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.must_contain_nonsilence); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_trailing_silence); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_utterance_length); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], enable_endpoint); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_file); + NCNN_ASR_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_score); + + if (config_body.contains("awake_delay")) + awake_delay_ = config_body["awake_delay"].get(); + else if (file_body["mode_param"].contains("awake_delay")) + awake_delay_ = file_body["mode_param"]["awake_delay"]; + + if (config_body.contains("rule1")) { + ncnn_config_.endpoint_config.rule1.min_trailing_silence = config_body["rule1"].get(); + ncnn_config_.endpoint_config.rule1.must_contain_nonsilence = + (ncnn_config_.endpoint_config.rule1.min_trailing_silence == 0.0f) ? false : true; + } + if (config_body.contains("rule2")) { + ncnn_config_.endpoint_config.rule2.min_trailing_silence = config_body["rule2"].get(); + ncnn_config_.endpoint_config.rule2.must_contain_nonsilence = + (ncnn_config_.endpoint_config.rule2.min_trailing_silence == 0.0f) ? false : true; + } + if (config_body.contains("rule3")) { + ncnn_config_.endpoint_config.rule3.min_utterance_length = config_body["rule3"].get(); + ncnn_config_.endpoint_config.rule3.must_contain_nonsilence = + (ncnn_config_.endpoint_config.rule3.min_utterance_length == 0.0f) ? false : true; + } + + ncnn_config_.model_config.tokens = base_model + ncnn_config_.model_config.tokens; + ncnn_config_.model_config.encoder_param = base_model + ncnn_config_.model_config.encoder_param; + ncnn_config_.model_config.encoder_bin = base_model + ncnn_config_.model_config.encoder_bin; + ncnn_config_.model_config.decoder_param = base_model + ncnn_config_.model_config.decoder_param; + ncnn_config_.model_config.decoder_bin = base_model + ncnn_config_.model_config.decoder_bin; + ncnn_config_.model_config.joiner_param = base_model + ncnn_config_.model_config.joiner_param; + ncnn_config_.model_config.joiner_bin = base_model + ncnn_config_.model_config.joiner_bin; + + ncnn_recognizer_ = std::make_unique(ncnn_config_); + return 0; + } + + int load_onnx_model(const nlohmann::json &config_body, const nlohmann::json &file_body) + { + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model (onnx) %s", base_model.c_str()); + + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.sampling_rate); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.feature_dim); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.low_freq); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.dither); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.normalize_samples); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.snip_edges); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.frame_shift_ms); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.frame_length_ms); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_librosa); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.remove_dc_offset); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.preemph_coeff); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.window_type); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.nemo_normalize_type); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.num_ceps); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.use_energy); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_mfcc); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_whisper); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_t_one); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.round_to_power_of_two); + + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.encoder_filename); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.decoder_filename); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.joiner_filename); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.paraformer.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.nemo_ctc.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.whisper.encoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.whisper.decoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.whisper.language); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.whisper.task); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.whisper.tail_paddings); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.fire_red_asr.encoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.fire_red_asr.decoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.tdnn.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_ctc.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.wenet_ctc.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.sense_voice.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.sense_voice.language); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.sense_voice.use_itn); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.moonshine.preprocessor); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.moonshine.encoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.moonshine.uncached_decoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.moonshine.cached_decoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.dolphin.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.canary.encoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.canary.decoder); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.canary.src_lang); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.canary.tgt_lang); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.canary.use_pnc); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.omnilingual.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.telespeech_ctc); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.tokens); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.num_threads); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.debug); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.provider); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.model_type); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.modeling_unit); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], model_config.bpe_vocab); + + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.model); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.scale); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lm_num_threads); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lm_provider); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_fst); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_scale); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_backoff_id); + + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], ctc_fst_decoder_config.graph); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], ctc_fst_decoder_config.max_active); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], decoding_method); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], max_active_paths); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_file); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_score); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], blank_penalty); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], rule_fsts); + ONNX_ASR_CONFIG_AUTO_SET(file_body["mode_param"], rule_fars); + + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.model); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.threshold); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.min_silence_duration); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.min_speech_duration); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.window_size); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.max_speech_duration); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], ten_vad.model); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], ten_vad.threshold); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], ten_vad.min_silence_duration); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], ten_vad.min_speech_duration); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], ten_vad.window_size); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], ten_vad.max_speech_duration); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], sample_rate); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], num_threads); + ONNX_VAD_CONFIG_AUTO_SET(file_body["mode_param"], debug); + + if (config_body.contains("awake_delay")) + awake_delay_ = config_body["awake_delay"].get(); + else if (file_body["mode_param"].contains("awake_delay")) + awake_delay_ = file_body["mode_param"]["awake_delay"]; + + if (config_body.contains("silence_timeout")) + silence_timeout = config_body["silence_timeout"].get(); + else if (file_body["mode_param"].contains("silence_timeout")) + silence_timeout = file_body["mode_param"]["silence_timeout"]; + + onnx_asr_config_.model_config.sense_voice.model = base_model + onnx_asr_config_.model_config.sense_voice.model; + onnx_asr_config_.model_config.tokens = base_model + onnx_asr_config_.model_config.tokens; + vad_config_.silero_vad.model = base_model + vad_config_.silero_vad.model; + + onnx_recognizer_ = std::make_unique(onnx_asr_config_); + vad_ = std::make_unique(vad_config_); + return 0; + } + int load_model(const nlohmann::json &config_body) { if (parse_config(config_body)) { return -1; } + nlohmann::json file_body; std::list config_file_paths = get_config_file_paths(base_model_path_, base_model_config_path_, model_); + try { for (auto file_name : config_file_paths) { std::ifstream config_file(file_name); @@ -110,68 +327,16 @@ public: SLOGE("all config file miss"); return -2; } - std::string base_model = base_model_path_ + model_ + "/"; - SLOGI("base_model %s", base_model.c_str()); - CONFIG_AUTO_SET(file_body["mode_param"], feat_config.sampling_rate); - CONFIG_AUTO_SET(file_body["mode_param"], feat_config.feature_dim); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.encoder_param); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.encoder_bin); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.decoder_param); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.decoder_bin); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.joiner_param); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.joiner_bin); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.tokens); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.encoder_opt.num_threads); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.decoder_opt.num_threads); - CONFIG_AUTO_SET(file_body["mode_param"], model_config.joiner_opt.num_threads); - CONFIG_AUTO_SET(file_body["mode_param"], decoder_config.method); - CONFIG_AUTO_SET(file_body["mode_param"], decoder_config.num_active_paths); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.must_contain_nonsilence); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_trailing_silence); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_utterance_length); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.must_contain_nonsilence); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_trailing_silence); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_utterance_length); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.must_contain_nonsilence); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_trailing_silence); - CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_utterance_length); - CONFIG_AUTO_SET(file_body["mode_param"], enable_endpoint); - CONFIG_AUTO_SET(file_body["mode_param"], hotwords_file); - CONFIG_AUTO_SET(file_body["mode_param"], hotwords_score); - if (config_body.contains("awake_delay")) - awake_delay_ = config_body["awake_delay"].get(); - else if (file_body["mode_param"].contains("awake_delay")) - awake_delay_ = file_body["mode_param"]["awake_delay"]; - if (config_body.contains("rule1")) { - mode_config_.endpoint_config.rule1.min_trailing_silence = config_body["rule1"].get(); - mode_config_.endpoint_config.rule1.must_contain_nonsilence = - (mode_config_.endpoint_config.rule1.min_trailing_silence == 0.0f) ? false : true; + if (engine_type_ == ENGINE_NCNN) { + return load_ncnn_model(config_body, file_body); + } else { + return load_onnx_model(config_body, file_body); } - if (config_body.contains("rule2")) { - mode_config_.endpoint_config.rule2.min_trailing_silence = config_body["rule2"].get(); - mode_config_.endpoint_config.rule2.must_contain_nonsilence = - (mode_config_.endpoint_config.rule2.min_trailing_silence == 0.0f) ? false : true; - } - if (config_body.contains("rule3")) { - mode_config_.endpoint_config.rule3.min_utterance_length = config_body["rule3"].get(); - mode_config_.endpoint_config.rule3.must_contain_nonsilence = - (mode_config_.endpoint_config.rule3.min_utterance_length == 0.0f) ? false : true; - } - - mode_config_.model_config.tokens = base_model + mode_config_.model_config.tokens; - mode_config_.model_config.encoder_param = base_model + mode_config_.model_config.encoder_param; - mode_config_.model_config.encoder_bin = base_model + mode_config_.model_config.encoder_bin; - mode_config_.model_config.decoder_param = base_model + mode_config_.model_config.decoder_param; - mode_config_.model_config.decoder_bin = base_model + mode_config_.model_config.decoder_bin; - mode_config_.model_config.joiner_param = base_model + mode_config_.model_config.joiner_param; - mode_config_.model_config.joiner_bin = base_model + mode_config_.model_config.joiner_bin; - recognizer_ = std::make_unique(mode_config_); } catch (...) { SLOGE("config false"); return -3; } - return 0; } void set_output(task_callback_t out_callback) @@ -179,7 +344,7 @@ public: out_callback_ = out_callback; } - void sys_pcm_on_data(const std::string &raw) + void sys_pcm_on_data_ncnn(const std::string &raw) { static int count = 0; if (count < delay_audio_frame_) { @@ -191,48 +356,110 @@ public: buffer_position_set(pcmdata, 0); std::vector floatSamples; - { - int16_t audio_val; - while (buffer_read_i16(pcmdata, &audio_val, 1)) { - float normalizedSample = static_cast(audio_val) / INT16_MAX; - floatSamples.push_back(normalizedSample); - } + int16_t audio_val; + while (buffer_read_i16(pcmdata, &audio_val, 1)) { + float normalizedSample = static_cast(audio_val) / INT16_MAX; + floatSamples.push_back(normalizedSample); } - buffer_resize(pcmdata, 0); count = 0; - if (awake_flage_ && recognizer_stream_) { - recognizer_stream_.reset(); + if (awake_flage_ && ncnn_stream_) { + ncnn_stream_.reset(); awake_flage_ = false; } - if (!recognizer_stream_) { - recognizer_stream_ = recognizer_->CreateStream(); + if (!ncnn_stream_) { + ncnn_stream_ = ncnn_recognizer_->CreateStream(); } - recognizer_stream_->AcceptWaveform(mode_config_.feat_config.sampling_rate, floatSamples.data(), - floatSamples.size()); - while (recognizer_->IsReady(recognizer_stream_.get())) { - recognizer_->DecodeStream(recognizer_stream_.get()); + + ncnn_stream_->AcceptWaveform(ncnn_config_.feat_config.sampling_rate, floatSamples.data(), floatSamples.size()); + + while (ncnn_recognizer_->IsReady(ncnn_stream_.get())) { + ncnn_recognizer_->DecodeStream(ncnn_stream_.get()); } - std::string text = recognizer_->GetResult(recognizer_stream_.get()).text; + + std::string text = ncnn_recognizer_->GetResult(ncnn_stream_.get()).text; std::string lower_text; lower_text.resize(text.size()); std::transform(text.begin(), text.end(), lower_text.begin(), [](const char c) { return std::tolower(c); }); + if ((!lower_text.empty()) && out_callback_) out_callback_(lower_text, false); - bool is_endpoint = recognizer_->IsEndpoint(recognizer_stream_.get()); + + bool is_endpoint = ncnn_recognizer_->IsEndpoint(ncnn_stream_.get()); if (is_endpoint) { - std::cout << "asr have a is_endpoint \n"; - recognizer_stream_->Finalize(); + ncnn_stream_->Finalize(); if ((!lower_text.empty()) && out_callback_) { out_callback_(lower_text, true); } - recognizer_stream_.reset(); + ncnn_stream_.reset(); if (ensleep_) { if (pause) pause(); } } } + void sys_pcm_on_data_onnx(const std::string &raw) + { + static int count = 0; + if (count < delay_audio_frame_) { + buffer_write_char(pcmdata, raw.data(), raw.length()); + count++; + return; + } + buffer_write_char(pcmdata, raw.data(), raw.length()); + buffer_position_set(pcmdata, 0); + + std::vector floatSamples; + int16_t audio_val; + while (buffer_read_i16(pcmdata, &audio_val, 1)) { + float normalizedSample = static_cast(audio_val) / INT16_MAX; + floatSamples.push_back(normalizedSample); + } + buffer_resize(pcmdata, 0); + count = 0; + + vad_->AcceptWaveform(floatSamples.data(), floatSamples.size()); + while (!vad_->Empty()) { + const auto &segment = vad_->Front(); + auto s = onnx_recognizer_->CreateStream(); + s->AcceptWaveform(onnx_asr_config_.feat_config.sampling_rate, segment.samples.data(), + segment.samples.size()); + onnx_recognizer_->DecodeStream(s.get()); + const auto &result = s->GetResult(); + if (!result.text.empty() && out_callback_) { + SLOGI("onnx-asr result: %s", result.text.c_str()); + out_callback_(result.text, true); + } + vad_->Pop(); + } + + { + bool detected = vad_->IsSpeechDetected(); + float chunk_ms = (delay_audio_frame_ + 1) * 10.0f; + + if (detected) { + silence_ms_accum_ = 0.0f; + } else { + silence_ms_accum_ += chunk_ms; + } + if (silence_ms_accum_ >= silence_timeout) { + if (ensleep_) { + if (pause) pause(); + } + silence_ms_accum_ = 0.0f; + } + } + } + + void sys_pcm_on_data(const std::string &raw) + { + if (engine_type_ == ENGINE_NCNN) { + sys_pcm_on_data_ncnn(raw); + } else { + sys_pcm_on_data_onnx(raw); + } + } + void kws_awake() { awake_flage_ = true; @@ -240,7 +467,8 @@ public: bool delete_model() { - recognizer_.reset(); + ncnn_recognizer_.reset(); + onnx_recognizer_.reset(); return true; } @@ -254,7 +482,6 @@ public: void start() { } - void stop() { } @@ -266,8 +493,6 @@ public: } }; -#undef CONFIG_AUTO_SET - class llm_asr : public StackFlow { public: enum { EVENT_LOAD_CONFIG = EVENT_EXPORT + 1, EVENT_TASK_PAUSE }; @@ -292,12 +517,14 @@ public: if (!(llm_task_obj && llm_channel)) { return; } + std::string tmp_msg1; const std::string *next_data = &data; if (finish) { tmp_msg1 = data + "."; next_data = &tmp_msg1; } + if (llm_channel->enstream_) { static int count = 0; nlohmann::json data_body; @@ -317,7 +544,7 @@ public: { int post = 0; if (in.length() > 10) - for (int i = 0; i < in.length() - 4; i++) { + for (int i = 0; i < (int)in.length() - 4; i++) { if ((in[i] == 'd') && (in[i + 1] == 'a') && (in[i + 2] == 't') && (in[i + 3] == 'a')) { post = i + 8; break; @@ -329,7 +556,6 @@ public: } else { return 0; } - return 0; } int decode_mp3(const std::string &in, std::string &out) @@ -350,15 +576,17 @@ public: send("None", "None", error_body, unit_name_); return; } + std::string tmp_msg1; const std::string *next_data = &data; int ret; + if (object.find("stream") != std::string::npos) { static std::unordered_map stream_buff; try { if (decode_stream(data, tmp_msg1, stream_buff)) { return; - }; + } } catch (...) { stream_buff.clear(); error_body["code"] = -25; @@ -368,6 +596,7 @@ public: } next_data = &tmp_msg1; } + std::string tmp_msg2; if (object.find("base64") != std::string::npos) { ret = decode_base64((*next_data), tmp_msg2); @@ -379,6 +608,7 @@ public: } next_data = &tmp_msg2; } + std::string tmp_msg3; if (object.find("wav") != std::string::npos) { ret = decode_wav((*next_data), tmp_msg3); @@ -387,6 +617,7 @@ public: } next_data = &tmp_msg3; } + std::string tmp_msg4; if (object.find("mp3") != std::string::npos) { ret = decode_mp3((*next_data), tmp_msg4); @@ -395,6 +626,7 @@ public: } next_data = &tmp_msg4; } + llm_task_obj->sys_pcm_on_data((*next_data)); } @@ -430,7 +662,9 @@ public: if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { std::weak_ptr _llm_task_obj = llm_task_obj; llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::shared_ptr &raw) { - _llm_task_obj.lock()->sys_pcm_on_data(raw->string()); + if (auto p = _llm_task_obj.lock()) { + p->sys_pcm_on_data(raw->string()); + } }); llm_task_obj->audio_flage_ = true; } @@ -452,7 +686,6 @@ public: void work(const std::string &work_id, const std::string &object, const std::string &data) override { SLOGI("llm_asr::work:%s", data.c_str()); - nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); if (llm_task_.find(work_id_num) == llm_task_.end()) { @@ -467,8 +700,7 @@ public: void pause(const std::string &work_id, const std::string &object, const std::string &data) override { - SLOGI("llm_asr::work:%s", data.c_str()); - + SLOGI("llm_asr::pause:%s", data.c_str()); nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); if (llm_task_.find(work_id_num) == llm_task_.end()) { @@ -502,9 +734,10 @@ public: SLOGE("setup json format error."); error_body["code"] = -2; error_body["message"] = "json format error."; - send("None", "None", error_body, "kws"); + send("None", "None", error_body, "asr"); return -2; } + int ret = llm_task_obj->load_model(config_body); if (ret == 0) { llm_channel->set_output(llm_task_obj->enoutput_); @@ -514,13 +747,15 @@ public: std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); - for (const auto input : llm_task_obj->inputs_) { + for (const auto &input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { audio_url_ = unit_call("audio", "cap", input); std::weak_ptr _llm_task_obj = llm_task_obj; llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::shared_ptr &raw) { - _llm_task_obj.lock()->sys_pcm_on_data(raw->string()); + if (auto p = _llm_task_obj.lock()) { + p->sys_pcm_on_data(raw->string()); + } }); llm_task_obj->audio_flage_ = true; } else if (input.find("asr") != std::string::npos) { @@ -538,12 +773,13 @@ public: std::placeholders::_2)); } } + llm_task_[work_id_num] = llm_task_obj; - SLOGI("load_mode success"); + SLOGI("load_model success"); send("None", "None", LLM_NO_ERROR, work_id); return 0; } else { - SLOGE("load_mode Failed"); + SLOGE("load_model Failed"); error_body["code"] = -5; error_body["message"] = "Model loading failed."; send("None", "None", error_body, "asr"); @@ -563,13 +799,17 @@ public: send("None", "None", error_body, work_id); return; } + auto llm_channel = get_channel(work_id); auto llm_task_obj = llm_task_[work_id_num]; + if (data.find("sys") != std::string::npos) { if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data); std::weak_ptr _llm_task_obj = llm_task_obj; llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::shared_ptr &raw) { - _llm_task_obj.lock()->sys_pcm_on_data(raw->string()); + if (auto p = _llm_task_obj.lock()) { + p->sys_pcm_on_data(raw->string()); + } }); llm_task_obj->audio_flage_ = true; llm_task_obj->inputs_.push_back(data); @@ -581,6 +821,7 @@ public: std::weak_ptr(llm_channel), std::placeholders::_1, std::placeholders::_2)); llm_task_obj->inputs_.push_back(data); } + if (ret) { error_body["code"] = -20; error_body["message"] = "link false"; @@ -594,7 +835,6 @@ public: void unlink(const std::string &work_id, const std::string &object, const std::string &data) override { SLOGI("llm_asr::unlink:%s", data.c_str()); - int ret = 0; nlohmann::json error_body; int work_id_num = sample_get_work_id_num(work_id); if (llm_task_.find(work_id_num) == llm_task_.end()) { @@ -603,8 +843,10 @@ public: send("None", "None", error_body, work_id); return; } + auto llm_channel = get_channel(work_id); llm_channel->stop_subscriber_work_id(data); + auto llm_task_obj = llm_task_[work_id_num]; for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) { if (*it == data) { @@ -619,13 +861,12 @@ public: void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { SLOGI("llm_asr::taskinfo:%s", data.c_str()); - nlohmann::json req_body; int work_id_num = sample_get_work_id_num(work_id); if (WORK_ID_NONE == work_id_num) { std::vector task_list; std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), - [](const auto task_channel) { return task_channel.second->work_id_; }); + [](const auto &task_channel) { return task_channel.second->work_id_; }); req_body = task_list; send("asr.tasklist", req_body, LLM_NO_ERROR, work_id); } else { @@ -669,17 +910,17 @@ public: ~llm_asr() { while (1) { - auto iteam = llm_task_.begin(); - if (iteam == llm_task_.end()) { + auto it = llm_task_.begin(); + if (it == llm_task_.end()) { break; } - iteam->second->stop(); - if (iteam->second->audio_flage_) { + it->second->stop(); + if (it->second->audio_flage_) { unit_call("audio", "cap_stop", "None"); } - get_channel(iteam->first)->stop_subscriber(""); - iteam->second.reset(); - llm_task_.erase(iteam->first); + get_channel(it->first)->stop_subscriber(""); + it->second.reset(); + llm_task_.erase(it->first); } } }; diff --git a/projects/llm_framework/main_audio/SConstruct b/projects/llm_framework/main_audio/SConstruct index 444248e..35b7023 100644 --- a/projects/llm_framework/main_audio/SConstruct +++ b/projects/llm_framework/main_audio/SConstruct @@ -5,7 +5,11 @@ with open(env['PROJECT_TOOL_S']) as f: exec(f.read()) # SRCS = append_srcs_dir(ADir('src')) -SRCS = Glob('src/*.c*') +if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: + SRCS = [AFile('src/sample_audio.c'), AFile('src/main.cpp')] +else: + SRCS = [AFile('src/alsa_audio.c'), AFile('src/main.cpp')] + INCLUDE = [ADir('include'), ADir('.')] PRIVATE_INCLUDE = [] REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'single_header_libs'] diff --git a/projects/llm_framework/main_audio/src/alsa_audio.c b/projects/llm_framework/main_audio/src/alsa_audio.c index 944497a..fa86a5e 100644 --- a/projects/llm_framework/main_audio/src/alsa_audio.c +++ b/projects/llm_framework/main_audio/src/alsa_audio.c @@ -6,7 +6,11 @@ #include #include -static int gcapLoopExit = 0; +static struct pcm *g_play_pcm = NULL; +static int gplayLoopExit = 1; +static int gcapLoopExit = 1; +AlsaConfig cap_config; +AlsaConfig play_config; void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, AUDIOCallback callback) @@ -23,7 +27,7 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch memset(&config, 0, sizeof(config)); config.channels = channel; - config.rate = 48000; // TODO: 部分USB MIC仅支持48k，暂时固定采集为48k + config.rate = 48000; config.period_size = 120; config.period_count = 4; config.format = PCM_FORMAT_S16_LE; @@ -57,14 +61,16 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch SRC_STATE *src_state = NULL; float *in_float = NULL, *out_float = NULL; + int in_frames = pcm_get_buffer_size(pcm); int out_frames = (int)((float)in_frames * ((float)rate / 48000.0f) + 1); - int out_bytes = out_frames * channel * sizeof(short); + + int src_channels = 1; if (rate != 48000) { - src_state = src_new(SRC_SINC_FASTEST, channel, NULL); - in_float = malloc(in_frames * channel * sizeof(float)); - out_float = malloc(out_frames * channel * sizeof(float)); + src_state = src_new(SRC_SINC_FASTEST, src_channels, NULL); + in_float = malloc(in_frames * src_channels * sizeof(float)); + out_float = malloc(out_frames * src_channels * sizeof(float)); if (!src_state || !in_float || !out_float) { fprintf(stderr, "Unable to allocate resample buffers\n"); free(buffer); @@ -82,16 +88,36 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch fprintf(stderr, "Error capturing samples - %d (%s)\n", errno, strerror(errno)); break; } + frames_read = ret; total_frames_read += frames_read; if (rate == 48000) { - callback(buffer, frames_read * bytes_per_frame); - } else { - short *in_short = (short *)buffer; - for (int i = 0; i < frames_read * channel; ++i) { - in_float[i] = in_short[i] / 32768.0f; + int in_channels = channel; // 比如 4 + int16_t *in = (int16_t *)buffer; + + int16_t *ch0 = malloc(frames_read * sizeof(int16_t)); + if (!ch0) { + fprintf(stderr, "Unable to allocate ch0 buffer\n"); + break; } + + for (unsigned int i = 0; i < frames_read; ++i) { + ch0[i] = in[i * in_channels + 0]; + } + + callback((const char *)ch0, frames_read * sizeof(int16_t)); + free(ch0); + + } else { + int in_channels = channel; // 比如 4 + short *in_short = (short *)buffer; + + for (int i = 0; i < frames_read; ++i) { + short s = in_short[i * in_channels + 0]; + in_float[i] = s / 32768.0f; + } + SRC_DATA src_data; src_data.data_in = in_float; src_data.input_frames = frames_read; @@ -99,20 +125,25 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch src_data.output_frames = out_frames; src_data.src_ratio = (double)rate / 48000.0; src_data.end_of_input = 0; - int error = src_process(src_state, &src_data); + + int error = src_process(src_state, &src_data); if (error) { fprintf(stderr, "SRC error: %s\n", src_strerror(error)); break; } - // float转short - short *out_short = malloc(src_data.output_frames_gen * channel * sizeof(short)); - for (int i = 0; i < src_data.output_frames_gen * channel; ++i) { + int out_samples = src_data.output_frames_gen; + short *out_short = malloc(out_samples * sizeof(short)); + if (!out_short) { + fprintf(stderr, "Unable to allocate out_short buffer\n"); + break; + } + for (int i = 0; i < out_samples; ++i) { float sample = out_float[i]; if (sample > 1.0f) sample = 1.0f; if (sample < -1.0f) sample = -1.0f; out_short[i] = (short)(sample * 32767.0f); } - callback((const char *)out_short, src_data.output_frames_gen * channel * sizeof(short)); + callback((const char *)out_short, out_samples * sizeof(short)); free(out_short); } } @@ -124,7 +155,6 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch } free(buffer); pcm_close(pcm); - printf("Total frames captured: %u\n", total_frames_read); } void alsa_close_cap() @@ -135,4 +165,52 @@ void alsa_close_cap() int alsa_cap_status() { return gcapLoopExit; +} + +void alsa_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data, + int size) +{ + gplayLoopExit = 0; + + struct pcm_config config; + memset(&config, 0, sizeof(config)); + config.channels = channel; + config.rate = rate; + config.period_size = 1024; + config.period_count = 2; + config.format = PCM_FORMAT_S16_LE; + config.silence_threshold = config.period_size * config.period_count; + config.stop_threshold = config.period_size * config.period_count; + config.start_threshold = config.period_size; + + unsigned int pcm_open_flags = PCM_OUT; + + struct pcm *pcm = pcm_open(card, device, pcm_open_flags, &config); + if (!pcm || !pcm_is_ready(pcm)) { + fprintf(stderr, "Unable to open PCM playback device (%s)\n", pcm ? pcm_get_error(pcm) : "invalid pcm"); + if (pcm) { + pcm_close(pcm); + } + gplayLoopExit = 2; + return; + } + + int frames = pcm_bytes_to_frames(pcm, size); + int written_frames = pcm_writei(pcm, data, frames); + if (written_frames < 0) { + fprintf(stderr, "PCM playback error %s\n", pcm_get_error(pcm)); + } + printf("Played %d frames\n", written_frames); + pcm_close(pcm); + gplayLoopExit = 2; +} + +void alsa_close_play() +{ + gplayLoopExit = 1; +} + +int alsa_play_status() +{ + return gplayLoopExit; } \ No newline at end of file diff --git a/projects/llm_framework/main_audio/src/alsa_audio.h b/projects/llm_framework/main_audio/src/alsa_audio.h index 95c469a..646af36 100644 --- a/projects/llm_framework/main_audio/src/alsa_audio.h +++ b/projects/llm_framework/main_audio/src/alsa_audio.h @@ -1,5 +1,18 @@ #pragma once +typedef struct AlsaConfig +{ + unsigned int card; + unsigned int device; + float volume; + int channel; + int rate; + int bit; +} AlsaConfig; + +extern AlsaConfig cap_config; +extern AlsaConfig play_config; + #ifdef __cplusplus extern "C" { #endif @@ -10,6 +23,13 @@ void alsa_cap_start(unsigned int card, unsigned int device, float Volume, int ch AUDIOCallback callback); void alsa_close_cap(); +void alsa_play(unsigned int card, unsigned int device, float Volume, int channel, int rate, int bit, const void *data, + int size); + +void alsa_close_play(); +int alsa_cap_status(); +int alsa_play_status(); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index de6165f..3500a44 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -13,6 +13,7 @@ #include #include #include "../../../../SDK/components/utilities/include/sample_log.h" +#include int main_exit_flage = 0; static void __sigint(int iSigNo) @@ -21,8 +22,11 @@ static void __sigint(int iSigNo) main_exit_flage = 1; } +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) #include "sample_audio.h" +#else #include "alsa_audio.h" +#endif #define CONFIG_AUTO_SET(obj, key) \ if (config_body.contains(#key)) \ @@ -80,8 +84,13 @@ private: } std::lock_guard guard(ax_play_mtx); +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) ax_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate, - play_config.bit, final_data.c_str(), final_data.length()); + play_config.bit, audio_data.c_str(), audio_data.length()); +#else + alsa_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate, + play_config.bit, final_data.c_str(), final_data.length()); +#endif } void hw_play(const std::string &audio_data) @@ -92,8 +101,13 @@ private: final_data = mono_to_stereo_s16le(audio_data); } std::lock_guard guard(ax_play_mtx); +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) ax_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate, play_config.bit, audio_data.c_str(), audio_data.length()); +#else + alsa_play(play_config.card, play_config.device, play_config.volume, play_config.channel, play_config.rate, + play_config.bit, final_data.c_str(), final_data.length()); +#endif } void hw_cap() @@ -117,7 +131,11 @@ private: void _play_stop() { +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) ax_close_play(); +#else + alsa_close_play(); +#endif if (audio_play_thread_) { audio_play_thread_->join(); audio_play_thread_.reset(); @@ -134,7 +152,7 @@ private: void _cap_stop() { -#if (defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED)) && !defined(CONFIG_AXCL_ENABLED) +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) ax_close_cap(); #else @@ -219,8 +237,10 @@ public: send("None", "None", error_body, "audio"); return -2; } - AX_AUDIO_SAMPLE_CONFIG_t mode_config_; + try { +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) + AX_AUDIO_SAMPLE_CONFIG_t mode_config_; memset(&mode_config_, 0, sizeof(AX_AUDIO_SAMPLE_CONFIG_t)); if (object == "audio.play") { CONFIG_AUTO_SET(file_body["play_param"], stPoolConfig.MetaSize); @@ -244,22 +264,7 @@ public: CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.enAgcMode); CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.s16TargetLevel); CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.s16Gain); -#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) - CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.bEnable); - CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.s32GainDb); - CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.s32Freq); - CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.bEnable); - CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.s32GainDb); - CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.s32Samplerate); - CONFIG_AUTO_SET(file_body["play_param"], stLpfAttr.s32Freq); - CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.bEnable); - CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[0]); - CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[1]); - CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[2]); - CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[3]); - CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[4]); - CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32Samplerate); -#endif + CONFIG_AUTO_SET(file_body["play_param"], gResample); CONFIG_AUTO_SET(file_body["play_param"], enInSampleRate); CONFIG_AUTO_SET(file_body["play_param"], gInstant); @@ -272,12 +277,12 @@ public: CONFIG_AUTO_SET(file_body["play_param"], bit); if (config_body.contains("stPoolConfig.PartitionName")) { std::string PartitionName = config_body["stPoolConfig.PartitionName"]; - for (int i = 0; i < PartitionName.length(); i++) { + for (int i = 0; i < (int)PartitionName.length(); i++) { mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i]; } - } else if (file_body["cap_param"].contains("stPoolConfig.PartitionName")) { - std::string PartitionName = file_body["cap_param"]["stPoolConfig.PartitionName"]; - for (int i = 0; i < PartitionName.length(); i++) { + } else if (file_body["play_param"].contains("stPoolConfig.PartitionName")) { + std::string PartitionName = file_body["play_param"]["stPoolConfig.PartitionName"]; + for (int i = 0; i < (int)PartitionName.length(); i++) { mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i]; } } @@ -292,7 +297,7 @@ public: CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enBitwidth); CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enLinkMode); CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enSamplerate); - // CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enLayoutMode); + if (config_body.contains("aistAttr.enLayoutMode")) mode_config_.aistAttr.enLayoutMode = config_body["aistAttr.enLayoutMode"]; else if (file_body["cap_param"].contains("aistAttr.enLayoutMode")) { @@ -316,23 +321,7 @@ public: CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAgcCfg.s16TargetLevel); CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAgcCfg.s16Gain); CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAecCfg.enAecMode); -#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) - CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.bEnable); - CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32GainDb); - CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32Samplerate); - CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32Freq); - CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.bEnable); - CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.s32GainDb); - CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.s32Samplerate); - CONFIG_AUTO_SET(file_body["cap_param"], stLpfAttr.s32Freq); - CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.bEnable); - CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[0]); - CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[1]); - CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[2]); - CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[3]); - CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[4]); - CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32Samplerate); -#endif + CONFIG_AUTO_SET(file_body["cap_param"], gResample); CONFIG_AUTO_SET(file_body["cap_param"], enOutSampleRate); CONFIG_AUTO_SET(file_body["cap_param"], gDbDetection); @@ -345,12 +334,12 @@ public: if (config_body.contains("stPoolConfig.PartitionName")) { std::string PartitionName = config_body["stPoolConfig.PartitionName"]; - for (int i = 0; i < PartitionName.length(); i++) { + for (int i = 0; i < (int)PartitionName.length(); i++) { mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i]; } } else if (file_body["cap_param"].contains("stPoolConfig.PartitionName")) { std::string PartitionName = file_body["cap_param"]["stPoolConfig.PartitionName"]; - for (int i = 0; i < PartitionName.length(); i++) { + for (int i = 0; i < (int)PartitionName.length(); i++) { mode_config_.stPoolConfig.PartitionName[i] = PartitionName[i]; } } @@ -361,6 +350,31 @@ public: } memcpy(&cap_config, &mode_config_, sizeof(AX_AUDIO_SAMPLE_CONFIG_t)); } + +#else + AlsaConfig mode_config_; + memset(&mode_config_, 0, sizeof(AlsaConfig)); + + if (object == "audio.play") { + CONFIG_AUTO_SET(file_body["play_param"], card); + CONFIG_AUTO_SET(file_body["play_param"], device); + CONFIG_AUTO_SET(file_body["play_param"], volume); + CONFIG_AUTO_SET(file_body["play_param"], channel); + CONFIG_AUTO_SET(file_body["play_param"], rate); + CONFIG_AUTO_SET(file_body["play_param"], bit); + memcpy(&play_config, &mode_config_, sizeof(AlsaConfig)); + } + + if (object == "audio.cap") { + CONFIG_AUTO_SET(file_body["cap_param"], card); + CONFIG_AUTO_SET(file_body["cap_param"], device); + CONFIG_AUTO_SET(file_body["cap_param"], volume); + CONFIG_AUTO_SET(file_body["cap_param"], channel); + CONFIG_AUTO_SET(file_body["cap_param"], rate); + CONFIG_AUTO_SET(file_body["cap_param"], bit); + memcpy(&cap_config, &mode_config_, sizeof(AlsaConfig)); + } +#endif } catch (...) { error_body["code"] = -22; error_body["message"] = "Parameter format error."; @@ -471,33 +485,23 @@ public: std::string audio_status(pzmq *_pzmq, const std::shared_ptr &rawdata) { std::string _rawdata = rawdata->string(); + +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) + auto play_state = ax_play_status(); + auto cap_state = ax_cap_status(); +#else + auto play_state = alsa_play_status(); + auto cap_state = alsa_cap_status(); +#endif + if (_rawdata == "play") { - if (ax_play_status()) { - return std::string("None"); - } else { - return std::string("Runing"); - } + return play_state ? "None" : "Runing"; } else if (_rawdata == "cap") { - if (ax_cap_status()) { - return std::string("None"); - } else { - return std::string("Runing"); - } + return cap_state ? "None" : "Runing"; } else { std::ostringstream return_val; - return_val << "{\"play\":"; - if (ax_play_status()) { - return_val << "\"None\""; - } else { - return_val << "\"Runing\""; - } - return_val << "\"cap\":"; - if (ax_cap_status()) { - return_val << "\"None\""; - } else { - return_val << "\"Runing\""; - } - return_val << "\"}"; + return_val << "{\"play\":" << (play_state ? "\"None\"" : "\"Runing\"") + << ",\"cap\":" << (cap_state ? "\"None\"" : "\"Runing\"") << "}"; return return_val.str(); } } diff --git a/projects/llm_framework/main_kws/SConstruct b/projects/llm_framework/main_kws/SConstruct index b46a1eb..f746d33 100644 --- a/projects/llm_framework/main_kws/SConstruct +++ b/projects/llm_framework/main_kws/SConstruct @@ -29,11 +29,11 @@ INCLUDE += [ADir('../static_lib/include/sherpa'), ] LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')] -LDFLAGS += ['-l:libcargs.a', - '-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', +REQUIREMENTS += ['onnxruntime', 'cargs'] + +LDFLAGS += ['-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a','-l:libkissfft-float.a', '-l:libkaldi-decoder-core.a', '-l:libssentencepiece_core.a'] -REQUIREMENTS += ['onnxruntime'] STATIC_FILES += [os.path.join(python_venv, 'sherpa-onnx')] STATIC_FILES += Glob('llm-kws_text2token.py') diff --git a/projects/llm_framework/main_kws_new/SConstruct b/projects/llm_framework/main_kws_new/SConstruct index 2458e81..d9e0eca 100644 --- a/projects/llm_framework/main_kws_new/SConstruct +++ b/projects/llm_framework/main_kws_new/SConstruct @@ -57,6 +57,8 @@ ignore['ignore'] = list(set(ignore['ignore'])) with open('../dist/fileignore', 'w') as f: json.dump(ignore, f, indent=4) +STATIC_FILES += Glob('mode_*.json') + env['COMPONENTS'].append({'target':'llm_kws_new-1.9', 'SRCS':SRCS, 'INCLUDE':INCLUDE, diff --git a/projects/llm_framework/main_kws_new/mode_kws.json b/projects/llm_framework/main_kws_new/mode_kws.json new file mode 100644 index 0000000..5647a8a --- /dev/null +++ b/projects/llm_framework/main_kws_new/mode_kws.json @@ -0,0 +1,41 @@ +{ + "mode": "kws", + "type": "kws", + "homepage": "", + "capabilities": [ + "Keyword_spotting", + "English" + ], + "input_type": [ + "sys.pcm", + "sys.cap.0_0" + ], + "output_type": [ + "kws.bool" + ], + "mode_param": { + "model": "kws.onnx", + "wake_wav_file": "/opt/m5stack/data/audio/wakeup_zh_cn.wav", + "chunk_size": 32, + "threshold": 0.9, + "min_continuous_frames": 5, + "REFRACTORY_TIME_MS": 2000, + "RESAMPLE_RATE": 16000, + "FEAT_DIM": 80, + "frame_opts.samp_freq": 16000, + "frame_opts.frame_length_ms": 25.0, + "frame_opts.frame_shift_ms": 10.0, + "frame_opts.snip_edges": false, + "frame_opts.dither": 0.0, + "frame_opts.preemph_coeff": 0.97, + "frame_opts.remove_dc_offset": true, + "frame_opts.window_type": "povey", + "mel_opts.num_bins": 80, + "mel_opts.low_freq": 20, + "mel_opts.high_freq": 0, + "energy_floor": 0.0, + "use_energy": false, + "raw_energy": true + }, + "mode_param_bak": {} +} \ No newline at end of file diff --git a/projects/llm_framework/main_kws_new/src/main.cpp b/projects/llm_framework/main_kws_new/src/main.cpp index b56f3c6..a395fd9 100644 --- a/projects/llm_framework/main_kws_new/src/main.cpp +++ b/projects/llm_framework/main_kws_new/src/main.cpp @@ -191,9 +191,7 @@ public: bool detect_wakeup(const std::vector &scores) { bool triggered = false; - SLOGE("%d", scores.size()); for (auto score : scores) { - printf("%f ", score); if (score > threshold) { count_frames++; if (count_frames >= min_continuous_frames) { @@ -208,7 +206,6 @@ public: } frame_index_global++; } - SLOGE("\n"); return triggered; } diff --git a/projects/llm_framework/main_vad/SConstruct b/projects/llm_framework/main_vad/SConstruct index 2d80206..f7d02e7 100644 --- a/projects/llm_framework/main_vad/SConstruct +++ b/projects/llm_framework/main_vad/SConstruct @@ -23,7 +23,8 @@ LINK_SEARCH_PATH += [ADir('../static_lib')] INCLUDE += [ADir('../static_lib/include/sherpa')] LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')] -LDFLAGS += ['-l:libsherpa-onnx-core.a'] + +LDFLAGS += ['-l:libsherpa-onnx-core.a', '-l:libkaldi-native-fbank-core.a', '-l:libkissfft-float.a'] REQUIREMENTS += ['onnxruntime']