diff --git a/projects/llm_framework/main_asr/SConstruct b/projects/llm_framework/main_asr/SConstruct index 9bdac46..217bbc5 100644 --- a/projects/llm_framework/main_asr/SConstruct +++ b/projects/llm_framework/main_asr/SConstruct @@ -21,7 +21,11 @@ DEFINITIONS += ['-std=c++17', '-fopenmp'] LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] -INCLUDE += [ADir('../static_lib/include/sherpa'), ADir('../static_lib/include/sherpa/sherpa-ncnn')] +INCLUDE += [ADir('../static_lib/include/sherpa'), + ADir('../static_lib/include/sherpa/sherpa-ncnn'), + ADir('../static_lib/include/sherpa/sherpa-onnx'), + ADir('../static_lib/include/sherpa/sherpa-onnx/openfst-src'), + ADir('../static_lib/include/sherpa/sherpa-onnx/onnxruntime-src')] LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/ncnn')] LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')] REQUIREMENTS += ['ncnn', '', 'onnxruntime'] diff --git a/projects/llm_framework/main_asr/mode_sherpa-onnx-zipformer-bilingual-zh-en-t.json b/projects/llm_framework/main_asr/mode_sherpa-onnx-zipformer-bilingual-zh-en-t.json new file mode 100644 index 0000000..e22da79 --- /dev/null +++ b/projects/llm_framework/main_asr/mode_sherpa-onnx-zipformer-bilingual-zh-en-t.json @@ -0,0 +1,70 @@ +{ + "mode": "sherpa-onnx-zipformer-bilingual-zh-en-t", + "type": "asr", + "homepage": "", + "capabilities": [ + "Automatic_Speech_Recognition", + "Chinese", + "English" + ], + "input_type": [ + "sys.pcm", + "sys.cap.0_0" + ], + "output_type": [ + "asr.utf-8", + "asr.bool" + ], + "mode_param": { + "model_config.transducer.encoder": "encoder.axmodel", + "model_config.transducer.decoder": "decoder.axmodel", + "model_config.transducer.joiner": "joiner.axmodel", + "model_config.tokens": "tokens.txt", + "feat_config.feature_dim": 80, + "feat_config.sampling_rate": 16000, + "endpoint_config.rule1.min_trailing_silence": 2.4, + "endpoint_config.rule2.min_trailing_silence": 1.2, + "endpoint_config.rule3.min_utterance_length": 30, + "enable_endpoint": true, + "awake_delay": 50, + "model_config.provider_config.provider": "axera", + "model_config.zipformer_meta.encoder_dims": [ + 256, + 256, + 256, + 256, + 256 + ], + "model_config.zipformer_meta.attention_dims": [ + 192, + 192, + 192, + 192, + 192 + ], + "model_config.zipformer_meta.num_encoder_layers": [ + 2, + 2, + 2, + 2, + 2 + ], + "model_config.zipformer_meta.cnn_module_kernels": [ + 31, + 31, + 31, + 31, + 31 + ], + "model_config.zipformer_meta.left_context_len": [ + 192, + 96, + 48, + 24, + 96 + ], + "model_config.zipformer_meta.T": 103, + "model_config.zipformer_meta.decode_chunk_len": 96, + "model_config.zipformer_meta.context_size": 2 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_asr/src/main.cpp b/projects/llm_framework/main_asr/src/main.cpp index 19fb336..eb3c80c 100644 --- a/projects/llm_framework/main_asr/src/main.cpp +++ b/projects/llm_framework/main_asr/src/main.cpp @@ -6,6 +6,7 @@ #include "StackFlow.h" #include "sherpa-ncnn/csrc/recognizer.h" #include "sherpa-onnx/csrc/offline-recognizer.h" +#include "sherpa-onnx/csrc/online-recognizer.h" #include "sherpa-onnx/csrc/voice-activity-detector.h" #include @@ -44,6 +45,12 @@ typedef std::function task_callback_ else if (obj.contains(#key)) \ ncnn_config_.key = obj[#key]; +#define ONNX_ONLINE_CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + config_body.at(#key).get_to(onnx_online_config.key); \ + else if ((obj).contains(#key)) \ + (obj).at(#key).get_to(onnx_online_config.key); + #define ONNX_ASR_CONFIG_AUTO_SET(obj, key) \ if (config_body.contains(#key)) \ onnx_asr_config_.key = config_body[#key]; \ @@ -63,13 +70,18 @@ private: std::unique_ptr ncnn_stream_; sherpa_onnx::OfflineRecognizerConfig onnx_asr_config_; + sherpa_onnx::OnlineRecognizerConfig onnx_online_config; + sherpa_onnx::VadModelConfig vad_config_; std::unique_ptr onnx_recognizer_; + std::unique_ptr onnx_online_recognizer_; + std::unique_ptr online_stream; std::unique_ptr vad_; enum EngineType { - ENGINE_NCNN = 0, - ENGINE_ONNX = 1, + ENGINE_NCNN = 0, + ENGINE_ONNX = 1, + ENGINE_ONLINE = 3, } engine_type_ = ENGINE_NCNN; public: @@ -112,6 +124,8 @@ public: if (model_.rfind("sherpa-ncnn", 0) == 0) { engine_type_ = ENGINE_NCNN; + } else if (model_.rfind("sherpa-onnx", 0) == 0) { + engine_type_ = ENGINE_ONLINE; } else { engine_type_ = ENGINE_ONNX; } @@ -301,6 +315,118 @@ public: return 0; } + int load_online_model(const nlohmann::json &config_body, const nlohmann::json &file_body) + { + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model (onnx) %s", base_model.c_str()); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.sampling_rate); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.feature_dim); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.low_freq); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.high_freq); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.dither); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.normalize_samples); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.snip_edges); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.frame_shift_ms); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.frame_length_ms); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_librosa); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.remove_dc_offset); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.preemph_coeff); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.window_type); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.nemo_normalize_type); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.num_ceps); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.use_energy); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_mfcc); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_whisper); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_t_one); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.round_to_power_of_two); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.encoder); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.decoder); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.joiner); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.provider_config.provider); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.paraformer.encoder); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.paraformer.decoder); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.wenet_ctc.model); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.wenet_ctc.chunk_size); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.wenet_ctc.num_left_chunks); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer2_ctc.model); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.nemo_ctc.model); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.t_one_ctc.model); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.encoder_dims); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.attention_dims); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.num_encoder_layers); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.cnn_module_kernels); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.left_context_len); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.T); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.decode_chunk_len); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.context_size); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.tokens); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.num_threads); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.warm_up); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.model_type); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.modeling_unit); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.bpe_vocab); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.tokens_buf); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.model); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.scale); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lm_num_threads); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lm_provider); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_fst); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_scale); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_backoff_id); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.shallow_fusion); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.must_contain_nonsilence); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_trailing_silence); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_utterance_length); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.must_contain_nonsilence); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_trailing_silence); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_utterance_length); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.must_contain_nonsilence); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_trailing_silence); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_utterance_length); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], ctc_fst_decoder_config.graph); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], ctc_fst_decoder_config.max_active); + + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], enable_endpoint); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], decoding_method); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], max_active_paths); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_file); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_score); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], blank_penalty); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], temperature_scale); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], rule_fsts); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], rule_fars); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], reset_encoder); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hr.dict_dir); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hr.lexicon); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hr.rule_fsts); + ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_buf); + + if (config_body.contains("awake_delay")) + awake_delay_ = config_body["awake_delay"].get(); + else if (file_body["mode_param"].contains("awake_delay")) + awake_delay_ = file_body["mode_param"]["awake_delay"]; + + onnx_online_config.model_config.transducer.encoder = + base_model + onnx_online_config.model_config.transducer.encoder; + onnx_online_config.model_config.transducer.decoder = + base_model + onnx_online_config.model_config.transducer.decoder; + onnx_online_config.model_config.transducer.joiner = + base_model + onnx_online_config.model_config.transducer.joiner; + onnx_online_config.model_config.tokens = base_model + onnx_online_config.model_config.tokens; + + onnx_online_recognizer_ = std::make_unique(onnx_online_config); + + return 0; + } + int load_model(const nlohmann::json &config_body) { if (parse_config(config_body)) { @@ -330,6 +456,8 @@ public: if (engine_type_ == ENGINE_NCNN) { return load_ncnn_model(config_body, file_body); + } else if (engine_type_ == ENGINE_ONLINE) { + return load_online_model(config_body, file_body); } else { return load_onnx_model(config_body, file_body); } @@ -451,10 +579,60 @@ public: } } + void sys_pcm_on_data_online(const std::string &raw) + { + static int count = 0; + if (count < delay_audio_frame_) { + buffer_write_char(pcmdata, raw.data(), raw.length()); + count++; + return; + } + buffer_write_char(pcmdata, raw.data(), raw.length()); + buffer_position_set(pcmdata, 0); + + std::vector floatSamples; + int16_t audio_val; + while (buffer_read_i16(pcmdata, &audio_val, 1)) { + float normalizedSample = static_cast(audio_val) / INT16_MAX; + floatSamples.push_back(normalizedSample); + } + buffer_resize(pcmdata, 0); + count = 0; + + if (!online_stream) online_stream = onnx_online_recognizer_->CreateStream(); + online_stream->AcceptWaveform(onnx_online_config.feat_config.sampling_rate, floatSamples.data(), + floatSamples.size()); + + while (onnx_online_recognizer_->IsReady(online_stream.get())) { + onnx_online_recognizer_->DecodeStream(online_stream.get()); + } + + auto text = onnx_online_recognizer_->GetResult(online_stream.get()).text; + std::string lower_text; + lower_text.resize(text.size()); + std::transform(text.begin(), text.end(), lower_text.begin(), [](auto c) { return std::tolower(c); }); + + if ((!lower_text.empty()) && out_callback_) out_callback_(lower_text, false); + + bool is_endpoint = onnx_online_recognizer_->IsEndpoint(online_stream.get()); + + if (is_endpoint) { + if ((!lower_text.empty()) && out_callback_) { + out_callback_(lower_text, true); + } + online_stream.reset(); + if (ensleep_) { + if (pause) pause(); + } + } + } + void sys_pcm_on_data(const std::string &raw) { if (engine_type_ == ENGINE_NCNN) { sys_pcm_on_data_ncnn(raw); + } else if (engine_type_ == ENGINE_ONLINE) { + sys_pcm_on_data_online(raw); } else { sys_pcm_on_data_onnx(raw); } diff --git a/projects/llm_framework/main_kws/mode_kws.json b/projects/llm_framework/main_kws/mode_kws.json index c1e11a9..8cff9bf 100644 --- a/projects/llm_framework/main_kws/mode_kws.json +++ b/projects/llm_framework/main_kws/mode_kws.json @@ -14,7 +14,7 @@ "kws.bool" ], "mode_param": { - "model": "kws.onnx", + "model": "kws.axmodel", "model_type": "onnx", "wake_wav_file": "/opt/m5stack/data/audio/wakeup_zh_cn.wav", "chunk_size": 32,