m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] llm_asr supported zipformer stream model.

Browse Source
This commit is contained in:
LittleMouse
2025-12-18 19:35:02 +08:00
parent d5685d4419
commit bea45abf1a
4 changed files with 256 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/llm_framework/main_asr/SConstruct
+5 -1
View File
@@ -21,7 +21,11 @@ DEFINITIONS += ['-std=c++17', '-fopenmp']
LINK_SEARCH_PATH += [ADir('../static_lib')]
REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
INCLUDE += [ADir('../static_lib/include/sherpa'), ADir('../static_lib/include/sherpa/sherpa-ncnn')]
INCLUDE += [ADir('../static_lib/include/sherpa'),
ADir('../static_lib/include/sherpa/sherpa-ncnn'),
ADir('../static_lib/include/sherpa/sherpa-onnx'),
ADir('../static_lib/include/sherpa/sherpa-onnx/openfst-src'),
ADir('../static_lib/include/sherpa/sherpa-onnx/onnxruntime-src')]
LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/ncnn')]
LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')]
REQUIREMENTS += ['ncnn', '', 'onnxruntime']
projects/llm_framework/main_asr/mode_sherpa-onnx-zipformer-bilingual-zh-en-t.json
+70
View File
@@ -0,0 +1,70 @@
{
"mode": "sherpa-onnx-zipformer-bilingual-zh-en-t",
"type": "asr",
"homepage": "",
"capabilities": [
"Automatic_Speech_Recognition",
"Chinese",
"English"
],
"input_type": [
"sys.pcm",
"sys.cap.0_0"
],
"output_type": [
"asr.utf-8",
"asr.bool"
],
"mode_param": {
"model_config.transducer.encoder": "encoder.axmodel",
"model_config.transducer.decoder": "decoder.axmodel",
"model_config.transducer.joiner": "joiner.axmodel",
"model_config.tokens": "tokens.txt",
"feat_config.feature_dim": 80,
"feat_config.sampling_rate": 16000,
"endpoint_config.rule1.min_trailing_silence": 2.4,
"endpoint_config.rule2.min_trailing_silence": 1.2,
"endpoint_config.rule3.min_utterance_length": 30,
"enable_endpoint": true,
"awake_delay": 50,
"model_config.provider_config.provider": "axera",
"model_config.zipformer_meta.encoder_dims": [
256,
256,
256,
256,
256
],
"model_config.zipformer_meta.attention_dims": [
192,
192,
192,
192,
192
],
"model_config.zipformer_meta.num_encoder_layers": [
2,
2,
2,
2,
2
],
"model_config.zipformer_meta.cnn_module_kernels": [
31,
31,
31,
31,
31
],
"model_config.zipformer_meta.left_context_len": [
192,
96,
48,
24,
96
],
"model_config.zipformer_meta.T": 103,
"model_config.zipformer_meta.decode_chunk_len": 96,
"model_config.zipformer_meta.context_size": 2
}
}
projects/llm_framework/main_asr/src/main.cpp
+180 -2
View File
@@ -6,6 +6,7 @@
#include "StackFlow.h"
#include "sherpa-ncnn/csrc/recognizer.h"
#include "sherpa-onnx/csrc/offline-recognizer.h"
#include "sherpa-onnx/csrc/online-recognizer.h"
#include "sherpa-onnx/csrc/voice-activity-detector.h"
#include <iostream>
@@ -44,6 +45,12 @@ typedef std::function<void(const std::string &data, bool finish)> task_callback_
else if (obj.contains(#key)) \
ncnn_config_.key = obj[#key];
#define ONNX_ONLINE_CONFIG_AUTO_SET(obj, key) \
if (config_body.contains(#key)) \
config_body.at(#key).get_to(onnx_online_config.key); \
else if ((obj).contains(#key)) \
(obj).at(#key).get_to(onnx_online_config.key);
#define ONNX_ASR_CONFIG_AUTO_SET(obj, key) \
if (config_body.contains(#key)) \
onnx_asr_config_.key = config_body[#key]; \
@@ -63,13 +70,18 @@ private:
std::unique_ptr<sherpa_ncnn::Stream> ncnn_stream_;
sherpa_onnx::OfflineRecognizerConfig onnx_asr_config_;
sherpa_onnx::OnlineRecognizerConfig onnx_online_config;
sherpa_onnx::VadModelConfig vad_config_;
std::unique_ptr<sherpa_onnx::OfflineRecognizer> onnx_recognizer_;
std::unique_ptr<sherpa_onnx::OnlineRecognizer> onnx_online_recognizer_;
std::unique_ptr<sherpa_onnx::OnlineStream> online_stream;
std::unique_ptr<sherpa_onnx::VoiceActivityDetector> vad_;
enum EngineType {
ENGINE_NCNN = 0,
ENGINE_ONNX = 1,
ENGINE_NCNN = 0,
ENGINE_ONNX = 1,
ENGINE_ONLINE = 3,
} engine_type_ = ENGINE_NCNN;
public:
@@ -112,6 +124,8 @@ public:
if (model_.rfind("sherpa-ncnn", 0) == 0) {
engine_type_ = ENGINE_NCNN;
} else if (model_.rfind("sherpa-onnx", 0) == 0) {
engine_type_ = ENGINE_ONLINE;
} else {
engine_type_ = ENGINE_ONNX;
}
@@ -301,6 +315,118 @@ public:
return 0;
}
int load_online_model(const nlohmann::json &config_body, const nlohmann::json &file_body)
{
std::string base_model = base_model_path_ + model_ + "/";
SLOGI("base_model (onnx) %s", base_model.c_str());
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.sampling_rate);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.feature_dim);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.low_freq);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.high_freq);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.dither);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.normalize_samples);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.snip_edges);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.frame_shift_ms);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.frame_length_ms);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_librosa);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.remove_dc_offset);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.preemph_coeff);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.window_type);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.nemo_normalize_type);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.num_ceps);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.use_energy);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_mfcc);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_whisper);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.is_t_one);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], feat_config.round_to_power_of_two);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.encoder);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.decoder);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.transducer.joiner);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.provider_config.provider);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.paraformer.encoder);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.paraformer.decoder);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.wenet_ctc.model);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.wenet_ctc.chunk_size);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.wenet_ctc.num_left_chunks);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer2_ctc.model);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.nemo_ctc.model);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.t_one_ctc.model);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.encoder_dims);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.attention_dims);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.num_encoder_layers);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.cnn_module_kernels);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.left_context_len);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.T);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.decode_chunk_len);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.zipformer_meta.context_size);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.tokens);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.num_threads);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.warm_up);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.model_type);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.modeling_unit);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.bpe_vocab);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], model_config.tokens_buf);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.model);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.scale);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lm_num_threads);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lm_provider);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_fst);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_scale);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.lodr_backoff_id);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], lm_config.shallow_fusion);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.must_contain_nonsilence);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_trailing_silence);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule1.min_utterance_length);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.must_contain_nonsilence);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_trailing_silence);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule2.min_utterance_length);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.must_contain_nonsilence);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_trailing_silence);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], endpoint_config.rule3.min_utterance_length);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], ctc_fst_decoder_config.graph);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], ctc_fst_decoder_config.max_active);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], enable_endpoint);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], decoding_method);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], max_active_paths);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_file);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_score);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], blank_penalty);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], temperature_scale);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], rule_fsts);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], rule_fars);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], reset_encoder);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hr.dict_dir);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hr.lexicon);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hr.rule_fsts);
ONNX_ONLINE_CONFIG_AUTO_SET(file_body["mode_param"], hotwords_buf);
if (config_body.contains("awake_delay"))
awake_delay_ = config_body["awake_delay"].get<int>();
else if (file_body["mode_param"].contains("awake_delay"))
awake_delay_ = file_body["mode_param"]["awake_delay"];
onnx_online_config.model_config.transducer.encoder =
base_model + onnx_online_config.model_config.transducer.encoder;
onnx_online_config.model_config.transducer.decoder =
base_model + onnx_online_config.model_config.transducer.decoder;
onnx_online_config.model_config.transducer.joiner =
base_model + onnx_online_config.model_config.transducer.joiner;
onnx_online_config.model_config.tokens = base_model + onnx_online_config.model_config.tokens;
onnx_online_recognizer_ = std::make_unique<sherpa_onnx::OnlineRecognizer>(onnx_online_config);
return 0;
}
int load_model(const nlohmann::json &config_body)
{
if (parse_config(config_body)) {
@@ -330,6 +456,8 @@ public:
if (engine_type_ == ENGINE_NCNN) {
return load_ncnn_model(config_body, file_body);
} else if (engine_type_ == ENGINE_ONLINE) {
return load_online_model(config_body, file_body);
} else {
return load_onnx_model(config_body, file_body);
}
@@ -451,10 +579,60 @@ public:
}
}
void sys_pcm_on_data_online(const std::string &raw)
{
static int count = 0;
if (count < delay_audio_frame_) {
buffer_write_char(pcmdata, raw.data(), raw.length());
count++;
return;
}
buffer_write_char(pcmdata, raw.data(), raw.length());
buffer_position_set(pcmdata, 0);
std::vector<float> floatSamples;
int16_t audio_val;
while (buffer_read_i16(pcmdata, &audio_val, 1)) {
float normalizedSample = static_cast<float>(audio_val) / INT16_MAX;
floatSamples.push_back(normalizedSample);
}
buffer_resize(pcmdata, 0);
count = 0;
if (!online_stream) online_stream = onnx_online_recognizer_->CreateStream();
online_stream->AcceptWaveform(onnx_online_config.feat_config.sampling_rate, floatSamples.data(),
floatSamples.size());
while (onnx_online_recognizer_->IsReady(online_stream.get())) {
onnx_online_recognizer_->DecodeStream(online_stream.get());
}
auto text = onnx_online_recognizer_->GetResult(online_stream.get()).text;
std::string lower_text;
lower_text.resize(text.size());
std::transform(text.begin(), text.end(), lower_text.begin(), [](auto c) { return std::tolower(c); });
if ((!lower_text.empty()) && out_callback_) out_callback_(lower_text, false);
bool is_endpoint = onnx_online_recognizer_->IsEndpoint(online_stream.get());
if (is_endpoint) {
if ((!lower_text.empty()) && out_callback_) {
out_callback_(lower_text, true);
}
online_stream.reset();
if (ensleep_) {
if (pause) pause();
}
}
}
void sys_pcm_on_data(const std::string &raw)
{
if (engine_type_ == ENGINE_NCNN) {
sys_pcm_on_data_ncnn(raw);
} else if (engine_type_ == ENGINE_ONLINE) {
sys_pcm_on_data_online(raw);
} else {
sys_pcm_on_data_onnx(raw);
}
projects/llm_framework/main_kws/mode_kws.json
+1 -1
View File
@@ -14,7 +14,7 @@
"kws.bool"
],
"mode_param": {
"model": "kws.onnx",
"model": "kws.axmodel",
"model_type": "onnx",
"wake_wav_file": "/opt/m5stack/data/audio/wakeup_zh_cn.wav",
"chunk_size": 32,