m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] add asr, kws model config

Browse Source
This commit is contained in:
LittleMouse
2025-12-19 11:59:50 +08:00
parent bea45abf1a
commit 3f608d2d60
11 changed files with 281 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/llm_framework/main_asr/mode_sense-voice-small-10s-ax650.json
+2 -2
View File
@@ -1,7 +1,7 @@
{
"mode": "sense-voice-small-10s-ax650",
"type": "asr",
"homepage": "https://huggingface.co/yunyu1258/qwen2.5-0.5b-ha",
"homepage": "https://huggingface.co/M5Stack/SenseVoiceSmall-axmodel",
"compile_flage": "pulsar2 build --input model-10-seconds.onnx --config config_sensevoice_main_u16.json --output_dir sensevoice-axmodel --output_name model-10-seconds.axmodel --target_hardware AX650 --compiler.check 0",
"pulsar_version": "5.0-patch1-fd447d0d",
"capabilities": [
@@ -21,7 +21,7 @@
"mode_param": {
"model_config.sense_voice.model": "model.axmodel",
"model_config.tokens": "tokens.txt",
"silero_vad.model": "silero_vad.onnx",
"silero_vad.model": "silero_vad.ort",
"model_config.provider": "axera",
"silence_timeout": 2000,
"awake_delay": 50
projects/llm_framework/main_asr/mode_sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-ax650.json
+70
View File
@@ -0,0 +1,70 @@
{
"mode": "sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-ax650",
"type": "asr",
"homepage": "https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t",
"capabilities": [
"Automatic_Speech_Recognition",
"Chinese",
"English"
],
"input_type": [
"sys.pcm",
"sys.cap.0_0"
],
"output_type": [
"asr.utf-8",
"asr.bool"
],
"mode_param": {
"model_config.transducer.encoder": "encoder.axmodel",
"model_config.transducer.decoder": "decoder.axmodel",
"model_config.transducer.joiner": "joiner.axmodel",
"model_config.tokens": "tokens.txt",
"feat_config.feature_dim": 80,
"feat_config.sampling_rate": 16000,
"endpoint_config.rule1.min_trailing_silence": 2.4,
"endpoint_config.rule2.min_trailing_silence": 1.2,
"endpoint_config.rule3.min_utterance_length": 30,
"enable_endpoint": true,
"awake_delay": 50,
"model_config.provider_config.provider": "axera",
"model_config.zipformer_meta.encoder_dims": [
256,
256,
256,
256,
256
],
"model_config.zipformer_meta.attention_dims": [
192,
192,
192,
192,
192
],
"model_config.zipformer_meta.num_encoder_layers": [
2,
2,
2,
2,
2
],
"model_config.zipformer_meta.cnn_module_kernels": [
31,
31,
31,
31,
31
],
"model_config.zipformer_meta.left_context_len": [
192,
96,
48,
24,
96
],
"model_config.zipformer_meta.T": 103,
"model_config.zipformer_meta.decode_chunk_len": 96,
"model_config.zipformer_meta.context_size": 2
}
}
projects/llm_framework/main_asr/mode_sherpa-onnx-zipformer-bilingual-zh-en-t.json → projects/llm_framework/main_asr/mode_sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16.json
+2 -2
View File
@@ -1,7 +1,7 @@
{
"mode": "sherpa-onnx-zipformer-bilingual-zh-en-t",
"mode": "sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16",
"type": "asr",
"homepage": "",
"homepage": "https://huggingface.co/csukuangfj/k2fsa-zipformer-bilingual-zh-en-t",
"capabilities": [
"Automatic_Speech_Recognition",
"Chinese",
projects/llm_framework/main_asr/src/main.cpp
+7 -6
View File
@@ -73,6 +73,7 @@ private:
sherpa_onnx::OnlineRecognizerConfig onnx_online_config;
sherpa_onnx::VadModelConfig vad_config_;
std::unique_ptr<sherpa_onnx::OfflineStream> offline_stream_;
std::unique_ptr<sherpa_onnx::OfflineRecognizer> onnx_recognizer_;
std::unique_ptr<sherpa_onnx::OnlineRecognizer> onnx_online_recognizer_;
std::unique_ptr<sherpa_onnx::OnlineStream> online_stream;
@@ -549,16 +550,16 @@ public:
vad_->AcceptWaveform(floatSamples.data(), floatSamples.size());
while (!vad_->Empty()) {
const auto &segment = vad_->Front();
auto s = onnx_recognizer_->CreateStream();
s->AcceptWaveform(onnx_asr_config_.feat_config.sampling_rate, segment.samples.data(),
segment.samples.size());
onnx_recognizer_->DecodeStream(s.get());
const auto &result = s->GetResult();
if (!offline_stream_) offline_stream_ = onnx_recognizer_->CreateStream();
offline_stream_->AcceptWaveform(onnx_asr_config_.feat_config.sampling_rate, segment.samples.data(),
segment.samples.size());
onnx_recognizer_->DecodeStream(offline_stream_.get());
const auto &result = offline_stream_->GetResult();
if (!result.text.empty() && out_callback_) {
SLOGI("onnx-asr result: %s", result.text.c_str());
out_callback_(result.text, true);
}
vad_->Pop();
offline_stream_.reset();
}
{
projects/llm_framework/main_audio/SConstruct
+1 -1
View File
@@ -29,7 +29,7 @@ REQUIREMENTS += ['ax_sys', 'ax_audio', 'ax_audio_3a', 'tinyalsa', 'fdk-aac', 'op
if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ:
REQUIREMENTS += ['ax_interpreter', 'ax_fdk', 'ax_opus']
STATIC_FILES += [AFile('audio.json'), AFile('audio_kit.json')]
STATIC_FILES += [AFile('audio.json'), AFile('audio_kit.json'), AFile('audio_pyramid.json')]
STATIC_FILES += Glob('mode_*.json')
env['COMPONENTS'].append({'target':'llm_audio-1.7',
projects/llm_framework/main_audio/audio_pyramid.json
+115
View File
@@ -0,0 +1,115 @@
{
"mode": "None",
"type": "audio",
"capabilities": [
"play",
"cap"
],
"input_type": [
"rpc.audio.wav.base64",
"rpc.audio.pcm.base64"
],
"output_type": [
"audio.pcm.stream"
],
"play_param": {
"card": 0,
"device": 0,
"volume": 0.5,
"channel": 2,
"rate": 48000,
"bit": 16,
"stPoolConfig.MetaSize": 8192,
"stPoolConfig.BlkSize": 32768,
"stPoolConfig.BlkCnt": 37,
"stPoolConfig.IsMergeMode": 0,
"stPoolConfig.CacheMode": 0,
"stPoolConfig.PartitionName": "anonymous",
"stAttr.enBitwidth": 1,
"stAttr.enSoundmode": 0,
"stAttr.u32ChnCnt": 2,
"stAttr.enLinkMode": 0,
"stAttr.enSamplerate": 16000,
"stAttr.U32Depth": 30,
"stAttr.u32PeriodSize": 160,
"stAttr.u32PeriodCount": 8,
"stAttr.bInsertSilence": 0,
"stVqeAttr.s32SampleRate": 16000,
"stVqeAttr.u32FrameSamples": 160,
"stVqeAttr.stNsCfg.bNsEnable": 0,
"stVqeAttr.stNsCfg.enAggressivenessLevel": 2,
"stVqeAttr.stAgcCfg.bAgcEnable": 0,
"stVqeAttr.stAgcCfg.enAgcMode": 2,
"stVqeAttr.stAgcCfg.s16TargetLevel": -3,
"stVqeAttr.stAgcCfg.s16Gain": 9,
"stHpfAttr.bEnable": 0,
"stHpfAttr.s32GainDb": -3,
"stHpfAttr.s32Samplerate": 16000,
"stHpfAttr.s32Freq": 200,
"stLpfAttr.bEnable": 0,
"stLpfAttr.s32GainDb": 0,
"stLpfAttr.s32Samplerate": 16000,
"stLpfAttr.s32Freq": 3000,
"stEqAttr.bEnable": 0,
"stEqAttr.s32GainDb[0]": -10,
"stEqAttr.s32GainDb[1]": -3,
"stEqAttr.s32GainDb[2]": 3,
"stEqAttr.s32GainDb[3]": 5,
"stEqAttr.s32GainDb[4]": 10,
"stEqAttr.s32Samplerate": 16000,
"gResample": 0,
"enInSampleRate": 16000,
"gInstant": 0,
"gInsertSilence": 0
},
"cap_param": {
"sys_pcm_cap_channel": "ipc:///tmp/llm/pcm.cap.socket",
"card": 0,
"device": 1,
"volume": 1.0,
"channel": 4,
"rate": 16000,
"bit": 16,
"stPoolConfig.MetaSize": 8192,
"stPoolConfig.BlkSize": 7680,
"stPoolConfig.BlkCnt": 33,
"stPoolConfig.IsMergeMode": 0,
"stPoolConfig.CacheMode": 0,
"stPoolConfig.PartitionName": "anonymous",
"aistAttr.enBitwidth": 1,
"aistAttr.enLinkMode": 0,
"aistAttr.enSamplerate": 16000,
"aistAttr.enLayoutMode": 2,
"aistAttr.U32Depth": 30,
"aistAttr.u32PeriodSize": 160,
"aistAttr.u32PeriodCount": 8,
"aistAttr.u32ChnCnt": 2,
"aistVqeAttr.s32SampleRate": 16000,
"aistVqeAttr.u32FrameSamples": 160,
"aistVqeAttr.stNsCfg.bNsEnable": 1,
"aistVqeAttr.stNsCfg.enAggressivenessLevel": 2,
"aistVqeAttr.stAgcCfg.bAgcEnable": 0,
"aistVqeAttr.stAgcCfg.enAgcMode": 2,
"aistVqeAttr.stAgcCfg.s16TargetLevel": -3,
"aistVqeAttr.stAgcCfg.s16Gain": 9,
"aistVqeAttr.stAecCfg.enAecMode": 2,
"stHpfAttr.bEnable": 0,
"stHpfAttr.s32GainDb": -3,
"stHpfAttr.s32Samplerate": 16000,
"stHpfAttr.s32Freq": 200,
"stLpfAttr.bEnable": 0,
"stLpfAttr.s32GainDb": 0,
"stLpfAttr.s32Samplerate": 16000,
"stLpfAttr.s32Freq": 3000,
"stEqAttr.bEnable": 0,
"stEqAttr.s32GainDb[0]": -10,
"stEqAttr.s32GainDb[1]": -3,
"stEqAttr.s32GainDb[2]": 3,
"stEqAttr.s32GainDb[3]": 5,
"stEqAttr.s32GainDb[4]": 10,
"stEqAttr.s32Samplerate": 16000,
"gResample": 0,
"enOutSampleRate": 16000,
"gDbDetection": 0
}
}
projects/llm_framework/main_audio/src/main.cpp
+1 -1
View File
@@ -209,7 +209,7 @@ public:
}
#else
std::list<std::string> config_file_paths =
get_config_file_paths(base_model_path, base_model_config_path, "audio");
get_config_file_paths(base_model_path, base_model_config_path, "audio_pyramid");
#endif
try {
config_body = nlohmann::json::parse(data);
projects/llm_framework/main_kws/mode_kws-ax650.json
+41
View File
@@ -0,0 +1,41 @@
{
"mode": "kws-ax650",
"type": "kws",
"homepage": "",
"capabilities": [
"Keyword_spotting",
"English"
],
"input_type": [
"sys.pcm",
"sys.cap.0_0"
],
"output_type": [
"kws.bool"
],
"mode_param": {
"model": "kws.axmodel",
"model_type": "axera",
"wake_wav_file": "/opt/m5stack/data/audio/wakeup_zh_cn.wav",
"chunk_size": 32,
"threshold": 0.9,
"min_continuous_frames": 5,
"REFRACTORY_TIME_MS": 2000,
"RESAMPLE_RATE": 16000,
"FEAT_DIM": 80,
"frame_opts.samp_freq": 16000,
"frame_opts.frame_length_ms": 25.0,
"frame_opts.frame_shift_ms": 10.0,
"frame_opts.snip_edges": false,
"frame_opts.dither": 0.0,
"frame_opts.preemph_coeff": 0.97,
"frame_opts.remove_dc_offset": true,
"frame_opts.window_type": "povey",
"mel_opts.num_bins": 80,
"mel_opts.low_freq": 20,
"mel_opts.high_freq": 0,
"energy_floor": 0.0,
"use_energy": false,
"raw_energy": true
}
}
projects/llm_framework/main_kws/mode_kws.json
+2 -3
View File
@@ -15,7 +15,7 @@
],
"mode_param": {
"model": "kws.axmodel",
"model_type": "onnx",
"model_type": "axera",
"wake_wav_file": "/opt/m5stack/data/audio/wakeup_zh_cn.wav",
"chunk_size": 32,
"threshold": 0.9,
@@ -37,6 +37,5 @@
"energy_floor": 0.0,
"use_energy": false,
"raw_energy": true
},
"mode_param_bak": {}
}
}
projects/llm_framework/main_kws/src/main.cpp
+28 -29
View File
@@ -39,7 +39,7 @@ typedef std::function<void(const std::string &data, bool finish)> task_callback_
#include "sherpa-onnx/csrc/keyword-spotter.h"
#include "kaldi-native-fbank/csrc/online-feature.h"
typedef struct mode_config_onnx {
typedef struct mode_config_axera {
int chunk_size = 32;
float threshold = 0.9f;
int min_continuous_frames = 5;
@@ -47,7 +47,7 @@ typedef struct mode_config_onnx {
int RESAMPLE_RATE = 16000;
int FEAT_DIM = 80;
int delay_audio_frame_ = 32;
} kws_config_onnx;
} kws_config_axera;
class llm_task {
private:
@@ -72,12 +72,11 @@ private:
std::unique_ptr<sherpa_onnx::KeywordSpotter> sherpa_spotter_;
std::unique_ptr<sherpa_onnx::OnlineStream> sherpa_stream_;
kws_config_onnx onnx_config_;
kws_config_axera axera_config_;
std::vector<float> axera_cache_;
std::unique_ptr<EngineWrapper> axera_session_;
knf::FbankOptions fbank_opts_;
std::unique_ptr<knf::OnlineFbank> fbank_;
Ort::Env onnx_env_{ORT_LOGGING_LEVEL_WARNING, "kws"};
Ort::SessionOptions session_options_;
int count_frames_ = 0;
long long last_trigger_time_ms_ = -1e9;
@@ -117,7 +116,7 @@ public:
if (model_.rfind("sherpa-onnx", 0) == 0) {
model_type_ = "sherpa";
} else {
model_type_ = "onnx";
model_type_ = "axera";
}
if (config_body.contains("enwake_audio")) {
@@ -294,11 +293,11 @@ public:
}
#undef CONFIG_AUTO_SET_SHERPA
#define CONFIG_AUTO_SET_ONNX(obj, key) \
#define CONFIG_AUTO_SET_AXERA(obj, key) \
if (config_body.contains(#key)) \
onnx_config_.key = config_body[#key]; \
axera_config_.key = config_body[#key]; \
else if (obj.contains(#key)) \
onnx_config_.key = obj[#key];
axera_config_.key = obj[#key];
#define OPTS_AUTO_SET(obj, key) \
if (config_body.contains(#key)) \
@@ -306,7 +305,7 @@ public:
else if (obj.contains(#key)) \
fbank_opts_.key = obj[#key];
int load_model_onnx(const nlohmann::json &config_body)
int load_model_axera(const nlohmann::json &config_body)
{
nlohmann::json file_body;
std::list<std::string> config_file_paths =
@@ -345,13 +344,13 @@ public:
axera_cache_.assign(1 * 32 * 88, 0.0f);
auto &mp = file_body["mode_param"];
CONFIG_AUTO_SET_ONNX(mp, chunk_size);
CONFIG_AUTO_SET_ONNX(mp, threshold);
CONFIG_AUTO_SET_ONNX(mp, min_continuous_frames);
CONFIG_AUTO_SET_ONNX(mp, REFRACTORY_TIME_MS);
CONFIG_AUTO_SET_ONNX(mp, RESAMPLE_RATE);
CONFIG_AUTO_SET_ONNX(mp, FEAT_DIM);
CONFIG_AUTO_SET_ONNX(mp, delay_audio_frame_);
CONFIG_AUTO_SET_AXERA(mp, chunk_size);
CONFIG_AUTO_SET_AXERA(mp, threshold);
CONFIG_AUTO_SET_AXERA(mp, min_continuous_frames);
CONFIG_AUTO_SET_AXERA(mp, REFRACTORY_TIME_MS);
CONFIG_AUTO_SET_AXERA(mp, RESAMPLE_RATE);
CONFIG_AUTO_SET_AXERA(mp, FEAT_DIM);
CONFIG_AUTO_SET_AXERA(mp, delay_audio_frame_);
OPTS_AUTO_SET(mp, frame_opts.samp_freq);
OPTS_AUTO_SET(mp, frame_opts.frame_length_ms);
@@ -373,7 +372,7 @@ public:
SLOGE("config file read false");
return -3;
}
delay_audio_frame_ = onnx_config_.delay_audio_frame_;
delay_audio_frame_ = axera_config_.delay_audio_frame_;
return 0;
}
#undef CONFIG_AUTO_SET_ONNX
@@ -383,11 +382,11 @@ public:
{
bool triggered = false;
for (auto score : scores) {
if (score > onnx_config_.threshold) {
if (score > axera_config_.threshold) {
count_frames_++;
if (count_frames_ >= onnx_config_.min_continuous_frames) {
long long trigger_time_ms = (frame_index_global_ - onnx_config_.min_continuous_frames + 1) * 10;
if (trigger_time_ms - last_trigger_time_ms_ >= onnx_config_.REFRACTORY_TIME_MS) {
if (count_frames_ >= axera_config_.min_continuous_frames) {
long long trigger_time_ms = (frame_index_global_ - axera_config_.min_continuous_frames + 1) * 10;
if (trigger_time_ms - last_trigger_time_ms_ >= axera_config_.REFRACTORY_TIME_MS) {
last_trigger_time_ms_ = trigger_time_ms;
triggered = true;
}
@@ -420,11 +419,11 @@ public:
std::vector<float> run_inference(const std::vector<float> &audio_chunk_16k)
{
std::vector<std::vector<float>> fbank_feats =
compute_fbank_kaldi(audio_chunk_16k, onnx_config_.RESAMPLE_RATE, onnx_config_.FEAT_DIM);
compute_fbank_kaldi(audio_chunk_16k, axera_config_.RESAMPLE_RATE, axera_config_.FEAT_DIM);
if (fbank_feats.empty()) return {};
constexpr int FIX_T = 32;
const int FEAT_DIM = onnx_config_.FEAT_DIM;
const int FEAT_DIM = axera_config_.FEAT_DIM;
std::vector<float> mat_flattened;
mat_flattened.resize(FIX_T * FEAT_DIM, 0.0f);
@@ -467,11 +466,11 @@ public:
if (parse_config(config_body)) {
return -1;
}
if (model_type_ == "onnx") {
SLOGE("load onnx kws model");
return load_model_onnx(config_body);
if (model_type_ == "axera") {
SLOGI("load axera kws model");
return load_model_axera(config_body);
} else {
SLOGE("load sherpa kws model");
SLOGI("load sherpa kws model");
return load_model_sherpa(config_body);
}
}
@@ -498,7 +497,7 @@ public:
int16_t audio_val;
while (buffer_read_i16(pcmdata, &audio_val, 1)) {
int16Samples.push_back(audio_val);
if (model_type_ == "onnx") {
if (model_type_ == "axera") {
floatSamples.push_back(static_cast<float>(audio_val) / 1.0f);
} else {
floatSamples.push_back(static_cast<float>(audio_val) / INT16_MAX);
@@ -507,7 +506,7 @@ public:
buffer_resize(pcmdata, 0);
count = 0;
if (model_type_ == "onnx") {
if (model_type_ == "axera") {
auto scores = run_inference(floatSamples);
if (detect_wakeup(scores)) {
if (enwake_audio_ && (!wake_wav_file_.empty()) && play_awake_wav) {
projects/llm_framework/tools/llm_pack.py
+12
View File
@@ -397,6 +397,18 @@ if __name__ == "__main__":
'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23', data_version, src_folder, revision],
'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01', '0.3', src_folder, revision],
'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01', '0.3', src_folder, revision],
'llm-model-sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16':[create_data_deb,'llm-model-sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16', '0.4', src_folder, revision],
'llm-model-sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-ax650':[create_data_deb,'llm-model-sherpa-onnx-streaming-zipformer-small-bilingual-zh-en-2023-02-16-ax650', '0.4', src_folder, revision],
# KWS model
## AX630C
'llm-model-kws':[create_data_deb,'llm-model-kws', '0.4', src_folder, revision],
## AX650
'llm-model-kws-ax650':[create_data_deb,'llm-model-kws-ax650', '0.4', src_folder, revision],
# ASR model
## AX630C
'llm-model-sense-voice-small-10s':[create_data_deb,'llm-model-sense-voice-small-10s', '0.4', src_folder, revision],
## AX650
'llm-model-sense-voice-small-10s-ax650':[create_data_deb,'llm-model-sense-voice-small-10s-ax650', '0.4', src_folder, revision],
# TTS model
'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', '0.3', src_folder, revision],
'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', '0.3', src_folder, revision],