m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] add vad model.update sherpa-onnx head file.

Browse Source
This commit is contained in:
LittleMouse
2024-12-17 10:13:54 +08:00
parent 13c41021d6
commit 2452f8bd2c
9 changed files with 542 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/llm_framework/README.md
+1 -1
View File
@@ -218,7 +218,7 @@ send :
"action":"setup",
"object":"yolo.setup",
"data":{
"model":"yolo11n_anquanmao",
"model":"yolo11n",
"response_format":"yolo.yolobox",
"input":"camera.1000",
"enoutput":true
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model-config.h
+5
View File
@@ -27,6 +27,11 @@ struct SileroVadModelConfig {
// 256, 512, 768 samples for 800 Hz
int32_t window_size = 512; // in samples
// If a speech segment is longer than this value, then we increase
// the threshold to 0.9. After finishing detecting the segment,
// the threshold value is reset to its original value.
float max_speech_duration = 20; // in seconds
SileroVadModelConfig() = default;
void Register(ParseOptions *po);
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model.h
+2 -8
View File
@@ -6,11 +6,6 @@
#include <memory>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "sherpa-onnx/csrc/vad-model.h"
namespace sherpa_onnx {
@@ -19,9 +14,8 @@ class SileroVadModel : public VadModel {
public:
explicit SileroVadModel(const VadModelConfig &config);
#if __ANDROID_API__ >= 9
SileroVadModel(AAssetManager *mgr, const VadModelConfig &config);
#endif
template <typename Manager>
SileroVadModel(Manager *mgr, const VadModelConfig &config);
~SileroVadModel() override;
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/vad-model.h
+2 -8
View File
@@ -6,11 +6,6 @@
#include <memory>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "sherpa-onnx/csrc/vad-model-config.h"
namespace sherpa_onnx {
@@ -21,10 +16,9 @@ class VadModel {
static std::unique_ptr<VadModel> Create(const VadModelConfig &config);
#if __ANDROID_API__ >= 9
static std::unique_ptr<VadModel> Create(AAssetManager *mgr,
template <typename Manager>
static std::unique_ptr<VadModel> Create(Manager *mgr,
const VadModelConfig &config);
#endif
// reset the internal model states
virtual void Reset() = 0;
projects/llm_framework/include/sherpa/sherpa-onnx/csrc/voice-activity-detector.h
+2 -8
View File
@@ -7,11 +7,6 @@
#include <memory>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "sherpa-onnx/csrc/vad-model-config.h"
namespace sherpa_onnx {
@@ -26,10 +21,9 @@ class VoiceActivityDetector {
explicit VoiceActivityDetector(const VadModelConfig &config,
float buffer_size_in_seconds = 60);
#if __ANDROID_API__ >= 9
VoiceActivityDetector(AAssetManager *mgr, const VadModelConfig &config,
template <typename Manager>
VoiceActivityDetector(Manager *mgr, const VadModelConfig &config,
float buffer_size_in_seconds = 60);
#endif
~VoiceActivityDetector();
projects/llm_framework/main_vad/Kconfig
View File
projects/llm_framework/main_vad/SConstruct
+44
View File
@@ -0,0 +1,44 @@
import os
Import('env')
with open(env['PROJECT_TOOL_S']) as f:
exec(f.read())
SRCS = Glob('src/*.c*')
INCLUDE = [ADir('include'), ADir('.')]
PRIVATE_INCLUDE = []
REQUIREMENTS = ['pthread', 'dl', 'utilities', 'eventpp', 'StackFlow', 'single_header_libs']
STATIC_LIB = []
DYNAMIC_LIB = []
DEFINITIONS = []
DEFINITIONS_PRIVATE = []
LDFLAGS = []
LINK_SEARCH_PATH = []
STATIC_FILES = []
DEFINITIONS += ['-std=c++17']
LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
LINK_SEARCH_PATH += [ADir('../static_lib')]
INCLUDE += [ADir('../include/sherpa')]
LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')]
LDFLAGS += ['-l:libsherpa-onnx-core.a',
'-l:libonnxruntime.a']
STATIC_FILES += Glob('mode_*.json')
env['COMPONENTS'].append({'target':'llm_vad',
'SRCS':SRCS,
'INCLUDE':INCLUDE,
'PRIVATE_INCLUDE':PRIVATE_INCLUDE,
'REQUIREMENTS':REQUIREMENTS,
'STATIC_LIB':STATIC_LIB,
'DYNAMIC_LIB':DYNAMIC_LIB,
'DEFINITIONS':DEFINITIONS,
'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE,
'LDFLAGS':LDFLAGS,
'LINK_SEARCH_PATH':LINK_SEARCH_PATH,
'STATIC_FILES':STATIC_FILES,
'REGISTER':'project'
})
projects/llm_framework/main_vad/mode_silero-vad-model.json
+26
View File
@@ -0,0 +1,26 @@
{
"mode": "silero_vad",
"type": "vad",
"capabilities": [
"Voice_activity_detection"
],
"input_type": [
"sys.pcm",
"sys.cap.0_0"
],
"output_type": [
"vad.bool"
],
"mode_param": {
"silero_vad.model": "silero_vad.onnx"
},
"mode_param_bak": {
"silero_vad.threshold": 0.5,
"silero_vad.min_silence_duration": 0.5,
"silero_vad.min_speech_duration": 0.25,
"silero_vad.window_size": 512,
"sample_rate": 16000,
"num_threads": 1,
"provider": "cpu"
}
}
projects/llm_framework/main_vad/src/main.cpp
+460
View File
@@ -0,0 +1,460 @@
/*
* SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
*
* SPDX-License-Identifier: MIT
*/
#include "StackFlow.h"
#include "sherpa-onnx/csrc/voice-activity-detector.h"
#include <signal.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <fstream>
#include <thread_safe_list.h>
#include "../../../../SDK/components/utilities/include/sample_log.h"
#define BUFFER_IMPLEMENTATION
#include <stdint.h>
#include "libs/buffer.h"
#include <iostream>
using namespace StackFlows;
int main_exit_flage = 0;
static void __sigint(int iSigNo)
{
SLOGW("llm_vad will be exit!");
main_exit_flage = 1;
}
static std::string base_model_path_;
static std::string base_model_config_path_;
#define CONFIG_AUTO_SET(obj, key) \
if (config_body.contains(#key)) \
mode_config_.key = config_body[#key]; \
else if (obj.contains(#key)) \
mode_config_.key = obj[#key];
class llm_task {
private:
sherpa_onnx::VadModelConfig mode_config_;
std::unique_ptr<sherpa_onnx::VoiceActivityDetector> vad_;
public:
std::string model_;
std::string response_format_;
std::vector<std::string> inputs_;
bool enoutput_;
bool enstream_;
bool printed = false;
std::atomic_bool audio_flage_;
int delay_audio_frame_ = 100;
buffer_t *pcmdata;
std::string wake_wav_file_;
std::function<void(const std::string &)> out_callback_;
bool parse_config(const nlohmann::json &config_body)
{
fprintf(stderr, "%s\n", mode_config_.ToString().c_str());
try {
model_ = config_body.at("model");
response_format_ = config_body.at("response_format");
enoutput_ = config_body.at("enoutput");
if (config_body.contains("input")) {
if (config_body["input"].is_string()) {
inputs_.push_back(config_body["input"].get<std::string>());
} else if (config_body["input"].is_array()) {
for (auto _in : config_body["input"]) {
inputs_.push_back(_in.get<std::string>());
}
}
}
} catch (...) {
SLOGE("setup config_body error");
return true;
}
enstream_ = response_format_.find("stream") == std::string::npos ? false : true;
return false;
}
int load_model(const nlohmann::json &config_body)
{
if (parse_config(config_body)) {
return -1;
}
nlohmann::json file_body;
std::list<std::string> config_file_paths =
get_config_file_paths(base_model_path_, base_model_config_path_, model_);
try {
for (auto file_name : config_file_paths) {
std::ifstream config_file(file_name);
if (!config_file.is_open()) {
SLOGW("config file :%s miss", file_name.c_str());
continue;
}
config_file >> file_body;
config_file.close();
break;
}
if (file_body.empty()) {
SLOGE("all config file miss");
return -2;
}
std::string base_model = base_model_path_ + model_ + "/";
SLOGI("base_model %s", base_model.c_str());
CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.model);
CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.threshold);
CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.min_silence_duration);
CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.min_speech_duration);
CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.window_size);
CONFIG_AUTO_SET(file_body["mode_param"], sample_rate);
CONFIG_AUTO_SET(file_body["mode_param"], num_threads);
CONFIG_AUTO_SET(file_body["mode_param"], provider);
if (config_body.contains("wake_wav_file"))
wake_wav_file_ = config_body["wake_wav_file"];
else if (file_body["mode_param"].contains("wake_wav_file"))
wake_wav_file_ = file_body["mode_param"]["wake_wav_file"];
mode_config_.silero_vad.model = base_model + mode_config_.silero_vad.model;
if (!mode_config_.Validate()) {
fprintf(stderr, "Errors in config!\n");
return -1;
}
vad_ = std::make_unique<sherpa_onnx::VoiceActivityDetector>(mode_config_);
} catch (...) {
SLOGE("config file read false");
return -3;
}
return 0;
}
void set_output(std::function<void(const std::string &)> out_callback)
{
out_callback_ = out_callback;
}
void sys_pcm_on_data(const std::string &raw)
{
static int count = 0;
int32_t k = 0;
if (count < delay_audio_frame_) {
buffer_write_char(pcmdata, raw.c_str(), raw.length());
count++;
return;
}
buffer_write_char(pcmdata, raw.c_str(), raw.length());
buffer_position_set(pcmdata, 0);
count = 0;
std::vector<float> floatSamples;
{
int16_t audio_val;
while (buffer_read_u16(pcmdata, (unsigned short *)&audio_val, 1)) {
float normalizedSample = (float)audio_val / INT16_MAX;
floatSamples.push_back(normalizedSample);
}
}
buffer_position_set(pcmdata, 0);
vad_->AcceptWaveform(floatSamples.data(), floatSamples.size());
if (vad_->IsSpeechDetected() && !printed) {
printed = true;
SLOGI("Detected speech!");
}
if (!vad_->IsSpeechDetected()) {
printed = false;
}
int32_t sample_rate = 16000;
while (!vad_->Empty()) {
const auto &segment = vad_->Front();
float duration = segment.samples.size() / static_cast<float>(sample_rate);
SLOGI("Duration: %.3f seconds", duration);
k += 1;
vad_->Pop();
}
}
bool delete_model()
{
vad_.reset();
return true;
}
llm_task(const std::string &workid) : audio_flage_(false)
{
pcmdata = buffer_create();
}
~llm_task()
{
if (vad_) {
vad_.reset();
}
buffer_destroy(pcmdata);
}
};
#undef CONFIG_AUTO_SET
class llm_kws : public StackFlow {
private:
int task_count_;
std::string audio_url_;
std::unordered_map<int, std::shared_ptr<llm_task>> llm_task_;
public:
llm_kws() : StackFlow("vad")
{
task_count_ = 1;
}
void task_pause(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak)
{
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
return;
}
if (llm_task_obj->audio_flage_) {
if (!audio_url_.empty()) llm_channel->stop_subscriber(audio_url_);
llm_task_obj->audio_flage_ = false;
}
}
void task_work(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak)
{
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
return;
}
if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) {
std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
_llm_task_obj.lock()->sys_pcm_on_data(raw);
});
llm_task_obj->audio_flage_ = true;
}
}
void work(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_asr::work:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
task_work(llm_task_[work_id_num], get_channel(work_id_num));
send("None", "None", LLM_NO_ERROR, work_id);
}
void pause(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_asr::work:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
task_pause(llm_task_[work_id_num], get_channel(work_id_num));
send("None", "None", LLM_NO_ERROR, work_id);
}
void task_user_data(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak, const std::string &object,
const std::string &data)
{
nlohmann::json error_body;
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
error_body["code"] = -11;
error_body["message"] = "Model run failed.";
send("None", "None", error_body, unit_name_);
return;
}
std::string tmp_msg1;
const std::string *next_data = &data;
int ret;
if (object.find("stream") != std::string::npos) {
static std::unordered_map<int, std::string> stream_buff;
try {
if (decode_stream(data, tmp_msg1, stream_buff)) {
return;
};
} catch (...) {
stream_buff.clear();
error_body["code"] = -25;
error_body["message"] = "Stream data index error.";
send("None", "None", error_body, unit_name_);
return;
}
next_data = &tmp_msg1;
}
std::string tmp_msg2;
if (object.find("base64") != std::string::npos) {
ret = decode_base64((*next_data), tmp_msg2);
if (ret == -1) {
error_body["code"] = -23;
error_body["message"] = "Base64 decoding error.";
send("None", "None", error_body, unit_name_);
return;
}
next_data = &tmp_msg2;
}
llm_task_obj->sys_pcm_on_data((*next_data));
}
int setup(const std::string &work_id, const std::string &object, const std::string &data) override
{
nlohmann::json error_body;
if ((llm_task_channel_.size() - 1) == task_count_) {
error_body["code"] = -21;
error_body["message"] = "task full";
send("None", "None", error_body, "vad");
return -1;
}
int work_id_num = sample_get_work_id_num(work_id);
auto llm_channel = get_channel(work_id);
auto llm_task_obj = std::make_shared<llm_task>(work_id);
nlohmann::json config_body;
try {
config_body = nlohmann::json::parse(data);
} catch (...) {
SLOGE("setup json format error.");
error_body["code"] = -2;
error_body["message"] = "json format error.";
send("None", "None", error_body, "vad");
return -2;
}
int ret = llm_task_obj->load_model(config_body);
if (ret == 0) {
llm_channel->set_output(llm_task_obj->enoutput_);
llm_channel->set_stream(llm_task_obj->enstream_);
llm_task_obj->set_output([llm_task_obj, llm_channel](const std::string &data) {
llm_channel->send(llm_task_obj->response_format_, true, LLM_NO_ERROR);
});
for (const auto input : llm_task_obj->inputs_) {
if (input.find("sys") != std::string::npos) {
audio_url_ = unit_call("audio", "cap", "None");
std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
_llm_task_obj.lock()->sys_pcm_on_data(raw);
});
llm_task_obj->audio_flage_ = true;
} else if (input.find("vad") != std::string::npos) {
llm_channel->subscriber_work_id(
"", std::bind(&llm_kws::task_user_data, this, std::weak_ptr<llm_task>(llm_task_obj),
std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
std::placeholders::_2));
}
}
llm_task_[work_id_num] = llm_task_obj;
SLOGI("load_mode success");
send("None", "None", LLM_NO_ERROR, work_id);
return 0;
} else {
SLOGE("load_mode Failed");
error_body["code"] = -5;
error_body["message"] = "Model loading failed.";
send("None", "None", error_body, "vad");
return -1;
}
}
void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_vad::taskinfo:%s", data.c_str());
nlohmann::json req_body;
int work_id_num = sample_get_work_id_num(work_id);
if (WORK_ID_NONE == work_id_num) {
std::vector<std::string> task_list;
std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list),
[](const auto task_channel) { return task_channel.second->work_id_; });
req_body = task_list;
send("vad.tasklist", req_body, LLM_NO_ERROR, work_id);
} else {
if (llm_task_.find(work_id_num) == llm_task_.end()) {
req_body["code"] = -6;
req_body["message"] = "Unit Does Not Exist";
send("None", "None", req_body, work_id);
return;
}
auto llm_task_obj = llm_task_[work_id_num];
req_body["model"] = llm_task_obj->model_;
req_body["response_format"] = llm_task_obj->response_format_;
req_body["enoutput"] = llm_task_obj->enoutput_;
req_body["inputs"] = llm_task_obj->inputs_;
send("vad.taskinfo", req_body, LLM_NO_ERROR, work_id);
}
}
int exit(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_kws::exit:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return -1;
}
auto llm_channel = get_channel(work_id_num);
llm_channel->stop_subscriber("");
if (llm_task_[work_id_num]->audio_flage_) {
unit_call("audio", "cap_stop", "None");
}
llm_task_.erase(work_id_num);
send("None", "None", LLM_NO_ERROR, work_id);
return 0;
}
~llm_kws()
{
while (1) {
auto iteam = llm_task_.begin();
if (iteam == llm_task_.end()) {
break;
}
if (iteam->second->audio_flage_) {
unit_call("audio", "cap_stop", "None");
}
get_channel(iteam->first)->stop_subscriber("");
iteam->second.reset();
llm_task_.erase(iteam->first);
}
}
};
int main(int argc, char *argv[])
{
signal(SIGTERM, __sigint);
signal(SIGINT, __sigint);
mkdir("/tmp/llm", 0777);
llm_kws llm;
while (!main_exit_flage) {
sleep(1);
}
return 0;
}