diff --git a/projects/llm_framework/README.md b/projects/llm_framework/README.md index f59c904..8fa6e29 100644 --- a/projects/llm_framework/README.md +++ b/projects/llm_framework/README.md @@ -218,7 +218,7 @@ send : "action":"setup", "object":"yolo.setup", "data":{ - "model":"yolo11n_anquanmao", + "model":"yolo11n", "response_format":"yolo.yolobox", "input":"camera.1000", "enoutput":true diff --git a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model-config.h b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model-config.h index b9679dd..5ae06f5 100644 --- a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model-config.h +++ b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model-config.h @@ -27,6 +27,11 @@ struct SileroVadModelConfig { // 256, 512, 768 samples for 800 Hz int32_t window_size = 512; // in samples + // If a speech segment is longer than this value, then we increase + // the threshold to 0.9. After finishing detecting the segment, + // the threshold value is reset to its original value. + float max_speech_duration = 20; // in seconds + SileroVadModelConfig() = default; void Register(ParseOptions *po); diff --git a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model.h b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model.h index 169cb72..8391399 100644 --- a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model.h +++ b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/silero-vad-model.h @@ -6,11 +6,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/vad-model.h" namespace sherpa_onnx { @@ -19,9 +14,8 @@ class SileroVadModel : public VadModel { public: explicit SileroVadModel(const VadModelConfig &config); -#if __ANDROID_API__ >= 9 - SileroVadModel(AAssetManager *mgr, const VadModelConfig &config); -#endif + template + SileroVadModel(Manager *mgr, const VadModelConfig &config); ~SileroVadModel() override; diff --git a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/vad-model.h b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/vad-model.h index 81028f2..aac68bb 100644 --- a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/vad-model.h +++ b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/vad-model.h @@ -6,11 +6,6 @@ #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/vad-model-config.h" namespace sherpa_onnx { @@ -21,10 +16,9 @@ class VadModel { static std::unique_ptr Create(const VadModelConfig &config); -#if __ANDROID_API__ >= 9 - static std::unique_ptr Create(AAssetManager *mgr, + template + static std::unique_ptr Create(Manager *mgr, const VadModelConfig &config); -#endif // reset the internal model states virtual void Reset() = 0; diff --git a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/voice-activity-detector.h b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/voice-activity-detector.h index 9eb53c5..015cedf 100644 --- a/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/voice-activity-detector.h +++ b/projects/llm_framework/include/sherpa/sherpa-onnx/csrc/voice-activity-detector.h @@ -7,11 +7,6 @@ #include #include -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - #include "sherpa-onnx/csrc/vad-model-config.h" namespace sherpa_onnx { @@ -26,10 +21,9 @@ class VoiceActivityDetector { explicit VoiceActivityDetector(const VadModelConfig &config, float buffer_size_in_seconds = 60); -#if __ANDROID_API__ >= 9 - VoiceActivityDetector(AAssetManager *mgr, const VadModelConfig &config, + template + VoiceActivityDetector(Manager *mgr, const VadModelConfig &config, float buffer_size_in_seconds = 60); -#endif ~VoiceActivityDetector(); diff --git a/projects/llm_framework/main_vad/Kconfig b/projects/llm_framework/main_vad/Kconfig new file mode 100644 index 0000000..e69de29 diff --git a/projects/llm_framework/main_vad/SConstruct b/projects/llm_framework/main_vad/SConstruct new file mode 100644 index 0000000..f7ad093 --- /dev/null +++ b/projects/llm_framework/main_vad/SConstruct @@ -0,0 +1,44 @@ +import os + +Import('env') +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +SRCS = Glob('src/*.c*') +INCLUDE = [ADir('include'), ADir('.')] +PRIVATE_INCLUDE = [] +REQUIREMENTS = ['pthread', 'dl', 'utilities', 'eventpp', 'StackFlow', 'single_header_libs'] +STATIC_LIB = [] +DYNAMIC_LIB = [] +DEFINITIONS = [] +DEFINITIONS_PRIVATE = [] +LDFLAGS = [] +LINK_SEARCH_PATH = [] +STATIC_FILES = [] + +DEFINITIONS += ['-std=c++17'] +LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] +LINK_SEARCH_PATH += [ADir('../static_lib')] + +INCLUDE += [ADir('../include/sherpa')] + +LINK_SEARCH_PATH += [ADir('../static_lib/sherpa/onnx')] +LDFLAGS += ['-l:libsherpa-onnx-core.a', + '-l:libonnxruntime.a'] + +STATIC_FILES += Glob('mode_*.json') + +env['COMPONENTS'].append({'target':'llm_vad', + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'STATIC_FILES':STATIC_FILES, + 'REGISTER':'project' + }) diff --git a/projects/llm_framework/main_vad/mode_silero-vad-model.json b/projects/llm_framework/main_vad/mode_silero-vad-model.json new file mode 100644 index 0000000..5bac296 --- /dev/null +++ b/projects/llm_framework/main_vad/mode_silero-vad-model.json @@ -0,0 +1,26 @@ +{ + "mode": "silero_vad", + "type": "vad", + "capabilities": [ + "Voice_activity_detection" + ], + "input_type": [ + "sys.pcm", + "sys.cap.0_0" + ], + "output_type": [ + "vad.bool" + ], + "mode_param": { + "silero_vad.model": "silero_vad.onnx" + }, + "mode_param_bak": { + "silero_vad.threshold": 0.5, + "silero_vad.min_silence_duration": 0.5, + "silero_vad.min_speech_duration": 0.25, + "silero_vad.window_size": 512, + "sample_rate": 16000, + "num_threads": 1, + "provider": "cpu" + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_vad/src/main.cpp b/projects/llm_framework/main_vad/src/main.cpp new file mode 100644 index 0000000..91557b5 --- /dev/null +++ b/projects/llm_framework/main_vad/src/main.cpp @@ -0,0 +1,460 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include "sherpa-onnx/csrc/voice-activity-detector.h" + +#include +#include +#include +#include +#include +#include +#include "../../../../SDK/components/utilities/include/sample_log.h" + +#define BUFFER_IMPLEMENTATION +#include +#include "libs/buffer.h" + +#include + +using namespace StackFlows; + +int main_exit_flage = 0; +static void __sigint(int iSigNo) +{ + SLOGW("llm_vad will be exit!"); + main_exit_flage = 1; +} + +static std::string base_model_path_; +static std::string base_model_config_path_; + +#define CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_config_.key = obj[#key]; + +class llm_task { +private: + sherpa_onnx::VadModelConfig mode_config_; + std::unique_ptr vad_; + +public: + std::string model_; + std::string response_format_; + std::vector inputs_; + bool enoutput_; + bool enstream_; + bool printed = false; + std::atomic_bool audio_flage_; + int delay_audio_frame_ = 100; + buffer_t *pcmdata; + std::string wake_wav_file_; + + std::function out_callback_; + + bool parse_config(const nlohmann::json &config_body) + { + fprintf(stderr, "%s\n", mode_config_.ToString().c_str()); + try { + model_ = config_body.at("model"); + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } + } catch (...) { + SLOGE("setup config_body error"); + return true; + } + enstream_ = response_format_.find("stream") == std::string::npos ? false : true; + return false; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + + nlohmann::json file_body; + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); + try { + for (auto file_name : config_file_paths) { + std::ifstream config_file(file_name); + if (!config_file.is_open()) { + SLOGW("config file :%s miss", file_name.c_str()); + continue; + } + config_file >> file_body; + config_file.close(); + break; + } + if (file_body.empty()) { + SLOGE("all config file miss"); + return -2; + } + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model %s", base_model.c_str()); + + CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.model); + CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.threshold); + CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.min_silence_duration); + CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.min_speech_duration); + CONFIG_AUTO_SET(file_body["mode_param"], silero_vad.window_size); + CONFIG_AUTO_SET(file_body["mode_param"], sample_rate); + CONFIG_AUTO_SET(file_body["mode_param"], num_threads); + CONFIG_AUTO_SET(file_body["mode_param"], provider); + + if (config_body.contains("wake_wav_file")) + wake_wav_file_ = config_body["wake_wav_file"]; + else if (file_body["mode_param"].contains("wake_wav_file")) + wake_wav_file_ = file_body["mode_param"]["wake_wav_file"]; + + mode_config_.silero_vad.model = base_model + mode_config_.silero_vad.model; + if (!mode_config_.Validate()) { + fprintf(stderr, "Errors in config!\n"); + return -1; + } + vad_ = std::make_unique(mode_config_); + } catch (...) { + SLOGE("config file read false"); + return -3; + } + return 0; + } + + void set_output(std::function out_callback) + { + out_callback_ = out_callback; + } + + void sys_pcm_on_data(const std::string &raw) + { + static int count = 0; + int32_t k = 0; + if (count < delay_audio_frame_) { + buffer_write_char(pcmdata, raw.c_str(), raw.length()); + count++; + return; + } + buffer_write_char(pcmdata, raw.c_str(), raw.length()); + buffer_position_set(pcmdata, 0); + count = 0; + std::vector floatSamples; + { + int16_t audio_val; + while (buffer_read_u16(pcmdata, (unsigned short *)&audio_val, 1)) { + float normalizedSample = (float)audio_val / INT16_MAX; + floatSamples.push_back(normalizedSample); + } + } + buffer_position_set(pcmdata, 0); + vad_->AcceptWaveform(floatSamples.data(), floatSamples.size()); + + if (vad_->IsSpeechDetected() && !printed) { + printed = true; + SLOGI("Detected speech!"); + } + if (!vad_->IsSpeechDetected()) { + printed = false; + } + int32_t sample_rate = 16000; + + while (!vad_->Empty()) { + const auto &segment = vad_->Front(); + float duration = segment.samples.size() / static_cast(sample_rate); + SLOGI("Duration: %.3f seconds", duration); + k += 1; + vad_->Pop(); + } + } + + bool delete_model() + { + vad_.reset(); + return true; + } + + llm_task(const std::string &workid) : audio_flage_(false) + { + pcmdata = buffer_create(); + } + + ~llm_task() + { + if (vad_) { + vad_.reset(); + } + buffer_destroy(pcmdata); + } +}; +#undef CONFIG_AUTO_SET + +class llm_kws : public StackFlow { +private: + int task_count_; + std::string audio_url_; + std::unordered_map> llm_task_; + +public: + llm_kws() : StackFlow("vad") + { + task_count_ = 1; + } + + void task_pause(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if (llm_task_obj->audio_flage_) { + if (!audio_url_.empty()) llm_channel->stop_subscriber(audio_url_); + llm_task_obj->audio_flage_ = false; + } + } + + void task_work(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) { + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); + llm_task_obj->audio_flage_ = true; + } + } + + void work(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_asr::work:%s", data.c_str()); + + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + task_work(llm_task_[work_id_num], get_channel(work_id_num)); + send("None", "None", LLM_NO_ERROR, work_id); + } + + void pause(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_asr::work:%s", data.c_str()); + + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return; + } + task_pause(llm_task_[work_id_num], get_channel(work_id_num)); + send("None", "None", LLM_NO_ERROR, work_id); + } + + void task_user_data(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &object, + const std::string &data) + { + nlohmann::json error_body; + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + error_body["code"] = -11; + error_body["message"] = "Model run failed."; + send("None", "None", error_body, unit_name_); + return; + } + std::string tmp_msg1; + const std::string *next_data = &data; + int ret; + if (object.find("stream") != std::string::npos) { + static std::unordered_map stream_buff; + try { + if (decode_stream(data, tmp_msg1, stream_buff)) { + return; + }; + } catch (...) { + stream_buff.clear(); + error_body["code"] = -25; + error_body["message"] = "Stream data index error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg1; + } + std::string tmp_msg2; + if (object.find("base64") != std::string::npos) { + ret = decode_base64((*next_data), tmp_msg2); + if (ret == -1) { + error_body["code"] = -23; + error_body["message"] = "Base64 decoding error."; + send("None", "None", error_body, unit_name_); + return; + } + next_data = &tmp_msg2; + } + llm_task_obj->sys_pcm_on_data((*next_data)); + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override + { + nlohmann::json error_body; + if ((llm_task_channel_.size() - 1) == task_count_) { + error_body["code"] = -21; + error_body["message"] = "task full"; + send("None", "None", error_body, "vad"); + return -1; + } + + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + SLOGE("setup json format error."); + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, "vad"); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + llm_task_obj->set_output([llm_task_obj, llm_channel](const std::string &data) { + llm_channel->send(llm_task_obj->response_format_, true, LLM_NO_ERROR); + }); + + for (const auto input : llm_task_obj->inputs_) { + if (input.find("sys") != std::string::npos) { + audio_url_ = unit_call("audio", "cap", "None"); + std::weak_ptr _llm_task_obj = llm_task_obj; + llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) { + _llm_task_obj.lock()->sys_pcm_on_data(raw); + }); + llm_task_obj->audio_flage_ = true; + } else if (input.find("vad") != std::string::npos) { + llm_channel->subscriber_work_id( + "", std::bind(&llm_kws::task_user_data, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); + } + } + llm_task_[work_id_num] = llm_task_obj; + SLOGI("load_mode success"); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } else { + SLOGE("load_mode Failed"); + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, "vad"); + return -1; + } + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_vad::taskinfo:%s", data.c_str()); + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("vad.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["model"] = llm_task_obj->model_; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs"] = llm_task_obj->inputs_; + send("vad.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override + { + SLOGI("llm_kws::exit:%s", data.c_str()); + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + if (llm_task_[work_id_num]->audio_flage_) { + unit_call("audio", "cap_stop", "None"); + } + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_kws() + { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + if (iteam->second->audio_flage_) { + unit_call("audio", "cap_stop", "None"); + } + get_channel(iteam->first)->stop_subscriber(""); + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main(int argc, char *argv[]) +{ + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_kws llm; + while (!main_exit_flage) { + sleep(1); + } + return 0; +} \ No newline at end of file