m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] update vad & whisper, vad support kws, whisper suport kws.

Browse Source
This commit is contained in:
LittleMouse
2025-01-15 10:37:32 +08:00
parent ee276a346b
commit 8a4e7e08de
2 changed files with 191 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/llm_framework/main_vad/src/main.cpp
+182 -66
View File
@@ -23,6 +23,7 @@
using namespace StackFlows;
int main_exit_flage = 0;
static void __sigint(int iSigNo)
{
SLOGW("llm_vad will be exit!");
@@ -51,13 +52,20 @@ public:
std::vector<std::string> inputs_;
bool enoutput_;
bool enstream_;
bool ensleep_;
bool printed = false;
task_callback_t out_callback_;
std::atomic_bool superior_flage_;
std::atomic_bool audio_flage_;
std::atomic_bool awake_flage_;
std::string superior_id_;
task_callback_t out_callback_;
int awake_delay_ = 50;
int delay_audio_frame_ = 100;
buffer_t *pcmdata;
std::string wake_wav_file_;
std::function<void(void)> pause;
bool parse_config(const nlohmann::json &config_body)
{
try {
@@ -184,9 +192,17 @@ public:
if (out_callback_) {
out_callback_(false);
}
if (ensleep_) {
if (pause) pause();
}
}
}
void kws_awake()
{
awake_flage_ = true;
}
bool delete_model()
{
vad_.reset();
@@ -195,7 +211,9 @@ public:
llm_task(const std::string &workid) : audio_flage_(false)
{
pcmdata = buffer_create();
ensleep_ = false;
awake_flage_ = false;
pcmdata = buffer_create();
}
~llm_task()
@@ -215,9 +233,12 @@ private:
std::unordered_map<int, std::shared_ptr<llm_task>> llm_task_;
public:
enum { EVENT_LOAD_CONFIG = EVENT_EXPORT + 1, EVENT_TASK_PAUSE };
llm_vad() : StackFlow("vad")
{
task_count_ = 1;
event_queue_.appendListener(
EVENT_TASK_PAUSE, std::bind(&llm_vad::_task_pause, this, std::placeholders::_1, std::placeholders::_2));
}
void task_output(const std::weak_ptr<llm_task> llm_task_obj_weak,
@@ -233,69 +254,6 @@ public:
llm_channel->send(llm_task_obj->response_format_, (*next_data), LLM_NO_ERROR);
}
void task_pause(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak)
{
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
return;
}
if (llm_task_obj->audio_flage_) {
if (!audio_url_.empty()) llm_channel->stop_subscriber(audio_url_);
llm_task_obj->audio_flage_ = false;
}
}
void task_work(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak)
{
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
return;
}
if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) {
std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
_llm_task_obj.lock()->sys_pcm_on_data(raw);
});
llm_task_obj->audio_flage_ = true;
}
}
void work(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_asr::work:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
task_work(llm_task_[work_id_num], get_channel(work_id_num));
send("None", "None", LLM_NO_ERROR, work_id);
}
void pause(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_asr::work:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
task_pause(llm_task_[work_id_num], get_channel(work_id_num));
send("None", "None", LLM_NO_ERROR, work_id);
}
void task_user_data(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak, const std::string &object,
const std::string &data)
@@ -341,6 +299,87 @@ public:
llm_task_obj->sys_pcm_on_data((*next_data));
}
void _task_pause(const std::string &work_id, const std::string &data)
{
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
return;
}
auto llm_task_obj = llm_task_[work_id_num];
auto llm_channel = get_channel(work_id_num);
if (llm_task_obj->audio_flage_) {
if (!audio_url_.empty()) llm_channel->stop_subscriber(audio_url_);
llm_task_obj->audio_flage_ = false;
}
}
void task_pause(const std::string &work_id, const std::string &data)
{
event_queue_.enqueue(EVENT_TASK_PAUSE, work_id, "");
}
void task_work(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak)
{
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
return;
}
if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) {
std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
_llm_task_obj.lock()->sys_pcm_on_data(raw);
});
llm_task_obj->audio_flage_ = true;
}
}
void kws_awake(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak, const std::string &object,
const std::string &data)
{
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
return;
}
std::this_thread::sleep_for(std::chrono::milliseconds(llm_task_obj->awake_delay_));
task_work(llm_task_obj, llm_channel);
}
void work(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_vad::work:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
task_work(llm_task_[work_id_num], get_channel(work_id_num));
send("None", "None", LLM_NO_ERROR, work_id);
}
void pause(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_vad::work:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
task_pause(work_id, "");
send("None", "None", LLM_NO_ERROR, work_id);
}
int setup(const std::string &work_id, const std::string &object, const std::string &data) override
{
nlohmann::json error_body;
@@ -354,6 +393,7 @@ public:
int work_id_num = sample_get_work_id_num(work_id);
auto llm_channel = get_channel(work_id);
auto llm_task_obj = std::make_shared<llm_task>(work_id);
nlohmann::json config_body;
try {
config_body = nlohmann::json::parse(data);
@@ -368,6 +408,7 @@ public:
if (ret == 0) {
llm_channel->set_output(llm_task_obj->enoutput_);
llm_channel->set_stream(llm_task_obj->enstream_);
llm_task_obj->pause = std::bind(&llm_vad::task_pause, this, work_id, "");
llm_task_obj->set_output(std::bind(&llm_vad::task_output, this, std::weak_ptr<llm_task>(llm_task_obj),
std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1));
@@ -384,6 +425,13 @@ public:
"", std::bind(&llm_vad::task_user_data, this, std::weak_ptr<llm_task>(llm_task_obj),
std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
std::placeholders::_2));
} else if (input.find("kws") != std::string::npos) {
llm_task_obj->ensleep_ = true;
task_pause(work_id, "");
llm_channel->subscriber_work_id(
input, std::bind(&llm_vad::kws_awake, this, std::weak_ptr<llm_task>(llm_task_obj),
std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
std::placeholders::_2));
}
}
llm_task_[work_id_num] = llm_task_obj;
@@ -399,6 +447,74 @@ public:
}
}
void link(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_vad::link:%s", data.c_str());
int ret = 1;
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
auto llm_channel = get_channel(work_id);
auto llm_task_obj = llm_task_[work_id_num];
if (data.find("sys") != std::string::npos) {
if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data);
std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
_llm_task_obj.lock()->sys_pcm_on_data(raw);
});
llm_task_obj->audio_flage_ = true;
llm_task_obj->inputs_.push_back(data);
} else if (data.find("kws") != std::string::npos) {
llm_task_obj->ensleep_ = true;
ret = llm_channel->subscriber_work_id(
data,
std::bind(&llm_vad::kws_awake, this, std::weak_ptr<llm_task>(llm_task_obj),
std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1, std::placeholders::_2));
llm_task_obj->inputs_.push_back(data);
}
if (ret) {
error_body["code"] = -20;
error_body["message"] = "link false";
send("None", "None", error_body, work_id);
return;
} else {
send("None", "None", LLM_NO_ERROR, work_id);
}
}
void unlink(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_vad::unlink:%s", data.c_str());
int ret = 0;
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
error_body["code"] = -6;
error_body["message"] = "Unit Does Not Exist";
send("None", "None", error_body, work_id);
return;
}
auto llm_channel = get_channel(work_id);
auto llm_task_obj = llm_task_[work_id_num];
if (llm_task_obj->superior_id_ == work_id) {
llm_task_obj->superior_flage_ = false;
}
llm_channel->stop_subscriber_work_id(data);
for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) {
if (*it == data) {
it = llm_task_obj->inputs_.erase(it);
} else {
++it;
}
}
send("None", "None", LLM_NO_ERROR, work_id);
}
void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_vad::taskinfo:%s", data.c_str());
@@ -428,7 +544,7 @@ public:
int exit(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_kws::exit:%s", data.c_str());
SLOGI("llm_vad::exit:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {
projects/llm_framework/main_whisper/src/main.cpp
+9 -8
View File
@@ -32,7 +32,7 @@ int main_exit_flage = 0;
static void __sigint(int iSigNo)
{
SLOGW("llm_sys will be exit!");
SLOGW("llm_whisper will be exit!");
main_exit_flage = 1;
}
@@ -77,11 +77,12 @@ typedef std::function<void(const std::string &data, bool finish)> task_callback_
class llm_task {
private:
public:
whisper_config mode_config_;
std::unique_ptr<Encoder> encoder_;
std::unique_ptr<DecoderMain> decoder_main_;
std::unique_ptr<DecoderLoop> decoder_loop_;
public:
std::vector<float> positional_embedding;
std::string model_;
std::string response_format_;
@@ -738,7 +739,7 @@ public:
void work(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_asr::work:%s", data.c_str());
SLOGI("llm_whisper::work:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
@@ -754,7 +755,7 @@ public:
void pause(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_asr::work:%s", data.c_str());
SLOGI("llm_whisper::pause:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
@@ -774,7 +775,7 @@ public:
if ((llm_task_channel_.size() - 1) == task_count_) {
error_body["code"] = -21;
error_body["message"] = "task full";
send("None", "None", error_body, "asr");
send("None", "None", error_body, "whisper");
return -1;
}
@@ -809,7 +810,7 @@ public:
_llm_task_obj.lock()->sys_pcm_on_data(raw);
});
llm_task_obj->audio_flage_ = true;
} else if (input.find("asr") != std::string::npos) {
} else if (input.find("whisper") != std::string::npos) {
llm_channel->subscriber_work_id(
"", std::bind(&llm_whisper::task_user_data, this, std::weak_ptr<llm_task>(llm_task_obj),
std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
@@ -845,7 +846,7 @@ public:
void link(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_melotts::link:%s", data.c_str());
SLOGI("llm_whisper::link:%s", data.c_str());
int ret = 1;
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
@@ -947,7 +948,7 @@ public:
int exit(const std::string &work_id, const std::string &object, const std::string &data) override
{
SLOGI("llm_asr::exit:%s", data.c_str());
SLOGI("llm_whisper::exit:%s", data.c_str());
nlohmann::json error_body;
int work_id_num = sample_get_work_id_num(work_id);
if (llm_task_.find(work_id_num) == llm_task_.end()) {