m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] KWS sets multiple keywords, fix melotts

Browse Source 
File missing, Optimize generated audio
This commit is contained in:
LittleMouse
2025-07-18 14:12:12 +08:00
parent 59958869a0
commit 9abe069db6
3 changed files with 148 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/llm_framework/main_kws/src/main.cpp
+60 -16
View File
@@ -34,6 +34,8 @@ static void __sigint(int iSigNo)
static std::string base_model_path_;
static std::string base_model_config_path_;
typedef std::function<void(const std::string &data, bool finish)> task_callback_t;
#define CONFIG_AUTO_SET(obj, key) \
if (config_body.contains(#key)) \
mode_config_.key = config_body[#key]; \
@@ -50,16 +52,17 @@ public:
std::string model_;
std::string response_format_;
std::vector<std::string> inputs_;
std::string kws_;
std::vector<std::string> kws_;
bool enoutput_;
bool enoutput_json_;
bool enstream_;
bool enwake_audio_;
std::atomic_bool audio_flage_;
task_callback_t out_callback_;
int delay_audio_frame_ = 100;
buffer_t *pcmdata;
std::string wake_wav_file_;
std::function<void(const std::string &)> out_callback_;
std::function<void(const std::string &)> play_awake_wav;
bool parse_config(const nlohmann::json &config_body)
@@ -68,7 +71,6 @@ public:
model_ = config_body.at("model");
response_format_ = config_body.at("response_format");
enoutput_ = config_body.at("enoutput");
kws_ = config_body.at("kws");
if (config_body.contains("enwake_audio")) {
enwake_audio_ = config_body["enwake_audio"];
} else {
@@ -83,6 +85,16 @@ public:
}
}
}
if (config_body.contains("kws")) {
if (config_body["kws"].is_string()) {
kws_.push_back(config_body["kws"].get<std::string>());
} else if (config_body["kws"].is_array()) {
for (auto _in : config_body["kws"]) {
kws_.push_back(_in.get<std::string>());
}
}
}
enoutput_json_ = response_format_.find("json") == std::string::npos ? false : true;
} catch (...) {
SLOGE("setup config_body error");
return true;
@@ -173,7 +185,9 @@ public:
mode_config_.keywords_file = base_model + mode_config_.keywords_file;
std::ofstream temp_awake_key("/tmp/kws_awake.txt.tmp");
temp_awake_key << kws_;
for (const auto &keyword : kws_) {
temp_awake_key << keyword << std::endl;
}
temp_awake_key.close();
std::ostringstream awake_key_compile_cmd;
if (file_exists("/opt/m5stack/scripts/text2token.py"))
@@ -206,7 +220,7 @@ public:
return 0;
}
void set_output(std::function<void(const std::string &)> out_callback)
void set_output(task_callback_t out_callback)
{
out_callback_ = out_callback;
}
@@ -242,16 +256,17 @@ public:
play_awake_wav(wake_wav_file_);
}
if (out_callback_) {
out_callback_("True");
if (enoutput_json_)
out_callback_(r.AsJsonString(), true);
else
out_callback_("", true);
}
}
}
void trigger()
{
if (out_callback_) {
out_callback_("True");
}
if (out_callback_) out_callback_("", true);
}
bool delete_model()
@@ -301,6 +316,39 @@ public:
});
}
void task_output(const std::weak_ptr<llm_task> llm_task_obj_weak,
const std::weak_ptr<llm_channel_obj> llm_channel_weak, const std::string &data, bool finish)
{
auto llm_task_obj = llm_task_obj_weak.lock();
auto llm_channel = llm_channel_weak.lock();
if (!(llm_task_obj && llm_channel)) {
return;
}
std::string tmp_msg1;
const std::string *next_data = &data;
if (data.empty()) {
llm_channel->send(llm_task_obj->response_format_, true, LLM_NO_ERROR);
return;
}
if (finish) {
tmp_msg1 = data + ".";
next_data = &tmp_msg1;
}
if (llm_channel->enstream_) {
static int count = 0;
nlohmann::json data_body;
data_body["index"] = count++;
data_body["delta"] = (*next_data);
data_body["finish"] = finish;
if (finish) count = 0;
SLOGI("send stream:%s", next_data->c_str());
llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR);
} else if (finish) {
SLOGI("send utf-8:%s", next_data->c_str());
llm_channel->send(llm_task_obj->response_format_, (*next_data), LLM_NO_ERROR);
}
}
void play_awake_wav(const std::string &wav_file)
{
FILE *fp = fopen(wav_file.c_str(), "rb");
@@ -465,13 +513,9 @@ public:
llm_channel->set_output(llm_task_obj->enoutput_);
llm_channel->set_stream(llm_task_obj->enstream_);
llm_task_obj->play_awake_wav = std::bind(&llm_kws::play_awake_wav, this, std::placeholders::_1);
llm_task_obj->set_output([_llm_task_obj, _llm_channel](const std::string &data) {
auto llm_task_obj = _llm_task_obj.lock();
auto llm_channel = _llm_channel.lock();
if (llm_task_obj && llm_channel) {
llm_channel->send(llm_task_obj->response_format_, true, LLM_NO_ERROR);
}
});
llm_task_obj->set_output(std::bind(&llm_kws::task_output, this, std::weak_ptr<llm_task>(llm_task_obj),
std::weak_ptr<llm_channel_obj>(llm_channel), std::placeholders::_1,
std::placeholders::_2));
for (const auto input : llm_task_obj->inputs_) {
if (input.find("sys") != std::string::npos) {
projects/llm_framework/main_melotts/src/main.cpp
+84 -142
View File
@@ -184,7 +184,10 @@ public:
else if (file_body["mode_param"].contains("awake_delay"))
awake_delay_ = file_body["mode_param"]["awake_delay"];
if (!std::filesystem::exists(mode_config_.tagger) || !std::filesystem::exists(mode_config_.verbalizer)) {
if (!std::filesystem::exists(mode_config_.tagger) ||
!std::filesystem::is_regular_file(mode_config_.tagger) ||
!std::filesystem::exists(mode_config_.verbalizer) ||
!std::filesystem::is_regular_file(mode_config_.verbalizer)) {
SLOGW("Either tagger or verbalizer file does not exist, using alternative lexicon.");
lexicon_ = std::make_unique<Lexicon>(mode_config_.lexicon, mode_config_.tokens);
} else {
@@ -281,195 +284,132 @@ public:
int dec_len = zp_size / zp_shape[1];
int audio_slice_len = decoder_->GetOutputSize(0) / sizeof(float);
const int pad_frames = 24;
const int samples_per_frame = 512;
const int overlap_size = 1024;
const int fade_size = 512;
const int effective_frames = dec_len - 2 * pad_frames;
int dec_slice_num = static_cast<int>(std::ceil(static_cast<double>(zp_shape[2]) / dec_len));
int dec_slice_num =
static_cast<int>(std::ceil(static_cast<double>(zp_shape[2]) / static_cast<double>(effective_frames)));
const int sola_buffer_frame = pad_frames * samples_per_frame;
const int sola_search_frame = pad_frames * samples_per_frame;
const int block_frame = (dec_len - 2 * pad_frames) * samples_per_frame;
std::vector<float> fade_in_window(sola_buffer_frame);
std::vector<float> fade_out_window(sola_buffer_frame);
for (int i = 0; i < sola_buffer_frame; i++) {
fade_in_window[i] = static_cast<float>(i) / sola_buffer_frame;
fade_out_window[i] = 1.0f - fade_in_window[i];
std::vector<float> fade_in(fade_size);
std::vector<float> fade_out(fade_size);
for (int i = 0; i < fade_size; i++) {
float t = static_cast<float>(i) / fade_size;
fade_in[i] = t;
fade_out[i] = 1.0f - t;
}
std::vector<float> sola_buffer(sola_buffer_frame, 0.0f);
bool first_frame = true;
std::vector<float> pcmlist;
std::vector<float> previous_tail;
for (int i = 0; i < dec_slice_num; i++) {
int input_start = i * effective_frames;
if (i > 0) {
input_start -= pad_frames;
}
input_start = std::max(0, input_start);
int actual_len = std::min(dec_len, static_cast<int>(zp_shape[2] - input_start));
int output_start_frame, output_end_frame;
if (i == 0) {
output_start_frame = 0;
output_end_frame = effective_frames - 1;
} else if (i == dec_slice_num - 1) {
output_start_frame = i * effective_frames;
output_end_frame = static_cast<int>(zp_shape[2]) - 1;
} else {
output_start_frame = i * effective_frames;
output_end_frame = (i + 1) * effective_frames - 1;
}
int input_start = i * dec_len;
int actual_size = std::min(dec_len, static_cast<int>(zp_shape[2] - input_start));
std::vector<float> zp(zp_size, 0);
for (int n = 0; n < zp_shape[1]; n++) {
int copy_size = std::min(actual_len, static_cast<int>(zp_shape[2] - input_start));
if (copy_size > 0) {
if (actual_size > 0) {
memcpy(zp.data() + n * dec_len, zp_data + n * zp_shape[2] + input_start,
sizeof(float) * copy_size);
sizeof(float) * actual_size);
}
}
std::vector<float> decoder_output(audio_slice_len);
decoder_->SetInput(zp.data(), 0);
decoder_->SetInput(g_matrix.data(), 1);
if (0 != decoder_->Run()) {
SLOGE("Decoder run failed at slice %d", i);
throw std::string("decoder_ RunSync error");
}
std::vector<float> decoder_output(audio_slice_len);
decoder_->GetOutput(decoder_output.data(), 0);
if (first_frame) {
int audio_start = 0;
int audio_len = decoder_output.size() - sola_buffer_frame;
audio_len = std::max(0, audio_len);
if (i == 0) {
int main_part_size = static_cast<int>(decoder_output.size()) - overlap_size;
main_part_size = std::max(0, main_part_size);
if (audio_len > 0) {
pcmlist.insert(pcmlist.end(), decoder_output.begin() + audio_start,
decoder_output.begin() + audio_start + audio_len);
pcmlist.insert(pcmlist.end(), decoder_output.begin(), decoder_output.begin() + main_part_size);
if (decoder_output.size() > main_part_size) {
previous_tail.assign(decoder_output.begin() + main_part_size, decoder_output.end());
}
int buffer_start = audio_len;
if (buffer_start + sola_buffer_frame <= decoder_output.size()) {
std::copy(decoder_output.begin() + buffer_start,
decoder_output.begin() + buffer_start + sola_buffer_frame, sola_buffer.begin());
} else {
int available = static_cast<int>(decoder_output.size() - buffer_start);
if (available > 0) {
std::copy(decoder_output.begin() + buffer_start, decoder_output.end(), sola_buffer.begin());
std::fill(sola_buffer.begin() + available, sola_buffer.end(), 0.0f);
} else {
std::fill(sola_buffer.begin(), sola_buffer.end(), 0.0f);
}
}
first_frame = false;
} else {
int audio_start = pad_frames * samples_per_frame;
std::vector<float> search_window(sola_buffer_frame + sola_search_frame);
std::copy(decoder_output.begin() + audio_start,
decoder_output.begin() + audio_start + search_window.size(), search_window.begin());
int best_offset = 0;
float best_correlation = -1.0;
for (int offset = 0; offset <= sola_search_frame; offset++) {
float correlation = 0.0;
float energy = 0.0;
for (int j = 0; j < sola_buffer_frame; j++) {
correlation += sola_buffer[j] * search_window[j + offset];
energy += search_window[j + offset] * search_window[j + offset];
}
float normalized_correlation = (energy > 1e-8) ? correlation / std::sqrt(energy) : 0.0f;
if (normalized_correlation > best_correlation) {
best_correlation = normalized_correlation;
best_offset = offset;
}
if (previous_tail.empty()) {
pcmlist.insert(pcmlist.end(), decoder_output.begin(), decoder_output.end());
continue;
}
int aligned_start = audio_start + best_offset;
int blend_size = std::min(
{fade_size, static_cast<int>(previous_tail.size()), static_cast<int>(decoder_output.size())});
std::vector<float> crossfade_region(sola_buffer_frame);
for (int j = 0; j < sola_buffer_frame; j++) {
crossfade_region[j] =
decoder_output[aligned_start + j] * fade_in_window[j] + sola_buffer[j] * fade_out_window[j];
std::vector<float> blended_region(blend_size);
for (int j = 0; j < blend_size; j++) {
blended_region[j] = previous_tail[j] * fade_out[j * fade_size / blend_size] +
decoder_output[j] * fade_in[j * fade_size / blend_size];
}
pcmlist.insert(pcmlist.end(), crossfade_region.begin(), crossfade_region.end());
pcmlist.insert(pcmlist.end(), blended_region.begin(), blended_region.end());
int remaining_start = aligned_start + sola_buffer_frame;
if (static_cast<int>(previous_tail.size()) > blend_size) {
pcmlist.insert(pcmlist.end(), previous_tail.begin() + blend_size, previous_tail.end());
}
int current_remaining_start = blend_size;
int current_remaining_size = static_cast<int>(decoder_output.size()) - current_remaining_start;
if (i == dec_slice_num - 1) {
int total_expected_samples = audio_len * samples_per_frame / 512;
int processed_samples = static_cast<int>(pcmlist.size());
int remaining_needed = total_expected_samples - processed_samples;
remaining_needed = std::max(0, remaining_needed);
int remaining_len =
std::min(remaining_needed, static_cast<int>(decoder_output.size() - remaining_start));
if (remaining_len > 0) {
pcmlist.insert(pcmlist.end(), decoder_output.begin() + remaining_start,
decoder_output.begin() + remaining_start + remaining_len);
}
} else {
int remaining_len = (dec_len - 2 * pad_frames) * samples_per_frame - sola_buffer_frame;
remaining_len =
std::min(remaining_len, static_cast<int>(decoder_output.size() - remaining_start));
if (remaining_len > 0) {
pcmlist.insert(pcmlist.end(), decoder_output.begin() + remaining_start,
decoder_output.begin() + remaining_start + remaining_len);
}
int buffer_start = remaining_start + remaining_len;
if (buffer_start + sola_buffer_frame <= decoder_output.size()) {
std::copy(decoder_output.begin() + buffer_start,
decoder_output.begin() + buffer_start + sola_buffer_frame, sola_buffer.begin());
} else {
int avail = static_cast<int>(decoder_output.size() - buffer_start);
if (avail > 0) {
std::copy(decoder_output.begin() + buffer_start, decoder_output.end(),
sola_buffer.begin());
}
std::fill(sola_buffer.begin() + avail, sola_buffer.end(), 0.0f);
}
int total_expected = audio_len;
int current_total = static_cast<int>(pcmlist.size());
current_remaining_size = std::min(current_remaining_size, total_expected - current_total);
}
if (current_remaining_size > overlap_size && i < dec_slice_num - 1) {
int main_part_size = current_remaining_size - overlap_size;
pcmlist.insert(pcmlist.end(), decoder_output.begin() + current_remaining_start,
decoder_output.begin() + current_remaining_start + main_part_size);
previous_tail.assign(decoder_output.begin() + current_remaining_start + main_part_size,
decoder_output.begin() + current_remaining_start + current_remaining_size);
} else {
if (current_remaining_size > 0) {
pcmlist.insert(pcmlist.end(), decoder_output.begin() + current_remaining_start,
decoder_output.begin() + current_remaining_start + current_remaining_size);
}
previous_tail.clear();
}
}
if (static_cast<int>(pcmlist.size()) >= audio_len) {
break;
}
}
if (pcmlist.size() > audio_len) {
if (static_cast<int>(pcmlist.size()) > audio_len) {
pcmlist.resize(audio_len);
}
float max_val = 0.0f;
int clip_count = 0;
for (float sample : pcmlist) {
max_val = std::max(max_val, std::abs(sample));
if (std::abs(sample) > 0.95f) clip_count++;
}
double src_ratio =
static_cast<double>(mode_config_.audio_rate) / static_cast<double>(mode_config_.mode_rate);
std::vector<float> tmp_pcm((pcmlist.size() * src_ratio + 1));
std::vector<float> tmp_pcm(static_cast<size_t>(pcmlist.size() * src_ratio + 1));
int len;
resample_audio(pcmlist.data(), pcmlist.size(), tmp_pcm.data(), &len, src_ratio);
wav_pcm_data.reserve(len);
std::transform(tmp_pcm.begin(), tmp_pcm.begin() + len, std::back_inserter(wav_pcm_data),
[](const auto val) { return static_cast<int16_t>(val * INT16_MAX); });
for (int i = 0; i < len; i++) {
float val = tmp_pcm[i];
if (std::abs(val) > 0.95f) {
val = val > 0 ? 0.95f : -0.95f;
}
wav_pcm_data.push_back(static_cast<int16_t>(val * INT16_MAX));
}
if (out_callback_) {
out_callback_(
@@ -478,8 +418,10 @@ public:
}
} catch (const std::exception &e) {
SLOGE("Exception: %s", e.what());
return true;
} catch (...) {
SLOGE("Unknown exception occurred");
return true;
}
return false;
projects/llm_framework/main_melotts/src/runner/Lexicon.hpp
+4 -4
View File
@@ -37,7 +37,7 @@ public:
const std::string& verbalizer_filename)
: max_phrase_length(0)
{
SLOGD("Dictionary loading: %s Pronunciation table loading: %s tagger_filename: %s verbalizer_filename: %s",
SLOGI("Dictionary loading: %s Pronunciation table loading: %s tagger_filename: %s verbalizer_filename: %s",
tokens_filename.c_str(), lexicon_filename.c_str(), tagger_filename.c_str(), verbalizer_filename.c_str());
m_processor = new wetext::Processor(tagger_filename, verbalizer_filename);
@@ -92,13 +92,13 @@ public:
lexicon["。"] = lexicon["."];
lexicon["！"] = lexicon["!"];
lexicon["？"] = lexicon["?"];
SLOGD("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(),
SLOGI("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(),
max_phrase_length);
}
Lexicon(const std::string& lexicon_filename, const std::string& tokens_filename) : max_phrase_length(0)
{
SLOGD("Dictionary loading: %s Pronunciation table loading: %s", tokens_filename.c_str(),
SLOGI("Dictionary loading: %s Pronunciation table loading: %s", tokens_filename.c_str(),
lexicon_filename.c_str());
std::unordered_map<std::string, int> tokens;
@@ -151,7 +151,7 @@ public:
lexicon["。"] = lexicon["."];
lexicon["！"] = lexicon["!"];
lexicon["？"] = lexicon["?"];
SLOGD("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(),
SLOGI("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(),
max_phrase_length);
}