diff --git a/projects/llm_framework/main_kws/src/main.cpp b/projects/llm_framework/main_kws/src/main.cpp index 4ea6abc..5b8a21b 100644 --- a/projects/llm_framework/main_kws/src/main.cpp +++ b/projects/llm_framework/main_kws/src/main.cpp @@ -34,6 +34,8 @@ static void __sigint(int iSigNo) static std::string base_model_path_; static std::string base_model_config_path_; +typedef std::function task_callback_t; + #define CONFIG_AUTO_SET(obj, key) \ if (config_body.contains(#key)) \ mode_config_.key = config_body[#key]; \ @@ -50,16 +52,17 @@ public: std::string model_; std::string response_format_; std::vector inputs_; - std::string kws_; + std::vector kws_; bool enoutput_; + bool enoutput_json_; bool enstream_; bool enwake_audio_; std::atomic_bool audio_flage_; + task_callback_t out_callback_; int delay_audio_frame_ = 100; buffer_t *pcmdata; std::string wake_wav_file_; - std::function out_callback_; std::function play_awake_wav; bool parse_config(const nlohmann::json &config_body) @@ -68,7 +71,6 @@ public: model_ = config_body.at("model"); response_format_ = config_body.at("response_format"); enoutput_ = config_body.at("enoutput"); - kws_ = config_body.at("kws"); if (config_body.contains("enwake_audio")) { enwake_audio_ = config_body["enwake_audio"]; } else { @@ -83,6 +85,16 @@ public: } } } + if (config_body.contains("kws")) { + if (config_body["kws"].is_string()) { + kws_.push_back(config_body["kws"].get()); + } else if (config_body["kws"].is_array()) { + for (auto _in : config_body["kws"]) { + kws_.push_back(_in.get()); + } + } + } + enoutput_json_ = response_format_.find("json") == std::string::npos ? false : true; } catch (...) { SLOGE("setup config_body error"); return true; @@ -173,7 +185,9 @@ public: mode_config_.keywords_file = base_model + mode_config_.keywords_file; std::ofstream temp_awake_key("/tmp/kws_awake.txt.tmp"); - temp_awake_key << kws_; + for (const auto &keyword : kws_) { + temp_awake_key << keyword << std::endl; + } temp_awake_key.close(); std::ostringstream awake_key_compile_cmd; if (file_exists("/opt/m5stack/scripts/text2token.py")) @@ -206,7 +220,7 @@ public: return 0; } - void set_output(std::function out_callback) + void set_output(task_callback_t out_callback) { out_callback_ = out_callback; } @@ -242,16 +256,17 @@ public: play_awake_wav(wake_wav_file_); } if (out_callback_) { - out_callback_("True"); + if (enoutput_json_) + out_callback_(r.AsJsonString(), true); + else + out_callback_("", true); } } } void trigger() { - if (out_callback_) { - out_callback_("True"); - } + if (out_callback_) out_callback_("", true); } bool delete_model() @@ -301,6 +316,39 @@ public: }); } + void task_output(const std::weak_ptr llm_task_obj_weak, + const std::weak_ptr llm_channel_weak, const std::string &data, bool finish) + { + auto llm_task_obj = llm_task_obj_weak.lock(); + auto llm_channel = llm_channel_weak.lock(); + if (!(llm_task_obj && llm_channel)) { + return; + } + std::string tmp_msg1; + const std::string *next_data = &data; + if (data.empty()) { + llm_channel->send(llm_task_obj->response_format_, true, LLM_NO_ERROR); + return; + } + if (finish) { + tmp_msg1 = data + "."; + next_data = &tmp_msg1; + } + if (llm_channel->enstream_) { + static int count = 0; + nlohmann::json data_body; + data_body["index"] = count++; + data_body["delta"] = (*next_data); + data_body["finish"] = finish; + if (finish) count = 0; + SLOGI("send stream:%s", next_data->c_str()); + llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); + } else if (finish) { + SLOGI("send utf-8:%s", next_data->c_str()); + llm_channel->send(llm_task_obj->response_format_, (*next_data), LLM_NO_ERROR); + } + } + void play_awake_wav(const std::string &wav_file) { FILE *fp = fopen(wav_file.c_str(), "rb"); @@ -465,13 +513,9 @@ public: llm_channel->set_output(llm_task_obj->enoutput_); llm_channel->set_stream(llm_task_obj->enstream_); llm_task_obj->play_awake_wav = std::bind(&llm_kws::play_awake_wav, this, std::placeholders::_1); - llm_task_obj->set_output([_llm_task_obj, _llm_channel](const std::string &data) { - auto llm_task_obj = _llm_task_obj.lock(); - auto llm_channel = _llm_channel.lock(); - if (llm_task_obj && llm_channel) { - llm_channel->send(llm_task_obj->response_format_, true, LLM_NO_ERROR); - } - }); + llm_task_obj->set_output(std::bind(&llm_kws::task_output, this, std::weak_ptr(llm_task_obj), + std::weak_ptr(llm_channel), std::placeholders::_1, + std::placeholders::_2)); for (const auto input : llm_task_obj->inputs_) { if (input.find("sys") != std::string::npos) { diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 6b8e1c1..c4e2122 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -184,7 +184,10 @@ public: else if (file_body["mode_param"].contains("awake_delay")) awake_delay_ = file_body["mode_param"]["awake_delay"]; - if (!std::filesystem::exists(mode_config_.tagger) || !std::filesystem::exists(mode_config_.verbalizer)) { + if (!std::filesystem::exists(mode_config_.tagger) || + !std::filesystem::is_regular_file(mode_config_.tagger) || + !std::filesystem::exists(mode_config_.verbalizer) || + !std::filesystem::is_regular_file(mode_config_.verbalizer)) { SLOGW("Either tagger or verbalizer file does not exist, using alternative lexicon."); lexicon_ = std::make_unique(mode_config_.lexicon, mode_config_.tokens); } else { @@ -281,195 +284,132 @@ public: int dec_len = zp_size / zp_shape[1]; int audio_slice_len = decoder_->GetOutputSize(0) / sizeof(float); - const int pad_frames = 24; - const int samples_per_frame = 512; + const int overlap_size = 1024; + const int fade_size = 512; - const int effective_frames = dec_len - 2 * pad_frames; + int dec_slice_num = static_cast(std::ceil(static_cast(zp_shape[2]) / dec_len)); - int dec_slice_num = - static_cast(std::ceil(static_cast(zp_shape[2]) / static_cast(effective_frames))); - - const int sola_buffer_frame = pad_frames * samples_per_frame; - const int sola_search_frame = pad_frames * samples_per_frame; - const int block_frame = (dec_len - 2 * pad_frames) * samples_per_frame; - - std::vector fade_in_window(sola_buffer_frame); - std::vector fade_out_window(sola_buffer_frame); - - for (int i = 0; i < sola_buffer_frame; i++) { - fade_in_window[i] = static_cast(i) / sola_buffer_frame; - fade_out_window[i] = 1.0f - fade_in_window[i]; + std::vector fade_in(fade_size); + std::vector fade_out(fade_size); + for (int i = 0; i < fade_size; i++) { + float t = static_cast(i) / fade_size; + fade_in[i] = t; + fade_out[i] = 1.0f - t; } - std::vector sola_buffer(sola_buffer_frame, 0.0f); - bool first_frame = true; - std::vector pcmlist; + std::vector previous_tail; for (int i = 0; i < dec_slice_num; i++) { - int input_start = i * effective_frames; - if (i > 0) { - input_start -= pad_frames; - } - input_start = std::max(0, input_start); - - int actual_len = std::min(dec_len, static_cast(zp_shape[2] - input_start)); - - int output_start_frame, output_end_frame; - - if (i == 0) { - output_start_frame = 0; - output_end_frame = effective_frames - 1; - } else if (i == dec_slice_num - 1) { - output_start_frame = i * effective_frames; - output_end_frame = static_cast(zp_shape[2]) - 1; - } else { - output_start_frame = i * effective_frames; - output_end_frame = (i + 1) * effective_frames - 1; - } + int input_start = i * dec_len; + int actual_size = std::min(dec_len, static_cast(zp_shape[2] - input_start)); std::vector zp(zp_size, 0); - for (int n = 0; n < zp_shape[1]; n++) { - int copy_size = std::min(actual_len, static_cast(zp_shape[2] - input_start)); - if (copy_size > 0) { + if (actual_size > 0) { memcpy(zp.data() + n * dec_len, zp_data + n * zp_shape[2] + input_start, - sizeof(float) * copy_size); + sizeof(float) * actual_size); } } - std::vector decoder_output(audio_slice_len); decoder_->SetInput(zp.data(), 0); decoder_->SetInput(g_matrix.data(), 1); if (0 != decoder_->Run()) { + SLOGE("Decoder run failed at slice %d", i); throw std::string("decoder_ RunSync error"); } + std::vector decoder_output(audio_slice_len); decoder_->GetOutput(decoder_output.data(), 0); - if (first_frame) { - int audio_start = 0; - int audio_len = decoder_output.size() - sola_buffer_frame; - audio_len = std::max(0, audio_len); + if (i == 0) { + int main_part_size = static_cast(decoder_output.size()) - overlap_size; + main_part_size = std::max(0, main_part_size); - if (audio_len > 0) { - pcmlist.insert(pcmlist.end(), decoder_output.begin() + audio_start, - decoder_output.begin() + audio_start + audio_len); + pcmlist.insert(pcmlist.end(), decoder_output.begin(), decoder_output.begin() + main_part_size); + + if (decoder_output.size() > main_part_size) { + previous_tail.assign(decoder_output.begin() + main_part_size, decoder_output.end()); } - int buffer_start = audio_len; - - if (buffer_start + sola_buffer_frame <= decoder_output.size()) { - std::copy(decoder_output.begin() + buffer_start, - decoder_output.begin() + buffer_start + sola_buffer_frame, sola_buffer.begin()); - } else { - int available = static_cast(decoder_output.size() - buffer_start); - if (available > 0) { - std::copy(decoder_output.begin() + buffer_start, decoder_output.end(), sola_buffer.begin()); - std::fill(sola_buffer.begin() + available, sola_buffer.end(), 0.0f); - } else { - std::fill(sola_buffer.begin(), sola_buffer.end(), 0.0f); - } - } - - first_frame = false; - } else { - int audio_start = pad_frames * samples_per_frame; - - std::vector search_window(sola_buffer_frame + sola_search_frame); - std::copy(decoder_output.begin() + audio_start, - decoder_output.begin() + audio_start + search_window.size(), search_window.begin()); - - int best_offset = 0; - float best_correlation = -1.0; - - for (int offset = 0; offset <= sola_search_frame; offset++) { - float correlation = 0.0; - float energy = 0.0; - - for (int j = 0; j < sola_buffer_frame; j++) { - correlation += sola_buffer[j] * search_window[j + offset]; - energy += search_window[j + offset] * search_window[j + offset]; - } - - float normalized_correlation = (energy > 1e-8) ? correlation / std::sqrt(energy) : 0.0f; - - if (normalized_correlation > best_correlation) { - best_correlation = normalized_correlation; - best_offset = offset; - } + if (previous_tail.empty()) { + pcmlist.insert(pcmlist.end(), decoder_output.begin(), decoder_output.end()); + continue; } - int aligned_start = audio_start + best_offset; + int blend_size = std::min( + {fade_size, static_cast(previous_tail.size()), static_cast(decoder_output.size())}); - std::vector crossfade_region(sola_buffer_frame); - - for (int j = 0; j < sola_buffer_frame; j++) { - crossfade_region[j] = - decoder_output[aligned_start + j] * fade_in_window[j] + sola_buffer[j] * fade_out_window[j]; + std::vector blended_region(blend_size); + for (int j = 0; j < blend_size; j++) { + blended_region[j] = previous_tail[j] * fade_out[j * fade_size / blend_size] + + decoder_output[j] * fade_in[j * fade_size / blend_size]; } - pcmlist.insert(pcmlist.end(), crossfade_region.begin(), crossfade_region.end()); + pcmlist.insert(pcmlist.end(), blended_region.begin(), blended_region.end()); - int remaining_start = aligned_start + sola_buffer_frame; + if (static_cast(previous_tail.size()) > blend_size) { + pcmlist.insert(pcmlist.end(), previous_tail.begin() + blend_size, previous_tail.end()); + } + + int current_remaining_start = blend_size; + int current_remaining_size = static_cast(decoder_output.size()) - current_remaining_start; if (i == dec_slice_num - 1) { - int total_expected_samples = audio_len * samples_per_frame / 512; - int processed_samples = static_cast(pcmlist.size()); - int remaining_needed = total_expected_samples - processed_samples; - remaining_needed = std::max(0, remaining_needed); - - int remaining_len = - std::min(remaining_needed, static_cast(decoder_output.size() - remaining_start)); - - if (remaining_len > 0) { - pcmlist.insert(pcmlist.end(), decoder_output.begin() + remaining_start, - decoder_output.begin() + remaining_start + remaining_len); - } - - } else { - int remaining_len = (dec_len - 2 * pad_frames) * samples_per_frame - sola_buffer_frame; - remaining_len = - std::min(remaining_len, static_cast(decoder_output.size() - remaining_start)); - - if (remaining_len > 0) { - pcmlist.insert(pcmlist.end(), decoder_output.begin() + remaining_start, - decoder_output.begin() + remaining_start + remaining_len); - } - - int buffer_start = remaining_start + remaining_len; - - if (buffer_start + sola_buffer_frame <= decoder_output.size()) { - std::copy(decoder_output.begin() + buffer_start, - decoder_output.begin() + buffer_start + sola_buffer_frame, sola_buffer.begin()); - } else { - int avail = static_cast(decoder_output.size() - buffer_start); - if (avail > 0) { - std::copy(decoder_output.begin() + buffer_start, decoder_output.end(), - sola_buffer.begin()); - } - std::fill(sola_buffer.begin() + avail, sola_buffer.end(), 0.0f); - } + int total_expected = audio_len; + int current_total = static_cast(pcmlist.size()); + current_remaining_size = std::min(current_remaining_size, total_expected - current_total); } + + if (current_remaining_size > overlap_size && i < dec_slice_num - 1) { + int main_part_size = current_remaining_size - overlap_size; + + pcmlist.insert(pcmlist.end(), decoder_output.begin() + current_remaining_start, + decoder_output.begin() + current_remaining_start + main_part_size); + + previous_tail.assign(decoder_output.begin() + current_remaining_start + main_part_size, + decoder_output.begin() + current_remaining_start + current_remaining_size); + } else { + if (current_remaining_size > 0) { + pcmlist.insert(pcmlist.end(), decoder_output.begin() + current_remaining_start, + decoder_output.begin() + current_remaining_start + current_remaining_size); + } + previous_tail.clear(); + } + } + + if (static_cast(pcmlist.size()) >= audio_len) { + break; } } - if (pcmlist.size() > audio_len) { + if (static_cast(pcmlist.size()) > audio_len) { pcmlist.resize(audio_len); } + float max_val = 0.0f; + int clip_count = 0; + for (float sample : pcmlist) { + max_val = std::max(max_val, std::abs(sample)); + if (std::abs(sample) > 0.95f) clip_count++; + } + double src_ratio = static_cast(mode_config_.audio_rate) / static_cast(mode_config_.mode_rate); - std::vector tmp_pcm((pcmlist.size() * src_ratio + 1)); + std::vector tmp_pcm(static_cast(pcmlist.size() * src_ratio + 1)); int len; - resample_audio(pcmlist.data(), pcmlist.size(), tmp_pcm.data(), &len, src_ratio); wav_pcm_data.reserve(len); - std::transform(tmp_pcm.begin(), tmp_pcm.begin() + len, std::back_inserter(wav_pcm_data), - [](const auto val) { return static_cast(val * INT16_MAX); }); + for (int i = 0; i < len; i++) { + float val = tmp_pcm[i]; + if (std::abs(val) > 0.95f) { + val = val > 0 ? 0.95f : -0.95f; + } + wav_pcm_data.push_back(static_cast(val * INT16_MAX)); + } if (out_callback_) { out_callback_( @@ -478,8 +418,10 @@ public: } } catch (const std::exception &e) { + SLOGE("Exception: %s", e.what()); return true; } catch (...) { + SLOGE("Unknown exception occurred"); return true; } return false; diff --git a/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp b/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp index 1884c92..cf2c1fc 100644 --- a/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp +++ b/projects/llm_framework/main_melotts/src/runner/Lexicon.hpp @@ -37,7 +37,7 @@ public: const std::string& verbalizer_filename) : max_phrase_length(0) { - SLOGD("Dictionary loading: %s Pronunciation table loading: %s tagger_filename: %s verbalizer_filename: %s", + SLOGI("Dictionary loading: %s Pronunciation table loading: %s tagger_filename: %s verbalizer_filename: %s", tokens_filename.c_str(), lexicon_filename.c_str(), tagger_filename.c_str(), verbalizer_filename.c_str()); m_processor = new wetext::Processor(tagger_filename, verbalizer_filename); @@ -92,13 +92,13 @@ public: lexicon["。"] = lexicon["."]; lexicon["!"] = lexicon["!"]; lexicon["?"] = lexicon["?"]; - SLOGD("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(), + SLOGI("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(), max_phrase_length); } Lexicon(const std::string& lexicon_filename, const std::string& tokens_filename) : max_phrase_length(0) { - SLOGD("Dictionary loading: %s Pronunciation table loading: %s", tokens_filename.c_str(), + SLOGI("Dictionary loading: %s Pronunciation table loading: %s", tokens_filename.c_str(), lexicon_filename.c_str()); std::unordered_map tokens; @@ -151,7 +151,7 @@ public: lexicon["。"] = lexicon["."]; lexicon["!"] = lexicon["!"]; lexicon["?"] = lexicon["?"]; - SLOGD("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(), + SLOGI("Dictionary loading complete, containing %zu entries, longest phrase length: %zu", lexicon.size(), max_phrase_length); }