From 1fac79e5bb796b91fc42d3fa5dd8d29963f6fc85 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Mon, 26 Jan 2026 18:41:30 +0800 Subject: [PATCH] [update] update llm-tts supported kokoro tts --- projects/llm_framework/SConstruct | 2 +- projects/llm_framework/main_tts/SConstruct | 10 +- .../main_tts/mode_kokoro-ax650.json | 33 ++ projects/llm_framework/main_tts/src/Kokoro.h | 68 ++++ projects/llm_framework/main_tts/src/main.cpp | 317 ++++++++++++++++-- projects/llm_framework/tools/llm_pack.py | 3 +- 6 files changed, 391 insertions(+), 42 deletions(-) create mode 100644 projects/llm_framework/main_tts/mode_kokoro-ax650.json create mode 100644 projects/llm_framework/main_tts/src/Kokoro.h diff --git a/projects/llm_framework/SConstruct b/projects/llm_framework/SConstruct index 731210d..98fe9c0 100644 --- a/projects/llm_framework/SConstruct +++ b/projects/llm_framework/SConstruct @@ -5,7 +5,7 @@ import shutil os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK')) os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components')) -version = 'v0.1.3' +version = 'v0.1.5' static_lib = 'static_lib' update = False diff --git a/projects/llm_framework/main_tts/SConstruct b/projects/llm_framework/main_tts/SConstruct index a38e56f..4ec7405 100644 --- a/projects/llm_framework/main_tts/SConstruct +++ b/projects/llm_framework/main_tts/SConstruct @@ -8,7 +8,7 @@ with open(env['PROJECT_TOOL_S']) as f: SRCS = Glob('src/*.c*') INCLUDE = [ADir('include'), ADir('.')] PRIVATE_INCLUDE = [] -REQUIREMENTS = ['pthread', 'utilities', 'gomp', 'eventpp', 'StackFlow'] +REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'gomp', 'eventpp', 'StackFlow'] STATIC_LIB = [] DYNAMIC_LIB = [] DEFINITIONS = [] @@ -20,14 +20,18 @@ STATIC_FILES = [] DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] LINK_SEARCH_PATH += [ADir('../static_lib')] -REQUIREMENTS += ['tts'] +REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] +REQUIREMENTS += ['samplerate'] +REQUIREMENTS += ['tts', 'kokoro', 'libkokoro_backend'] INCLUDE += [ADir('src/runner/eigen-3.4.0'), ADir('src/runner/src/tn/header'), ADir('src/runner/include'), ADir('src/runner/src/header')] +REQUIREMENTS += ['onnxruntime'] + STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_tts-1.6', +env['COMPONENTS'].append({'target':'llm_tts-1.7', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_tts/mode_kokoro-ax650.json b/projects/llm_framework/main_tts/mode_kokoro-ax650.json new file mode 100644 index 0000000..034ea69 --- /dev/null +++ b/projects/llm_framework/main_tts/mode_kokoro-ax650.json @@ -0,0 +1,33 @@ +{ + "mode": "kokoro-ax650", + "type": "tts", + "homepage": "https://huggingface.co/AXERA-TECH/kokoro.axera", + "capabilities": [ + "tts", + "Chinese", + "English" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "axmodel_dir": "./models", + "lang": "z", + "voice_path": "./voices", + "voice_name": "zf_xiaoxiao", + "vocab_path": "./vocab.txt", + "espeak_data_path": "./espeak-ng-data", + "dict_dir": "./dict", + "spacker_speed": 1.0, + "max_len": 96, + "mode_rate": 24000, + "audio_rate": 48000, + "speed": 1.0, + "pause": 0.0, + "fade_out": 0.0 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_tts/src/Kokoro.h b/projects/llm_framework/main_tts/src/Kokoro.h new file mode 100644 index 0000000..4d05a8b --- /dev/null +++ b/projects/llm_framework/main_tts/src/Kokoro.h @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once + +#include +#include +#include + +namespace kokoro { +struct SentenceInfo { + std::string sentence; + std::vector input_ids; + std::string phonemes; + int content_len = 0; + bool is_long = false; + std::vector sub_results; + + SentenceInfo() = default; + SentenceInfo(const std::string &s, const std::vector &ids, const std::string &ph, int len) + : sentence(s), input_ids(ids), phonemes(ph), content_len(len), is_long(false) + { + } + + SentenceInfo(const std::string &s, const std::vector &subs) + : sentence(s), is_long(true), sub_results(subs) + { + } +}; + +struct MergedGroup { + bool is_long_split = false; + std::vector input_ids; + std::string phonemes; + std::vector sub_results; + + MergedGroup() = default; + MergedGroup(const std::vector &ids, const std::string &ph) : is_long_split(false), input_ids(ids), phonemes(ph) + { + } + + MergedGroup(const std::vector &subs) : is_long_split(true), sub_results(subs) + { + } +}; + +class Kokoro { +public: + Kokoro(); + ~Kokoro(); + Kokoro(const Kokoro &) = delete; + Kokoro &operator=(const Kokoro &) = delete; + + bool init(const std::string &model_path, int max_seq_len = 96, const std::string &lang_code = "z", + const std::string &voices_path = "./voices", const std::string &voice_name = "af_heart", + const std::string &vocab_path = "dict/vocab.txt", + const std::string &espeak_data_path = "./espeak-ng-data", const std::string &dict_dir = "./dict"); + + bool tts(const std::string &text, const std::string &voice_name, float speed, int sample_rate, float fade_out, + float pause_duration, std::vector &generated_audio); + +private: + struct Impl; + std::unique_ptr impl_; +}; +} // namespace kokoro \ No newline at end of file diff --git a/projects/llm_framework/main_tts/src/main.cpp b/projects/llm_framework/main_tts/src/main.cpp index 88ef4a8..42a5ad4 100644 --- a/projects/llm_framework/main_tts/src/main.cpp +++ b/projects/llm_framework/main_tts/src/main.cpp @@ -6,7 +6,9 @@ #include "StackFlow.h" #include "utils.h" #include "SynthesizerTrn.h" - +#include "Kokoro.h" +#include +#include #include #include #include @@ -14,6 +16,7 @@ #include #include #include +#include #include "../../../../SDK/components/utilities/include/sample_log.h" using namespace StackFlows; @@ -21,13 +24,31 @@ using namespace StackFlows; int main_exit_flage = 0; static void __sigint(int iSigNo) { - SLOGW("llm_sys will be exit!"); + SLOGW("llm_tts will be exit!"); main_exit_flage = 1; } static std::string base_model_path_; static std::string base_model_config_path_; +typedef struct { + std::string axmodel_dir; + std::string lang; + std::string voice_path; + std::string voice_name; + std::string vocab_path; + std::string espeak_data_path; + std::string dict_dir; + float spacker_speed = 1.0; + int max_len = 96; + int mode_rate = 44100; + int audio_rate = 16000; + float speed = 1.0f; + float pause = 0.0f; + float fade_out = 0.0f; + +} kokoro_config; + struct SynthesizerTrn_config { int spacker_role = 0; float spacker_speed = 1.0; @@ -42,29 +63,35 @@ typedef std::function task_callback_ else if (obj.contains(#key)) \ mode_config_.key = obj[#key]; +#define CONFIG_KOKORO_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + mode_kokoro_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + mode_kokoro_config_.key = obj[#key]; + class llm_task { private: float *dataW = NULL; int modelSize; public: + kokoro_config mode_kokoro_config_; std::unique_ptr synthesizer_; + std::unique_ptr kokoro_; + std::string model_type_; + std::string model_; SynthesizerTrn_config mode_config_; + std::string response_format_; std::vector inputs_; bool enoutput_; bool enstream_; std::atomic_bool superior_flage_; std::string superior_id_; - std::string model_; - std::string response_format_; + static int ax_init_flage_; task_callback_t out_callback_; + bool enaudio_; int awake_delay_ = 1000; - void set_output(task_callback_t out_callback) - { - out_callback_ = out_callback; - } - bool parse_config(const nlohmann::json &config_body) { try { @@ -72,6 +99,13 @@ public: response_format_ = config_body.at("response_format"); enoutput_ = config_body.at("enoutput"); + if (model_.rfind("single-speaker", 0) == 0) { + model_type_ = "summer_tts"; + } else { + model_type_ = "kokoro_tts"; + } + + if (config_body.contains("enaudio")) enaudio_ = config_body.at("enaudio"); if (config_body.contains("input")) { if (config_body["input"].is_string()) { inputs_.push_back(config_body["input"].get()); @@ -85,11 +119,11 @@ public: SLOGE("setup config_body error"); return true; } - enstream_ = (response_format_.find("stream") != std::string::npos); + enstream_ = response_format_.find("stream") == std::string::npos ? false : true; return false; } - int load_model(const nlohmann::json &config_body) + int load_summer_tts_model(const nlohmann::json &config_body) { if (parse_config(config_body)) { return -1; @@ -137,24 +171,203 @@ public: SLOGI("Available speakers in the model are %d", spkNum); } catch (...) { SLOGE("config false"); - return -3; + return -6; } return 0; } - bool TTS(const std::string &msg) + int load_kokoro_tts_model(const nlohmann::json &config_body) + { + nlohmann::json file_body; + std::list config_file_paths = + get_config_file_paths(base_model_path_, base_model_config_path_, model_); + try { + for (auto file_name : config_file_paths) { + std::ifstream config_file(file_name); + if (!config_file.is_open()) { + SLOGW("config file :%s miss", file_name.c_str()); + continue; + } + SLOGI("config file :%s read", file_name.c_str()); + config_file >> file_body; + config_file.close(); + break; + } + if (file_body.empty()) { + SLOGE("all config file miss"); + return -2; + } + std::string base_model = base_model_path_ + model_ + "/"; + SLOGI("base_model %s", base_model.c_str()); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], axmodel_dir); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], lang); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], voice_path); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], voice_name); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], vocab_path); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], espeak_data_path); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], dict_dir); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], spacker_speed); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], max_len); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], mode_rate); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], audio_rate); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], speed); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], pause); + CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], fade_out); + mode_kokoro_config_.axmodel_dir = base_model + mode_kokoro_config_.axmodel_dir; + mode_kokoro_config_.voice_path = base_model + mode_kokoro_config_.voice_path; + mode_kokoro_config_.vocab_path = base_model + mode_kokoro_config_.vocab_path; + mode_kokoro_config_.espeak_data_path = base_model + mode_kokoro_config_.espeak_data_path; + mode_kokoro_config_.dict_dir = base_model + mode_kokoro_config_.dict_dir; + kokoro_ = std::make_unique(); + if (!kokoro_->init(mode_kokoro_config_.axmodel_dir, mode_kokoro_config_.max_len, mode_kokoro_config_.lang, + mode_kokoro_config_.voice_path, mode_kokoro_config_.voice_name, + mode_kokoro_config_.vocab_path, mode_kokoro_config_.espeak_data_path, + mode_kokoro_config_.dict_dir)) { + SLOGE("encoder init failed!"); + return -4; + } + } catch (...) { + SLOGE("config false"); + return -6; + } + return 0; + } + + int load_model(const nlohmann::json &config_body) + { + if (parse_config(config_body)) { + return -1; + } + if (model_type_ == "summer_tts") { + SLOGI("load summer tts model"); + return load_summer_tts_model(config_body); + } else { + SLOGI("load kokoro tts model"); + return load_kokoro_tts_model(config_body); + } + } + + void set_output(task_callback_t out_callback) + { + out_callback_ = out_callback; + } + + void resample_audio(const float *input_buffer, int input_length, float *output_buffer, int *output_length, + double src_ratio) + { + int error = 0; + SRC_STATE *src_state = src_new(SRC_SINC_FASTEST, 1, &error); + if (!src_state) { + SLOGE("src_new failed: %s", src_strerror(error)); + *output_length = 0; + return; + } + + SRC_DATA src_data; + memset(&src_data, 0, sizeof(src_data)); + src_data.data_in = input_buffer; + src_data.input_frames = input_length; + src_data.data_out = output_buffer; + src_data.output_frames = static_cast(input_length * src_ratio) + 1; + src_data.src_ratio = src_ratio; + src_data.end_of_input = 1; + + error = src_process(src_state, &src_data); + if (error) { + SLOGE("src_process failed: %s", src_strerror(error)); + *output_length = 0; + src_delete(src_state); + return; + } + + *output_length = src_data.output_frames_gen; + src_delete(src_state); + } + + bool TTS(const std::string &msg, bool finish) { SLOGI("TTS msg:%s", msg.c_str()); - int32_t dataLen; - int16_t *rawData = synthesizer_->infer(msg, mode_config_.spacker_role, mode_config_.spacker_speed, dataLen); - if (!rawData) { - SLOGW("tts infer false!"); - return true; + if (msg.empty()) { + if (out_callback_) { + out_callback_(std::string(), finish); + } + return false; } - std::vector wavData(rawData, rawData + dataLen); - out_callback_(std::string((char *)rawData, dataLen * sizeof(int16_t)), true); - free(rawData); - return false; + + if (model_type_ == "summer_tts") { + int32_t dataLen = 0; + int16_t *rawData = synthesizer_->infer(msg, mode_config_.spacker_role, mode_config_.spacker_speed, dataLen); + + if (!rawData || dataLen <= 0) { + SLOGW("summer tts infer failed!"); + return true; + } + + if (out_callback_) { + out_callback_(std::string(reinterpret_cast(rawData), dataLen * sizeof(int16_t)), finish); + } + + free(rawData); + return false; + + } else if (model_type_ == "kokoro_tts") { + std::vector audio; + int src_rate = mode_kokoro_config_.mode_rate; + int dst_rate = mode_kokoro_config_.audio_rate; + + if (!kokoro_->tts(msg, mode_kokoro_config_.voice_name, mode_kokoro_config_.speed, src_rate, + mode_kokoro_config_.fade_out, mode_kokoro_config_.pause, audio)) { + SLOGE("kokoro tts run failed!"); + return true; + } + + if (audio.empty()) { + if (out_callback_) { + out_callback_(std::string(), finish); + } + SLOGE("kokoro tts audio empty!"); + return false; + } + + std::vector resampled_pcm; + const float *pcm_ptr = audio.data(); + int pcm_len = static_cast(audio.size()); + + if (src_rate != dst_rate) { + double ratio = static_cast(dst_rate) / static_cast(src_rate); + + int max_dst_len = static_cast(pcm_len * ratio) + 1; + resampled_pcm.resize(max_dst_len); + + int out_len = 0; + resample_audio(pcm_ptr, pcm_len, resampled_pcm.data(), &out_len, ratio); + resampled_pcm.resize(out_len); + pcm_ptr = resampled_pcm.data(); + pcm_len = out_len; + } + + std::vector wav_pcm_data; + wav_pcm_data.reserve(pcm_len); + for (int i = 0; i < pcm_len; ++i) { + float v = pcm_ptr[i]; + + if (std::abs(v) > 0.95f) { + v = v > 0 ? 0.95f : -0.95f; + } + + wav_pcm_data.push_back(static_cast(v * INT16_MAX)); + } + + if (out_callback_) { + out_callback_( + std::string(reinterpret_cast(wav_pcm_data.data()), wav_pcm_data.size() * sizeof(int16_t)), + finish); + } + + return false; + } + + return true; } bool delete_model() @@ -163,8 +376,38 @@ public: return true; } + void _ax_init() + { + if (!ax_init_flage_) { + int ret = AX_SYS_Init(); + if (0 != ret) { + fprintf(stderr, "AX_SYS_Init failed! ret = 0x%x\n", ret); + } + AX_ENGINE_NPU_ATTR_T npu_attr; + memset(&npu_attr, 0, sizeof(npu_attr)); + ret = AX_ENGINE_Init(&npu_attr); + if (0 != ret) { + fprintf(stderr, "Init ax-engine failed{0x%8x}.\n", ret); + } + } + ax_init_flage_++; + } + + void _ax_deinit() + { + if (ax_init_flage_ > 0) { + --ax_init_flage_; + if (!ax_init_flage_) { + AX_ENGINE_Deinit(); + AX_SYS_Deinit(); + } + } + } + llm_task(const std::string &workid) { + enaudio_ = true; + _ax_init(); } void start() @@ -178,9 +421,11 @@ public: ~llm_task() { stop(); + if (kokoro_) kokoro_.reset(); + _ax_deinit(); } }; - +int llm_task::ax_init_flage_ = 0; #undef CONFIG_AUTO_SET class llm_tts : public StackFlow { @@ -203,15 +448,17 @@ public: return; } std::string base64_data; - int len = encode_base64(data, base64_data); + if (!data.empty()) { + int len = encode_base64(data, base64_data); + } if (llm_channel->enstream_) { static int count = 0; nlohmann::json data_body; data_body["index"] = count++; - if (!finish) + if (!data.empty()) data_body["delta"] = base64_data; else - data_body["delta"] = std::string(""); + data_body["delta"] = ""; data_body["finish"] = finish; if (finish) count = 0; llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); @@ -265,10 +512,10 @@ public: int ret; std::string tmp_msg1; if (enstream) { - std::string finish = sample_json_str_get((*next_data), "finish"); - tmp_msg1 = sample_json_str_get((*next_data), "delta"); - finish_flage = (finish == "true") ? true : false; - next_data = &tmp_msg1; + std::string finish_str = sample_json_str_get((*next_data), "finish"); + finish_flage = (finish_str.find("true") != std::string::npos); + tmp_msg1 = sample_json_str_get((*next_data), "delta"); + next_data = &tmp_msg1; } std::string tmp_msg2; if (enbase64) { @@ -284,7 +531,7 @@ public: if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" || cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") { faster_stream_buff += cutf8; - ret = llm_task_obj->TTS(faster_stream_buff); + ret = llm_task_obj->TTS(faster_stream_buff, false); faster_stream_buff.clear(); if (ret) { error_body["code"] = -11; @@ -298,7 +545,7 @@ public: if (finish_flage) { if (!faster_stream_buff.empty()) { faster_stream_buff.push_back('.'); - ret = llm_task_obj->TTS(faster_stream_buff); + ret = llm_task_obj->TTS(faster_stream_buff, true); faster_stream_buff.clear(); if (ret) { error_body["code"] = -11; @@ -306,6 +553,7 @@ public: llm_channel->send("None", "None", error_body, llm_channel->work_id_); } } + llm_task_obj->TTS("", true); } } @@ -323,10 +571,6 @@ public: if (llm_task_obj->response_format_.find("sys") != std::string::npos) { unit_call("audio", "queue_play_stop", data); } - std::this_thread::sleep_for(std::chrono::milliseconds(llm_task_obj->awake_delay_)); - if (llm_task_obj->response_format_.find("sys") != std::string::npos) { - unit_call("audio", "play_stop", data); - } llm_channel->subscriber_work_id( llm_task_obj->superior_id_, std::bind(&llm_tts::task_user_data, this, std::weak_ptr(llm_task_obj), @@ -458,7 +702,6 @@ public: for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) { if (*it == data) { it = llm_task_obj->inputs_.erase(it); - break; } else { ++it; } diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index e0c6a8e..9ac017e 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -378,7 +378,7 @@ if __name__ == "__main__": 'llm-kws':[create_bin_deb,'llm-kws', '1.11', src_folder, revision], 'llm-asr':[create_bin_deb,'llm-asr', '1.10', src_folder, revision], 'llm-llm':[create_bin_deb,'llm-llm', '1.12', src_folder, revision], - 'llm-tts':[create_bin_deb,'llm-tts', '1.6', src_folder, revision], + 'llm-tts':[create_bin_deb,'llm-tts', '1.7', src_folder, revision], 'llm-melotts':[create_bin_deb,'llm-melotts', '1.11', src_folder, revision], 'llm-camera':[create_bin_deb,'llm-camera', '1.9', src_folder, revision, 'lib-llm'], 'llm-vlm':[create_bin_deb,'llm-vlm', '1.11', src_folder, revision], @@ -412,6 +412,7 @@ if __name__ == "__main__": # TTS model 'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', '0.3', src_folder, revision], 'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', '0.3', src_folder, revision], + 'llm-model-kokoro-ax650':[create_data_deb,'llm-model-kokoro-ax650', '0.4', src_folder, revision], # VAD model 'llm-model-silero-vad':[create_data_deb,'llm-model-silero-vad', '0.4', src_folder, revision], # MeloTTS model