[update] update llm-tts supported kokoro tts

projects/llm_framework/SConstruct

@@ -5,7 +5,7 @@ import shutil
 os.environ['SDK_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'SDK'))
 os.environ['EXT_COMPONENTS_PATH'] = os.path.normpath(str(Path(os.getcwd())/'..'/'..'/'ext_components'))
 
-version = 'v0.1.3'
+version = 'v0.1.5'
 static_lib = 'static_lib'
 update = False
 

projects/llm_framework/main_tts/SConstruct

@@ -8,7 +8,7 @@ with open(env['PROJECT_TOOL_S']) as f:
 SRCS = Glob('src/*.c*')
 INCLUDE = [ADir('include'), ADir('.')]
 PRIVATE_INCLUDE = []
-REQUIREMENTS = ['pthread', 'utilities', 'gomp', 'eventpp', 'StackFlow']
+REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'gomp', 'eventpp', 'StackFlow']
 STATIC_LIB = []
 DYNAMIC_LIB = []
 DEFINITIONS = []
@@ -20,14 +20,18 @@ STATIC_FILES = []
 DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17']
 LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./']
 LINK_SEARCH_PATH += [ADir('../static_lib')]
-REQUIREMENTS += ['tts']
+REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys']
+REQUIREMENTS += ['samplerate']
+REQUIREMENTS += ['tts', 'kokoro', 'libkokoro_backend']
 
 
 INCLUDE += [ADir('src/runner/eigen-3.4.0'), ADir('src/runner/src/tn/header'), ADir('src/runner/include'), ADir('src/runner/src/header')]
 
+REQUIREMENTS += ['onnxruntime']
+
 STATIC_FILES += Glob('mode_*.json')
 
-env['COMPONENTS'].append({'target':'llm_tts-1.6',
+env['COMPONENTS'].append({'target':'llm_tts-1.7',
                           'SRCS':SRCS,
                           'INCLUDE':INCLUDE,
                           'PRIVATE_INCLUDE':PRIVATE_INCLUDE,

projects/llm_framework/main_tts/mode_kokoro-ax650.json

@@ -0,0 +1,33 @@
+{
+    "mode": "kokoro-ax650",
+    "type": "tts",
+    "homepage": "https://huggingface.co/AXERA-TECH/kokoro.axera",
+    "capabilities": [
+        "tts",
+        "Chinese",
+        "English"
+    ],
+    "input_type": [
+        "tts.utf-8"
+    ],
+    "output_type": [
+        "tts.wav",
+        "sys.play.0_1"
+    ],
+    "mode_param": {
+        "axmodel_dir": "./models",
+        "lang": "z",
+        "voice_path": "./voices",
+        "voice_name": "zf_xiaoxiao",
+        "vocab_path": "./vocab.txt",
+        "espeak_data_path": "./espeak-ng-data",
+        "dict_dir": "./dict",
+        "spacker_speed": 1.0,
+        "max_len": 96,
+        "mode_rate": 24000,
+        "audio_rate": 48000,
+        "speed": 1.0,
+        "pause": 0.0,
+        "fade_out": 0.0
+    }
+}

projects/llm_framework/main_tts/src/Kokoro.h

@@ -0,0 +1,68 @@
+/*
+ * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+
+namespace kokoro {
+struct SentenceInfo {
+    std::string sentence;
+    std::vector<int> input_ids;
+    std::string phonemes;
+    int content_len = 0;
+    bool is_long    = false;
+    std::vector<SentenceInfo> sub_results;
+
+    SentenceInfo() = default;
+    SentenceInfo(const std::string &s, const std::vector<int> &ids, const std::string &ph, int len)
+        : sentence(s), input_ids(ids), phonemes(ph), content_len(len), is_long(false)
+    {
+    }
+
+    SentenceInfo(const std::string &s, const std::vector<SentenceInfo> &subs)
+        : sentence(s), is_long(true), sub_results(subs)
+    {
+    }
+};
+
+struct MergedGroup {
+    bool is_long_split = false;
+    std::vector<int> input_ids;
+    std::string phonemes;
+    std::vector<SentenceInfo> sub_results;
+
+    MergedGroup() = default;
+    MergedGroup(const std::vector<int> &ids, const std::string &ph) : is_long_split(false), input_ids(ids), phonemes(ph)
+    {
+    }
+
+    MergedGroup(const std::vector<SentenceInfo> &subs) : is_long_split(true), sub_results(subs)
+    {
+    }
+};
+
+class Kokoro {
+public:
+    Kokoro();
+    ~Kokoro();
+    Kokoro(const Kokoro &)            = delete;
+    Kokoro &operator=(const Kokoro &) = delete;
+
+    bool init(const std::string &model_path, int max_seq_len = 96, const std::string &lang_code = "z",
+              const std::string &voices_path = "./voices", const std::string &voice_name = "af_heart",
+              const std::string &vocab_path       = "dict/vocab.txt",
+              const std::string &espeak_data_path = "./espeak-ng-data", const std::string &dict_dir = "./dict");
+
+    bool tts(const std::string &text, const std::string &voice_name, float speed, int sample_rate, float fade_out,
+             float pause_duration, std::vector<float> &generated_audio);
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+};
+}  // namespace kokoro

projects/llm_framework/main_tts/src/main.cpp

@@ -6,7 +6,9 @@
 #include "StackFlow.h"
 #include "utils.h"
 #include "SynthesizerTrn.h"
-
+#include "Kokoro.h"
+#include <ax_sys_api.h>
+#include <ax_engine_api.h>
 #include <signal.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -14,6 +16,7 @@
 #include <base64.h>
 #include <fstream>
 #include <stdexcept>
+#include <samplerate.h>
 #include "../../../../SDK/components/utilities/include/sample_log.h"
 
 using namespace StackFlows;
@@ -21,13 +24,31 @@ using namespace StackFlows;
 int main_exit_flage = 0;
 static void __sigint(int iSigNo)
 {
-    SLOGW("llm_sys will be exit!");
+    SLOGW("llm_tts will be exit!");
     main_exit_flage = 1;
 }
 
 static std::string base_model_path_;
 static std::string base_model_config_path_;
 
+typedef struct {
+    std::string axmodel_dir;
+    std::string lang;
+    std::string voice_path;
+    std::string voice_name;
+    std::string vocab_path;
+    std::string espeak_data_path;
+    std::string dict_dir;
+    float spacker_speed = 1.0;
+    int max_len         = 96;
+    int mode_rate       = 44100;
+    int audio_rate      = 16000;
+    float speed         = 1.0f;
+    float pause         = 0.0f;
+    float fade_out      = 0.0f;
+
+} kokoro_config;
+
 struct SynthesizerTrn_config {
     int spacker_role    = 0;
     float spacker_speed = 1.0;
@@ -42,29 +63,35 @@ typedef std::function<void(const std::string &data, bool finish)> task_callback_
     else if (obj.contains(#key))              \
         mode_config_.key = obj[#key];
 
+#define CONFIG_KOKORO_AUTO_SET(obj, key)             \
+    if (config_body.contains(#key))                  \
+        mode_kokoro_config_.key = config_body[#key]; \
+    else if (obj.contains(#key))                     \
+        mode_kokoro_config_.key = obj[#key];
+
 class llm_task {
 private:
     float *dataW = NULL;
     int modelSize;
 
 public:
+    kokoro_config mode_kokoro_config_;
     std::unique_ptr<SynthesizerTrn> synthesizer_;
+    std::unique_ptr<kokoro::Kokoro> kokoro_;
+    std::string model_type_;
+    std::string model_;
     SynthesizerTrn_config mode_config_;
+    std::string response_format_;
     std::vector<std::string> inputs_;
     bool enoutput_;
     bool enstream_;
     std::atomic_bool superior_flage_;
     std::string superior_id_;
-    std::string model_;
-    std::string response_format_;
+    static int ax_init_flage_;
     task_callback_t out_callback_;
+    bool enaudio_;
     int awake_delay_ = 1000;
 
-    void set_output(task_callback_t out_callback)
-    {
-        out_callback_ = out_callback;
-    }
-
     bool parse_config(const nlohmann::json &config_body)
     {
         try {
@@ -72,6 +99,13 @@ public:
             response_format_ = config_body.at("response_format");
             enoutput_        = config_body.at("enoutput");
 
+            if (model_.rfind("single-speaker", 0) == 0) {
+                model_type_ = "summer_tts";
+            } else {
+                model_type_ = "kokoro_tts";
+            }
+
+            if (config_body.contains("enaudio")) enaudio_ = config_body.at("enaudio");
             if (config_body.contains("input")) {
                 if (config_body["input"].is_string()) {
                     inputs_.push_back(config_body["input"].get<std::string>());
@@ -85,11 +119,11 @@ public:
             SLOGE("setup config_body error");
             return true;
         }
-        enstream_ = (response_format_.find("stream") != std::string::npos);
+        enstream_ = response_format_.find("stream") == std::string::npos ? false : true;
         return false;
     }
 
-    int load_model(const nlohmann::json &config_body)
+    int load_summer_tts_model(const nlohmann::json &config_body)
     {
         if (parse_config(config_body)) {
             return -1;
@@ -137,24 +171,203 @@ public:
             SLOGI("Available speakers in the model are %d", spkNum);
         } catch (...) {
             SLOGE("config false");
-            return -3;
+            return -6;
         }
         return 0;
     }
 
-    bool TTS(const std::string &msg)
+    int load_kokoro_tts_model(const nlohmann::json &config_body)
+    {
+        nlohmann::json file_body;
+        std::list<std::string> config_file_paths =
+            get_config_file_paths(base_model_path_, base_model_config_path_, model_);
+        try {
+            for (auto file_name : config_file_paths) {
+                std::ifstream config_file(file_name);
+                if (!config_file.is_open()) {
+                    SLOGW("config file :%s miss", file_name.c_str());
+                    continue;
+                }
+                SLOGI("config file :%s read", file_name.c_str());
+                config_file >> file_body;
+                config_file.close();
+                break;
+            }
+            if (file_body.empty()) {
+                SLOGE("all config file miss");
+                return -2;
+            }
+            std::string base_model = base_model_path_ + model_ + "/";
+            SLOGI("base_model %s", base_model.c_str());
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], axmodel_dir);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], lang);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], voice_path);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], voice_name);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], vocab_path);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], espeak_data_path);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], dict_dir);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], spacker_speed);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], max_len);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], mode_rate);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], audio_rate);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], speed);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], pause);
+            CONFIG_KOKORO_AUTO_SET(file_body["mode_param"], fade_out);
+            mode_kokoro_config_.axmodel_dir      = base_model + mode_kokoro_config_.axmodel_dir;
+            mode_kokoro_config_.voice_path       = base_model + mode_kokoro_config_.voice_path;
+            mode_kokoro_config_.vocab_path       = base_model + mode_kokoro_config_.vocab_path;
+            mode_kokoro_config_.espeak_data_path = base_model + mode_kokoro_config_.espeak_data_path;
+            mode_kokoro_config_.dict_dir         = base_model + mode_kokoro_config_.dict_dir;
+            kokoro_                              = std::make_unique<kokoro::Kokoro>();
+            if (!kokoro_->init(mode_kokoro_config_.axmodel_dir, mode_kokoro_config_.max_len, mode_kokoro_config_.lang,
+                               mode_kokoro_config_.voice_path, mode_kokoro_config_.voice_name,
+                               mode_kokoro_config_.vocab_path, mode_kokoro_config_.espeak_data_path,
+                               mode_kokoro_config_.dict_dir)) {
+                SLOGE("encoder init failed!");
+                return -4;
+            }
+        } catch (...) {
+            SLOGE("config false");
+            return -6;
+        }
+        return 0;
+    }
+
+    int load_model(const nlohmann::json &config_body)
+    {
+        if (parse_config(config_body)) {
+            return -1;
+        }
+        if (model_type_ == "summer_tts") {
+            SLOGI("load summer tts model");
+            return load_summer_tts_model(config_body);
+        } else {
+            SLOGI("load kokoro tts model");
+            return load_kokoro_tts_model(config_body);
+        }
+    }
+
+    void set_output(task_callback_t out_callback)
+    {
+        out_callback_ = out_callback;
+    }
+
+    void resample_audio(const float *input_buffer, int input_length, float *output_buffer, int *output_length,
+                        double src_ratio)
+    {
+        int error            = 0;
+        SRC_STATE *src_state = src_new(SRC_SINC_FASTEST, 1, &error);
+        if (!src_state) {
+            SLOGE("src_new failed: %s", src_strerror(error));
+            *output_length = 0;
+            return;
+        }
+
+        SRC_DATA src_data;
+        memset(&src_data, 0, sizeof(src_data));
+        src_data.data_in       = input_buffer;
+        src_data.input_frames  = input_length;
+        src_data.data_out      = output_buffer;
+        src_data.output_frames = static_cast<long>(input_length * src_ratio) + 1;
+        src_data.src_ratio     = src_ratio;
+        src_data.end_of_input  = 1;
+
+        error = src_process(src_state, &src_data);
+        if (error) {
+            SLOGE("src_process failed: %s", src_strerror(error));
+            *output_length = 0;
+            src_delete(src_state);
+            return;
+        }
+
+        *output_length = src_data.output_frames_gen;
+        src_delete(src_state);
+    }
+
+    bool TTS(const std::string &msg, bool finish)
     {
         SLOGI("TTS msg:%s", msg.c_str());
-        int32_t dataLen;
-        int16_t *rawData = synthesizer_->infer(msg, mode_config_.spacker_role, mode_config_.spacker_speed, dataLen);
-        if (!rawData) {
-            SLOGW("tts infer false!");
-            return true;
+        if (msg.empty()) {
+            if (out_callback_) {
+                out_callback_(std::string(), finish);
+            }
+            return false;
         }
-        std::vector<int16_t> wavData(rawData, rawData + dataLen);
-        out_callback_(std::string((char *)rawData, dataLen * sizeof(int16_t)), true);
-        free(rawData);
-        return false;
+
+        if (model_type_ == "summer_tts") {
+            int32_t dataLen  = 0;
+            int16_t *rawData = synthesizer_->infer(msg, mode_config_.spacker_role, mode_config_.spacker_speed, dataLen);
+
+            if (!rawData || dataLen <= 0) {
+                SLOGW("summer tts infer failed!");
+                return true;
+            }
+
+            if (out_callback_) {
+                out_callback_(std::string(reinterpret_cast<char *>(rawData), dataLen * sizeof(int16_t)), finish);
+            }
+
+            free(rawData);
+            return false;
+
+        } else if (model_type_ == "kokoro_tts") {
+            std::vector<float> audio;
+            int src_rate = mode_kokoro_config_.mode_rate;
+            int dst_rate = mode_kokoro_config_.audio_rate;
+
+            if (!kokoro_->tts(msg, mode_kokoro_config_.voice_name, mode_kokoro_config_.speed, src_rate,
+                              mode_kokoro_config_.fade_out, mode_kokoro_config_.pause, audio)) {
+                SLOGE("kokoro tts run failed!");
+                return true;
+            }
+
+            if (audio.empty()) {
+                if (out_callback_) {
+                    out_callback_(std::string(), finish);
+                }
+                SLOGE("kokoro tts audio empty!");
+                return false;
+            }
+
+            std::vector<float> resampled_pcm;
+            const float *pcm_ptr = audio.data();
+            int pcm_len          = static_cast<int>(audio.size());
+
+            if (src_rate != dst_rate) {
+                double ratio = static_cast<double>(dst_rate) / static_cast<double>(src_rate);
+
+                int max_dst_len = static_cast<int>(pcm_len * ratio) + 1;
+                resampled_pcm.resize(max_dst_len);
+
+                int out_len = 0;
+                resample_audio(pcm_ptr, pcm_len, resampled_pcm.data(), &out_len, ratio);
+                resampled_pcm.resize(out_len);
+                pcm_ptr = resampled_pcm.data();
+                pcm_len = out_len;
+            }
+
+            std::vector<int16_t> wav_pcm_data;
+            wav_pcm_data.reserve(pcm_len);
+            for (int i = 0; i < pcm_len; ++i) {
+                float v = pcm_ptr[i];
+
+                if (std::abs(v) > 0.95f) {
+                    v = v > 0 ? 0.95f : -0.95f;
+                }
+
+                wav_pcm_data.push_back(static_cast<int16_t>(v * INT16_MAX));
+            }
+
+            if (out_callback_) {
+                out_callback_(
+                    std::string(reinterpret_cast<char *>(wav_pcm_data.data()), wav_pcm_data.size() * sizeof(int16_t)),
+                    finish);
+            }
+
+            return false;
+        }
+
+        return true;
     }
 
     bool delete_model()
@@ -163,8 +376,38 @@ public:
         return true;
     }
 
+    void _ax_init()
+    {
+        if (!ax_init_flage_) {
+            int ret = AX_SYS_Init();
+            if (0 != ret) {
+                fprintf(stderr, "AX_SYS_Init failed! ret = 0x%x\n", ret);
+            }
+            AX_ENGINE_NPU_ATTR_T npu_attr;
+            memset(&npu_attr, 0, sizeof(npu_attr));
+            ret = AX_ENGINE_Init(&npu_attr);
+            if (0 != ret) {
+                fprintf(stderr, "Init ax-engine failed{0x%8x}.\n", ret);
+            }
+        }
+        ax_init_flage_++;
+    }
+
+    void _ax_deinit()
+    {
+        if (ax_init_flage_ > 0) {
+            --ax_init_flage_;
+            if (!ax_init_flage_) {
+                AX_ENGINE_Deinit();
+                AX_SYS_Deinit();
+            }
+        }
+    }
+
     llm_task(const std::string &workid)
     {
+        enaudio_ = true;
+        _ax_init();
     }
 
     void start()
@@ -178,9 +421,11 @@ public:
     ~llm_task()
     {
         stop();
+        if (kokoro_) kokoro_.reset();
+        _ax_deinit();
     }
 };
-
+int llm_task::ax_init_flage_ = 0;
 #undef CONFIG_AUTO_SET
 
 class llm_tts : public StackFlow {
@@ -203,15 +448,17 @@ public:
             return;
         }
         std::string base64_data;
-        int len = encode_base64(data, base64_data);
+        if (!data.empty()) {
+            int len = encode_base64(data, base64_data);
+        }
         if (llm_channel->enstream_) {
             static int count = 0;
             nlohmann::json data_body;
             data_body["index"] = count++;
-            if (!finish)
+            if (!data.empty())
                 data_body["delta"] = base64_data;
             else
-                data_body["delta"] = std::string("");
+                data_body["delta"] = "";
             data_body["finish"] = finish;
             if (finish) count = 0;
             llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR);
@@ -265,10 +512,10 @@ public:
         int ret;
         std::string tmp_msg1;
         if (enstream) {
-            std::string finish = sample_json_str_get((*next_data), "finish");
-            tmp_msg1           = sample_json_str_get((*next_data), "delta");
-            finish_flage       = (finish == "true") ? true : false;
-            next_data          = &tmp_msg1;
+            std::string finish_str = sample_json_str_get((*next_data), "finish");
+            finish_flage           = (finish_str.find("true") != std::string::npos);
+            tmp_msg1               = sample_json_str_get((*next_data), "delta");
+            next_data              = &tmp_msg1;
         }
         std::string tmp_msg2;
         if (enbase64) {
@@ -284,7 +531,7 @@ public:
             if (cutf8 == "，" || cutf8 == "、" || cutf8 == "," || cutf8 == "。" || cutf8 == "." || cutf8 == "!" ||
                 cutf8 == "！" || cutf8 == "?" || cutf8 == "？" || cutf8 == ";" || cutf8 == "；") {
                 faster_stream_buff += cutf8;
-                ret = llm_task_obj->TTS(faster_stream_buff);
+                ret = llm_task_obj->TTS(faster_stream_buff, false);
                 faster_stream_buff.clear();
                 if (ret) {
                     error_body["code"]    = -11;
@@ -298,7 +545,7 @@ public:
         if (finish_flage) {
             if (!faster_stream_buff.empty()) {
                 faster_stream_buff.push_back('.');
-                ret = llm_task_obj->TTS(faster_stream_buff);
+                ret = llm_task_obj->TTS(faster_stream_buff, true);
                 faster_stream_buff.clear();
                 if (ret) {
                     error_body["code"]    = -11;
@@ -306,6 +553,7 @@ public:
                     llm_channel->send("None", "None", error_body, llm_channel->work_id_);
                 }
             }
+            llm_task_obj->TTS("", true);
         }
     }
 
@@ -323,10 +571,6 @@ public:
             if (llm_task_obj->response_format_.find("sys") != std::string::npos) {
                 unit_call("audio", "queue_play_stop", data);
             }
-            std::this_thread::sleep_for(std::chrono::milliseconds(llm_task_obj->awake_delay_));
-            if (llm_task_obj->response_format_.find("sys") != std::string::npos) {
-                unit_call("audio", "play_stop", data);
-            }
             llm_channel->subscriber_work_id(
                 llm_task_obj->superior_id_,
                 std::bind(&llm_tts::task_user_data, this, std::weak_ptr<llm_task>(llm_task_obj),
@@ -458,7 +702,6 @@ public:
         for (auto it = llm_task_obj->inputs_.begin(); it != llm_task_obj->inputs_.end();) {
             if (*it == data) {
                 it = llm_task_obj->inputs_.erase(it);
-                break;
             } else {
                 ++it;
             }

projects/llm_framework/tools/llm_pack.py

@@ -378,7 +378,7 @@ if __name__ == "__main__":
         'llm-kws':[create_bin_deb,'llm-kws', '1.11', src_folder, revision],
         'llm-asr':[create_bin_deb,'llm-asr', '1.10', src_folder, revision],
         'llm-llm':[create_bin_deb,'llm-llm', '1.12', src_folder, revision],
-        'llm-tts':[create_bin_deb,'llm-tts', '1.6', src_folder, revision],
+        'llm-tts':[create_bin_deb,'llm-tts', '1.7', src_folder, revision],
         'llm-melotts':[create_bin_deb,'llm-melotts', '1.11', src_folder, revision],
         'llm-camera':[create_bin_deb,'llm-camera', '1.9', src_folder, revision, 'lib-llm'],
         'llm-vlm':[create_bin_deb,'llm-vlm', '1.11', src_folder, revision],
@@ -412,6 +412,7 @@ if __name__ == "__main__":
         # TTS model
         'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', '0.3', src_folder, revision],
         'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', '0.3', src_folder, revision],
+        'llm-model-kokoro-ax650':[create_data_deb,'llm-model-kokoro-ax650', '0.4', src_folder, revision],
         # VAD model
         'llm-model-silero-vad':[create_data_deb,'llm-model-silero-vad', '0.4', src_folder, revision],
         # MeloTTS model