diff --git a/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json b/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json index d4c167f..b3f7669 100644 --- a/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json +++ b/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json @@ -22,8 +22,25 @@ "filename_post_axmodel": "qwen2_post.axmodel", "filename_decoder_axmodel": "llm_decoder.axmodel", "template_filename_axmodel": "qwen2_p128_l%d_together.axmodel", - "token2wav_axmodel_dir": "", - "prompt_files": "zh_man1", + "flow_input_embedding": "flow.input_embedding.float16.bin", + "rand_noise": "rand_noise_1_80_300.txt", + "speech_window": "speech_window_2x8x480.txt", + "flow_encoder_28": "flow_encoder_28.axmodel", + "flow_encoder_53": "flow_encoder_53.axmodel", + "flow_encoder_78": "flow_encoder_78.axmodel", + "flow_encoder_50_final": "flow_encoder_50_final.axmodel", + "flow_estimator_200": "flow_estimator_200.axmodel", + "flow_estimator_250": "flow_estimator_250.axmodel", + "flow_estimator_300": "flow_estimator_300.axmodel", + "hift_p2_50_first": "hift_p2_50_first.axmodel", + "hift_p2_58": "hift_p2_58.axmodel", + "hift_p1_50_first": "hift_p1_50_first.onnx", + "hift_p1_58": "hift_p1_58.onnx", + "prompt_dir": "prompt_data", + "prompt_text": "prompt_text.txt", + "llm_prompt_speech_token": "llm_prompt_speech_token.txt", + "prompt_speech_feat": "prompt_speech_feat.txt", + "flow_embedding": "flow_embedding.txt", "b_use_topk": false, "b_bos": false, "b_eos": false, diff --git a/projects/llm_framework/main_cosy_voice/src/main.cpp b/projects/llm_framework/main_cosy_voice/src/main.cpp index a688e10..5bb1559 100644 --- a/projects/llm_framework/main_cosy_voice/src/main.cpp +++ b/projects/llm_framework/main_cosy_voice/src/main.cpp @@ -6,6 +6,7 @@ #include "StackFlow.h" #include "runner/LLM.hpp" #include "runner/Token2wav.hpp" +#include "runner/utils/wav.hpp" #include #include @@ -46,6 +47,12 @@ typedef std::function task_callback_ else if (obj.contains(#key)) \ mode_config_.key = obj[#key]; +#define INFER_CONFIG_AUTO_SET(obj, key) \ + if (config_body.contains(#key)) \ + infer_mode_config_.key = config_body[#key]; \ + else if (obj.contains(#key)) \ + infer_mode_config_.key = obj[#key]; + class llm_task { private: static std::atomic next_port_; @@ -71,6 +78,7 @@ private: public: enum inference_status { INFERENCE_NONE = 0, INFERENCE_RUNNING }; LLMAttrType mode_config_; + Token2WavAttr infer_mode_config_; std::unique_ptr lLaMa_; std::string model_; std::string response_format_; @@ -185,6 +193,26 @@ public: CONFIG_AUTO_SET(file_body["mode_param"], b_use_mmap_load_embed); CONFIG_AUTO_SET(file_body["mode_param"], b_dynamic_load_axmodel_layer); CONFIG_AUTO_SET(file_body["mode_param"], max_token_len); + CONFIG_AUTO_SET(file_body["mode_param"], output_path); + + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_input_embedding); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], rand_noise); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], speech_window); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_encoder_28); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_encoder_53); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_encoder_78); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_encoder_50_final); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_estimator_200); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_estimator_250); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], flow_estimator_300); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], hift_p2_50_first); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], hift_p2_58); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], hift_p1_50_first); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], hift_p1_58); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], prompt_dir); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], prompt_dir); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], prompt_dir); + INFER_CONFIG_AUTO_SET(file_body["mode_param"], prompt_dir); { auto has_http = [](const std::string &s) { return s.find("http") != std::string::npos; }; @@ -256,8 +284,32 @@ public: mode_config_.template_filename_axmodel = base_model + mode_config_.template_filename_axmodel; mode_config_.filename_speech_embed = base_model + mode_config_.filename_speech_embed; mode_config_.filename_decoder_axmodel = base_model + mode_config_.filename_decoder_axmodel; - mode_config_.token2wav_axmodel_dir = base_model + mode_config_.token2wav_axmodel_dir; - mode_config_.prompt_files = base_model + mode_config_.prompt_files; + + infer_mode_config_.flow_input_embedding = base_model + infer_mode_config_.flow_input_embedding; + infer_mode_config_.rand_noise = base_model + infer_mode_config_.rand_noise; + infer_mode_config_.speech_window = base_model + infer_mode_config_.speech_window; + infer_mode_config_.flow_encoder_28 = base_model + infer_mode_config_.flow_encoder_28; + infer_mode_config_.flow_encoder_53 = base_model + infer_mode_config_.flow_encoder_53; + infer_mode_config_.flow_encoder_78 = base_model + infer_mode_config_.flow_encoder_78; + infer_mode_config_.flow_encoder_50_final = base_model + infer_mode_config_.flow_encoder_50_final; + infer_mode_config_.flow_estimator_200 = base_model + infer_mode_config_.flow_estimator_200; + infer_mode_config_.flow_estimator_250 = base_model + infer_mode_config_.flow_estimator_250; + infer_mode_config_.flow_estimator_300 = base_model + infer_mode_config_.flow_estimator_300; + infer_mode_config_.hift_p2_50_first = base_model + infer_mode_config_.hift_p2_50_first; + infer_mode_config_.hift_p2_58 = base_model + infer_mode_config_.hift_p2_58; + infer_mode_config_.hift_p1_50_first = base_model + infer_mode_config_.hift_p1_50_first; + infer_mode_config_.hift_p1_58 = base_model + infer_mode_config_.hift_p1_58; + + infer_mode_config_.prompt_text = + (fs::path(base_model) / infer_mode_config_.prompt_dir / infer_mode_config_.prompt_text).string(); + infer_mode_config_.llm_prompt_speech_token = + (fs::path(base_model) / infer_mode_config_.prompt_dir / infer_mode_config_.llm_prompt_speech_token) + .string(); + infer_mode_config_.prompt_speech_feat = + (fs::path(base_model) / infer_mode_config_.prompt_dir / infer_mode_config_.prompt_speech_feat).string(); + infer_mode_config_.flow_embedding = + (fs::path(base_model) / infer_mode_config_.prompt_dir / infer_mode_config_.flow_embedding).string(); + mode_config_.runing_callback = [this](int *p_token, int n_token, const char *p_str, float token_per_sec, void *reserve) { if (this->out_callback_) { @@ -265,10 +317,10 @@ public: } }; - readtxt(mode_config_.prompt_files + "/prompt_text.txt", prompt_text_token); - readtxt(mode_config_.prompt_files + "/llm_prompt_speech_token.txt", prompt_speech_token); - readtxt(mode_config_.prompt_files + "/prompt_speech_feat.txt", prompt_feat); - readtxt(mode_config_.prompt_files + "/flow_embedding.txt", spk_embeds); + readtxt(infer_mode_config_.prompt_text, prompt_text_token); + readtxt(infer_mode_config_.llm_prompt_speech_token, prompt_speech_token); + readtxt(infer_mode_config_.prompt_speech_feat, prompt_feat); + readtxt(infer_mode_config_.flow_embedding, spk_embeds); lLaMa_ = std::make_unique(); if (!lLaMa_->Init(mode_config_)) { @@ -276,7 +328,9 @@ public: lLaMa_.reset(); return -2; } - if (!lToken2Wav.Init(mode_config_.token2wav_axmodel_dir, mode_config_.n_timesteps)) { + if (!lToken2Wav.Init(infer_mode_config_)) { + lLaMa_->Deinit(); + lLaMa_.reset(); return -1; } lLaMa_->TextToken2Embeds(prompt_text_token, prompt_text_embeds); @@ -347,6 +401,18 @@ public: src_delete(src_state); } + static std::string generateFilename(const fs::path &dir) + { + auto now = std::chrono::system_clock::now(); + std::time_t now_time = std::chrono::system_clock::to_time_t(now); + std::tm tm_now; + localtime_r(&now_time, &tm_now); + + std::ostringstream oss; + oss << "audio_" << std::put_time(&tm_now, "%Y%m%d_%H%M%S") << ".wav"; + return (dir / oss.str()).string(); + } + int tts(const std::string &text, std::vector &prompt_text_embeds, std::vector &prompt_speech_embeds, const std::vector &prompt_feat, const std::vector &prompt_speech_embeds_flow, std::vector &spk_embeds) @@ -363,7 +429,7 @@ public: std::thread llm_thread(llm_thread_func); int token_offset = 0; - int prompt_token_len = prompt_speech_embeds_flow.size() / lToken2Wav.flow_embed_size; + int prompt_token_len = prompt_speech_embeds_flow.size() / lToken2Wav._attr.flow_embed_size; if (prompt_token_len < 75) { SLOGE("Error, prompt speech token len %d < 75", prompt_token_len); return -1; @@ -382,26 +448,27 @@ public: int this_token_hop_len; int i = 0; while (true) { - this_token_hop_len = - (token_offset == 0) ? lToken2Wav.token_hop_len + promot_token_pad : lToken2Wav.token_hop_len; + this_token_hop_len = (token_offset == 0) ? lToken2Wav._attr.token_hop_len + promot_token_pad + : lToken2Wav._attr.token_hop_len; std::unique_lock lock(g_buffer_mutex); g_buffer_cv.wait(lock, [&] { return (g_token_buffer.size() - token_offset >= - this_token_hop_len + lToken2Wav.pre_lookahead_len) || + this_token_hop_len + lToken2Wav._attr.pre_lookahead_len) || g_llm_finished.load() || g_stop.load(); }); if (g_stop) { lock.unlock(); break; - } else if (g_token_buffer.size() - token_offset >= this_token_hop_len + lToken2Wav.pre_lookahead_len) { + } else if (g_token_buffer.size() - token_offset >= + this_token_hop_len + lToken2Wav._attr.pre_lookahead_len) { std::vector token; - int start = token_offset - std::min(int(token_offset / lToken2Wav.token_hop_len), - lToken2Wav.max_infer_chunk_num - 1) * - lToken2Wav.token_hop_len; - int end = token_offset + this_token_hop_len + lToken2Wav.pre_lookahead_len; + int start = token_offset - std::min(int(token_offset / lToken2Wav._attr.token_hop_len), + lToken2Wav._attr.max_infer_chunk_num - 1) * + lToken2Wav._attr.token_hop_len; + int end = token_offset + this_token_hop_len + lToken2Wav._attr.pre_lookahead_len; token.insert(token.end(), g_token_buffer.begin() + start, g_token_buffer.begin() + end); @@ -448,12 +515,13 @@ public: } std::vector token; - int start = g_token_buffer.size() - std::min(int(g_token_buffer.size() / lToken2Wav.token_hop_len), - lToken2Wav.max_infer_chunk_num - 1) * - lToken2Wav.token_hop_len; + int start = g_token_buffer.size() - std::min(int(g_token_buffer.size() / lToken2Wav._attr.token_hop_len), + lToken2Wav._attr.max_infer_chunk_num - 1) * + lToken2Wav._attr.token_hop_len; token.insert(token.end(), g_token_buffer.begin() + start, g_token_buffer.end()); auto speech = lToken2Wav.infer(token, prompt_speech_embeds_flow1, prompt_feat1, spk_embeds, token_offset - start, true); + output.insert(output.end(), speech.begin(), speech.end()); double src_ratio = static_cast(mode_config_.audio_rate) / static_cast(mode_config_.mode_rate); std::vector resampled_pcm(static_cast(speech.size() * src_ratio + 1)); @@ -473,6 +541,35 @@ public: std::string(reinterpret_cast(wav_pcm_data.data()), wav_pcm_data.size() * sizeof(int16_t)), true); } + if (response_format_.find("file") != std::string::npos) { + double src_ratio = + static_cast(mode_config_.audio_rate) / static_cast(mode_config_.mode_rate); + std::vector resampled_pcm(static_cast(output.size() * src_ratio + 1)); + int resampled_len = 0; + resample_audio(output.data(), output.size(), resampled_pcm.data(), &resampled_len, src_ratio); + + std::vector wav_pcm_data_full; + wav_pcm_data_full.reserve(resampled_len); + for (int i = 0; i < resampled_len; i++) { + float val = resampled_pcm[i]; + if (val > 1.0f) val = 1.0f; + if (val < -1.0f) val = -1.0f; + wav_pcm_data_full.push_back(static_cast(val * 32767.0f)); + } + + std::string wav_path; + if (mode_config_.output_path.empty()) { + wav_path = generateFilename("/tmp"); + } else { + fs::path user_path(mode_config_.output_path); + if (!user_path.has_filename()) { + wav_path = generateFilename(user_path); + } else { + wav_path = user_path.string(); + } + } + saveVectorAsWavFloat(resampled_pcm, wav_path, mode_config_.audio_rate, 1); + } SLOGI("tts total use time: %.3f s", time_total.cost() / 1000); reset(); diff --git a/projects/llm_framework/main_cosy_voice/src/runner/LLM.hpp b/projects/llm_framework/main_cosy_voice/src/runner/LLM.hpp index 0b35885..b77d7f0 100644 --- a/projects/llm_framework/main_cosy_voice/src/runner/LLM.hpp +++ b/projects/llm_framework/main_cosy_voice/src/runner/LLM.hpp @@ -36,6 +36,7 @@ struct LLMAttrType { std::string filename_tokenizer_model = "http://127.0.0.1:12345"; std::string filename_decoder_axmodel; std::string token2wav_axmodel_dir; + std::string output_path; int prefill_token_num = 96; // auto calc int prefill_max_token_num = 512; diff --git a/projects/llm_framework/main_cosy_voice/src/runner/Token2wav.hpp b/projects/llm_framework/main_cosy_voice/src/runner/Token2wav.hpp index 1723931..dc5c12a 100644 --- a/projects/llm_framework/main_cosy_voice/src/runner/Token2wav.hpp +++ b/projects/llm_framework/main_cosy_voice/src/runner/Token2wav.hpp @@ -6,8 +6,10 @@ #include #include #include -#include // For size_t -#include // For std::invalid_argument +#include +#include +#include + #include "bfloat16.hpp" #include "Tokenizer/Tokenizer.hpp" #include "LLMEmbedSelector.hpp" @@ -18,199 +20,126 @@ #include "ax_cmm_utils.hpp" #include "cqdm.h" #include "timer.hpp" -// #include "opencv2/opencv.hpp" #include "axcl_manager.h" #include "BaseRunner.hpp" -class Token2Wav -{ -public: - int devid = 0; - int flow_embed_num = 6561; - int flow_embed_size = 512; - int token_mel_ratio = 2; - int token_hop_len = 25; - int max_infer_chunk_num = 3; - int mel_cache_len = 8; - int source_cache_len = mel_cache_len * 480; - int pre_lookahead_len = 3; +struct Token2WavAttr { + int devid = 0; + int flow_embed_num = 6561; + int flow_embed_size = 512; + int token_mel_ratio = 2; + int token_hop_len = 25; + int max_infer_chunk_num = 3; + int mel_cache_len = 8; + int source_cache_len = mel_cache_len * 480; + int pre_lookahead_len = 3; float inference_cfg_rate = 0.7; + std::string flow_input_embedding = "flow.input_embedding.float16.bin"; + std::string rand_noise = "rand_noise_1_80_300.txt"; + std::string speech_window = "speech_window_2x8x480.txt"; + std::string flow_encoder_28 = "flow_encoder_28.axmodel"; + std::string flow_encoder_53 = "flow_encoder_53.axmodel"; + std::string flow_encoder_78 = "flow_encoder_78.axmodel"; + std::string flow_encoder_50_final = "flow_encoder_50_final.axmodel"; + std::string flow_estimator_200 = "flow_estimator_200.axmodel"; + std::string flow_estimator_250 = "flow_estimator_250.axmodel"; + std::string flow_estimator_300 = "flow_estimator_300.axmodel"; + std::string hift_p2_50_first = "hift_p2_50_first.axmodel"; + std::string hift_p2_58 = "hift_p2_58.axmodel"; + std::string hift_p1_50_first = "hift_p1_50_first.onnx"; + std::string hift_p1_58 = "hift_p1_58.onnx"; + std::string prompt_dir = "prompt_data/"; + std::string prompt_text = "prompt_text.txt"; + std::string llm_prompt_speech_token = "llm_prompt_speech_token.txt"; + std::string prompt_speech_feat = "prompt_speech_feat.txt"; + std::string flow_embedding = "flow_embedding.txt"; + int n_timesteps = 10; +}; + +class Token2Wav { +public: + Token2WavAttr _attr; + private: ax_runner_ax650 flow_encoder_28; ax_runner_ax650 flow_encoder_53; ax_runner_ax650 flow_encoder_78; ax_runner_ax650 flow_encoder_50_final; - ax_runner_ax650 flow_estimator_200; ax_runner_ax650 flow_estimator_250; ax_runner_ax650 flow_estimator_300; - ax_runner_ax650 hift_p2_50_first; ax_runner_ax650 hift_p2_58; - - std::shared_ptr hift_p1_50_first; std::shared_ptr hift_p1_58; - std::vector rand_noise; std::vector t_span; - LLaMaEmbedSelector flow_embed_selector; - std::unordered_map> hift_cache_dict; - std::vector speech_window; // np.hamming(2 * 8 * 480) + std::vector speech_window; - int init_noise(std::string model_dir) + int init_noise(const std::string &path) { - return readtxt(model_dir+"/rand_noise_1_80_300.txt", rand_noise); + return readtxt(path, rand_noise); } - - int init_speech_window(std::string model_dir) + int init_speech_window(const std::string &path) { - return readtxt(model_dir+"/speech_window_2x8x480.txt", speech_window); + return readtxt(path, speech_window); } - int init_tspan(int n_timesteps) { - // std::vector t_span_10 = {0.0000, 0.0123, 0.0489, 0.1090, 0.1910, 0.2929, 0.4122, 0.5460, 0.6910,0.8436, 1.0000}; - // std::vector t_span_7 = {0.0000, 0.1429, 0.2857, 0.4286, 0.5714, 0.7143, 0.8571, 1.0000}; // n_timesteps = 7 - // std::vector t_span_6 = {0.0000, 0.1667, 0.3333, 0.5000, 0.6667, 0.8333, 1.0000}; // n_timesteps = 6 - // std::vector t_span_5 = {0.0000, 0.2000, 0.4000, 0.6000, 0.8000, 1.0000 }; - // std::vector t_span_4 = {0.0000, 0.2500, 0.5000, 0.7500, 1.0000}; - - if(n_timesteps <4) - { - return -1; - } - - n_timesteps = n_timesteps; + if (n_timesteps < 4) return -1; t_span = linspace(0.0, 1.0, n_timesteps + 1); return 0; } public: - bool Init(std::string model_dir, int n_timesteps) + bool Init(const Token2WavAttr &attr) { - int ret; + _attr = attr; - ret = init_tspan(n_timesteps); - if(ret != 0){ - ALOGE("init_tspan failed, n_timesteps:%d", n_timesteps); + if (init_tspan(_attr.n_timesteps) != 0) { + ALOGE("init_tspan failed, n_timesteps:%d", _attr.n_timesteps); return false; } - - ret = init_noise(model_dir); - if(ret != 0){ - ALOGE("init rand noise(%s) failed", "rand_noise_1_80_300.txt"); + if (init_noise(_attr.rand_noise) != 0) { + ALOGE("init rand noise failed"); return false; } - - ret = init_speech_window(model_dir); - if(ret != 0){ - ALOGE("init speech_window(%s) failed", "speech_window_2x8x480.txt"); + if (init_speech_window(_attr.speech_window) != 0) { + ALOGE("init speech_window failed"); return false; } - - if (!flow_embed_selector.Init((model_dir+"/flow.input_embedding.float16.bin").c_str(), flow_embed_num, flow_embed_size, false)) - { - ALOGE("flow_embed_selector.Init(%s, %d, %d) failed", (model_dir+"/flow.input_embedding.float16.bin").c_str(),flow_embed_num, flow_embed_size); - return false; - } - - // if (axcl_Init(devid) != 0) - // { - // ALOGE("axcl_Init(%d) failed", devid); - // return false; - // } - - ret = flow_encoder_28.init((model_dir+"/flow_encoder_28.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/flow_encoder_28.axmodel").c_str()); - return false; - } - - ret = flow_encoder_53.init((model_dir+"/flow_encoder_53.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/flow_encoder_53.axmodel").c_str()); - return false; - } - - ret = flow_encoder_78.init((model_dir+"/flow_encoder_78.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/flow_encoder_78.axmodel").c_str()); - return false; - } - - ret = flow_encoder_50_final.init((model_dir+"/flow_encoder_50_final.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/flow_encoder_50_final.axmodel").c_str()); - return false; - } - - ret = flow_estimator_200.init((model_dir+"/flow_estimator_200.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/flow_estimator_200.axmodel").c_str()); - return false; - } - - ret = flow_estimator_250.init((model_dir+"/flow_estimator_250.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/flow_estimator_250.axmodel").c_str()); - return false; - } - - ret = flow_estimator_300.init((model_dir+"/flow_estimator_300.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/flow_estimator_300.axmodel").c_str()); - return false; - } - - ret = hift_p2_50_first.init((model_dir+"/hift_p2_50_first.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/hift_p2_50_first.axmodel").c_str()); - return false; - } - - ret = hift_p2_58.init((model_dir+"/hift_p2_58.axmodel").c_str(), devid); - if (ret != 0) - { - ALOGE("init axmodel(%s) failed", (model_dir+"/hift_p2_58.axmodel").c_str()); + if (!flow_embed_selector.Init(_attr.flow_input_embedding.c_str(), _attr.flow_embed_num, _attr.flow_embed_size, + false)) { + ALOGE("flow_embed_selector init failed"); return false; } + if (flow_encoder_28.init(_attr.flow_encoder_28.c_str(), _attr.devid) != 0) return false; + if (flow_encoder_53.init(_attr.flow_encoder_53.c_str(), _attr.devid) != 0) return false; + if (flow_encoder_78.init(_attr.flow_encoder_78.c_str(), _attr.devid) != 0) return false; + if (flow_encoder_50_final.init(_attr.flow_encoder_50_final.c_str(), _attr.devid) != 0) return false; + if (flow_estimator_200.init(_attr.flow_estimator_200.c_str(), _attr.devid) != 0) return false; + if (flow_estimator_250.init(_attr.flow_estimator_250.c_str(), _attr.devid) != 0) return false; + if (flow_estimator_300.init(_attr.flow_estimator_300.c_str(), _attr.devid) != 0) return false; + if (hift_p2_50_first.init(_attr.hift_p2_50_first.c_str(), _attr.devid) != 0) return false; + if (hift_p2_58.init(_attr.hift_p2_58.c_str(), _attr.devid) != 0) return false; hift_p1_50_first = CreateRunner(RT_OnnxRunner); - if(hift_p1_50_first == nullptr) - { - ALOGE("init hift_p1_50_first failed"); - return false; - } - BaseConfig config_50; - config_50.nthread = 8; - config_50.onnx_model = model_dir+"/hift_p1_50_first.onnx"; - hift_p1_50_first->load(config_50); + if (!hift_p1_50_first) return false; + BaseConfig cfg50; + cfg50.nthread = 8; + cfg50.onnx_model = _attr.hift_p1_50_first; + hift_p1_50_first->load(cfg50); hift_p1_58 = CreateRunner(RT_OnnxRunner); - if(hift_p1_58 == nullptr) - { - ALOGE("init hift_p1_58 failed"); - return false; - } - BaseConfig config_58; - config_58.nthread = 8; - config_58.onnx_model = model_dir+"/hift_p1_58.onnx"; - hift_p1_58->load(config_58); + if (!hift_p1_58) return false; + BaseConfig cfg58; + cfg58.nthread = 8; + cfg58.onnx_model = _attr.hift_p1_58; + hift_p1_58->load(cfg58); - int remain_cmm = axcl_GetCMMRemain(devid); - ALOGI("Token2Wav init ok, remain_cmm(%d MB)", remain_cmm); return true; } @@ -226,7 +155,6 @@ public: hift_p2_50_first.release(); hift_p2_58.release(); flow_embed_selector.Deinit(); - // axcl_Exit(devid); } void SetTimesteps(int n_timesteps) @@ -234,218 +162,216 @@ public: init_tspan(n_timesteps); } - int SpeechToken2Embeds(std::vector & token_ids, std::vector &token_embeds) - { - if(token_embeds.empty() || token_embeds.size() != token_ids.size()* flow_embed_size) - { - token_embeds.resize(token_ids.size()* flow_embed_size); + int SpeechToken2Embeds(std::vector &token_ids, std::vector &token_embeds) + { + if (token_embeds.empty() || token_embeds.size() != token_ids.size() * _attr.flow_embed_size) { + token_embeds.resize(token_ids.size() * _attr.flow_embed_size); } - std::vector speech_embeds_one(flow_embed_size); - for (size_t i = 0; i < token_ids.size(); i++) - { + std::vector speech_embeds_one(_attr.flow_embed_size); + for (size_t i = 0; i < token_ids.size(); i++) { flow_embed_selector.getByIndex(token_ids[i], speech_embeds_one.data()); - for (int j = 0; j < flow_embed_size; j++) - { - unsigned int proc = speech_embeds_one[j] << 16; - token_embeds[i * flow_embed_size + j] = *reinterpret_cast(&proc); - } + for (int j = 0; j < _attr.flow_embed_size; j++) { + unsigned int proc = speech_embeds_one[j] << 16; + token_embeds[i * _attr.flow_embed_size + j] = *reinterpret_cast(&proc); + } } return token_embeds.size(); } - int infer_flow_encoder( - std::vector & token_embeds, std::vector & prompt_feat, std::vector & spk_embeds, int token_len, bool finalize, - std::vector & mu, std::vector & spks, std::vector & cond - ) + int infer_flow_encoder(std::vector &token_embeds, std::vector &prompt_feat, + std::vector &spk_embeds, int token_len, bool finalize, std::vector &mu, + std::vector &spks, std::vector &cond) { - ax_runner_ax650 * model; - if(!finalize) - { - if(token_len == 28) - { + ax_runner_ax650 *model; + if (!finalize) { + if (token_len == 28) { model = &flow_encoder_28; - }else if(token_len == 53) - { + } else if (token_len == 53) { model = &flow_encoder_53; - }else if(token_len == 78) - { + } else if (token_len == 78) { model = &flow_encoder_78; - }else{ + } else { return -1; } - }else if(token_len == 50){ + } else if (token_len == 50) { model = &flow_encoder_50_final; - }else{ + } else { return -1; } - void * p = (void *)model->get_input("token_embedding").phyAddr; - axcl_Memcpy(p, token_embeds.data(), token_embeds.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + void *p = (void *)model->get_input("token_embedding").phyAddr; + axcl_Memcpy(p, token_embeds.data(), token_embeds.size() * sizeof(float), + axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); p = (void *)model->get_input("prompt_feat").phyAddr; - axcl_Memcpy(p, prompt_feat.data(), prompt_feat.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, prompt_feat.data(), prompt_feat.size() * sizeof(float), + axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); p = (void *)model->get_input("embedding").phyAddr; - axcl_Memcpy(p, spk_embeds.data(), spk_embeds.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, spk_embeds.data(), spk_embeds.size() * sizeof(float), + axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); model->inference(); auto &output_mu = model->get_output("mu"); - if(mu.empty()) - { + if (mu.empty()) { mu.resize(output_mu.nSize / sizeof(float)); } - // axcl_Memcpy(mu.data(), (void *)output_mu.phyAddr, output_mu.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); - axcl_Memcpy((void *)output_mu.pVirAddr, (void *)output_mu.phyAddr, output_mu.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + // axcl_Memcpy(mu.data(), (void *)output_mu.phyAddr, output_mu.nSize, + // axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + axcl_Memcpy((void *)output_mu.pVirAddr, (void *)output_mu.phyAddr, output_mu.nSize, + axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, _attr.devid); memcpy(mu.data(), (void *)output_mu.pVirAddr, output_mu.nSize); auto &output_spks = model->get_output("spks"); - if(spks.empty()) - { + if (spks.empty()) { spks.resize(output_spks.nSize / sizeof(float)); } - // axcl_Memcpy(spks.data(), (void *)output_spks.phyAddr, output_spks.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); - axcl_Memcpy((void *)output_spks.pVirAddr, (void *)output_spks.phyAddr, output_spks.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + // axcl_Memcpy(spks.data(), (void *)output_spks.phyAddr, output_spks.nSize, + // axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + axcl_Memcpy((void *)output_spks.pVirAddr, (void *)output_spks.phyAddr, output_spks.nSize, + axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, _attr.devid); memcpy(spks.data(), (void *)output_spks.pVirAddr, output_spks.nSize); auto &output_cond = model->get_output("cond"); - if(cond.empty()) - { + if (cond.empty()) { cond.resize(output_cond.nSize / sizeof(float)); } - // axcl_Memcpy(cond.data(), (void *)output_cond.phyAddr, output_cond.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); - axcl_Memcpy((void *)output_cond.pVirAddr, (void *)output_cond.phyAddr, output_cond.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + // axcl_Memcpy(cond.data(), (void *)output_cond.phyAddr, output_cond.nSize, + // axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + axcl_Memcpy((void *)output_cond.pVirAddr, (void *)output_cond.phyAddr, output_cond.nSize, + axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, _attr.devid); mempcpy(cond.data(), (void *)output_cond.pVirAddr, output_cond.nSize); return 0; } - int infer_flow_estimator( - std::vector & x, std::vector & mask, std::vector & t, - std::vector & mu, std::vector & spks, std::vector & cond, - std::vector & dphi_dt - ) + int infer_flow_estimator(std::vector &x, std::vector &mask, std::vector &t, + std::vector &mu, std::vector &spks, std::vector &cond, + std::vector &dphi_dt) { - ax_runner_ax650 * model; - int len = x.size()/(2*80); - if(len == 200){ + ax_runner_ax650 *model; + int len = x.size() / (2 * 80); + if (len == 200) { model = &flow_estimator_200; - }else if(len == 250){ + } else if (len == 250) { model = &flow_estimator_250; - }else if(len == 300){ + } else if (len == 300) { model = &flow_estimator_300; - }else{ + } else { return -1; } - void * p = (void *)model->get_input("x").phyAddr; - axcl_Memcpy(p, x.data(), x.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + void *p = (void *)model->get_input("x").phyAddr; + axcl_Memcpy(p, x.data(), x.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); p = (void *)model->get_input("mask").phyAddr; - axcl_Memcpy(p, mask.data(), mask.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, mask.data(), mask.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, + _attr.devid); p = (void *)model->get_input("t").phyAddr; - axcl_Memcpy(p, t.data(), t.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, t.data(), t.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); p = (void *)model->get_input("mu").phyAddr; - axcl_Memcpy(p, mu.data(), mu.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, mu.data(), mu.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); p = (void *)model->get_input("spks").phyAddr; - axcl_Memcpy(p, spks.data(), spks.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, spks.data(), spks.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, + _attr.devid); p = (void *)model->get_input("cond").phyAddr; - axcl_Memcpy(p, cond.data(), cond.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, cond.data(), cond.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, + _attr.devid); model->inference(); auto &output_dphi_dt = model->get_output("y"); - if(dphi_dt.empty() || dphi_dt.size() != output_dphi_dt.nSize / sizeof(float)) - { + if (dphi_dt.empty() || dphi_dt.size() != output_dphi_dt.nSize / sizeof(float)) { dphi_dt.resize(output_dphi_dt.nSize / sizeof(float)); } - // axcl_Memcpy(dphi_dt.data(), (void *)output_dphi_dt.phyAddr, output_dphi_dt.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); - axcl_Memcpy((void *)output_dphi_dt.pVirAddr, (void *)output_dphi_dt.phyAddr, output_dphi_dt.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + // axcl_Memcpy(dphi_dt.data(), (void *)output_dphi_dt.phyAddr, output_dphi_dt.nSize, + // axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + axcl_Memcpy((void *)output_dphi_dt.pVirAddr, (void *)output_dphi_dt.phyAddr, output_dphi_dt.nSize, + axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, _attr.devid); memcpy(dphi_dt.data(), (void *)output_dphi_dt.pVirAddr, output_dphi_dt.nSize); return 0; } - int infer_hift(std::vector &mel, std::vector &cache_source, - std::vector & tts_speech, std::vector & tts_source) + int infer_hift(std::vector &mel, std::vector &cache_source, std::vector &tts_speech, + std::vector &tts_source) { std::shared_ptr model_p1; - ax_runner_ax650 * model_p2; - int len = mel.size()/(80); - - if(len == 50 && cache_source.empty()) - { + ax_runner_ax650 *model_p2; + int len = mel.size() / (80); + + if (len == 50 && cache_source.empty()) { model_p1 = hift_p1_50_first; model_p2 = &hift_p2_50_first; - }else if(len == 58 && !cache_source.empty()) - { + } else if (len == 58 && !cache_source.empty()) { model_p1 = hift_p1_58; model_p2 = &hift_p2_58; - }else - { + } else { ALOGE("invalid size: %d", len); return -1; } - float * p_input = (float *)model_p1->getInputPtr(0); - memcpy(p_input, mel.data(), mel.size()*sizeof(float)); + float *p_input = (float *)model_p1->getInputPtr(0); + memcpy(p_input, mel.data(), mel.size() * sizeof(float)); model_p1->inference(); auto p_s = model_p2->get_input("s"); memcpy(p_s.pVirAddr, (void *)model_p1->getOutputPtr(0), len * 480 * sizeof(float)); - axcl_Memcpy((void *)p_s.phyAddr, (void *)p_s.pVirAddr, len * 480 * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy((void *)p_s.phyAddr, (void *)p_s.pVirAddr, len * 480 * sizeof(float), + axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); - void * p = (void *)model_p2->get_input("mel").phyAddr; - axcl_Memcpy(p, mel.data(), mel.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); - - if(!cache_source.empty()) - { + void *p = (void *)model_p2->get_input("mel").phyAddr; + axcl_Memcpy(p, mel.data(), mel.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, + _attr.devid); + + if (!cache_source.empty()) { p = (void *)model_p2->get_input("hift_cache_source").phyAddr; - axcl_Memcpy(p, cache_source.data(), cache_source.size() * sizeof(float), axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, devid); + axcl_Memcpy(p, cache_source.data(), cache_source.size() * sizeof(float), + axclrtMemcpyKind::AXCL_MEMCPY_HOST_TO_DEVICE, _attr.devid); } - + model_p2->inference(); - + auto &output_speech = model_p2->get_output("audio"); - if(tts_speech.empty() || tts_speech.size() != output_speech.nSize / sizeof(float)) - { + if (tts_speech.empty() || tts_speech.size() != output_speech.nSize / sizeof(float)) { tts_speech.resize(output_speech.nSize / sizeof(float)); } - // axcl_Memcpy(tts_speech.data(), (void *)output_speech.phyAddr, output_speech.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); - axcl_Memcpy((void *)output_speech.pVirAddr, (void *)output_speech.phyAddr, output_speech.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + // axcl_Memcpy(tts_speech.data(), (void *)output_speech.phyAddr, output_speech.nSize, + // axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + axcl_Memcpy((void *)output_speech.pVirAddr, (void *)output_speech.phyAddr, output_speech.nSize, + axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, _attr.devid); memcpy(tts_speech.data(), (void *)output_speech.pVirAddr, output_speech.nSize); auto &output_source = model_p2->get_output(1); - if(tts_source.empty() || tts_source.size() != output_source.nSize / sizeof(float)) - { + if (tts_source.empty() || tts_source.size() != output_source.nSize / sizeof(float)) { tts_source.resize(output_source.nSize / sizeof(float)); } - // axcl_Memcpy(tts_source.data(), (void *)output_source.phyAddr, output_source.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); - axcl_Memcpy((void *)output_source.pVirAddr, (void *)output_source.phyAddr, output_source.nSize, axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + // axcl_Memcpy(tts_source.data(), (void *)output_source.phyAddr, output_source.nSize, + // axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, devid); + axcl_Memcpy((void *)output_source.pVirAddr, (void *)output_source.phyAddr, output_source.nSize, + axclrtMemcpyKind::AXCL_MEMCPY_DEVICE_TO_HOST, _attr.devid); memcpy(tts_source.data(), (void *)output_source.pVirAddr, output_source.nSize); return 0; } - int infer_flow_decoder_solve_euler( - std::vector & x, std::vector & mu, std::vector & spks, std::vector & cond, std::vector & mask, - std::vector & mel - ) + int infer_flow_decoder_solve_euler(std::vector &x, std::vector &mu, std::vector &spks, + std::vector &cond, std::vector &mask, std::vector &mel) { - int len = mu.size()/80; + int len = mu.size() / 80; - float t = t_span[0]; + float t = t_span[0]; float dt = t_span[1] - t_span[0]; - std::vector x_in(2*80*len, 0); - std::vector mask_in(2*1*len, 0); - std::vector mu_in(2*80*len,0); - std::vector t_in(2,0); - std::vector spks_in(2*80, 0); - std::vector cond_in(2*80*len, 0); - for(int step=1; step x_in(2 * 80 * len, 0); + std::vector mask_in(2 * 1 * len, 0); + std::vector mu_in(2 * 80 * len, 0); + std::vector t_in(2, 0); + std::vector spks_in(2 * 80, 0); + std::vector cond_in(2 * 80 * len, 0); + for (int step = 1; step < t_span.size(); step++) { memcpy(x_in.data(), x.data(), x.size() * sizeof(float)); - memcpy(x_in.data()+x.size(), x.data(), x.size() * sizeof(float)); + memcpy(x_in.data() + x.size(), x.data(), x.size() * sizeof(float)); memcpy(mask_in.data(), mask.data(), mask.size() * sizeof(float)); - memcpy(mask_in.data()+mask.size(), mask.data(), mask.size() * sizeof(float)); + memcpy(mask_in.data() + mask.size(), mask.data(), mask.size() * sizeof(float)); memcpy(mu_in.data(), mu.data(), mu.size() * sizeof(float)); @@ -457,41 +383,33 @@ public: std::vector dphi_dt; int ret = infer_flow_estimator(x_in, mask_in, t_in, mu_in, spks_in, cond_in, dphi_dt); - if(ret != 0) - { + if (ret != 0) { return ret; } - for(int i=0; i<80*len; i++) - { - - dphi_dt[i] = (1.0 + inference_cfg_rate) * dphi_dt[i] - inference_cfg_rate * dphi_dt[80 * len + i]; + for (int i = 0; i < 80 * len; i++) { + dphi_dt[i] = + (1.0 + _attr.inference_cfg_rate) * dphi_dt[i] - _attr.inference_cfg_rate * dphi_dt[80 * len + i]; x[i] = x[i] + dt * dphi_dt[i]; } - + t = t + dt; - if(step < t_span.size()-1) - { - dt = t_span[step+1] - t; - } - else{ - if(mel.empty() || mel.size()!=x.size()) - { + if (step < t_span.size() - 1) { + dt = t_span[step + 1] - t; + } else { + if (mel.empty() || mel.size() != x.size()) { mel.resize(x.size()); } memcpy(mel.data(), x.data(), x.size() * sizeof(float)); } - } return 0; } - int infer_flow_decoder( - std::vector & mu, std::vector & spks, std::vector & cond, std::vector & mask, - std::vector & mel - ) + int infer_flow_decoder(std::vector &mu, std::vector &spks, std::vector &cond, + std::vector &mask, std::vector &mel) { std::vector z; z.insert(z.end(), rand_noise.begin(), rand_noise.begin() + mu.size()); @@ -500,55 +418,51 @@ public: return ret; } - std::vector infer_flow( - std::vector & token_embeds, std::vector & prompt_feat, std::vector & spk_embeds, int token_len, bool finalize - ) + std::vector infer_flow(std::vector &token_embeds, std::vector &prompt_feat, + std::vector &spk_embeds, int token_len, bool finalize) { - int ret; + int ret; int len; std::vector mu; std::vector spks; std::vector cond; ret = infer_flow_encoder(token_embeds, prompt_feat, spk_embeds, token_len, finalize, mu, spks, cond); - if(ret != 0) - { + if (ret != 0) { return std::vector{}; } - len = mu.size()/80; + len = mu.size() / 80; std::vector mask(len, 1.0); std::vector all_mel; - + ret = infer_flow_decoder(mu, spks, cond, mask, all_mel); - if(ret != 0) - { + if (ret != 0) { return std::vector{}; } - - int len_mel1 = prompt_feat.size()/80; - int len_mel2 = all_mel.size()/80 - len_mel1; - + + int len_mel1 = prompt_feat.size() / 80; + int len_mel2 = all_mel.size() / 80 - len_mel1; + std::vector mel(len_mel2 * 80, 0); - auto result = slice_3d_last_dim_from(all_mel, 1, 80, all_mel.size()/80, len_mel1); - + auto result = slice_3d_last_dim_from(all_mel, 1, 80, all_mel.size() / 80, len_mel1); + return result; } - void fade_in_out(std::vector& fade_in_mel_data, - const std::vector& fade_out_mel_data, - const std::vector& window) { - + void fade_in_out(std::vector &fade_in_mel_data, const std::vector &fade_out_mel_data, + const std::vector &window) + { // --- Constants based on window = np.hamming(2 * 8 * 480) --- - const size_t WINDOW_SIZE = 2 * 8 * 480; // 7680 - const size_t MEL_OVERLAP_LEN = WINDOW_SIZE / 2; // 3840 + const size_t WINDOW_SIZE = 2 * 8 * 480; // 7680 + const size_t MEL_OVERLAP_LEN = WINDOW_SIZE / 2; // 3840 // dim0 is implicitly 1 for both inputs - size_t dim1_in = fade_in_mel_data.size(); + size_t dim1_in = fade_in_mel_data.size(); size_t dim1_out = fade_out_mel_data.size(); // --- Input Validation --- // For 2D arrays [1, L], the 1D vector size is just L. - + if (window.size() != WINDOW_SIZE) { throw std::invalid_argument("window size (" + std::to_string(window.size()) + ") does not match expected size (7680)."); @@ -569,7 +483,7 @@ public: for (size_t i = 0; i < MEL_OVERLAP_LEN; ++i) { // Indices are simply 'i' for the start of fade_in_mel // and 'dim1_out - MEL_OVERLAP_LEN + i' for the end of fade_out_mel - const size_t in_idx = i; + const size_t in_idx = i; const size_t out_idx = dim1_out - MEL_OVERLAP_LEN + i; // Perform the weighted sum: result = in_val * win_in + out_val * win_out @@ -577,8 +491,8 @@ public: // out_val = fade_out_mel_data[out_idx] // win_in = window[i] // win_out = window[MEL_OVERLAP_LEN + i] - fade_in_mel_data[in_idx] = fade_in_mel_data[in_idx] * window[i] + - fade_out_mel_data[out_idx] * window[MEL_OVERLAP_LEN + i]; + fade_in_mel_data[in_idx] = + fade_in_mel_data[in_idx] * window[i] + fade_out_mel_data[out_idx] * window[MEL_OVERLAP_LEN + i]; } // fade_in_mel_data is now modified in-place with the faded result. } @@ -588,109 +502,97 @@ public: std::unordered_map>().swap(hift_cache_dict); } - std::vector infer(std::vector & text_speech_token, std::vector & prompt_speech_embeds, std::vector & prompt_feat, - std::vector & spk_embeds, int token_offset, bool finalize) + std::vector infer(std::vector &text_speech_token, std::vector &prompt_speech_embeds, + std::vector &prompt_feat, std::vector &spk_embeds, int token_offset, + bool finalize) { int ret = 0; - std::vector speech_embeds( text_speech_token.size()*flow_embed_size + prompt_speech_embeds.size(), 0.0f); - std::vector speech_embeds_one(flow_embed_size, 0); + std::vector speech_embeds(text_speech_token.size() * _attr.flow_embed_size + prompt_speech_embeds.size(), + 0.0f); + std::vector speech_embeds_one(_attr.flow_embed_size, 0); memcpy(speech_embeds.data(), prompt_speech_embeds.data(), prompt_speech_embeds.size() * sizeof(float)); - for (size_t i = 0; i < text_speech_token.size(); i++) - { + for (size_t i = 0; i < text_speech_token.size(); i++) { flow_embed_selector.getByIndex(text_speech_token[i], speech_embeds_one.data()); - for (int j = 0; j < flow_embed_size; j++) - { - unsigned int proc = speech_embeds_one[j] << 16; - speech_embeds[prompt_speech_embeds.size() + i * flow_embed_size + j] = *reinterpret_cast(&proc); - } + for (int j = 0; j < _attr.flow_embed_size; j++) { + unsigned int proc = speech_embeds_one[j] << 16; + speech_embeds[prompt_speech_embeds.size() + i * _attr.flow_embed_size + j] = + *reinterpret_cast(&proc); + } } std::vector mel; - - mel = infer_flow(speech_embeds, prompt_feat, spk_embeds, text_speech_token.size(), finalize); + + mel = infer_flow(speech_embeds, prompt_feat, spk_embeds, text_speech_token.size(), finalize); std::vector tts_mel; - int neg_offset=0, start; - if(finalize) - { - neg_offset = token_offset * token_mel_ratio - mel.size()/80; - start = - token_hop_len * token_mel_ratio; + int neg_offset = 0, start; + if (finalize) { + neg_offset = token_offset * _attr.token_mel_ratio - mel.size() / 80; + start = -_attr.token_hop_len * _attr.token_mel_ratio; + } else { + start = std::min(int(token_offset / _attr.token_hop_len), _attr.max_infer_chunk_num - 1) * + _attr.token_hop_len * _attr.token_mel_ratio; } - else{ - start = std::min( int(token_offset / token_hop_len), max_infer_chunk_num-1) * token_hop_len * token_mel_ratio; - } - - tts_mel = slice_3d_last_dim_from(mel, 1, 80, mel.size()/80, start); - + + tts_mel = slice_3d_last_dim_from(mel, 1, 80, mel.size() / 80, start); + std::vector hift_cache_source; std::vector tts_mel1; std::vector speech, source, tts_speech; - if (!hift_cache_dict.empty()) - { + if (!hift_cache_dict.empty()) { auto hift_cache_mel = hift_cache_dict["mel"]; - hift_cache_source = hift_cache_dict["source"]; - tts_mel1 = concat_3d_dim2(hift_cache_mel, 1, 80, hift_cache_mel.size()/80, tts_mel, 1, 80, tts_mel.size()/80); - } - else{ + hift_cache_source = hift_cache_dict["source"]; + tts_mel1 = concat_3d_dim2(hift_cache_mel, 1, 80, hift_cache_mel.size() / 80, tts_mel, 1, 80, + tts_mel.size() / 80); + } else { tts_mel1 = tts_mel; - } + } ret = infer_hift(tts_mel1, hift_cache_source, speech, source); - - if(ret != 0){ + + if (ret != 0) { ALOGE("failed"); return std::vector{}; } - if(!finalize) - { - - if(!hift_cache_dict.empty()) - { + if (!finalize) { + if (!hift_cache_dict.empty()) { fade_in_out(speech, hift_cache_dict["speech"], speech_window); } - hift_cache_dict["mel"] = slice_3d_last_dim_from(tts_mel1, 1, 80, tts_mel1.size()/80, -mel_cache_len); + hift_cache_dict["mel"] = + slice_3d_last_dim_from(tts_mel1, 1, 80, tts_mel1.size() / 80, -_attr.mel_cache_len); int offset = speech.size(); - if(speech.size() > source_cache_len) - { - offset = source_cache_len; + if (speech.size() > _attr.source_cache_len) { + offset = _attr.source_cache_len; } - hift_cache_dict["source"].assign(source.end()-offset, source.end()); - hift_cache_dict["speech"].assign(speech.end()-offset, speech.end()); - tts_speech.assign(speech.begin(), speech.end()-offset); + hift_cache_dict["source"].assign(source.end() - offset, source.end()); + hift_cache_dict["speech"].assign(speech.end() - offset, speech.end()); + tts_speech.assign(speech.begin(), speech.end() - offset); - } - else{ - - if(speech.size() < source_cache_len){ + } else { + if (speech.size() < _attr.source_cache_len) { tts_speech.assign(speech.begin(), speech.end()); - } - else if (- neg_offset*480 >= source_cache_len) - { - tts_speech.assign(speech.end() + neg_offset*480, speech.end()); + } else if (-neg_offset * 480 >= _attr.source_cache_len) { + tts_speech.assign(speech.end() + neg_offset * 480, speech.end()); - if(!hift_cache_dict.empty()) - { + if (!hift_cache_dict.empty()) { fade_in_out(tts_speech, hift_cache_dict["speech"], speech_window); } - } - else{ - tts_speech.assign(speech.end()-source_cache_len, speech.end()); + } else { + tts_speech.assign(speech.end() - _attr.source_cache_len, speech.end()); - if(!hift_cache_dict.empty()) - { + if (!hift_cache_dict.empty()) { fade_in_out(tts_speech, hift_cache_dict["speech"], speech_window); } - int offset = speech.size() + neg_offset*480 - (speech.size() - source_cache_len); + int offset = speech.size() + neg_offset * 480 - (speech.size() - _attr.source_cache_len); tts_speech.assign(tts_speech.begin() + offset, tts_speech.end()); } - } return tts_speech; diff --git a/projects/llm_framework/main_cosy_voice/src/runner/utils/wav.hpp b/projects/llm_framework/main_cosy_voice/src/runner/utils/wav.hpp index dbbd979..11573c6 100644 --- a/projects/llm_framework/main_cosy_voice/src/runner/utils/wav.hpp +++ b/projects/llm_framework/main_cosy_voice/src/runner/utils/wav.hpp @@ -2,97 +2,101 @@ #include #include #include -#include // for memcpy -#include // for std::clamp (C++17) +#include +#include +#include +namespace fs = std::filesystem; -// 将 32 位整数以小端序写入文件 -void write_little_endian_32(std::ofstream& file, uint32_t value) { +void write_little_endian_32(std::ofstream& file, uint32_t value) +{ file.put(static_cast(value & 0xFF)); file.put(static_cast((value >> 8) & 0xFF)); file.put(static_cast((value >> 16) & 0xFF)); file.put(static_cast((value >> 24) & 0xFF)); } -// 将 16 位整数以小端序写入文件 -void write_little_endian_16(std::ofstream& file, uint16_t value) { +void write_little_endian_16(std::ofstream& file, uint16_t value) +{ file.put(static_cast(value & 0xFF)); file.put(static_cast((value >> 8) & 0xFF)); } -// 将 32 位浮点数以小端序写入文件 (IEEE 754) -void write_little_endian_float32(std::ofstream& file, float value) { - // IEEE 754 float 在大多数系统上已经是小端序 - // 但为了确保跨平台兼容性，我们显式处理字节序 +void write_little_endian_float32(std::ofstream& file, float value) +{ static_assert(sizeof(float) == 4, "Float must be 32 bits"); char bytes[4]; std::memcpy(bytes, &value, 4); - // bytes[0] 是最低有效字节 (LSB) file.put(bytes[0]); file.put(bytes[1]); file.put(bytes[2]); file.put(bytes[3]); } -// 将 float 范围 [-1.0, 1.0] 转换为 16-bit signed integer [-32768, 32767] -int16_t float_to_int16(float sample) { - // 确保输入在有效范围内 - // C++17 std::clamp, 或手动实现 - // sample = std::clamp(sample, -1.0f, 1.0f); +int16_t float_to_int16(float sample) +{ if (sample > 1.0f) sample = 1.0f; if (sample < -1.0f) sample = -1.0f; - - // 转换 return static_cast(sample * 32767.0f); } +bool ensure_directory_exists(const std::string& filename) +{ + try { + fs::path filepath(filename); + fs::path dir = filepath.parent_path(); + if (!dir.empty() && !fs::exists(dir)) { + fs::create_directories(dir); + } + return true; + } catch (const fs::filesystem_error& e) { + std::cerr << "Failed to create directories: " << e.what() << std::endl; + return false; + } +} -bool saveVectorAsWavFloat(const std::vector& audio_data, const std::string& filename, int sample_rate, int channels) { +bool saveVectorAsWavFloat(const std::vector& audio_data, const std::string& filename, int sample_rate, + int channels) +{ if (audio_data.empty() || channels <= 0 || sample_rate <= 0) { std::cerr << "Invalid input parameters." << std::endl; return false; } + if (!ensure_directory_exists(filename)) return false; + std::ofstream file(filename, std::ios::binary); if (!file.is_open()) { std::cerr << "Could not open file " << filename << " for writing." << std::endl; return false; } - // --- 计算关键参数 --- - uint32_t num_samples = static_cast(audio_data.size()); - uint16_t bits_per_sample = 32; // IEEE Float - uint32_t byte_rate = sample_rate * channels * (bits_per_sample / 8); - uint16_t block_align = channels * (bits_per_sample / 8); + uint32_t num_samples = static_cast(audio_data.size()); + uint16_t bits_per_sample = 32; // IEEE Float + uint32_t byte_rate = sample_rate * channels * (bits_per_sample / 8); + uint16_t block_align = channels * (bits_per_sample / 8); uint32_t data_chunk_size = num_samples * (bits_per_sample / 8); - uint32_t riff_chunk_size = 4 + (8 + 16) + (8 + data_chunk_size); // fmt chunk is 24 bytes including header + uint32_t riff_chunk_size = 4 + (8 + 16) + (8 + data_chunk_size); - // --- 写入 RIFF Header --- file.write("RIFF", 4); write_little_endian_32(file, riff_chunk_size); file.write("WAVE", 4); - - // --- 写入 fmt Subchunk --- file.write("fmt ", 4); - write_little_endian_32(file, 16); // Subchunk1Size for PCM - write_little_endian_16(file, 3); // AudioFormat: 3 = IEEE Float + write_little_endian_32(file, 16); + write_little_endian_16(file, 3); write_little_endian_16(file, static_cast(channels)); write_little_endian_32(file, sample_rate); write_little_endian_32(file, byte_rate); write_little_endian_16(file, block_align); write_little_endian_16(file, bits_per_sample); - - // --- 写入 data Subchunk --- file.write("data", 4); write_little_endian_32(file, data_chunk_size); - // --- 写入音频数据 --- for (const float& sample : audio_data) { write_little_endian_float32(file, sample); } if (file.fail()) { std::cerr << "Error occurred while writing audio data." << std::endl; - file.close(); return false; } @@ -101,60 +105,54 @@ bool saveVectorAsWavFloat(const std::vector& audio_data, const std::strin return true; } -bool saveVectorAsWavInt16(const std::vector& audio_data, const std::string& filename, int sample_rate, int channels) { +bool saveVectorAsWavInt16(const std::vector& audio_data, const std::string& filename, int sample_rate, + int channels) +{ if (audio_data.empty() || channels <= 0 || sample_rate <= 0) { std::cerr << "Invalid input parameters." << std::endl; return false; } + if (!ensure_directory_exists(filename)) return false; + std::ofstream file(filename, std::ios::binary); if (!file.is_open()) { std::cerr << "Could not open file " << filename << " for writing." << std::endl; return false; } - // --- 计算关键参数 --- - uint32_t num_samples = static_cast(audio_data.size()); - uint16_t bits_per_sample = 16; // 16-bit signed integer - uint32_t byte_rate = sample_rate * channels * (bits_per_sample / 8); - uint16_t block_align = channels * (bits_per_sample / 8); + uint32_t num_samples = static_cast(audio_data.size()); + uint16_t bits_per_sample = 16; + uint32_t byte_rate = sample_rate * channels * (bits_per_sample / 8); + uint16_t block_align = channels * (bits_per_sample / 8); uint32_t data_chunk_size = num_samples * (bits_per_sample / 8); - uint32_t riff_chunk_size = 4 + (8 + 16) + (8 + data_chunk_size); // fmt chunk is 24 bytes including header + uint32_t riff_chunk_size = 4 + (8 + 16) + (8 + data_chunk_size); - // --- 写入 RIFF Header --- file.write("RIFF", 4); write_little_endian_32(file, riff_chunk_size); file.write("WAVE", 4); - - // --- 写入 fmt Subchunk --- file.write("fmt ", 4); - write_little_endian_32(file, 16); // Subchunk1Size for PCM - write_little_endian_16(file, 1); // AudioFormat: 1 = PCM (Integer) + write_little_endian_32(file, 16); + write_little_endian_16(file, 1); write_little_endian_16(file, static_cast(channels)); write_little_endian_32(file, sample_rate); write_little_endian_32(file, byte_rate); write_little_endian_16(file, block_align); write_little_endian_16(file, bits_per_sample); - - // --- 写入 data Subchunk --- file.write("data", 4); write_little_endian_32(file, data_chunk_size); - // --- 写入音频数据 --- for (const float& sample : audio_data) { int16_t int_sample = float_to_int16(sample); - write_little_endian_16(file, static_cast(int_sample)); // int16_t -> uint16_t for raw bytes + write_little_endian_16(file, static_cast(int_sample)); } if (file.fail()) { std::cerr << "Error occurred while writing audio data." << std::endl; - file.close(); return false; } file.close(); std::cout << "Successfully saved audio to " << filename << " (16-bit Integer PCM)." << std::endl; return true; -} - - +} \ No newline at end of file