diff --git a/doc/projects_llm_framework_doc/llm_cosyvoice2.md b/doc/projects_llm_framework_doc/llm_cosyvoice2.md index c3ca007..de36bfd 100644 --- a/doc/projects_llm_framework_doc/llm_cosyvoice2.md +++ b/doc/projects_llm_framework_doc/llm_cosyvoice2.md @@ -17,7 +17,7 @@ cosy_voice "object": "cosy_voice.setup", "data": { "model": "CosyVoice2-0.5B-axcl", - "response_format": "sys.pcm", + "response_format": "file", "input": "tts.utf-8", "enoutput": false } @@ -30,7 +30,8 @@ cosy_voice - action：调用的方法为 `setup`。 - object：传输的数据类型为 `cosy_voice.setup`。 - model：使用的模型为 `CosyVoice2-0.5B-axcl` 模型。 -- response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。 +- prompt_files：要克隆的音频信息文件。 +- response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。返回结果为 `file`, 生成的音频写 wav 文件，可用 `prompt_data` 指定路径或文件名。 - input：输入的为 `tts.utf-8`,代表的是从用户输入。 - enoutput：是否起用用户结果输出。 diff --git a/projects/llm_framework/main_cosy_voice/SConstruct b/projects/llm_framework/main_cosy_voice/SConstruct index 75876a6..48fbba8 100644 --- a/projects/llm_framework/main_cosy_voice/SConstruct +++ b/projects/llm_framework/main_cosy_voice/SConstruct @@ -17,7 +17,10 @@ LDFLAGS = [] LINK_SEARCH_PATH = [] STATIC_FILES = [] -python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-cosy-voice-python-venv_v1.7.tar.gz", 'm5stack_llm-cosy-voice-python-venv_v1.7.tar.gz') +if "CONFIG_AXCL_ENABLED" in os.environ: + python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-cosy-voice-python-axcl-venv_v1.8.tar.gz", 'm5stack_llm-cosy-voice-python-axc-venv_v1.8.tar.gz') +else: + python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-cosy-voice-python-venv_v1.7.tar.gz", 'm5stack_llm-cosy-voice-python-venv_v1.7.tar.gz') # REQUIREMENTS += ['Backward_cpp'] # DYNAMIC_LIB += [ AFile('../static_lib/libdw.so.1'), @@ -45,6 +48,7 @@ INCLUDE += [ADir('../static_lib/include/sentencepiece'), static_file = Glob('../static_lib/module-llm/libabsl_*') static_file += [AFile('../static_lib/module-llm/libre2.a'), AFile('../static_lib/module-llm/libsentencepiece.a'), AFile('../static_lib/module-llm/libsentencepiece_train.a')] STATIC_LIB += static_file * 4 + REQUIREMENTS += ['onnxruntime'] STATIC_FILES += [os.path.join(python_venv, 'cosy-voice')] STATIC_FILES += Glob('scripts/tokenizer*') diff --git a/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json b/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json index b3f7669..3485208 100644 --- a/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json +++ b/projects/llm_framework/main_cosy_voice/models/mode_CosyVoice2-0.5B-axcl.json @@ -34,8 +34,8 @@ "flow_estimator_300": "flow_estimator_300.axmodel", "hift_p2_50_first": "hift_p2_50_first.axmodel", "hift_p2_58": "hift_p2_58.axmodel", - "hift_p1_50_first": "hift_p1_50_first.onnx", - "hift_p1_58": "hift_p1_58.onnx", + "hift_p1_50_first": "hift_p1_50_first.ort", + "hift_p1_58": "hift_p1_58.ort", "prompt_dir": "prompt_data", "prompt_text": "prompt_text.txt", "llm_prompt_speech_token": "llm_prompt_speech_token.txt", diff --git a/projects/llm_framework/main_cosy_voice/src/main.cpp b/projects/llm_framework/main_cosy_voice/src/main.cpp index 5bb1559..a2c5cc6 100644 --- a/projects/llm_framework/main_cosy_voice/src/main.cpp +++ b/projects/llm_framework/main_cosy_voice/src/main.cpp @@ -317,10 +317,10 @@ public: } }; - readtxt(infer_mode_config_.prompt_text, prompt_text_token); - readtxt(infer_mode_config_.llm_prompt_speech_token, prompt_speech_token); - readtxt(infer_mode_config_.prompt_speech_feat, prompt_feat); - readtxt(infer_mode_config_.flow_embedding, spk_embeds); + if (readtxt(infer_mode_config_.prompt_text, prompt_text_token)) return -3; + if (readtxt(infer_mode_config_.llm_prompt_speech_token, prompt_speech_token)) return -3; + if (readtxt(infer_mode_config_.prompt_speech_feat, prompt_feat)) return -3; + if (readtxt(infer_mode_config_.flow_embedding, spk_embeds)) return -3; lLaMa_ = std::make_unique(); if (!lLaMa_->Init(mode_config_)) { @@ -432,6 +432,7 @@ public: int prompt_token_len = prompt_speech_embeds_flow.size() / lToken2Wav._attr.flow_embed_size; if (prompt_token_len < 75) { SLOGE("Error, prompt speech token len %d < 75", prompt_token_len); + if (llm_thread.joinable()) llm_thread.join(); return -1; } int prompt_token_align_len = 75; diff --git a/projects/llm_framework/main_cosy_voice/src/runner/utils/utils.hpp b/projects/llm_framework/main_cosy_voice/src/runner/utils/utils.hpp index 58e9bb0..6d07701 100644 --- a/projects/llm_framework/main_cosy_voice/src/runner/utils/utils.hpp +++ b/projects/llm_framework/main_cosy_voice/src/runner/utils/utils.hpp @@ -3,71 +3,52 @@ #include #include #include -#include // 用于类型检查 +#include #include #include #include #include #include #include -#include // 用于异常处理 +#include -// 函数模板：支持任意元素类型的vector template -void savetxt(const std::string& filename, - const std::vector& data, - char delimiter = ' ', - int precision = 6) // 默认精度调整为6位 +void savetxt(const std::string& filename, const std::vector& data, char delimiter = ' ', int precision = 6) { std::ofstream outfile(filename); if (!outfile.is_open()) { - throw std::runtime_error("无法打开文件: " + filename); + throw std::runtime_error("Unable to open file: " + filename); } - - // 保存流的原始格式状态 std::ios_base::fmtflags original_flags = outfile.flags(); - - // 仅对浮点类型启用科学计数法 if constexpr (std::is_floating_point_v) { outfile << std::scientific << std::setprecision(precision); } - - // 优化输出：先输出第一个元素，再循环输出后续元素 if (!data.empty()) { outfile << data[0]; for (size_t i = 1; i < data.size(); ++i) { outfile << delimiter << data[i]; } } - - // 恢复流的原始格式状态 outfile.flags(original_flags); outfile.close(); } - - -// 函数模板：读取文本文件到 vector template -int readtxt(const std::string& filename, std::vector& data) { +int readtxt(const std::string& filename, std::vector& data) +{ std::ifstream file(filename); if (!file.is_open()) { - std::cerr << "错误：无法打开文件 " << filename << std::endl; + std::cerr << "Error: Unable to open file " << filename << std::endl; return -1; } - std::string line; while (std::getline(file, line)) { - // 跳过空行 if (line.empty()) continue; - std::istringstream iss(line); T value; - // 解析行中的每个数值 while (iss >> value) { data.push_back(value); - // 跳过分隔符（兼容逗号、分号等） if (iss.peek() == ',' || iss.peek() == ';') iss.ignore(); } } @@ -75,23 +56,19 @@ int readtxt(const std::string& filename, std::vector& data) { return 0; } -std::vector linspace(float start, float end, std::size_t num_steps) { +std::vector linspace(float start, float end, std::size_t num_steps) +{ if (num_steps == 0) { - return {}; // 返回空向量 + return {}; } if (num_steps == 1) { - return {start}; // 如果只需要一个点，返回起始值 + return {start}; } - std::vector result(num_steps); - float step_size = (end - start) / (num_steps - 1); // 计算步长 - + float step_size = (end - start) / (num_steps - 1); for (std::size_t i = 0; i < num_steps; ++i) { result[i] = start + i * step_size; } - - // 确保最后一个值精确等于 end，避免浮点数精度带来的误差 result.back() = end; - return result; } \ No newline at end of file diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index f56f4b7..fa86708 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -150,52 +150,67 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): shutil.rmtree(deb_folder) return package_name + " creat success!" -def create_data_deb(package_name, version, src_folder, revision = 'm5stack1', depends = 'lib-llm (>= 1.6)'): +def create_data_deb(package_name, version, src_folder, revision='m5stack1', depends='lib-llm (>= 1.6)'): deb_file = f"{package_name}_{version}-{revision}_arm64.deb" - deb_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'debian-{}'.format(package_name)) + deb_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'debian-{package_name}') + if os.path.exists(deb_folder): shutil.rmtree(deb_folder) - os.makedirs(deb_folder, exist_ok = True) + os.makedirs(deb_folder, exist_ok=True) zip_file = f"m5stack_{package_name[0:4]+package_name[10:]}_{version}_data.tar.gz" down_url = f"https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_{package_name[0:4]+package_name[10:]}_{version}_data.tar.gz" zip_file_extrpath = f"m5stack_{package_name}_{version}_data" + if not os.path.exists(zip_file_extrpath): - # Downloading via HTTP (more common) if not os.path.exists(zip_file): response = requests.get(down_url) if response.status_code == 200: with open(zip_file, 'wb') as file: file.write(response.content) else: - print("{} down failed".format(down_url)) + print(f"{down_url} download failed.") with tarfile.open(zip_file, 'r:gz') as tar: tar.extractall(path=zip_file_extrpath) - print("The {} download successful.".format(down_url)) + print(f"The {down_url} download successful.") + if os.path.exists(zip_file_extrpath): shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/data')) RED = "\033[31m" RESET = "\033[0m" - os.makedirs(os.path.join(deb_folder, 'opt/m5stack/data/models'), exist_ok = True) - mode_config_file = os.path.join(src_folder,'mode_{}.json'.format(package_name[10:])) + os.makedirs(os.path.join(deb_folder, 'opt/m5stack/data/models'), exist_ok=True) + + mode_config_file = os.path.join(src_folder, f'mode_{package_name[10:]}.json') if os.path.exists(mode_config_file): - shutil.copy2(mode_config_file, os.path.join(deb_folder, 'opt/m5stack/data/models', 'mode_{}.json'.format(package_name[10:]))) + shutil.copy2(mode_config_file, os.path.join(deb_folder, 'opt/m5stack/data/models', f'mode_{package_name[10:]}.json')) + try: with open(mode_config_file, 'r', encoding='utf-8') as file: data = json.load(file) - for scripts_file in data['mode_param']['ext_scripts']: - tokenizer_py_file = os.path.join(src_folder, scripts_file) - if os.path.exists(tokenizer_py_file): - os.makedirs(os.path.join(deb_folder, 'opt/m5stack/scripts'), exist_ok = True) - shutil.copy2(tokenizer_py_file, os.path.join(deb_folder, 'opt/m5stack/scripts', scripts_file)) - except: - pass - else: - print(RED, mode_config_file, " miss", RESET) - os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True) - with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f: + for scripts_file in data['mode_param'].get('ext_scripts', []): + src_path = os.path.join(src_folder, scripts_file) + dest_dir = os.path.join(deb_folder, 'opt/m5stack/scripts') + os.makedirs(dest_dir, exist_ok=True) + + if os.path.isfile(src_path): + shutil.copy2(src_path, os.path.join(dest_dir, os.path.basename(scripts_file))) + print(f"Copied file: {scripts_file}") + elif os.path.isdir(src_path): + shutil.copytree(src_path, os.path.join(dest_dir, os.path.basename(scripts_file)), dirs_exist_ok=True) + print(f"Copied folder: {scripts_file}") + else: + print(f"Warning: {scripts_file} not found in {src_folder}") + + except Exception as e: + print(f"Error reading ext_scripts: {e}") + + else: + print(RED, mode_config_file, " missing", RESET) + + os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok=True) + with open(os.path.join(deb_folder, 'DEBIAN/control'), 'w') as f: f.write(f'Package: {package_name}\n') f.write(f'Version: {version}\n') f.write(f'Architecture: arm64\n') @@ -207,21 +222,22 @@ def create_data_deb(package_name, version, src_folder, revision = 'm5stack1', de f.write(f'Homepage: https://www.m5stack.com\n') if deb_file.startswith('llm-model-'): deb_name = deb_file[:deb_file.find('_')] - old_deb_name = deb_name.replace('model-','').lower() - f.write(f'Conflicts: {old_deb_name}\n') - f.write(f'Packaged-Date: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n') + old_deb_name = deb_name.replace('model-', '').lower() + f.write(f'Conflicts: {old_deb_name}\n') + f.write(f'Packaged-Date: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n') f.write(f'Description: llm-module\n') f.write(f' bsp.\n') - with open(os.path.join(deb_folder, 'DEBIAN/postinst'),'w') as f: - f.write(f'#!/bin/sh\n') - f.write(f'exit 0\n') - with open(os.path.join(deb_folder, 'DEBIAN/prerm'),'w') as f: - f.write(f'#!/bin/sh\n') - f.write(f'exit 0\n') + + with open(os.path.join(deb_folder, 'DEBIAN/postinst'), 'w') as f: + f.write('#!/bin/sh\nexit 0\n') + with open(os.path.join(deb_folder, 'DEBIAN/prerm'), 'w') as f: + f.write('#!/bin/sh\nexit 0\n') os.chmod(os.path.join(deb_folder, 'DEBIAN/postinst'), 0o755) os.chmod(os.path.join(deb_folder, 'DEBIAN/prerm'), 0o755) + subprocess.run(["dpkg-deb", "-b", deb_folder, deb_file], check=True) print(f"Debian package created: {deb_file}") + shutil.rmtree(deb_folder) return package_name + " creat success!" @@ -262,6 +278,10 @@ def create_bin_deb(package_name, version, src_folder, revision = 'm5stack1', dep vlm_dir = os.path.join(src_folder, 'vlm') if os.path.exists(vlm_dir): shutil.copytree(vlm_dir, os.path.join(deb_folder, 'opt/m5stack/lib/vlm')) + if package_name == 'llm-cosy-voice': + cosy_voice_dir = os.path.join(src_folder, 'cosy-voice') + if os.path.exists(cosy_voice_dir): + shutil.copytree(cosy_voice_dir, os.path.join(deb_folder, 'opt/m5stack/lib/cosy-voice')) bin_file_name = package_name.replace("-", "_") if version_info != 0.0: @@ -368,6 +388,7 @@ if __name__ == "__main__": 'llm-vad':[create_bin_deb,'llm-vad', '1.8', src_folder, revision], 'llm-whisper':[create_bin_deb,'llm-whisper', '1.8', src_folder, revision], 'llm-openai-api':[create_bin_deb,'llm-openai-api', '1.8', src_folder, revision], + 'llm-cosy-voice':[create_bin_deb,'llm-cosy-voice', '1.8', src_folder, revision], # keyword spotting Audio file 'llm-model-audio-en-us':[create_data_deb,'llm-model-audio-en-us', data_version, src_folder, revision], 'llm-model-audio-zh-cn':[create_data_deb,'llm-model-audio-zh-cn', data_version, src_folder, revision], @@ -409,6 +430,9 @@ if __name__ == "__main__": 'llm-model-melotts-en-us-axcl':[create_data_deb,'llm-model-melotts-en-us-axcl', '0.6', src_folder, revision], 'llm-model-melotts-ja-jp-axcl':[create_data_deb,'llm-model-melotts-ja-jp-axcl', '0.6', src_folder, revision], 'llm-model-melotts-es-es-axcl':[create_data_deb,'llm-model-melotts-es-es-axcl', '0.5', src_folder, revision], + # CosyVoice2 model + ## AXCL + 'llm-model-CosyVoice2-0.5B-axcl':[create_data_deb,'llm-model-CosyVoice2-0.5B-axcl', '0.6', src_folder, revision], # Yolo model ## AX630C 'llm-model-yolo11n':[create_data_deb,'llm-model-yolo11n', data_version, src_folder, revision],