From 23cc53d9ef2fb5cd7b0c21412aad268ce02ad0f5 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Fri, 18 Apr 2025 18:48:40 +0800 Subject: [PATCH 01/14] [update] Update import package method --- benchmark/benchmodulellm.py | 2 +- benchmark/utils/__init__.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 benchmark/utils/__init__.py diff --git a/benchmark/benchmodulellm.py b/benchmark/benchmodulellm.py index 00dfcd0..7934b58 100644 --- a/benchmark/benchmodulellm.py +++ b/benchmark/benchmodulellm.py @@ -7,7 +7,7 @@ import logging from pathlib import Path -from utils.llm import LLMClient +from utils import LLMClient FILE = Path(__file__).resolve() ROOT = FILE.parents[0] diff --git a/benchmark/utils/__init__.py b/benchmark/utils/__init__.py new file mode 100644 index 0000000..5714a53 --- /dev/null +++ b/benchmark/utils/__init__.py @@ -0,0 +1,3 @@ +from .llm import LLMClient + +__all__ = ["LLMClient"] \ No newline at end of file From 66a47708d1f650972fead3bffc18cd988ef34f36 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Mon, 21 Apr 2025 11:51:16 +0800 Subject: [PATCH 02/14] [update] Update model name --- benchmark/utils/llm.py | 10 +--- .../llm_depth_anything_en.md | 2 +- .../llm_depth_anything_zh.md | 2 +- .../llm_melotts_en.md | 8 ++-- .../llm_melotts_zh.md | 8 ++-- doc/projects_llm_framework_doc/llm_tts_en.md | 47 +++++++++++++++++-- doc/projects_llm_framework_doc/llm_tts_zh.md | 46 ++++++++++++++++-- .../mode_depth-anything-ax630c.json | 4 +- .../main_melotts/mode_melotts-zh-cn.json | 4 +- .../mode_single-speaker-english-fast.json | 2 +- .../main_tts/mode_single-speaker-fast.json | 2 +- .../main_vad/mode_silero-vad.json | 2 +- projects/llm_framework/tools/llm_pack.py | 40 ++++++++-------- 13 files changed, 123 insertions(+), 54 deletions(-) diff --git a/benchmark/utils/llm.py b/benchmark/utils/llm.py index e2d6a0b..bf77670 100644 --- a/benchmark/utils/llm.py +++ b/benchmark/utils/llm.py @@ -30,7 +30,6 @@ class LLMClient: while True: chunk = self.sock.recv(4096) - logging.info(f"Received chunk: {chunk}") response += chunk while b'\n' in response: @@ -149,21 +148,14 @@ class LLMClient: def test(self, model, input_text): logging.info("Setting up...") setup_response = self.setup(model) - logging.info("Setup response: %s", setup_response) logging.info("Running inference...") inference_result = self.inference(input_text) - logging.info("Inference result: %s", inference_result) logging.info("Exiting...") exit_response = self.exit() - logging.info("Exit response: %s", exit_response) - return { - "setup_response": setup_response, - "inference_result": inference_result, - "exit_response": exit_response - } + return {} if __name__ == "__main__": host = "192.168.20.186" diff --git a/doc/projects_llm_framework_doc/llm_depth_anything_en.md b/doc/projects_llm_framework_doc/llm_depth_anything_en.md index 301cc54..b1d0b51 100644 --- a/doc/projects_llm_framework_doc/llm_depth_anything_en.md +++ b/doc/projects_llm_framework_doc/llm_depth_anything_en.md @@ -27,7 +27,7 @@ Send JSON: - work_id: When configuring the unit, it is `depth_anything`. - action: The method called is `setup`. - object: The data type being transmitted is `depth_anything.setup`. -- model: The model used is the `depth_anything` model. +- model: The model used is the `depth-anything-ax630c` model. - response_format: The return result is `jpeg.base64.stream`. - input: The input is `camera.1001`, which refers to the input from the camera unit, as detailed in the camera unit documentation. diff --git a/doc/projects_llm_framework_doc/llm_depth_anything_zh.md b/doc/projects_llm_framework_doc/llm_depth_anything_zh.md index 846ca34..85f07f4 100644 --- a/doc/projects_llm_framework_doc/llm_depth_anything_zh.md +++ b/doc/projects_llm_framework_doc/llm_depth_anything_zh.md @@ -27,7 +27,7 @@ depth_anything 视觉单元，用于提供图片深度信息。 - work_id：配置单元时，为 `depth_anything`。 - action：调用的方法为 `setup`。 - object：传输的数据类型为 `depth_anything.setup`。 -- model：使用的模型为 `depth_anything` 模型。 +- model：使用的模型为 `depth-anything-ax630c` 模型。 - response_format：返回结果为 `jpeg.base64.stream`。 - input：输入的为 `camera.1001`，代表的是从 camera 单元内部输入，详见 camera 单位文档。 - enoutput：是否启用用户结果输出。 diff --git a/doc/projects_llm_framework_doc/llm_melotts_en.md b/doc/projects_llm_framework_doc/llm_melotts_en.md index f45eb0e..d488deb 100644 --- a/doc/projects_llm_framework_doc/llm_melotts_en.md +++ b/doc/projects_llm_framework_doc/llm_melotts_en.md @@ -16,7 +16,7 @@ Send JSON: "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -28,7 +28,7 @@ Send JSON: - work_id: For configuration, it is `melotts`. - action: The method to be called is `setup`. - object: The data type being transmitted is `melotts.setup`. -- model: The model being used is the Chinese model `melotts_zh-cn`. +- model: The model being used is the Chinese model `melotts-zh-cn`. - response_format: The result is returned as `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. - input: The input is `tts.utf-8`, representing user input. @@ -139,7 +139,7 @@ Example: "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -335,7 +335,7 @@ Response JSON: "inputs_": [ "tts.utf-8" ], - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm" }, "error": { diff --git a/doc/projects_llm_framework_doc/llm_melotts_zh.md b/doc/projects_llm_framework_doc/llm_melotts_zh.md index 7d23b51..8c86e82 100644 --- a/doc/projects_llm_framework_doc/llm_melotts_zh.md +++ b/doc/projects_llm_framework_doc/llm_melotts_zh.md @@ -15,7 +15,7 @@ "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -27,7 +27,7 @@ - work_id：配置单元时，为 `melotts`。 - action：调用的方法为 `setup`。 - object：传输的数据类型为 `melotts.setup`。 -- model：使用的模型为 `melotts_zh-cn` 中文模型。 +- model：使用的模型为 `melotts-zh-cn` 中文模型。 - response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。 - input：输入的为 `tts.utf-8`,代表的是从用户输入。 - enoutput：是否起用用户结果输出。 @@ -134,7 +134,7 @@ error::code 为 0 表示执行成功。 "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -328,7 +328,7 @@ error::code 为 0 表示执行成功。 "inputs_": [ "tts.utf-8" ], - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm" }, "error": { diff --git a/doc/projects_llm_framework_doc/llm_tts_en.md b/doc/projects_llm_framework_doc/llm_tts_en.md index 7833828..61d0e45 100644 --- a/doc/projects_llm_framework_doc/llm_tts_en.md +++ b/doc/projects_llm_framework_doc/llm_tts_en.md @@ -16,7 +16,7 @@ Send JSON: "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-english-fast", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -28,7 +28,7 @@ Send JSON: - work_id: For configuring the unit, it is `tts`. - action: The method to call is `setup`. - object: The type of data being transmitted is `tts.setup`. -- model: The model used is the `single_speaker_fast` Chinese model. +- model: The model used is the `single-speaker-english-fast` English model. - response_format: The returned result is `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. - input: Input is `tts.utf-8`, representing user input. @@ -46,13 +46,50 @@ Response JSON: }, "object": "None", "request_id": "2", - "work_id": "llm.1003" + "work_id": "tts.1003" } ``` - created: Message creation time, in Unix time. - work_id: The successfully created work_id unit. +## inference + +### streaming input + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8.stream", + "data": { + "delta": "What's ur name?", + "index": 0, + "finish": true + } +} +``` +- object: The data type transmitted is tts.utf-8.stream, indicating a streaming input from the user's UTF-8. +- delta: Segment data of the streaming input. +- index: Index of the segment in the streaming input. +- finish: A flag indicating whether the streaming input has completed. + +### non-streaming input + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8", + "data": "What's ur name?" +} +``` + +- object: The data type transmitted is tts.utf-8, indicating a non-streaming input from the user's UTF-8. +- data: Data for non-streaming input. + ## link Link the output of the upper unit. @@ -102,7 +139,7 @@ Example: "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -298,7 +335,7 @@ Response JSON: "inputs_": [ "tts.utf-8" ], - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm" }, "error": { diff --git a/doc/projects_llm_framework_doc/llm_tts_zh.md b/doc/projects_llm_framework_doc/llm_tts_zh.md index 719d4fb..7e4ab18 100644 --- a/doc/projects_llm_framework_doc/llm_tts_zh.md +++ b/doc/projects_llm_framework_doc/llm_tts_zh.md @@ -15,7 +15,7 @@ "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -27,7 +27,7 @@ - work_id：配置单元时，为 `tts`。 - action：调用的方法为 `setup`。 - object：传输的数据类型为 `tts.setup`。 -- model：使用的模型为 `single_speaker_fast` 中文模型。 +- model：使用的模型为 `single-speaker-fast` 中文模型。 - response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。 - input：输入的为 `tts.utf-8`,代表的是从用户输入。 - enoutput：是否起用用户结果输出。 @@ -44,13 +44,49 @@ }, "object": "None", "request_id": "2", - "work_id": "llm.1003" + "work_id": "tts.1003" } ``` - created：消息创建时间，unix 时间。 - work_id：返回成功创建的 work_id 单元。 +## inference + +### 流式输入 + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8.stream", + "data": { + "delta": "今天天气真好！", + "index": 0, + "finish": true + } +} +``` +- object：传输的数据类型为 `tts.utf-8.stream` 代表的是从用户 utf-8 的流式输入 +- delta：流式输入的分段数据 +- index：流式输入的分段索引 +- finish:流式输入是否完成的标志位 + +### 非流式输入 + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8", + "data": "今天天气真好！" +} +``` +- object：传输的数据类型为 `tts.utf-8` 代表的是从用户 utf-8 的非流式输入 +- data：非流式输入的数据 + ## link 链接上级单元的输出。 @@ -98,7 +134,7 @@ error::code 为 0 表示执行成功。 "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -294,7 +330,7 @@ error::code 为 0 表示执行成功。 "inputs_": [ "tts.utf-8" ], - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm" }, "error": { diff --git a/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json b/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json index ddf0a6c..fd91eb3 100644 --- a/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json +++ b/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json @@ -1,7 +1,9 @@ { - "mode":"depth_anything", + "mode":"depth-anything-ax630c", "type":"cv", "homepage":"https://github.com/DepthAnything/Depth-Anything-V2", + "compile_flage":"pulsar2 build --input depth_anything.onnx --config config.json --output_dir output --output_name depth_anything.axmodel --target_hardware AX620E", + "pulsar_version":"2.5-2d5d0fa3", "capabilities":[ "Segmentation" ], diff --git a/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json b/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json index e7af2c5..ee9f57c 100644 --- a/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json +++ b/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json @@ -1,7 +1,9 @@ { - "mode": "melotts_zh-cn", + "mode": "melotts-zh-cn", "type": "tts", "homepage":"https://huggingface.co/myshell-ai/MeloTTS-Chinese", + "compile_flage":"pulsar2 build --input decoder.onnx --config config_decoder_u16.json --output_dir decoder --output_name decoder.axmodel --target_hardware AX620E --npu_mode NPU2 --compiler.check 0", + "pulsar_version":"3.2-99f14d0a", "capabilities": [ "tts", "Chinese" diff --git a/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json b/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json index e311a4a..cbaccaa 100644 --- a/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json +++ b/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json @@ -1,5 +1,5 @@ { - "mode": "single_speaker_english_fast", + "mode": "single-speaker-english-fast", "type": "tts", "homepage":"https://github.com/huakunyang/SummerTTS", "capabilities": [ diff --git a/projects/llm_framework/main_tts/mode_single-speaker-fast.json b/projects/llm_framework/main_tts/mode_single-speaker-fast.json index ca1ede7..6fadf2f 100644 --- a/projects/llm_framework/main_tts/mode_single-speaker-fast.json +++ b/projects/llm_framework/main_tts/mode_single-speaker-fast.json @@ -1,5 +1,5 @@ { - "mode": "single_speaker_fast", + "mode": "single-speaker-fast", "type": "tts", "homepage":"https://github.com/huakunyang/SummerTTS", "capabilities": [ diff --git a/projects/llm_framework/main_vad/mode_silero-vad.json b/projects/llm_framework/main_vad/mode_silero-vad.json index d262af4..a4d8eae 100644 --- a/projects/llm_framework/main_vad/mode_silero-vad.json +++ b/projects/llm_framework/main_vad/mode_silero-vad.json @@ -1,5 +1,5 @@ { - "mode": "silero_vad", + "mode": "silero-vad", "type": "vad", "homepage":"https://github.com/snakers4/silero-vad", "capabilities": [ diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 38ca64e..21c3831 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -69,23 +69,23 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): # if os.path.exists(zip_file_extrpath): # shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/scripts')) - # zip_file = 'm5stack_dist-packages.tar.gz' - # down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz' - # zip_file_extrpath = 'm5stack_dist-packages' - # if not os.path.exists(zip_file_extrpath): - # # Downloading via HTTP (more common) - # if not os.path.exists(zip_file): - # response = requests.get(down_url) - # if response.status_code == 200: - # with open(zip_file, 'wb') as file: - # file.write(response.content) - # else: - # print("{} down failed".format(down_url)) - # with tarfile.open(zip_file, 'r:gz') as tar: - # tar.extractall(path=zip_file_extrpath) - # print("The {} download successful.".format(down_url)) - # if os.path.exists(zip_file_extrpath): - # shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages')) + zip_file = 'm5stack_dist-packages.tar.gz' + down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz' + zip_file_extrpath = 'm5stack_dist-packages' + if not os.path.exists(zip_file_extrpath): + # Downloading via HTTP (more common) + if not os.path.exists(zip_file): + response = requests.get(down_url) + if response.status_code == 200: + with open(zip_file, 'wb') as file: + file.write(response.content) + else: + print("{} down failed".format(down_url)) + with tarfile.open(zip_file, 'r:gz') as tar: + tar.extractall(path=zip_file_extrpath) + print("The {} download successful.".format(down_url)) + if os.path.exists(zip_file_extrpath): + shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages')) os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True) with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f: @@ -374,9 +374,9 @@ if __name__ == "__main__": 'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23', data_version, src_folder, revision], 'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01', '0.3', src_folder, revision], 'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01', '0.3', src_folder, revision], - 'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', data_version, src_folder, revision], - 'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', data_version, src_folder, revision], - 'llm-model-melotts-zh-cn':[create_data_deb,'llm-model-melotts-zh-cn', '0.4', src_folder, revision], + 'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', '0.3', src_folder, revision], + 'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', '0.3', src_folder, revision], + 'llm-model-melotts-zh-cn':[create_data_deb,'llm-model-melotts-zh-cn', '0.5', src_folder, revision], 'llm-model-yolo11n':[create_data_deb,'llm-model-yolo11n', data_version, src_folder, revision], 'llm-model-yolo11n-pose':[create_data_deb,'llm-model-yolo11n-pose', '0.3', src_folder, revision], 'llm-model-yolo11n-hand-pose':[create_data_deb,'llm-model-yolo11n-hand-pose', '0.3', src_folder, revision], From 783200e804dbbd266766524e4a2149b328809ed6 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Mon, 21 Apr 2025 15:23:17 +0800 Subject: [PATCH 03/14] [update] Update openai_api version --- projects/llm_framework/main_openai_api/SConstruct | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_framework/main_openai_api/SConstruct b/projects/llm_framework/main_openai_api/SConstruct index 17adecc..db38738 100644 --- a/projects/llm_framework/main_openai_api/SConstruct +++ b/projects/llm_framework/main_openai_api/SConstruct @@ -18,8 +18,8 @@ LINK_SEARCH_PATH = [] STATIC_FILES = [] -ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/m5stack/ModuleLLM-OpenAI-Plugin.git', '33477071ca362d20cd65cfad43f5b05480724711', True) -python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.5.tar.gz", 'm5stack_llm-openai-api-python-venv_v1.5.tar.gz') +ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/m5stack/ModuleLLM-OpenAI-Plugin.git', '6f783440eb2b4ff6566c171ffc2815cb64024928', True) +python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.6.tar.gz", 'm5stack_llm-openai-api-python-venv_v1.6.tar.gz') DEFINITIONS += ['-O3', '-fopenmp', '-std=c++17'] From 8f2643d27076c449171122df5f32d02c668e7580 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Mon, 21 Apr 2025 15:49:18 +0800 Subject: [PATCH 04/14] [update] update llm-asr doc --- doc/projects_llm_framework_doc/llm_asr_en.md | 6 +++--- doc/projects_llm_framework_doc/llm_asr_zh.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/projects_llm_framework_doc/llm_asr_en.md b/doc/projects_llm_framework_doc/llm_asr_en.md index 8e96f25..282bb68 100644 --- a/doc/projects_llm_framework_doc/llm_asr_en.md +++ b/doc/projects_llm_framework_doc/llm_asr_en.md @@ -119,9 +119,9 @@ Example: "endpoint_config.rule1.min_trailing_silence": 2.4, "endpoint_config.rule2.min_trailing_silence": 1.2, "endpoint_config.rule3.min_trailing_silence": 30.1, - "endpoint_config.rule1.must_contain_nonsilence": false, - "endpoint_config.rule2.must_contain_nonsilence": false, - "endpoint_config.rule3.must_contain_nonsilence": false + "endpoint_config.rule1.must_contain_nonsilence": true, + "endpoint_config.rule2.must_contain_nonsilence": true, + "endpoint_config.rule3.must_contain_nonsilence": true } } ``` diff --git a/doc/projects_llm_framework_doc/llm_asr_zh.md b/doc/projects_llm_framework_doc/llm_asr_zh.md index e139f28..97c8e90 100644 --- a/doc/projects_llm_framework_doc/llm_asr_zh.md +++ b/doc/projects_llm_framework_doc/llm_asr_zh.md @@ -117,9 +117,9 @@ error::code 为 0 表示执行成功。 "endpoint_config.rule1.min_trailing_silence": 2.4, "endpoint_config.rule2.min_trailing_silence": 1.2, "endpoint_config.rule3.min_trailing_silence": 30.1, - "endpoint_config.rule1.must_contain_nonsilence": false, - "endpoint_config.rule2.must_contain_nonsilence": false, - "endpoint_config.rule3.must_contain_nonsilence": false + "endpoint_config.rule1.must_contain_nonsilence": true, + "endpoint_config.rule2.must_contain_nonsilence": true, + "endpoint_config.rule3.must_contain_nonsilence": true } } ``` From 14d7e5b01b31b804b08d618fa98be1652cb2bb76 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Mon, 21 Apr 2025 16:55:48 +0800 Subject: [PATCH 05/14] [fix] llm_whisper --- projects/llm_framework/main_whisper/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_framework/main_whisper/src/main.cpp b/projects/llm_framework/main_whisper/src/main.cpp index 776408b..73515f1 100644 --- a/projects/llm_framework/main_whisper/src/main.cpp +++ b/projects/llm_framework/main_whisper/src/main.cpp @@ -316,7 +316,7 @@ public: if (endpoint_flage_) return; } endpoint_flage_ = true; - buffer_resize(pcmdata, 0); + if (delay_audio_frame_ == 0) buffer_resize(pcmdata, 0); buffer_write_char(pcmdata, raw.c_str(), raw.length()); buffer_position_set(pcmdata, 0); count = 0; From 4b477eeac3ee351f3d8fb83ecc5d90c83c220230 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 22 Apr 2025 15:50:00 +0800 Subject: [PATCH 06/14] [update] Add melotts-en-us model --- .../main_melotts/mode_melotts-en-us.json | 29 +++++++++++++++++++ .../llm_framework/main_melotts/src/main.cpp | 13 ++++++--- projects/llm_framework/tools/llm_pack.py | 1 + 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 projects/llm_framework/main_melotts/mode_melotts-en-us.json diff --git a/projects/llm_framework/main_melotts/mode_melotts-en-us.json b/projects/llm_framework/main_melotts/mode_melotts-en-us.json new file mode 100644 index 0000000..6a375c9 --- /dev/null +++ b/projects/llm_framework/main_melotts/mode_melotts-en-us.json @@ -0,0 +1,29 @@ +{ + "mode": "melotts-en-us", + "type": "tts", + "homepage":"https://huggingface.co/myshell-ai/MeloTTS-English", + "compile_flage":"pulsar2 build --input decoder-en.onnx --config config_decoder_u16.json --output_dir decoder-en --output_name decoder-en.axmodel --target_hardware AX620E --npu_mode NPU2 --compiler.check 0", + "pulsar_version":"3.4-3dfd5692", + "capabilities": [ + "tts", + "English" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-en.ort", + "decoder": "decoder-en.axmodel", + "gbin": "g-en.bin", + "tokens": "tokens.txt", + "lexicon": "lexicon.txt", + "spacker_speed": 1.0, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/src/main.cpp b/projects/llm_framework/main_melotts/src/main.cpp index 9890fac..b5a27cb 100644 --- a/projects/llm_framework/main_melotts/src/main.cpp +++ b/projects/llm_framework/main_melotts/src/main.cpp @@ -37,6 +37,7 @@ static std::string base_model_path_; static std::string base_model_config_path_; typedef struct { + std::string mode; std::string encoder; std::string decoder; std::string lexicon; @@ -112,6 +113,9 @@ public: return false; } + std::unordered_map MELOTTS_LANG_IDS_MAP{ + {"melotts-ja-jp", 1}, {"melotts-en-us", 2}, {"melotts_zh-cn", 3}, {"melotts-zh-cn", 3}}; + std::vector intersperse(const std::vector &lst, int item) { std::vector result(lst.size() * 2 + 1, item); @@ -251,10 +255,11 @@ public: std::vector phones_bef, tones_bef; lexicon_->convert(msg_str, phones_bef, tones_bef); // Add blank between words - auto phones = intersperse(phones_bef, 0); - auto tones = intersperse(tones_bef, 0); - int phone_len = phones.size(); - std::vector langids(phone_len, 3); + auto phones = intersperse(phones_bef, 0); + auto tones = intersperse(tones_bef, 0); + int phone_len = phones.size(); + int MELOTTS_LANG_IDS = MELOTTS_LANG_IDS_MAP[mode_config_.mode]; + std::vector langids(phone_len, MELOTTS_LANG_IDS); auto encoder_output = encoder_->Run(phones, tones, langids, g_matrix, mode_config_.noise_scale, mode_config_.noise_scale_w, mode_config_.get_length_scale(), mode_config_.sdp_ratio); diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 21c3831..808771e 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -377,6 +377,7 @@ if __name__ == "__main__": 'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', '0.3', src_folder, revision], 'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', '0.3', src_folder, revision], 'llm-model-melotts-zh-cn':[create_data_deb,'llm-model-melotts-zh-cn', '0.5', src_folder, revision], + 'llm-model-melotts-en-us':[create_data_deb,'llm-model-melotts-en-us', '0.5', src_folder, revision], 'llm-model-yolo11n':[create_data_deb,'llm-model-yolo11n', data_version, src_folder, revision], 'llm-model-yolo11n-pose':[create_data_deb,'llm-model-yolo11n-pose', '0.3', src_folder, revision], 'llm-model-yolo11n-hand-pose':[create_data_deb,'llm-model-yolo11n-hand-pose', '0.3', src_folder, revision], From aa3441d307aad9725e6293f523b4b9b545d95f82 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 22 Apr 2025 17:14:49 +0800 Subject: [PATCH 07/14] [update] Update package version --- projects/llm_framework/main_asr/SConstruct | 2 +- .../main_depth_anything/SConstruct | 2 +- projects/llm_framework/main_kws/SConstruct | 2 +- projects/llm_framework/main_melotts/SConstruct | 2 +- .../llm_framework/main_openai_api/SConstruct | 2 +- projects/llm_framework/main_vad/SConstruct | 2 +- projects/llm_framework/main_vlm/SConstruct | 2 +- projects/llm_framework/main_whisper/SConstruct | 2 +- projects/llm_framework/tools/llm_pack.py | 18 +++++++++--------- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/projects/llm_framework/main_asr/SConstruct b/projects/llm_framework/main_asr/SConstruct index 496838c..344162b 100644 --- a/projects/llm_framework/main_asr/SConstruct +++ b/projects/llm_framework/main_asr/SConstruct @@ -26,7 +26,7 @@ REQUIREMENTS += ['ncnn', 'sherpa-ncnn-core'] STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_asr-1.5', +env['COMPONENTS'].append({'target':'llm_asr-1.6', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_depth_anything/SConstruct b/projects/llm_framework/main_depth_anything/SConstruct index de14d98..a90887f 100644 --- a/projects/llm_framework/main_depth_anything/SConstruct +++ b/projects/llm_framework/main_depth_anything/SConstruct @@ -30,7 +30,7 @@ STATIC_LIB += static_file * 2 STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_depth_anything-1.5', +env['COMPONENTS'].append({'target':'llm_depth_anything-1.6', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_kws/SConstruct b/projects/llm_framework/main_kws/SConstruct index 9b4dee4..c09ca41 100644 --- a/projects/llm_framework/main_kws/SConstruct +++ b/projects/llm_framework/main_kws/SConstruct @@ -55,7 +55,7 @@ ignore['ignore'] = list(set(ignore['ignore'])) with open('../dist/fileignore', 'w') as f: json.dump(ignore, f, indent=4) -env['COMPONENTS'].append({'target':'llm_kws-1.6', +env['COMPONENTS'].append({'target':'llm_kws-1.7', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_melotts/SConstruct b/projects/llm_framework/main_melotts/SConstruct index d3c3cba..6663ca3 100644 --- a/projects/llm_framework/main_melotts/SConstruct +++ b/projects/llm_framework/main_melotts/SConstruct @@ -31,7 +31,7 @@ LDFLAGS += ['-l:libcargs.a', '-l:libonnxruntime.a'] STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_melotts-1.6', +env['COMPONENTS'].append({'target':'llm_melotts-1.7', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_openai_api/SConstruct b/projects/llm_framework/main_openai_api/SConstruct index db38738..490cebc 100644 --- a/projects/llm_framework/main_openai_api/SConstruct +++ b/projects/llm_framework/main_openai_api/SConstruct @@ -52,7 +52,7 @@ ignore['ignore'] = list(set(ignore['ignore'])) with open('../dist/fileignore', 'w') as f: json.dump(ignore, f, indent=4) -env['COMPONENTS'].append({'target':'llm_openai_api-1.6', +env['COMPONENTS'].append({'target':'llm_openai_api-1.7', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_vad/SConstruct b/projects/llm_framework/main_vad/SConstruct index 0551cae..2493e15 100644 --- a/projects/llm_framework/main_vad/SConstruct +++ b/projects/llm_framework/main_vad/SConstruct @@ -28,7 +28,7 @@ LDFLAGS += ['-l:libsherpa-onnx-core.a', STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_vad-1.5', +env['COMPONENTS'].append({'target':'llm_vad-1.6', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_vlm/SConstruct b/projects/llm_framework/main_vlm/SConstruct index a42d7fb..d1046f2 100644 --- a/projects/llm_framework/main_vlm/SConstruct +++ b/projects/llm_framework/main_vlm/SConstruct @@ -73,7 +73,7 @@ ignore['ignore'] = list(set(ignore['ignore'])) with open('../dist/fileignore', 'w') as f: json.dump(ignore, f, indent=4) -env['COMPONENTS'].append({'target':'llm_vlm-1.6', +env['COMPONENTS'].append({'target':'llm_vlm-1.7', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_whisper/SConstruct b/projects/llm_framework/main_whisper/SConstruct index 40ab63d..4c61edc 100644 --- a/projects/llm_framework/main_whisper/SConstruct +++ b/projects/llm_framework/main_whisper/SConstruct @@ -33,7 +33,7 @@ LDFLAGS += ['-l:libopencc.a', '-l:libmarisa.a'] STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_whisper-1.6', +env['COMPONENTS'].append({'target':'llm_whisper-1.7', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 808771e..d32adfd 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -355,19 +355,19 @@ if __name__ == "__main__": 'lib-llm':[create_lib_deb,'lib-llm', '1.7', src_folder, revision], 'llm-sys':[create_bin_deb,'llm-sys', version, src_folder, revision], 'llm-audio':[create_bin_deb,'llm-audio', version, src_folder, revision], - 'llm-kws':[create_bin_deb,'llm-kws', '1.6', src_folder, revision], - 'llm-asr':[create_bin_deb,'llm-asr', version, src_folder, revision], + 'llm-kws':[create_bin_deb,'llm-kws', '1.7', src_folder, revision], + 'llm-asr':[create_bin_deb,'llm-asr', '1.6', src_folder, revision], 'llm-llm':[create_bin_deb,'llm-llm', '1.7', src_folder, revision], 'llm-tts':[create_bin_deb,'llm-tts', version, src_folder, revision], - 'llm-melotts':[create_bin_deb,'llm-melotts', '1.6', src_folder, revision], + 'llm-melotts':[create_bin_deb,'llm-melotts', '1.7', src_folder, revision], 'llm-camera':[create_bin_deb,'llm-camera', '1.7', src_folder, revision, 'lib-llm'], - 'llm-vlm':[create_bin_deb,'llm-vlm', '1.6', src_folder, revision], - 'llm-yolo':[create_bin_deb,'llm-yolo', '1.6', src_folder, revision], + 'llm-vlm':[create_bin_deb,'llm-vlm', '1.7', src_folder, revision], + 'llm-yolo':[create_bin_deb,'llm-yolo', '1.7', src_folder, revision], 'llm-skel':[create_bin_deb,'llm-skel', version, src_folder, revision], - 'llm-depth-anything':[create_bin_deb,'llm-depth-anything', version, src_folder, revision], - 'llm-vad':[create_bin_deb,'llm-vad', version, src_folder, revision], - 'llm-whisper':[create_bin_deb,'llm-whisper', '1.6', src_folder, revision], - 'llm-openai-api':[create_bin_deb,'llm-openai-api', '1.6', src_folder, revision], + 'llm-depth-anything':[create_bin_deb,'llm-depth-anything', '1.6', src_folder, revision], + 'llm-vad':[create_bin_deb,'llm-vad', '1.6', src_folder, revision], + 'llm-whisper':[create_bin_deb,'llm-whisper', '1.7', src_folder, revision], + 'llm-openai-api':[create_bin_deb,'llm-openai-api', '1.7', src_folder, revision], 'llm-model-audio-en-us':[create_data_deb,'llm-model-audio-en-us', data_version, src_folder, revision], 'llm-model-audio-zh-cn':[create_data_deb,'llm-model-audio-zh-cn', data_version, src_folder, revision], 'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17', data_version, src_folder, revision], From 8e263d0f44290033d8202e36e4fb48c5352191e5 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 22 Apr 2025 17:36:12 +0800 Subject: [PATCH 08/14] [update] Update other package version --- projects/llm_framework/main_audio/SConstruct | 2 +- projects/llm_framework/main_camera/SConstruct | 2 +- projects/llm_framework/main_llm/SConstruct | 2 +- projects/llm_framework/main_sys/SConstruct | 2 +- projects/llm_framework/main_tts/SConstruct | 2 +- projects/llm_framework/main_yolo/SConstruct | 2 +- projects/llm_framework/tools/llm_pack.py | 12 ++++++------ 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/llm_framework/main_audio/SConstruct b/projects/llm_framework/main_audio/SConstruct index 25b3f1a..27a04b6 100644 --- a/projects/llm_framework/main_audio/SConstruct +++ b/projects/llm_framework/main_audio/SConstruct @@ -29,7 +29,7 @@ REQUIREMENTS += ['tinyalsa', 'opus', 'samplerate', 'fdk-aac'] STATIC_FILES += [AFile('audio.json')] STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_audio-1.5', +env['COMPONENTS'].append({'target':'llm_audio-1.6', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_camera/SConstruct b/projects/llm_framework/main_camera/SConstruct index db43b04..eb5190c 100644 --- a/projects/llm_framework/main_camera/SConstruct +++ b/projects/llm_framework/main_camera/SConstruct @@ -68,7 +68,7 @@ STATIC_LIB += static_file * 4 STATIC_FILES += [AFile('camera.json')] STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_camera-1.7', +env['COMPONENTS'].append({'target':'llm_camera-1.8', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_llm/SConstruct b/projects/llm_framework/main_llm/SConstruct index 52b0778..1b12643 100644 --- a/projects/llm_framework/main_llm/SConstruct +++ b/projects/llm_framework/main_llm/SConstruct @@ -66,7 +66,7 @@ ignore['ignore'] = list(set(ignore['ignore'])) with open('../dist/fileignore', 'w') as f: json.dump(ignore, f, indent=4) -env['COMPONENTS'].append({'target':'llm_llm-1.7', +env['COMPONENTS'].append({'target':'llm_llm-1.8', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_sys/SConstruct b/projects/llm_framework/main_sys/SConstruct index 0093d21..d9a4830 100644 --- a/projects/llm_framework/main_sys/SConstruct +++ b/projects/llm_framework/main_sys/SConstruct @@ -37,7 +37,7 @@ LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '- STATIC_FILES += [AFile('sys_config.json')] REQUIREMENTS += ['simdjson_component'] -env['COMPONENTS'].append({'target':'llm_sys-1.5', +env['COMPONENTS'].append({'target':'llm_sys-1.6', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_tts/SConstruct b/projects/llm_framework/main_tts/SConstruct index bba7997..a38e56f 100644 --- a/projects/llm_framework/main_tts/SConstruct +++ b/projects/llm_framework/main_tts/SConstruct @@ -27,7 +27,7 @@ INCLUDE += [ADir('src/runner/eigen-3.4.0'), ADir('src/runner/src/tn/header'), AD STATIC_FILES += Glob('mode_*.json') -env['COMPONENTS'].append({'target':'llm_tts-1.5', +env['COMPONENTS'].append({'target':'llm_tts-1.6', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/main_yolo/SConstruct b/projects/llm_framework/main_yolo/SConstruct index e432321..99f1e1e 100644 --- a/projects/llm_framework/main_yolo/SConstruct +++ b/projects/llm_framework/main_yolo/SConstruct @@ -39,7 +39,7 @@ STATIC_FILES += Glob('mode_*.json') # AFile('../static_lib/libbz2.so.1.0')] # DEFINITIONS += ["-DENABLE_BACKWARD"] -env['COMPONENTS'].append({'target':'llm_yolo-1.7', +env['COMPONENTS'].append({'target':'llm_yolo-1.8', 'SRCS':SRCS, 'INCLUDE':INCLUDE, 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index d32adfd..370c8b2 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -353,16 +353,16 @@ if __name__ == "__main__": # cpu_count = 50 Tasks = { 'lib-llm':[create_lib_deb,'lib-llm', '1.7', src_folder, revision], - 'llm-sys':[create_bin_deb,'llm-sys', version, src_folder, revision], - 'llm-audio':[create_bin_deb,'llm-audio', version, src_folder, revision], + 'llm-sys':[create_bin_deb,'llm-sys', '1.6', src_folder, revision], + 'llm-audio':[create_bin_deb,'llm-audio', '1.6', src_folder, revision], 'llm-kws':[create_bin_deb,'llm-kws', '1.7', src_folder, revision], 'llm-asr':[create_bin_deb,'llm-asr', '1.6', src_folder, revision], - 'llm-llm':[create_bin_deb,'llm-llm', '1.7', src_folder, revision], - 'llm-tts':[create_bin_deb,'llm-tts', version, src_folder, revision], + 'llm-llm':[create_bin_deb,'llm-llm', '1.8', src_folder, revision], + 'llm-tts':[create_bin_deb,'llm-tts', '1.6', src_folder, revision], 'llm-melotts':[create_bin_deb,'llm-melotts', '1.7', src_folder, revision], - 'llm-camera':[create_bin_deb,'llm-camera', '1.7', src_folder, revision, 'lib-llm'], + 'llm-camera':[create_bin_deb,'llm-camera', '1.8', src_folder, revision, 'lib-llm'], 'llm-vlm':[create_bin_deb,'llm-vlm', '1.7', src_folder, revision], - 'llm-yolo':[create_bin_deb,'llm-yolo', '1.7', src_folder, revision], + 'llm-yolo':[create_bin_deb,'llm-yolo', '1.8', src_folder, revision], 'llm-skel':[create_bin_deb,'llm-skel', version, src_folder, revision], 'llm-depth-anything':[create_bin_deb,'llm-depth-anything', '1.6', src_folder, revision], 'llm-vad':[create_bin_deb,'llm-vad', '1.6', src_folder, revision], From efa978bd773e01daacf41241161171f1692d5eea Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 22 Apr 2025 17:38:17 +0800 Subject: [PATCH 09/14] [update] update melotts doc --- doc/projects_llm_framework_doc/llm_melotts_en.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/projects_llm_framework_doc/llm_melotts_en.md b/doc/projects_llm_framework_doc/llm_melotts_en.md index d488deb..08222df 100644 --- a/doc/projects_llm_framework_doc/llm_melotts_en.md +++ b/doc/projects_llm_framework_doc/llm_melotts_en.md @@ -16,7 +16,7 @@ Send JSON: "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts-zh-cn", + "model": "melotts-en-us", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -28,7 +28,7 @@ Send JSON: - work_id: For configuration, it is `melotts`. - action: The method to be called is `setup`. - object: The data type being transmitted is `melotts.setup`. -- model: The model being used is the Chinese model `melotts-zh-cn`. +- model: The model being used is the English model `melotts-en-us`. - response_format: The result is returned as `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. - input: The input is `tts.utf-8`, representing user input. @@ -139,7 +139,7 @@ Example: "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts-zh-cn", + "model": "melotts-en-us", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -335,7 +335,7 @@ Response JSON: "inputs_": [ "tts.utf-8" ], - "model": "melotts-zh-cn", + "model": "melotts-en-us", "response_format": "sys.pcm" }, "error": { From 16dfe700359107a2ec074bdfa0630449577ef56e Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Tue, 22 Apr 2025 18:16:32 +0800 Subject: [PATCH 10/14] [update] update model version --- projects/llm_framework/tools/llm_pack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 370c8b2..14cab95 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -382,11 +382,11 @@ if __name__ == "__main__": 'llm-model-yolo11n-pose':[create_data_deb,'llm-model-yolo11n-pose', '0.3', src_folder, revision], 'llm-model-yolo11n-hand-pose':[create_data_deb,'llm-model-yolo11n-hand-pose', '0.3', src_folder, revision], 'llm-model-yolo11n-seg':[create_data_deb,'llm-model-yolo11n-seg', '0.3', src_folder, revision], - 'llm-model-depth-anything-ax630c':[create_data_deb,'llm-model-depth-anything-ax630c', '0.3', src_folder, revision], + 'llm-model-depth-anything-ax630c':[create_data_deb,'llm-model-depth-anything-ax630c', '0.4', src_folder, revision], 'llm-model-whisper-tiny':[create_data_deb,'llm-model-whisper-tiny', '0.3', src_folder, revision], 'llm-model-whisper-base':[create_data_deb,'llm-model-whisper-base', '0.3', src_folder, revision], 'llm-model-whisper-small':[create_data_deb,'llm-model-whisper-small', '0.3', src_folder, revision], - 'llm-model-silero-vad':[create_data_deb,'llm-model-silero-vad', '0.3', src_folder, revision], + 'llm-model-silero-vad':[create_data_deb,'llm-model-silero-vad', '0.4', src_folder, revision], 'llm-model-qwen2.5-0.5B-prefill-20e':[create_data_deb,'llm-model-qwen2.5-0.5B-prefill-20e', data_version, src_folder, revision], 'llm-model-qwen2.5-0.5B-p256-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-p256-ax630c', '0.4', src_folder, revision], 'llm-model-qwen2.5-0.5B-Int4-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-Int4-ax630c', '0.4', src_folder, revision], From 43c34385c93bfbb744e57d4344d348b5259e704f Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Wed, 23 Apr 2025 09:05:27 +0800 Subject: [PATCH 11/14] [fix] Fix non-utf-8 characters --- .../llm_framework/main_whisper/src/main.cpp | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/projects/llm_framework/main_whisper/src/main.cpp b/projects/llm_framework/main_whisper/src/main.cpp index 73515f1..cc59a97 100644 --- a/projects/llm_framework/main_whisper/src/main.cpp +++ b/projects/llm_framework/main_whisper/src/main.cpp @@ -192,6 +192,34 @@ public: return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0; } + bool is_valid_utf8(const std::string &str) + { + int bytes = 0; + for (unsigned char c : str) { + if (bytes == 0) { + if ((c >> 5) == 0b110) + bytes = 1; + else if ((c >> 4) == 0b1110) + bytes = 2; + else if ((c >> 3) == 0b11110) + bytes = 3; + else if ((c >> 7)) + return false; + } else { + if ((c >> 6) != 0b10) return false; + bytes--; + } + } + return bytes == 0; + } + + void fix_utf8_string(std::string &s) + { + while (!s.empty() && !is_valid_utf8(s)) { + s.pop_back(); + } + } + int load_model(const nlohmann::json &config_body) { if (parse_config(config_body)) { @@ -475,7 +503,7 @@ public: (uint32)mode_config_.token_tables[i].size(), str); s += str; } - + fix_utf8_string(s); if (mode_config_.language == "en" || mode_config_.language == "ja") { if (out_callback_) out_callback_(s, true); } else { From dce7e4d034ecb6913563cbb0f8053e9bc41adabd Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Wed, 23 Apr 2025 14:51:06 +0800 Subject: [PATCH 12/14] [update] Update OpenAI-Plugin --- projects/llm_framework/main_openai_api/SConstruct | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/llm_framework/main_openai_api/SConstruct b/projects/llm_framework/main_openai_api/SConstruct index 490cebc..9098a67 100644 --- a/projects/llm_framework/main_openai_api/SConstruct +++ b/projects/llm_framework/main_openai_api/SConstruct @@ -18,7 +18,7 @@ LINK_SEARCH_PATH = [] STATIC_FILES = [] -ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/m5stack/ModuleLLM-OpenAI-Plugin.git', '6f783440eb2b4ff6566c171ffc2815cb64024928', True) +ModuleLLMOpenAIPluginPath = wget_github_commit('https://github.com/m5stack/ModuleLLM-OpenAI-Plugin.git', '9612b20800eb1708e648744e3cb333c4f743811c', True) python_venv = check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_llm-openai-api-python-venv_v1.6.tar.gz", 'm5stack_llm-openai-api-python-venv_v1.6.tar.gz') From 076ecace17df6cd6e21a875ed46fe185b27821b7 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Thu, 24 Apr 2025 11:30:53 +0800 Subject: [PATCH 13/14] [update] Update LLM VLM STT benchmark --- benchmark/RESULTS.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 benchmark/RESULTS.md diff --git a/benchmark/RESULTS.md b/benchmark/RESULTS.md new file mode 100644 index 0000000..9c49b35 --- /dev/null +++ b/benchmark/RESULTS.md @@ -0,0 +1,33 @@ +# Results + +## ModuleLLM (AX630C) + +### LLM +| model | ttft (ms) | avg-token/s | model version | llm version | +|---------------------------------|------------|-------------|---------------|-------------| +| qwen2.5-0.5B-prefill-20e | 359.8 | 10.32 | v0.2 | v1.8 | +| qwen2.5-0.5B-p256-ax630c | 1126.19 | 10.30 | v0.4 | v1.8 | +| qwen2.5-0.5B-Int4-ax630c | 442.95 | 12.52 | v0.4 | v1.8 | +| qwen2.5-coder-0.5B-ax630c | 361.81 | 10.28 | v0.2 | v1.8 | +| qwen2.5-1.5B-ax630c | 1029.41 | 3.59 | v0.3 | v1.8 | +| qwen2.5-1.5B-p256-ax630c | 3056.54 | 3.57 | v0.4 | v1.8 | +| qwen2.5-1.5B-Int4-ax630c | 1219.54 | 4.63 | v0.4 | v1.8 | +| deepseek-r1-1.5B-ax630c | 1075.04 | 3.57 | v0.3 | v1.8 | +| deepseek-r1-1.5B-p256-ax630c | 3056.86 | 3.57 | v0.4 | v1.8 | +| llama3.2-1B-prefill-ax630c | 891.00 | 4.48 | v0.2 | v1.8 | +| llama3.2-1B-p256-ax630c | 2601.11 | 4.49 | v0.4 | v1.8 | +| openbuddy-llama3.2-1B-ax630c | 891.02 | 4.52 | v0.2 | v1.8 | + +### VLM +| model | ttft (ms) | avg-token/s | image encode (ms) | model version | vlm version | +|---------------------------------|------------|-------------|-------------------|---------------|-------------| +| internvl2.5-1B-364-ax630c | 1117.27 | 10.56 | 1164.61 | v0.4 | v1.7 | +| smolvlm-256M-ax630c | 185.75 | 30.16 | 799.11 | v0.4 | v1.7 | +| smolvlm-500M-ax630c | 365.69 | 13.14 | 838.30 | v0.4 | v1.7 | + +### STT +| model | encode (ms) | avg-decode (ms) | model version | whisper version | +|--------------------|-------------|-----------------|---------------|-----------------| +| whisper-tiny | 248.0 | 32.54 | v0.4 | v1.7 | +| whisper-base | 660.31 | 51.11 | v0.4 | v1.7 | +| whisper-small | 1606.08 | 148.92 | v0.4 | v1.7 | \ No newline at end of file From 3bdf8222d6ff5cf1f8c511ee5e59d7d7791fec0c Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Thu, 24 Apr 2025 11:42:06 +0800 Subject: [PATCH 14/14] [update] Update benchmark --- benchmark/RESULTS.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmark/RESULTS.md b/benchmark/RESULTS.md index 9c49b35..da44e3f 100644 --- a/benchmark/RESULTS.md +++ b/benchmark/RESULTS.md @@ -18,6 +18,8 @@ | llama3.2-1B-p256-ax630c | 2601.11 | 4.49 | v0.4 | v1.8 | | openbuddy-llama3.2-1B-ax630c | 891.02 | 4.52 | v0.2 | v1.8 | +`The input text used by the llm test is "hello!“` + ### VLM | model | ttft (ms) | avg-token/s | image encode (ms) | model version | vlm version | |---------------------------------|------------|-------------|-------------------|---------------|-------------| @@ -25,9 +27,13 @@ | smolvlm-256M-ax630c | 185.75 | 30.16 | 799.11 | v0.4 | v1.7 | | smolvlm-500M-ax630c | 365.69 | 13.14 | 838.30 | v0.4 | v1.7 | +`The image encoding test uses a jpg image with a size of 810*1080` + ### STT | model | encode (ms) | avg-decode (ms) | model version | whisper version | |--------------------|-------------|-----------------|---------------|-----------------| | whisper-tiny | 248.0 | 32.54 | v0.4 | v1.7 | | whisper-base | 660.31 | 51.11 | v0.4 | v1.7 | -| whisper-small | 1606.08 | 148.92 | v0.4 | v1.7 | \ No newline at end of file +| whisper-small | 1606.08 | 148.92 | v0.4 | v1.7 | + +`The STT test uses a 30-second wav English audio` \ No newline at end of file