diff --git a/benchmark/utils/llm.py b/benchmark/utils/llm.py index e2d6a0b..bf77670 100644 --- a/benchmark/utils/llm.py +++ b/benchmark/utils/llm.py @@ -30,7 +30,6 @@ class LLMClient: while True: chunk = self.sock.recv(4096) - logging.info(f"Received chunk: {chunk}") response += chunk while b'\n' in response: @@ -149,21 +148,14 @@ class LLMClient: def test(self, model, input_text): logging.info("Setting up...") setup_response = self.setup(model) - logging.info("Setup response: %s", setup_response) logging.info("Running inference...") inference_result = self.inference(input_text) - logging.info("Inference result: %s", inference_result) logging.info("Exiting...") exit_response = self.exit() - logging.info("Exit response: %s", exit_response) - return { - "setup_response": setup_response, - "inference_result": inference_result, - "exit_response": exit_response - } + return {} if __name__ == "__main__": host = "192.168.20.186" diff --git a/doc/projects_llm_framework_doc/llm_depth_anything_en.md b/doc/projects_llm_framework_doc/llm_depth_anything_en.md index 301cc54..b1d0b51 100644 --- a/doc/projects_llm_framework_doc/llm_depth_anything_en.md +++ b/doc/projects_llm_framework_doc/llm_depth_anything_en.md @@ -27,7 +27,7 @@ Send JSON: - work_id: When configuring the unit, it is `depth_anything`. - action: The method called is `setup`. - object: The data type being transmitted is `depth_anything.setup`. -- model: The model used is the `depth_anything` model. +- model: The model used is the `depth-anything-ax630c` model. - response_format: The return result is `jpeg.base64.stream`. - input: The input is `camera.1001`, which refers to the input from the camera unit, as detailed in the camera unit documentation. diff --git a/doc/projects_llm_framework_doc/llm_depth_anything_zh.md b/doc/projects_llm_framework_doc/llm_depth_anything_zh.md index 846ca34..85f07f4 100644 --- a/doc/projects_llm_framework_doc/llm_depth_anything_zh.md +++ b/doc/projects_llm_framework_doc/llm_depth_anything_zh.md @@ -27,7 +27,7 @@ depth_anything 视觉单元，用于提供图片深度信息。 - work_id：配置单元时，为 `depth_anything`。 - action：调用的方法为 `setup`。 - object：传输的数据类型为 `depth_anything.setup`。 -- model：使用的模型为 `depth_anything` 模型。 +- model：使用的模型为 `depth-anything-ax630c` 模型。 - response_format：返回结果为 `jpeg.base64.stream`。 - input：输入的为 `camera.1001`，代表的是从 camera 单元内部输入，详见 camera 单位文档。 - enoutput：是否启用用户结果输出。 diff --git a/doc/projects_llm_framework_doc/llm_melotts_en.md b/doc/projects_llm_framework_doc/llm_melotts_en.md index f45eb0e..d488deb 100644 --- a/doc/projects_llm_framework_doc/llm_melotts_en.md +++ b/doc/projects_llm_framework_doc/llm_melotts_en.md @@ -16,7 +16,7 @@ Send JSON: "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -28,7 +28,7 @@ Send JSON: - work_id: For configuration, it is `melotts`. - action: The method to be called is `setup`. - object: The data type being transmitted is `melotts.setup`. -- model: The model being used is the Chinese model `melotts_zh-cn`. +- model: The model being used is the Chinese model `melotts-zh-cn`. - response_format: The result is returned as `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. - input: The input is `tts.utf-8`, representing user input. @@ -139,7 +139,7 @@ Example: "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -335,7 +335,7 @@ Response JSON: "inputs_": [ "tts.utf-8" ], - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm" }, "error": { diff --git a/doc/projects_llm_framework_doc/llm_melotts_zh.md b/doc/projects_llm_framework_doc/llm_melotts_zh.md index 7d23b51..8c86e82 100644 --- a/doc/projects_llm_framework_doc/llm_melotts_zh.md +++ b/doc/projects_llm_framework_doc/llm_melotts_zh.md @@ -15,7 +15,7 @@ "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -27,7 +27,7 @@ - work_id：配置单元时，为 `melotts`。 - action：调用的方法为 `setup`。 - object：传输的数据类型为 `melotts.setup`。 -- model：使用的模型为 `melotts_zh-cn` 中文模型。 +- model：使用的模型为 `melotts-zh-cn` 中文模型。 - response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。 - input：输入的为 `tts.utf-8`,代表的是从用户输入。 - enoutput：是否起用用户结果输出。 @@ -134,7 +134,7 @@ error::code 为 0 表示执行成功。 "action": "setup", "object": "melotts.setup", "data": { - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -328,7 +328,7 @@ error::code 为 0 表示执行成功。 "inputs_": [ "tts.utf-8" ], - "model": "melotts_zh-cn", + "model": "melotts-zh-cn", "response_format": "sys.pcm" }, "error": { diff --git a/doc/projects_llm_framework_doc/llm_tts_en.md b/doc/projects_llm_framework_doc/llm_tts_en.md index 7833828..61d0e45 100644 --- a/doc/projects_llm_framework_doc/llm_tts_en.md +++ b/doc/projects_llm_framework_doc/llm_tts_en.md @@ -16,7 +16,7 @@ Send JSON: "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-english-fast", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -28,7 +28,7 @@ Send JSON: - work_id: For configuring the unit, it is `tts`. - action: The method to call is `setup`. - object: The type of data being transmitted is `tts.setup`. -- model: The model used is the `single_speaker_fast` Chinese model. +- model: The model used is the `single-speaker-english-fast` English model. - response_format: The returned result is `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. - input: Input is `tts.utf-8`, representing user input. @@ -46,13 +46,50 @@ Response JSON: }, "object": "None", "request_id": "2", - "work_id": "llm.1003" + "work_id": "tts.1003" } ``` - created: Message creation time, in Unix time. - work_id: The successfully created work_id unit. +## inference + +### streaming input + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8.stream", + "data": { + "delta": "What's ur name?", + "index": 0, + "finish": true + } +} +``` +- object: The data type transmitted is tts.utf-8.stream, indicating a streaming input from the user's UTF-8. +- delta: Segment data of the streaming input. +- index: Index of the segment in the streaming input. +- finish: A flag indicating whether the streaming input has completed. + +### non-streaming input + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8", + "data": "What's ur name?" +} +``` + +- object: The data type transmitted is tts.utf-8, indicating a non-streaming input from the user's UTF-8. +- data: Data for non-streaming input. + ## link Link the output of the upper unit. @@ -102,7 +139,7 @@ Example: "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -298,7 +335,7 @@ Response JSON: "inputs_": [ "tts.utf-8" ], - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm" }, "error": { diff --git a/doc/projects_llm_framework_doc/llm_tts_zh.md b/doc/projects_llm_framework_doc/llm_tts_zh.md index 719d4fb..7e4ab18 100644 --- a/doc/projects_llm_framework_doc/llm_tts_zh.md +++ b/doc/projects_llm_framework_doc/llm_tts_zh.md @@ -15,7 +15,7 @@ "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm", "input": "tts.utf-8", "enoutput": false @@ -27,7 +27,7 @@ - work_id：配置单元时，为 `tts`。 - action：调用的方法为 `setup`。 - object：传输的数据类型为 `tts.setup`。 -- model：使用的模型为 `single_speaker_fast` 中文模型。 +- model：使用的模型为 `single-speaker-fast` 中文模型。 - response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。 - input：输入的为 `tts.utf-8`,代表的是从用户输入。 - enoutput：是否起用用户结果输出。 @@ -44,13 +44,49 @@ }, "object": "None", "request_id": "2", - "work_id": "llm.1003" + "work_id": "tts.1003" } ``` - created：消息创建时间，unix 时间。 - work_id：返回成功创建的 work_id 单元。 +## inference + +### 流式输入 + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8.stream", + "data": { + "delta": "今天天气真好！", + "index": 0, + "finish": true + } +} +``` +- object：传输的数据类型为 `tts.utf-8.stream` 代表的是从用户 utf-8 的流式输入 +- delta：流式输入的分段数据 +- index：流式输入的分段索引 +- finish:流式输入是否完成的标志位 + +### 非流式输入 + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "inference", + "object": "tts.utf-8", + "data": "今天天气真好！" +} +``` +- object：传输的数据类型为 `tts.utf-8` 代表的是从用户 utf-8 的非流式输入 +- data：非流式输入的数据 + ## link 链接上级单元的输出。 @@ -98,7 +134,7 @@ error::code 为 0 表示执行成功。 "action": "setup", "object": "tts.setup", "data": { - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm", "input": [ "tts.utf-8", @@ -294,7 +330,7 @@ error::code 为 0 表示执行成功。 "inputs_": [ "tts.utf-8" ], - "model": "single_speaker_fast", + "model": "single-speaker-fast", "response_format": "sys.pcm" }, "error": { diff --git a/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json b/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json index ddf0a6c..fd91eb3 100644 --- a/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json +++ b/projects/llm_framework/main_depth_anything/mode_depth-anything-ax630c.json @@ -1,7 +1,9 @@ { - "mode":"depth_anything", + "mode":"depth-anything-ax630c", "type":"cv", "homepage":"https://github.com/DepthAnything/Depth-Anything-V2", + "compile_flage":"pulsar2 build --input depth_anything.onnx --config config.json --output_dir output --output_name depth_anything.axmodel --target_hardware AX620E", + "pulsar_version":"2.5-2d5d0fa3", "capabilities":[ "Segmentation" ], diff --git a/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json b/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json index e7af2c5..ee9f57c 100644 --- a/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json +++ b/projects/llm_framework/main_melotts/mode_melotts-zh-cn.json @@ -1,7 +1,9 @@ { - "mode": "melotts_zh-cn", + "mode": "melotts-zh-cn", "type": "tts", "homepage":"https://huggingface.co/myshell-ai/MeloTTS-Chinese", + "compile_flage":"pulsar2 build --input decoder.onnx --config config_decoder_u16.json --output_dir decoder --output_name decoder.axmodel --target_hardware AX620E --npu_mode NPU2 --compiler.check 0", + "pulsar_version":"3.2-99f14d0a", "capabilities": [ "tts", "Chinese" diff --git a/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json b/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json index e311a4a..cbaccaa 100644 --- a/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json +++ b/projects/llm_framework/main_tts/mode_single-speaker-english-fast.json @@ -1,5 +1,5 @@ { - "mode": "single_speaker_english_fast", + "mode": "single-speaker-english-fast", "type": "tts", "homepage":"https://github.com/huakunyang/SummerTTS", "capabilities": [ diff --git a/projects/llm_framework/main_tts/mode_single-speaker-fast.json b/projects/llm_framework/main_tts/mode_single-speaker-fast.json index ca1ede7..6fadf2f 100644 --- a/projects/llm_framework/main_tts/mode_single-speaker-fast.json +++ b/projects/llm_framework/main_tts/mode_single-speaker-fast.json @@ -1,5 +1,5 @@ { - "mode": "single_speaker_fast", + "mode": "single-speaker-fast", "type": "tts", "homepage":"https://github.com/huakunyang/SummerTTS", "capabilities": [ diff --git a/projects/llm_framework/main_vad/mode_silero-vad.json b/projects/llm_framework/main_vad/mode_silero-vad.json index d262af4..a4d8eae 100644 --- a/projects/llm_framework/main_vad/mode_silero-vad.json +++ b/projects/llm_framework/main_vad/mode_silero-vad.json @@ -1,5 +1,5 @@ { - "mode": "silero_vad", + "mode": "silero-vad", "type": "vad", "homepage":"https://github.com/snakers4/silero-vad", "capabilities": [ diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 38ca64e..21c3831 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -69,23 +69,23 @@ def create_lib_deb(package_name, version, src_folder, revision = 'm5stack1'): # if os.path.exists(zip_file_extrpath): # shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'opt/m5stack/scripts')) - # zip_file = 'm5stack_dist-packages.tar.gz' - # down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz' - # zip_file_extrpath = 'm5stack_dist-packages' - # if not os.path.exists(zip_file_extrpath): - # # Downloading via HTTP (more common) - # if not os.path.exists(zip_file): - # response = requests.get(down_url) - # if response.status_code == 200: - # with open(zip_file, 'wb') as file: - # file.write(response.content) - # else: - # print("{} down failed".format(down_url)) - # with tarfile.open(zip_file, 'r:gz') as tar: - # tar.extractall(path=zip_file_extrpath) - # print("The {} download successful.".format(down_url)) - # if os.path.exists(zip_file_extrpath): - # shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages')) + zip_file = 'm5stack_dist-packages.tar.gz' + down_url = 'https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_dist-packages.tar.gz' + zip_file_extrpath = 'm5stack_dist-packages' + if not os.path.exists(zip_file_extrpath): + # Downloading via HTTP (more common) + if not os.path.exists(zip_file): + response = requests.get(down_url) + if response.status_code == 200: + with open(zip_file, 'wb') as file: + file.write(response.content) + else: + print("{} down failed".format(down_url)) + with tarfile.open(zip_file, 'r:gz') as tar: + tar.extractall(path=zip_file_extrpath) + print("The {} download successful.".format(down_url)) + if os.path.exists(zip_file_extrpath): + shutil.copytree(zip_file_extrpath, os.path.join(deb_folder, 'usr/local/lib/python3.10/dist-packages')) os.makedirs(os.path.join(deb_folder, 'DEBIAN'), exist_ok = True) with open(os.path.join(deb_folder, 'DEBIAN/control'),'w') as f: @@ -374,9 +374,9 @@ if __name__ == "__main__": 'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23', data_version, src_folder, revision], 'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01', '0.3', src_folder, revision], 'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01', '0.3', src_folder, revision], - 'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', data_version, src_folder, revision], - 'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', data_version, src_folder, revision], - 'llm-model-melotts-zh-cn':[create_data_deb,'llm-model-melotts-zh-cn', '0.4', src_folder, revision], + 'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', '0.3', src_folder, revision], + 'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', '0.3', src_folder, revision], + 'llm-model-melotts-zh-cn':[create_data_deb,'llm-model-melotts-zh-cn', '0.5', src_folder, revision], 'llm-model-yolo11n':[create_data_deb,'llm-model-yolo11n', data_version, src_folder, revision], 'llm-model-yolo11n-pose':[create_data_deb,'llm-model-yolo11n-pose', '0.3', src_folder, revision], 'llm-model-yolo11n-hand-pose':[create_data_deb,'llm-model-yolo11n-hand-pose', '0.3', src_folder, revision],