diff --git a/doc/projects_llm_framework_doc/llm_asr_en.md b/doc/projects_llm_framework_doc/llm_asr_en.md index 451b5a0..adf681a 100644 --- a/doc/projects_llm_framework_doc/llm_asr_en.md +++ b/doc/projects_llm_framework_doc/llm_asr_en.md @@ -1,30 +1,35 @@ # llm-asr -A speech-to-text module that provides speech-to-text services. It supports both Chinese and English models for converting speech to text in these languages. + +A speech-to-text module that provides speech-to-text services. It supports both Chinese and English models for +converting speech to text in these languages. ## setup + Configure the unit. Send JSON: + ```json { - "request_id": "2", - "work_id": "asr", - "action": "setup", - "object": "asr.setup", - "data": { - "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", - "response_format": "asr.utf-8.stream", - "input": "sys.pcm", - "enoutput": true, - "endpoint_config.rule1.min_trailing_silence": 2.4, - "endpoint_config.rule2.min_trailing_silence": 1.2, - "endpoint_config.rule3.min_trailing_silence": 30.1, - "endpoint_config.rule1.must_contain_nonsilence": true, - "endpoint_config.rule2.must_contain_nonsilence": true, - "endpoint_config.rule3.must_contain_nonsilence": true - } + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": "sys.pcm", + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence": 2.4, + "endpoint_config.rule2.min_trailing_silence": 1.2, + "endpoint_config.rule3.min_trailing_silence": 30.1, + "endpoint_config.rule1.must_contain_nonsilence": true, + "endpoint_config.rule2.must_contain_nonsilence": true, + "endpoint_config.rule3.must_contain_nonsilence": true + } } ``` + - request_id: Reference basic data explanation. - work_id: For configuration units, it is `asr`. - action: The method to be called is `setup`. @@ -36,36 +41,39 @@ Send JSON: - endpoint_config.rule1.min_trailing_silence: A speech break occurs 2.4 seconds after wake-up. - endpoint_config.rule2.min_trailing_silence: A speech break occurs 1.2 seconds after a speech is recognized. - endpoint_config.rule3.min_trailing_silence: A speech break occurs after a maximum of 30.1 seconds of recognition. - + Response JSON: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "2", - "work_id": "asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "asr.1001" } ``` + - created: Message creation time, Unix time. - work_id: The successfully created work_id unit. ## link + Link the output of the upstream unit. Send JSON: + ```json { - "request_id": "3", - "work_id": "asr.1001", - "action": "link", - "object": "work_id", - "data": "kws.1000" + "request_id": "3", + "work_id": "asr.1001", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` @@ -73,20 +81,22 @@ Response JSON: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "3", - "work_id": "asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "asr.1001" } ``` + error::code of 0 indicates successful execution. -Link the asr and kws units so that when kws sends wake-up data, the asr unit starts recognizing the user's speech, pauses automatically after recognition, and waits until the next wake-up. +Link the asr and kws units so that when kws sends wake-up data, the asr unit starts recognizing the user's speech, +pauses automatically after recognition, and waits until the next wake-up. > **Ensure that kws is already configured and working when linking. Linking can also be done during the setup phase.** @@ -94,36 +104,41 @@ Example: ```json { - "request_id": "2", - "work_id": "asr", - "action": "setup", - "object": "asr.setup", - "data": { - "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", - "response_format": "asr.utf-8.stream", - "input": ["sys.pcm", "kws.1000"], - "enoutput": true, - "endpoint_config.rule1.min_trailing_silence": 2.4, - "endpoint_config.rule2.min_trailing_silence": 1.2, - "endpoint_config.rule3.min_trailing_silence": 30.1, - "endpoint_config.rule1.must_contain_nonsilence": false, - "endpoint_config.rule2.must_contain_nonsilence": false, - "endpoint_config.rule3.must_contain_nonsilence": false - } + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": [ + "sys.pcm", + "kws.1000" + ], + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence": 2.4, + "endpoint_config.rule2.min_trailing_silence": 1.2, + "endpoint_config.rule3.min_trailing_silence": 30.1, + "endpoint_config.rule1.must_contain_nonsilence": false, + "endpoint_config.rule2.must_contain_nonsilence": false, + "endpoint_config.rule3.must_contain_nonsilence": false + } } ``` ## unlink + Unlink. Send JSON: + ```json { - "request_id": "4", - "work_id": "asr.1001", - "action": "unlink", - "object": "work_id", - "data": "kws.1000" + "request_id": "4", + "work_id": "asr.1001", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` @@ -131,28 +146,31 @@ Response JSON: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "4", - "work_id": "asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "asr.1001" } ``` + error::code of 0 indicates successful execution. ## pause + Pause the unit's work. Send JSON: + ```json { - "request_id": "5", - "work_id": "asr.1001", - "action": "pause" + "request_id": "5", + "work_id": "asr.1001", + "action": "pause" } ``` @@ -160,28 +178,31 @@ Response JSON: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "5", - "work_id": "asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "asr.1001" } ``` + error::code of 0 indicates successful execution. ## work + Resume the unit's work. Send JSON: + ```json { - "request_id": "6", - "work_id": "asr.1001", - "action": "work" + "request_id": "6", + "work_id": "asr.1001", + "action": "work" } ``` @@ -189,28 +210,31 @@ Response JSON: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "6", - "work_id": "asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "asr.1001" } ``` + error::code of 0 indicates successful execution. ## exit + Exit the unit. Send JSON: + ```json { - "request_id": "7", - "work_id": "asr.1001", - "action": "exit" + "request_id": "7", + "work_id": "asr.1001", + "action": "exit" } ``` @@ -218,17 +242,18 @@ Response JSON: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "7", - "work_id": "asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "asr.1001" } ``` + error::code of 0 indicates successful execution. ## Task Information @@ -236,62 +261,67 @@ error::code of 0 indicates successful execution. Get task list. Send JSON: + ```json { - "request_id": "2", - "work_id": "asr", - "action": "taskinfo" + "request_id": "2", + "work_id": "asr", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731580350, - "data":[ - "asr.1001" - ], - "error":{ - "code":0, - "message":"" - }, - "object":"asr.tasklist", - "request_id":"2", - "work_id":"asr" + "created": 1731580350, + "data": [ + "asr.1001" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "asr.tasklist", + "request_id": "2", + "work_id": "asr" } ``` Get task runtime parameters. Send JSON: + ```json { - "request_id": "2", - "work_id": "asr.1001", - "action": "taskinfo" + "request_id": "2", + "work_id": "asr.1001", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731579679, - "data":{ - "enoutput":false, - "inputs_":[ - "sys.pcm" - ], - "model":"sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", - "response_format":"asr.utf-8-stream" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"asr.taskinfo", - "request_id":"2", - "work_id":"asr.1001" + "created": 1731579679, + "data": { + "enoutput": false, + "inputs_": [ + "sys.pcm" + ], + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8-stream" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "asr.taskinfo", + "request_id": "2", + "work_id": "asr.1001" } ``` -> **Note: The work_id increases according to the initialization registration order of the unit and is not a fixed index value.** \ No newline at end of file +> **Note: The work_id increases according to the initialization registration order of the unit and is not a fixed index +value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_asr_zh.md b/doc/projects_llm_framework_doc/llm_asr_zh.md index 7c07409..e139f28 100644 --- a/doc/projects_llm_framework_doc/llm_asr_zh.md +++ b/doc/projects_llm_framework_doc/llm_asr_zh.md @@ -1,30 +1,34 @@ # llm-asr -语音转文字单元，用于提供语音转文字服务，可选择中英文模型，用于提供中英文语音转文字服务。 + +语音转文字单元，用于提供语音转文字服务，可选择中英文模型，用于提供中英文语音转文字服务。 ## setup + 配置单元工作。 发送 json： + ```json { - "request_id": "2", - "work_id": "asr", - "action": "setup", - "object": "asr.setup", - "data": { - "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", - "response_format": "asr.utf-8.stream", - "input": "sys.pcm", - "enoutput": true, - "endpoint_config.rule1.min_trailing_silence":2.4, - "endpoint_config.rule2.min_trailing_silence":1.2, - "endpoint_config.rule3.min_trailing_silence":30.1, - "endpoint_config.rule1.must_contain_nonsilence":true, - "endpoint_config.rule2.must_contain_nonsilence":true, - "endpoint_config.rule3.must_contain_nonsilence":true - } + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": "sys.pcm", + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence": 2.4, + "endpoint_config.rule2.min_trailing_silence": 1.2, + "endpoint_config.rule3.min_trailing_silence": 30.1, + "endpoint_config.rule1.must_contain_nonsilence": true, + "endpoint_config.rule2.must_contain_nonsilence": true, + "endpoint_config.rule3.must_contain_nonsilence": true + } } ``` + - request_id：参考基本数据解释。 - work_id：配置单元时，为 `asr`。 - action：调用的方法为 `setup`。 @@ -41,15 +45,15 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "asr.1001" } ``` @@ -57,16 +61,18 @@ - work_id：返回成功创建的 work_id 单元。 ## link + 链接上级单元的输出。 发送 json： + ```json { - "request_id": "3", - "work_id": "asr.1001", - "action": "link", - "object":"work_id", - "data":"kws.1000" + "request_id": "3", + "work_id": "asr.1001", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` @@ -74,17 +80,18 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "asr.1001" } ``` + error::code 为 0 表示执行成功。 将 asr 和 kws 单元链接起来，当 kws 发出唤醒数据时，asr 单元开始识别用户的语音，识别完成后自动暂停，直到下一次的唤醒。 @@ -95,36 +102,41 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "asr", - "action": "setup", - "object": "asr.setup", - "data": { - "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", - "response_format": "asr.utf-8.stream", - "input": ["sys.pcm","kws.1000"], - "enoutput": true, - "endpoint_config.rule1.min_trailing_silence":2.4, - "endpoint_config.rule2.min_trailing_silence":1.2, - "endpoint_config.rule3.min_trailing_silence":30.1, - "endpoint_config.rule1.must_contain_nonsilence":false, - "endpoint_config.rule2.must_contain_nonsilence":false, - "endpoint_config.rule3.must_contain_nonsilence":false - } + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": [ + "sys.pcm", + "kws.1000" + ], + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence": 2.4, + "endpoint_config.rule2.min_trailing_silence": 1.2, + "endpoint_config.rule3.min_trailing_silence": 30.1, + "endpoint_config.rule1.must_contain_nonsilence": false, + "endpoint_config.rule2.must_contain_nonsilence": false, + "endpoint_config.rule3.must_contain_nonsilence": false + } } ``` ## unlink + 取消链接。 发送 json： + ```json { - "request_id": "4", - "work_id": "asr.1001", - "action": "unlink", - "object":"work_id", - "data":"kws.1000" + "request_id": "4", + "work_id": "asr.1001", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` @@ -132,29 +144,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "asr.1001" } ``` error::code 为 0 表示执行成功。 ## pause + 暂停单元工作。 发送 json： + ```json { - "request_id": "5", - "work_id": "asr.1001", - "action": "pause", + "request_id": "5", + "work_id": "asr.1001", + "action": "pause" } ``` @@ -162,28 +176,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "asr.1001" } ``` + error::code 为 0 表示执行成功。 ## work + 恢复单元工作。 发送 json： + ```json { - "request_id": "6", - "work_id": "asr.1001", - "action": "work", + "request_id": "6", + "work_id": "asr.1001", + "action": "work" } ``` @@ -191,28 +208,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"6", - "work_id":"asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "asr.1001" } ``` + error::code 为 0 表示执行成功。 ## exit + 单元退出。 发送 json： + ```json { - "request_id": "7", - "work_id": "asr.1001", - "action": "exit", + "request_id": "7", + "work_id": "asr.1001", + "action": "exit" } ``` @@ -220,17 +240,18 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"asr.1001" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "asr.1001" } ``` + error::code 为 0 表示执行成功。 ## taskinfo @@ -238,11 +259,12 @@ error::code 为 0 表示执行成功。 获取任务列表。 发送 json： + ```json { - "request_id": "2", - "work_id": "asr", - "action": "taskinfo" + "request_id": "2", + "work_id": "asr", + "action": "taskinfo" } ``` @@ -250,17 +272,17 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731580350, - "data":[ - "asr.1001" - ], - "error":{ - "code":0, - "message":"" - }, - "object":"asr.tasklist", - "request_id":"2", - "work_id":"asr" + "created": 1731580350, + "data": [ + "asr.1001" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "asr.tasklist", + "request_id": "2", + "work_id": "asr" } ``` @@ -268,9 +290,9 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "asr.1001", - "action": "taskinfo" + "request_id": "2", + "work_id": "asr.1001", + "action": "taskinfo" } ``` @@ -278,24 +300,23 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731579679, - "data":{ - "enoutput":false, - "inputs_":[ - "sys.pcm" - ], - "model":"sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", - "response_format":"asr.utf-8-stream" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"asr.taskinfo", - "request_id":"2", - "work_id":"asr.1001" + "created": 1731579679, + "data": { + "enoutput": false, + "inputs_": [ + "sys.pcm" + ], + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8-stream" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "asr.taskinfo", + "request_id": "2", + "work_id": "asr.1001" } ``` - > **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_audio_en.md b/doc/projects_llm_framework_doc/llm_audio_en.md index c11521e..117b8ed 100644 --- a/doc/projects_llm_framework_doc/llm_audio_en.md +++ b/doc/projects_llm_framework_doc/llm_audio_en.md @@ -1,13 +1,16 @@ # llm-audio + System audio unit for providing system audio playback and recording. ## API + - play: Play audio. This call will interrupt any previously unfinished audio playback. - queue_play: Queue playback. Add audio to the playback queue. - play_stop: Stop playback. - queue_play_stop: Clear the playback queue. - audio_status: Get the current playback status. - cap: Start a recording task, can be called repeatedly. -- cap_stop: Stop a recording task, can be called repeatedly. The last call will stop the data output of the recording channel. +- cap_stop: Stop a recording task, can be called repeatedly. The last call will stop the data output of the recording + channel. - cap_stop_all: Force stop all ongoing recording tasks. - setup: Configure the working parameters of the recording unit. \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_audio_zh.md b/doc/projects_llm_framework_doc/llm_audio_zh.md index 24ea293..b62ab5a 100644 --- a/doc/projects_llm_framework_doc/llm_audio_zh.md +++ b/doc/projects_llm_framework_doc/llm_audio_zh.md @@ -1,8 +1,9 @@ # llm-audio + 系统音频单元，用于提供系统音频播放和录音。 - ## API + - play: 播放音频，本次调用会打断上次未播放的完的音频。 - queue_play：队列播放，将音频加入播放队列。 - play_stop：停止播放。 diff --git a/doc/projects_llm_framework_doc/llm_camera.md b/doc/projects_llm_framework_doc/llm_camera.md deleted file mode 100644 index 891c7cb..0000000 --- a/doc/projects_llm_framework_doc/llm_camera.md +++ /dev/null @@ -1,147 +0,0 @@ -# llm-camera -视频源单元，用于从 USB V4L2 视频设备中获取视频流到内部通道。 - -## setup -配置单元工作。 - -发送 json： -```json -{ - "request_id":"2", - "work_id":"camera", - "action":"setup", - "object":"camera.setup", - "data":{ - "response_format":"camera.raw", - "input":"/dev/video0", - "enoutput":false, - "frame_width":320, - "frame_height":320 - } -} -``` -- request_id：参考基本数据解释。 -- work_id：配置单元时，为 `camera`。 -- action：调用的方法为 `setup`。 -- object：传输的数据类型为 `camera.setup`。 -- response_format：返回结果为 `camera.raw`，是 yuv422 格式。 -- input：读取的设备名。 -- frame_width：输出的视频帧宽。 -- frame_height：输出的视频帧高。 -- enoutput：是否起用用户结果输出。如果不需要获取摄像头图片，请不要开启该参数，视频流会增加信道的通信压力。 - -响应 json： - -```json -{ - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"camera.1003" -} -``` -- created：消息创建时间，unix 时间。 -- work_id：返回成功创建的 work_id 单元。 - - -## exit - -单元退出。 - -发送 json： - -```json -{ - "request_id": "7", - "work_id": "camera.1003", - "action": "exit", -} -``` - -响应 json： - -```json -{ - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"camera.1003" -} -``` - -error::code 为 0 表示执行成功。 - -## taskinfo - -获取任务列表。 - -发送 json： -```json -{ - "request_id": "2", - "work_id": "camera", - "action": "taskinfo" -} -``` - -响应 json： - -```json -{ - "created":1731652311, - "data":["camera.1003"], - "error":{ - "code":0, - "message":"" - }, - "object":"camera.tasklist", - "request_id":"2", - "work_id":"camera" -} -``` - -获取任务运行参数。 - -发送 json： -```json -{ - "request_id": "2", - "work_id": "camera.1003", - "action": "taskinfo" -} -``` - -响应 json： - -```json -{ - "created":1731652344, - "data":{ - "enoutput":false, - "response_format":"camera.raw", - "input":"/dev/video0", - "frame_width":320, - "frame_height":320 - }, - "error":{ - "code":0, - "message":"" - }, - "object":"camera.taskinfo", - "request_id":"2", - "work_id":"camera.1003" -} -``` - - -> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_camera_en.md b/doc/projects_llm_framework_doc/llm_camera_en.md new file mode 100644 index 0000000..f7a93f9 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_camera_en.md @@ -0,0 +1,156 @@ +# llm-camera + +Video source unit for obtaining video streams from USB V4L2 video devices to internal channels. + +## setup + +Configuration unit operation. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "camera", + "action": "setup", + "object": "camera.setup", + "data": { + "response_format": "camera.raw", + "input": "/dev/video0", + "enoutput": false, + "frame_width": 320, + "frame_height": 320 + } +} +``` + +- request_id: Reference basic data explanation. +- work_id: When configuring the unit, it is `camera`. +- action: The method being called is `setup`. +- object: The data type being transmitted is `camera.setup`. +- response_format: The returned result is `camera.raw`, which is in YUV422 format. +- input: The name of the device being read. +- frame_width: The output video frame width. +- frame_height: The output video frame height. +- enoutput: Whether to enable the user result output. If camera images are not needed, do not enable this parameter, as + the video stream will increase the communication load of the channel. + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "camera.1003" +} +``` + +- created: Message creation time, in Unix time. +- work_id: The work_id unit successfully created. + +## exit + +Exit the unit. + +Send JSON: + +```json +{ + "request_id": "7", + "work_id": "camera.1003", + "action": "exit" +} +``` + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "camera.1003" +} +``` + +error::code of 0 indicates successful execution. + +## taskinfo + +Get the task list. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "camera", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1731652311, + "data": [ + "camera.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "camera.tasklist", + "request_id": "2", + "work_id": "camera" +} +``` + +Get task running parameters. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "camera.1003", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1731652344, + "data": { + "enoutput": false, + "response_format": "camera.raw", + "input": "/dev/video0", + "frame_width": 320, + "frame_height": 320 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "camera.taskinfo", + "request_id": "2", + "work_id": "camera.1003" +} +``` + +> **Note: The work_id increases according to the order of unit initialization registration, not as a fixed index value. +** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_camera_zh.md b/doc/projects_llm_framework_doc/llm_camera_zh.md new file mode 100644 index 0000000..3610a81 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_camera_zh.md @@ -0,0 +1,154 @@ +# llm-camera + +视频源单元，用于从 USB V4L2 视频设备中获取视频流到内部通道。 + +## setup + +配置单元工作。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "camera", + "action": "setup", + "object": "camera.setup", + "data": { + "response_format": "camera.raw", + "input": "/dev/video0", + "enoutput": false, + "frame_width": 320, + "frame_height": 320 + } +} +``` + +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `camera`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `camera.setup`。 +- response_format：返回结果为 `camera.raw`，是 yuv422 格式。 +- input：读取的设备名。 +- frame_width：输出的视频帧宽。 +- frame_height：输出的视频帧高。 +- enoutput：是否起用用户结果输出。如果不需要获取摄像头图片，请不要开启该参数，视频流会增加信道的通信压力。 + +响应 json： + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "camera.1003" +} +``` + +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## exit + +单元退出。 + +发送 json： + +```json +{ + "request_id": "7", + "work_id": "camera.1003", + "action": "exit" +} +``` + +响应 json： + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "camera.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "camera", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1731652311, + "data": [ + "camera.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "camera.tasklist", + "request_id": "2", + "work_id": "camera" +} +``` + +获取任务运行参数。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "camera.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1731652344, + "data": { + "enoutput": false, + "response_format": "camera.raw", + "input": "/dev/video0", + "frame_width": 320, + "frame_height": 320 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "camera.taskinfo", + "request_id": "2", + "work_id": "camera.1003" +} +``` + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_kws_en.md b/doc/projects_llm_framework_doc/llm_kws_en.md index b379d4f..7504e26 100644 --- a/doc/projects_llm_framework_doc/llm_kws_en.md +++ b/doc/projects_llm_framework_doc/llm_kws_en.md @@ -1,25 +1,30 @@ # llm-kws -Voice wake-up unit, used to provide voice wake-up services. You can choose between Chinese and English models to provide wake-up services in either language. + +Voice wake-up unit, used to provide voice wake-up services. You can choose between Chinese and English models to provide +wake-up services in either language. ## setup + Configure the unit to work. Send JSON: + ```json { - "request_id": "2", - "work_id": "kws", - "action": "setup", - "object": "kws.setup", - "data": { - "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", - "response_format": "kws.bool", - "input": "sys.pcm", - "enoutput": true, - "kws": "你好你好" - } + "request_id": "2", + "work_id": "kws", + "action": "setup", + "object": "kws.setup", + "data": { + "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format": "kws.bool", + "input": "sys.pcm", + "enoutput": true, + "kws": "你好你好" + } } ``` + - request_id: Refer to basic data explanation. - work_id: When configuring the unit, it is `kws`. - action: The method called is `setup`. @@ -34,29 +39,32 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "kws.1000" } ``` + - created: Message creation time in Unix time. - work_id: The successfully created work_id unit returned. ## pause + Pause the unit's work. Send JSON: + ```json { - "request_id": "3", - "work_id": "kws.1000", - "action": "pause", + "request_id": "3", + "work_id": "kws.1000", + "action": "pause" } ``` @@ -64,28 +72,31 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "kws.1000" } ``` + error::code of 0 indicates successful execution. ## work + Resume the unit's work. Send JSON: + ```json { - "request_id": "4", - "work_id": "kws.1000", - "action": "work", + "request_id": "4", + "work_id": "kws.1000", + "action": "work" } ``` @@ -93,28 +104,31 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "kws.1000" } ``` + error::code of 0 indicates successful execution. ## exit + Exit the unit. Send JSON: + ```json { - "request_id": "5", - "work_id": "kws.1000", - "action": "exit", + "request_id": "5", + "work_id": "kws.1000", + "action": "exit" } ``` @@ -122,17 +136,18 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "kws.1000" } ``` + error::code of 0 indicates successful execution. ## Task Information @@ -140,60 +155,67 @@ error::code of 0 indicates successful execution. Get task list. Send JSON: + ```json { - "request_id": "2", - "work_id": "kws", - "action": "taskinfo" + "request_id": "2", + "work_id": "kws", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731580350, - "data":[ - "kws.1000" - ], - "error":{ - "code":0, - "message":"" - }, - "object":"kws.tasklist", - "request_id":"2", - "work_id":"kws" + "created": 1731580350, + "data": [ + "kws.1000" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "kws.tasklist", + "request_id": "2", + "work_id": "kws" } ``` Get task runtime parameters. Send JSON: + ```json { - "request_id": "2", - "work_id": "kws.1000", - "action": "taskinfo" + "request_id": "2", + "work_id": "kws.1000", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731652086, - "data":{ - "enoutput":true, - "inputs_":["sys.pcm"], - "model":"sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", - "response_format":"kws.bool" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"kws.taskinfo", - "request_id":"2", - "work_id":"kws.1000" + "created": 1731652086, + "data": { + "enoutput": true, + "inputs_": [ + "sys.pcm" + ], + "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format": "kws.bool" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "kws.taskinfo", + "request_id": "2", + "work_id": "kws.1000" } ``` -> **Note: work_id increases according to the initialization registration order of the unit and is not a fixed index value.** \ No newline at end of file +> **Note: work_id increases according to the initialization registration order of the unit and is not a fixed index +value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_kws_zh.md b/doc/projects_llm_framework_doc/llm_kws_zh.md index 8633aa3..86dcbfa 100644 --- a/doc/projects_llm_framework_doc/llm_kws_zh.md +++ b/doc/projects_llm_framework_doc/llm_kws_zh.md @@ -1,25 +1,29 @@ # llm-kws -语音唤醒单元，用于提供语音唤醒服务，可选择中英文模型，用于提供中英文唤醒服务。 + +语音唤醒单元，用于提供语音唤醒服务，可选择中英文模型，用于提供中英文唤醒服务。 ## setup + 配置单元工作。 发送 json： + ```json { - "request_id": "2", - "work_id": "kws", - "action": "setup", - "object": "kws.setup", - "data": { - "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", - "response_format": "kws.bool", - "input": "sys.pcm", - "enoutput": true, - "kws": "你好你好" - } + "request_id": "2", + "work_id": "kws", + "action": "setup", + "object": "kws.setup", + "data": { + "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format": "kws.bool", + "input": "sys.pcm", + "enoutput": true, + "kws": "你好你好" + } } ``` + - request_id：参考基本数据解释。 - work_id：配置单元时，为 `kws`。 - action：调用的方法为 `setup`。 @@ -34,15 +38,15 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "kws.1000" } ``` @@ -50,14 +54,16 @@ - work_id：返回成功创建的 work_id 单元。 ## pause + 暂停单元工作。 发送 json： + ```json { - "request_id": "3", - "work_id": "kws.1000", - "action": "pause", + "request_id": "3", + "work_id": "kws.1000", + "action": "pause" } ``` @@ -65,29 +71,31 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "kws.1000" } ``` error::code 为 0 表示执行成功。 ## work + 恢复单元工作。 发送 json： + ```json { - "request_id": "4", - "work_id": "kws.1000", - "action": "work", + "request_id": "4", + "work_id": "kws.1000", + "action": "work" } ``` @@ -95,29 +103,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "kws.1000" } ``` error::code 为 0 表示执行成功。 ## exit + 单元退出。 发送 json： + ```json { - "request_id": "5", - "work_id": "kws.1000", - "action": "exit", + "request_id": "5", + "work_id": "kws.1000", + "action": "exit" } ``` @@ -125,15 +135,15 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"kws.1000" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "kws.1000" } ``` @@ -144,11 +154,12 @@ error::code 为 0 表示执行成功。 获取任务列表。 发送 json： + ```json { - "request_id": "2", - "work_id": "kws", - "action": "taskinfo" + "request_id": "2", + "work_id": "kws", + "action": "taskinfo" } ``` @@ -156,17 +167,17 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731580350, - "data":[ - "kws.1000" - ], - "error":{ - "code":0, - "message":"" - }, - "object":"kws.tasklist", - "request_id":"2", - "work_id":"kws" + "created": 1731580350, + "data": [ + "kws.1000" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "kws.tasklist", + "request_id": "2", + "work_id": "kws" } ``` @@ -174,9 +185,9 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "kws.1000", - "action": "taskinfo" + "request_id": "2", + "work_id": "kws.1000", + "action": "taskinfo" } ``` @@ -184,22 +195,23 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652086, - "data":{ - "enoutput":true, - "inputs_":["sys.pcm"], - "model":"sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", - "response_format":"kws.bool" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"kws.taskinfo", - "request_id":"2", - "work_id":"kws.1000" + "created": 1731652086, + "data": { + "enoutput": true, + "inputs_": [ + "sys.pcm" + ], + "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format": "kws.bool" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "kws.taskinfo", + "request_id": "2", + "work_id": "kws.1000" } ``` - > **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** diff --git a/doc/projects_llm_framework_doc/llm_llm_en.md b/doc/projects_llm_framework_doc/llm_llm_en.md index 9ab5b72..9e6d909 100644 --- a/doc/projects_llm_framework_doc/llm_llm_en.md +++ b/doc/projects_llm_framework_doc/llm_llm_en.md @@ -1,26 +1,30 @@ # llm-llm + Large Model Unit, used to provide large model inference services. ## setup + Configure the unit. Send json: + ```json { - "request_id": "2", - "work_id": "llm", - "action": "setup", - "object": "llm.setup", - "data": { - "model": "qwen2.5-0.5B-prefill-20e", - "response_format": "llm.utf-8.stream", - "input": "llm.utf-8", - "enoutput": true, - "max_token_len": 256, - "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." - } + "request_id": "2", + "work_id": "llm", + "action": "setup", + "object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": "llm.utf-8", + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } } ``` + - request_id: Refer to the basic data explanation. - work_id: For configuration unit, it is `llm`. - action: The method called is `setup`. @@ -36,31 +40,34 @@ Response json: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "2", - "work_id": "llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "llm.1002" } ``` + - created: Message creation time, unix time. - work_id: The successfully created work_id unit. ## link + Link the output of the upper unit. Send json: + ```json { - "request_id": "3", - "work_id": "llm.1002", - "action": "link", - "object": "work_id", - "data": "kws.1000" + "request_id": "3", + "work_id": "llm.1002", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` @@ -68,53 +75,62 @@ Response json: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "3", - "work_id": "llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "llm.1002" } ``` + error::code of 0 indicates successful execution. -Link the llm and kws units so that when kws issues wake-up data, the llm unit stops the previous unfinished inference for repeated wake-up functionality. +Link the llm and kws units so that when kws issues wake-up data, the llm unit stops the previous unfinished inference +for repeated wake-up functionality. -> **When linking, ensure that kws has been configured and is in working status. Linking can also be done during the setup phase.** +> **When linking, ensure that kws has been configured and is in working status. Linking can also be done during the +setup phase.** Example: ```json { - "request_id": "2", - "work_id": "llm", - "action": "setup", - "object": "llm.setup", - "data": { - "model": "qwen2.5-0.5B-prefill-20e", - "response_format": "llm.utf-8.stream", - "input": ["llm.utf-8", "asr.1001", "kws.1000"], - "enoutput": true, - "max_token_len": 256, - "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." - } + "request_id": "2", + "work_id": "llm", + "action": "setup", + "object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": [ + "llm.utf-8", + "asr.1001", + "kws.1000" + ], + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } } ``` ## unlink + Unlink. Send json: + ```json { - "request_id": "4", - "work_id": "llm.1002", - "action": "unlink", - "object": "work_id", - "data": "kws.1000" + "request_id": "4", + "work_id": "llm.1002", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` @@ -122,28 +138,31 @@ Response json: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "4", - "work_id": "llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "llm.1002" } ``` + error::code of 0 indicates successful execution. ## pause + Pause the unit. Send json: + ```json { - "request_id": "5", - "work_id": "llm.1002", - "action": "pause" + "request_id": "5", + "work_id": "llm.1002", + "action": "pause" } ``` @@ -151,28 +170,31 @@ Response json: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "5", - "work_id": "llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "llm.1002" } ``` + error::code of 0 indicates successful execution. ## work + Resume the unit. Send json: + ```json { - "request_id": "6", - "work_id": "llm.1002", - "action": "work" + "request_id": "6", + "work_id": "llm.1002", + "action": "work" } ``` @@ -180,28 +202,31 @@ Response json: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "6", - "work_id": "llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "llm.1002" } ``` + error::code of 0 indicates successful execution. ## exit + Exit the unit. Send json: + ```json { - "request_id": "7", - "work_id": "llm.1002", - "action": "exit" + "request_id": "7", + "work_id": "llm.1002", + "action": "exit" } ``` @@ -209,17 +234,18 @@ Response json: ```json { - "created": 1731488402, - "data": "None", - "error": { - "code": 0, - "message": "" - }, - "object": "None", - "request_id": "7", - "work_id": "llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "llm.1002" } ``` + error::code of 0 indicates successful execution. ## Task Information @@ -227,58 +253,67 @@ error::code of 0 indicates successful execution. Get task list. Send JSON: + ```json { - "request_id": "2", - "work_id": "llm", - "action": "taskinfo" + "request_id": "2", + "work_id": "llm", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731652149, - "data":["llm.1002"], - "error":{ - "code":0, - "message":"" - }, - "object":"llm.tasklist", - "request_id":"2", - "work_id":"llm" + "created": 1731652149, + "data": [ + "llm.1002" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "llm.tasklist", + "request_id": "2", + "work_id": "llm" } ``` Get task runtime parameters. Send JSON: + ```json { - "request_id": "2", - "work_id": "llm.1002", - "action": "taskinfo" + "request_id": "2", + "work_id": "llm.1002", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731652187, - "data":{ - "enoutput":true, - "inputs_":["llm.utf-8"], - "model":"qwen2.5-0.5B-prefill-20e", - "response_format":"llm.utf-8.stream" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"llm.taskinfo", - "request_id":"2", - "work_id":"llm.1002" + "created": 1731652187, + "data": { + "enoutput": true, + "inputs_": [ + "llm.utf-8" + ], + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "llm.taskinfo", + "request_id": "2", + "work_id": "llm.1002" } ``` -> **Note: work_id increases according to the initialization registration order of the unit and is not a fixed index value.** \ No newline at end of file +> **Note: work_id increases according to the initialization registration order of the unit and is not a fixed index +value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_llm_zh.md b/doc/projects_llm_framework_doc/llm_llm_zh.md index da945d1..f018e20 100644 --- a/doc/projects_llm_framework_doc/llm_llm_zh.md +++ b/doc/projects_llm_framework_doc/llm_llm_zh.md @@ -1,25 +1,30 @@ # llm-llm -大模型单元，用于提供大模型推理服务。 + +大模型单元，用于提供大模型推理服务。 ## setup + 配置单元工作。 发送 json： + ```json { - "request_id": "2", - "work_id": "llm", - "action": "setup","object": "llm.setup", - "data": { - "model": "qwen2.5-0.5B-prefill-20e", - "response_format": "llm.utf-8.stream", - "input": "llm.utf-8", - "enoutput": true, - "max_token_len": 256, - "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." - } + "request_id": "2", + "work_id": "llm", + "action": "setup", + "object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": "llm.utf-8", + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } } ``` + - request_id：参考基本数据解释。 - work_id：配置单元时，为 `llm`。 - action：调用的方法为 `setup`。 @@ -29,21 +34,21 @@ - input：输入的为 `llm.utf-8`,代表的是从用户输入。 - enoutput：是否起用用户结果输出。 - max_token_len：最大输出 token,该值的最大值受到模型的最大限制。 -- prompt：模型的提示词。 +- prompt：模型的系统提示词。 响应 json： ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "llm.1002" } ``` @@ -51,16 +56,18 @@ - work_id：返回成功创建的 work_id 单元。 ## link + 链接上级单元的输出。 发送 json： + ```json { - "request_id": "3", - "work_id": "llm.1002", - "action": "link", - "object":"work_id", - "data":"kws.1000" + "request_id": "3", + "work_id": "llm.1002", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` @@ -68,17 +75,18 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "llm.1002" } ``` + error::code 为 0 表示执行成功。 将 llm 和 kws 单元链接起来，当 kws 发出唤醒数据时，llm 单元停止上次未完成的推理，用于重复唤醒功能。 @@ -89,31 +97,38 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "llm", - "action": "setup","object": "llm.setup", - "data": { - "model": "qwen2.5-0.5B-prefill-20e", - "response_format": "llm.utf-8.stream", - "input": ["llm.utf-8", "asr.1001", "kws.1000"], - "enoutput": true, - "max_token_len": 256, - "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." - } + "request_id": "2", + "work_id": "llm", + "action": "setup", + "object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": [ + "llm.utf-8", + "asr.1001", + "kws.1000" + ], + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } } ``` ## unlink + 取消链接。 发送 json： + ```json { - "request_id": "4", - "work_id": "llm.1002", - "action": "unlink", - "object":"work_id", - "data":"kws.1000" + "request_id": "4", + "work_id": "llm.1002", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` @@ -121,29 +136,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "llm.1002" } ``` error::code 为 0 表示执行成功。 ## pause + 暂停单元工作。 发送 json： + ```json { - "request_id": "5", - "work_id": "llm.1002", - "action": "pause", + "request_id": "5", + "work_id": "llm.1002", + "action": "pause" } ``` @@ -151,29 +168,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "llm.1002" } ``` error::code 为 0 表示执行成功。 ## work + 恢复单元工作。 发送 json： + ```json { - "request_id": "6", - "work_id": "llm.1002", - "action": "work", + "request_id": "6", + "work_id": "llm.1002", + "action": "work" } ``` @@ -181,29 +200,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"6", - "work_id":"llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "llm.1002" } ``` error::code 为 0 表示执行成功。 ## exit -恢复单元工作。 + +退出单元工作。 发送 json： + ```json { - "request_id": "7", - "work_id": "llm.1002", - "action": "exit", + "request_id": "7", + "work_id": "llm.1002", + "action": "exit" } ``` @@ -211,15 +232,15 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"llm.1002" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "llm.1002" } ``` @@ -230,11 +251,12 @@ error::code 为 0 表示执行成功。 获取任务列表。 发送 json： + ```json { - "request_id": "2", - "work_id": "llm", - "action": "taskinfo" + "request_id": "2", + "work_id": "llm", + "action": "taskinfo" } ``` @@ -242,15 +264,17 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652149, - "data":["llm.1002"], - "error":{ - "code":0, - "message":"" - }, - "object":"llm.tasklist", - "request_id":"2", - "work_id":"llm" + "created": 1731652149, + "data": [ + "llm.1002" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "llm.tasklist", + "request_id": "2", + "work_id": "llm" } ``` @@ -258,9 +282,9 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "llm.1002", - "action": "taskinfo" + "request_id": "2", + "work_id": "llm.1002", + "action": "taskinfo" } ``` @@ -268,20 +292,22 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652187, - "data":{ - "enoutput":true, - "inputs_":["llm.utf-8"], - "model":"qwen2.5-0.5B-prefill-20e", - "response_format":"llm.utf-8.stream" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"llm.taskinfo", - "request_id":"2", - "work_id":"llm.1002" + "created": 1731652187, + "data": { + "enoutput": true, + "inputs_": [ + "llm.utf-8" + ], + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "llm.taskinfo", + "request_id": "2", + "work_id": "llm.1002" } ``` diff --git a/doc/projects_llm_framework_doc/llm_melotts_en.md b/doc/projects_llm_framework_doc/llm_melotts_en.md index f5dd34d..66a705f 100644 --- a/doc/projects_llm_framework_doc/llm_melotts_en.md +++ b/doc/projects_llm_framework_doc/llm_melotts_en.md @@ -1,212 +1,247 @@ # llm-melotts -Text-to-speech unit accelerated by NPU, used to provide text-to-speech services. It supports both Chinese and English models for text-to-speech conversion. + +Text-to-speech unit accelerated by NPU, used to provide text-to-speech services. It supports both Chinese and English +models for text-to-speech conversion. ## setup + Configure the unit. Send JSON: + ```json { - "request_id": "2", - "work_id": "melotts", - "action": "setup", - "object": "melotts.setup", - "data": { - "model": "melotts_zh-cn", - "response_format": "sys.pcm", - "input": "tts.utf-8", - "enoutput": false - } + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } } ``` + - request_id: Refer to the basic data explanation. - work_id: For configuration, it is `melotts`. - action: The method to be called is `setup`. - object: The data type being transmitted is `melotts.setup`. - model: The model being used is the Chinese model `melotts_zh-cn`. -- response_format: The result is returned as `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. +- response_format: The result is returned as `sys.pcm`, system audio data, which is directly sent to the llm-audio + module for playback. - input: The input is `tts.utf-8`, representing user input. - enoutput: Whether to enable user result output. Response JSON: + ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "melotts.1003" } ``` + - created: Message creation time, UNIX time. - work_id: Successfully created work_id unit. ## link + Link the output of the upper-level unit. Send JSON: + ```json { - "request_id": "3", - "work_id": "melotts.1003", - "action": "link", - "object":"work_id", - "data":"kws.1000" + "request_id": "3", + "work_id": "melotts.1003", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` Response JSON: + ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "melotts.1003" } ``` + error::code being 0 indicates success. -Link the llm and melotts units. When the kws melotts unit stops the unfinished inference from the last time, it is used for repeated wake-up functionality. +Link the llm and melotts units. When the kws melotts unit stops the unfinished inference from the last time, it is used +for repeated wake-up functionality. -> **Ensure that kws is already configured and in working status during link. Link can also be performed during the setup stage.** +> **Ensure that kws is already configured and in working status during link. Link can also be performed during the setup +stage.** Example: + ```json { - "request_id": "2", - "work_id": "melotts", - "action": "setup", - "object": "melotts.setup", - "data": { - "model": "melotts_zh-cn", - "response_format": "sys.pcm", - "input": ["tts.utf-8", "llm.1002", "kws.1000"], - "enoutput": false - } + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": [ + "tts.utf-8", + "llm.1002", + "kws.1000" + ], + "enoutput": false + } } ``` ## unlink + Unlink. Send JSON: + ```json { - "request_id": "4", - "work_id": "melotts.1003", - "action": "unlink", - "object":"work_id", - "data":"kws.1000" + "request_id": "4", + "work_id": "melotts.1003", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` Response JSON: + ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "melotts.1003" } ``` + error::code being 0 indicates success. ## pause + Pause the unit. Send JSON: + ```json { - "request_id": "5", - "work_id": "llm.1003", - "action": "pause", + "request_id": "5", + "work_id": "llm.1003", + "action": "pause" } ``` Response JSON: + ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "llm.1003" } ``` + error::code being 0 indicates success. ## work + Resume the unit. Send JSON: + ```json { - "request_id": "6", - "work_id": "llm.1003", - "action": "work", + "request_id": "6", + "work_id": "llm.1003", + "action": "work" } ``` Response JSON: + ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"6", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "llm.1003" } ``` + error::code being 0 indicates success. ## exit + Exit the unit. Send JSON: + ```json { - "request_id": "7", - "work_id": "llm.1003", - "action": "exit", + "request_id": "7", + "work_id": "llm.1003", + "action": "exit" } ``` Response JSON: + ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "llm.1003" } ``` + error::code being 0 indicates success. ## Task Information @@ -214,59 +249,68 @@ error::code being 0 indicates success. Get task list. Send JSON: + ```json { - "request_id": "2", - "work_id": "melotts", - "action": "taskinfo" + "request_id": "2", + "work_id": "melotts", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731652311, - "data":["melotts.1003"], - "error":{ - "code":0, - "message":"" - }, - "object":"melotts.tasklist", - "request_id":"2", - "work_id":"melotts" + "created": 1731652311, + "data": [ + "melotts.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "melotts.tasklist", + "request_id": "2", + "work_id": "melotts" } ``` Get task runtime parameters. Send JSON: + ```json { - "request_id": "2", - "work_id": "melotts.1003", - "action": "taskinfo" + "request_id": "2", + "work_id": "melotts.1003", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731652344, - "data":{ - "enoutput":false, - "inputs_":["tts.utf-8"], - "model":"melotts_zh-cn", - "response_format":"sys.pcm" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"melotts.taskinfo", - "request_id":"2", - "work_id":"melotts.1003" + "created": 1731652344, + "data": { + "enoutput": false, + "inputs_": [ + "tts.utf-8" + ], + "model": "melotts_zh-cn", + "response_format": "sys.pcm" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "melotts.taskinfo", + "request_id": "2", + "work_id": "melotts.1003" } ``` > **Note: work_id increases in the order of the unit's initialization registration and is not a fixed index value.** -> **The same type of unit cannot configure multiple units to work simultaneously, or unknown errors may occur. For example, tts and melo tts cannot be activated to work at the same time.** \ No newline at end of file +> **The same type of unit cannot configure multiple units to work simultaneously, or unknown errors may occur. For +example, tts and melo tts cannot be activated to work at the same time.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_melotts_zh.md b/doc/projects_llm_framework_doc/llm_melotts_zh.md index e276c61..40d40c8 100644 --- a/doc/projects_llm_framework_doc/llm_melotts_zh.md +++ b/doc/projects_llm_framework_doc/llm_melotts_zh.md @@ -1,24 +1,28 @@ # llm-melotts -使用 npu 加速的文字转语音单元，用于提供文字转语音服务，可选择中英文模型，用于提供中英文文字转语音服务。 + +使用 npu 加速的文字转语音单元，用于提供文字转语音服务，可选择中英文模型，用于提供中英文文字转语音服务。 ## setup + 配置单元工作。 发送 json： + ```json { - "request_id": "2", - "work_id": "melotts", - "action": "setup", - "object": "melotts.setup", - "data": { - "model": "melotts_zh-cn", - "response_format": "sys.pcm", - "input": "tts.utf-8", - "enoutput": false - } + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } } ``` + - request_id：参考基本数据解释。 - work_id：配置单元时，为 `melotts`。 - action：调用的方法为 `setup`。 @@ -32,31 +36,34 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "melotts.1003" } ``` + - created：消息创建时间，unix 时间。 - work_id：返回成功创建的 work_id 单元。 ## link + 链接上级单元的输出。 发送 json： + ```json { - "request_id": "3", - "work_id": "melotts.1003", - "action": "link", - "object":"work_id", - "data":"kws.1000" + "request_id": "3", + "work_id": "melotts.1003", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` @@ -64,17 +71,18 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "melotts.1003" } ``` + error::code 为 0 表示执行成功。 将 llm 和 melotts 单元链接起来，当 kws melotts 单元停止上次未完成的推理，用于重复唤醒功能。 @@ -85,30 +93,36 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "melotts", - "action": "setup", - "object": "melotts.setup", - "data": { - "model": "melotts_zh-cn", - "response_format": "sys.pcm", - "input": ["tts.utf-8", "llm.1002", "kws.1000"], - "enoutput": false - } + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": [ + "tts.utf-8", + "llm.1002", + "kws.1000" + ], + "enoutput": false + } } ``` ## unlink + 取消链接。 发送 json： + ```json { - "request_id": "4", - "work_id": "melotts.1003", - "action": "unlink", - "object":"work_id", - "data":"kws.1000" + "request_id": "4", + "work_id": "melotts.1003", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` @@ -116,28 +130,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "melotts.1003" } ``` + error::code 为 0 表示执行成功。 ## pause + 暂停单元工作。 发送 json： + ```json { - "request_id": "5", - "work_id": "melotts.1003", - "action": "pause", + "request_id": "5", + "work_id": "melotts.1003", + "action": "pause" } ``` @@ -145,29 +162,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "melotts.1003" } ``` error::code 为 0 表示执行成功。 ## work + 恢复单元工作。 发送 json： + ```json { - "request_id": "6", - "work_id": "melotts.1003", - "action": "work", + "request_id": "6", + "work_id": "melotts.1003", + "action": "work" } ``` @@ -175,29 +194,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"6", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "melotts.1003" } ``` error::code 为 0 表示执行成功。 ## exit + 单元退出。 发送 json： + ```json { - "request_id": "7", - "work_id": "melotts.1003", - "action": "exit", + "request_id": "7", + "work_id": "melotts.1003", + "action": "exit" } ``` @@ -205,15 +226,15 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"melotts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "melotts.1003" } ``` @@ -224,11 +245,12 @@ error::code 为 0 表示执行成功。 获取任务列表。 发送 json： + ```json { - "request_id": "2", - "work_id": "melotts", - "action": "taskinfo" + "request_id": "2", + "work_id": "melotts", + "action": "taskinfo" } ``` @@ -236,15 +258,17 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652311, - "data":["melotts.1003"], - "error":{ - "code":0, - "message":"" - }, - "object":"melotts.tasklist", - "request_id":"2", - "work_id":"melotts" + "created": 1731652311, + "data": [ + "melotts.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "melotts.tasklist", + "request_id": "2", + "work_id": "melotts" } ``` @@ -252,9 +276,9 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "melotts.1003", - "action": "taskinfo" + "request_id": "2", + "work_id": "melotts.1003", + "action": "taskinfo" } ``` @@ -262,20 +286,22 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652344, - "data":{ - "enoutput":false, - "inputs_":["tts.utf-8"], - "model":"melotts_zh-cn", - "response_format":"sys.pcm" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"melotts.taskinfo", - "request_id":"2", - "work_id":"melotts.1003" + "created": 1731652344, + "data": { + "enoutput": false, + "inputs_": [ + "tts.utf-8" + ], + "model": "melotts_zh-cn", + "response_format": "sys.pcm" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "melotts.taskinfo", + "request_id": "2", + "work_id": "melotts.1003" } ``` diff --git a/doc/projects_llm_framework_doc/llm_sys_en.md b/doc/projects_llm_framework_doc/llm_sys_en.md index 40e6ed9..56232d4 100644 --- a/doc/projects_llm_framework_doc/llm_sys_en.md +++ b/doc/projects_llm_framework_doc/llm_sys_en.md @@ -1,8 +1,10 @@ # llm-sys -The basic service unit of StackFlow, providing external serial and TCP channels and some system function services, while internally handling port resource allocation and a simple in-memory database. +The basic service unit of StackFlow, providing external serial and TCP channels and some system function services, while +internally handling port resource allocation and a simple in-memory database. ## External API + - sys.ping: Test if communication with the LLM is possible. - sys.lsmode: Models that have existed in the system in the past. - sys.bashexec: Execute bash commands. @@ -13,8 +15,9 @@ The basic service unit of StackFlow, providing external serial and TCP channels - sys.version: Get the version of the LLM framework program. ## Internal API + - sql_select: Query key-value pairs in the small in-memory KV database. - register_unit: Register a unit. - release_unit: Release a unit. - sql_set: Set key-value pairs in the small in-memory KV database. -- sql_unset: Delete key-value pairs in the small in-memory KV database. \ No newline at end of file +- sql_unset: Delete key-value pairs in the small in-memory KV database. diff --git a/doc/projects_llm_framework_doc/llm_sys_zh.md b/doc/projects_llm_framework_doc/llm_sys_zh.md index 1869cb2..e51de20 100644 --- a/doc/projects_llm_framework_doc/llm_sys_zh.md +++ b/doc/projects_llm_framework_doc/llm_sys_zh.md @@ -1,7 +1,9 @@ # llm-sys + StackFlow 的基本服务单元，对外提供串口和 TCP 外部信道和一部分系统功能服务，对内进行端口资源分配，和一个简单的内存数据库。 ## 外部 API + - sys.ping：测试是否能够和 LLM 进行通信。 - sys.lsmode：过去系统中存在的模型。 - sys.bashexec：执行 bash 命令。 @@ -12,9 +14,9 @@ StackFlow 的基本服务单元，对外提供串口和 TCP 外部信道和一 - sys.version：获取 LLM 框架程序版本。 ## 内部 API： + - sql_select：查寻小型内存 KV 数据库键值。 - register_unit：注册单元。 - release_unit：释放单元。 - sql_set：设定小型内存 KV 数据库键值。 - sql_unset：删除小型内存 KV 数据库键值。 - diff --git a/doc/projects_llm_framework_doc/llm_tts_en.md b/doc/projects_llm_framework_doc/llm_tts_en.md index ce61e00..7833828 100644 --- a/doc/projects_llm_framework_doc/llm_tts_en.md +++ b/doc/projects_llm_framework_doc/llm_tts_en.md @@ -1,30 +1,36 @@ # llm-tts -Text-to-Speech unit, used to provide text-to-speech services, with options for Chinese and English models to provide text-to-speech services in both languages. + +Text-to-Speech unit, used to provide text-to-speech services, with options for Chinese and English models to provide +text-to-speech services in both languages. ## setup + Configure the unit. Send JSON: + ```json { - "request_id": "2", - "work_id": "tts", - "action": "setup", - "object": "tts.setup", - "data": { - "model": "single_speaker_fast", - "response_format": "sys.pcm", - "input": "tts.utf-8", - "enoutput": false - } + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } } ``` + - request_id: Refer to the basic data explanation. - work_id: For configuring the unit, it is `tts`. - action: The method to call is `setup`. - object: The type of data being transmitted is `tts.setup`. - model: The model used is the `single_speaker_fast` Chinese model. -- response_format: The returned result is `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. +- response_format: The returned result is `sys.pcm`, system audio data, which is directly sent to the llm-audio module + for playback. - input: Input is `tts.utf-8`, representing user input. - enoutput: Whether to enable user result output. @@ -32,15 +38,15 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "llm.1003" } ``` @@ -48,16 +54,18 @@ Response JSON: - work_id: The successfully created work_id unit. ## link + Link the output of the upper unit. Send JSON: + ```json { - "request_id": "3", - "work_id": "tts.1003", - "action": "link", - "object":"work_id", - "data":"kws.1000" + "request_id": "3", + "work_id": "tts.1003", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` @@ -65,51 +73,60 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"tts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "tts.1003" } ``` + error::code 0 indicates successful execution. -Link the llm and tts units, so that when kws wakes up, the tts unit stops the previous unfinished inference, used for repeat wake-up functionality. +Link the llm and tts units, so that when kws wakes up, the tts unit stops the previous unfinished inference, used for +repeat wake-up functionality. -> **When linking, ensure that kws is already configured and in working state. Linking can also be done during the setup phase.** +> **When linking, ensure that kws is already configured and in working state. Linking can also be done during the setup +phase.** Example: ```json { - "request_id": "2", - "work_id": "tts", - "action": "setup", - "object": "tts.setup", - "data": { - "model": "single_speaker_fast", - "response_format": "sys.pcm", - "input": ["tts.utf-8", "llm.1002", "kws.1000"], - "enoutput": false - } + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": [ + "tts.utf-8", + "llm.1002", + "kws.1000" + ], + "enoutput": false + } } ``` ## unlink + Unlink. Send JSON: + ```json { - "request_id": "4", - "work_id": "tts.1003", - "action": "unlink", - "object":"work_id", - "data":"kws.1000" + "request_id": "4", + "work_id": "tts.1003", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` @@ -117,28 +134,31 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"tts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "tts.1003" } ``` + error::code 0 indicates successful execution. ## pause + Pause unit work. Send JSON: + ```json { - "request_id": "5", - "work_id": "llm.1003", - "action": "pause", + "request_id": "5", + "work_id": "llm.1003", + "action": "pause" } ``` @@ -146,28 +166,31 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "llm.1003" } ``` + error::code 0 indicates successful execution. ## work + Resume unit work. Send JSON: + ```json { - "request_id": "6", - "work_id": "llm.1003", - "action": "work", + "request_id": "6", + "work_id": "llm.1003", + "action": "work" } ``` @@ -175,28 +198,31 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"6", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "llm.1003" } ``` + error::code 0 indicates successful execution. ## exit + Exit the unit. Send JSON: + ```json { - "request_id": "7", - "work_id": "llm.1003", - "action": "exit", + "request_id": "7", + "work_id": "llm.1003", + "action": "exit" } ``` @@ -204,17 +230,18 @@ Response JSON: ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "llm.1003" } ``` + error::code 0 indicates successful execution. ## Task Information @@ -222,59 +249,69 @@ error::code 0 indicates successful execution. Get task list. Send JSON: + ```json { - "request_id": "2", - "work_id": "tts", - "action": "taskinfo" + "request_id": "2", + "work_id": "tts", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731652311, - "data":["tts.1003"], - "error":{ - "code":0, - "message":"" - }, - "object":"tts.tasklist", - "request_id":"2", - "work_id":"tts" + "created": 1731652311, + "data": [ + "tts.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "tts.tasklist", + "request_id": "2", + "work_id": "tts" } ``` Get task runtime parameters. Send JSON: + ```json { - "request_id": "2", - "work_id": "tts.1003", - "action": "taskinfo" + "request_id": "2", + "work_id": "tts.1003", + "action": "taskinfo" } ``` Response JSON: + ```json { - "created":1731652344, - "data":{ - "enoutput":false, - "inputs_":["tts.utf-8"], - "model":"single_speaker_fast", - "response_format":"sys.pcm" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"tts.taskinfo", - "request_id":"2", - "work_id":"tts.1003" + "created": 1731652344, + "data": { + "enoutput": false, + "inputs_": [ + "tts.utf-8" + ], + "model": "single_speaker_fast", + "response_format": "sys.pcm" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "tts.taskinfo", + "request_id": "2", + "work_id": "tts.1003" } ``` -> **Note: work_id increases according to the order of unit initialization registration and is not a fixed index value.** -> **The same type of unit cannot have multiple units working simultaneously, as it may cause unknown errors. For example, tts and melo tts cannot be enabled to work at the same time.** \ No newline at end of file +> **Note: work_id increases according to the order of unit initialization registration and is not a fixed index value. +** +> **The same type of unit cannot have multiple units working simultaneously, as it may cause unknown errors. For +example, tts and melo tts cannot be enabled to work at the same time.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_tts_zh.md b/doc/projects_llm_framework_doc/llm_tts_zh.md index 04227ec..719d4fb 100644 --- a/doc/projects_llm_framework_doc/llm_tts_zh.md +++ b/doc/projects_llm_framework_doc/llm_tts_zh.md @@ -1,24 +1,28 @@ # llm-tts -文字转语音单元，用于提供文字转语音服务，可选择中英文模型，用于提供中英文文字转语音服务。 + +文字转语音单元，用于提供文字转语音服务，可选择中英文模型，用于提供中英文文字转语音服务。 ## setup + 配置单元工作。 发送 json： + ```json { - "request_id": "2", - "work_id": "tts", - "action": "setup", - "object": "tts.setup", - "data": { - "model": "single_speaker_fast", - "response_format": "sys.pcm", - "input": "tts.utf-8", - "enoutput": false - } + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } } ``` + - request_id：参考基本数据解释。 - work_id：配置单元时，为 `tts`。 - action：调用的方法为 `setup`。 @@ -32,31 +36,34 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"llm.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "llm.1003" } ``` + - created：消息创建时间，unix 时间。 - work_id：返回成功创建的 work_id 单元。 ## link + 链接上级单元的输出。 发送 json： + ```json { - "request_id": "3", - "work_id": "tts.1003", - "action": "link", - "object":"work_id", - "data":"kws.1000" + "request_id": "3", + "work_id": "tts.1003", + "action": "link", + "object": "work_id", + "data": "kws.1000" } ``` @@ -64,17 +71,18 @@ ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"3", - "work_id":"tts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "tts.1003" } ``` + error::code 为 0 表示执行成功。 将 llm 和 tts 单元链接起来，当 kws 唤醒时 tts 单元停止上次未完成的推理，用于重复唤醒功能。 @@ -85,31 +93,36 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "2", - "work_id": "tts", - "action": "setup", - "object": "tts.setup", - "data": { - "model": "single_speaker_fast", - "response_format": "sys.pcm", - "input": ["tts.utf-8", "llm.1002", "kws.1000"], - "enoutput": false - } + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": [ + "tts.utf-8", + "llm.1002", + "kws.1000" + ], + "enoutput": false + } } ``` ## unlink + 取消链接。 发送 json： ```json { - "request_id": "4", - "work_id": "tts.1003", - "action": "unlink", - "object":"work_id", - "data":"kws.1000" + "request_id": "4", + "work_id": "tts.1003", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" } ``` @@ -117,30 +130,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"4", - "work_id":"tts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "tts.1003" } ``` error::code 为 0 表示执行成功。 ## pause + 暂停单元工作。 发送 json： ```json { - "request_id": "5", - "work_id": "tts.1003", - "action": "pause", + "request_id": "5", + "work_id": "tts.1003", + "action": "pause" } ``` @@ -148,29 +162,31 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"5", - "work_id":"tts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "tts.1003" } ``` + error::code 为 0 表示执行成功。 ## work + 恢复单元工作。 发送 json： ```json { - "request_id": "6", - "work_id": "tts.1003", - "action": "work", + "request_id": "6", + "work_id": "tts.1003", + "action": "work" } ``` @@ -178,17 +194,18 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"6", - "work_id":"tts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "tts.1003" } ``` + error::code 为 0 表示执行成功。 ## exit @@ -199,9 +216,9 @@ error::code 为 0 表示执行成功。 ```json { - "request_id": "7", - "work_id": "tts.1003", - "action": "exit", + "request_id": "7", + "work_id": "tts.1003", + "action": "exit" } ``` @@ -209,15 +226,15 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"tts.1003" + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "tts.1003" } ``` @@ -228,11 +245,12 @@ error::code 为 0 表示执行成功。 获取任务列表。 发送 json： + ```json { - "request_id": "2", - "work_id": "tts", - "action": "taskinfo" + "request_id": "2", + "work_id": "tts", + "action": "taskinfo" } ``` @@ -240,26 +258,29 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652311, - "data":["tts.1003"], - "error":{ - "code":0, - "message":"" - }, - "object":"tts.tasklist", - "request_id":"2", - "work_id":"tts" + "created": 1731652311, + "data": [ + "tts.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "tts.tasklist", + "request_id": "2", + "work_id": "tts" } ``` 获取任务运行参数。 发送 json： + ```json { - "request_id": "2", - "work_id": "tts.1003", - "action": "taskinfo" + "request_id": "2", + "work_id": "tts.1003", + "action": "taskinfo" } ``` @@ -267,23 +288,24 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652344, - "data":{ - "enoutput":false, - "inputs_":["tts.utf-8"], - "model":"single_speaker_fast", - "response_format":"sys.pcm" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"tts.taskinfo", - "request_id":"2", - "work_id":"tts.1003" + "created": 1731652344, + "data": { + "enoutput": false, + "inputs_": [ + "tts.utf-8" + ], + "model": "single_speaker_fast", + "response_format": "sys.pcm" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "tts.taskinfo", + "request_id": "2", + "work_id": "tts.1003" } ``` - > **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** > **同类型单元不能配置多个单元同时工作，否则会产生未知错误。例如 tts 和 melo tts 不能同时拍起用工作。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_vad_en.md b/doc/projects_llm_framework_doc/llm_vad_en.md new file mode 100644 index 0000000..fbdfa1d --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_vad_en.md @@ -0,0 +1,215 @@ +# llm-vad + +The voice activity detection unit is used to provide the service of identifying whether there is speech content in an audio signal. + +## setup + +Configure the unit for operation. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "vad", + "action": "setup", + "object": "vad.setup", + "data": { + "model": "silero-vad", + "response_format": "vad.bool", + "input": "sys.pcm", + "enoutput": true + } +} +``` + +- request_id: Refers to the basic data explanation. +- work_id: For configuring the unit, use `vad`. +- action: The method called is `setup`. +- object: The data type being transferred is `vad.setup`. +- model: The model used is the Chinese version of `silero-vad`. +- response_format: The returned result is of type `vad.bool`. +- input: The input is `sys.pcm`, which represents system audio. +- enoutput: Whether to enable the output for user results. + +Response JSON: + +```json +{ + "created": 1737595236, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "vad.1000" +} +``` + +- created: Message creation time, in Unix time. +- work_id: The work_id of the successfully created unit. + +## pause + +Pause the unit's operation. + +Send JSON: + +```json +{ + "request_id": "3", + "work_id": "vad.1000", + "action": "pause" +} +``` + +Response JSON: + +```json +{ + "created": 1737595314, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "vad.1000" +} +``` + +`error::code` of 0 indicates successful execution. + +## work + +Resume the unit's operation. + +Send JSON: + +```json +{ + "request_id": "4", + "work_id": "vad.1000", + "action": "work" +} +``` + +Response JSON: + +```json +{ + "created": 1737595419, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "vad.1000" +} +``` + +`error::code` of 0 indicates successful execution. + +## exit + +Exit the unit. + +Send JSON: + +```json +{ + "request_id": "5", + "work_id": "vad.1000", + "action": "exit" +} +``` + +Response JSON: + +```json +{ + "created": 1737595478, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "vad.1000" +} +``` + +`error::code` of 0 indicates successful execution. + +## taskinfo + +Get the task list. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "vad", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737595606, + "data": [ + "vad.1000" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "vad.tasklist", + "request_id": "2", + "work_id": "vad" +} +``` + +Get the task's running parameters. + +```json +{ + "request_id": "2", + "work_id": "vad.1000", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737596100, + "data": { + "enoutput": true, + "inputs": [ + "sys.pcm" + ], + "model": "silero-vad", + "response_format": "vad.bool" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vad.taskinfo", + "request_id": "2", + "work_id": "vad.1000" +} +``` + +> **Note: The work_id increases according to the order in which the units are initialized and is not a fixed index value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_vad_zh.md b/doc/projects_llm_framework_doc/llm_vad_zh.md new file mode 100644 index 0000000..63d3ae9 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_vad_zh.md @@ -0,0 +1,215 @@ +# llm-vad + +语音活动检测单元，用于提供识别音频信号中是否存在语音成分服务。 + +## setup + +配置单元工作。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "vad", + "action": "setup", + "object": "vad.setup", + "data": { + "model": "silero-vad", + "response_format": "vad.bool", + "input": "sys.pcm", + "enoutput": true + } +} +``` + +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `vad`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `vad.setup`。 +- model：使用的模型为 `silero-vad` 中文模型。 +- response_format：返回结果为 `vad.bool` 型。 +- input：输入的为 `sys.pcm`,代表的是系统音频。 +- enoutput：是否起用用户结果输出。 + +响应 json： + +```json +{ + "created": 1737595236, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "vad.1000" +} +``` + +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## pause + +暂停单元工作。 + +发送 json： + +```json +{ + "request_id": "3", + "work_id": "vad.1000", + "action": "pause" +} +``` + +响应 json： + +```json +{ + "created": 1737595314, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "vad.1000" +} +``` + +error::code 为 0 表示执行成功。 + +## work + +恢复单元工作。 + +发送 json： + +```json +{ + "request_id": "4", + "work_id": "vad.1000", + "action": "work" +} +``` + +响应 json： + +```json +{ + "created": 1737595419, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "vad.1000" +} +``` + +error::code 为 0 表示执行成功。 + +## exit + +单元退出。 + +发送 json： + +```json +{ + "request_id": "5", + "work_id": "vad.1000", + "action": "exit" +} +``` + +响应 json： + +```json +{ + "created": 1737595478, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "vad.1000" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "vad", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1737595606, + "data": [ + "vad.1000" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "vad.tasklist", + "request_id": "2", + "work_id": "vad" +} +``` + +获取任务运行参数。 + +```json +{ + "request_id": "2", + "work_id": "vad.1000", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1737596100, + "data": { + "enoutput": true, + "inputs": [ + "sys.pcm" + ], + "model": "silero-vad", + "response_format": "vad.bool" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vad.taskinfo", + "request_id": "2", + "work_id": "vad.1000" +} +``` + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** diff --git a/doc/projects_llm_framework_doc/llm_vlm_en.md b/doc/projects_llm_framework_doc/llm_vlm_en.md new file mode 100644 index 0000000..7619669 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_vlm_en.md @@ -0,0 +1,463 @@ +# llm-vlm + +A large model unit designed to provide multimodal inference services. + +## setup + +Configures the unit's operation. + +Send the following JSON: + +```json +{ + "request_id": "2", + "work_id": "vlm", + "action": "setup", + "object": "vlm.setup", + "data": { + "model": "internvl2.5-1B-ax630c", + "response_format": "vlm.utf-8.stream", + "input": "vlm.utf-8", + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` + +- request_id: Reference for basic data explanation. +- work_id: Set to `vlm` when configuring the unit. +- action: The method being called is `setup`. +- object: Data type being transferred is `vlm.setup`. +- model: The model used is `internvl2.5-1B-ax630c`, a multimodal model. +- response_format: The output is in `vlm.utf-8.stream`, a UTF-8 stream format. +- input: The input is `vlm.utf-8`, representing user input. +- enoutput: Specifies whether to enable user output. +- max_token_len: Maximum output tokens, subject to the model's maximum limit. +- prompt: System prompt for the model. + +Response JSON: + +```json +{ + "created": 1737599810, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "vlm.1003" +} +``` + +- created: Message creation time in Unix timestamp. +- work_id: Returned work_id after successful creation. + +## inference + +Infer user data. + +Send the following JSON: + +```json +{ + "request_id": "4", + "work_id": "vlm.1003", + "action": "inference", + "object": "vlm.utf-8.stream", + "data": { + "delta": "May i know your name?", + "index": 0, + "finish": true + } +} +``` + +Non-streaming response JSON: + +```json +{ + "created": 1737600915, + "data": "I am an AI assistant whose name is LittleAI. How can I help you today?", + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +Streaming response: + +```json +{ + "created": 1737600539, + "data": { + "delta": "I am an", + "finish": false, + "index": 0 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600539, + "data": { + "delta": " AI assistant whose", + "finish": false, + "index": 1 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600540, + "data": { + "delta": " name is Intern", + "finish": false, + "index": 2 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600540, + "data": { + "delta": "A.", + "finish": false, + "index": 3 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600540, + "data": { + "delta": "", + "finish": true, + "index": 4 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +Send the following JSON: + +```json +{ + "request_id": "4", + "work_id": "vlm.1003", + "action": "inference", + "object": "vlm.jpeg.stream.base64", + "data": { + "delta": "U29tZSByYW5kb20gYmFzZTY0IGRhdGEgZm9yIHlvdSB0byBjb25zdWx0IGFwcGxpY2F0aW9ucy4=", + "index": 0, + "finish": true + } +} +``` + +> **Due to large image data, the above JSON only shows a segment. When used in practice, you must send the complete +image data.** + +## link + +Link to the upper unit's output. + +Send the following JSON: + +```json +{ + "request_id": "3", + "work_id": "vlm.1003", + "action": "link", + "object": "work_id", + "data": "kws.1000" +} +``` + +Response JSON: + +```json +{ + "created": 1737599866, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "vlm.1003" +} +``` + +Error code `0` indicates success. + +This links the `vlm` and `kws` units. When `kws` sends wake-up data, `vlm` stops the previous unfinished inference to +repeat the wake-up function. + +> **Linking must ensure that `kws` is configured and ready to work. Linking can also be done during the setup phase.** + +Example: + +```json +{ + "request_id": "2", + "work_id": "vlm", + "action": "setup", + "object": "vlm.setup", + "data": { + "model": "internvl2.5-1B-ax630c", + "response_format": "vlm.utf-8.stream", + "input": [ + "vlm.utf-8", + "kws.1000", + "whisper.1001" + ], + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` + +## unlink + +Unlink units. + +Send the following JSON: + +```json +{ + "request_id": "4", + "work_id": "vlm.1003", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" +} +``` + +Response JSON: + +```json +{ + "created": 1737600178, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +Error code `0` indicates success. + +## pause + +Pause unit work. + +Send the following JSON: + +```json +{ + "request_id": "5", + "work_id": "vlm.1003", + "action": "pause" +} +``` + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "vlm.1003" +} +``` + +Error code `0` indicates success. + +## work + +Resume unit work. + +Send the following JSON: + +```json +{ + "request_id": "6", + "work_id": "vlm.1003", + "action": "work" +} +``` + +Response JSON: + +```json +{ + "created": 1737600236, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "vlm.1003" +} +``` + +Error code `0` indicates success. + +## exit + +Exit unit work. + +Send the following JSON: + +```json +{ + "request_id": "7", + "work_id": "vlm.1003", + "action": "exit" +} +``` + +Response JSON: + +```json +{ + "created": 1737600704, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "vlm.1003" +} +``` + +Error code `0` indicates success. + +## taskinfo + +Get the task list. + +Send the following JSON: + +```json +{ + "request_id": "2", + "work_id": "vlm", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737600076, + "data": [ + "vlm.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.tasklist", + "request_id": "2", + "work_id": "vlm" +} +``` + +Get task running parameters. + +```json +{ + "request_id": "2", + "work_id": "vlm.1003", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737600098, + "data": { + "enoutput": true, + "inputs": [ + "vlm.utf-8", + "kws.1000" + ], + "model": "internvl2.5-1B-ax630c", + "response_format": "vlm.utf-8.stream" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.taskinfo", + "request_id": "2", + "work_id": "vlm.1003" +} +``` + +> **Note: `work_id` increases sequentially as units are initialized, it is not a fixed index value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_vlm_zh.md b/doc/projects_llm_framework_doc/llm_vlm_zh.md new file mode 100644 index 0000000..161f48c --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_vlm_zh.md @@ -0,0 +1,461 @@ +# llm-vlm + +大模型单元，用于提供多模态大模型推理服务。 + +## setup + +配置单元工作。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "vlm", + "action": "setup", + "object": "vlm.setup", + "data": { + "model": "internvl2.5-1B-ax630c", + "response_format": "vlm.utf-8.stream", + "input": "vlm.utf-8", + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` + +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `vlm`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `vlm.setup`。 +- model：使用的模型为 `internvl2.5-1B-ax630c` 多模态模型。 +- response_format：返回结果为 `vlm.utf-8.stream`, utf-8 的流式输出。 +- input：输入的为 `vlm.utf-8`,代表的是从用户输入。 +- enoutput：是否起用用户结果输出。 +- max_token_len：最大输出 token,该值的最大值受到模型的最大限制。 +- prompt：模型的系统提示词。 + +响应 json： + +```json +{ + "created": 1737599810, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "vlm.1003" +} +``` + +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## inference + +推理用户数据 + +发送 json： + +```json +{ + "request_id": "4", + "work_id": "vlm.1003", + "action": "inference", + "object": "vlm.utf-8.stream", + "data": { + "delta": "May i know your name?", + "index": 0, + "finish": true + } +} +``` + +非流式响应 json： + +```json +{ + "created": 1737600915, + "data": "I am an AI assistant whose name is LittleAI. How can I help you today?", + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +流式响应 + +```json +{ + "created": 1737600539, + "data": { + "delta": "I am an", + "finish": false, + "index": 0 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600539, + "data": { + "delta": " AI assistant whose", + "finish": false, + "index": 1 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600540, + "data": { + "delta": " name is Intern", + "finish": false, + "index": 2 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600540, + "data": { + "delta": "A.", + "finish": false, + "index": 3 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +```json +{ + "created": 1737600540, + "data": { + "delta": "", + "finish": true, + "index": 4 + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.utf-8.stream", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +发送 json： + +```json +{ + "request_id": "4", + "work_id": "vlm.1003", + "action": "inference", + "object": "vlm.jpeg.stream.base64", + "data": { + "delta": "U29tZSByYW5kb20gYmFzZTY0IGRhdGEgZm9yIHlvdSB0byBjb25zdWx0IGFwcGxpY2F0aW9ucy4=", + "index": 0, + "finish": true + } +} +``` + +> **由于图片数据较多，因此上述 json 仅展示片段，实际使用时，需要发送完整的图片数据** + +## link + +链接上级单元的输出。 + +发送 json： + +```json +{ + "request_id": "3", + "work_id": "vlm.1003", + "action": "link", + "object": "work_id", + "data": "kws.1000" +} +``` + +响应 json： + +```json +{ + "created": 1737599866, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "vlm.1003" +} +``` + +error::code 为 0 表示执行成功。 + +将 vlm 和 kws 单元链接起来，当 kws 发出唤醒数据时，vlm 单元停止上次未完成的推理，用于重复唤醒功能。 + +> **link 时必须保证 kws 此时已经配置好进入工作状态。link 也可以在 setup 阶段进行。** + +示例: + +```json +{ + "request_id": "2", + "work_id": "vlm", + "action": "setup", + "object": "vlm.setup", + "data": { + "model": "internvl2.5-1B-ax630c", + "response_format": "vlm.utf-8.stream", + "input": [ + "vlm.utf-8", + "kws.1000", + "whisper.1001" + ], + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` + +## unlink + +取消链接。 + +发送 json： + +```json +{ + "request_id": "4", + "work_id": "vlm.1003", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" +} +``` + +响应 json： + +```json +{ + "created": 1737600178, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "vlm.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## pause + +暂停单元工作。 + +发送 json： + +```json +{ + "request_id": "5", + "work_id": "vlm.1003", + "action": "pause" +} +``` + +响应 json： + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "vlm.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## work + +恢复单元工作。 + +发送 json： + +```json +{ + "request_id": "6", + "work_id": "vlm.1003", + "action": "work" +} +``` + +响应 json： + +```json +{ + "created": 1737600236, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "vlm.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## exit + +退出单元工作。 + +发送 json： + +```json +{ + "request_id": "7", + "work_id": "vlm.1003", + "action": "exit" +} +``` + +响应 json： + +```json +{ + "created": 1737600704, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "vlm.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "vlm", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1737600076, + "data": [ + "vlm.1003" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.tasklist", + "request_id": "2", + "work_id": "vlm" +} +``` + +获取任务运行参数。 + +```json +{ + "request_id": "2", + "work_id": "vlm.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1737600098, + "data": { + "enoutput": true, + "inputs": [ + "vlm.utf-8", + "kws.1000" + ], + "model": "internvl2.5-1B-ax630c", + "response_format": "vlm.utf-8.stream" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "vlm.taskinfo", + "request_id": "2", + "work_id": "vlm.1003" +} +``` + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_whisper_en.md b/doc/projects_llm_framework_doc/llm_whisper_en.md new file mode 100644 index 0000000..45c5877 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_whisper_en.md @@ -0,0 +1,318 @@ +# llm-asr + +The speech-to-text module, which allows you to choose the language model for recognizing speech, provides speech-to-text +services in multiple languages. + +## setup + +Configure the module to work. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "whisper", + "action": "setup", + "object": "whisper.setup", + "data": { + "model": "whisper-tiny", + "response_format": "asr.utf-8", + "input": "sys.pcm", + "language": "zh", + "enoutput": true + } +} +``` + +- `request_id`: Reference basic data explanation. +- `work_id`: For configuration, use `whisper`. +- `action`: The method called is `setup`. +- `object`: The data type being transferred is `whisper.setup`. +- `model`: The model being used is `whisper-tiny`. +- `response_format`: The returned result is `asr.utf-8`, which is a non-streaming UTF-8 output. +- `input`: The input is `sys.pcm`, representing system audio. +- `language`: The language for the model to recognize. +- `enoutput`: Whether to enable user result output. + +Response JSON: + +```json +{ + "created": 1737597583, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "whisper.1002" +} +``` + +- `created`: The message creation time, in Unix time. +- `work_id`: The work ID of the successfully created unit. + +## link + +Link the output of the upper-level unit. + +Send JSON: + +```json +{ + "request_id": "3", + "work_id": "whisper.1002", + "action": "link", + "object": "work_id", + "data": "vad.1000" +} +``` + +Response JSON: + +```json +{ + "created": 1737597688, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "whisper.1002" +} +``` + +Error `code: 0` indicates successful execution. + +Link the `vad` and `whisper` units, so when `vad` detects the start of speech data, the `whisper` unit will begin +recognizing the user's speech. When `vad` detects the end of speech, the `whisper` unit will output the recognition +result, continuing until the next speech is detected by `vad`. + +Link `kws` and `whisper` units. When `kws` detects a wake-up signal, the `whisper` unit will start recognizing the +user's speech, and when `vad` detects the end of speech, `whisper` will output the result until the next wake-up event. + +> **When linking, make sure that `kws` and `vad` are properly configured and linked together. See the `vad` +documentation for more details. Linking can also be done during the setup phase.** + +Example: + +```json +{ + "request_id": "2", + "work_id": "whisper", + "action": "setup", + "object": "whisper.setup", + "data": { + "model": "whisper-tiny", + "response_format": "asr.utf-8", + "input": [ + "sys.pcm", + "kws.1000", + "vad.1001" + ], + "language": "zh", + "enoutput": true + } +} +``` + +## unlink + +Unlink the units. + +Send JSON: + +```json +{ + "request_id": "4", + "work_id": "whisper.1002", + "action": "unlink", + "object": "work_id", + "data": "vad.1000" +} +``` + +Response JSON: + +```json +{ + "created": 1737598243, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "whisper.1002" +} +``` + +Error `code: 0` indicates successful execution. + +## pause + +Pause the module. + +Send JSON: + +```json +{ + "request_id": "5", + "work_id": "whisper.1002", + "action": "pause" +} +``` + +Response JSON: + +```json +{ + "created": 1737598297, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "whisper.1002" +} +``` + +Error `code: 0` indicates successful execution. + +## work + +Resume the module. + +Send JSON: + +```json +{ + "request_id": "6", + "work_id": "whisper.1002", + "action": "work" +} +``` + +Response JSON: + +```json +{ + "created": 1737598333, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "whisper.1002" +} +``` + +Error `code: 0` indicates successful execution. + +## exit + +Exit the module. + +Send JSON: + +```json +{ + "request_id": "7", + "work_id": "whisper.1002", + "action": "exit" +} +``` + +Response JSON: + +```json +{ + "created": 1737598447, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "whisper.1002" +} +``` + +Error `code: 0` indicates successful execution. + +## taskinfo + +Get the task list. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "whisper", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737598371, + "data": [ + "whisper.1002" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "whisper.tasklist", + "request_id": "2", + "work_id": "whisper" +} +``` + +Get task parameters: + +```json +{ + "request_id": "2", + "work_id": "whisper.1002", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737598415, + "data": { + "enoutput": true, + "inputs": [ + "sys.pcm" + ], + "model": "whisper-tiny", + "response_format": "asr.utf-8" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "whisper.taskinfo", + "request_id": "2", + "work_id": "whisper.1002" +} +``` + +> **Note: `work_id` increases according to the initialization order of units and is not a fixed index value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_whisper_zh.md b/doc/projects_llm_framework_doc/llm_whisper_zh.md new file mode 100644 index 0000000..62baf54 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_whisper_zh.md @@ -0,0 +1,316 @@ +# llm-asr + +语音转文字单元，可选择模型识别语言，用于提供多种语言的语音转文字服务。 + +## setup + +配置单元工作。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "whisper", + "action": "setup", + "object": "whisper.setup", + "data": { + "model": "whisper-tiny", + "response_format": "asr.utf-8", + "input": "sys.pcm", + "language": "zh", + "enoutput": true + } +} +``` + +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `whisper`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `whisper.setup`。 +- model：使用的模型为 `whisper-tiny` 模型。 +- response_format：返回结果为 `asr.utf-8`, utf-8 的非流式输出。 +- input：输入的为 `sys.pcm`,代表的是系统音频。 +- language:选择模型识别的语言。 +- enoutput：是否起用用户结果输出。 + +响应 json： + +```json +{ + "created": 1737597583, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "whisper.1002" +} +``` + +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## link + +链接上级单元的输出。 + +发送 json： + +```json +{ + "request_id": "3", + "work_id": "whisper.1002", + "action": "link", + "object": "work_id", + "data": "vad.1000" +} +``` + +响应 json： + +```json +{ + "created": 1737597688, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "whisper.1002" +} +``` + +error::code 为 0 表示执行成功。 + +将 vad 和 whisper 单元链接起来，当 vad 发出检测到语言数据开始时，whisper 单元开始识别用户的语音，当 vad 发出检测到语言结束时，whisper +单元输出识别结果，直到下一次 vad 检测到语音。 + +将 kws 和 whisper 单元链接起来，当 kws 发出唤醒数据时，whisper 单元开始识别用户的语音，当 vad 发出检测到语言结束时，whisper +单元输出识别结果，直到下一次的唤醒。 + +> **link 时必须保证 kws 和 vad 此时已经配置好进入工作状态，且 kws 和 vad 已经链接，详见 vad 文档。link 也可以在 setup 阶段进行。 +** + +示例: + +```json +{ + "request_id": "2", + "work_id": "whisper", + "action": "setup", + "object": "whisper.setup", + "data": { + "model": "whisper-tiny", + "response_format": "asr.utf-8", + "input": [ + "sys.pcm", + "kws.1000", + "vad.1001" + ], + "language": "zh", + "enoutput": true + } +} +``` + +## unlink + +取消链接。 + +发送 json： + +```json +{ + "request_id": "4", + "work_id": "whisper.1002", + "action": "unlink", + "object": "work_id", + "data": "vad.1000" +} +``` + +响应 json： + +```json +{ + "created": 1737598243, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "whisper.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## pause + +暂停单元工作。 + +发送 json： + +```json +{ + "request_id": "5", + "work_id": "whisper.1002", + "action": "pause" +} +``` + +响应 json： + +```json +{ + "created": 1737598297, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "whisper.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## work + +恢复单元工作。 + +发送 json： + +```json +{ + "request_id": "6", + "work_id": "whisper.1002", + "action": "work" +} +``` + +响应 json： + +```json +{ + "created": 1737598333, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "whisper.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## exit + +单元退出。 + +发送 json： + +```json +{ + "request_id": "7", + "work_id": "whisper.1002", + "action": "exit" +} +``` + +响应 json： + +```json +{ + "created": 1737598447, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "whisper.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表。 + +发送 json： + +```json +{ + "request_id": "2", + "work_id": "whisper", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1737598371, + "data": [ + "whisper.1002" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "whisper.tasklist", + "request_id": "2", + "work_id": "whisper" +} +``` + +获取任务运行参数。 + +```json +{ + "request_id": "2", + "work_id": "whisper.1002", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created": 1737598415, + "data": { + "enoutput": true, + "inputs": [ + "sys.pcm" + ], + "model": "whisper-tiny", + "response_format": "asr.utf-8" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "whisper.taskinfo", + "request_id": "2", + "work_id": "whisper.1002" +} +``` + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_yolo_en.md b/doc/projects_llm_framework_doc/llm_yolo_en.md new file mode 100644 index 0000000..ac279b5 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_yolo_en.md @@ -0,0 +1,153 @@ +# llm-yolo + +YOLO vision detection unit for providing image detection services. Various YOLO models can be selected. + +## setup + +Configuring the unit to work. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "yolo", + "action": "setup", + "object": "yolo.setup", + "data": { + "model": "yolo11n", + "response_format": "yolo.box", + "input": "yolo.jpg.base64", + "enoutput": true + } +} +``` + +- request_id: Reference for basic data explanation. +- work_id: Set to `yolo` during configuration. +- action: The method to call is `setup`. +- object: Data type being transmitted is `yolo.setup`. +- model: The model used is `yolo11n`. +- response_format: The returned result is `yolo.box`. +- input: The input is `yolo.jpg.base64`, which represents the data from the user input, encoded in jpg and base64. +- enoutput: Whether to enable user result output. + +Response JSON: + +```json +{ + "created": 1737596640, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "yolo.1001" +} +``` + +- created: Message creation time, Unix time. +- work_id: The successfully created work_id unit. + +## exit + +Unit exit. + +Send JSON: + +```json +{ + "request_id": "7", + "work_id": "yolo.1001", + "action": "exit" +} +``` + +Response JSON: + +```json +{ + "created": 1737596871, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "yolo.1001" +} +``` + +`error::code` of 0 means the execution was successful. + +## taskinfo + +Retrieve the task list. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "yolo", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737596668, + "data": [ + "yolo.1001" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "yolo.tasklist", + "request_id": "2", + "work_id": "yolo" +} +``` + +Retrieve task run parameters. + +Send JSON: + +```json +{ + "request_id": "2", + "work_id": "yolo.1003", + "action": "taskinfo" +} +``` + +Response JSON: + +```json +{ + "created": 1737596698, + "data": { + "enoutput": true, + "inputs": [ + "yolo.jpg.base64" + ], + "model": "yolo11n", + "response_format": "yolo.box" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "yolo.taskinfo", + "request_id": "2", + "work_id": "yolo.1001" +} +``` + +> **Note: work_id increases according to the order of unit initialization, not a fixed index value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_yolo_zh.md b/doc/projects_llm_framework_doc/llm_yolo_zh.md index 66c0f6d..717f007 100644 --- a/doc/projects_llm_framework_doc/llm_yolo_zh.md +++ b/doc/projects_llm_framework_doc/llm_yolo_zh.md @@ -1,29 +1,33 @@ # llm-yolo -yolo 视觉检测单元，用于提供图片检测服务。可选择多用 yolo 模型。 + +yolo 视觉检测单元，用于提供图片检测服务。可选择多种 yolo 模型。 ## setup + 配置单元工作。 发送 json： + ```json { - "request_id": "2", - "work_id": "yolo", - "action": "setup", - "object": "yolo.setup", - "data": { - "model": "yolov5", - "response_format": "yolo.box", - "input": "yolo.jpg.base64", - "enoutput": true - } + "request_id": "2", + "work_id": "yolo", + "action": "setup", + "object": "yolo.setup", + "data": { + "model": "yolo11n", + "response_format": "yolo.box", + "input": "yolo.jpg.base64", + "enoutput": true + } } ``` + - request_id：参考基本数据解释。 - work_id：配置单元时，为 `yolo`。 - action：调用的方法为 `setup`。 - object：传输的数据类型为 `yolo.setup`。 -- model：使用的模型为 `yolov5` 模型。 +- model：使用的模型为 `yolo11n` 模型。 - response_format：返回结果为 `yolo.box`。 - input：输入的为 `yolo.jpg.base64`,代表的是从用户输入，数据类型为 jpg, base64编码。 - enoutput：是否起用用户结果输出。 @@ -32,21 +36,21 @@ yolo 视觉检测单元，用于提供图片检测服务。可选择多用 yolo ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"2", - "work_id":"yolo.1003" + "created": 1737596640, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "yolo.1001" } ``` + - created：消息创建时间，unix 时间。 - work_id：返回成功创建的 work_id 单元。 - ## exit 单元退出。 @@ -55,9 +59,9 @@ yolo 视觉检测单元，用于提供图片检测服务。可选择多用 yolo ```json { - "request_id": "7", - "work_id": "tts.1003", - "action": "exit", + "request_id": "7", + "work_id": "yolo.1001", + "action": "exit" } ``` @@ -65,15 +69,15 @@ yolo 视觉检测单元，用于提供图片检测服务。可选择多用 yolo ```json { - "created":1731488402, - "data":"None", - "error":{ - "code":0, - "message":"" - }, - "object":"None", - "request_id":"7", - "work_id":"tts.1003" + "created": 1737596871, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "yolo.1001" } ``` @@ -84,11 +88,12 @@ error::code 为 0 表示执行成功。 获取任务列表。 发送 json： + ```json { - "request_id": "2", - "work_id": "yolo", - "action": "taskinfo" + "request_id": "2", + "work_id": "yolo", + "action": "taskinfo" } ``` @@ -96,26 +101,29 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652311, - "data":["yolo.1003"], - "error":{ - "code":0, - "message":"" - }, - "object":"yolo.tasklist", - "request_id":"2", - "work_id":"yolo" + "created": 1737596668, + "data": [ + "yolo.1001" + ], + "error": { + "code": 0, + "message": "" + }, + "object": "yolo.tasklist", + "request_id": "2", + "work_id": "yolo" } ``` 获取任务运行参数。 发送 json： + ```json { - "request_id": "2", - "work_id": "yolo.1003", - "action": "taskinfo" + "request_id": "2", + "work_id": "yolo.1003", + "action": "taskinfo" } ``` @@ -123,22 +131,23 @@ error::code 为 0 表示执行成功。 ```json { - "created":1731652344, - "data":{ - "enoutput":false, - "inputs_":["yolo.jpg.base64"], - "model":"yolov5", - "response_format":"yolo.box" - }, - "error":{ - "code":0, - "message":"" - }, - "object":"tts.taskinfo", - "request_id":"2", - "work_id":"yolo.1003" + "created": 1737596698, + "data": { + "enoutput": true, + "inputs": [ + "yolo.jpg.base64" + ], + "model": "yolo11n", + "response_format": "yolo.box" + }, + "error": { + "code": 0, + "message": "" + }, + "object": "yolo.taskinfo", + "request_id": "2", + "work_id": "yolo.1001" } ``` - > **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file