From 4c26c7251b0510881843da318575d42fe795c865 Mon Sep 17 00:00:00 2001 From: dianjixz Date: Mon, 18 Nov 2024 12:14:11 +0800 Subject: [PATCH] [init] StackFlow --- README.md | 91 ++ README_zh.md | 105 +++ SDK | 1 + doc/component_doc/StackFlow_en.md | 418 ++++++++++ doc/component_doc/StackFlow_zh.md | 418 ++++++++++ doc/projects_llm_framework_doc/llm_asr_en.md | 297 +++++++ doc/projects_llm_framework_doc/llm_asr_zh.md | 300 +++++++ .../llm_audio_en.md | 13 + .../llm_audio_zh.md | 16 + doc/projects_llm_framework_doc/llm_kws_en.md | 199 +++++ doc/projects_llm_framework_doc/llm_kws_zh.md | 205 +++++ doc/projects_llm_framework_doc/llm_llm_en.md | 284 +++++++ doc/projects_llm_framework_doc/llm_llm_zh.md | 287 +++++++ .../llm_melotts_en.md | 272 ++++++ .../llm_melotts_zh.md | 282 +++++++ doc/projects_llm_framework_doc/llm_sys_en.md | 20 + doc/projects_llm_framework_doc/llm_sys_zh.md | 20 + doc/projects_llm_framework_doc/llm_tts_en.md | 279 +++++++ doc/projects_llm_framework_doc/llm_tts_zh.md | 282 +++++++ ext_components/StackFlow/Kconfig | 10 + ext_components/StackFlow/SConstruct | 42 + .../StackFlow/stackflow/StackFlow.cpp | 512 ++++++++++++ .../StackFlow/stackflow/StackFlow.h | 361 ++++++++ .../StackFlow/stackflow/StackFlowUtil.cpp | 391 +++++++++ .../StackFlow/stackflow/StackFlowUtil.h | 20 + .../StackFlow/stackflow/libzmq/zmq.h | 787 ++++++++++++++++++ .../StackFlow/stackflow/libzmq/zmq_utils.h | 23 + ext_components/StackFlow/stackflow/pzmq.hpp | 288 +++++++ ext_components/ax_msp/Kconfig | 19 + ext_components/ax_msp/SConstruct | 86 ++ 30 files changed, 6328 insertions(+) create mode 100644 README.md create mode 100644 README_zh.md create mode 160000 SDK create mode 100644 doc/component_doc/StackFlow_en.md create mode 100644 doc/component_doc/StackFlow_zh.md create mode 100644 doc/projects_llm_framework_doc/llm_asr_en.md create mode 100644 doc/projects_llm_framework_doc/llm_asr_zh.md create mode 100644 doc/projects_llm_framework_doc/llm_audio_en.md create mode 100644 doc/projects_llm_framework_doc/llm_audio_zh.md create mode 100644 doc/projects_llm_framework_doc/llm_kws_en.md create mode 100644 doc/projects_llm_framework_doc/llm_kws_zh.md create mode 100644 doc/projects_llm_framework_doc/llm_llm_en.md create mode 100644 doc/projects_llm_framework_doc/llm_llm_zh.md create mode 100644 doc/projects_llm_framework_doc/llm_melotts_en.md create mode 100644 doc/projects_llm_framework_doc/llm_melotts_zh.md create mode 100644 doc/projects_llm_framework_doc/llm_sys_en.md create mode 100644 doc/projects_llm_framework_doc/llm_sys_zh.md create mode 100644 doc/projects_llm_framework_doc/llm_tts_en.md create mode 100644 doc/projects_llm_framework_doc/llm_tts_zh.md create mode 100644 ext_components/StackFlow/Kconfig create mode 100644 ext_components/StackFlow/SConstruct create mode 100644 ext_components/StackFlow/stackflow/StackFlow.cpp create mode 100644 ext_components/StackFlow/stackflow/StackFlow.h create mode 100644 ext_components/StackFlow/stackflow/StackFlowUtil.cpp create mode 100644 ext_components/StackFlow/stackflow/StackFlowUtil.h create mode 100644 ext_components/StackFlow/stackflow/libzmq/zmq.h create mode 100644 ext_components/StackFlow/stackflow/libzmq/zmq_utils.h create mode 100644 ext_components/StackFlow/stackflow/pzmq.hpp create mode 100644 ext_components/ax_msp/Kconfig create mode 100644 ext_components/ax_msp/SConstruct diff --git a/README.md b/README.md new file mode 100644 index 0000000..2b90d1f --- /dev/null +++ b/README.md @@ -0,0 +1,91 @@ +# Module-LLM + +
+ +## Description + +**Module LLM** is an integrated offline Large Language Model (LLM) inference module designed for terminal devices that require efficient and intelligent interaction. Whether for smart homes, voice assistants, or industrial control, Module LLM provides a smooth and natural AI experience without relying on the cloud, ensuring privacy and stability. Integrated with the **StackFlow** framework and **Arduino/UiFlow** libraries, smart features can be easily implemented with just a few lines of code.
+Powered by the advanced **AX630C** SoC processor, it integrates a 3.2 TOPs high-efficiency NPU with native support for Transformer models, handling complex AI tasks with ease. Equipped with **4GB LPDDR4** memory (1GB available for user applications, 3GB dedicated to hardware acceleration) and **32GB eMMC** storage, it supports parallel loading and sequential inference of multiple models, ensuring smooth multitasking. The main chip's runtime power consumption of approximately 1.5W, making it highly efficient and suitable for long-term operation.
+It features a built-in microphone, speaker, TF storage card, **USB OTG**, and RGB status light, meeting diverse application needs with support for voice interaction and data transfer. The module offers flexible expansion: the onboard SD card slot supports cold/hot firmware upgrades, and the **UART** communication interface simplifies connection and debugging, ensuring continuous optimization and expansion of module functionality. The USB port supports master-slave auto-switching, serving as both a debugging port and allowing connection to additional USB devices like cameras. Users can purchase the LLM debugging kit to add a 100 Mbps Ethernet port and kernel serial port, using it as an SBC.
+The module is compatible with multiple models and comes pre-installed with the **Qwen2.5-0.5B** language model. It features **KWS** (wake word), **ASR** (speech recognition), **LLM** (large language model), and **TTS** (text-to-speech) functionalities, with support for standalone calls or **pipeline** automatic transfer for convenient development. Future support includes Qwen2.5-1.5B, Llama3.2-1B, and InternVL2-1B models, allowing hot model updates to keep up with community trends and accommodate various complex AI tasks. Vision recognition capabilities include support for CLIP, YoloWorld, and future updates for DepthAnything, SegmentAnything, and other advanced models to enhance intelligent recognition and analysis.
+Plug and play with **M5 hosts**, Module LLM offers an easy-to-use AI interaction experience. Users can quickly integrate it into existing smart devices without complex settings, enabling smart functionality and improving device intelligence. This product is suitable for offline voice assistants, text-to-speech conversion, smart home control, interactive robots, and more. + + + +## Product Features + +- Offline inference, 3.2T@INT8 precision computing power +- Integrated KWS (wake word), ASR (speech recognition), LLM (large language model), TTS (text-to-speech generation) +- Multi-model parallel processing +- Onboard 32GB eMMC storage and 4GB LPDDR4 memory +- Onboard microphone and speaker +- Serial communication +- SD card firmware upgrade +- Supports ADB debugging +- RGB indicator light +- Built-in Ubuntu system +- Supports OTG functionality +- Compatible with Arduino/UIFlow + +> + +## Applications + +- Offline voice assistants +- Text-to-speech conversion +- Smart home control +- Interactive robots + +## Specifications + +| Specification | Parameter | +| ---------------- | ------------------------------------------------------------------------------------------- | +| Processor SoC | AX630C@Dual Cortex A53 1.2 GHz
MAX.12.8 TOPS @INT4 and 3.2 TOPS @INT8 | +| Memory | 4GB LPDDR4 (1GB system memory + 3GB dedicated for hardware acceleration) | +| Storage | 32GB eMMC5.1 | +| Communication | Serial communication default baud rate 115200@8N1 (adjustable) | +| Microphone | MSM421A | +| Audio Driver | AW8737 | +| Speaker | 8Ω@1W, Size:2014 cavity speaker | +| Built-in Units | KWS (wake word), ASR (speech recognition), LLM (large language model), TTS (text-to-speech) | +| RGB Light | 3x RGB LED@2020 driven by LP5562 (status indication) | +| Power | Idle: 5V@0.5W, Full load: 5V@1.5W | +| Button | For entering download mode for firmware upgrade | +| Upgrade Port | SD card / Type-C port | +| Working Temp | 0-40°C | +| Product Size | 54*54*13mm | +| Packaging Size | 133*95*16mm | +| Product Weight | 17.4g | +| Packaging Weight | 32.0g | + +
+ +
+ +## Related Links + +- [AX630C](https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/products/module/Module%20LLM/AX630C.pdf) + +## PinMap + +| Module LLM | RXD | TXD | +| ------------ | --- | --- | +| Core (Basic) | G16 | G17 | +| Core2 | G13 | G14 | +| CoreS3 | G18 | G17 | + +>LLM Module Pin Switching| LLM Module has reserved soldering pads for pin switching. In cases of pin multiplexing conflicts, the PCB trace can be cut and reconnected to other sets of pins. + +module size + +> Taking `CoreS3` as an example, the first column (left green box) is the TX pin for serial communication, where users can choose one out of four options as needed (from top to bottom, the pins are G18, G7, G14, and G10). The default is set to IO18. To switch to a different pin, cut the connection on the solder pad (at the red line) — it’s recommended to use a blade for this — and then connect to one of the three remaining pins below. The second column (right green box) is for RX pin selection, and, as with the TX pin, it also allows a choice of one out of four options. + + + +## Video + +- Module LLM product introduction and example showcase [Module_LLM_Video.mp4](https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/products/module/Module%20LLM/Module_LLM_Video.mp4) + +## AI Benchmark Comparison + +compare \ No newline at end of file diff --git a/README_zh.md b/README_zh.md new file mode 100644 index 0000000..e8fc0b0 --- /dev/null +++ b/README_zh.md @@ -0,0 +1,105 @@ +# Module-LLM + +
+ +## 描述 + +**Module LLM**是一款集成化的离线大语言模型 (LLM) 推理模块，专为需要高效，智能交互的终端设备设计，无论是在智能家居，语音助手，还是在工业控制中，Module LLM 都能为您带来流畅，自然的 AI 体验，无需依赖云端，确保隐私安全与稳定性。集成 **StackFlow** 框架，配合 **Arduino/UiFlow** 库，几行代码就可轻松实现端侧智能。
+搭载爱芯 **AX630C** SoC 先进处理器，集成 3.2 TOPs 高能效 NPU，原生支持 Transformer 模型，轻松应对复杂 AI 任务。配备 **4GB LPDDR4** 内存（其中1GB供用户使用，3GB专用于硬件加速）和**32GB eMMC**存储，支持多模型并行加载与串联推理，确保多任务处理流畅无阻。运行功耗仅约 1.5W，远低于同类产品，节能高效，适合长时间运行。
+集成麦克风，扬声器，TF存储卡，**USB OTG** 及 RGB状态灯，满足多样化应用需求，轻松实现语音交互与数据传输。灵活扩展：板载 SD 卡槽支持固件冷/热升级，**UART** 通信接口简化连接与调试，确保模块功能持续优化与扩展。USB 口支持主从自动切换，既可以做调试口，也可以外接更多 USB 设备如摄像头。搭配 LLM 调试套件，用于扩展百兆以太网口，及内核串口，作为 SBC 使用。
+多模型兼容，出厂预装 **Qwen2.5-0.5B** 大语言模型，内置**KWS**（唤醒词），**ASR**（语音识别），**LLM**（大语言模型）及**TTS**（文本转语音）功能，且支持分开调用或 **pipeline** 自动流转，方便开发。后续将支持Qwen2.5-1.5B、Llama3.2-1B及InternVL2-1B等多种端侧LLM/VLM模型，支持热更新模型，紧跟社区潮流，适应不同复杂度的AI任务；视觉识别能力：支持CLIP、YoloWorld等Open world模型，未来将持续更新DepthAnything、SegmentAnything等先进模型，赋能智能识别与分析；
+即插即用，搭配**M5主机**，Module LLM 实现即插即用的AI交互体验。用户无需繁琐设置，即可将其集成到现有智能设备中，快速启用智能化功能，提升设备智能水平。该产品适用于离线语音助手，文本语音转换，智能家居控制，互动机器人等领域。 + + + + +## 产品特性 + +- 离线推理,3.2T@INT8精度算力 + +- 集成KWS（唤醒词），ASR（语音识别），LLM（大语言模型），TTS（文本生成语音） + +- 多模型并行 + +- 板载32GB eMMC存储和4GB LPDDR4内存 + +- 板载麦克风及扬声器 + +- 串口通信 + +- SD卡固件升级 + +- 支持ADB调试 + +- RGB提示灯 + +- 内置ubuntu系统 + +- 支持OTG功能 + +- 支持Arduino/UIFlow + + + + +## 应用 + +- 离线语音助手 +- 文本语音转换 +- 智能家居控制 +- 互动机器人 + +## 规格参数 + +| 规格 | 参数 | +| ------------ | --------------------------------------------------------------------------- | +| 处理器SoC | AX630C@Dual Cortex A53 1.2 GHz
MAX.12.8 TOPS @INT4,and 3.2 TOPS @INT8 | +| 内存 | 4GB LPDDR4(1GB系统内存 + 3GB 硬件加速专用内存) | +| 存储 | 32GB eMMC5.1 | +| 通信接口 | 串口通信 波特率默认 115200@8N1（可调） | +| 麦克风 | MSM421A | +| 音频驱动 | AW8737 | +| 扬声器 | 8Ω@1W,尺寸:2014 腔体喇叭 | +| 内置功能单元 | KWS（唤醒词），ASR（语音识别），LLM（大语言模型），TTS（文本生成语音） | +| RGB灯 | 3x RGB LED@2020 由LP5562驱动 (状态指示) | +| 功耗 | 空载：5V@0.5W，满载:5V@1.5W | +| 按键 | 用于升级固件进入下载模式 | +| 升级接口 | SD卡/Type C口 | +| 工作温度 | 0-40°C | +| 产品尺寸 | 54*54*13mm | +| 包装尺寸 | 133*95*16mm | +| 产品重量 | 17.4g | +| 包装重量 | 32.0g | + +
+ +
+ + +## 相关链接 + +- [AX630C](https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/products/module/Module%20LLM/AX630C.pdf) + +## PinMap + +| Module LLM | RXD | TXD | +| ------------ | --- | --- | +| Core (Basic) | G16 | G17 | +| Core2 | G13 | G14 | +| CoreS3 | G18 | G17 | + +>LLM Module引脚切换| LLM Module预留了引脚切换焊盘, 一些出现引脚复用冲突的情况下, 可以通过切割PCB线路然后跳线连接至其他组引脚。 + +module size + +> 以`CoreS3`为例子，第一列（左绿色框）是串口通讯的TX引脚，用户根据需要四选一，（从上到下分别代表引脚G18 G7 G14 G10）,默认是IO18引脚，如需要切换其他引脚，切断焊盘连接线（红线处）（建议使用刀片切割），然后连接下面三个引脚之间的一个即可；第二列（右绿色框）是RX引脚切换，如上所述，也是四选一操作 + + + +## 视频 + +- Module LLM 产品介绍以及例子展示 [Module_LLM_Video.mp4](https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/products/module/Module%20LLM/Module_LLM_Video.mp4) + +## AI Benchmark 对比 + +compare diff --git a/SDK b/SDK new file mode 160000 index 0000000..c767769 --- /dev/null +++ b/SDK @@ -0,0 +1 @@ +Subproject commit c767769d28ec511b613a24fc8680ec3acd4c769a diff --git a/doc/component_doc/StackFlow_en.md b/doc/component_doc/StackFlow_en.md new file mode 100644 index 0000000..5d542be --- /dev/null +++ b/doc/component_doc/StackFlow_en.md @@ -0,0 +1,418 @@ +# StackFlow +StackFlow is a communication framework developed by M5STACK for AI accelerated computing, mainly running on embedded Linux platforms, with [zmq](https://zeromq.org/) providing the underlying communication services. +StackFlow primarily offers three functions: first, remote RPC calls for function invocation between units; second, message communication, providing standard message stream services for better context integration; and third, resource allocation, mainly to avoid communication address conflicts between related units and temporary data storage. +![](../../assets/image/StackFlow_frame.png) + +## pzmq +The zmq API is repackaged to make the invocation of ZMQ_PUB, ZMQ_SUB, ZMQ_PUSH, and ZMQ_PULL simpler and more convenient, using asynchronous callback methods to provide message reception functionality. +On the basis of ZMQ_REP and ZMQ_REQ, simple RPC functionality is encapsulated to provide RPC services. + +### Related Examples +1. ZMQ_PULL +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +void pull_msg(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; +} +int main(int argc, char *argv[]) { + pzmq zpull_("ipc:///tmp.5000.socket", ZMQ_PULL, pull_msg); + while(1) { + sleep(1); + } + return 0; +} +``` + +2. ZMQ_PUSH +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +int main(int argc, char *argv[]) { + pzmq zpush_("ipc:///tmp.5000.socket", ZMQ_PUSH); + zpush_.send_data("nihao"); + return 0; +} +``` + +3. ZMQ_PUB +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +int main(int argc, char *argv[]) { + pzmq zpush_("ipc:///tmp.5001.socket", ZMQ_PUB); + zpush_.send_data("nihao"); + return 0; +} +``` + +4. ZMQ_SUB +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +void sub_msg(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; +} +int main(int argc, char *argv[]) { + pzmq zpull_("ipc:///tmp.5001.socket", ZMQ_SUB, sub_msg); + while(1) { + sleep(1); + } + return 0; +} +``` + +4. ZMQ_RPC_FUN +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +std::string fun1_(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; + return std::string("nihao"); +} +std::string fun2_(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; + return std::string("hello"); +} +int main(int argc, char *argv[]) { + pzmq _rpc("test"); + _rpc.register_rpc_action("fun1", fun1_); + _rpc.register_rpc_action("fun2", fun2_); + while(1) { + sleep(1); + } + return 0; +} +``` + +5. ZMQ_RPC_CALL +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +std::string fun1_(const std::string &raw_msg){ + return std::string("nihao"); +} +std::string fun2_(const std::string &raw_msg){ + return std::string("hello"); +} +int main(int argc, char *argv[]) { + pzmq _rpc("test"); + _rpc.call_rpc_action("fun1", "call fun1_", [](const std::string &raw_msg){std::cout << raw_msg << std::endl;}); + _rpc.call_rpc_action("fun2", "call fun2_", [](const std::string &raw_msg){std::cout << raw_msg << std::endl;}); + return 0; +} +``` + +## StackFlow Main Body +StackFlow encapsulates pzmq and eventpp, providing basic RPC functions, asynchronous processing, and channel establishment for accelerated units. +StackFlow provides seven basic RPC functions for basic function calls of the StackFlow JSON protocol. +- setup: Unit configuration function, a function that each unit must implement. +- pause: Suspend unit function. +- work: Unit start work function. +- exit: Unit exit function, a function that each unit must implement. +- link: Link to the upstream output function, used to build a message communication chain. +- unlink: Unlink from the upstream output function, stop receiving messages from the upstream. +- taskinfo: Get unit running information. + +StackFlow provides convenient APIs for unit usage: +- unit_call: Unit RPC call function, to call RPC functions of other units. +- sys_sql_select: sys unit simple key-value database query function. +- sys_sql_set: sys unit simple key-value database setting function. +- sys_sql_unset: sys unit simple key-value database deletion function. +- repeat_event: Asynchronous periodic execution function. +- send: Send user message function. +- sys_register_unit: Unit registration function, generally not needed to be called. +- sys_release_unit: Unit release function, generally not needed to be called. + +llm_channel_obj encapsulates the communication functions required by the unit, with one configuration corresponding to one llm_channel_obj object. +llm_channel_obj provides convenient communication APIs for unit usage: +- subscriber_work_id: Subscribe to the pub output of the upstream work_id unit. +- stop_subscriber_work_id: Unsubscribe from work_id. +- subscriber: Subscribe to the pub output of the zmq URL. +- stop_subscriber: Unsubscribe from zmq_url. +- send: Send messages of this unit through pub. + +### Basic Usage Example: +``` c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include +#include +#include +#include +#include +#include +#include +using namespace StackFlows; +int main_exit_flage = 0; +static void __sigint(int iSigNo) { + main_exit_flage = 1; +} +typedef std::function task_callback_t; +class llm_task { + private: + public: + std::string model_; + std::string response_format_; + std::vector inputs_; + task_callback_t out_callback_; + bool enoutput_; + bool enstream_; + + void set_output(task_callback_t out_callback) { + out_callback_ = out_callback; + } + + bool parse_config(const nlohmann::json &config_body) { + try { + model_ = config_body.at("model"); + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } + } catch (...) { + return true; + } + enstream_ = (response_format_.find("stream") != std::string::npos); + return false; + } + + int load_model(const nlohmann::json &config_body) { + if (parse_config(config_body)) { + return -1; + } + return 0; + } + + void inference(const std::string &msg) { + std::cout << msg << std::endl; + if (out_callback_) out_callback_(std::string("hello"), true); + } + + llm_task(const std::string &workid) { + } + + ~llm_task() { + } +}; + +class llm_llm : public StackFlow { + private: + int task_count_; + std::unordered_map> llm_task_; + + public: + llm_llm() : StackFlow("test") { + task_count_ = 1; + } + + void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, + const std::string &data, bool finish) { + if (llm_channel->enstream_) { + static int count = 0; + nlohmann::json data_body; + data_body["index"] = count++; + data_body["delta"] = data; + if (!finish) + data_body["delta"] = data; + else + data_body["delta"] = std::string(""); + data_body["finish"] = finish; + if (finish) count = 0; + llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); + } else if (finish) { + llm_channel->send(llm_task_obj->response_format_, data, LLM_NO_ERROR); + } + } + + void task_user_data(const std::shared_ptr llm_task_obj, + const std::shared_ptr llm_channel, const std::string &object, + const std::string &data) { + const std::string *next_data = &data; + int ret; + std::string tmp_msg1; + if (object.find("stream") != std::string::npos) { + static std::unordered_map stream_buff; + if (decode_stream(data, tmp_msg1, stream_buff)) return; + next_data = &tmp_msg1; + } + std::string tmp_msg2; + if (object.find("base64") != std::string::npos) { + ret = decode_base64((*next_data), tmp_msg2); + if (!ret) { + return; + } + next_data = &tmp_msg2; + } + llm_task_obj->inference((*next_data)); + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + nlohmann::json error_body; + if ((llm_task_channel_.size() - 1) == task_count_) { + error_body["code"] = -21; + error_body["message"] = "task full"; + send("None", "None", error_body, unit_name_); + return -1; + } + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, unit_name_); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id("", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + llm_task_[work_id_num] = llm_task_obj; + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } else { + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, unit_name_); + return -1; + } + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("llm.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["model"] = llm_task_obj->model_; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs_"] = llm_task_obj->inputs_; + send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_llm() { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main(int argc, char *argv[]) { + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_llm llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} +``` + +## StackFlowUtil +Provides a convenient API: +- sample_json_str_get: Simple function to read key values inside a JSON, used to quickly read JSON keys without parsing the JSON object. +- sample_get_work_id_num: Read the numeric index from the work_id string. +- sample_get_work_id_name: Read the unit name from the work_id string. +- sample_get_work_id: Used to synthesize the work_id string. +- sample_escapeString: Simple encoding of escape characters in a string. +- sample_unescapeString: Simple decoding of escape strings in a string. +- decode_stream: Parse streaming data flow. +- decode_base64: Decode base64. +- encode_base64: Encode base64. \ No newline at end of file diff --git a/doc/component_doc/StackFlow_zh.md b/doc/component_doc/StackFlow_zh.md new file mode 100644 index 0000000..c9fbe4a --- /dev/null +++ b/doc/component_doc/StackFlow_zh.md @@ -0,0 +1,418 @@ +# StackFlow +StackFlow 是 M5STACK 为 AI 加速计算开发的一个通信框架，主要运行在嵌入式 Linux 平台，由 [zmq](https://zeromq.org/) 提供底层通信服务。 +StackFlow 主要提供三种功能，一、远程 RPC 调用，承载单元之间的函数调用。二、消息通信，提供标准消息流服务，更好的串通上下文。三、资源分配，主要用于避免相关单元的通信地址冲突和临时数据储存。 +![](../../assets/image/StackFlow_frame.png) + +## pzmq +重新包装了 zmq 的 api，让 ZMQ_PUB、ZMQ_SUB、ZMQ_PUSH、ZMQ_PULL 的调用更加简单便捷，采用异步式回调的方法提供接收消息功能。 +在 ZMQ_REP、ZMQ_REQ 的基础上封装了简单的 RPC 功能，提供 RPC 服务。 + +### 相关示例 +1、ZMQ_PULL +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +void pull_msg(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; +} +int main(int argc, char *argv[]) { + pzmq zpull_("ipc:///tmp.5000.socket", ZMQ_PULL, pull_msg); + while(1) { + sleep(1); + } + return 0; +} +``` + +2、ZMQ_PUSH +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +int main(int argc, char *argv[]) { + pzmq zpush_("ipc:///tmp.5000.socket", ZMQ_PUSH); + zpush_.send_data("nihao"); + return 0; +} +``` + +3、ZMQ_PUB +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +int main(int argc, char *argv[]) { + pzmq zpush_("ipc:///tmp.5001.socket", ZMQ_PUB); + zpush_.send_data("nihao"); + return 0; +} +``` + +4、ZMQ_SUB +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +void sub_msg(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; +} +int main(int argc, char *argv[]) { + pzmq zpull_("ipc:///tmp.5001.socket", ZMQ_SUB, sub_msg); + while(1) { + sleep(1); + } + return 0; +} +``` + +4、ZMQ_RPC_FUN +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +std::string fun1_(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; + return std::string("nihao"); +} +std::string fun2_(const std::string &raw_msg){ + std::cout << raw_msg << std::endl; + return std::string("hello"); +} +int main(int argc, char *argv[]) { + pzmq _rpc("test"); + _rpc.register_rpc_action("fun1", fun1_); + _rpc.register_rpc_action("fun2", fun2_); + while(1) { + sleep(1); + } + return 0; +} +``` + +5、ZMQ_RPC_CALL +```c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include "pzmq.hpp" +#include +using namespace StackFlows; +std::string fun1_(const std::string &raw_msg){ + return std::string("nihao"); +} +std::string fun2_(const std::string &raw_msg){ + return std::string("hello"); +} +int main(int argc, char *argv[]) { + pzmq _rpc("test"); + _rpc.call_rpc_action("fun1", "call fun1_", [](const std::string &raw_msg){std::cout << raw_msg << std::endl;}); + _rpc.call_rpc_action("fun2", "call fun2_", [](const std::string &raw_msg){std::cout << raw_msg << std::endl;}); + return 0; +} +``` + +## StackFlow 主体 +StackFlow 封装了 pzmq 和 eventpp，为加速单元提供基础的 RPC 函数、异步处理和信道建立。 +StackFlow 提供基本的七个 RPC 函数，用于 StackFlow json 协议的基础功能调用。 +- setup：单元配置函数，是每个单元必须实现的函数。 +- pause：暂停单元函数。 +- work：单元开始工作函数。 +- exit：单元退出函数，是每个单元必须实现的函数。 +- link：链接单元上级输出函数，用于构建消息通信链。 +- unlink：解除上级输出函数，不在接收上级的消息。 +- taskinfo：获取单元运行信息。 + +StackFlow 提供了简便 API 方便单元使用： +- unit_call: 单元 RPC 调用函数，调用其他单元的 RPC 函数。 +- sys_sql_select: sys 单元的简单键值数据库查寻函数。 +- sys_sql_set: sys 单元的简单键值数据库键值设置函数。 +- sys_sql_unset: sys 单元的简单键值数据库删除键值函数。 +- repeat_event: 异步的定时重复执行函数。 +- send: 发送用户消息函数。 +- sys_register_unit: 单元注册函数，一般情况不需要调用。 +- sys_release_unit: 单元释放函数，一般情况不需要调用。 + +llm_channel_obj 封装了单元所需的通信函数，一份配置对应一个 llm_channel_obj 对象。 +llm_channel_obj 提供了通信简便 API 方便单元使用： +- subscriber_work_id: 订阅上级 work_id 单元的 pub 输出。 +- stop_subscriber_work_id：取消订阅 work_id 。 +- subscriber： 订阅 zmq url 的 pub 输出。 +- stop_subscriber： 取消订阅 zmq_url 。 +- send：将本单元的消息通过 pub 发送出去。 + +### 基本使用示例： +``` c++ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include +#include +#include +#include +#include +#include +#include +using namespace StackFlows; +int main_exit_flage = 0; +static void __sigint(int iSigNo) { + main_exit_flage = 1; +} +typedef std::function task_callback_t; +class llm_task { + private: + public: + std::string model_; + std::string response_format_; + std::vector inputs_; + task_callback_t out_callback_; + bool enoutput_; + bool enstream_; + + void set_output(task_callback_t out_callback) { + out_callback_ = out_callback; + } + + bool parse_config(const nlohmann::json &config_body) { + try { + model_ = config_body.at("model"); + response_format_ = config_body.at("response_format"); + enoutput_ = config_body.at("enoutput"); + if (config_body.contains("input")) { + if (config_body["input"].is_string()) { + inputs_.push_back(config_body["input"].get()); + } else if (config_body["input"].is_array()) { + for (auto _in : config_body["input"]) { + inputs_.push_back(_in.get()); + } + } + } + } catch (...) { + return true; + } + enstream_ = (response_format_.find("stream") != std::string::npos); + return false; + } + + int load_model(const nlohmann::json &config_body) { + if (parse_config(config_body)) { + return -1; + } + return 0; + } + + void inference(const std::string &msg) { + std::cout << msg << std::endl; + if (out_callback_) out_callback_(std::string("hello"), true); + } + + llm_task(const std::string &workid) { + } + + ~llm_task() { + } +}; + +class llm_llm : public StackFlow { + private: + int task_count_; + std::unordered_map> llm_task_; + + public: + llm_llm() : StackFlow("test") { + task_count_ = 1; + } + + void task_output(const std::shared_ptr llm_task_obj, const std::shared_ptr llm_channel, + const std::string &data, bool finish) { + if (llm_channel->enstream_) { + static int count = 0; + nlohmann::json data_body; + data_body["index"] = count++; + data_body["delta"] = data; + if (!finish) + data_body["delta"] = data; + else + data_body["delta"] = std::string(""); + data_body["finish"] = finish; + if (finish) count = 0; + llm_channel->send(llm_task_obj->response_format_, data_body, LLM_NO_ERROR); + } else if (finish) { + llm_channel->send(llm_task_obj->response_format_, data, LLM_NO_ERROR); + } + } + + void task_user_data(const std::shared_ptr llm_task_obj, + const std::shared_ptr llm_channel, const std::string &object, + const std::string &data) { + const std::string *next_data = &data; + int ret; + std::string tmp_msg1; + if (object.find("stream") != std::string::npos) { + static std::unordered_map stream_buff; + if (decode_stream(data, tmp_msg1, stream_buff)) return; + next_data = &tmp_msg1; + } + std::string tmp_msg2; + if (object.find("base64") != std::string::npos) { + ret = decode_base64((*next_data), tmp_msg2); + if (!ret) { + return; + } + next_data = &tmp_msg2; + } + llm_task_obj->inference((*next_data)); + } + + int setup(const std::string &work_id, const std::string &object, const std::string &data) override { + nlohmann::json error_body; + if ((llm_task_channel_.size() - 1) == task_count_) { + error_body["code"] = -21; + error_body["message"] = "task full"; + send("None", "None", error_body, unit_name_); + return -1; + } + int work_id_num = sample_get_work_id_num(work_id); + auto llm_channel = get_channel(work_id); + auto llm_task_obj = std::make_shared(work_id); + nlohmann::json config_body; + try { + config_body = nlohmann::json::parse(data); + } catch (...) { + error_body["code"] = -2; + error_body["message"] = "json format error."; + send("None", "None", error_body, unit_name_); + return -2; + } + int ret = llm_task_obj->load_model(config_body); + if (ret == 0) { + llm_channel->set_output(llm_task_obj->enoutput_); + llm_channel->set_stream(llm_task_obj->enstream_); + llm_task_obj->set_output(std::bind(&llm_llm::task_output, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + llm_channel->subscriber_work_id("", std::bind(&llm_llm::task_user_data, this, llm_task_obj, llm_channel, + std::placeholders::_1, std::placeholders::_2)); + llm_task_[work_id_num] = llm_task_obj; + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } else { + error_body["code"] = -5; + error_body["message"] = "Model loading failed."; + send("None", "None", error_body, unit_name_); + return -1; + } + } + + void taskinfo(const std::string &work_id, const std::string &object, const std::string &data) override { + nlohmann::json req_body; + int work_id_num = sample_get_work_id_num(work_id); + if (WORK_ID_NONE == work_id_num) { + std::vector task_list; + std::transform(llm_task_channel_.begin(), llm_task_channel_.end(), std::back_inserter(task_list), + [](const auto task_channel) { return task_channel.second->work_id_; }); + req_body = task_list; + send("llm.tasklist", req_body, LLM_NO_ERROR, work_id); + } else { + if (llm_task_.find(work_id_num) == llm_task_.end()) { + req_body["code"] = -6; + req_body["message"] = "Unit Does Not Exist"; + send("None", "None", req_body, work_id); + return; + } + auto llm_task_obj = llm_task_[work_id_num]; + req_body["model"] = llm_task_obj->model_; + req_body["response_format"] = llm_task_obj->response_format_; + req_body["enoutput"] = llm_task_obj->enoutput_; + req_body["inputs_"] = llm_task_obj->inputs_; + send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id); + } + } + + int exit(const std::string &work_id, const std::string &object, const std::string &data) override { + nlohmann::json error_body; + int work_id_num = sample_get_work_id_num(work_id); + if (llm_task_.find(work_id_num) == llm_task_.end()) { + error_body["code"] = -6; + error_body["message"] = "Unit Does Not Exist"; + send("None", "None", error_body, work_id); + return -1; + } + auto llm_channel = get_channel(work_id_num); + llm_channel->stop_subscriber(""); + llm_task_.erase(work_id_num); + send("None", "None", LLM_NO_ERROR, work_id); + return 0; + } + + ~llm_llm() { + while (1) { + auto iteam = llm_task_.begin(); + if (iteam == llm_task_.end()) { + break; + } + iteam->second.reset(); + llm_task_.erase(iteam->first); + } + } +}; + +int main(int argc, char *argv[]) { + signal(SIGTERM, __sigint); + signal(SIGINT, __sigint); + mkdir("/tmp/llm", 0777); + llm_llm llm; + while (!main_exit_flage) { + sleep(1); + } + llm.llm_firework_exit(); + return 0; +} +``` + +## StackFlowUtil +提供一个简便使用的 API： +- sample_json_str_get: 简单的读取 json 内的键值函数，用于在不解析 json 对象的情况下快速读取 json 键值。 +- sample_get_work_id_num： 从 work_id 字符串中读取数字索引。 +- sample_get_work_id_name： 从 work_id 字符串中读取单元名。 +- sample_get_work_id： 用于合成 work_id 字符串。 +- sample_escapeString：简单的对字符串中的转义字符进行编码。 +- sample_unescapeString：简单的对字符串中的转义字符串进行解码。 +- decode_stream：解析流式数据流。 +- decode_base64：解码 base64 +- encode_base64：编码 base64 \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_asr_en.md b/doc/projects_llm_framework_doc/llm_asr_en.md new file mode 100644 index 0000000..a7fc7a1 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_asr_en.md @@ -0,0 +1,297 @@ +# llm-asr +A speech-to-text module that provides speech-to-text services. It supports both Chinese and English models for converting speech to text in these languages. + +## setup +Configure the unit. + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": "sys.pcm", + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence": 2.4, + "endpoint_config.rule2.min_trailing_silence": 1.2, + "endpoint_config.rule3.min_trailing_silence": 30.1, + "endpoint_config.rule1.must_contain_nonsilence": true, + "endpoint_config.rule2.must_contain_nonsilence": true, + "endpoint_config.rule3.must_contain_nonsilence": true + } +} +``` +- request_id: Reference basic data explanation. +- work_id: For configuration units, it is `asr`. +- action: The method to be called is `setup`. +- object: The type of data being transmitted is `asr.setup`. +- model: The model used is the Chinese model `sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23`. +- response_format: The result format is `asr.utf-8.stream`, a UTF-8 stream output. +- input: The input is `sys.pcm`, representing system audio. +- enoutput: Whether to enable user result output. +- endpoint_config.rule1.min_trailing_silence: A speech break occurs 2.4 seconds after wake-up. +- endpoint_config.rule2.min_trailing_silence: A speech break occurs 1.2 seconds after a speech is recognized. +- endpoint_config.rule3.min_trailing_silence: A speech break occurs after a maximum of 30.1 seconds of recognition. + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "asr.1001" +} +``` +- created: Message creation time, Unix time. +- work_id: The successfully created work_id unit. + +## link +Link the output of the upstream unit. + +Send JSON: +```json +{ + "request_id": "3", + "work_id": "asr.1001", + "action": "link", + "object": "work_id", + "data": "kws.1000" +} +``` + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "asr.1001" +} +``` +error::code of 0 indicates successful execution. + +Link the asr and kws units so that when kws sends wake-up data, the asr unit starts recognizing the user's speech, pauses automatically after recognition, and waits until the next wake-up. + +> **Ensure that kws is already configured and working when linking. Linking can also be done during the setup phase.** + +Example: + +```json +{ + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": ["sys.pcm", "kws.1000"], + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence": 2.4, + "endpoint_config.rule2.min_trailing_silence": 1.2, + "endpoint_config.rule3.min_trailing_silence": 30.1, + "endpoint_config.rule1.must_contain_nonsilence": false, + "endpoint_config.rule2.must_contain_nonsilence": false, + "endpoint_config.rule3.must_contain_nonsilence": false + } +} +``` + +## unlink +Unlink + +Send JSON: +```json +{ + "request_id": "4", + "work_id": "asr.1001", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" +} +``` + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "asr.1001" +} +``` +error::code of 0 indicates successful execution. + +## pause +Pause the unit's work. + +Send JSON: +```json +{ + "request_id": "5", + "work_id": "asr.1001", + "action": "pause" +} +``` + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "asr.1001" +} +``` +error::code of 0 indicates successful execution. + +## work +Resume the unit's work. + +Send JSON: +```json +{ + "request_id": "6", + "work_id": "asr.1001", + "action": "work" +} +``` + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "asr.1001" +} +``` +error::code of 0 indicates successful execution. + +## exit +Exit the unit's work. + +Send JSON: +```json +{ + "request_id": "7", + "work_id": "asr.1001", + "action": "exit" +} +``` + +Response JSON: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "asr.1001" +} +``` +error::code of 0 indicates successful execution. + +## Task Information + +Get task list: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "asr", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731580350, + "data":[ + "asr.1001" + ], + "error":{ + "code":0, + "message":"" + }, + "object":"asr.tasklist", + "request_id":"2", + "work_id":"asr" +} +``` + +Get task runtime parameters: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "asr.1001", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731579679, + "data":{ + "enoutput":false, + "inputs_":[ + "sys.pcm" + ], + "model":"sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format":"asr.utf-8-stream" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"asr.taskinfo", + "request_id":"2", + "work_id":"asr.1001" +} +``` + +> **Note: The work_id increases according to the initialization registration order of the unit and is not a fixed index value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_asr_zh.md b/doc/projects_llm_framework_doc/llm_asr_zh.md new file mode 100644 index 0000000..5c19d04 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_asr_zh.md @@ -0,0 +1,300 @@ +# llm-asr +语音转文字单元，用于提供语音转文字服务，可选择中英文模型，用于提供中英文语音转文字服务。 + +## setup +配置单元工作. + +发送 json： +```json +{ + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": "sys.pcm", + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence":2.4, + "endpoint_config.rule2.min_trailing_silence":1.2, + "endpoint_config.rule3.min_trailing_silence":30.1, + "endpoint_config.rule1.must_contain_nonsilence":true, + "endpoint_config.rule2.must_contain_nonsilence":true, + "endpoint_config.rule3.must_contain_nonsilence":true + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `asr`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `asr.setup`。 +- model：使用的模型为 `sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23` 中文模型。 +- response_format：返回结果为 `asr.utf-8.stream`, utf-8 的流式输出。 +- input：输入的为 `sys.pcm`,代表的是系统音频。 +- enoutput：是否起用用户结果输出。 +- endpoint_config.rule1.min_trailing_silence：唤醒后 2.4 s 后语音产生断点。 +- endpoint_config.rule2.min_trailing_silence：识别语音后停顿 1.2 s 后语音产生断点。 +- endpoint_config.rule3.min_trailing_silence：最长能够识别 30.1 s 后语音产生断点。 +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"asr.1001" +} +``` + +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## link +链接上级单元的输出。 + +发送 json： +```json +{ + "request_id": "3", + "work_id": "asr.1001", + "action": "link", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"asr.1001" +} +``` +error::code 为 0 表示执行成功。 + +将 asr 和 kws 单元链接起来，当 kws 发出唤醒数据时，asr 单元开始识别用户的语音，识别完成后自动暂停，直到下一次的唤醒。 + +> **link时必须保证 kws 此时已经配置好进入工作状态。link 也可以在 setup 阶段进行。** + +示例: + +```json +{ + "request_id": "2", + "work_id": "asr", + "action": "setup", + "object": "asr.setup", + "data": { + "model": "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format": "asr.utf-8.stream", + "input": ["sys.pcm","kws.1000"], + "enoutput": true, + "endpoint_config.rule1.min_trailing_silence":2.4, + "endpoint_config.rule2.min_trailing_silence":1.2, + "endpoint_config.rule3.min_trailing_silence":30.1, + "endpoint_config.rule1.must_contain_nonsilence":false, + "endpoint_config.rule2.must_contain_nonsilence":false, + "endpoint_config.rule3.must_contain_nonsilence":false + } +} +``` + +## unlink +取消链接 + +发送 json： +```json +{ + "request_id": "4", + "work_id": "asr.1001", + "action": "unlink", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"asr.1001" +} +``` + +error::code 为 0 表示执行成功。 + +## pause +暂停单元工作。 + +发送 json： +```json +{ + "request_id": "5", + "work_id": "asr.1001", + "action": "pause", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"asr.1001" +} +``` +error::code 为 0 表示执行成功。 + +## work +恢复单元工作. + +发送 json： +```json +{ + "request_id": "6", + "work_id": "asr.1001", + "action": "work", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"6", + "work_id":"asr.1001" +} +``` +error::code 为 0 表示执行成功。 + +## exit +恢复单元工作. + +发送 json： +```json +{ + "request_id": "7", + "work_id": "asr.1001", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"asr.1001" +} +``` +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表： + +发送 json： +```json +{ + "request_id": "2", + "work_id": "asr", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731580350, + "data":[ + "asr.1001" + ], + "error":{ + "code":0, + "message":"" + }, + "object":"asr.tasklist", + "request_id":"2", + "work_id":"asr" +} +``` + +获取任务运行参数： + +```json +{ + "request_id": "2", + "work_id": "asr.1001", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731579679, + "data":{ + "enoutput":false, + "inputs_":[ + "sys.pcm" + ], + "model":"sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23", + "response_format":"asr.utf-8-stream" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"asr.taskinfo", + "request_id":"2", + "work_id":"asr.1001" +} +``` + + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_audio_en.md b/doc/projects_llm_framework_doc/llm_audio_en.md new file mode 100644 index 0000000..c11521e --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_audio_en.md @@ -0,0 +1,13 @@ +# llm-audio +System audio unit for providing system audio playback and recording. + +## API +- play: Play audio. This call will interrupt any previously unfinished audio playback. +- queue_play: Queue playback. Add audio to the playback queue. +- play_stop: Stop playback. +- queue_play_stop: Clear the playback queue. +- audio_status: Get the current playback status. +- cap: Start a recording task, can be called repeatedly. +- cap_stop: Stop a recording task, can be called repeatedly. The last call will stop the data output of the recording channel. +- cap_stop_all: Force stop all ongoing recording tasks. +- setup: Configure the working parameters of the recording unit. \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_audio_zh.md b/doc/projects_llm_framework_doc/llm_audio_zh.md new file mode 100644 index 0000000..24ea293 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_audio_zh.md @@ -0,0 +1,16 @@ +# llm-audio +系统音频单元，用于提供系统音频播放和录音。 + + +## API +- play: 播放音频，本次调用会打断上次未播放的完的音频。 +- queue_play：队列播放，将音频加入播放队列。 +- play_stop：停止播放。 +- queue_play_stop：清理播放队列。 +- audio_status：当前播放状态。 +- cap：开始一个录音任务，可重复调用。 +- cap_stop：停止一个录音任务，可重复调用，最后一个调用会停止录音信道的数据输出。 +- cap_stop_all：强制停止所有开启的录音任务。 +- setup: 配置录音单元的工作参数。 + + diff --git a/doc/projects_llm_framework_doc/llm_kws_en.md b/doc/projects_llm_framework_doc/llm_kws_en.md new file mode 100644 index 0000000..ebb19ce --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_kws_en.md @@ -0,0 +1,199 @@ +# llm-kws +Voice wake-up unit, used to provide voice wake-up services. You can choose between Chinese and English models to provide wake-up services in either language. + +## setup +Configure the unit to work. + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "kws", + "action": "setup", + "object": "kws.setup", + "data": { + "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format": "kws.bool", + "input": "sys.pcm", + "enoutput": true, + "kws": "你好你好" + } +} +``` +- request_id: Refer to basic data explanation. +- work_id: When configuring the unit, it is `kws`. +- action: The method called is `setup`. +- object: The type of data being transmitted is `kws.setup`. +- model: The model used is the Chinese model `sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01`. +- response_format: The result returned is in `kws.bool` format. +- input: The input is `sys.pcm`, representing system audio. +- enoutput: Whether to enable user result output. +- kws: The Chinese wake-up word is `"你好你好"`. + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"kws.1000" +} +``` +- created: Message creation time in Unix time. +- work_id: The successfully created work_id unit returned. + +## pause +Pause the unit's work. + +Send JSON: +```json +{ + "request_id": "3", + "work_id": "kws.1000", + "action": "pause", +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"kws.1000" +} +``` +error::code of 0 indicates successful execution. + +## work +Resume the unit's work. + +Send JSON: +```json +{ + "request_id": "4", + "work_id": "kws.1000", + "action": "work", +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"kws.1000" +} +``` +error::code of 0 indicates successful execution. + +## exit +Resume the unit's work. + +Send JSON: +```json +{ + "request_id": "5", + "work_id": "kws.1000", + "action": "exit", +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"kws.1000" +} +``` +error::code of 0 indicates successful execution. + +## Task Information + +Get task list: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "kws", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731580350, + "data":[ + "kws.1000" + ], + "error":{ + "code":0, + "message":"" + }, + "object":"kws.tasklist", + "request_id":"2", + "work_id":"kws" +} +``` + +Get task runtime parameters: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "kws.1000", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731652086, + "data":{ + "enoutput":true, + "inputs_":["sys.pcm"], + "model":"sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format":"kws.bool" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"kws.taskinfo", + "request_id":"2", + "work_id":"kws.1000" +} +``` + +> **Note: work_id increases according to the initialization registration order of the unit and is not a fixed index value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_kws_zh.md b/doc/projects_llm_framework_doc/llm_kws_zh.md new file mode 100644 index 0000000..9ebb728 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_kws_zh.md @@ -0,0 +1,205 @@ +# llm-kws +语音唤醒单元，用于提供语音唤醒服务，可选择中英文模型，用于提供中英文唤醒服务。 + +## setup +配置单元工作. + +发送 json： +```json +{ + "request_id": "2", + "work_id": "kws", + "action": "setup", + "object": "kws.setup", + "data": { + "model": "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format": "kws.bool", + "input": "sys.pcm", + "enoutput": true, + "kws": "你好你好" + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `kws`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `kws.setup`。 +- model：使用的模型为 `sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01` 中文模型。 +- response_format：返回结果为 `kws.bool` 型。 +- input：输入的为 `sys.pcm`,代表的是系统音频。 +- enoutput：是否起用用户结果输出。 +- kws：中文唤醒词为 `"你好你好"`。 + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"kws.1000" +} +``` + +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## pause +暂停单元工作. + +发送 json： +```json +{ + "request_id": "3", + "work_id": "kws.1000", + "action": "pause", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"kws.1000" +} +``` + +error::code 为 0 表示执行成功。 + +## work +恢复单元工作. + +发送 json： +```json +{ + "request_id": "4", + "work_id": "kws.1000", + "action": "work", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"kws.1000" +} +``` + +error::code 为 0 表示执行成功。 + +## exit +恢复单元工作. + +发送 json： +```json +{ + "request_id": "5", + "work_id": "kws.1000", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"kws.1000" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表： + +发送 json： +```json +{ + "request_id": "2", + "work_id": "kws", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731580350, + "data":[ + "kws.1000" + ], + "error":{ + "code":0, + "message":"" + }, + "object":"kws.tasklist", + "request_id":"2", + "work_id":"kws" +} +``` + +获取任务运行参数： + +```json +{ + "request_id": "2", + "work_id": "kws.1000", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652086, + "data":{ + "enoutput":true, + "inputs_":["sys.pcm"], + "model":"sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01", + "response_format":"kws.bool" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"kws.taskinfo", + "request_id":"2", + "work_id":"kws.1000" +} +``` + + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** diff --git a/doc/projects_llm_framework_doc/llm_llm_en.md b/doc/projects_llm_framework_doc/llm_llm_en.md new file mode 100644 index 0000000..4070d3e --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_llm_en.md @@ -0,0 +1,284 @@ +# llm-llm +Large Model Unit, used to provide large model inference services. + +## setup +Configure the unit. + +Send json: +```json +{ + "request_id": "2", + "work_id": "llm", + "action": "setup", + "object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": "llm.utf-8", + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` +- request_id: Refer to the basic data explanation. +- work_id: For configuration unit, it is `llm`. +- action: The method called is `setup`. +- object: The type of data transmitted is `llm.setup`. +- model: The model used is the Chinese model `qwen2.5-0.5B-prefill-20e`. +- response_format: The result returned is in `llm.utf-8.stream`, utf-8 streaming output. +- input: The input is `llm.utf-8`, representing input from the user. +- enoutput: Whether to enable user result output. +- max_token_len: Maximum output token, this value is limited by the model's maximum limit. +- prompt: The prompt for the model. + +Response json: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "2", + "work_id": "llm.1002" +} +``` +- created: Message creation time, unix time. +- work_id: The successfully created work_id unit. + +## link +Link the output of the upper unit. + +Send json: +```json +{ + "request_id": "3", + "work_id": "llm.1002", + "action": "link", + "object": "work_id", + "data": "kws.1000" +} +``` + +Response json: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "3", + "work_id": "llm.1002" +} +``` +error::code of 0 indicates successful execution. + +Link the llm and kws units so that when kws issues wake-up data, the llm unit stops the previous unfinished inference for repeated wake-up functionality. + +> **When linking, ensure that kws has been configured and is in working status. Linking can also be done during the setup phase.** + +Example: + +```json +{ + "request_id": "2", + "work_id": "llm", + "action": "setup", + "object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": ["llm.utf-8", "asr.1001", "kws.1000"], + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` + +## unlink +Unlink + +Send json: +```json +{ + "request_id": "4", + "work_id": "llm.1002", + "action": "unlink", + "object": "work_id", + "data": "kws.1000" +} +``` + +Response json: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "4", + "work_id": "llm.1002" +} +``` +error::code of 0 indicates successful execution. + +## pause +Pause the unit. + +Send json: +```json +{ + "request_id": "5", + "work_id": "llm.1002", + "action": "pause" +} +``` + +Response json: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "5", + "work_id": "llm.1002" +} +``` +error::code of 0 indicates successful execution. + +## work +Resume the unit. + +Send json: +```json +{ + "request_id": "6", + "work_id": "llm.1002", + "action": "work" +} +``` + +Response json: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "6", + "work_id": "llm.1002" +} +``` +error::code of 0 indicates successful execution. + +## exit +Exit the unit. + +Send json: +```json +{ + "request_id": "7", + "work_id": "llm.1002", + "action": "exit" +} +``` + +Response json: + +```json +{ + "created": 1731488402, + "data": "None", + "error": { + "code": 0, + "message": "" + }, + "object": "None", + "request_id": "7", + "work_id": "llm.1002" +} +``` +error::code of 0 indicates successful execution. + +## Task Information + +Get task list: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "llm", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731652149, + "data":["llm.1002"], + "error":{ + "code":0, + "message":"" + }, + "object":"llm.tasklist", + "request_id":"2", + "work_id":"llm" +} +``` + +Get task runtime parameters: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "llm.1002", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731652187, + "data":{ + "enoutput":true, + "inputs_":["llm.utf-8"], + "model":"qwen2.5-0.5B-prefill-20e", + "response_format":"llm.utf-8.stream" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"llm.taskinfo", + "request_id":"2", + "work_id":"llm.1002" +} +``` + +> **Note: work_id increases according to the initialization registration order of the unit and is not a fixed index value.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_llm_zh.md b/doc/projects_llm_framework_doc/llm_llm_zh.md new file mode 100644 index 0000000..f314034 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_llm_zh.md @@ -0,0 +1,287 @@ +# llm-llm +大模型单元，用于提供大模型推理服务。 + +## setup +配置单元工作. + +发送 json： +```json +{ + "request_id": "2", + "work_id": "llm", + "action": "setup","object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": "llm.utf-8", + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `llm`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `llm.setup`。 +- model：使用的模型为 `qwen2.5-0.5B-prefill-20e` 中文模型。 +- response_format：返回结果为 `llm.utf-8.stream`, utf-8 的流式输出。 +- input：输入的为 `llm.utf-8`,代表的是从用户输入。 +- enoutput：是否起用用户结果输出。 +- max_token_len：最大输出 token,该值的最大值受到模型的最大限制。 +- prompt：模型的提示词。 +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"llm.1002" +} +``` + +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## link +链接上级单元的输出。 + +发送 json： +```json +{ + "request_id": "3", + "work_id": "llm.1002", + "action": "link", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"llm.1002" +} +``` +error::code 为 0 表示执行成功。 + +将 llm 和 kws 单元链接起来，当 kws 发出唤醒数据时，llm 单元停止上次未完成的推理，用于重复唤醒功能。 + +> **link时必须保证 kws 此时已经配置好进入工作状态。link 也可以在 setup 阶段进行。** + +示例: + +```json +{ + "request_id": "2", + "work_id": "llm", + "action": "setup","object": "llm.setup", + "data": { + "model": "qwen2.5-0.5B-prefill-20e", + "response_format": "llm.utf-8.stream", + "input": ["llm.utf-8", "asr.1001", "kws.1000"], + "enoutput": true, + "max_token_len": 256, + "prompt": "You are a knowledgeable assistant capable of answering various questions and providing information." + } +} +``` + +## unlink +取消链接 + +发送 json： +```json +{ + "request_id": "4", + "work_id": "llm.1002", + "action": "unlink", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"llm.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## pause +暂停单元工作. + +发送 json： +```json +{ + "request_id": "5", + "work_id": "llm.1002", + "action": "pause", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"llm.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## work +恢复单元工作. + +发送 json： +```json +{ + "request_id": "6", + "work_id": "llm.1002", + "action": "work", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"6", + "work_id":"llm.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## exit +恢复单元工作. + +发送 json： +```json +{ + "request_id": "7", + "work_id": "llm.1002", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"llm.1002" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表： + +发送 json： +```json +{ + "request_id": "2", + "work_id": "llm", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652149, + "data":["llm.1002"], + "error":{ + "code":0, + "message":"" + }, + "object":"llm.tasklist", + "request_id":"2", + "work_id":"llm" +} +``` + +获取任务运行参数： + +```json +{ + "request_id": "2", + "work_id": "llm.1002", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652187, + "data":{ + "enoutput":true, + "inputs_":["llm.utf-8"], + "model":"qwen2.5-0.5B-prefill-20e", + "response_format":"llm.utf-8.stream" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"llm.taskinfo", + "request_id":"2", + "work_id":"llm.1002" +} +``` + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_melotts_en.md b/doc/projects_llm_framework_doc/llm_melotts_en.md new file mode 100644 index 0000000..7b75a1f --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_melotts_en.md @@ -0,0 +1,272 @@ +# llm-melotts +Text-to-speech unit accelerated by NPU, used to provide text-to-speech services. It supports both Chinese and English models for text-to-speech conversion. + +## setup +Configure the unit. + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } +} +``` +- request_id: Refer to the basic data explanation. +- work_id: For configuration, it is `melotts`. +- action: The method to be called is `setup`. +- object: The data type being transmitted is `melotts.setup`. +- model: The model being used is the Chinese model `melotts_zh-cn`. +- response_format: The result is returned as `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. +- input: The input is `tts.utf-8`, representing user input. +- enoutput: Whether to enable user result output. + +Response JSON: +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"melotts.1003" +} +``` +- created: Message creation time, UNIX time. +- work_id: Successfully created work_id unit. + +## link +Link the output of the upper-level unit. + +Send JSON: +```json +{ + "request_id": "3", + "work_id": "melotts.1003", + "action": "link", + "object":"work_id", + "data":"kws.1000" +} +``` + +Response JSON: +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"melotts.1003" +} +``` +error::code being 0 indicates success. + +Link the llm and melotts units. When the kws melotts unit stops the unfinished inference from the last time, it is used for repeated wake-up functionality. + +> **Ensure that kws is already configured and in working status during link. Link can also be performed during the setup stage.** + +Example: +```json +{ + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": ["tts.utf-8", "llm.1002", "kws.1000"], + "enoutput": false + } +} +``` + +## unlink +Unlink + +Send JSON: +```json +{ + "request_id": "4", + "work_id": "melotts.1003", + "action": "unlink", + "object":"work_id", + "data":"kws.1000" +} +``` + +Response JSON: +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"melotts.1003" +} +``` +error::code being 0 indicates success. + +## pause +Pause the unit. + +Send JSON: +```json +{ + "request_id": "5", + "work_id": "llm.1003", + "action": "pause", +} +``` + +Response JSON: +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"llm.1003" +} +``` +error::code being 0 indicates success. + +## work +Resume the unit. + +Send JSON: +```json +{ + "request_id": "6", + "work_id": "llm.1003", + "action": "work", +} +``` + +Response JSON: +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"6", + "work_id":"llm.1003" +} +``` +error::code being 0 indicates success. + +## exit +Exit the unit. + +Send JSON: +```json +{ + "request_id": "7", + "work_id": "llm.1003", + "action": "exit", +} +``` + +Response JSON: +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"llm.1003" +} +``` +error::code being 0 indicates success. + +## Task Information + +Get task list: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "melotts", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731652311, + "data":["melotts.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"melotts.tasklist", + "request_id":"2", + "work_id":"melotts" +} +``` + +Get task runtime parameters: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "melotts.1003", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "inputs_":["tts.utf-8"], + "model":"melotts_zh-cn", + "response_format":"sys.pcm" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"melotts.taskinfo", + "request_id":"2", + "work_id":"melotts.1003" +} +``` + +> **Note: work_id increases in the order of the unit's initialization registration and is not a fixed index value.** +> **The same type of unit cannot configure multiple units to work simultaneously, or unknown errors may occur. For example, tts and melo tts cannot be activated to work at the same time.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_melotts_zh.md b/doc/projects_llm_framework_doc/llm_melotts_zh.md new file mode 100644 index 0000000..e64fe17 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_melotts_zh.md @@ -0,0 +1,282 @@ +# llm-melotts +使用 npu 加速的文字转语音单元，用于提供文字转语音服务，可选择中英文模型，用于提供中英文文字转语音服务。 + +## setup +配置单元工作. + +发送 json： +```json +{ + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `melotts`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `melotts.setup`。 +- model：使用的模型为 `melotts_zh-cn` 中文模型。 +- response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。 +- input：输入的为 `tts.utf-8`,代表的是从用户输入。 +- enoutput：是否起用用户结果输出。 +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"melotts.1003" +} +``` +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## link +链接上级单元的输出。 + +发送 json： +```json +{ + "request_id": "3", + "work_id": "melotts.1003", + "action": "link", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"melotts.1003" +} +``` +error::code 为 0 表示执行成功。 + +将 llm 和 melotts 单元链接起来，当 kws melotts 单元停止上次未完成的推理，用于重复唤醒功能。 + +> **link时必须保证 kws 此时已经配置好进入工作状态。link 也可以在 setup 阶段进行。** + +示例: + +```json +{ + "request_id": "2", + "work_id": "melotts", + "action": "setup", + "object": "melotts.setup", + "data": { + "model": "melotts_zh-cn", + "response_format": "sys.pcm", + "input": ["tts.utf-8", "llm.1002", "kws.1000"], + "enoutput": false + } +} +``` + +## unlink +取消链接 + +发送 json： +```json +{ + "request_id": "4", + "work_id": "melotts.1003", + "action": "unlink", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"melotts.1003" +} +``` +error::code 为 0 表示执行成功。 + +## pause +暂停单元工作. + +发送 json： +```json +{ + "request_id": "5", + "work_id": "melotts.1003", + "action": "pause", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"melotts.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## work +恢复单元工作. + +发送 json： +```json +{ + "request_id": "6", + "work_id": "melotts.1003", + "action": "work", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"6", + "work_id":"melotts.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## exit +恢复单元工作. + +发送 json： +```json +{ + "request_id": "7", + "work_id": "melotts.1003", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"melotts.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表： + +发送 json： +```json +{ + "request_id": "2", + "work_id": "melotts", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652311, + "data":["melotts.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"melotts.tasklist", + "request_id":"2", + "work_id":"melotts" +} +``` + +获取任务运行参数： + +```json +{ + "request_id": "2", + "work_id": "melotts.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "inputs_":["tts.utf-8"], + "model":"melotts_zh-cn", + "response_format":"sys.pcm" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"melotts.taskinfo", + "request_id":"2", + "work_id":"melotts.1003" +} +``` + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** +> **同类型单元不能配置多个单元同时工作，否则会产生未知错误。例如 tts 和 melo tts 不能同时拍起用工作。** diff --git a/doc/projects_llm_framework_doc/llm_sys_en.md b/doc/projects_llm_framework_doc/llm_sys_en.md new file mode 100644 index 0000000..40e6ed9 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_sys_en.md @@ -0,0 +1,20 @@ +# llm-sys + +The basic service unit of StackFlow, providing external serial and TCP channels and some system function services, while internally handling port resource allocation and a simple in-memory database. + +## External API +- sys.ping: Test if communication with the LLM is possible. +- sys.lsmode: Models that have existed in the system in the past. +- sys.bashexec: Execute bash commands. +- sys.hwinfo: Retrieve onboard CPU, memory, and temperature parameters of the LLM. +- sys.uartsetup: Set serial port parameters, effective for a single session. +- sys.reset: Reset the entire LLM framework application. +- sys.reboot: Restart the system. +- sys.version: Get the version of the LLM framework program. + +## Internal API +- sql_select: Query key-value pairs in the small in-memory KV database. +- register_unit: Register a unit. +- release_unit: Release a unit. +- sql_set: Set key-value pairs in the small in-memory KV database. +- sql_unset: Delete key-value pairs in the small in-memory KV database. \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_sys_zh.md b/doc/projects_llm_framework_doc/llm_sys_zh.md new file mode 100644 index 0000000..1869cb2 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_sys_zh.md @@ -0,0 +1,20 @@ +# llm-sys +StackFlow 的基本服务单元，对外提供串口和 TCP 外部信道和一部分系统功能服务，对内进行端口资源分配，和一个简单的内存数据库。 + +## 外部 API +- sys.ping：测试是否能够和 LLM 进行通信。 +- sys.lsmode：过去系统中存在的模型。 +- sys.bashexec：执行 bash 命令。 +- sys.hwinfo：获取 LLM 板载 cpu、内存、温度参数。 +- sys.uartsetup：设置串口参数，单次生效。 +- sys.reset：复位整个 LLM 框架的应用程序。 +- sys.reboot：重启系统。 +- sys.version：获取 LLM 框架程序版本。 + +## 内部 API： +- sql_select：查寻小型内存 KV 数据库键值。 +- register_unit：注册单元。 +- release_unit：释放单元。 +- sql_set：设定小型内存 KV 数据库键值。 +- sql_unset：删除小型内存 KV 数据库键值。 + diff --git a/doc/projects_llm_framework_doc/llm_tts_en.md b/doc/projects_llm_framework_doc/llm_tts_en.md new file mode 100644 index 0000000..95418b3 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_tts_en.md @@ -0,0 +1,279 @@ +# llm-tts +Text-to-Speech unit, used to provide text-to-speech services, with options for Chinese and English models to provide text-to-speech services in both languages. + +## setup +Configure the unit. + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } +} +``` +- request_id: Refer to the basic data explanation. +- work_id: For configuring the unit, it is `tts`. +- action: The method to call is `setup`. +- object: The type of data being transmitted is `tts.setup`. +- model: The model used is the `single_speaker_fast` Chinese model. +- response_format: The returned result is `sys.pcm`, system audio data, which is directly sent to the llm-audio module for playback. +- input: Input is `tts.utf-8`, representing user input. +- enoutput: Whether to enable user result output. +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"llm.1003" +} +``` + +- created: Message creation time, in Unix time. +- work_id: The successfully created work_id unit. + +## link +Link the output of the upper unit. + +Send JSON: +```json +{ + "request_id": "3", + "work_id": "tts.1003", + "action": "link", + "object":"work_id", + "data":"kws.1000" +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"tts.1003" +} +``` +error::code 0 indicates successful execution. + +Link the llm and tts units, so that when kws wakes up, the tts unit stops the previous unfinished inference, used for repeat wake-up functionality. + +> **When linking, ensure that kws is already configured and in working state. Linking can also be done during the setup phase.** + +Example: + +```json +{ + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": ["tts.utf-8", "llm.1002", "kws.1000"], + "enoutput": false + } +} +``` + +## unlink +Unlink + +Send JSON: +```json +{ + "request_id": "4", + "work_id": "tts.1003", + "action": "unlink", + "object":"work_id", + "data":"kws.1000" +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"tts.1003" +} +``` +error::code 0 indicates successful execution. + +## pause +Pause unit work. + +Send JSON: +```json +{ + "request_id": "5", + "work_id": "llm.1003", + "action": "pause", +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"llm.1003" +} +``` +error::code 0 indicates successful execution. + +## work +Resume unit work. + +Send JSON: +```json +{ + "request_id": "6", + "work_id": "llm.1003", + "action": "work", +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"6", + "work_id":"llm.1003" +} +``` +error::code 0 indicates successful execution. + +## exit +Exit unit work. + +Send JSON: +```json +{ + "request_id": "7", + "work_id": "llm.1003", + "action": "exit", +} +``` + +Response JSON: + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"llm.1003" +} +``` +error::code 0 indicates successful execution. + +## Task Information + +Get task list: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "tts", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731652311, + "data":["tts.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"tts.tasklist", + "request_id":"2", + "work_id":"tts" +} +``` + +Get task runtime parameters: + +Send JSON: +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "taskinfo" +} +``` + +Response JSON: +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "inputs_":["tts.utf-8"], + "model":"single_speaker_fast", + "response_format":"sys.pcm" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"tts.taskinfo", + "request_id":"2", + "work_id":"tts.1003" +} +``` + +> **Note: work_id increases according to the order of unit initialization registration and is not a fixed index value.** +> **The same type of unit cannot have multiple units working simultaneously, as it may cause unknown errors. For example, tts and melo tts cannot be enabled to work at the same time.** \ No newline at end of file diff --git a/doc/projects_llm_framework_doc/llm_tts_zh.md b/doc/projects_llm_framework_doc/llm_tts_zh.md new file mode 100644 index 0000000..86b1139 --- /dev/null +++ b/doc/projects_llm_framework_doc/llm_tts_zh.md @@ -0,0 +1,282 @@ +# llm-tts +文字转语音单元，用于提供文字转语音服务，可选择中英文模型，用于提供中英文文字转语音服务。 + +## setup +配置单元工作. + +发送 json： +```json +{ + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": "tts.utf-8", + "enoutput": false + } +} +``` +- request_id：参考基本数据解释。 +- work_id：配置单元时，为 `tts`。 +- action：调用的方法为 `setup`。 +- object：传输的数据类型为 `tts.setup`。 +- model：使用的模型为 `single_speaker_fast` 中文模型。 +- response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。 +- input：输入的为 `tts.utf-8`,代表的是从用户输入。 +- enoutput：是否起用用户结果输出。 +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"2", + "work_id":"llm.1003" +} +``` +- created：消息创建时间，unix 时间。 +- work_id：返回成功创建的 work_id 单元。 + +## link +链接上级单元的输出。 + +发送 json： +```json +{ + "request_id": "3", + "work_id": "tts.1003", + "action": "link", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"3", + "work_id":"tts.1003" +} +``` +error::code 为 0 表示执行成功。 + +将 llm 和 tts 单元链接起来，当 kws 唤醒时 tts 单元停止上次未完成的推理，用于重复唤醒功能。 + +> **link时必须保证 kws 此时已经配置好进入工作状态。link 也可以在 setup 阶段进行。** + +示例: + +```json +{ + "request_id": "2", + "work_id": "tts", + "action": "setup", + "object": "tts.setup", + "data": { + "model": "single_speaker_fast", + "response_format": "sys.pcm", + "input": ["tts.utf-8", "llm.1002", "kws.1000"], + "enoutput": false + } +} +``` + +## unlink +取消链接 + +发送 json： +```json +{ + "request_id": "4", + "work_id": "tts.1003", + "action": "unlink", + "object":"work_id", + "data":"kws.1000" +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"4", + "work_id":"tts.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## pause +暂停单元工作. + +发送 json： +```json +{ + "request_id": "5", + "work_id": "tts.1003", + "action": "pause", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"5", + "work_id":"tts.1003" +} +``` +error::code 为 0 表示执行成功。 + +## work +恢复单元工作. + +发送 json： +```json +{ + "request_id": "6", + "work_id": "tts.1003", + "action": "work", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"6", + "work_id":"tts.1003" +} +``` +error::code 为 0 表示执行成功。 + +## exit +恢复单元工作. + +发送 json： +```json +{ + "request_id": "7", + "work_id": "tts.1003", + "action": "exit", +} +``` + +响应 json： + +```json +{ + "created":1731488402, + "data":"None", + "error":{ + "code":0, + "message":"" + }, + "object":"None", + "request_id":"7", + "work_id":"tts.1003" +} +``` + +error::code 为 0 表示执行成功。 + +## taskinfo + +获取任务列表： + +发送 json： +```json +{ + "request_id": "2", + "work_id": "tts", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652311, + "data":["tts.1003"], + "error":{ + "code":0, + "message":"" + }, + "object":"tts.tasklist", + "request_id":"2", + "work_id":"tts" +} +``` + +获取任务运行参数： + +```json +{ + "request_id": "2", + "work_id": "tts.1003", + "action": "taskinfo" +} +``` + +响应 json： + +```json +{ + "created":1731652344, + "data":{ + "enoutput":false, + "inputs_":["tts.utf-8"], + "model":"single_speaker_fast", + "response_format":"sys.pcm" + }, + "error":{ + "code":0, + "message":"" + }, + "object":"tts.taskinfo", + "request_id":"2", + "work_id":"tts.1003" +} +``` + + +> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。** +> **同类型单元不能配置多个单元同时工作，否则会产生未知错误。例如 tts 和 melo tts 不能同时拍起用工作。** \ No newline at end of file diff --git a/ext_components/StackFlow/Kconfig b/ext_components/StackFlow/Kconfig new file mode 100644 index 0000000..8247a07 --- /dev/null +++ b/ext_components/StackFlow/Kconfig @@ -0,0 +1,10 @@ + +menuconfig STACKFLOW_ENABLED + bool "Enable stackflow" + default n + + + + + + diff --git a/ext_components/StackFlow/SConstruct b/ext_components/StackFlow/SConstruct new file mode 100644 index 0000000..c127b60 --- /dev/null +++ b/ext_components/StackFlow/SConstruct @@ -0,0 +1,42 @@ +# component/SConscript +Import('env') +import os +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +if 'CONFIG_STACKFLOW_ENABLED' in os.environ: + SRCS = [] + INCLUDE = [] + PRIVATE_INCLUDE = [] + REQUIREMENTS = [] + STATIC_LIB = [] + DYNAMIC_LIB = [] + DEFINITIONS = [] + DEFINITIONS_PRIVATE = [] + LDFLAGS = [] + LINK_SEARCH_PATH = [] + + SRCS += Glob('stackflow/*.cpp') + INCLUDE.append(ADir("stackflow")) + PRIVATE_INCLUDE.append(ADir("stackflow")) + PRIVATE_INCLUDE.append(ADir("stackflow/libzmq")) + + REQUIREMENTS += ['eventpp', 'utilities', 'zmq'] + + # STATIC_LIB += [ + # AFile("uintv2_ncnn/lib/libncnn.a") + # ] + + env['COMPONENTS'].append({'target':os.path.basename(env['component_dir']), + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'REGISTER':'static' + }) diff --git a/ext_components/StackFlow/stackflow/StackFlow.cpp b/ext_components/StackFlow/stackflow/StackFlow.cpp new file mode 100644 index 0000000..dace7cb --- /dev/null +++ b/ext_components/StackFlow/stackflow/StackFlow.cpp @@ -0,0 +1,512 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlow.h" +#include "sample_log.h" + +using namespace StackFlows; + +std::string llm_channel_obj::uart_push_url; + +llm_channel_obj::llm_channel_obj(const std::string &_publisher_url, const std::string &inference_url, + const std::string &unit_name) + : unit_name_(unit_name), inference_url_(inference_url) { + zmq_url_index_ = -1000; + zmq_[-1] = std::make_unique(_publisher_url, ZMQ_PUB); + zmq_[-2].reset(); +} + +llm_channel_obj::~llm_channel_obj() { +} + +void llm_channel_obj::subscriber_event_call(const std::function &call, + const std::string &raw) { + // SLOGI("object%s", sample_json_str_get(raw, "object").c_str()); + // SLOGI("request_id%s", sample_json_str_get(raw, "request_id").c_str()); + // SLOGI("work_id%s", sample_json_str_get(raw, "work_id").c_str()); + // SLOGI("action%s", sample_json_str_get(raw, "action").c_str()); + // SLOGI("data%s", sample_json_str_get(raw, "data").c_str()); + if (sample_json_str_get(raw, "action") == "inference") { + std::string zmq_com = sample_json_str_get(raw, "zmq_com"); + if (!zmq_com.empty()) set_push_url(zmq_com); + request_id_ = sample_json_str_get(raw, "request_id"); + work_id_ = sample_json_str_get(raw, "work_id"); + } + call(sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); +} + +int llm_channel_obj::subscriber_work_id(const std::string &work_id, + const std::function &call) { + int id_num; + std::string subscriber_url; + std::regex pattern(R"((\w+)\.(\d+))"); + std::smatch matches; + if ((!work_id.empty()) && std::regex_match(work_id, matches, pattern)) { + if (matches.size() == 3) { + // std::string part1 = matches[1].str(); + id_num = std::stoi(matches[2].str()); + std::string input_url_name = work_id + ".out_port"; + std::string input_url; + pzmq _call("sys"); + _call.call_rpc_action("sql_select", input_url_name, [&](const std::string &data) { input_url = data; }); + if (input_url.empty()) { + return -1; + } + subscriber_url = input_url; + } + } else { + id_num = 0; + subscriber_url = inference_url_; + } + zmq_[id_num] = std::make_unique( + subscriber_url, ZMQ_SUB, std::bind(&llm_channel_obj::subscriber_event_call, this, call, std::placeholders::_1)); + return 0; +} + +void llm_channel_obj::stop_subscriber_work_id(const std::string &work_id) { + int id_num; + std::regex pattern(R"((\w+)\.(\d+))"); + std::smatch matches; + if (std::regex_match(work_id, matches, pattern)) { + if (matches.size() == 3) { + // std::string part1 = matches[1].str(); + id_num = std::stoi(matches[2].str()); + } + } else { + id_num = 0; + } + if (zmq_.find(id_num) != zmq_.end()) zmq_.erase(id_num); +} + +void llm_channel_obj::subscriber(const std::string &zmq_url, const std::function &call) { + zmq_url_map_[zmq_url] = zmq_url_index_--; + zmq_[zmq_url_map_[zmq_url]] = std::make_unique(zmq_url, ZMQ_SUB, call); +} + +void llm_channel_obj::stop_subscriber(const std::string &zmq_url) { + if (zmq_url.empty()) { + zmq_.clear(); + zmq_url_map_.clear(); + } else if (zmq_url_map_.find(zmq_url) != zmq_url_map_.end()) { + zmq_.erase(zmq_url_map_[zmq_url]); + zmq_url_map_.erase(zmq_url); + } +} + +int llm_channel_obj::send_raw_to_pub(const std::string &raw) { + return zmq_[-1]->send_data(raw); +} + +int llm_channel_obj::send_raw_to_usr(const std::string &raw) { + if (zmq_[-2]) { + return zmq_[-2]->send_data(raw); + } else { + return -1; + } +} + +void llm_channel_obj::set_push_url(const std::string &url) { + if (output_url_ != url) { + output_url_ = url; + zmq_[-2].reset(new pzmq(output_url_, ZMQ_PUSH)); + } +} + +void llm_channel_obj::cear_push_url() { + zmq_[-2].reset(); +} + +int llm_channel_obj::send_raw_for_url(const std::string &zmq_url, const std::string &raw) { + pzmq _zmq(zmq_url, ZMQ_PUSH); + return _zmq.send_data(raw); +} + +int llm_channel_obj::output_to_uart(const std::string &data) { + return send_raw_for_url(uart_push_url, data); +} + +StackFlow::StackFlow::StackFlow(const std::string &unit_name) + : unit_name_(unit_name), rpc_ctx_(std::make_unique(unit_name)) { + event_queue_.appendListener(EVENT_NONE, + std::bind(&StackFlow::_none_event, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_PAUSE, + std::bind(&StackFlow::_pause, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_WORK, + std::bind(&StackFlow::_work, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_EXIT, + std::bind(&StackFlow::_exit, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_SETUP, + std::bind(&StackFlow::_setup, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_LINK, + std::bind(&StackFlow::_link, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_UNLINK, + std::bind(&StackFlow::_unlink, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_TASKINFO, + std::bind(&StackFlow::_taskinfo, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener(EVENT_SYS_INIT, + std::bind(&StackFlow::_sys_init, this, std::placeholders::_1, std::placeholders::_2)); + event_queue_.appendListener( + EVENT_REPEAT_EVENT, std::bind(&StackFlow::_repeat_loop, this, std::placeholders::_1, std::placeholders::_2)); + rpc_ctx_->register_rpc_action("setup", std::bind(&StackFlow::_rpc_setup, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("pause", std::bind(&StackFlow::_rpc_pause, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("work", std::bind(&StackFlow::_rpc_work, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("exit", std::bind(&StackFlow::_rpc_exit, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("link", std::bind(&StackFlow::_rpc_link, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("unlink", std::bind(&StackFlow::_rpc_unlink, this, std::placeholders::_1)); + rpc_ctx_->register_rpc_action("taskinfo", std::bind(&StackFlow::_rpc_taskinfo, this, std::placeholders::_1)); + + status_.store(0); + exit_flage_.store(false); + even_loop_thread_ = std::make_unique(std::bind(&StackFlow::even_loop, this)); + event_queue_.enqueue(EVENT_SYS_INIT, "", ""); +} + +StackFlow::~StackFlow() { + while (1) { + auto iteam = llm_task_channel_.begin(); + if (iteam == llm_task_channel_.end()) { + break; + } + sys_release_unit(iteam->first, ""); + llm_task_channel_.erase(iteam->first); + } + exit_flage_.store(true); + event_queue_.enqueue(EVENT_NONE, "", ""); + even_loop_thread_->join(); +} + +void StackFlow::even_loop() { + while (!exit_flage_.load()) { + event_queue_.wait(); + event_queue_.process(); + } +} + +void StackFlow::_none_event(const std::string &data1, const std::string &data2) { +} + +void StackFlow::_sys_init(const std::string &zmq_url, const std::string &data) { + // serial_zmq_url + std::string serial_zmq_url; + serial_zmq_url = sys_sql_select("serial_zmq_url"); + if (!serial_zmq_url.empty()) { + SLOGI("serial_zmq_url:%s", serial_zmq_url.c_str()); + llm_channel_obj::uart_push_url = serial_zmq_url; + status_.store(1); + } else { + sleep(1); + event_queue_.enqueue(EVENT_SYS_INIT, "", ""); + } +} + +std::string StackFlow::_rpc_setup(const std::string &data) { + SLOGI("_rpc_setup:%s", data.c_str()); + event_queue_.enqueue(EVENT_SETUP, sample_json_str_get(data, "zmq_com"), + sample_unescapeString(sample_json_str_get(data, "raw_data"))); + return std::string("None"); +} + +int StackFlow::setup(const std::string &zmq_url, const std::string &raw) { + SLOGI("StackFlow::setup raw zmq_url:%s raw:%s", zmq_url.c_str(), raw.c_str()); + int workid_num = sys_register_unit(unit_name_); + std::string work_id = unit_name_ + "." + std::to_string(workid_num); + auto task_channel = get_channel(workid_num); + task_channel->set_push_url(zmq_url); + task_channel->request_id_ = sample_json_str_get(raw, "request_id"); + task_channel->work_id_ = work_id; + if (setup(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data"))) { + sys_release_unit(workid_num, work_id); + } + return 0; +} + +int StackFlow::setup(const std::string &work_id, const std::string &object, const std::string &data) { + SLOGI("StackFlow::setup"); + if (_setup_) { + return _setup_(work_id, object, data); + } + nlohmann::json error_body; + error_body["code"] = -18; + error_body["message"] = "not have unit action!"; + send("None", "None", error_body, work_id); + return -1; +} + +std::string StackFlow::_rpc_link(const std::string &data) { + event_queue_.enqueue(EVENT_LINK, sample_json_str_get(data, "zmq_com"), + sample_unescapeString(sample_json_str_get(data, "raw_data"))); + return std::string("None"); +} + +void StackFlow::link(const std::string &zmq_url, const std::string &raw) { + SLOGI("StackFlow::link raw"); + std::string work_id = sample_json_str_get(raw, "work_id"); + try { + auto task_channel = get_channel(sample_get_work_id_num(work_id)); + task_channel->set_push_url(zmq_url); + } catch (...) { + } + link(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); +} + +void StackFlow::link(const std::string &work_id, const std::string &object, const std::string &data) { + SLOGI("StackFlow::link"); + if (_link_) { + _link_(work_id, object, data); + return; + } + nlohmann::json error_body; + error_body["code"] = -18; + error_body["message"] = "not have unit action!"; + send("None", "None", error_body, work_id); +} + +std::string StackFlow::_rpc_unlink(const std::string &data) { + event_queue_.enqueue(EVENT_UNLINK, sample_json_str_get(data, "zmq_com"), + sample_unescapeString(sample_json_str_get(data, "raw_data"))); + return std::string("None"); +} + +void StackFlow::unlink(const std::string &zmq_url, const std::string &raw) { + SLOGI("StackFlow::unlink raw"); + std::string work_id = sample_json_str_get(raw, "work_id"); + try { + auto task_channel = get_channel(sample_get_work_id_num(work_id)); + task_channel->set_push_url(zmq_url); + } catch (...) { + } + unlink(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); +} + +void StackFlow::unlink(const std::string &work_id, const std::string &object, const std::string &data) { + SLOGI("StackFlow::unlink"); + if (_unlink_) { + _unlink_(work_id, object, data); + return; + } + nlohmann::json error_body; + error_body["code"] = -18; + error_body["message"] = "not have unit action!"; + send("None", "None", error_body, work_id); +} + +std::string StackFlow::_rpc_work(const std::string &data) { + event_queue_.enqueue(EVENT_WORK, sample_json_str_get(data, "zmq_com"), + sample_unescapeString(sample_json_str_get(data, "raw_data"))); + return std::string("None"); +} + +void StackFlow::work(const std::string &zmq_url, const std::string &raw) { + SLOGI("StackFlow::work raw"); + std::string work_id = sample_json_str_get(raw, "work_id"); + try { + auto task_channel = get_channel(sample_get_work_id_num(work_id)); + task_channel->set_push_url(zmq_url); + } catch (...) { + } + work(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); +} + +void StackFlow::work(const std::string &work_id, const std::string &object, const std::string &data) { + SLOGI("StackFlow::work"); + if (_work_) { + _work_(work_id, object, data); + return; + } + nlohmann::json error_body; + error_body["code"] = -18; + error_body["message"] = "not have unit action!"; + send("None", "None", error_body, work_id); +} + +std::string StackFlow::_rpc_exit(const std::string &data) { + event_queue_.enqueue(EVENT_EXIT, sample_json_str_get(data, "zmq_com"), + sample_unescapeString(sample_json_str_get(data, "raw_data"))); + return std::string("None"); +} + +int StackFlow::exit(const std::string &zmq_url, const std::string &raw) { + SLOGI("StackFlow::exit raw"); + std::string work_id = sample_json_str_get(raw, "work_id"); + try { + auto task_channel = get_channel(sample_get_work_id_num(work_id)); + task_channel->set_push_url(zmq_url); + } catch (...) { + } + if (exit(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")) == 0) { + return (int)sys_release_unit(-1, work_id); + } + return 0; +} + +int StackFlow::exit(const std::string &work_id, const std::string &object, const std::string &data) { + SLOGI("StackFlow::exit"); + if (_exit_) { + return _exit_(work_id, object, data); + } + nlohmann::json error_body; + error_body["code"] = -18; + error_body["message"] = "not have unit action!"; + send("None", "None", error_body, work_id); + return 0; +} + +std::string StackFlow::_rpc_pause(const std::string &data) { + event_queue_.enqueue(EVENT_PAUSE, sample_json_str_get(data, "zmq_com"), + sample_unescapeString(sample_json_str_get(data, "raw_data"))); + return std::string("None"); +} + +void StackFlow::pause(const std::string &zmq_url, const std::string &raw) { + SLOGI("StackFlow::pause raw"); + std::string work_id = sample_json_str_get(raw, "work_id"); + try { + auto task_channel = get_channel(sample_get_work_id_num(work_id)); + task_channel->set_push_url(zmq_url); + } catch (...) { + } + pause(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); +} + +void StackFlow::pause(const std::string &work_id, const std::string &object, const std::string &data) { + SLOGI("StackFlow::pause"); + if (_pause_) { + _pause_(work_id, object, data); + return; + } + nlohmann::json error_body; + error_body["code"] = -18; + error_body["message"] = "not have unit action!"; + send("None", "None", error_body, work_id); +} + +std::string StackFlow::_rpc_taskinfo(const std::string &data) { + event_queue_.enqueue(EVENT_TASKINFO, sample_json_str_get(data, "zmq_com"), + sample_unescapeString(sample_json_str_get(data, "raw_data"))); + return std::string("None"); +} + +void StackFlow::taskinfo(const std::string &zmq_url, const std::string &raw) { + SLOGI("StackFlow::taskinfo raw"); + std::string work_id = sample_json_str_get(raw, "work_id"); + try { + auto task_channel = get_channel(sample_get_work_id_num(work_id)); + task_channel->set_push_url(zmq_url); + } catch (...) { + } + taskinfo(work_id, sample_json_str_get(raw, "object"), sample_json_str_get(raw, "data")); +} + +void StackFlow::taskinfo(const std::string &work_id, const std::string &object, const std::string &data) { + SLOGI("StackFlow::taskinfo"); + if (_taskinfo_) { + _taskinfo_(work_id, object, data); + return; + } + nlohmann::json error_body; + error_body["code"] = -18; + error_body["message"] = "not have unit action!"; + send("None", "None", error_body, work_id); +} + +int StackFlow::sys_register_unit(const std::string &unit_name) { + int work_id_number; + std::string out_port; + std::string inference_port; + + pzmq _call("sys"); + _call.call_rpc_action("register_unit", unit_name, [&](const std::string &unit_info) { + work_id_number = std::stoi(sample_json_str_get(unit_info, "work_id_number")); + out_port = sample_json_str_get(unit_info, "out_port"); + inference_port = sample_json_str_get(unit_info, "inference_port"); + }); + SLOGI("work_id_number:%d, out_port:%s, inference_port:%s ", work_id_number, out_port.c_str(), + inference_port.c_str()); + llm_task_channel_[work_id_number] = std::make_shared(out_port, inference_port, unit_name_); + return work_id_number; +} + +bool StackFlow::sys_release_unit(int work_id_num, const std::string &work_id) { + std::string _work_id; + int _work_id_num; + if (work_id.empty()) { + _work_id = sample_get_work_id(work_id_num, unit_name_); + _work_id_num = work_id_num; + } else { + _work_id = work_id; + _work_id_num = sample_get_work_id_num(work_id); + } + pzmq _call("sys"); + _call.call_rpc_action("release_unit", _work_id, [&](const std::string &unit_info) {}); + llm_task_channel_[_work_id_num].reset(); + llm_task_channel_.erase(_work_id_num); + SLOGI("release work_id %s success", _work_id.c_str()); + return false; +} + +std::string StackFlow::sys_sql_select(const std::string &key) { + std::string val; + pzmq _call("sys"); + _call.call_rpc_action("sql_select", key, [&val](const std::string &data) { val = data; }); + return val; +} + +void StackFlow::sys_sql_set(const std::string &key, const std::string &val) { + nlohmann::json out_body; + out_body["key"] = key; + out_body["val"] = val; + pzmq _call("sys"); + _call.call_rpc_action("sql_set", out_body.dump(), [](const std::string &data) {}); +} + +void StackFlow::sys_sql_unset(const std::string &key) { + std::string val; + pzmq _call("sys"); + _call.call_rpc_action("sql_select", key, [](const std::string &data) {}); +} + +std::string StackFlow::unit_call(const std::string &unit_name, const std::string &unit_action, + const std::string &data) { + std::string value; + pzmq _call(unit_name); + _call.call_rpc_action(unit_action, data, [&value](const std::string &raw) { value = raw; }); + return value; +} + +void StackFlow::_repeat_loop(const std::string &action, const std::string &ms) { + repeat_callback_fun_mutex_.lock(); + const auto call_fun = repeat_callback_fun_[action]; + repeat_callback_fun_mutex_.unlock(); + if (call_fun()) { + int delayms = std::stoi(ms); + if (delayms) + std::thread([this, action, delayms, ms]() { + std::this_thread::sleep_for(std::chrono::milliseconds(delayms)); + this->event_queue_.enqueue(EVENT_REPEAT_EVENT, action, ms); + }).detach(); + else { + event_queue_.enqueue(EVENT_REPEAT_EVENT, action, ms); + } + } else { + repeat_callback_fun_mutex_.lock(); + repeat_callback_fun_.erase(action); + repeat_callback_fun_mutex_.unlock(); + } +} + +void StackFlow::repeat_event(int ms, std::function repeat_fun, bool now) { + repeat_callback_fun_mutex_.lock(); + std::string action = std::to_string(repeat_callback_fun_.size() + 1); + repeat_callback_fun_[action] = repeat_fun; + repeat_callback_fun_mutex_.unlock(); + if (!now) + std::thread([this, action, ms]() { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + this->event_queue_.enqueue(EVENT_REPEAT_EVENT, action, std::to_string(ms)); + }).detach(); + else { + event_queue_.enqueue(EVENT_REPEAT_EVENT, action, std::to_string(ms)); + } +} diff --git a/ext_components/StackFlow/stackflow/StackFlow.h b/ext_components/StackFlow/stackflow/StackFlow.h new file mode 100644 index 0000000..f7c911f --- /dev/null +++ b/ext_components/StackFlow/stackflow/StackFlow.h @@ -0,0 +1,361 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once + +// #define __cplusplus 1 + +#include +#include + +// #define CONFIG_SUPPORTTHREADSAFE 0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include "json.hpp" +#include +#include "pzmq.hpp" +#include "StackFlowUtil.h" +namespace StackFlows { +template +class ThreadSafeWrapper { + public: + template + ThreadSafeWrapper(Args &&...args) : object(std::make_unique(std::forward(args)...)) { + } + + template + auto access(Func func) -> decltype(func(std::declval())) { + std::lock_guard lock(mutex); + return func(*object); + } + + ThreadSafeWrapper &operator=(const T &newValue) { + std::lock_guard lock(mutex); + *object = newValue; + return *this; + } + + ThreadSafeWrapper &operator=(T &&newValue) { + std::lock_guard lock(mutex); + *object = std::move(newValue); + return *this; + } + + private: + std::unique_ptr object; + mutable std::mutex mutex; +}; + +#define LLM_NO_ERROR std::string("") +#define LLM_NONE std::string("None") + +class llm_channel_obj { + private: + std::unordered_map> zmq_; + std::atomic zmq_url_index_; + std::unordered_map zmq_url_map_; + + public: + std::string unit_name_; + bool enoutput_; + bool enstream_; + std::string request_id_; + std::string work_id_; + std::string inference_url_; + std::string output_url_; + static std::string uart_push_url; + std::string publisher_url; + + llm_channel_obj(const std::string &_publisher_url, const std::string &inference_url, const std::string &unit_name); + ~llm_channel_obj(); + inline void set_output(bool flage) { + enoutput_ = flage; + } + inline bool get_output() { + return enoutput_; + } + inline void set_stream(bool flage) { + enstream_ = flage; + } + inline bool get_stream() { + return enstream_; + } + void subscriber_event_call(const std::function &call, + const std::string &raw); + int subscriber_work_id(const std::string &work_id, + const std::function &call); + void stop_subscriber_work_id(const std::string &work_id); + void subscriber(const std::string &zmq_url, const std::function &call); + void stop_subscriber(const std::string &zmq_url); + int check_zmq_errno(void *ctx, void *com, int code); + int send_raw_to_pub(const std::string &raw); + int send_raw_to_usr(const std::string &raw); + template + int output_data(const std::string &object, const T &data, const U &error_msg) { + return output_data(request_id_, work_id_, object, data, error_msg); + } + void set_push_url(const std::string &url); + void cear_push_url(); + static int output_to_uart(const std::string &data); + static int send_raw_for_url(const std::string &zmq_url, const std::string &raw); + template + static int output_data_for_url(const std::string &zmq_url, const std::string &request_id, + const std::string &work_id, const std::string &object, const U &data, + const T &error_msg, bool outuart = false) { + nlohmann::json out_body; + out_body["request_id"] = request_id; + out_body["work_id"] = work_id; + out_body["created"] = time(NULL); + out_body["object"] = object; + out_body["data"] = data; + if (error_msg.empty()) { + out_body["error"]["code"] = 0; + out_body["error"]["message"] = ""; + } else + out_body["error"] = error_msg; + std::string out = out_body.dump(); + if (outuart) + return output_to_uart(out); + else + return send_raw_for_url(zmq_url, out); + } + template + int send(const std::string &object, const U &data, const T &error_msg, const std::string &work_id = "", + bool outuart = false) { + nlohmann::json out_body; + out_body["request_id"] = request_id_; + out_body["work_id"] = work_id.empty() ? work_id_ : work_id; + out_body["created"] = time(NULL); + out_body["object"] = object; + out_body["data"] = data; + if (error_msg.empty()) { + out_body["error"]["code"] = 0; + out_body["error"]["message"] = ""; + } else + out_body["error"] = error_msg; + + std::string out = out_body.dump(); + send_raw_to_pub(out); + if (enoutput_) return send_raw_to_usr(out); + return 0; + } + template + int output_data(const std::string &request_id, const std::string &work_id, const std::string &object, const T &data, + const U &error_msg) { + nlohmann::json out_body; + out_body["request_id"] = request_id; + out_body["work_id"] = work_id; + out_body["created"] = time(NULL); + out_body["object"] = object; + out_body["data"] = data; + if (error_msg.empty()) { + out_body["error"]["code"] = 0; + out_body["error"]["message"] = ""; + } else + out_body["error"] = error_msg; + + std::string out = out_body.dump(); + send_raw_to_pub(out); + if (enoutput_) return send_raw_to_usr(out); + return 0; + } +}; + +class StackFlow { + private: + protected: + std::string unit_name_; + typedef enum { + EVENT_NONE = 0, + EVENT_SETUP, + EVENT_EXIT, + EVENT_PAUSE, + EVENT_WORK, + EVENT_STOP, + EVENT_LINK, + EVENT_UNLINK, + EVENT_TASKINFO, + EVENT_SWITCH_INPUT, + EVENT_SYS_INIT, + EVENT_CREAT_CHANNEL, + EVENT_GET_CHANNL, + EVENT_REPEAT_EVENT, + EVENT_EXPORT, + } local_event_t; + + eventpp::EventQueue event_queue_; + std::unique_ptr even_loop_thread_; + std::unique_ptr rpc_ctx_; + std::atomic status_; + std::unordered_map> llm_task_channel_; + std::unordered_map> repeat_callback_fun_; + std::mutex repeat_callback_fun_mutex_; + + void _repeat_loop(const std::string &zmq_url, const std::string &raw); + + public: + std::string request_id_; + std::string out_zmq_url_; + + std::function _link_; + std::function _unlink_; + std::function _setup_; + std::function _exit_; + std::function _pause_; + std::function _work_; + std::function _taskinfo_; + std::function _get_channl_; + + std::atomic exit_flage_; + + StackFlow(const std::string &unit_name); + void even_loop(); + void _none_event(const std::string &data1, const std::string &data2); + + template + std::shared_ptr get_channel(T workid) { + int _work_id_num; + if constexpr (std::is_same::value) { + _work_id_num = workid; + } else if constexpr (std::is_same::value) { + _work_id_num = sample_get_work_id_num(workid); + } else { + return nullptr; + } + return llm_task_channel_.at(_work_id_num); + } + + std::string _rpc_setup(const std::string &data); + void _setup(const std::string &zmq_url, const std::string &data) { + // printf("void _setup run \n"); + request_id_ = sample_json_str_get(data, "request_id"); + out_zmq_url_ = zmq_url; + if (status_.load()) setup(zmq_url, data); + }; + virtual int setup(const std::string &zmq_url, const std::string &raw); + virtual int setup(const std::string &work_id, const std::string &object, const std::string &data); + + std::string _rpc_link(const std::string &data); + void _link(const std::string &zmq_url, const std::string &data) { + // printf("void _link run \n"); + request_id_ = sample_json_str_get(data, "request_id"); + out_zmq_url_ = zmq_url; + if (status_.load()) link(zmq_url, data); + }; + virtual void link(const std::string &zmq_url, const std::string &raw); + virtual void link(const std::string &work_id, const std::string &object, const std::string &data); + + std::string _rpc_unlink(const std::string &data); + void _unlink(const std::string &zmq_url, const std::string &data) { + // printf("void _unlink run \n"); + request_id_ = sample_json_str_get(data, "request_id"); + out_zmq_url_ = zmq_url; + if (status_.load()) unlink(zmq_url, data); + }; + virtual void unlink(const std::string &zmq_url, const std::string &raw); + virtual void unlink(const std::string &work_id, const std::string &object, const std::string &data); + + std::string _rpc_exit(const std::string &data); + void _exit(const std::string &zmq_url, const std::string &data) { + request_id_ = sample_json_str_get(data, "request_id"); + out_zmq_url_ = zmq_url; + if (status_.load()) exit(zmq_url, data); + } + virtual int exit(const std::string &zmq_url, const std::string &raw); + virtual int exit(const std::string &work_id, const std::string &object, const std::string &data); + + std::string _rpc_work(const std::string &data); + void _work(const std::string &zmq_url, const std::string &data) { + request_id_ = sample_json_str_get(data, "request_id"); + out_zmq_url_ = zmq_url; + if (status_.load()) work(zmq_url, data); + } + virtual void work(const std::string &zmq_url, const std::string &raw); + virtual void work(const std::string &work_id, const std::string &object, const std::string &data); + + std::string _rpc_pause(const std::string &data); + void _pause(const std::string &zmq_url, const std::string &data) { + request_id_ = sample_json_str_get(data, "request_id"); + out_zmq_url_ = zmq_url; + if (status_.load()) pause(zmq_url, data); + } + virtual void pause(const std::string &zmq_url, const std::string &raw); + virtual void pause(const std::string &work_id, const std::string &object, const std::string &data); + + std::string _rpc_taskinfo(const std::string &data); + void _taskinfo(const std::string &zmq_url, const std::string &data) { + request_id_ = sample_json_str_get(data, "request_id"); + out_zmq_url_ = zmq_url; + if (status_.load()) taskinfo(zmq_url, data); + } + virtual void taskinfo(const std::string &zmq_url, const std::string &raw); + virtual void taskinfo(const std::string &work_id, const std::string &object, const std::string &data); + + void _sys_init(const std::string &zmq_url, const std::string &data); + + void user_output(const std::string &zmq_url, const std::string &request_id, const std::string &data); + template + int send(const std::string &object, const U &data, const T &error_msg, const std::string &work_id, + const std::string &zmq_url = "") { + nlohmann::json out_body; + out_body["request_id"] = request_id_; + out_body["work_id"] = work_id; + out_body["created"] = time(NULL); + out_body["object"] = object; + out_body["data"] = data; + if (error_msg.empty()) { + out_body["error"]["code"] = 0; + out_body["error"]["message"] = ""; + } else + out_body["error"] = error_msg; + if (zmq_url.empty()) { + pzmq _zmq(out_zmq_url_, ZMQ_PUSH); + return _zmq.send_data(out_body.dump()); + } else { + pzmq _zmq(zmq_url, ZMQ_PUSH); + return _zmq.send_data(out_body.dump()); + } + } + + void llm_firework_exit() { + } + + std::string unit_call(const std::string &unit_name, const std::string &unit_action, const std::string &data); + std::string sys_sql_select(const std::string &key); + void sys_sql_set(const std::string &key, const std::string &val); + void sys_sql_unset(const std::string &key); + int sys_register_unit(const std::string &unit_name); + template + bool sys_release_unit(T workid) { + std::string _work_id; + int _work_id_num; + if constexpr (std::is_same::value) { + _work_id = sample_get_work_id(workid, unit_name_); + _work_id_num = workid; + } else if constexpr (std::is_same::value) { + _work_id = workid; + _work_id_num = sample_get_work_id_num(workid); + } else { + return false; + } + pzmq _call("sys"); + _call.call_rpc_action("release_unit", _work_id, [](const std::string &data) {}); + llm_task_channel_[_work_id_num].reset(); + llm_task_channel_.erase(_work_id_num); + // SLOGI("release work_id %s success", _work_id.c_str()); + return false; + } + bool sys_release_unit(int work_id_num, const std::string &work_id); + void repeat_event(int ms, std::function repeat_fun, bool now = true); + ~StackFlow(); +}; +}; // namespace StackFlows \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.cpp b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp new file mode 100644 index 0000000..fde8ecf --- /dev/null +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.cpp @@ -0,0 +1,391 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "StackFlowUtil.h" +#include + +std::string StackFlows::sample_json_str_get(const std::string &json_str, const std::string &json_key) { + std::string key_val; + std::string format_val; + // SLOGD("json_str: %s json_key:%s\n", json_str.c_str(), json_key.c_str()); + std::string find_key = "\"" + json_key + "\""; + int subs_start = json_str.find(find_key); + if (subs_start == std::string::npos) { + return key_val; + } + int status = 0; + char last_c = '\0'; + int obj_flage = 0; + for (auto c : json_str.substr(subs_start + find_key.length())) { + switch (status) { + case 0: { + switch (c) { + case '"': { + status = 100; + } break; + case '{': { + key_val.push_back(c); + obj_flage = 1; + status = 10; + } break; + case ':': + obj_flage = 1; + break; + case ',': + case '}': { + obj_flage = 0; + status = -1; + } break; + case ' ': + break; + default: { + if (obj_flage) { + key_val.push_back(c); + } + } break; + } + } break; + case 10: { + key_val.push_back(c); + if (c == '{') { + obj_flage++; + } + if (c == '}') { + obj_flage--; + } + if (obj_flage == 0) { + if (!key_val.empty()) { + status = -1; + } + } + } break; + case 100: { + if ((c == '"') && (last_c != '\\')) { + obj_flage = 0; + status = -1; + } else { + key_val.push_back(c); + } + } break; + default: + break; + } + last_c = c; + } + if (obj_flage != 0) { + key_val.clear(); + } + // SLOGD("key_val:%s\n", key_val.c_str()); + return key_val; +} + +int StackFlows::sample_get_work_id_num(const std::string &work_id) { + int a = work_id.find("."); + if ((a == std::string::npos) || (a == work_id.length() - 1)) { + return WORK_ID_NONE; + } + return std::stoi(work_id.substr(a + 1)); +} + +std::string StackFlows::sample_get_work_id_name(const std::string &work_id) { + int a = work_id.find("."); + if (a == std::string::npos) { + return work_id; + } else { + return work_id.substr(0, a); + } +} + +std::string StackFlows::sample_get_work_id(int work_id_num, const std::string &unit_name) { + return unit_name + "." + std::to_string(work_id_num); +} + +std::string StackFlows::sample_escapeString(const std::string &input) { + std::string escaped; + for (char c : input) { + switch (c) { + case '\n': + escaped += "\\n"; + break; + case '\t': + escaped += "\\t"; + break; + case '\\': + escaped += "\\\\"; + break; + case '\"': + escaped += "\\\""; + break; + case '\r': + escaped += "\\r"; + break; + case '\b': + escaped += "\\b"; + break; + default: + escaped += c; + break; + } + } + return escaped; +} + +std::string StackFlows::sample_unescapeString(const std::string &input) { + std::string unescaped; + for (size_t i = 0; i < input.length(); ++i) { + if (input[i] == '\\' && i + 1 < input.length()) { + switch (input[i + 1]) { + case 'n': + unescaped += '\n'; + ++i; + break; + case 't': + unescaped += '\t'; + ++i; + break; + case '\\': + unescaped += '\\'; + ++i; + break; + case '\"': + unescaped += '\"'; + ++i; + break; + case 'r': + unescaped += '\r'; + ++i; + break; + case 'b': + unescaped += '\b'; + ++i; + break; + default: + unescaped += input[i]; + break; + } + } else { + unescaped += input[i]; + } + } + return unescaped; +} + +bool StackFlows::decode_stream(const std::string &in, std::string &out, + std::unordered_map &stream_buff) { + try { + int index = std::stoi(StackFlows::sample_json_str_get(in, "index")); + std::string finish = StackFlows::sample_json_str_get(in, "finish"); + std::string delta = StackFlows::sample_json_str_get(in, "delta"); + stream_buff[index] = delta; + if (finish == "true") { + for (size_t i = 0; i < stream_buff.size(); i++) { + out += stream_buff.at(i); + } + stream_buff.clear(); + return false; + } else if (finish != "false") { + throw true; + } + } catch (...) { + stream_buff.clear(); + } + return true; +} + +#define BASE64_ENCODE_OUT_SIZE(s) (((s) + 2) / 3 * 4) +#define BASE64_DECODE_OUT_SIZE(s) (((s)) / 4 * 3) +#include +/* BASE 64 encode table */ +static const char base64en[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', +}; + +#define BASE64_PAD '=' +#define BASE64DE_FIRST '+' +#define BASE64DE_LAST 'z' +/* ASCII order for BASE 64 decode, -1 in unused character */ +static const signed char base64de[] = { + /* '+', ',', '-', '.', '/', '0', '1', '2', */ + 62, + -1, + -1, + -1, + 63, + 52, + 53, + 54, + + /* '3', '4', '5', '6', '7', '8', '9', ':', */ + 55, + 56, + 57, + 58, + 59, + 60, + 61, + -1, + + /* ';', '<', '=', '>', '?', '@', 'A', 'B', */ + -1, + -1, + -1, + -1, + -1, + -1, + 0, + 1, + + /* 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', */ + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + + /* 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', */ + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + + /* 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', */ + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + + /* '[', '\', ']', '^', '_', '`', 'a', 'b', */ + -1, + -1, + -1, + -1, + -1, + -1, + 26, + 27, + + /* 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', */ + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + + /* 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', */ + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + + /* 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', */ + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, +}; + +static int base64_encode(const unsigned char *in, unsigned int inlen, char *out) { + unsigned int i = 0, j = 0; + + for (; i < inlen; i++) { + int s = i % 3; + + switch (s) { + case 0: + out[j++] = base64en[(in[i] >> 2) & 0x3F]; + continue; + case 1: + out[j++] = base64en[((in[i - 1] & 0x3) << 4) + ((in[i] >> 4) & 0xF)]; + continue; + case 2: + out[j++] = base64en[((in[i - 1] & 0xF) << 2) + ((in[i] >> 6) & 0x3)]; + out[j++] = base64en[in[i] & 0x3F]; + } + } + + /* move back */ + i -= 1; + + /* check the last and add padding */ + if ((i % 3) == 0) { + out[j++] = base64en[(in[i] & 0x3) << 4]; + out[j++] = BASE64_PAD; + out[j++] = BASE64_PAD; + } else if ((i % 3) == 1) { + out[j++] = base64en[(in[i] & 0xF) << 2]; + out[j++] = BASE64_PAD; + } + + return j; +} + +static int base64_decode(const char *in, unsigned int inlen, unsigned char *out) { + unsigned int i = 0, j = 0; + + for (; i < inlen; i++) { + int c; + int s = i % 4; + + if (in[i] == '=') return j; + + if (in[i] < BASE64DE_FIRST || in[i] > BASE64DE_LAST || (c = base64de[in[i] - BASE64DE_FIRST]) == -1) return -1; + + switch (s) { + case 0: + out[j] = ((unsigned int)c << 2) & 0xFF; + continue; + case 1: + out[j++] += ((unsigned int)c >> 4) & 0x3; + + /* if not last char with padding */ + if (i < (inlen - 3) || in[inlen - 2] != '=') out[j] = ((unsigned int)c & 0xF) << 4; + continue; + case 2: + out[j++] += ((unsigned int)c >> 2) & 0xF; + + /* if not last char with padding */ + if (i < (inlen - 2) || in[inlen - 1] != '=') out[j] = ((unsigned int)c & 0x3) << 6; + continue; + case 3: + out[j++] += (unsigned char)c; + } + } + + return j; +} + +int StackFlows::decode_base64(const std::string &in, std::string &out) { + out.resize(BASE64_DECODE_OUT_SIZE(in.length())); + return base64_decode((const char *)in.c_str(), in.length(), (unsigned char *)out.data()); +} + +int StackFlows::encode_base64(const std::string &in, std::string &out) { + out.resize(BASE64_ENCODE_OUT_SIZE(in.length())); + return base64_encode((const unsigned char *)in.c_str(), in.length(), (char *)out.data()); +} \ No newline at end of file diff --git a/ext_components/StackFlow/stackflow/StackFlowUtil.h b/ext_components/StackFlow/stackflow/StackFlowUtil.h new file mode 100644 index 0000000..1b5df5f --- /dev/null +++ b/ext_components/StackFlow/stackflow/StackFlowUtil.h @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once +#include +#include +#define WORK_ID_NONE -100 +namespace StackFlows { +std::string sample_json_str_get(const std::string &json_str, const std::string &json_key); +int sample_get_work_id_num(const std::string &work_id); +std::string sample_get_work_id_name(const std::string &work_id); +std::string sample_get_work_id(int work_id_num, const std::string &unit_name); +std::string sample_escapeString(const std::string &input); +std::string sample_unescapeString(const std::string &input); +bool decode_stream(const std::string &in, std::string &out, std::unordered_map &stream_buff); +int decode_base64(const std::string &in, std::string &out); +int encode_base64(const std::string &in, std::string &out); +}; // namespace StackFlows diff --git a/ext_components/StackFlow/stackflow/libzmq/zmq.h b/ext_components/StackFlow/stackflow/libzmq/zmq.h new file mode 100644 index 0000000..7b94061 --- /dev/null +++ b/ext_components/StackFlow/stackflow/libzmq/zmq.h @@ -0,0 +1,787 @@ +/* SPDX-License-Identifier: MPL-2.0 */ +/* ************************************************************************* + NOTE to contributors. This file comprises the principal public contract + for ZeroMQ API users. Any change to this file supplied in a stable + release SHOULD not break existing applications. + In practice this means that the value of constants must not change, and + that old values may not be reused for new constants. + ************************************************************************* +*/ + +#ifndef __ZMQ_H_INCLUDED__ +#define __ZMQ_H_INCLUDED__ + +/* Version macros for compile-time API version detection */ +#define ZMQ_VERSION_MAJOR 4 +#define ZMQ_VERSION_MINOR 3 +#define ZMQ_VERSION_PATCH 6 + +#define ZMQ_MAKE_VERSION(major, minor, patch) \ + ((major) *10000 + (minor) *100 + (patch)) +#define ZMQ_VERSION \ + ZMQ_MAKE_VERSION (ZMQ_VERSION_MAJOR, ZMQ_VERSION_MINOR, ZMQ_VERSION_PATCH) + +#ifdef __cplusplus +extern "C" { +#endif + +#if !defined _WIN32_WCE +#include +#endif +#include +#include + +/* Handle DSO symbol visibility */ +#if defined ZMQ_NO_EXPORT +#define ZMQ_EXPORT +#else +#if defined _WIN32 +#if defined ZMQ_STATIC +#define ZMQ_EXPORT +#elif defined DLL_EXPORT +#define ZMQ_EXPORT __declspec(dllexport) +#else +#define ZMQ_EXPORT __declspec(dllimport) +#endif +#else +#if defined __SUNPRO_C || defined __SUNPRO_CC +#define ZMQ_EXPORT __global +#elif (defined __GNUC__ && __GNUC__ >= 4) || defined __INTEL_COMPILER +#define ZMQ_EXPORT __attribute__ ((visibility ("default"))) +#else +#define ZMQ_EXPORT +#endif +#endif +#endif + +/* Define integer types needed for event interface */ +#define ZMQ_DEFINED_STDINT 1 +#if defined ZMQ_HAVE_SOLARIS || defined ZMQ_HAVE_OPENVMS +#include +#elif defined _MSC_VER && _MSC_VER < 1600 +#ifndef uint64_t +typedef unsigned __int64 uint64_t; +#endif +#ifndef int32_t +typedef __int32 int32_t; +#endif +#ifndef uint32_t +typedef unsigned __int32 uint32_t; +#endif +#ifndef uint16_t +typedef unsigned __int16 uint16_t; +#endif +#ifndef uint8_t +typedef unsigned __int8 uint8_t; +#endif +#else +#include +#endif + +#if !defined _WIN32 +// needed for sigset_t definition in zmq_ppoll +#include +#endif + +// 32-bit AIX's pollfd struct members are called reqevents and rtnevents so it +// defines compatibility macros for them. Need to include that header first to +// stop build failures since zmq_pollset_t defines them as events and revents. +#ifdef ZMQ_HAVE_AIX +#include +#endif + + +/******************************************************************************/ +/* 0MQ errors. */ +/******************************************************************************/ + +/* A number random enough not to collide with different errno ranges on */ +/* different OSes. The assumption is that error_t is at least 32-bit type. */ +#define ZMQ_HAUSNUMERO 156384712 + +/* On Windows platform some of the standard POSIX errnos are not defined. */ +#ifndef ENOTSUP +#define ENOTSUP (ZMQ_HAUSNUMERO + 1) +#endif +#ifndef EPROTONOSUPPORT +#define EPROTONOSUPPORT (ZMQ_HAUSNUMERO + 2) +#endif +#ifndef ENOBUFS +#define ENOBUFS (ZMQ_HAUSNUMERO + 3) +#endif +#ifndef ENETDOWN +#define ENETDOWN (ZMQ_HAUSNUMERO + 4) +#endif +#ifndef EADDRINUSE +#define EADDRINUSE (ZMQ_HAUSNUMERO + 5) +#endif +#ifndef EADDRNOTAVAIL +#define EADDRNOTAVAIL (ZMQ_HAUSNUMERO + 6) +#endif +#ifndef ECONNREFUSED +#define ECONNREFUSED (ZMQ_HAUSNUMERO + 7) +#endif +#ifndef EINPROGRESS +#define EINPROGRESS (ZMQ_HAUSNUMERO + 8) +#endif +#ifndef ENOTSOCK +#define ENOTSOCK (ZMQ_HAUSNUMERO + 9) +#endif +#ifndef EMSGSIZE +#define EMSGSIZE (ZMQ_HAUSNUMERO + 10) +#endif +#ifndef EAFNOSUPPORT +#define EAFNOSUPPORT (ZMQ_HAUSNUMERO + 11) +#endif +#ifndef ENETUNREACH +#define ENETUNREACH (ZMQ_HAUSNUMERO + 12) +#endif +#ifndef ECONNABORTED +#define ECONNABORTED (ZMQ_HAUSNUMERO + 13) +#endif +#ifndef ECONNRESET +#define ECONNRESET (ZMQ_HAUSNUMERO + 14) +#endif +#ifndef ENOTCONN +#define ENOTCONN (ZMQ_HAUSNUMERO + 15) +#endif +#ifndef ETIMEDOUT +#define ETIMEDOUT (ZMQ_HAUSNUMERO + 16) +#endif +#ifndef EHOSTUNREACH +#define EHOSTUNREACH (ZMQ_HAUSNUMERO + 17) +#endif +#ifndef ENETRESET +#define ENETRESET (ZMQ_HAUSNUMERO + 18) +#endif + +/* Native 0MQ error codes. */ +#define EFSM (ZMQ_HAUSNUMERO + 51) +#define ENOCOMPATPROTO (ZMQ_HAUSNUMERO + 52) +#define ETERM (ZMQ_HAUSNUMERO + 53) +#define EMTHREAD (ZMQ_HAUSNUMERO + 54) + +/* This function retrieves the errno as it is known to 0MQ library. The goal */ +/* of this function is to make the code 100% portable, including where 0MQ */ +/* compiled with certain CRT library (on Windows) is linked to an */ +/* application that uses different CRT library. */ +ZMQ_EXPORT int zmq_errno (void); + +/* Resolves system errors and 0MQ errors to human-readable string. */ +ZMQ_EXPORT const char *zmq_strerror (int errnum_); + +/* Run-time API version detection */ +ZMQ_EXPORT void zmq_version (int *major_, int *minor_, int *patch_); + +/******************************************************************************/ +/* 0MQ infrastructure (a.k.a. context) initialisation & termination. */ +/******************************************************************************/ + +/* Context options */ +#define ZMQ_IO_THREADS 1 +#define ZMQ_MAX_SOCKETS 2 +#define ZMQ_SOCKET_LIMIT 3 +#define ZMQ_THREAD_PRIORITY 3 +#define ZMQ_THREAD_SCHED_POLICY 4 +#define ZMQ_MAX_MSGSZ 5 +#define ZMQ_MSG_T_SIZE 6 +#define ZMQ_THREAD_AFFINITY_CPU_ADD 7 +#define ZMQ_THREAD_AFFINITY_CPU_REMOVE 8 +#define ZMQ_THREAD_NAME_PREFIX 9 + +/* Default for new contexts */ +#define ZMQ_IO_THREADS_DFLT 1 +#define ZMQ_MAX_SOCKETS_DFLT 1023 +#define ZMQ_THREAD_PRIORITY_DFLT -1 +#define ZMQ_THREAD_SCHED_POLICY_DFLT -1 + +ZMQ_EXPORT void *zmq_ctx_new (void); +ZMQ_EXPORT int zmq_ctx_term (void *context_); +ZMQ_EXPORT int zmq_ctx_shutdown (void *context_); +ZMQ_EXPORT int zmq_ctx_set (void *context_, int option_, int optval_); +ZMQ_EXPORT int zmq_ctx_get (void *context_, int option_); + +/* Old (legacy) API */ +ZMQ_EXPORT void *zmq_init (int io_threads_); +ZMQ_EXPORT int zmq_term (void *context_); +ZMQ_EXPORT int zmq_ctx_destroy (void *context_); + + +/******************************************************************************/ +/* 0MQ message definition. */ +/******************************************************************************/ + +/* Some architectures, like sparc64 and some variants of aarch64, enforce pointer + * alignment and raise sigbus on violations. Make sure applications allocate + * zmq_msg_t on addresses aligned on a pointer-size boundary to avoid this issue. + */ +typedef struct zmq_msg_t +{ +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + __declspec(align (8)) unsigned char _[64]; +#elif defined(_MSC_VER) \ + && (defined(_M_IX86) || defined(_M_ARM_ARMV7VE) || defined(_M_ARM)) + __declspec(align (4)) unsigned char _[64]; +#elif defined(__GNUC__) || defined(__INTEL_COMPILER) \ + || (defined(__SUNPRO_C) && __SUNPRO_C >= 0x590) \ + || (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x590) + unsigned char _[64] __attribute__ ((aligned (sizeof (void *)))); +#else + unsigned char _[64]; +#endif +} zmq_msg_t; + +typedef void (zmq_free_fn) (void *data_, void *hint_); + +ZMQ_EXPORT int zmq_msg_init (zmq_msg_t *msg_); +ZMQ_EXPORT int zmq_msg_init_size (zmq_msg_t *msg_, size_t size_); +ZMQ_EXPORT int zmq_msg_init_data ( + zmq_msg_t *msg_, void *data_, size_t size_, zmq_free_fn *ffn_, void *hint_); +ZMQ_EXPORT int zmq_msg_send (zmq_msg_t *msg_, void *s_, int flags_); +ZMQ_EXPORT int zmq_msg_recv (zmq_msg_t *msg_, void *s_, int flags_); +ZMQ_EXPORT int zmq_msg_close (zmq_msg_t *msg_); +ZMQ_EXPORT int zmq_msg_move (zmq_msg_t *dest_, zmq_msg_t *src_); +ZMQ_EXPORT int zmq_msg_copy (zmq_msg_t *dest_, zmq_msg_t *src_); +ZMQ_EXPORT void *zmq_msg_data (zmq_msg_t *msg_); +ZMQ_EXPORT size_t zmq_msg_size (const zmq_msg_t *msg_); +ZMQ_EXPORT int zmq_msg_more (const zmq_msg_t *msg_); +ZMQ_EXPORT int zmq_msg_get (const zmq_msg_t *msg_, int property_); +ZMQ_EXPORT int zmq_msg_set (zmq_msg_t *msg_, int property_, int optval_); +ZMQ_EXPORT const char *zmq_msg_gets (const zmq_msg_t *msg_, + const char *property_); + +/******************************************************************************/ +/* 0MQ socket definition. */ +/******************************************************************************/ + +/* Socket types. */ +#define ZMQ_PAIR 0 +#define ZMQ_PUB 1 +#define ZMQ_SUB 2 +#define ZMQ_REQ 3 +#define ZMQ_REP 4 +#define ZMQ_DEALER 5 +#define ZMQ_ROUTER 6 +#define ZMQ_PULL 7 +#define ZMQ_PUSH 8 +#define ZMQ_XPUB 9 +#define ZMQ_XSUB 10 +#define ZMQ_STREAM 11 + +/* Deprecated aliases */ +#define ZMQ_XREQ ZMQ_DEALER +#define ZMQ_XREP ZMQ_ROUTER + +/* Socket options. */ +#define ZMQ_AFFINITY 4 +#define ZMQ_ROUTING_ID 5 +#define ZMQ_SUBSCRIBE 6 +#define ZMQ_UNSUBSCRIBE 7 +#define ZMQ_RATE 8 +#define ZMQ_RECOVERY_IVL 9 +#define ZMQ_SNDBUF 11 +#define ZMQ_RCVBUF 12 +#define ZMQ_RCVMORE 13 +#define ZMQ_FD 14 +#define ZMQ_EVENTS 15 +#define ZMQ_TYPE 16 +#define ZMQ_LINGER 17 +#define ZMQ_RECONNECT_IVL 18 +#define ZMQ_BACKLOG 19 +#define ZMQ_RECONNECT_IVL_MAX 21 +#define ZMQ_MAXMSGSIZE 22 +#define ZMQ_SNDHWM 23 +#define ZMQ_RCVHWM 24 +#define ZMQ_MULTICAST_HOPS 25 +#define ZMQ_RCVTIMEO 27 +#define ZMQ_SNDTIMEO 28 +#define ZMQ_LAST_ENDPOINT 32 +#define ZMQ_ROUTER_MANDATORY 33 +#define ZMQ_TCP_KEEPALIVE 34 +#define ZMQ_TCP_KEEPALIVE_CNT 35 +#define ZMQ_TCP_KEEPALIVE_IDLE 36 +#define ZMQ_TCP_KEEPALIVE_INTVL 37 +#define ZMQ_IMMEDIATE 39 +#define ZMQ_XPUB_VERBOSE 40 +#define ZMQ_ROUTER_RAW 41 +#define ZMQ_IPV6 42 +#define ZMQ_MECHANISM 43 +#define ZMQ_PLAIN_SERVER 44 +#define ZMQ_PLAIN_USERNAME 45 +#define ZMQ_PLAIN_PASSWORD 46 +#define ZMQ_CURVE_SERVER 47 +#define ZMQ_CURVE_PUBLICKEY 48 +#define ZMQ_CURVE_SECRETKEY 49 +#define ZMQ_CURVE_SERVERKEY 50 +#define ZMQ_PROBE_ROUTER 51 +#define ZMQ_REQ_CORRELATE 52 +#define ZMQ_REQ_RELAXED 53 +#define ZMQ_CONFLATE 54 +#define ZMQ_ZAP_DOMAIN 55 +#define ZMQ_ROUTER_HANDOVER 56 +#define ZMQ_TOS 57 +#define ZMQ_CONNECT_ROUTING_ID 61 +#define ZMQ_GSSAPI_SERVER 62 +#define ZMQ_GSSAPI_PRINCIPAL 63 +#define ZMQ_GSSAPI_SERVICE_PRINCIPAL 64 +#define ZMQ_GSSAPI_PLAINTEXT 65 +#define ZMQ_HANDSHAKE_IVL 66 +#define ZMQ_SOCKS_PROXY 68 +#define ZMQ_XPUB_NODROP 69 +#define ZMQ_BLOCKY 70 +#define ZMQ_XPUB_MANUAL 71 +#define ZMQ_XPUB_WELCOME_MSG 72 +#define ZMQ_STREAM_NOTIFY 73 +#define ZMQ_INVERT_MATCHING 74 +#define ZMQ_HEARTBEAT_IVL 75 +#define ZMQ_HEARTBEAT_TTL 76 +#define ZMQ_HEARTBEAT_TIMEOUT 77 +#define ZMQ_XPUB_VERBOSER 78 +#define ZMQ_CONNECT_TIMEOUT 79 +#define ZMQ_TCP_MAXRT 80 +#define ZMQ_THREAD_SAFE 81 +#define ZMQ_MULTICAST_MAXTPDU 84 +#define ZMQ_VMCI_BUFFER_SIZE 85 +#define ZMQ_VMCI_BUFFER_MIN_SIZE 86 +#define ZMQ_VMCI_BUFFER_MAX_SIZE 87 +#define ZMQ_VMCI_CONNECT_TIMEOUT 88 +#define ZMQ_USE_FD 89 +#define ZMQ_GSSAPI_PRINCIPAL_NAMETYPE 90 +#define ZMQ_GSSAPI_SERVICE_PRINCIPAL_NAMETYPE 91 +#define ZMQ_BINDTODEVICE 92 + +/* Message options */ +#define ZMQ_MORE 1 +#define ZMQ_SHARED 3 + +/* Send/recv options. */ +#define ZMQ_DONTWAIT 1 +#define ZMQ_SNDMORE 2 + +/* Security mechanisms */ +#define ZMQ_NULL 0 +#define ZMQ_PLAIN 1 +#define ZMQ_CURVE 2 +#define ZMQ_GSSAPI 3 + +/* RADIO-DISH protocol */ +#define ZMQ_GROUP_MAX_LENGTH 255 + +/* Deprecated options and aliases */ +#define ZMQ_IDENTITY ZMQ_ROUTING_ID +#define ZMQ_CONNECT_RID ZMQ_CONNECT_ROUTING_ID +#define ZMQ_TCP_ACCEPT_FILTER 38 +#define ZMQ_IPC_FILTER_PID 58 +#define ZMQ_IPC_FILTER_UID 59 +#define ZMQ_IPC_FILTER_GID 60 +#define ZMQ_IPV4ONLY 31 +#define ZMQ_DELAY_ATTACH_ON_CONNECT ZMQ_IMMEDIATE +#define ZMQ_NOBLOCK ZMQ_DONTWAIT +#define ZMQ_FAIL_UNROUTABLE ZMQ_ROUTER_MANDATORY +#define ZMQ_ROUTER_BEHAVIOR ZMQ_ROUTER_MANDATORY + +/* Deprecated Message options */ +#define ZMQ_SRCFD 2 + +/******************************************************************************/ +/* GSSAPI definitions */ +/******************************************************************************/ + +/* GSSAPI principal name types */ +#define ZMQ_GSSAPI_NT_HOSTBASED 0 +#define ZMQ_GSSAPI_NT_USER_NAME 1 +#define ZMQ_GSSAPI_NT_KRB5_PRINCIPAL 2 + +/******************************************************************************/ +/* 0MQ socket events and monitoring */ +/******************************************************************************/ + +/* Socket transport events (TCP, IPC and TIPC only) */ + +#define ZMQ_EVENT_CONNECTED 0x0001 +#define ZMQ_EVENT_CONNECT_DELAYED 0x0002 +#define ZMQ_EVENT_CONNECT_RETRIED 0x0004 +#define ZMQ_EVENT_LISTENING 0x0008 +#define ZMQ_EVENT_BIND_FAILED 0x0010 +#define ZMQ_EVENT_ACCEPTED 0x0020 +#define ZMQ_EVENT_ACCEPT_FAILED 0x0040 +#define ZMQ_EVENT_CLOSED 0x0080 +#define ZMQ_EVENT_CLOSE_FAILED 0x0100 +#define ZMQ_EVENT_DISCONNECTED 0x0200 +#define ZMQ_EVENT_MONITOR_STOPPED 0x0400 +#define ZMQ_EVENT_ALL 0xFFFF +/* Unspecified system errors during handshake. Event value is an errno. */ +#define ZMQ_EVENT_HANDSHAKE_FAILED_NO_DETAIL 0x0800 +/* Handshake complete successfully with successful authentication (if * + * enabled). Event value is unused. */ +#define ZMQ_EVENT_HANDSHAKE_SUCCEEDED 0x1000 +/* Protocol errors between ZMTP peers or between server and ZAP handler. * + * Event value is one of ZMQ_PROTOCOL_ERROR_* */ +#define ZMQ_EVENT_HANDSHAKE_FAILED_PROTOCOL 0x2000 +/* Failed authentication requests. Event value is the numeric ZAP status * + * code, i.e. 300, 400 or 500. */ +#define ZMQ_EVENT_HANDSHAKE_FAILED_AUTH 0x4000 +#define ZMQ_PROTOCOL_ERROR_ZMTP_UNSPECIFIED 0x10000000 +#define ZMQ_PROTOCOL_ERROR_ZMTP_UNEXPECTED_COMMAND 0x10000001 +#define ZMQ_PROTOCOL_ERROR_ZMTP_INVALID_SEQUENCE 0x10000002 +#define ZMQ_PROTOCOL_ERROR_ZMTP_KEY_EXCHANGE 0x10000003 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_UNSPECIFIED 0x10000011 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_MESSAGE 0x10000012 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_HELLO 0x10000013 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_INITIATE 0x10000014 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_ERROR 0x10000015 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_READY 0x10000016 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MALFORMED_COMMAND_WELCOME 0x10000017 +#define ZMQ_PROTOCOL_ERROR_ZMTP_INVALID_METADATA 0x10000018 +// the following two may be due to erroneous configuration of a peer +#define ZMQ_PROTOCOL_ERROR_ZMTP_CRYPTOGRAPHIC 0x11000001 +#define ZMQ_PROTOCOL_ERROR_ZMTP_MECHANISM_MISMATCH 0x11000002 +#define ZMQ_PROTOCOL_ERROR_ZAP_UNSPECIFIED 0x20000000 +#define ZMQ_PROTOCOL_ERROR_ZAP_MALFORMED_REPLY 0x20000001 +#define ZMQ_PROTOCOL_ERROR_ZAP_BAD_REQUEST_ID 0x20000002 +#define ZMQ_PROTOCOL_ERROR_ZAP_BAD_VERSION 0x20000003 +#define ZMQ_PROTOCOL_ERROR_ZAP_INVALID_STATUS_CODE 0x20000004 +#define ZMQ_PROTOCOL_ERROR_ZAP_INVALID_METADATA 0x20000005 +#define ZMQ_PROTOCOL_ERROR_WS_UNSPECIFIED 0x30000000 + +ZMQ_EXPORT void *zmq_socket (void *, int type_); +ZMQ_EXPORT int zmq_close (void *s_); +ZMQ_EXPORT int +zmq_setsockopt (void *s_, int option_, const void *optval_, size_t optvallen_); +ZMQ_EXPORT int +zmq_getsockopt (void *s_, int option_, void *optval_, size_t *optvallen_); +ZMQ_EXPORT int zmq_bind (void *s_, const char *addr_); +ZMQ_EXPORT int zmq_connect (void *s_, const char *addr_); +ZMQ_EXPORT int zmq_unbind (void *s_, const char *addr_); +ZMQ_EXPORT int zmq_disconnect (void *s_, const char *addr_); +ZMQ_EXPORT int zmq_send (void *s_, const void *buf_, size_t len_, int flags_); +ZMQ_EXPORT int +zmq_send_const (void *s_, const void *buf_, size_t len_, int flags_); +ZMQ_EXPORT int zmq_recv (void *s_, void *buf_, size_t len_, int flags_); +ZMQ_EXPORT int zmq_socket_monitor (void *s_, const char *addr_, int events_); + +/******************************************************************************/ +/* Hide socket fd type; this was before zmq_poller_event_t typedef below */ +/******************************************************************************/ + +#if defined _WIN32 +// Windows uses a pointer-sized unsigned integer to store the socket fd. +#if defined _WIN64 +typedef unsigned __int64 zmq_fd_t; +#else +typedef unsigned int zmq_fd_t; +#endif +#else +typedef int zmq_fd_t; +#endif + +/******************************************************************************/ +/* Deprecated I/O multiplexing. Prefer using zmq_poller API */ +/******************************************************************************/ + +#define ZMQ_POLLIN 1 +#define ZMQ_POLLOUT 2 +#define ZMQ_POLLERR 4 +#define ZMQ_POLLPRI 8 + +typedef struct zmq_pollitem_t +{ + void *socket; + zmq_fd_t fd; + short events; + short revents; +} zmq_pollitem_t; + +#define ZMQ_POLLITEMS_DFLT 16 + +ZMQ_EXPORT int zmq_poll (zmq_pollitem_t *items_, int nitems_, long timeout_); + +/******************************************************************************/ +/* Message proxying */ +/******************************************************************************/ + +ZMQ_EXPORT int zmq_proxy (void *frontend_, void *backend_, void *capture_); +ZMQ_EXPORT int zmq_proxy_steerable (void *frontend_, + void *backend_, + void *capture_, + void *control_); + +/******************************************************************************/ +/* Probe library capabilities */ +/******************************************************************************/ + +#define ZMQ_HAS_CAPABILITIES 1 +ZMQ_EXPORT int zmq_has (const char *capability_); + +/* Deprecated aliases */ +#define ZMQ_STREAMER 1 +#define ZMQ_FORWARDER 2 +#define ZMQ_QUEUE 3 + +/* Deprecated methods */ +ZMQ_EXPORT int zmq_device (int type_, void *frontend_, void *backend_); +ZMQ_EXPORT int zmq_sendmsg (void *s_, zmq_msg_t *msg_, int flags_); +ZMQ_EXPORT int zmq_recvmsg (void *s_, zmq_msg_t *msg_, int flags_); +struct iovec; +ZMQ_EXPORT int +zmq_sendiov (void *s_, struct iovec *iov_, size_t count_, int flags_); +ZMQ_EXPORT int +zmq_recviov (void *s_, struct iovec *iov_, size_t *count_, int flags_); + +/******************************************************************************/ +/* Encryption functions */ +/******************************************************************************/ + +/* Encode data with Z85 encoding. Returns encoded data */ +ZMQ_EXPORT char * +zmq_z85_encode (char *dest_, const uint8_t *data_, size_t size_); + +/* Decode data with Z85 encoding. Returns decoded data */ +ZMQ_EXPORT uint8_t *zmq_z85_decode (uint8_t *dest_, const char *string_); + +/* Generate z85-encoded public and private keypair with libsodium. */ +/* Returns 0 on success. */ +ZMQ_EXPORT int zmq_curve_keypair (char *z85_public_key_, char *z85_secret_key_); + +/* Derive the z85-encoded public key from the z85-encoded secret key. */ +/* Returns 0 on success. */ +ZMQ_EXPORT int zmq_curve_public (char *z85_public_key_, + const char *z85_secret_key_); + +/******************************************************************************/ +/* Atomic utility methods */ +/******************************************************************************/ + +ZMQ_EXPORT void *zmq_atomic_counter_new (void); +ZMQ_EXPORT void zmq_atomic_counter_set (void *counter_, int value_); +ZMQ_EXPORT int zmq_atomic_counter_inc (void *counter_); +ZMQ_EXPORT int zmq_atomic_counter_dec (void *counter_); +ZMQ_EXPORT int zmq_atomic_counter_value (void *counter_); +ZMQ_EXPORT void zmq_atomic_counter_destroy (void **counter_p_); + +/******************************************************************************/ +/* Scheduling timers */ +/******************************************************************************/ + +#define ZMQ_HAVE_TIMERS + +typedef void (zmq_timer_fn) (int timer_id, void *arg); + +ZMQ_EXPORT void *zmq_timers_new (void); +ZMQ_EXPORT int zmq_timers_destroy (void **timers_p); +ZMQ_EXPORT int +zmq_timers_add (void *timers, size_t interval, zmq_timer_fn handler, void *arg); +ZMQ_EXPORT int zmq_timers_cancel (void *timers, int timer_id); +ZMQ_EXPORT int +zmq_timers_set_interval (void *timers, int timer_id, size_t interval); +ZMQ_EXPORT int zmq_timers_reset (void *timers, int timer_id); +ZMQ_EXPORT long zmq_timers_timeout (void *timers); +ZMQ_EXPORT int zmq_timers_execute (void *timers); + + +/******************************************************************************/ +/* These functions are not documented by man pages -- use at your own risk. */ +/* If you need these to be part of the formal ZMQ API, then (a) write a man */ +/* page, and (b) write a test case in tests. */ +/******************************************************************************/ + +/* Helper functions are used by perf tests so that they don't have to care */ +/* about minutiae of time-related functions on different OS platforms. */ + +/* Starts the stopwatch. Returns the handle to the watch. */ +ZMQ_EXPORT void *zmq_stopwatch_start (void); + +/* Returns the number of microseconds elapsed since the stopwatch was */ +/* started, but does not stop or deallocate the stopwatch. */ +ZMQ_EXPORT unsigned long zmq_stopwatch_intermediate (void *watch_); + +/* Stops the stopwatch. Returns the number of microseconds elapsed since */ +/* the stopwatch was started, and deallocates that watch. */ +ZMQ_EXPORT unsigned long zmq_stopwatch_stop (void *watch_); + +/* Sleeps for specified number of seconds. */ +ZMQ_EXPORT void zmq_sleep (int seconds_); + +typedef void (zmq_thread_fn) (void *); + +/* Start a thread. Returns a handle to the thread. */ +ZMQ_EXPORT void *zmq_threadstart (zmq_thread_fn *func_, void *arg_); + +/* Wait for thread to complete then free up resources. */ +ZMQ_EXPORT void zmq_threadclose (void *thread_); + + +/******************************************************************************/ +/* These functions are DRAFT and disabled in stable releases, and subject to */ +/* change at ANY time until declared stable. */ +/******************************************************************************/ + +#ifdef ZMQ_BUILD_DRAFT_API + +/* DRAFT Socket types. */ +#define ZMQ_SERVER 12 +#define ZMQ_CLIENT 13 +#define ZMQ_RADIO 14 +#define ZMQ_DISH 15 +#define ZMQ_GATHER 16 +#define ZMQ_SCATTER 17 +#define ZMQ_DGRAM 18 +#define ZMQ_PEER 19 +#define ZMQ_CHANNEL 20 + +/* DRAFT Socket options. */ +#define ZMQ_ZAP_ENFORCE_DOMAIN 93 +#define ZMQ_LOOPBACK_FASTPATH 94 +#define ZMQ_METADATA 95 +#define ZMQ_MULTICAST_LOOP 96 +#define ZMQ_ROUTER_NOTIFY 97 +#define ZMQ_XPUB_MANUAL_LAST_VALUE 98 +#define ZMQ_SOCKS_USERNAME 99 +#define ZMQ_SOCKS_PASSWORD 100 +#define ZMQ_IN_BATCH_SIZE 101 +#define ZMQ_OUT_BATCH_SIZE 102 +#define ZMQ_WSS_KEY_PEM 103 +#define ZMQ_WSS_CERT_PEM 104 +#define ZMQ_WSS_TRUST_PEM 105 +#define ZMQ_WSS_HOSTNAME 106 +#define ZMQ_WSS_TRUST_SYSTEM 107 +#define ZMQ_ONLY_FIRST_SUBSCRIBE 108 +#define ZMQ_RECONNECT_STOP 109 +#define ZMQ_HELLO_MSG 110 +#define ZMQ_DISCONNECT_MSG 111 +#define ZMQ_PRIORITY 112 +#define ZMQ_BUSY_POLL 113 +#define ZMQ_HICCUP_MSG 114 +#define ZMQ_XSUB_VERBOSE_UNSUBSCRIBE 115 +#define ZMQ_TOPICS_COUNT 116 +#define ZMQ_NORM_MODE 117 +#define ZMQ_NORM_UNICAST_NACK 118 +#define ZMQ_NORM_BUFFER_SIZE 119 +#define ZMQ_NORM_SEGMENT_SIZE 120 +#define ZMQ_NORM_BLOCK_SIZE 121 +#define ZMQ_NORM_NUM_PARITY 122 +#define ZMQ_NORM_NUM_AUTOPARITY 123 +#define ZMQ_NORM_PUSH 124 + +/* DRAFT ZMQ_NORM_MODE options */ +#define ZMQ_NORM_FIXED 0 +#define ZMQ_NORM_CC 1 +#define ZMQ_NORM_CCL 2 +#define ZMQ_NORM_CCE 3 +#define ZMQ_NORM_CCE_ECNONLY 4 + +/* DRAFT ZMQ_RECONNECT_STOP options */ +#define ZMQ_RECONNECT_STOP_CONN_REFUSED 0x1 +#define ZMQ_RECONNECT_STOP_HANDSHAKE_FAILED 0x2 +#define ZMQ_RECONNECT_STOP_AFTER_DISCONNECT 0x4 + +/* DRAFT Context options */ +#define ZMQ_ZERO_COPY_RECV 10 + +/* DRAFT Context methods. */ +ZMQ_EXPORT int zmq_ctx_set_ext (void *context_, + int option_, + const void *optval_, + size_t optvallen_); +ZMQ_EXPORT int zmq_ctx_get_ext (void *context_, + int option_, + void *optval_, + size_t *optvallen_); + +/* DRAFT Socket methods. */ +ZMQ_EXPORT int zmq_join (void *s, const char *group); +ZMQ_EXPORT int zmq_leave (void *s, const char *group); +ZMQ_EXPORT uint32_t zmq_connect_peer (void *s_, const char *addr_); + +/* DRAFT Msg methods. */ +ZMQ_EXPORT int zmq_msg_set_routing_id (zmq_msg_t *msg, uint32_t routing_id); +ZMQ_EXPORT uint32_t zmq_msg_routing_id (zmq_msg_t *msg); +ZMQ_EXPORT int zmq_msg_set_group (zmq_msg_t *msg, const char *group); +ZMQ_EXPORT const char *zmq_msg_group (zmq_msg_t *msg); +ZMQ_EXPORT int +zmq_msg_init_buffer (zmq_msg_t *msg_, const void *buf_, size_t size_); + +/* DRAFT Msg property names. */ +#define ZMQ_MSG_PROPERTY_ROUTING_ID "Routing-Id" +#define ZMQ_MSG_PROPERTY_SOCKET_TYPE "Socket-Type" +#define ZMQ_MSG_PROPERTY_USER_ID "User-Id" +#define ZMQ_MSG_PROPERTY_PEER_ADDRESS "Peer-Address" + +/* Router notify options */ +#define ZMQ_NOTIFY_CONNECT 1 +#define ZMQ_NOTIFY_DISCONNECT 2 + +/******************************************************************************/ +/* Poller polling on sockets,fd and thread-safe sockets */ +/******************************************************************************/ + +#define ZMQ_HAVE_POLLER + +typedef struct zmq_poller_event_t +{ + void *socket; + zmq_fd_t fd; + void *user_data; + short events; +} zmq_poller_event_t; + +ZMQ_EXPORT void *zmq_poller_new (void); +ZMQ_EXPORT int zmq_poller_destroy (void **poller_p); +ZMQ_EXPORT int zmq_poller_size (void *poller); +ZMQ_EXPORT int +zmq_poller_add (void *poller, void *socket, void *user_data, short events); +ZMQ_EXPORT int zmq_poller_modify (void *poller, void *socket, short events); +ZMQ_EXPORT int zmq_poller_remove (void *poller, void *socket); +ZMQ_EXPORT int +zmq_poller_wait (void *poller, zmq_poller_event_t *event, long timeout); +ZMQ_EXPORT int zmq_poller_wait_all (void *poller, + zmq_poller_event_t *events, + int n_events, + long timeout); +ZMQ_EXPORT int zmq_poller_fd (void *poller, zmq_fd_t *fd); + +ZMQ_EXPORT int +zmq_poller_add_fd (void *poller, zmq_fd_t fd, void *user_data, short events); +ZMQ_EXPORT int zmq_poller_modify_fd (void *poller, zmq_fd_t fd, short events); +ZMQ_EXPORT int zmq_poller_remove_fd (void *poller, zmq_fd_t fd); + +ZMQ_EXPORT int zmq_socket_get_peer_state (void *socket, + const void *routing_id, + size_t routing_id_size); + +/* DRAFT Socket monitoring events */ +#define ZMQ_EVENT_PIPES_STATS 0x10000 + +#define ZMQ_CURRENT_EVENT_VERSION 1 +#define ZMQ_CURRENT_EVENT_VERSION_DRAFT 2 + +#define ZMQ_EVENT_ALL_V1 ZMQ_EVENT_ALL +#define ZMQ_EVENT_ALL_V2 ZMQ_EVENT_ALL_V1 | ZMQ_EVENT_PIPES_STATS + +ZMQ_EXPORT int zmq_socket_monitor_versioned ( + void *s_, const char *addr_, uint64_t events_, int event_version_, int type_); +ZMQ_EXPORT int zmq_socket_monitor_pipes_stats (void *s); + +#if !defined _WIN32 +ZMQ_EXPORT int zmq_ppoll (zmq_pollitem_t *items_, + int nitems_, + long timeout_, + const sigset_t *sigmask_); +#else +// Windows has no sigset_t +ZMQ_EXPORT int zmq_ppoll (zmq_pollitem_t *items_, + int nitems_, + long timeout_, + const void *sigmask_); +#endif + +#endif // ZMQ_BUILD_DRAFT_API + + +#undef ZMQ_EXPORT + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ext_components/StackFlow/stackflow/libzmq/zmq_utils.h b/ext_components/StackFlow/stackflow/libzmq/zmq_utils.h new file mode 100644 index 0000000..c409406 --- /dev/null +++ b/ext_components/StackFlow/stackflow/libzmq/zmq_utils.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MPL-2.0 */ + +/* This file is deprecated, and all its functionality provided by zmq.h */ +/* Note that -Wpedantic compilation requires GCC to avoid using its custom + extensions such as #warning, hence the trick below. Also, pragmas for + warnings or other messages are not standard, not portable, and not all + compilers even have an equivalent concept. + So in the worst case, this include file is treated as silently empty. */ + +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) \ + || defined(_MSC_VER) +#if defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic push +#pragma GCC diagnostic warning "-Wcpp" +#pragma GCC diagnostic ignored "-Werror" +#pragma GCC diagnostic ignored "-Wall" +#endif +#pragma message( \ + "Warning: zmq_utils.h is deprecated. All its functionality is provided by zmq.h.") +#if defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic pop +#endif +#endif diff --git a/ext_components/StackFlow/stackflow/pzmq.hpp b/ext_components/StackFlow/stackflow/pzmq.hpp new file mode 100644 index 0000000..b334097 --- /dev/null +++ b/ext_components/StackFlow/stackflow/pzmq.hpp @@ -0,0 +1,288 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#define ZMQ_RPC_FUN (ZMQ_REP | 0x80) +#define ZMQ_RPC_CALL (ZMQ_REQ | 0x80) +namespace StackFlows { +class pzmq { + private: + const std::string unix_socket_head_ = "ipc://"; + const std::string rpc_url_head_ = "ipc:///tmp/rpc."; + void *zmq_ctx_; + void *zmq_socket_; + std::unordered_map> zmq_fun_; + std::atomic flage_; + std::unique_ptr zmq_thread_; + int mode_; + std::string rpc_server_; + std::string zmq_url_; + int timeout_; + + public: + pzmq() : zmq_ctx_(NULL), zmq_socket_(NULL), flage_(true), timeout_(3000) { + } + pzmq(const std::string &server) + : zmq_ctx_(NULL), zmq_socket_(NULL), rpc_server_(server), flage_(true), timeout_(3000) { + } + pzmq(const std::string &url, int mode, const std::function &raw_call = nullptr) + : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000) { + if (mode_ != ZMQ_RPC_FUN) creat(url, raw_call); + } + void set_timeout(int ms) { + timeout_ = ms; + } + int get_timeout() { + return timeout_; + } + int register_rpc_action(const std::string &action, + const std::function &raw_call) { + if (zmq_fun_.find(action) != zmq_fun_.end()) { + zmq_fun_[action] = raw_call; + return 0; + } + std::string url = rpc_url_head_ + rpc_server_; + if (!flage_) { + flage_ = true; + zmq_ctx_shutdown(zmq_ctx_); + zmq_thread_->join(); + close_zmq(); + } + zmq_fun_[action] = raw_call; + mode_ = ZMQ_RPC_FUN; + return creat(url); + } + int call_rpc_action(const std::string &action, const std::string &data, + const std::function &raw_call) { + int ret; + zmq_msg_t msg; + zmq_msg_init(&msg); + try { + if (NULL == zmq_socket_) { + if (rpc_server_.empty()) return -1; + std::string url = rpc_url_head_ + rpc_server_; + mode_ = ZMQ_RPC_CALL; + ret = creat(url); + if (ret) { + throw ret; + } + } + // requist + { + zmq_send(zmq_socket_, action.c_str(), action.length(), ZMQ_SNDMORE); + zmq_send(zmq_socket_, data.c_str(), data.length(), 0); + } + // action + { zmq_msg_recv(&msg, zmq_socket_, 0); } + raw_call(std::string((const char *)zmq_msg_data(&msg), zmq_msg_size(&msg))); + } catch (int e) { + ret = e; + } + zmq_msg_close(&msg); + close_zmq(); + return ret; + } + int creat(const std::string &url, const std::function &raw_call = nullptr) { + zmq_url_ = url; + do { + zmq_ctx_ = zmq_ctx_new(); + } while (zmq_ctx_ == NULL); + do { + zmq_socket_ = zmq_socket(zmq_ctx_, mode_ & 0x3f); + } while (zmq_socket_ == NULL); + + switch (mode_) { + case ZMQ_PUB: { + return creat_pub(url); + } break; + case ZMQ_SUB: { + int reconnect_interval = 100; + zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL, &reconnect_interval, sizeof(reconnect_interval)); + + int max_reconnect_interval = 1000; // 5 seconds + zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL_MAX, &max_reconnect_interval, + sizeof(max_reconnect_interval)); + return subscriber_url(url, raw_call); + } break; + case ZMQ_PUSH: { + int reconnect_interval = 100; + zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL, &reconnect_interval, sizeof(reconnect_interval)); + + int max_reconnect_interval = 1000; // 5 seconds + zmq_setsockopt(zmq_socket_, ZMQ_RECONNECT_IVL_MAX, &max_reconnect_interval, + sizeof(max_reconnect_interval)); + return creat_push(url); + } break; + case ZMQ_PULL: { + return creat_pull(url, raw_call); + } break; + case ZMQ_RPC_FUN: { + return creat_rep(url, raw_call); + } break; + case ZMQ_RPC_CALL: { + return creat_req(url); + } break; + default: + break; + } + return 0; + } + int check_zmq_errno(int code) { + if (code == -1) { + // SLOGE("Error occurred: %s", zmq_strerror(errno)); + switch (errno) { + case EAGAIN: // In non-blocking mode, the sending buffer is full, so the message cannot be sent + // immediately. + break; + case ENOTSUP: // The requested operation is not supported. + break; + case EFSM: // Inconsistent state machine with the socket. + break; + case ETERM: // The ZeroMQ context has been terminated. + break; + case ENOTSOCK: // The provided socket is invalid. + break; + case EINTR: // The operation was interrupted by a signal. + break; + case EFAULT: // Invalid buffer or socket. + break; + default: + break; + } + } + return code; + } + int send_data(const std::string &raw) { + return zmq_send(zmq_socket_, raw.c_str(), raw.length(), 0); + } + int send_data(const char *raw, int size) { + return zmq_send(zmq_socket_, raw, size, 0); + } + inline int creat_pub(const std::string &url) { + return zmq_bind(zmq_socket_, url.c_str()); + } + inline int creat_push(const std::string &url) { + int timeout = 3000; + zmq_setsockopt(zmq_socket_, ZMQ_SNDTIMEO, &timeout, sizeof(timeout)); + zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout, sizeof(timeout)); + return zmq_connect(zmq_socket_, url.c_str()); + } + inline int creat_pull(const std::string &url, const std::function &raw_call) { + int ret = zmq_bind(zmq_socket_, url.c_str()); + flage_ = false; + zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); + return ret; + } + inline int subscriber_url(const std::string &url, const std::function &raw_call) { + int ret = zmq_connect(zmq_socket_, url.c_str()); + zmq_setsockopt(zmq_socket_, ZMQ_SUBSCRIBE, "", 0); + flage_ = false; + zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); + return ret; + } + inline int creat_rep(const std::string &url, const std::function &raw_call) { + int ret = zmq_bind(zmq_socket_, url.c_str()); + flage_ = false; + zmq_thread_ = std::make_unique(std::bind(&pzmq::zmq_event_loop, this, raw_call)); + return ret; + } + inline int creat_req(const std::string &url) { + int pos = url.find(unix_socket_head_); + if (pos != std::string::npos) { + std::string socket_file = url.substr(pos + unix_socket_head_.length()); + if (access(socket_file.c_str(), F_OK) != 0) { + return -1; + } + } + zmq_setsockopt(zmq_socket_, ZMQ_SNDTIMEO, &timeout_, sizeof(timeout_)); + zmq_setsockopt(zmq_socket_, ZMQ_RCVTIMEO, &timeout_, sizeof(timeout_)); + return zmq_connect(zmq_socket_, url.c_str()); + } + void zmq_event_loop(const std::function &raw_call) { + int ret; + zmq_pollitem_t items[1]; + if (mode_ == ZMQ_PULL) { + items[0].socket = zmq_socket_; + items[0].fd = 0; + items[0].events = ZMQ_POLLIN; + items[0].revents = 0; + }; + while (!flage_.load()) { + zmq_msg_t msg; + zmq_msg_init(&msg); + if (mode_ == ZMQ_PULL) { + ret = zmq_poll(items, 1, -1); + if (ret == -1) { + zmq_close(zmq_socket_); + continue; + } + if (!(items[0].revents & ZMQ_POLLIN)) { + continue; + } + } + // SLOGI("zmq_msg_recv"); + ret = zmq_msg_recv(&msg, zmq_socket_, 0); + if (ret <= 0) { + // SLOGE("zmq_connect false:%s", zmq_strerror(zmq_errno())); + zmq_msg_close(&msg); + continue; + } + std::string raw_data((const char *)zmq_msg_data(&msg), zmq_msg_size(&msg)); + if (mode_ == ZMQ_RPC_FUN) { + zmq_msg_t msg1; + zmq_msg_init(&msg1); + zmq_msg_recv(&msg1, zmq_socket_, 0); + std::string _raw_data((const char *)zmq_msg_data(&msg1), zmq_msg_size(&msg1)); + std::string retval; + try { + retval = zmq_fun_.at(raw_data)(_raw_data); + } catch (...) { + retval = "NotAction"; + } + zmq_send(zmq_socket_, retval.c_str(), retval.length(), 0); + zmq_msg_close(&msg1); + } else { + // SLOGI("zmq_msg:%s", zmq_msg_data(&msg)); + raw_call(raw_data); + } + zmq_msg_close(&msg); + } + } + void close_zmq() { + zmq_close(zmq_socket_); + zmq_ctx_destroy(zmq_ctx_); + int pos = zmq_url_.find(unix_socket_head_); + if ((mode_ == ZMQ_PUB) || (mode_ == ZMQ_PULL) || (mode_ == ZMQ_RPC_FUN)) { + if (pos != std::string::npos) { + std::string socket_file = zmq_url_.substr(pos + unix_socket_head_.length()); + if (access(socket_file.c_str(), F_OK) == 0) { + remove(socket_file.c_str()); + } + } + } + zmq_socket_ = NULL; + zmq_ctx_ = NULL; + } + ~pzmq() { + if (!zmq_socket_) { + return; + } + if (zmq_thread_) { + flage_ = true; + zmq_ctx_shutdown(zmq_ctx_); + zmq_thread_->join(); + } + close_zmq(); + } +}; +}; // namespace StackFlows \ No newline at end of file diff --git a/ext_components/ax_msp/Kconfig b/ext_components/ax_msp/Kconfig new file mode 100644 index 0000000..88d6048 --- /dev/null +++ b/ext_components/ax_msp/Kconfig @@ -0,0 +1,19 @@ +menuconfig AX_MSP_ENABLED + bool "Enable ax_msp support" + default n + help + enable axera bsp support + + choice + prompt "choice board" + default AX_620E_MSP_ENABLED + depends on AX_MSP_ENABLED + config AX_620E_MSP_ENABLED + bool "enable ax620e bsp" + help + build by AXERA! + config AX_520_MSP_ENABLED + bool "enable ax520 bsp" + help + build by AXERA! + endchoice \ No newline at end of file diff --git a/ext_components/ax_msp/SConstruct b/ext_components/ax_msp/SConstruct new file mode 100644 index 0000000..59678ea --- /dev/null +++ b/ext_components/ax_msp/SConstruct @@ -0,0 +1,86 @@ +# component2/SConscript +Import('env') +import os +with open(env['PROJECT_TOOL_S']) as f: + exec(f.read()) + +if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: + MSP_PATH = os.path.join(os.environ['GIT_REPO_PATH'], 'm5stack_msp') + if 'MSP_PATH' in os.environ: + MSP_PATH = os.environ['MSP_PATH'] + else: + if not os.path.exists(MSP_PATH): + zip_file = MSP_PATH + '.tar.gz' + zip_file_extrpath = MSP_PATH + '_tmp' + down_url = "https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_msp.tar.gz" + + if 'CONFIG_REPO_AUTOMATION' in os.environ: + down = 'y' + else: + down = input('{} does not exist. Please choose whether to download it automatically? Y/N :'.format('m5stack_msp.tar.gz')) + down = down.lower() + if down == 'y': + # from git import Repo + import requests + import parse + import tarfile + import shutil + try: + # Downloading via HTTP (more common) + if not os.path.exists(zip_file): + response = requests.get(down_url) + if response.status_code == 200: + with open(zip_file, 'wb') as file: + file.write(response.content) + else: + env.Fatal("{} down failed".format(down_url)) + with tarfile.open(zip_file, 'r:gz') as tar: + tar.extractall(path=zip_file_extrpath) + shutil.move(zip_file_extrpath, MSP_PATH) + # shutil.rmtree(zip_file_extrpath) + print("The {} download successful.".format(down_url)) + except Exception as e: + print('Please manually download {} to {} .'.format(down_url, zip_file)) + env.Fatal("Cloning failed.: {}".format(e)) + else: + env.Fatal('Please manually download {} to {} .'.format(down_url, zip_file)) + SRCS=[] + INCLUDE=[] + PRIVATE_INCLUDE=[] + REQUIREMENTS=[] + STATIC_LIB=[] + DYNAMIC_LIB=[] + DEFINITIONS=[] + DEFINITIONS_PRIVATE=[''] + LDFLAGS=[] + LINK_SEARCH_PATH=[] + + INCLUDE += [os.path.join(MSP_PATH, 'out/arm64_glibc/include'), os.path.join(MSP_PATH, 'sample/common')] + LINK_SEARCH_PATH += [os.path.join(MSP_PATH, 'out/arm64_glibc/lib')] + REQUIREMENTS += ['dl', 'm', 'pthread'] + + # REQUIREMENTS += ['ax_sys', 'ax_mipi', 'ax_proton', 'ax_engine','ax_interpreter','ax_ae','ax_awb','ax_af', + # 'ax_nt_stream', 'ax_venc', 'ax_ivps', 'ax_vdec', 'ax_nt_stream', 'ax_nt_ctrl', 'ax_ive', 'ax_audio', 'ax_opal', 'ax_fdk', + # 'ax_ives', 'ax_skel', 'ax_audio_3a', 'ax_opus', 'ax_opal', 'ax_audio'] + + third_party = ['cmdline','drm','faac','fdk-aac','ffmpeg','libsamplerate','live','mp4','opencv','openssl','optee','opus','swupdate','tinyalsa'] + for dirn in third_party: + INCLUDE.append(os.path.join(MSP_PATH,'third-party',dirn,'include')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party',dirn,'lib/arm64/glibc')) + + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'app/lib')) + + env['COMPONENTS'].append({'target':os.path.basename(env['component_dir']), + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'REGISTER':'static' + }) +