diff --git a/examples/KWS_ASR/KWS_ASR.ino b/examples/KWS_ASR/KWS_ASR.ino index 0633850..dac124e 100644 --- a/examples/KWS_ASR/KWS_ASR.ino +++ b/examples/KWS_ASR/KWS_ASR.ino @@ -12,15 +12,20 @@ M5ModuleLLM module_llm; /* Must be capitalized */ String wake_up_keyword = "HELLO"; - +// String wake_up_keyword = "你好你好"; String kws_work_id; String asr_work_id; +String language; void setup() { M5.begin(); M5.Display.setTextSize(2); M5.Display.setTextScroll(true); + // M5.Display.setFont(&fonts::efontCN_12); // Support Chinese display + + language = "en_US"; + // language = "zh_CN"; /* Init module serial port */ Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic @@ -50,11 +55,13 @@ void setup() M5.Display.printf(">> Setup kws..\n"); m5_module_llm::ApiKwsSetupConfig_t kws_config; kws_config.kws = wake_up_keyword; - kws_work_id = module_llm.kws.setup(kws_config); + kws_work_id = module_llm.kws.setup(kws_config, "kws_setup", language); /* Setup ASR module and save returned work id */ M5.Display.printf(">> Setup asr..\n"); - asr_work_id = module_llm.asr.setup(); + m5_module_llm::ApiAsrSetupConfig_t asr_config; + asr_config.input = {"sys.pcm", kws_work_id}; + asr_work_id = module_llm.asr.setup(asr_config, "asr_setup", language); M5.Display.printf(">> Setup ok\n>> Say \"%s\" to wakeup\n", wake_up_keyword.c_str()); } diff --git a/examples/SerialTextAssistant/SerialTextAssistant.ino b/examples/SerialTextAssistant/SerialTextAssistant.ino index d545c4d..2587018 100644 --- a/examples/SerialTextAssistant/SerialTextAssistant.ino +++ b/examples/SerialTextAssistant/SerialTextAssistant.ino @@ -19,6 +19,7 @@ void setup() M5.begin(); M5.Display.setTextSize(2); M5.Display.setTextScroll(true); + // M5.Display.setFont(&fonts::efontCN_12); // Support Chinese display /* Init usb serial */ CommSerialPort.begin(115200); @@ -45,7 +46,9 @@ void setup() /* Setup LLM module and save returned work id */ M5.Display.printf(">> Setup llm..\n"); - llm_work_id = module_llm.llm.setup(); + m5_module_llm::ApiLlmSetupConfig_t llm_config; + llm_config.max_token_len = 1023; + llm_work_id = module_llm.llm.setup(llm_config); M5.Display.printf(">> Setup finish\n"); M5.Display.printf(">> Try send your question via usb serial port\n"); diff --git a/examples/TTS/TTS.ino b/examples/TTS/TTS.ino index 909964f..73d536d 100644 --- a/examples/TTS/TTS.ino +++ b/examples/TTS/TTS.ino @@ -9,12 +9,17 @@ M5ModuleLLM module_llm; String tts_work_id; +String language; void setup() { M5.begin(); M5.Display.setTextSize(2); M5.Display.setTextScroll(true); + // M5.Display.setFont(&fonts::efontCN_12); // Support Chinese display + + language = "en_US"; + // language = "zh_CN"; /* Init module serial port */ Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic @@ -42,7 +47,8 @@ void setup() /* Setup TTS module and save returned work id */ M5.Display.printf(">> Setup tts..\n\n"); - tts_work_id = module_llm.tts.setup(); + m5_module_llm::ApiTtsSetupConfig_t tts_config; + tts_work_id = module_llm.tts.setup(tts_config, "tts_setup", language); } void loop() @@ -51,6 +57,7 @@ void loop() static int i = 0; i++; std::string text = std::to_string(i) + " plus " + std::to_string(i) + " equals " + std::to_string(i + i) + "."; + // std::string text = std::to_string(i) + " 加 " + std::to_string(i) + " 等于 " + std::to_string(i + i) + "."; M5.Display.setTextColor(TFT_GREEN); M5.Display.printf("<< %s\n\n", text.c_str()); diff --git a/examples/VoiceAssistant/VoiceAssistant.ino b/examples/VoiceAssistant/VoiceAssistant.ino index b4ddc77..3a4979b 100644 --- a/examples/VoiceAssistant/VoiceAssistant.ino +++ b/examples/VoiceAssistant/VoiceAssistant.ino @@ -14,6 +14,7 @@ M5ModuleLLM_VoiceAssistant voice_assistant(&module_llm); void on_asr_data_input(String data, bool isFinish, int index) { M5.Display.setTextColor(TFT_GREEN, TFT_BLACK); + // M5.Display.setFont(&fonts::efontCN_12); // Support Chinese display M5.Display.printf(">> %s\n", data.c_str()); /* If ASR data is finish */ @@ -59,6 +60,7 @@ void setup() /* Begin voice assistant preset */ M5.Display.printf(">> Begin voice assistant..\n"); int ret = voice_assistant.begin("HELLO"); + // int ret = voice_assistant.begin("你好你好", "", "zh_CN"); // Chinese kws and asr if (ret != MODULE_LLM_OK) { while (1) { M5.Display.setTextColor(TFT_RED); diff --git a/examples/YOLO/YOLO.ino b/examples/YOLO/YOLO.ino new file mode 100644 index 0000000..2be0ddf --- /dev/null +++ b/examples/YOLO/YOLO.ino @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include +#include +#include + +M5ModuleLLM module_llm; +String camera_work_id; +String yolo_work_id; + +void setup() +{ + M5.begin(); + M5.Display.setTextSize(2); + M5.Display.setTextScroll(true); + + /* Init module serial port */ + Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic + // Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2 + // Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3 + + /* Init module */ + module_llm.begin(&Serial2); + + /* Make sure module is connected */ + M5.Display.printf(">> Check ModuleLLM connection..\n"); + while (1) { + if (module_llm.checkConnection()) { + break; + } + } + + /* Reset ModuleLLM */ + M5.Display.printf(">> Reset ModuleLLM..\n"); + module_llm.sys.reset(); + + /* Setup Camera module */ + M5.Display.printf(">> Setup camera..\n"); + camera_work_id = module_llm.camera.setup(); + + /* Setup YOLO module and save returned work id */ + M5.Display.printf(">> Setup yolo..\n"); + m5_module_llm::ApiYoloSetupConfig_t yolo_config; + yolo_config.input = {camera_work_id}; + yolo_work_id = module_llm.yolo.setup(yolo_config, "yolo_setup"); + // M5.Display.printf(">> Yolo ready\n"); + M5.Display.drawString("class", 10, 80); + M5.Display.drawString("confidence", 180, 80); + M5.Display.drawString("x1", 10, 110); + M5.Display.drawString("y1", 10, 140); + M5.Display.drawString("x2", 10, 170); + M5.Display.drawString("y2", 10, 200); +} + +void loop() +{ + /* Update ModuleLLM */ + module_llm.update(); + + /* Handle module response messages */ + for (auto& msg : module_llm.msg.responseMsgList) { + /* If YOLO module message */ + if (msg.work_id == yolo_work_id) { + /* Check message object type */ + if (msg.object == "yolo.box.stream") { + /* Parse message json and get YOLO result */ + JsonDocument doc; + deserializeJson(doc, msg.raw_msg); + JsonArray delta = doc["data"]["delta"].as(); + + if (delta.size() > 0) { + JsonObject result = delta[0].as(); + String class_name = result["class"].as(); + float confidence = result["confidence"].as(); + JsonArray bboxArray = result["bbox"].as(); + + if (bboxArray.size() == 4) { + int x1 = bboxArray[0].as(); + int y1 = bboxArray[1].as(); + int x2 = bboxArray[2].as(); + int y2 = bboxArray[3].as(); + M5.Display.drawString(class_name, 80, 80); + M5.Display.drawFloat(confidence, 2, 200, 110); + M5.Display.drawNumber(x1, 40, 110); + M5.Display.drawNumber(y1, 40, 140); + M5.Display.drawNumber(x2, 40, 170); + M5.Display.drawNumber(y2, 40, 200); + } + } else { + M5.Display.drawString("None", 80, 80); + M5.Display.drawFloat(0, 2, 200, 110); + M5.Display.drawNumber(0, 40, 110); + M5.Display.drawNumber(0, 40, 140); + M5.Display.drawNumber(0, 40, 170); + M5.Display.drawNumber(0, 40, 200); + } + } + } + } + + /* Clear handled messages */ + module_llm.msg.responseMsgList.clear(); +} \ No newline at end of file diff --git a/src/M5ModuleLLM.cpp b/src/M5ModuleLLM.cpp index deb8426..0bac44b 100644 --- a/src/M5ModuleLLM.cpp +++ b/src/M5ModuleLLM.cpp @@ -14,14 +14,19 @@ bool M5ModuleLLM::begin(Stream* serialPort) llm.init(&msg); audio.init(&msg); tts.init(&msg); + melotts.init(&msg); kws.init(&msg); asr.init(&msg); + yolo.init(&msg); + camera.init(&msg); return true; } bool M5ModuleLLM::checkConnection() { - return sys.ping() == MODULE_LLM_OK; + const bool result = (sys.ping() == MODULE_LLM_OK); + llm_version = (sys.version() == MODULE_LLM_OK); + return result; } void M5ModuleLLM::update() diff --git a/src/M5ModuleLLM.h b/src/M5ModuleLLM.h index 9b2a113..42bae02 100644 --- a/src/M5ModuleLLM.h +++ b/src/M5ModuleLLM.h @@ -11,8 +11,12 @@ #include "api/api_llm.h" #include "api/api_audio.h" #include "api/api_tts.h" +#include "api/api_melotts.h" #include "api/api_kws.h" #include "api/api_asr.h" +#include "api/api_yolo.h" +#include "api/api_camera.h" +#include "api/api_version.h" class M5ModuleLLM { public: @@ -57,12 +61,24 @@ public: */ m5_module_llm::ApiAudio audio; + /** + * @brief Camera module api set + * + */ + m5_module_llm::ApiCamera camera; + /** * @brief TTS module api set * */ m5_module_llm::ApiTts tts; + /** + * @brief MELOTTS module api set + * + */ + m5_module_llm::ApiMelotts melotts; + /** * @brief KWS module api set * @@ -75,6 +91,12 @@ public: */ m5_module_llm::ApiAsr asr; + /** + * @brief YOLO module api set + * + */ + m5_module_llm::ApiYolo yolo; + /** * @brief MSG module to handle module response message * @@ -93,8 +115,10 @@ private: typedef std::function OnKeywordDetectedCallback_t; typedef std::function OnAsrDataInputCallback_t; typedef std::function OnLlmDataInputCallback_t; +typedef std::function OnYoloDataInputCallback_t; typedef std::function OnAsrDataInputRawCallback_t; typedef std::function OnLlmDataInputRawCallback_t; +typedef std::function OnYoloDataInputRawCallback_t; /** * @brief Voice assistant preset base on class M5ModuleLLM @@ -113,7 +137,7 @@ public: * @param prompt * @return int */ - int begin(String wakeUpKeyword = "HELLO", String prompt = ""); + int begin(String wakeUpKeyword = "HELLO", String prompt = "", String language = "en_US"); /** * @brief Update voice assistant preset, trigger callbacks @@ -163,6 +187,7 @@ private: String asr; String llm; String tts; + String melotts; }; WorkId_t _work_id; diff --git a/src/api/api_asr.cpp b/src/api/api_asr.cpp index 85283eb..900d531 100644 --- a/src/api/api_asr.cpp +++ b/src/api/api_asr.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: MIT */ #include "api_asr.h" +#include "api_version.h" using namespace m5_module_llm; @@ -12,7 +13,7 @@ void ApiAsr::init(ModuleMsg* moduleMsg) _module_msg = moduleMsg; } -String ApiAsr::setup(ApiAsrSetupConfig_t config, String request_id) +String ApiAsr::setup(ApiAsrSetupConfig_t config, String request_id, String language) { String cmd; { @@ -23,12 +24,20 @@ String ApiAsr::setup(ApiAsrSetupConfig_t config, String request_id) doc["object"] = "asr.setup"; doc["data"]["model"] = config.model; doc["data"]["response_format"] = config.response_format; - doc["data"]["input"] = config.input; doc["data"]["enoutput"] = config.enoutput; doc["data"]["enkws"] = config.enkws; doc["data"]["rule1"] = config.rule1; doc["data"]["rule2"] = config.rule2; doc["data"]["rule3"] = config.rule3; + if (!llm_version) { + doc["data"]["input"] = config.input[0]; + } else { + JsonArray inputArray = doc["data"]["input"].to(); + for (const String& str : config.input) { + inputArray.add(str); + } + } + if (language == "zh_CN") doc["data"]["model"] = "sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23"; serializeJson(doc, cmd); } diff --git a/src/api/api_asr.h b/src/api/api_asr.h index 87499f8..34d2364 100644 --- a/src/api/api_asr.h +++ b/src/api/api_asr.h @@ -10,14 +10,14 @@ namespace m5_module_llm { struct ApiAsrSetupConfig_t { - String model = "sherpa-ncnn-streaming-zipformer-20M-2023-02-17"; - String response_format = "asr.utf-8.stream"; - String input = "sys.pcm"; - bool enoutput = true; - bool enkws = true; - float rule1 = 2.4; - float rule2 = 1.2; - float rule3 = 30.0; + String model = "sherpa-ncnn-streaming-zipformer-20M-2023-02-17"; + String response_format = "asr.utf-8.stream"; + std::vector input = {"sys.pcm"}; + bool enoutput = true; + bool enkws = true; + float rule1 = 2.4; + float rule2 = 1.2; + float rule3 = 30.0; }; class ApiAsr { @@ -31,7 +31,8 @@ public: * @param request_id * @return String */ - String setup(ApiAsrSetupConfig_t config = ApiAsrSetupConfig_t(), String request_id = "asr_setup"); + String setup(ApiAsrSetupConfig_t config = ApiAsrSetupConfig_t(), String request_id = "asr_setup", + String language = "en_US"); private: ModuleMsg* _module_msg = nullptr; diff --git a/src/api/api_camera.cpp b/src/api/api_camera.cpp new file mode 100644 index 0000000..8b2c7da --- /dev/null +++ b/src/api/api_camera.cpp @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "api_camera.h" + +using namespace m5_module_llm; + +void ApiCamera::init(ModuleMsg* moduleMsg) +{ + _module_msg = moduleMsg; +} + +String ApiCamera::setup(ApiCameraSetupConfig_t config, String request_id) +{ + String cmd; + { + JsonDocument doc; + doc["request_id"] = request_id; + doc["work_id"] = "camera"; + doc["action"] = "setup"; + doc["object"] = "camera.setup"; + doc["data"]["response_format"] = config.response_format; + doc["data"]["input"] = config.input; + doc["data"]["enoutput"] = config.enoutput; + doc["data"]["frame_width"] = config.frame_width; + doc["data"]["frame_height"] = config.frame_height; + serializeJson(doc, cmd); + } + + String work_id; + _module_msg->sendCmdAndWaitToTakeMsg( + cmd.c_str(), request_id, + [&work_id](ResponseMsg_t& msg) { + // Copy work id + work_id = msg.work_id; + }, + 5000); + return work_id; +} diff --git a/src/api/api_camera.h b/src/api/api_camera.h new file mode 100644 index 0000000..f0d0ea3 --- /dev/null +++ b/src/api/api_camera.h @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once +#include "../utils/msg.h" +#include + +namespace m5_module_llm { + +struct ApiCameraSetupConfig_t { + String response_format = "camera.raw"; + String input = "/dev/video0"; + bool enoutput = false; + int frame_width = 320; + int frame_height = 320; +}; + +class ApiCamera { +public: + void init(ModuleMsg* moduleMsg); + + /** + * @brief Setup module camera, return work_id + * + * @param config + * @param request_id + * @return String + */ + String setup(ApiCameraSetupConfig_t config = ApiCameraSetupConfig_t(), String request_id = "camera_setup"); + +private: + ModuleMsg* _module_msg = nullptr; +}; + +} // namespace m5_module_llm diff --git a/src/api/api_kws.cpp b/src/api/api_kws.cpp index 47f2d32..07f417e 100644 --- a/src/api/api_kws.cpp +++ b/src/api/api_kws.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: MIT */ #include "api_kws.h" +#include "api_version.h" using namespace m5_module_llm; @@ -12,7 +13,7 @@ void ApiKws::init(ModuleMsg* moduleMsg) _module_msg = moduleMsg; } -String ApiKws::setup(ApiKwsSetupConfig_t config, String request_id) +String ApiKws::setup(ApiKwsSetupConfig_t config, String request_id, String language) { String cmd; { @@ -23,9 +24,17 @@ String ApiKws::setup(ApiKwsSetupConfig_t config, String request_id) doc["object"] = "kws.setup"; doc["data"]["model"] = config.model; doc["data"]["response_format"] = config.response_format; - doc["data"]["input"] = config.input; doc["data"]["enoutput"] = config.enoutput; doc["data"]["kws"] = config.kws; + if (!llm_version) { + doc["data"]["input"] = config.input[0]; + } else { + JsonArray inputArray = doc["data"]["input"].to(); + for (const String& str : config.input) { + inputArray.add(str); + } + } + if (language == "zh_CN") doc["data"]["model"] = "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01"; serializeJson(doc, cmd); } diff --git a/src/api/api_kws.h b/src/api/api_kws.h index 2e09553..6400eec 100644 --- a/src/api/api_kws.h +++ b/src/api/api_kws.h @@ -10,11 +10,11 @@ namespace m5_module_llm { struct ApiKwsSetupConfig_t { - String kws = "HELLO"; - String model = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; - String response_format = "kws.bool"; - String input = "sys.pcm"; - bool enoutput = true; + String kws = "HELLO"; + String model = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; + String response_format = "kws.bool"; + std::vector input = {"sys.pcm"}; + bool enoutput = true; }; class ApiKws { @@ -28,7 +28,8 @@ public: * @param request_id * @return String */ - String setup(ApiKwsSetupConfig_t config = ApiKwsSetupConfig_t(), String request_id = "kws_setup"); + String setup(ApiKwsSetupConfig_t config = ApiKwsSetupConfig_t(), String request_id = "kws_setup", + String language = "en_US"); private: ModuleMsg* _module_msg = nullptr; diff --git a/src/api/api_llm.cpp b/src/api/api_llm.cpp index e2f54ac..57b4ee9 100644 --- a/src/api/api_llm.cpp +++ b/src/api/api_llm.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: MIT */ #include "api_llm.h" +#include "api_version.h" using namespace m5_module_llm; @@ -23,11 +24,19 @@ String ApiLlm::setup(ApiLlmSetupConfig_t config, String request_id) doc["object"] = "llm.setup"; doc["data"]["model"] = config.model; doc["data"]["response_format"] = config.response_format; - doc["data"]["input"] = config.input; doc["data"]["enoutput"] = config.enoutput; doc["data"]["enkws"] = config.enkws; doc["data"]["max_token_len"] = config.max_token_len; doc["data"]["prompt"] = config.prompt; + if (!llm_version) { + doc["data"]["model"] = "qwen2.5-0.5b"; + doc["data"]["input"] = config.input[0]; + } else { + JsonArray inputArray = doc["data"]["input"].to(); + for (const String& str : config.input) { + inputArray.add(str); + } + } serializeJson(doc, cmd); } diff --git a/src/api/api_llm.h b/src/api/api_llm.h index 32c7eef..27536e9 100644 --- a/src/api/api_llm.h +++ b/src/api/api_llm.h @@ -11,12 +11,13 @@ namespace m5_module_llm { struct ApiLlmSetupConfig_t { String prompt; - String model = "qwen2.5-0.5b"; - String response_format = "llm.utf-8.stream"; - String input = "llm.utf-8.stream"; - bool enoutput = true; - bool enkws = true; - int max_token_len = 127; + String model = "qwen2.5-0.5B-prefill-20e"; + String response_format = "llm.utf-8.stream"; + std::vector input = {"llm.utf-8.stream"}; + bool enoutput = true; + bool enkws = true; + int max_token_len = 127; + // int max_token_len = 512; }; class ApiLlm { diff --git a/src/api/api_melotts.cpp b/src/api/api_melotts.cpp new file mode 100644 index 0000000..b4e6f8b --- /dev/null +++ b/src/api/api_melotts.cpp @@ -0,0 +1,76 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "api_melotts.h" + +using namespace m5_module_llm; + +void ApiMelotts::init(ModuleMsg* moduleMsg) +{ + _module_msg = moduleMsg; +} + +String ApiMelotts::setup(ApiMelottsSetupConfig_t config, String request_id, String language) +{ + String cmd; + { + JsonDocument doc; + doc["request_id"] = request_id; + doc["work_id"] = "melotts"; + doc["action"] = "setup"; + doc["object"] = "melotts.setup"; + doc["data"]["model"] = config.model; + doc["data"]["response_format"] = config.response_format; + JsonArray inputArray = doc["data"]["input"].to(); + for (const String& str : config.input) { + inputArray.add(str); + } + if (language == "zh_CN") doc["data"]["model"] = "melotts_zh-cn"; + doc["data"]["enoutput"] = config.enoutput; + doc["data"]["enaudio"] = config.enaudio; + serializeJson(doc, cmd); + } + + String work_id; + _module_msg->sendCmdAndWaitToTakeMsg( + cmd.c_str(), request_id, + [&work_id](ResponseMsg_t& msg) { + // Copy work id + work_id = msg.work_id; + }, + 15000); + return work_id; +} + +int ApiMelotts::inference(String work_id, String input, uint32_t timeout, String request_id) +{ + String cmd; + { + JsonDocument doc; + doc["request_id"] = request_id; + doc["work_id"] = work_id; + doc["action"] = "inference"; + doc["object"] = "tts.utf-8.stream"; + doc["data"]["delta"] = input; + doc["data"]["index"] = 0; + doc["data"]["finish"] = true; + serializeJson(doc, cmd); + } + + if (timeout == 0) { + _module_msg->sendCmd(cmd.c_str()); + return MODULE_LLM_OK; + } + + int ret = MODULE_LLM_WAIT_RESPONSE_TIMEOUT; + _module_msg->sendCmdAndWaitToTakeMsg( + cmd.c_str(), request_id, + [&ret](ResponseMsg_t& msg) { + // Copy error code + ret = msg.error.code; + }, + timeout); + return ret; +} diff --git a/src/api/api_melotts.h b/src/api/api_melotts.h new file mode 100644 index 0000000..7a35219 --- /dev/null +++ b/src/api/api_melotts.h @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once +#include "../utils/msg.h" +#include + +namespace m5_module_llm { + +struct ApiMelottsSetupConfig_t { + String model = "melotts_zh-cn"; + String response_format = "sys.pcm"; + std::vector input = {"tts.utf-8.stream"}; + bool enoutput = false; + bool enaudio = true; +}; + +class ApiMelotts { +public: + void init(ModuleMsg* moduleMsg); + + /** + * @brief Setup module TTS, return TTS work_id + * + * @param config + * @param request_id + * @return String + */ + String setup(ApiMelottsSetupConfig_t config = ApiMelottsSetupConfig_t(), String request_id = "melotts_setup", + String language = "en_US"); + + /** + * @brief Inference input data by TTS module + * + * @param work_id + * @param input + * @param timeout wait response timeout, default 0 (do not wait response) + * @param request_id + * @return int + */ + int inference(String work_id, String input, uint32_t timeout = 0, String request_id = "tts_inference"); + +private: + ModuleMsg* _module_msg = nullptr; +}; + +} // namespace m5_module_llm diff --git a/src/api/api_sys.cpp b/src/api/api_sys.cpp index 3f90b4c..2baaa68 100644 --- a/src/api/api_sys.cpp +++ b/src/api/api_sys.cpp @@ -13,8 +13,8 @@ static const char* _cmd_reset = "{\"request_id\":\"sys_reset\",\"work_id\":\"sys\",\"action\":\"reset\",\"object\":\"None\",\"data\":\"None\"}"; static const char* _cmd_reboot = "{\"request_id\":\"sys_reboot\",\"work_id\":\"sys\",\"action\":\"reboot\",\"object\":\"None\",\"data\":\"None\"}"; -// static const char* _cmd_ls_mode = -// "{\"request_id\":\"sys_lsmode\",\"work_id\":\"sys\",\"action\":\"lsmode\",\"object\":\"None\",\"data\":\"None\"}"; +static const char* _cmd_version = + "{\"request_id\":\"sys_version\",\"work_id\":\"sys\",\"action\":\"version\",\"object\":\"None\",\"data\":\"None\"}"; void ApiSys::init(ModuleMsg* moduleMsg) { @@ -29,6 +29,14 @@ int ApiSys::ping() return ret; } +int ApiSys::version() +{ + int ret = MODULE_LLM_WAIT_RESPONSE_TIMEOUT; + _module_msg->sendCmdAndWaitToTakeMsg( + _cmd_version, "sys_version", [&ret](ResponseMsg_t& msg) { ret = msg.error.code; }, 2000); + return ret; +} + int ApiSys::reset(bool waitResetFinish) { int ret = MODULE_LLM_WAIT_RESPONSE_TIMEOUT; diff --git a/src/api/api_sys.h b/src/api/api_sys.h index 2683d1b..796038b 100644 --- a/src/api/api_sys.h +++ b/src/api/api_sys.h @@ -26,6 +26,16 @@ public: * @param waitResetFinish * @return int */ + + int version(); + + /** + * @brief Check version + * + * @param waitCheckFinish + * @return int + */ + int reset(bool waitResetFinish = true); /** diff --git a/src/api/api_tts.cpp b/src/api/api_tts.cpp index 5dd8877..6cfe661 100644 --- a/src/api/api_tts.cpp +++ b/src/api/api_tts.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: MIT */ #include "api_tts.h" +#include "api_version.h" using namespace m5_module_llm; @@ -12,7 +13,7 @@ void ApiTts::init(ModuleMsg* moduleMsg) _module_msg = moduleMsg; } -String ApiTts::setup(ApiTtsSetupConfig_t config, String request_id) +String ApiTts::setup(ApiTtsSetupConfig_t config, String request_id, String language) { String cmd; { @@ -23,9 +24,20 @@ String ApiTts::setup(ApiTtsSetupConfig_t config, String request_id) doc["object"] = "tts.setup"; doc["data"]["model"] = config.model; doc["data"]["response_format"] = config.response_format; - doc["data"]["input"] = config.input; doc["data"]["enoutput"] = config.enoutput; doc["data"]["enkws"] = config.enkws; + doc["data"]["enaudio"] = config.enaudio; + if (!llm_version) { + doc["data"]["response_format"] = "tts.base64.wav"; + doc["data"]["input"] = config.input[0]; + doc["data"]["enoutput"] = true; + } else { + JsonArray inputArray = doc["data"]["input"].to(); + for (const String& str : config.input) { + inputArray.add(str); + } + } + if (language == "zh_CN") doc["data"]["model"] = "single_speaker_fast"; serializeJson(doc, cmd); } @@ -50,7 +62,7 @@ int ApiTts::inference(String work_id, String input, uint32_t timeout, String req doc["action"] = "inference"; doc["object"] = "tts.utf-8.stream"; doc["data"]["delta"] = input; - doc["data"]["index"] = 1; + doc["data"]["index"] = 0; doc["data"]["finish"] = true; serializeJson(doc, cmd); } diff --git a/src/api/api_tts.h b/src/api/api_tts.h index c3dd1b9..e5b06b9 100644 --- a/src/api/api_tts.h +++ b/src/api/api_tts.h @@ -10,11 +10,12 @@ namespace m5_module_llm { struct ApiTtsSetupConfig_t { - String model = "single_speaker_english_fast"; - String response_format = "tts.base64.wav"; - String input = "tts.utf-8.stream"; - bool enoutput = true; - bool enkws = true; + String model = "single_speaker_english_fast"; + String response_format = "sys.pcm"; + std::vector input = {"tts.utf-8.stream"}; + bool enoutput = false; + bool enaudio = true; + bool enkws = true; }; class ApiTts { @@ -28,7 +29,8 @@ public: * @param request_id * @return String */ - String setup(ApiTtsSetupConfig_t config = ApiTtsSetupConfig_t(), String request_id = "tts_setup"); + String setup(ApiTtsSetupConfig_t config = ApiTtsSetupConfig_t(), String request_id = "tts_setup", + String language = "en_US"); /** * @brief Inference input data by TTS module diff --git a/src/api/api_version.cpp b/src/api/api_version.cpp new file mode 100644 index 0000000..8cf6244 --- /dev/null +++ b/src/api/api_version.cpp @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "api_version.h" + +int llm_version = 0; diff --git a/src/api/api_version.h b/src/api/api_version.h new file mode 100644 index 0000000..64f5758 --- /dev/null +++ b/src/api/api_version.h @@ -0,0 +1,8 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once + +extern int llm_version; diff --git a/src/api/api_yolo.cpp b/src/api/api_yolo.cpp new file mode 100644 index 0000000..2774cc0 --- /dev/null +++ b/src/api/api_yolo.cpp @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "api_yolo.h" + +using namespace m5_module_llm; + +void ApiYolo::init(ModuleMsg* moduleMsg) +{ + _module_msg = moduleMsg; +} + +String ApiYolo::setup(ApiYoloSetupConfig_t config, String request_id) +{ + String cmd; + { + JsonDocument doc; + doc["request_id"] = request_id; + doc["work_id"] = "yolo"; + doc["action"] = "setup"; + doc["object"] = "yolo.setup"; + doc["data"]["model"] = config.model; + doc["data"]["response_format"] = config.response_format; + JsonArray inputArray = doc["data"]["input"].to(); + for (const String& str : config.input) { + inputArray.add(str); + } + doc["data"]["enoutput"] = config.enoutput; + serializeJson(doc, cmd); + } + + String work_id; + _module_msg->sendCmdAndWaitToTakeMsg( + cmd.c_str(), request_id, + [&work_id](ResponseMsg_t& msg) { + // Copy work id + work_id = msg.work_id; + }, + 5000); + return work_id; +} + +int ApiYolo::inference(String& work_id, uint8_t* input, size_t& raw_len, String request_id) +{ + String cmd; + { + JsonDocument doc; + doc["RAW"] = raw_len; + doc["request_id"] = request_id; + doc["work_id"] = work_id; + doc["action"] = "inference"; + doc["object"] = "cv.jpeg.base64"; + serializeJson(doc, cmd); + } + + _module_msg->sendCmd(cmd.c_str()); + _module_msg->sendRaw(input, raw_len); + return MODULE_LLM_OK; +} + +int ApiYolo::inferenceAndWaitResult(String& work_id, uint8_t* input, size_t& raw_len, + std::function onResult, uint32_t timeout, String request_id) +{ + inference(work_id, input, raw_len, request_id); + + uint32_t time_out_count = millis(); + bool is_time_out = false; + bool is_msg_finish = false; + while (1) { + _module_msg->update(); + _module_msg->takeMsg(request_id, [&time_out_count, &is_msg_finish, &onResult](ResponseMsg_t& msg) { + String response_msg; + { + JsonDocument doc; + deserializeJson(doc, msg.raw_msg); + response_msg = doc["data"]["delta"].as(); + if (!doc["data"]["finish"].isNull()) { + is_msg_finish = doc["data"]["finish"]; + if (is_msg_finish) { + response_msg += '\n'; + } + } + } + if (onResult) { + onResult(response_msg); + } + time_out_count = millis(); + }); + + if (is_msg_finish) { + break; + } + + if (millis() - time_out_count > timeout) { + is_time_out = true; + break; + } + } + + if (is_time_out) { + return MODULE_LLM_WAIT_RESPONSE_TIMEOUT; + } + return MODULE_LLM_OK; +} diff --git a/src/api/api_yolo.h b/src/api/api_yolo.h new file mode 100644 index 0000000..0817aa4 --- /dev/null +++ b/src/api/api_yolo.h @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once +#include "../utils/msg.h" +#include + +namespace m5_module_llm { +struct ApiYoloSetupConfig_t { + String model = "yolo11n"; + String response_format = "yolo.yolobox.stream"; + std::vector input = {"yolo.jpeg.base64"}; + bool enoutput = true; +}; + +class ApiYolo { +public: + void init(ModuleMsg* moduleMsg); + + /** + * @brief Setup module YOLO, return YOLO work_id + * + * @param config + * @param request_id + * @return String + */ + String setup(ApiYoloSetupConfig_t config = ApiYoloSetupConfig_t(), String request_id = "yolo_setup"); + + /** + * @brief Inference input data by module LLM + * + * @param raw_len + * @param work_id + * @param input + * @param request_id + * @return int + */ + int inference(String& work_id, uint8_t* input, size_t& raw_len, String request_id = "yolo_inference"); + + /** + * @brief Inference input data by module LLM, and wait inference result + * + * @param raw_len + * @param work_id + * @param input + * @param onResult On inference result callback + * @param timeout + * @param request_id + * @return int + */ + int inferenceAndWaitResult(String& work_id, uint8_t* input, size_t& raw_len, std::function onResult, + uint32_t timeout = 5000, String request_id = "yolo_inference"); + +private: + ModuleMsg* _module_msg = nullptr; +}; +} // namespace m5_module_llm diff --git a/src/presets/voice_assistant.cpp b/src/presets/voice_assistant.cpp index d34cc0d..9ce852a 100644 --- a/src/presets/voice_assistant.cpp +++ b/src/presets/voice_assistant.cpp @@ -4,12 +4,13 @@ * SPDX-License-Identifier: MIT */ #include "../M5ModuleLLM.h" +#include "../src/api/api_version.h" using namespace m5_module_llm; #define _debug(format, ...) printf("[\033[1;32mVoiceAssistant\033[0m] " format "\n", ##__VA_ARGS__) -int M5ModuleLLM_VoiceAssistant::begin(String wakeUpKeyword, String prompt) +int M5ModuleLLM_VoiceAssistant::begin(String wakeUpKeyword, String prompt, String language) { // Check connection if (!_m5_module_llm->checkConnection()) { @@ -30,14 +31,18 @@ int M5ModuleLLM_VoiceAssistant::begin(String wakeUpKeyword, String prompt) { ApiKwsSetupConfig_t config; config.kws = wakeUpKeyword; - _work_id.kws = _m5_module_llm->kws.setup(config); + _work_id.kws = _m5_module_llm->kws.setup(config, "kws_setup", language); } if (_work_id.kws.isEmpty()) { return MODULE_LLM_ERROR_NONE; } _debug("setup module asr.."); - _work_id.asr = _m5_module_llm->asr.setup(); + { + ApiAsrSetupConfig_t config; + config.input = {"sys.pcm", _work_id.kws}; + _work_id.asr = _m5_module_llm->asr.setup(config, "asr_setup", language); + } if (_work_id.asr.isEmpty()) { return MODULE_LLM_ERROR_NONE; } @@ -45,7 +50,7 @@ int M5ModuleLLM_VoiceAssistant::begin(String wakeUpKeyword, String prompt) _debug("setup module llm.."); { ApiLlmSetupConfig_t config; - config.input = _work_id.asr; + config.input = {_work_id.asr, _work_id.kws}; config.prompt = prompt; _work_id.llm = _m5_module_llm->llm.setup(config); } @@ -55,11 +60,17 @@ int M5ModuleLLM_VoiceAssistant::begin(String wakeUpKeyword, String prompt) _debug("setup module tts.."); { - ApiTtsSetupConfig_t config; - config.input = _work_id.llm; - _work_id.tts = _m5_module_llm->tts.setup(config); + if (!llm_version) { + ApiTtsSetupConfig_t config; + config.input = {_work_id.llm, _work_id.kws}; + _work_id.tts = _m5_module_llm->tts.setup(config, "tts_setup", language); + } else { + ApiMelottsSetupConfig_t config; + config.input = {_work_id.llm, _work_id.kws}; + _work_id.melotts = _m5_module_llm->melotts.setup(config, "tts_setup", language); + } } - if (_work_id.tts.isEmpty()) { + if (_work_id.tts.isEmpty() && _work_id.melotts.isEmpty()) { return MODULE_LLM_ERROR_NONE; } diff --git a/src/utils/comm.cpp b/src/utils/comm.cpp index e835b1a..7df0c4a 100644 --- a/src/utils/comm.cpp +++ b/src/utils/comm.cpp @@ -26,6 +26,11 @@ void ModuleComm::sendCmd(const char* cmd) _serial->print(cmd); } +void ModuleComm::sendRaw(const uint8_t* data, size_t& raw_len) +{ + _serial->write(data, raw_len); +} + ModuleComm::Respond_t ModuleComm::getResponse(uint32_t timeout) { Respond_t ret; diff --git a/src/utils/comm.h b/src/utils/comm.h index fa616d1..99c0344 100644 --- a/src/utils/comm.h +++ b/src/utils/comm.h @@ -24,6 +24,7 @@ public: bool init(Stream* serialPort); void sendCmd(const char* cmd); + void sendRaw(const uint8_t* data, size_t& raw_len); Respond_t getResponse(uint32_t timeout = 0xFFFFFFFF); private: diff --git a/src/utils/msg.h b/src/utils/msg.h index b73e414..60ad69d 100644 --- a/src/utils/msg.h +++ b/src/utils/msg.h @@ -59,6 +59,10 @@ public: _module_comm->sendCmd(cmd); } + inline void sendRaw(const uint8_t* data, size_t& raw_len) + { + _module_comm->sendRaw(data, raw_len); + } /** * @brief Module response message list *