diff --git a/examples/YOLO/YOLO.ino b/examples/YOLO/YOLO.ino new file mode 100644 index 0000000..2be0ddf --- /dev/null +++ b/examples/YOLO/YOLO.ino @@ -0,0 +1,107 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include +#include +#include +#include + +M5ModuleLLM module_llm; +String camera_work_id; +String yolo_work_id; + +void setup() +{ + M5.begin(); + M5.Display.setTextSize(2); + M5.Display.setTextScroll(true); + + /* Init module serial port */ + Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic + // Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2 + // Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3 + + /* Init module */ + module_llm.begin(&Serial2); + + /* Make sure module is connected */ + M5.Display.printf(">> Check ModuleLLM connection..\n"); + while (1) { + if (module_llm.checkConnection()) { + break; + } + } + + /* Reset ModuleLLM */ + M5.Display.printf(">> Reset ModuleLLM..\n"); + module_llm.sys.reset(); + + /* Setup Camera module */ + M5.Display.printf(">> Setup camera..\n"); + camera_work_id = module_llm.camera.setup(); + + /* Setup YOLO module and save returned work id */ + M5.Display.printf(">> Setup yolo..\n"); + m5_module_llm::ApiYoloSetupConfig_t yolo_config; + yolo_config.input = {camera_work_id}; + yolo_work_id = module_llm.yolo.setup(yolo_config, "yolo_setup"); + // M5.Display.printf(">> Yolo ready\n"); + M5.Display.drawString("class", 10, 80); + M5.Display.drawString("confidence", 180, 80); + M5.Display.drawString("x1", 10, 110); + M5.Display.drawString("y1", 10, 140); + M5.Display.drawString("x2", 10, 170); + M5.Display.drawString("y2", 10, 200); +} + +void loop() +{ + /* Update ModuleLLM */ + module_llm.update(); + + /* Handle module response messages */ + for (auto& msg : module_llm.msg.responseMsgList) { + /* If YOLO module message */ + if (msg.work_id == yolo_work_id) { + /* Check message object type */ + if (msg.object == "yolo.box.stream") { + /* Parse message json and get YOLO result */ + JsonDocument doc; + deserializeJson(doc, msg.raw_msg); + JsonArray delta = doc["data"]["delta"].as(); + + if (delta.size() > 0) { + JsonObject result = delta[0].as(); + String class_name = result["class"].as(); + float confidence = result["confidence"].as(); + JsonArray bboxArray = result["bbox"].as(); + + if (bboxArray.size() == 4) { + int x1 = bboxArray[0].as(); + int y1 = bboxArray[1].as(); + int x2 = bboxArray[2].as(); + int y2 = bboxArray[3].as(); + M5.Display.drawString(class_name, 80, 80); + M5.Display.drawFloat(confidence, 2, 200, 110); + M5.Display.drawNumber(x1, 40, 110); + M5.Display.drawNumber(y1, 40, 140); + M5.Display.drawNumber(x2, 40, 170); + M5.Display.drawNumber(y2, 40, 200); + } + } else { + M5.Display.drawString("None", 80, 80); + M5.Display.drawFloat(0, 2, 200, 110); + M5.Display.drawNumber(0, 40, 110); + M5.Display.drawNumber(0, 40, 140); + M5.Display.drawNumber(0, 40, 170); + M5.Display.drawNumber(0, 40, 200); + } + } + } + } + + /* Clear handled messages */ + module_llm.msg.responseMsgList.clear(); +} \ No newline at end of file diff --git a/src/M5ModuleLLM.cpp b/src/M5ModuleLLM.cpp index 4bf51a9..0bac44b 100644 --- a/src/M5ModuleLLM.cpp +++ b/src/M5ModuleLLM.cpp @@ -18,6 +18,7 @@ bool M5ModuleLLM::begin(Stream* serialPort) kws.init(&msg); asr.init(&msg); yolo.init(&msg); + camera.init(&msg); return true; } diff --git a/src/M5ModuleLLM.h b/src/M5ModuleLLM.h index 5f61d35..42bae02 100644 --- a/src/M5ModuleLLM.h +++ b/src/M5ModuleLLM.h @@ -15,6 +15,7 @@ #include "api/api_kws.h" #include "api/api_asr.h" #include "api/api_yolo.h" +#include "api/api_camera.h" #include "api/api_version.h" class M5ModuleLLM { @@ -60,6 +61,12 @@ public: */ m5_module_llm::ApiAudio audio; + /** + * @brief Camera module api set + * + */ + m5_module_llm::ApiCamera camera; + /** * @brief TTS module api set * diff --git a/src/api/api_camera.cpp b/src/api/api_camera.cpp new file mode 100644 index 0000000..8b2c7da --- /dev/null +++ b/src/api/api_camera.cpp @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#include "api_camera.h" + +using namespace m5_module_llm; + +void ApiCamera::init(ModuleMsg* moduleMsg) +{ + _module_msg = moduleMsg; +} + +String ApiCamera::setup(ApiCameraSetupConfig_t config, String request_id) +{ + String cmd; + { + JsonDocument doc; + doc["request_id"] = request_id; + doc["work_id"] = "camera"; + doc["action"] = "setup"; + doc["object"] = "camera.setup"; + doc["data"]["response_format"] = config.response_format; + doc["data"]["input"] = config.input; + doc["data"]["enoutput"] = config.enoutput; + doc["data"]["frame_width"] = config.frame_width; + doc["data"]["frame_height"] = config.frame_height; + serializeJson(doc, cmd); + } + + String work_id; + _module_msg->sendCmdAndWaitToTakeMsg( + cmd.c_str(), request_id, + [&work_id](ResponseMsg_t& msg) { + // Copy work id + work_id = msg.work_id; + }, + 5000); + return work_id; +} diff --git a/src/api/api_camera.h b/src/api/api_camera.h new file mode 100644 index 0000000..f0d0ea3 --- /dev/null +++ b/src/api/api_camera.h @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD + * + * SPDX-License-Identifier: MIT + */ +#pragma once +#include "../utils/msg.h" +#include + +namespace m5_module_llm { + +struct ApiCameraSetupConfig_t { + String response_format = "camera.raw"; + String input = "/dev/video0"; + bool enoutput = false; + int frame_width = 320; + int frame_height = 320; +}; + +class ApiCamera { +public: + void init(ModuleMsg* moduleMsg); + + /** + * @brief Setup module camera, return work_id + * + * @param config + * @param request_id + * @return String + */ + String setup(ApiCameraSetupConfig_t config = ApiCameraSetupConfig_t(), String request_id = "camera_setup"); + +private: + ModuleMsg* _module_msg = nullptr; +}; + +} // namespace m5_module_llm