Merge pull request #17 from Abandon-ht/dev

docs/en.md

@@ -0,0 +1,41 @@
+# LLM Module Arduino Quick Start
+
+## Overview
+
+The `LLM Module` can be used with various M5 controllers. This tutorial demonstrates how to control the LLM Module using the `M5Core` series in the `Arduino IDE` with the LLM Module driver library.
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_module_device_01.jpg" width="70%" />
+
+## Environment Setup
+
+- 1.Arduino IDE Installation: Refer to the [Arduino IDE Installation Guide](/en/arduino/arduino_ide) to complete the IDE installation.
+
+- 2.Board Manager Installation: Refer to the [Basic Environment Setup Guide](/en/arduino/arduino_board) to complete the M5Stack board manager installation and select the `M5Core` development board.
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/arduino/m5core/quickstart_arduino_core_selectboard.png" width="70%" />
+
+- 3.Library Installation: Refer to the [Library Management Guide](/en/arduino/arduino_library) to install the `LLM Module` driver library. (Follow prompts to install the dependency library `M5Unified`)
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_lib_01.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_lib_02.jpg" width="70%" />
+
+## Program Compilation & Upload
+
+Open the example program "kws_asr" in the driver library, click the upload button, and the program will automatically compile and upload.The wake-up word used in the example program is "HELLO". After waiting for the device to be initialized, it will be woken up using the keyword.
+
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_01.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_02.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_03.jpg" width="70%" />
+<img src="https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/docs/static/assets/img/guide/llm/llm/llm_arduino_example_04.jpg" width="70%" />
+
+- Examples:
+  - `kws_asr`: Uses KWS to wake up and triggers ASR for speech-to-text conversion. (KWS+ASR)
+  - `text_assistant`: Inputs text into the LLM model, performs inference, and outputs the result in text form. (LLM)
+  - `tts`: Uses the TTS unit to convert text to speech for playback. (TTS)
+  - `voice_assistant`: Uses KWS to wake up, triggers ASR for speech-to-text conversion, inputs the converted text into the LLM for inference, and outputs the inference result through TTS as speech. (KWS+ASR+LLM+TTS)
+
+## Related Links
+
+- [LLM Module Arduino Lib](https://github.com/m5stack/M5Module-LLM)
+- [LLM Module Arduino Lib API](/en/guide/llm/llm/arduino_api)
+

src/M5ModuleLLM.cpp

@@ -12,6 +12,7 @@ bool M5ModuleLLM::begin(Stream* serialPort)
     msg.init(&comm);
     sys.init(&msg);
     llm.init(&msg);
+    vlm.init(&msg);
     audio.init(&msg);
     tts.init(&msg);
     melotts.init(&msg);
@@ -19,6 +20,7 @@ bool M5ModuleLLM::begin(Stream* serialPort)
     asr.init(&msg);
     yolo.init(&msg);
     camera.init(&msg);
+    depthanything.init(&msg);
     return true;
 }
 

src/M5ModuleLLM.h

@@ -9,12 +9,14 @@
 #include "utils/msg.h"
 #include "api/api_sys.h"
 #include "api/api_llm.h"
+#include "api/api_vlm.h"
 #include "api/api_audio.h"
 #include "api/api_tts.h"
 #include "api/api_melotts.h"
 #include "api/api_kws.h"
 #include "api/api_asr.h"
 #include "api/api_yolo.h"
+#include "api/api_depth_anything.h"
 #include "api/api_camera.h"
 #include "api/api_version.h"
 
@@ -55,6 +57,12 @@ public:
      */
     m5_module_llm::ApiLlm llm;
 
+    /**
+     * @brief VLM module api set
+     *
+     */
+    m5_module_llm::ApiVlm vlm;
+
     /**
      * @brief Audio module api set
      *
@@ -97,6 +105,12 @@ public:
      */
     m5_module_llm::ApiYolo yolo;
 
+    /**
+     * @brief DepthAnything module api set
+     *
+     */
+    m5_module_llm::ApiDepthAnything depthanything;
+
     /**
      * @brief MSG module to handle module response message
      *

src/api/api_asr.cpp

@@ -51,3 +51,24 @@ String ApiAsr::setup(ApiAsrSetupConfig_t config, String request_id, String langu
         10000);
     return work_id;
 }
+
+String ApiAsr::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_asr.h

@@ -34,6 +34,15 @@ public:
     String setup(ApiAsrSetupConfig_t config = ApiAsrSetupConfig_t(), String request_id = "asr_setup",
                  String language = "en_US");
 
+    /**
+     * @brief Exit module ASR, return ASR work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "asr_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };

src/api/api_audio.cpp

@@ -40,3 +40,24 @@ String ApiAudio::setup(ApiAudioSetupConfig_t config, String request_id)
         5000);
     return work_id;
 }
+
+String ApiAudio::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_audio.h

@@ -23,7 +23,7 @@ public:
     void init(ModuleMsg* moduleMsg);
 
     /**
-     * @brief Setup module audio, return work_id
+     * @brief Setup module audio, return audio work_id
      *
      * @param config
      * @param request_id
@@ -31,6 +31,15 @@ public:
      */
     String setup(ApiAudioSetupConfig_t config = ApiAudioSetupConfig_t(), String request_id = "audio_setup");
 
+    /**
+     * @brief Exit module audio, return audio work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "audio_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };

src/api/api_camera.cpp

@@ -39,3 +39,24 @@ String ApiCamera::setup(ApiCameraSetupConfig_t config, String request_id)
         5000);
     return work_id;
 }
+
+String ApiCamera::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_camera.h

@@ -22,7 +22,7 @@ public:
     void init(ModuleMsg* moduleMsg);
 
     /**
-     * @brief Setup module camera, return work_id
+     * @brief Setup module camera, return camera work_id
      *
      * @param config
      * @param request_id
@@ -30,6 +30,15 @@ public:
      */
     String setup(ApiCameraSetupConfig_t config = ApiCameraSetupConfig_t(), String request_id = "camera_setup");
 
+    /**
+     * @brief Exit module camera, return camera work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "camera_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };

src/api/api_depth_anything.cpp

@@ -0,0 +1,127 @@
+/*
+ * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#include "api_depth_anything.h"
+
+using namespace m5_module_llm;
+
+void ApiDepthAnything::init(ModuleMsg* moduleMsg)
+{
+    _module_msg = moduleMsg;
+}
+
+String ApiDepthAnything::setup(ApiDepthAnythingSetupConfig_t config, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"]              = request_id;
+        doc["work_id"]                 = "depth_anything";
+        doc["action"]                  = "setup";
+        doc["object"]                  = "depth_anything.setup";
+        doc["data"]["model"]           = config.model;
+        doc["data"]["response_format"] = config.response_format;
+        JsonArray inputArray           = doc["data"]["input"].to<JsonArray>();
+        for (const String& str : config.input) {
+            inputArray.add(str);
+        }
+        doc["data"]["enoutput"] = config.enoutput;
+        serializeJson(doc, cmd);
+    }
+
+    String work_id;
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        5000);
+    return work_id;
+}
+
+String ApiDepthAnything::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}
+
+int ApiDepthAnything::inference(String& work_id, uint8_t* input, size_t& raw_len, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["RAW"]        = raw_len;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "inference";
+        doc["object"]     = "cv.jpeg.base64";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmd(cmd.c_str());
+    _module_msg->sendRaw(input, raw_len);
+    return MODULE_LLM_OK;
+}
+
+int ApiDepthAnything::inferenceAndWaitResult(String& work_id, uint8_t* input, size_t& raw_len,
+                                             std::function<void(String&)> onResult, uint32_t timeout, String request_id)
+{
+    inference(work_id, input, raw_len, request_id);
+
+    uint32_t time_out_count = millis();
+    bool is_time_out        = false;
+    bool is_msg_finish      = false;
+    while (1) {
+        _module_msg->update();
+        _module_msg->takeMsg(request_id, [&time_out_count, &is_msg_finish, &onResult](ResponseMsg_t& msg) {
+            String response_msg;
+            {
+                JsonDocument doc;
+                deserializeJson(doc, msg.raw_msg);
+                response_msg = doc["data"]["delta"].as<String>();
+                if (!doc["data"]["finish"].isNull()) {
+                    is_msg_finish = doc["data"]["finish"];
+                    if (is_msg_finish) {
+                        response_msg += '\n';
+                    }
+                }
+            }
+            if (onResult) {
+                onResult(response_msg);
+            }
+            time_out_count = millis();
+        });
+
+        if (is_msg_finish) {
+            break;
+        }
+
+        if (millis() - time_out_count > timeout) {
+            is_time_out = true;
+            break;
+        }
+    }
+
+    if (is_time_out) {
+        return MODULE_LLM_WAIT_RESPONSE_TIMEOUT;
+    }
+    return MODULE_LLM_OK;
+}

src/api/api_depth_anything.h

@@ -0,0 +1,69 @@
+/*
+ * SPDX-FileCopyrightText: 2024 M5Stack Technology CO LTD
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+#include "../utils/msg.h"
+#include <Arduino.h>
+
+namespace m5_module_llm {
+struct ApiDepthAnythingSetupConfig_t {
+    String model              = "depth_anything";
+    String response_format    = "jpeg.base64.stream";
+    std::vector<String> input = {"depth_anything.jpeg.raw"};
+    bool enoutput             = true;
+};
+
+class ApiDepthAnything {
+public:
+    void init(ModuleMsg* moduleMsg);
+
+    /**
+     * @brief Setup module DepthAnything, return DepthAnything work_id
+     *
+     * @param config
+     * @param request_id
+     * @return String
+     */
+    String setup(ApiDepthAnythingSetupConfig_t config = ApiDepthAnythingSetupConfig_t(),
+                 String request_id                    = "depth_anything_setup");
+
+    /**
+     * @brief Exit module DepthAnything, return DepthAnything work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "depth_anything_exit");
+
+    /**
+     * @brief Inference input data by module LLM
+     *
+     * @param raw_len
+     * @param work_id
+     * @param input
+     * @param request_id
+     * @return int
+     */
+    int inference(String& work_id, uint8_t* input, size_t& raw_len, String request_id = "depth_anything_inference");
+
+    /**
+     * @brief Inference input data by module LLM, and wait inference result
+     *
+     * @param raw_len
+     * @param work_id
+     * @param input
+     * @param onResult On inference result callback
+     * @param timeout
+     * @param request_id
+     * @return int
+     */
+    int inferenceAndWaitResult(String& work_id, uint8_t* input, size_t& raw_len, std::function<void(String&)> onResult,
+                               uint32_t timeout = 5000, String request_id = "depth_anything_inference");
+
+private:
+    ModuleMsg* _module_msg = nullptr;
+};
+}  // namespace m5_module_llm

src/api/api_kws.cpp

@@ -48,3 +48,24 @@ String ApiKws::setup(ApiKwsSetupConfig_t config, String request_id, String langu
         30000);
     return work_id;
 }
+
+String ApiKws::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}

src/api/api_kws.h

@@ -31,6 +31,15 @@ public:
     String setup(ApiKwsSetupConfig_t config = ApiKwsSetupConfig_t(), String request_id = "kws_setup",
                  String language = "en_US");
 
+    /**
+     * @brief Exit module KWS, return KWS work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "kws_exit");
+
 private:
     ModuleMsg* _module_msg = nullptr;
 };

src/api/api_llm.cpp

@@ -47,10 +47,31 @@ String ApiLlm::setup(ApiLlmSetupConfig_t config, String request_id)
             // Copy work id
             llm_work_id = msg.work_id;
         },
-        10000);
+        20000);
     return llm_work_id;
 }
 
+String ApiLlm::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}
+
 int ApiLlm::inference(String work_id, String input, String request_id)
 {
     String cmd;

src/api/api_llm.h

@@ -33,6 +33,15 @@ public:
      */
     String setup(ApiLlmSetupConfig_t config = ApiLlmSetupConfig_t(), String request_id = "llm_setup");
 
+    /**
+     * @brief Exit module LLM, return LLM work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "llm_exit");
+
     /**
      * @brief Inference input data by module LLM
      *

src/api/api_melotts.cpp

@@ -44,6 +44,27 @@ String ApiMelotts::setup(ApiMelottsSetupConfig_t config, String request_id, Stri
     return work_id;
 }
 
+String ApiMelotts::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}
+
 int ApiMelotts::inference(String work_id, String input, uint32_t timeout, String request_id)
 {
     String cmd;

src/api/api_melotts.h

@@ -31,6 +31,15 @@ public:
     String setup(ApiMelottsSetupConfig_t config = ApiMelottsSetupConfig_t(), String request_id = "melotts_setup",
                  String language = "en_US");
 
+    /**
+     * @brief Exit module TTS, return TTS work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "tts_exit");
+
     /**
      * @brief Inference input data by TTS module
      *

src/api/api_tts.cpp

@@ -52,6 +52,27 @@ String ApiTts::setup(ApiTtsSetupConfig_t config, String request_id, String langu
     return work_id;
 }
 
+String ApiTts::exit(String work_id, String request_id)
+{
+    String cmd;
+    {
+        JsonDocument doc;
+        doc["request_id"] = request_id;
+        doc["work_id"]    = work_id;
+        doc["action"]     = "exit";
+        serializeJson(doc, cmd);
+    }
+
+    _module_msg->sendCmdAndWaitToTakeMsg(
+        cmd.c_str(), request_id,
+        [&work_id](ResponseMsg_t& msg) {
+            // Copy work id
+            work_id = msg.work_id;
+        },
+        100);
+    return work_id;
+}
+
 int ApiTts::inference(String work_id, String input, uint32_t timeout, String request_id)
 {
     String cmd;

src/api/api_tts.h

@@ -32,6 +32,15 @@ public:
     String setup(ApiTtsSetupConfig_t config = ApiTtsSetupConfig_t(), String request_id = "tts_setup",
                  String language = "en_US");
 
+    /**
+     * @brief Exit module TTS, return TTS work_id
+     *
+     * @param work_id
+     * @param request_id
+     * @return String
+     */
+    String exit(String work_id, String request_id = "tts_exit");
+
     /**
      * @brief Inference input data by TTS module
      *