From 2d250067b04b0f01ee4370af584ccef468a28e9a Mon Sep 17 00:00:00 2001 From: Forairaaaaa Date: Fri, 25 Oct 2024 11:17:47 +0800 Subject: [PATCH] add description --- examples/kws_asr.ino | 24 ++++++--- examples/text_assistant.ino | 17 ++++-- examples/tts.ino | 10 +++- src/M5ModuleLLM.h | 101 ++++++++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 13 deletions(-) diff --git a/examples/kws_asr.ino b/examples/kws_asr.ino index f9ce104..0633850 100644 --- a/examples/kws_asr.ino +++ b/examples/kws_asr.ino @@ -9,8 +9,10 @@ #include M5ModuleLLM module_llm; -// Must be all uppercase letter + +/* Must be capitalized */ String wake_up_keyword = "HELLO"; + String kws_work_id; String asr_work_id; @@ -20,12 +22,15 @@ void setup() M5.Display.setTextSize(2); M5.Display.setTextScroll(true); + /* Init module serial port */ Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic // Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2 // Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3 + /* Init module */ module_llm.begin(&Serial2); + /* Make sure module is connected */ M5.Display.printf(">> Check ModuleLLM connection..\n"); while (1) { if (module_llm.checkConnection()) { @@ -33,17 +38,21 @@ void setup() } } + /* Reset ModuleLLM */ M5.Display.printf(">> Reset ModuleLLM..\n"); module_llm.sys.reset(); + /* Setup Audio module */ M5.Display.printf(">> Setup audio..\n"); module_llm.audio.setup(); + /* Setup KWS module and save returned work id */ M5.Display.printf(">> Setup kws..\n"); m5_module_llm::ApiKwsSetupConfig_t kws_config; kws_config.kws = wake_up_keyword; kws_work_id = module_llm.kws.setup(kws_config); + /* Setup ASR module and save returned work id */ M5.Display.printf(">> Setup asr..\n"); asr_work_id = module_llm.asr.setup(); @@ -52,20 +61,22 @@ void setup() void loop() { + /* Update ModuleLLM */ module_llm.update(); - // Handle response msg + /* Handle module response messages */ for (auto& msg : module_llm.msg.responseMsgList) { - // KWS msg + /* If KWS module message */ if (msg.work_id == kws_work_id) { M5.Display.setTextColor(TFT_GREENYELLOW); M5.Display.printf(">> Keyword detected\n"); } - // ASR msg + /* If ASR module message */ if (msg.work_id == asr_work_id) { + /* Check message object type */ if (msg.object == "asr.utf-8.stream") { - // Parse and get asr result + /* Parse message json and get ASR result */ JsonDocument doc; deserializeJson(doc, msg.raw_msg); String asr_result = doc["data"]["delta"].as(); @@ -76,5 +87,6 @@ void loop() } } + /* Clear handled messages */ module_llm.msg.responseMsgList.clear(); -} +} \ No newline at end of file diff --git a/examples/text_assistant.ino b/examples/text_assistant.ino index f8193e7..17bc101 100644 --- a/examples/text_assistant.ino +++ b/examples/text_assistant.ino @@ -16,12 +16,15 @@ void setup() M5.Display.setTextSize(2); M5.Display.setTextScroll(true); + /* Init module serial port */ Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic // Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2 // Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3 + /* Init module */ module_llm.begin(&Serial2); + /* Make sure module is connected */ M5.Display.printf(">> Check ModuleLLM connection..\n"); while (1) { if (module_llm.checkConnection()) { @@ -29,16 +32,18 @@ void setup() } } + /* Reset ModuleLLM */ M5.Display.printf(">> Reset ModuleLLM..\n"); module_llm.sys.reset(); + /* Setup LLM module and save returned work id */ M5.Display.printf(">> Setup llm..\n"); llm_work_id = module_llm.llm.setup(); } void loop() { - // Make question + /* Make a question: What is {i} + {i} equal to? */ static int i = 0; i++; std::string question = "What is " + std::to_string(i) + " + " + std::to_string(i) + " equal to?"; @@ -48,11 +53,13 @@ void loop() M5.Display.setTextColor(TFT_YELLOW); M5.Display.printf(">> "); - // Push question to llm and wait result - module_llm.llm.inferenceAndWaitResult(llm_work_id, question.c_str(), - [](String& result) { M5.Display.printf("%s", result.c_str()); }); + /* Push question to LLM module and wait inference result */ + module_llm.llm.inferenceAndWaitResult(llm_work_id, question.c_str(), [](String& result) { + /* Show result on screen */ + M5.Display.printf("%s", result.c_str()); + }); M5.Display.println(); delay(500); -} +} \ No newline at end of file diff --git a/examples/tts.ino b/examples/tts.ino index 4daa60c..909964f 100644 --- a/examples/tts.ino +++ b/examples/tts.ino @@ -16,12 +16,15 @@ void setup() M5.Display.setTextSize(2); M5.Display.setTextScroll(true); + /* Init module serial port */ Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic // Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2 // Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3 + /* Init module */ module_llm.begin(&Serial2); + /* Make sure module is connected */ M5.Display.printf(">> Check ModuleLLM connection..\n"); while (1) { if (module_llm.checkConnection()) { @@ -29,19 +32,22 @@ void setup() } } + /* Reset ModuleLLM */ M5.Display.printf(">> Reset ModuleLLM..\n"); module_llm.sys.reset(); + /* Setup Audio module */ M5.Display.printf(">> Setup audio..\n"); module_llm.audio.setup(); + /* Setup TTS module and save returned work id */ M5.Display.printf(">> Setup tts..\n\n"); tts_work_id = module_llm.tts.setup(); } void loop() { - // Make text + /* Make a text for speech: {i} plus {i} equals to {i + i} */ static int i = 0; i++; std::string text = std::to_string(i) + " plus " + std::to_string(i) + " equals " + std::to_string(i + i) + "."; @@ -49,7 +55,7 @@ void loop() M5.Display.setTextColor(TFT_GREEN); M5.Display.printf("<< %s\n\n", text.c_str()); - // Push text to tts and wait result + /* Push text to TTS module and wait inference result */ module_llm.tts.inference(tts_work_id, text.c_str(), 10000); delay(500); diff --git a/src/M5ModuleLLM.h b/src/M5ModuleLLM.h index 5aca2d1..9b2a113 100644 --- a/src/M5ModuleLLM.h +++ b/src/M5ModuleLLM.h @@ -16,17 +16,75 @@ class M5ModuleLLM { public: + /** + * @brief Init module + * + * @param targetPort module serial port + * @return true + * @return false + */ bool begin(Stream* targetPort); + + /** + * @brief Check if module is connected + * + * @return true + * @return false + */ bool checkConnection(); + + /** + * @brief Update module + * + */ void update(); + /** + * @brief SYS module api set + * + */ m5_module_llm::ApiSys sys; + + /** + * @brief LLM module api set + * + */ m5_module_llm::ApiLlm llm; + + /** + * @brief Audio module api set + * + */ m5_module_llm::ApiAudio audio; + + /** + * @brief TTS module api set + * + */ m5_module_llm::ApiTts tts; + + /** + * @brief KWS module api set + * + */ m5_module_llm::ApiKws kws; + + /** + * @brief ASR module api set + * + */ m5_module_llm::ApiAsr asr; + + /** + * @brief MSG module to handle module response message + * + */ m5_module_llm::ModuleMsg msg; + + /** + * @brief COMM module to handle module communication stream + * + */ m5_module_llm::ModuleComm comm; private: @@ -38,21 +96,64 @@ typedef std::function OnLlmDataInpu typedef std::function OnAsrDataInputRawCallback_t; typedef std::function OnLlmDataInputRawCallback_t; +/** + * @brief Voice assistant preset base on class M5ModuleLLM + * + */ class M5ModuleLLM_VoiceAssistant { public: M5ModuleLLM_VoiceAssistant(M5ModuleLLM* m5ModuleLlm) : _m5_module_llm(m5ModuleLlm) { } + /** + * @brief Start voice assistant preset + * + * @param wakeUpKeyword + * @param prompt + * @return int + */ int begin(String wakeUpKeyword = "HELLO", String prompt = ""); + /** + * @brief Update voice assistant preset, trigger callbacks + * + */ void update(); + /** + * @brief Register on key-word detected callback + * + * @param callback + */ void onKeywordDetected(OnKeywordDetectedCallback_t callback); + + /** + * @brief Register on ASR data input callback + * + * @param callback + */ void onAsrDataInput(OnAsrDataInputCallback_t callback); + + /** + * @brief Register on LLM data input callback + * + * @param callback + */ void onLlmDataInput(OnLlmDataInputCallback_t callback); + /** + * @brief Register on ASR raw data (module response json) input callback + * + * @param callback + */ void onAsrDataInputRaw(OnAsrDataInputRawCallback_t callback); + + /** + * @brief Register on LLM raw data (module response json) input callback + * + * @param callback + */ void onLlmDataInputRaw(OnLlmDataInputRawCallback_t callback); private: