m5/M5Module-LLM
0
0
Fork 0
mirror of https://github.com/m5stack/M5Module-LLM.git synced 2026-05-20 11:31:45 -07:00
Code Issues Packages Projects Releases Wiki Activity

add description

Browse Source
This commit is contained in:
Forairaaaaa
2024-10-25 11:17:47 +08:00
parent d5b487489e
commit 2d250067b0
4 changed files with 139 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/kws_asr.ino
+18 -6
View File
@@ -9,8 +9,10 @@
#include <ArduinoJson.h>
M5ModuleLLM module_llm;
// Must be all uppercase letter
/* Must be capitalized */
String wake_up_keyword = "HELLO";
String kws_work_id;
String asr_work_id;
@@ -20,12 +22,15 @@ void setup()
M5.Display.setTextSize(2);
M5.Display.setTextScroll(true);
/* Init module serial port */
Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic
// Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2
// Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3
/* Init module */
module_llm.begin(&Serial2);
/* Make sure module is connected */
M5.Display.printf(">> Check ModuleLLM connection..\n");
while (1) {
if (module_llm.checkConnection()) {
@@ -33,17 +38,21 @@ void setup()
}
}
/* Reset ModuleLLM */
M5.Display.printf(">> Reset ModuleLLM..\n");
module_llm.sys.reset();
/* Setup Audio module */
M5.Display.printf(">> Setup audio..\n");
module_llm.audio.setup();
/* Setup KWS module and save returned work id */
M5.Display.printf(">> Setup kws..\n");
m5_module_llm::ApiKwsSetupConfig_t kws_config;
kws_config.kws = wake_up_keyword;
kws_work_id = module_llm.kws.setup(kws_config);
/* Setup ASR module and save returned work id */
M5.Display.printf(">> Setup asr..\n");
asr_work_id = module_llm.asr.setup();
@@ -52,20 +61,22 @@ void setup()
void loop()
{
/* Update ModuleLLM */
module_llm.update();
// Handle response msg
/* Handle module response messages */
for (auto& msg : module_llm.msg.responseMsgList) {
// KWS msg
/* If KWS module message */
if (msg.work_id == kws_work_id) {
M5.Display.setTextColor(TFT_GREENYELLOW);
M5.Display.printf(">> Keyword detected\n");
}
// ASR msg
/* If ASR module message */
if (msg.work_id == asr_work_id) {
/* Check message object type */
if (msg.object == "asr.utf-8.stream") {
// Parse and get asr result
/* Parse message json and get ASR result */
JsonDocument doc;
deserializeJson(doc, msg.raw_msg);
String asr_result = doc["data"]["delta"].as<String>();
@@ -76,5 +87,6 @@ void loop()
}
}
/* Clear handled messages */
module_llm.msg.responseMsgList.clear();
}
}
examples/text_assistant.ino
+12 -5
View File
@@ -16,12 +16,15 @@ void setup()
M5.Display.setTextSize(2);
M5.Display.setTextScroll(true);
/* Init module serial port */
Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic
// Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2
// Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3
/* Init module */
module_llm.begin(&Serial2);
/* Make sure module is connected */
M5.Display.printf(">> Check ModuleLLM connection..\n");
while (1) {
if (module_llm.checkConnection()) {
@@ -29,16 +32,18 @@ void setup()
}
}
/* Reset ModuleLLM */
M5.Display.printf(">> Reset ModuleLLM..\n");
module_llm.sys.reset();
/* Setup LLM module and save returned work id */
M5.Display.printf(">> Setup llm..\n");
llm_work_id = module_llm.llm.setup();
}
void loop()
{
// Make question
/* Make a question: What is {i} + {i} equal to? */
static int i = 0;
i++;
std::string question = "What is " + std::to_string(i) + " + " + std::to_string(i) + " equal to?";
@@ -48,11 +53,13 @@ void loop()
M5.Display.setTextColor(TFT_YELLOW);
M5.Display.printf(">> ");
// Push question to llm and wait result
module_llm.llm.inferenceAndWaitResult(llm_work_id, question.c_str(),
[](String& result) { M5.Display.printf("%s", result.c_str()); });
/* Push question to LLM module and wait inference result */
module_llm.llm.inferenceAndWaitResult(llm_work_id, question.c_str(), [](String& result) {
/* Show result on screen */
M5.Display.printf("%s", result.c_str());
});
M5.Display.println();
delay(500);
}
}
examples/tts.ino
+8 -2
View File
@@ -16,12 +16,15 @@ void setup()
M5.Display.setTextSize(2);
M5.Display.setTextScroll(true);
/* Init module serial port */
Serial2.begin(115200, SERIAL_8N1, 16, 17); // Basic
// Serial2.begin(115200, SERIAL_8N1, 13, 14); // Core2
// Serial2.begin(115200, SERIAL_8N1, 18, 17); // CoreS3
/* Init module */
module_llm.begin(&Serial2);
/* Make sure module is connected */
M5.Display.printf(">> Check ModuleLLM connection..\n");
while (1) {
if (module_llm.checkConnection()) {
@@ -29,19 +32,22 @@ void setup()
}
}
/* Reset ModuleLLM */
M5.Display.printf(">> Reset ModuleLLM..\n");
module_llm.sys.reset();
/* Setup Audio module */
M5.Display.printf(">> Setup audio..\n");
module_llm.audio.setup();
/* Setup TTS module and save returned work id */
M5.Display.printf(">> Setup tts..\n\n");
tts_work_id = module_llm.tts.setup();
}
void loop()
{
// Make text
/* Make a text for speech: {i} plus {i} equals to {i + i} */
static int i = 0;
i++;
std::string text = std::to_string(i) + " plus " + std::to_string(i) + " equals " + std::to_string(i + i) + ".";
@@ -49,7 +55,7 @@ void loop()
M5.Display.setTextColor(TFT_GREEN);
M5.Display.printf("<< %s\n\n", text.c_str());
// Push text to tts and wait result
/* Push text to TTS module and wait inference result */
module_llm.tts.inference(tts_work_id, text.c_str(), 10000);
delay(500);
src/M5ModuleLLM.h
+101
View File
@@ -16,17 +16,75 @@
class M5ModuleLLM {
public:
/**
* @brief Init module
*
* @param targetPort module serial port
* @return true
* @return false
*/
bool begin(Stream* targetPort);
/**
* @brief Check if module is connected
*
* @return true
* @return false
*/
bool checkConnection();
/**
* @brief Update module
*
*/
void update();
/**
* @brief SYS module api set
*
*/
m5_module_llm::ApiSys sys;
/**
* @brief LLM module api set
*
*/
m5_module_llm::ApiLlm llm;
/**
* @brief Audio module api set
*
*/
m5_module_llm::ApiAudio audio;
/**
* @brief TTS module api set
*
*/
m5_module_llm::ApiTts tts;
/**
* @brief KWS module api set
*
*/
m5_module_llm::ApiKws kws;
/**
* @brief ASR module api set
*
*/
m5_module_llm::ApiAsr asr;
/**
* @brief MSG module to handle module response message
*
*/
m5_module_llm::ModuleMsg msg;
/**
* @brief COMM module to handle module communication stream
*
*/
m5_module_llm::ModuleComm comm;
private:
@@ -38,21 +96,64 @@ typedef std::function<void(String data, bool isFinish, int index)> OnLlmDataInpu
typedef std::function<void(String rawData)> OnAsrDataInputRawCallback_t;
typedef std::function<void(String rawData)> OnLlmDataInputRawCallback_t;
/**
* @brief Voice assistant preset base on class M5ModuleLLM
*
*/
class M5ModuleLLM_VoiceAssistant {
public:
M5ModuleLLM_VoiceAssistant(M5ModuleLLM* m5ModuleLlm) : _m5_module_llm(m5ModuleLlm)
{
}
/**
* @brief Start voice assistant preset
*
* @param wakeUpKeyword
* @param prompt
* @return int
*/
int begin(String wakeUpKeyword = "HELLO", String prompt = "");
/**
* @brief Update voice assistant preset, trigger callbacks
*
*/
void update();
/**
* @brief Register on key-word detected callback
*
* @param callback
*/
void onKeywordDetected(OnKeywordDetectedCallback_t callback);
/**
* @brief Register on ASR data input callback
*
* @param callback
*/
void onAsrDataInput(OnAsrDataInputCallback_t callback);
/**
* @brief Register on LLM data input callback
*
* @param callback
*/
void onLlmDataInput(OnLlmDataInputCallback_t callback);
/**
* @brief Register on ASR raw data (module response json) input callback
*
* @param callback
*/
void onAsrDataInputRaw(OnAsrDataInputRawCallback_t callback);
/**
* @brief Register on LLM raw data (module response json) input callback
*
* @param callback
*/
void onLlmDataInputRaw(OnLlmDataInputRawCallback_t callback);
private: