From 43c34385c93bfbb744e57d4344d348b5259e704f Mon Sep 17 00:00:00 2001
From: LittleMouse <niuchenyang@m5stack.com>
Date: Wed, 23 Apr 2025 09:05:27 +0800
Subject: [PATCH] [fix] Fix non-utf-8 characters

---
 .../llm_framework/main_whisper/src/main.cpp   | 30 ++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/projects/llm_framework/main_whisper/src/main.cpp b/projects/llm_framework/main_whisper/src/main.cpp
index 73515f1..cc59a97 100644
--- a/projects/llm_framework/main_whisper/src/main.cpp
+++ b/projects/llm_framework/main_whisper/src/main.cpp
@@ -192,6 +192,34 @@ public:
         return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
     }
 
+    bool is_valid_utf8(const std::string &str)
+    {
+        int bytes = 0;
+        for (unsigned char c : str) {
+            if (bytes == 0) {
+                if ((c >> 5) == 0b110)
+                    bytes = 1;
+                else if ((c >> 4) == 0b1110)
+                    bytes = 2;
+                else if ((c >> 3) == 0b11110)
+                    bytes = 3;
+                else if ((c >> 7))
+                    return false;
+            } else {
+                if ((c >> 6) != 0b10) return false;
+                bytes--;
+            }
+        }
+        return bytes == 0;
+    }
+
+    void fix_utf8_string(std::string &s)
+    {
+        while (!s.empty() && !is_valid_utf8(s)) {
+            s.pop_back();
+        }
+    }
+
     int load_model(const nlohmann::json &config_body)
     {
         if (parse_config(config_body)) {
@@ -475,7 +503,7 @@ public:
                           (uint32)mode_config_.token_tables[i].size(), str);
             s += str;
         }
-
+        fix_utf8_string(s);
         if (mode_config_.language == "en" || mode_config_.language == "ja") {
             if (out_callback_) out_callback_(s, true);
         } else {