diff --git a/application/basic_demo/main/app_claw.c b/application/basic_demo/main/app_claw.c index 5e70788..8b4c9af 100644 --- a/application/basic_demo/main/app_claw.c +++ b/application/basic_demo/main/app_claw.c @@ -40,9 +40,7 @@ static const char *TAG = "app_esp_claw"; #if CONFIG_BASIC_DEMO_MEMORY_MODE_FULL static const char *const BASIC_DEMO_LLM_VISIBLE_GROUPS[] = { - "cap_cli", "cap_files", - "cap_router_mgr", "cap_skill", "claw_memory", }; @@ -56,7 +54,7 @@ static const char *const BASIC_DEMO_LLM_VISIBLE_GROUPS[] = { #define BASIC_DEMO_IM_ATTACHMENT_MAX_BYTES (2 * 1024 * 1024) #define BASIC_DEMO_SYSTEM_PROMPT_COMMON \ - "You are the ESP-Claw running on ESP32. " \ + "You are the ESP-Claw. " \ "Answer briefly and plainly. " \ "Treat Skills List as a catalog of optional skills." \ "Use 'activate_skill' to load a skill, and you will gain more callable capabilities\n" \ @@ -362,7 +360,7 @@ esp_err_t app_claw_start(const basic_demo_settings_t *settings) core_config.task_stack_size = 6 * 1024; core_config.task_priority = 5; core_config.task_core = tskNO_AFFINITY; - core_config.max_tool_iterations = 10; + core_config.max_tool_iterations = 20; core_config.request_queue_len = 4; core_config.response_queue_len = 4; core_config.max_context_providers = 6; diff --git a/components/claw_capabilities/cap_im_attachment/src/cap_im_attachment.c b/components/claw_capabilities/cap_im_attachment/src/cap_im_attachment.c index 65a265f..bbc58f3 100644 --- a/components/claw_capabilities/cap_im_attachment/src/cap_im_attachment.c +++ b/components/claw_capabilities/cap_im_attachment/src/cap_im_attachment.c @@ -137,6 +137,30 @@ const char *cap_im_attachment_ext_from_mime(const char *mime) if (strcmp(mime, "image/webp") == 0) { return ".webp"; } + if (strcmp(mime, "audio/mpeg") == 0) { + return ".mp3"; + } + if (strcmp(mime, "audio/mp3") == 0) { + return ".mp3"; + } + if (strcmp(mime, "audio/wav") == 0 || strcmp(mime, "audio/x-wav") == 0) { + return ".wav"; + } + if (strcmp(mime, "audio/ogg") == 0) { + return ".ogg"; + } + if (strcmp(mime, "audio/aac") == 0) { + return ".aac"; + } + if (strcmp(mime, "audio/amr") == 0) { + return ".amr"; + } + if (strcmp(mime, "audio/silk") == 0) { + return ".silk"; + } + if (strcmp(mime, "audio/mp4") == 0) { + return ".m4a"; + } if (strcmp(mime, "application/pdf") == 0) { return ".pdf"; } diff --git a/components/claw_capabilities/cap_im_qq/src/cap_im_qq.c b/components/claw_capabilities/cap_im_qq/src/cap_im_qq.c index 6c5dfcc..b9eb099 100644 --- a/components/claw_capabilities/cap_im_qq/src/cap_im_qq.c +++ b/components/claw_capabilities/cap_im_qq/src/cap_im_qq.c @@ -97,6 +97,13 @@ typedef struct { size_t len; } cap_im_qq_inbound_frame_t; +typedef struct { + char *buf; + size_t len; + size_t cap; + int payload_len; +} cap_im_qq_ws_assembly_t; + typedef struct { char app_id[64]; char app_secret[128]; @@ -116,6 +123,7 @@ typedef struct { volatile bool ws_identify_pending; volatile bool ws_should_reconnect; volatile bool stop_requested; + cap_im_qq_ws_assembly_t ws_assembly; uint64_t seen_msg_keys[CAP_IM_QQ_DEDUP_CACHE_SIZE]; size_t seen_msg_idx; } cap_im_qq_state_t; @@ -186,6 +194,99 @@ static bool cap_im_qq_is_image_mime(const char *mime) return mime && strncmp(mime, "image/", 6) == 0; } +static bool cap_im_qq_is_audio_mime(const char *mime) +{ + return mime && strncmp(mime, "audio/", 6) == 0; +} + +static void cap_im_qq_reset_ws_assembly(void) +{ + free(s_qq.ws_assembly.buf); + s_qq.ws_assembly.buf = NULL; + s_qq.ws_assembly.len = 0; + s_qq.ws_assembly.cap = 0; + s_qq.ws_assembly.payload_len = 0; +} + +static esp_err_t cap_im_qq_queue_inbound_frame(const char *frame, size_t frame_len) +{ + cap_im_qq_inbound_frame_t item = {0}; + + if (!frame || frame_len == 0 || !s_qq.inbound_queue) { + return ESP_ERR_INVALID_ARG; + } + + item.frame = calloc(1, frame_len + 1); + if (!item.frame) { + return ESP_ERR_NO_MEM; + } + + memcpy(item.frame, frame, frame_len); + item.len = frame_len; + if (xQueueSend(s_qq.inbound_queue, &item, 0) != pdTRUE) { + free(item.frame); + return ESP_ERR_NO_MEM; + } + + return ESP_OK; +} + +static esp_err_t cap_im_qq_handle_inbound_ws_data(const esp_websocket_event_data_t *event) +{ + size_t needed = 0; + + if (!event || event->op_code != 0x01 || !event->data_ptr || event->data_len <= 0) { + return ESP_ERR_INVALID_ARG; + } + if (!s_qq.inbound_queue) { + return ESP_ERR_INVALID_STATE; + } + + if (event->payload_len <= 0 || + (event->payload_offset == 0 && event->payload_len == event->data_len)) { + return cap_im_qq_queue_inbound_frame(event->data_ptr, (size_t)event->data_len); + } + + if (event->payload_offset == 0) { + needed = (size_t)event->payload_len + 1; + if (needed > s_qq.ws_assembly.cap) { + char *tmp = realloc(s_qq.ws_assembly.buf, needed); + + if (!tmp) { + cap_im_qq_reset_ws_assembly(); + return ESP_ERR_NO_MEM; + } + s_qq.ws_assembly.buf = tmp; + s_qq.ws_assembly.cap = needed; + } + s_qq.ws_assembly.len = 0; + s_qq.ws_assembly.payload_len = event->payload_len; + } else if (!s_qq.ws_assembly.buf || + s_qq.ws_assembly.payload_len != event->payload_len || + s_qq.ws_assembly.len != (size_t)event->payload_offset) { + cap_im_qq_reset_ws_assembly(); + return ESP_FAIL; + } + + memcpy(s_qq.ws_assembly.buf + s_qq.ws_assembly.len, event->data_ptr, (size_t)event->data_len); + s_qq.ws_assembly.len += (size_t)event->data_len; + s_qq.ws_assembly.buf[s_qq.ws_assembly.len] = '\0'; + if ((int)s_qq.ws_assembly.len < s_qq.ws_assembly.payload_len) { + return ESP_OK; + } + if ((int)s_qq.ws_assembly.len != s_qq.ws_assembly.payload_len) { + cap_im_qq_reset_ws_assembly(); + return ESP_FAIL; + } + + { + esp_err_t err = cap_im_qq_queue_inbound_frame(s_qq.ws_assembly.buf, s_qq.ws_assembly.len); + + cap_im_qq_reset_ws_assembly(); + return err; + } +} + static esp_err_t cap_im_qq_http_event_handler(esp_http_client_event_t *event) { cap_im_qq_http_resp_t *resp = (cap_im_qq_http_resp_t *)event->user_data; @@ -741,6 +842,8 @@ static void cap_im_qq_handle_attachments(cJSON *attachments, } if (cap_im_qq_is_image_mime(mime)) { kind = "image"; + } else if (cap_im_qq_is_audio_mime(mime)) { + kind = "file"; } { @@ -762,13 +865,65 @@ static void cap_im_qq_handle_attachments(cJSON *attachments, } } +static void cap_im_qq_handle_attachment_field(cJSON *field, + const char *chat_id, + const char *sender_id, + const char *message_id) +{ + if (cJSON_IsArray(field)) { + cap_im_qq_handle_attachments(field, chat_id, sender_id, message_id); + } else if (cJSON_IsObject(field)) { + cJSON *array = cJSON_CreateArray(); + + if (!array) { + return; + } + cJSON_AddItemReferenceToArray(array, field); + cap_im_qq_handle_attachments(array, chat_id, sender_id, message_id); + cJSON_Delete(array); + } +} + +static void cap_im_qq_handle_all_media(cJSON *data, + const char *chat_id, + const char *sender_id, + const char *message_id) +{ + static const char *const media_fields[] = { + "attachments", + "audio", + "audios", + "voice", + "voices", + "record", + "records", + "file", + "files", + "video", + "videos", + }; + size_t i; + + if (!cJSON_IsObject(data)) { + return; + } + + for (i = 0; i < sizeof(media_fields) / sizeof(media_fields[0]); i++) { + cJSON *field = cJSON_GetObjectItem(data, media_fields[i]); + + if (!field) { + continue; + } + cap_im_qq_handle_attachment_field(field, chat_id, sender_id, message_id); + } +} + static void cap_im_qq_handle_dispatch(cJSON *data, const char *event_type) { char chat_id[96] = {0}; char sender_id[96] = {0}; const char *content = NULL; const char *message_id = NULL; - cJSON *attachments = NULL; if (!data || !event_type) { return; @@ -787,7 +942,6 @@ static void cap_im_qq_handle_dispatch(cJSON *data, const char *event_type) strlcpy(sender_id, openid->valuestring, sizeof(sender_id)); content = cJSON_IsString(content_json) ? content_json->valuestring : ""; message_id = id_json->valuestring; - attachments = cJSON_GetObjectItem(data, "attachments"); } else if (strcmp(event_type, "GROUP_AT_MESSAGE_CREATE") == 0) { cJSON *group = cJSON_GetObjectItem(data, "group_openid"); cJSON *author = cJSON_GetObjectItem(data, "author"); @@ -804,7 +958,6 @@ static void cap_im_qq_handle_dispatch(cJSON *data, const char *event_type) } content = cJSON_IsString(content_json) ? content_json->valuestring : ""; message_id = id_json->valuestring; - attachments = cJSON_GetObjectItem(data, "attachments"); } else { ESP_LOGI(TAG, "QQ dispatch type %s is not handled", event_type); return; @@ -814,7 +967,7 @@ static void cap_im_qq_handle_dispatch(cJSON *data, const char *event_type) return; } - cap_im_qq_handle_attachments(attachments, chat_id, sender_id, message_id); + cap_im_qq_handle_all_media(data, chat_id, sender_id, message_id); if (content && content[0]) { if (cap_im_qq_publish_inbound_text(chat_id, sender_id, message_id, content) == ESP_OK) { @@ -942,21 +1095,15 @@ static void cap_im_qq_ws_event_handler(void *arg, } if (s_qq.inbound_queue) { - cap_im_qq_inbound_frame_t item = { - .frame = calloc(1, (size_t)event->data_len + 1), - .len = (size_t)event->data_len, - }; + esp_err_t err = cap_im_qq_handle_inbound_ws_data(event); - if (!item.frame) { - ESP_LOGW(TAG, "QQ inbound queue alloc failed len=%d", event->data_len); - return; - } - - memcpy(item.frame, event->data_ptr, item.len); - if (xQueueSend(s_qq.inbound_queue, &item, 0) != pdTRUE) { - ESP_LOGW(TAG, "QQ inbound queue full, dropping frame len=%d", event->data_len); - free(item.frame); - } else { + if (err != ESP_OK && err != ESP_ERR_INVALID_ARG) { + ESP_LOGW(TAG, + "QQ inbound WS frame handling failed len=%d payload_len=%d offset=%d err=%s", + event->data_len, + event->payload_len, + event->payload_offset, + esp_err_to_name(err)); } } } @@ -1498,6 +1645,7 @@ static void cap_im_qq_ws_task(void *arg) static void cap_im_qq_reset_runtime_state(void) { + cap_im_qq_reset_ws_assembly(); s_qq.ws_client = NULL; s_qq.ws_task = NULL; s_qq.inbound_task = NULL; diff --git a/components/claw_capabilities/cap_time/skills/cap_time.md b/components/claw_capabilities/cap_time/skills/cap_time.md new file mode 100644 index 0000000..312f5f7 --- /dev/null +++ b/components/claw_capabilities/cap_time/skills/cap_time.md @@ -0,0 +1,64 @@ +# Time Sync + +Use this skill when the user needs the current real-world time from the network, especially when local device time may be stale or invalid. + +## When to use +- The user asks for the current date, time, weekday, or local time. +- A task depends on fresh real-world time rather than model memory. +- The user asks whether device time is valid or wants the device clock synced now. + +## Available capability +- `get_current_time`: fetch current network time, update the device clock, and return formatted local time. + +## Current scope +- The callable capability exposed to the agent is `get_current_time`. +- It takes an empty input object. +- It returns formatted plain text such as local date, time, timezone, and weekday. +- There is also device-side support for setting timezone through `cap_time_set_timezone(...)` and the `time --set-timezone` CLI command, but that is not currently exposed as a callable LLM capability. + +## Calling rules +- Call `get_current_time` directly when the user needs real current time. +- Input should be an empty object: + +```json +{} +``` + +- Prefer this capability over guessing from model knowledge when the exact current time matters. +- Do not claim a timezone change was performed through this skill unless some other runtime path explicitly handled it. + +## Output behavior +- On success, the capability returns formatted local time text. +- It also updates the device clock from the network. +- On failure, the capability returns an error string similar to: + - `Error: failed to fetch time (...)` + +## Timezone notes +- The returned time is formatted using the device's currently configured timezone. +- Default timezone in the component is `UTC0` until changed elsewhere. +- If the user asks to change timezone, do not pretend `get_current_time` can do it. Use an explicit non-LLM control path if available. + +## Recommended workflow +1. Decide whether the user needs real current time or just a stable explanation about time handling. +2. Call `get_current_time` with `{}`. +3. Return the formatted result to the user. +4. If the user also needs timezone reconfiguration, handle that separately from this skill. + +## Common failure causes +- Expecting `get_current_time` to accept parameters; it does not. +- Expecting this skill alone to change timezone. +- Calling it when network access is unavailable, causing sync failure. + +## Examples + +Get current local device time: + +```json +{} +``` + +Use before a time-sensitive answer: + +```json +{} +``` diff --git a/components/claw_capabilities/cap_time/skills/skills_list.json b/components/claw_capabilities/cap_time/skills/skills_list.json new file mode 100644 index 0000000..9ea4b99 --- /dev/null +++ b/components/claw_capabilities/cap_time/skills/skills_list.json @@ -0,0 +1,12 @@ +{ + "skills": [ + { + "id": "cap_time", + "file": "cap_time.md", + "summary": "Fetch and sync current network time through the get_current_time capability.", + "cap_groups": [ + "cap_time" + ] + } + ] +} diff --git a/components/claw_capabilities/cap_web_search/skills/cap_web_search.md b/components/claw_capabilities/cap_web_search/skills/cap_web_search.md new file mode 100644 index 0000000..36e8cbe --- /dev/null +++ b/components/claw_capabilities/cap_web_search/skills/cap_web_search.md @@ -0,0 +1,85 @@ +# Web Search + +Use this skill when the user needs current public web information that should be fetched through the built-in search capability instead of answered from memory. + +## When to use +- The user asks for current news, recent developments, live information, or up-to-date facts. +- The user asks to search the web for a topic, company, product, documentation page, or article. +- The task needs a few likely sources or result links before summarizing. +- The answer would be risky if guessed from stale model knowledge. + +## Available capability +- `web_search`: search the web with the configured provider and return concise formatted results. + +## Provider behavior +- The runtime prefers `Tavily` when a Tavily API key is configured. +- If Tavily is not configured but Brave Search is configured, it falls back to `Brave`. +- If neither provider key is configured, the capability returns an error instead of search results. + +## Calling rules +- Call `web_search` directly. Do not route web search through CLI wrappers unless the user explicitly asks for console commands. +- Input must be a JSON object with one required field: + +```json +{ + "query": "your search query" +} +``` + +- Keep queries short and concrete. Prefer one clear search intent per call. +- If the user asks a compound question, split it into separate searches when needed instead of cramming everything into one long query. +- Use search when freshness matters. Do not use it for stable facts that can be answered locally. + +## Output shape +- The capability returns plain text, not structured JSON. +- Results are formatted as a short numbered list. +- Each item typically contains: + - title + - URL + - short snippet/content +- If no result is found, the output is `No web results found.` +- Common error strings include: + - `Error: no search provider credentials configured` + - `Error: invalid input JSON` + - `Error: missing query` + - `Error: search request failed (...)` + - `Error: failed to parse search results` + +## Recommended workflow +1. Decide whether the user needs fresh web information or a normal answer. +2. Write one focused query. +3. Call `web_search`. +4. Read the returned snippets and URLs. +5. Summarize the findings for the user and mention uncertainty when the results are weak or mixed. + +## Common failure causes +- Calling `web_search` with no `query`. +- Using a vague query like `news` or `weather` without the target subject or location. +- Expecting structured JSON output; this capability returns formatted text. +- Assuming the provider is available when no API key is configured. + +## Examples + +Search for current documentation: + +```json +{ + "query": "ESP-IDF mDNS example" +} +``` + +Search for a recent topic: + +```json +{ + "query": "latest Espressif ESP32 AI news" +} +``` + +Search for a product or company: + +```json +{ + "query": "Brave Search API pricing" +} +``` diff --git a/components/claw_capabilities/cap_web_search/skills/skills_list.json b/components/claw_capabilities/cap_web_search/skills/skills_list.json new file mode 100644 index 0000000..8034be2 --- /dev/null +++ b/components/claw_capabilities/cap_web_search/skills/skills_list.json @@ -0,0 +1,12 @@ +{ + "skills": [ + { + "id": "cap_web_search", + "file": "cap_web_search.md", + "summary": "Search the public web for current information through the configured Tavily or Brave provider.", + "cap_groups": [ + "cap_web_search" + ] + } + ] +}