diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f42b1e3..3449d1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,8 @@ -spec: - inputs: - force_push_to_github: - type: boolean - default: false - ---- - stages: - pre-check - build - deploy + - deploy-github workflow: rules: @@ -28,10 +21,11 @@ workflow: variables: IOT_SOLUTION_PATH: "$CI_PROJECT_DIR" - CI_FORCE_PUSH_TO_GITHUB: "$[[ inputs.force_push_to_github ]]" include: - '.gitlab/ci/rules.yml' - '.gitlab/ci/pre-check.yml' - '.gitlab/ci/build.yml' + - '.gitlab/ci/build-doc.yml' - '.gitlab/ci/deploy.yml' + - '.gitlab/ci/deploy-github.yml' diff --git a/.gitlab/ci/build-doc.yml b/.gitlab/ci/build-doc.yml new file mode 100644 index 0000000..41ae43f --- /dev/null +++ b/.gitlab/ci/build-doc.yml @@ -0,0 +1,26 @@ +build_doc: + stage: build + tags: + - build + image: node:24 + extends: + - .rules:build:doc + needs: + - job: build_application_basic_demo + artifacts: true + optional: true + script: + - apt update && apt install -y curl + - curl -fsSL https://d2lang.com/install.sh | sh -s -- + - npm install -g pnpm + - cd docs + - pnpm install + - pnpm run generate-firmware-json ../merged_binary + - pnpm run build + artifacts: + when: always + paths: + - docs/public/**/* + - docs/dist/**/* + - docs/src/flash-tool/firmware.json + expire_in: 1 week diff --git a/.gitlab/ci/deploy-github.yml b/.gitlab/ci/deploy-github.yml new file mode 100644 index 0000000..bde5035 --- /dev/null +++ b/.gitlab/ci/deploy-github.yml @@ -0,0 +1,24 @@ +push_to_github: + stage: deploy-github + tags: + - build + image: alpine:latest + variables: + GIT_STRATEGY: clone + GIT_DEPTH: "0" + extends: + - .rules:push:github + script: + - apk add --no-cache git openssh-client curl + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - printf '%s' "$GH_PUSH_KEY" | base64 -d > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null || true + - git remote remove github 2>/dev/null || true + - git remote add github "$GH_REMOTE_URL" + - git push github "refs/heads/master:refs/heads/master" + - | + for tag in $(git tag --merged refs/heads/master); do + git push github "refs/tags/$tag:refs/tags/$tag" + done diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index 921b8a4..b6c3dfe 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -1,35 +1,35 @@ -collect_firmware_manifest: - extends: - - .rules:build:application_basic_demo +deploy_website: stage: deploy tags: - build - image: python:3.13-slim - allow_failure: true + image: node:24-alpine variables: GIT_STRATEGY: fetch + environment: + name: $DEPLOY_ENV_NAME + url: $DEPLOY_ENV_URL + deployment_tier: $DEPLOY_ENV_TIER rules: - - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "push"' + - if: '$CI_COMMIT_REF_NAME == "master" && $CI_COMMIT_TAG == null && $CI_PIPELINE_SOURCE == "push"' when: on_success variables: - DEPLOY_FIRMWARE: "1" - - if: '$CI_COMMIT_REF_NAME != "master" && $CI_PIPELINE_SOURCE == "merge_request_event"' + DEPLOY_ENV_NAME: docs-production + DEPLOY_ENV_URL: https://esp-claw.com + DEPLOY_ENV_TIER: production + DEPLOY_BRANCH: master + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + changes: + - "docs/**/*" when: on_success variables: - DEPLOY_FIRMWARE: "0" + DEPLOY_ENV_NAME: docs-preview/$CI_COMMIT_REF_SLUG + DEPLOY_ENV_URL: https://$CI_COMMIT_REF_SLUG.$PREVIEW_DOMAIN + DEPLOY_ENV_TIER: development + DEPLOY_BRANCH: $CI_COMMIT_REF_NAME - when: never needs: - - job: build_application_basic_demo + - job: build_doc artifacts: true - optional: true script: - - apt-get update - - apt-get install -y curl - - python .gitlab/ci/generate_firmware_json.py - - python .gitlab/ci/deploy_firmware.py - artifacts: - when: always - paths: - - firmware.json - - merged_binary/**/* - expire_in: 1 week + - cd docs + - npx wrangler pages deploy "dist/" --project-name "$SITE_DEPLOY_PROJECT_NAME" --branch="$DEPLOY_BRANCH" diff --git a/.gitlab/ci/deploy_firmware.py b/.gitlab/ci/deploy_firmware.py deleted file mode 100644 index 89d5058..0000000 --- a/.gitlab/ci/deploy_firmware.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -""" -Deploy firmware artifacts to the download server. - -Behavior is controlled by the DEPLOY_FIRMWARE environment variable: - "1" - package firmware.json + merged_binary/ into esp-claw-firmware.zip and upload - "0" - print a skip message and exit -""" - -import os -import subprocess -import sys -import zipfile -from pathlib import Path - -ARCHIVE_NAME = 'esp-claw-firmware.zip' - - -def build_archive() -> Path: - archive_path = Path(ARCHIVE_NAME) - with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) as zf: - firmware_json = Path('firmware.json') - if firmware_json.exists(): - zf.write(firmware_json, firmware_json.name) - print(f" added: {firmware_json}") - else: - print('WARNING: firmware.json not found, skipping.', file=sys.stderr) - - merged_binary = Path('merged_binary') - if merged_binary.exists(): - for file in sorted(merged_binary.rglob('*')): - if file.is_file(): - zf.write(file, file.as_posix()) - print(f" added: {file}") - else: - print('WARNING: merged_binary/ directory not found, skipping.', file=sys.stderr) - - print(f"Archive created: {archive_path} ({archive_path.stat().st_size} bytes)") - return archive_path - - -def upload_archive(archive_path: Path) -> None: - upload_key = os.environ.get('DL_UPLOAD_KEY') - upload_api_url = os.environ.get('DL_UPLOAD_API_URL') - upload_path = os.environ.get('DL_UPLOAD_PATH') - - missing = [var for var, val in [ - ('DL_UPLOAD_KEY', upload_key), - ('DL_UPLOAD_API_URL', upload_api_url), - ('DL_UPLOAD_PATH', upload_path), - ] if not val] - - if missing: - print(f"ERROR: Missing required environment variables: {', '.join(missing)}", file=sys.stderr) - sys.exit(1) - - cmd = [ - 'curl', - '--fail', - '-u', upload_key, - '-F', f"file=@{archive_path}", - '-F', f"path={upload_path}", - upload_api_url, - ] - - print(f"Uploading {archive_path} to {upload_api_url} (path={upload_path}) ...") - result = subprocess.run(cmd, check=False) - if result.returncode != 0: - print(f"ERROR: Upload failed with exit code {result.returncode}", file=sys.stderr) - sys.exit(result.returncode) - - print('Upload succeeded.') - - -def main() -> None: - deploy_firmware = os.environ.get('DEPLOY_FIRMWARE', '0') - - if deploy_firmware != '1': - print(f"DEPLOY_FIRMWARE={deploy_firmware!r}: skipping firmware deployment.") - sys.exit(0) - - print('DEPLOY_FIRMWARE=1: packaging and uploading firmware artifacts ...') - archive_path = build_archive() - upload_archive(archive_path) - - -if __name__ == '__main__': - main() diff --git a/.gitlab/ci/generate_firmware_json.py b/.gitlab/ci/generate_firmware_json.py deleted file mode 100644 index 1699b7f..0000000 --- a/.gitlab/ci/generate_firmware_json.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD -# -# SPDX-License-Identifier: Apache-2.0 - -import json -import os -import sys -from pathlib import Path -from typing import Any, Dict, List - - -def _log(msg: str) -> None: - print(msg, file=sys.stderr) - - -def _parse_flash_mb(value: Any) -> int: - if isinstance(value, int): - return value - - if not isinstance(value, str): - raise ValueError(f'unsupported min_flash_size type: {type(value)}') - - text = value.strip() - if not text: - raise ValueError('empty min_flash_size') - - upper = text.upper() - if upper.endswith('MB'): - return int(upper[:-2].strip(), 0) - if upper.endswith('M'): - return int(upper[:-1].strip(), 0) - - num = int(text, 0) - if num % (1024 * 1024) == 0: - return num // (1024 * 1024) - - return num - - -def _join_base_url(base_url: str, filename: str) -> str: - if not base_url: - base_url = '/' - - if base_url.endswith('/'): - return f'{base_url}{filename}' - - return f'{base_url}/{filename}' - - -def _load_metadata_files(merged_dir: Path) -> List[Dict[str, Any]]: - metadata_files = sorted(merged_dir.glob('*.json')) - if not metadata_files: - raise RuntimeError(f'No metadata json found in {merged_dir}') - - records: List[Dict[str, Any]] = [] - for path in metadata_files: - with path.open('r', encoding='utf-8') as fr: - data = json.load(fr) - if not isinstance(data, dict): - raise RuntimeError(f'Invalid metadata format: {path}') - records.append(data) - - return records - - -def main() -> int: - cwd = Path.cwd() - merged_dir = cwd / 'merged_binary' - - if not merged_dir.is_dir(): - _log(f'merged_binary directory not found: {merged_dir}') - return 1 - - base_url = os.getenv('MERGED_BINARY_BASE_URL', '/').strip() or '/' - - try: - records = _load_metadata_files(merged_dir) - except Exception as e: - _log(str(e)) - return 1 - - firmware: Dict[str, List[Dict[str, Any]]] = {} - - for record in records: - chip = record.get('chip') - board = record.get('board') - merged_binary = record.get('merged_binary') - min_flash_size = record.get('min_flash_size') - nvs_info = record.get('nvs_info') - - if not isinstance(chip, str) or not chip.strip(): - _log(f'skip one metadata: invalid chip ({record})') - continue - if not isinstance(board, str) or not board.strip(): - _log(f'skip one metadata: invalid board ({record})') - continue - if not isinstance(merged_binary, str) or not merged_binary.strip(): - _log(f'skip one metadata: invalid merged_binary ({record})') - continue - if not isinstance(nvs_info, dict): - _log(f'skip one metadata: invalid nvs_info ({record})') - continue - - try: - min_flash_mb = _parse_flash_mb(min_flash_size) - except Exception as e: - _log(f'skip one metadata: invalid min_flash_size ({record}) ({e})') - continue - - item = { - 'board': board, - 'features': [], - 'description': '', - 'merged_binary': _join_base_url(base_url, merged_binary), - 'min_flash_size': min_flash_mb, - 'min_psram_size': 8, - 'nvs_info': { - 'start_addr': str(nvs_info.get('start_addr', '')), - 'size': str(nvs_info.get('size', '')), - }, - } - - firmware.setdefault(chip, []).append(item) - - if not firmware: - _log('No valid metadata collected from merged_binary/*.json') - return 1 - - for chip in firmware: - firmware[chip].sort(key=lambda x: x['board']) - - out_file = cwd / 'firmware.json' - out_file.write_text(json.dumps(firmware, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') - print(f'Generated: {out_file}') - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index fc832f0..20015db 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -10,78 +10,38 @@ - "pytest.ini" # components folder, in the alphabetic order -.patterns-components_cap_cli: &patterns-components_cap_cli - - "components/cap_cli/**/*" +.patterns-components_capabilities: &patterns-components_capabilities + - "components/claw_capabilities/**/*" -.patterns-components_cap_files: &patterns-components_cap_files - - "components/cap_files/**/*" +.patterns-components_modules: &patterns-components_modules + - "components/claw_modules/**/*" -.patterns-components_cap_im_attachment: &patterns-components_cap_im_attachment - - "components/cap_im_attachment/**/*" - -.patterns-components_cap_im_qq: &patterns-components_cap_im_qq - - "components/cap_im_qq/**/*" - -.patterns-components_cap_im_tg: &patterns-components_cap_im_tg - - "components/cap_im_tg/**/*" - -.patterns-components_cap_im_wechat: &patterns-components_cap_im_wechat - - "components/cap_im_wechat/**/*" - -.patterns-components_cap_llm_inspect: &patterns-components_cap_llm_inspect - - "components/cap_llm_inspect/**/*" - -.patterns-components_cap_lua: &patterns-components_cap_lua - - "components/cap_lua/**/*" - -.patterns-components_cap_mcp_client: &patterns-components_cap_mcp_client - - "components/cap_mcp_client/**/*" - -.patterns-components_cap_mcp_server: &patterns-components_cap_mcp_server - - "components/cap_mcp_server/**/*" - -.patterns-components_cap_skill: &patterns-components_cap_skill - - "components/cap_skill/**/*" - -.patterns-components_cap_time: &patterns-components_cap_time - - "components/cap_time/**/*" - -.patterns-components_cap_web_search: &patterns-components_cap_web_search - - "components/cap_web_search/**/*" - -.patterns-components_claw_cap: &patterns-components_claw_cap - - "components/claw_cap/**/*" - -.patterns-components_claw_core: &patterns-components_claw_core - - "components/claw_core/**/*" - -.patterns-components_claw_event_router: &patterns-components_claw_event_router - - "components/claw_event_router/**/*" - -.patterns-components_claw_memory: &patterns-components_claw_memory - - "components/claw_memory/**/*" - -.patterns-components_claw_skill: &patterns-components_claw_skill - - "components/claw_skill/**/*" +.patterns-components_lua_modules: &patterns-components_lua_modules + - "components/lua_modules/**/*" # application folder, in the alphabetic order .patterns-application_basic_demo: &patterns-application_basic_demo - "application/basic_demo/**/*" -.patterns-yml: &patterns-yml - - "**/*.yml" - .patterns-docs: &patterns-docs - "docs/**/*" ############## # if anchors # ############## -.if-protected: &if-protected - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)' -.if-dev-push: &if-dev-push - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' +# Push to any protected ref +.if-protected-push: &if-protected-push + if: '$CI_COMMIT_REF_PROTECTED == "true" && $CI_PIPELINE_SOURCE == "push"' + +# master push only +.if-master-push: &if-master-push + if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "push"' + +# Merge request pipeline (dev branches). Non-protected branch push pipelines are +# disabled at workflow level (see .gitlab-ci.yml), so we only care about MR here. +.if-mr: &if-mr + if: '$CI_PIPELINE_SOURCE == "merge_request_event"' ################## # Auto Generated # @@ -92,69 +52,51 @@ .if-label-build: &if-label-build if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i' -.if-label-target_test: &if-label-target_test - if: '$BOT_LABEL_TARGET_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' - -.if-label-build_docs: &if-label-build_docs - if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i' - .if-label-pre_check: &if-label-pre_check if: '$BOT_LABEL_PRE_CHECK || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*pre_check(?:,[^,\n\r]+)*$/i' -.if_label-deploy: &if-label-deploy - if: '$BOT_LABEL_DEPLOY || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*deploy(?:,[^,\n\r]+)*$/i' +######### +# Rules # +######### -# rules for applications .rules:build:application_basic_demo: rules: - - <<: *if-protected + - <<: *if-protected-push - <<: *if-label-build - <<: *if-trigger-job - - <<: *if-dev-push + - <<: *if-mr changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-components_cap_cli - - <<: *if-dev-push - changes: *patterns-components_cap_files - - <<: *if-dev-push - changes: *patterns-components_cap_im_attachment - - <<: *if-dev-push - changes: *patterns-components_cap_im_qq - - <<: *if-dev-push - changes: *patterns-components_cap_im_tg - - <<: *if-dev-push - changes: *patterns-components_cap_im_wechat - - <<: *if-dev-push - changes: *patterns-components_cap_llm_inspect - - <<: *if-dev-push - changes: *patterns-components_cap_lua - - <<: *if-dev-push - changes: *patterns-components_cap_mcp_client - - <<: *if-dev-push - changes: *patterns-components_cap_mcp_server - - <<: *if-dev-push - changes: *patterns-components_cap_skill - - <<: *if-dev-push - changes: *patterns-components_cap_time - - <<: *if-dev-push - changes: *patterns-components_cap_web_search - - <<: *if-dev-push - changes: *patterns-components_claw_cap - - <<: *if-dev-push - changes: *patterns-components_claw_core - - <<: *if-dev-push - changes: *patterns-components_claw_event_router - - <<: *if-dev-push - changes: *patterns-components_claw_memory - - <<: *if-dev-push - changes: *patterns-components_claw_skill - - <<: *if-dev-push + - <<: *if-mr + changes: *patterns-components_capabilities + - <<: *if-mr + changes: *patterns-components_modules + - <<: *if-mr + changes: *patterns-components_lua_modules + - <<: *if-mr changes: *patterns-application_basic_demo + - <<: *if-mr + changes: *patterns-docs + - when: never .rules:pre_check:docs: rules: - - <<: *if-protected + - <<: *if-protected-push - <<: *if-label-pre_check - <<: *if-trigger-job - - <<: *if-dev-push + - <<: *if-mr changes: *patterns-docs + - when: never + +.rules:build:doc: + rules: + - <<: *if-protected-push + - <<: *if-label-build + - <<: *if-trigger-job + - <<: *if-mr + changes: *patterns-docs + - when: never + +.rules:push:github: + rules: + - <<: *if-master-push + - when: never diff --git a/README.md b/README.md index 73d0eb3..ee4149d 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,16 @@ esp-claw/ If this project inspires or helps you, please consider giving it a star. ⭐⭐⭐⭐⭐ +### Star History + + + + + + Star History Chart + + + ## Acknowledgements ESP-Claw is inspired by [OpenClaw](https://github.com/openclaw/openclaw). diff --git a/README_CN.md b/README_CN.md index c56ce59..555e87c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -168,6 +168,16 @@ esp-claw/ 如果这个项目对您有所启发和帮助，欢迎点亮一颗星！⭐⭐⭐⭐⭐ +### Star History + + + + + + Star History Chart + + + ## 致谢 灵感来自 [OpenClaw](https://github.com/openclaw/openclaw)。 diff --git a/docs/package.json5 b/docs/package.json5 index 1feccce..e9f1037 100644 --- a/docs/package.json5 +++ b/docs/package.json5 @@ -8,7 +8,8 @@ "build": "astro build", "preview": "astro preview", "astro": "astro", - "check:doc-lines": "node ./tools/check-doc-line-alignment.ts" + "check:doc-lines": "node ./tools/check-doc-line-alignment.ts", + "generate-firmware-json": "node ./tools/generate-firmware-json.ts" }, "dependencies": { "@astrojs/check": "^0.9.8", diff --git a/docs/public/_redirects b/docs/public/_redirects new file mode 100644 index 0000000..d6ad167 --- /dev/null +++ b/docs/public/_redirects @@ -0,0 +1,2 @@ +/ /zh-cn/ 302 +/esp-claw/* /:splat 302 diff --git a/docs/src/assets/features/chat-coding-en.webm b/docs/src/assets/features/chat-coding-en.webm new file mode 100644 index 0000000..9ae0a3f Binary files /dev/null and b/docs/src/assets/features/chat-coding-en.webm differ diff --git a/docs/src/assets/features/event-en.webm b/docs/src/assets/features/event-en.webm new file mode 100644 index 0000000..71d6615 Binary files /dev/null and b/docs/src/assets/features/event-en.webm differ diff --git a/docs/src/assets/features/locallua-en.webm b/docs/src/assets/features/locallua-en.webm new file mode 100644 index 0000000..97b2c22 Binary files /dev/null and b/docs/src/assets/features/locallua-en.webm differ diff --git a/docs/src/assets/features/mcp-en.webm b/docs/src/assets/features/mcp-en.webm new file mode 100644 index 0000000..928bf28 Binary files /dev/null and b/docs/src/assets/features/mcp-en.webm differ diff --git a/docs/src/assets/features/memory-en.webm b/docs/src/assets/features/memory-en.webm new file mode 100644 index 0000000..95a2d3b Binary files /dev/null and b/docs/src/assets/features/memory-en.webm differ diff --git a/docs/src/assets/features/multi-en.webm b/docs/src/assets/features/multi-en.webm new file mode 100644 index 0000000..978ba02 Binary files /dev/null and b/docs/src/assets/features/multi-en.webm differ diff --git a/docs/src/content/docs/en/reference-project/build-from-source.mdx b/docs/src/content/docs/en/reference-project/build-from-source.mdx index 73ffd93..e680b3e 100644 --- a/docs/src/content/docs/en/reference-project/build-from-source.mdx +++ b/docs/src/content/docs/en/reference-project/build-from-source.mdx @@ -25,10 +25,13 @@ import { Steps, Aside, Badge } from '@astrojs/starlight/components'; pip install esp-bmgr-assist ``` -