From 9a14d45799280d9558129a9357fdc340441bd2c7 Mon Sep 17 00:00:00 2001 From: LittleMouse Date: Wed, 27 Aug 2025 16:10:46 +0800 Subject: [PATCH] [update] support ax650. add ax650 model. --- ext_components/ax_msp/SConstruct | 57 +++++++++ projects/llm_framework/main_audio/SConstruct | 6 +- .../llm_framework/main_audio/src/main.cpp | 27 ++++- .../main_audio/src/sample_audio.c | 27 ++++- .../main_audio/src/sample_audio.h | 5 + projects/llm_framework/main_camera/SConstruct | 10 +- .../main_camera/src/axera_camera.c | 20 ++-- .../llm_framework/main_camera/src/main.cpp | 17 ++- .../mode_depth-anything-ax6350.json | 24 ++++ .../main_depth_anything/src/EngineWrapper.cpp | 2 +- .../models/mode_melotts-en-au-ax650.json | 31 +++++ .../models/mode_melotts-en-br-ax650.json | 31 +++++ .../models/mode_melotts-en-default-ax650.json | 31 +++++ .../models/mode_melotts-en-india-ax650.json | 31 +++++ .../models/mode_melotts-en-us-ax650.json | 31 +++++ .../models/mode_melotts-es-es-ax650.json | 29 +++++ .../models/mode_melotts-ja-jp-ax650.json | 31 +++++ .../models/mode_melotts-zh-cn-ax650.json | 31 +++++ .../main_melotts/src/runner/EngineWrapper.cpp | 2 +- projects/llm_framework/main_skel/SConstruct | 9 +- projects/llm_framework/main_skel/src/main.c | 8 +- .../main_whisper/src/runner/EngineWrapper.cpp | 2 +- .../main_yolo/models/mode_yolo11n-ax650.json | 110 ++++++++++++++++++ .../models/mode_yolo11n-hand-pose-ax650.json | 32 +++++ .../models/mode_yolo11n-pose-ax650.json | 32 +++++ .../models/mode_yolo11n-seg-ax650.json | 110 ++++++++++++++++++ .../main_yolo/src/EngineWrapper.cpp | 2 +- projects/llm_framework/tools/llm_pack.py | 61 +++++++++- 28 files changed, 769 insertions(+), 40 deletions(-) create mode 100644 projects/llm_framework/main_depth_anything/mode_depth-anything-ax6350.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-en-au-ax650.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-en-br-ax650.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-en-default-ax650.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-en-india-ax650.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-en-us-ax650.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-es-es-ax650.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-ja-jp-ax650.json create mode 100644 projects/llm_framework/main_melotts/models/mode_melotts-zh-cn-ax650.json create mode 100644 projects/llm_framework/main_yolo/models/mode_yolo11n-ax650.json create mode 100644 projects/llm_framework/main_yolo/models/mode_yolo11n-hand-pose-ax650.json create mode 100644 projects/llm_framework/main_yolo/models/mode_yolo11n-pose-ax650.json create mode 100644 projects/llm_framework/main_yolo/models/mode_yolo11n-seg-ax650.json diff --git a/ext_components/ax_msp/SConstruct b/ext_components/ax_msp/SConstruct index 3c5ecbe..c4e1ad0 100644 --- a/ext_components/ax_msp/SConstruct +++ b/ext_components/ax_msp/SConstruct @@ -55,3 +55,60 @@ if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: 'REGISTER':'static' }) +if 'CONFIG_AX_650N_MSP_ENABLED' in os.environ: + MSP_PATH = '' + if 'CONFIG_MSP_V3_0_0_ENABLED' in os.environ: + MSP_PATH = os.environ.get('EXT_MSP_PATH', check_wget_down("https://m5stack.oss-cn-shenzhen.aliyuncs.com/resource/linux/llm/m5stack_ax650n_msp.tar.gz", 'm5stack_ax650n_msp.tar.gz')) + + SRCS=[] + INCLUDE=[] + PRIVATE_INCLUDE=[] + REQUIREMENTS=[] + STATIC_LIB=[] + DYNAMIC_LIB=[] + DEFINITIONS=[] + DEFINITIONS_PRIVATE=[''] + LDFLAGS=[] + LINK_SEARCH_PATH=[] + + env["MSP_PATH"] = MSP_PATH + # , os.path.join(MSP_PATH, 'sample/common') + INCLUDE += [os.path.join(MSP_PATH, 'out/include'), os.path.join(MSP_PATH, 'sample'), os.path.join(MSP_PATH, 'sample/rtsp')] + LINK_SEARCH_PATH += [os.path.join(MSP_PATH, 'out/lib')] + REQUIREMENTS += ['dl', 'm', 'pthread'] + third_party = ['drm','faac','fdk-aac','ffmpeg','libsamplerate','mp4','opencv','opus','tinyalsa'] + for dirn in third_party: + INCLUDE.append(os.path.join(MSP_PATH,'third-party',dirn,'include')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party',dirn,'lib/arm64/glibc')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/tinyalsa/include/tinyalsa')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/live/out/include/liveMedia')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/live/out/include/groupsock')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/live/out/include/UsageEnvironment')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/live/out/include/BasicUsageEnvironment')) + INCLUDE.append(os.path.join(MSP_PATH,'component/isp_proton/sensor/platform/tcm')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/libsamplerate/lib')) + INCLUDE.append(os.path.join(MSP_PATH,'third-party/openssl/include')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/openssl/arm64/glibc/lib')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/tinyalsa/lib')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/fdk-aac/lib')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'third-party/opus/lib')) + LINK_SEARCH_PATH.append(os.path.join(MSP_PATH,'app/lib')) + + if 'CONFIG_SAMPLE_COMMON_ENABLED' in os.environ: + if 'CONFIG_MSP_V3_0_0_ENABLED' in os.environ: + INCLUDE.append(os.path.join(MSP_PATH,'sample/common')) + SRCS += AGlob(os.path.join(MSP_PATH,'sample/common/*.c*')) + + env['COMPONENTS'].append({'target':os.path.basename(env['component_dir']), + 'SRCS':SRCS, + 'INCLUDE':INCLUDE, + 'PRIVATE_INCLUDE':PRIVATE_INCLUDE, + 'REQUIREMENTS':REQUIREMENTS, + 'STATIC_LIB':STATIC_LIB, + 'DYNAMIC_LIB':DYNAMIC_LIB, + 'DEFINITIONS':DEFINITIONS, + 'DEFINITIONS_PRIVATE':DEFINITIONS_PRIVATE, + 'LDFLAGS':LDFLAGS, + 'LINK_SEARCH_PATH':LINK_SEARCH_PATH, + 'REGISTER':'static' + }) \ No newline at end of file diff --git a/projects/llm_framework/main_audio/SConstruct b/projects/llm_framework/main_audio/SConstruct index 18d7ca0..5491339 100644 --- a/projects/llm_framework/main_audio/SConstruct +++ b/projects/llm_framework/main_audio/SConstruct @@ -22,9 +22,9 @@ DEFINITIONS += ['-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] # INCLUDE += [ADir('../include'), ADir('../include/cppzmq'), ADir('../include/libzmq')] LINK_SEARCH_PATH += [ADir('../static_lib')] - -REQUIREMENTS += ['ax_sys', 'ax_interpreter', 'ax_audio', 'ax_audio_3a', 'ax_fdk', 'ax_opus'] -REQUIREMENTS += ['tinyalsa', 'opus', 'samplerate', 'fdk-aac'] +REQUIREMENTS += ['ax_sys', 'ax_audio', 'ax_audio_3a', 'tinyalsa', 'fdk-aac', 'opus', 'samplerate'] +if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: + REQUIREMENTS += ['ax_interpreter', 'ax_fdk', 'ax_opus'] STATIC_FILES += [AFile('audio.json'), AFile('audio_kit.json')] STATIC_FILES += Glob('mode_*.json') diff --git a/projects/llm_framework/main_audio/src/main.cpp b/projects/llm_framework/main_audio/src/main.cpp index e700753..fdd2506 100644 --- a/projects/llm_framework/main_audio/src/main.cpp +++ b/projects/llm_framework/main_audio/src/main.cpp @@ -69,8 +69,13 @@ private: void hw_cap() { +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) ax_cap_start(cap_config.card, cap_config.device, cap_config.volume, cap_config.channel, cap_config.rate, cap_config.bit, llm_audio::on_cap_sample); +#else + alsa_cap_start(cap_config.card, cap_config.device, cap_config.volume, cap_config.channel, cap_config.rate, + cap_config.bit, llm_audio::on_cap_sample); +#endif } void _play(const std::string &audio_data) @@ -143,13 +148,17 @@ public: nlohmann::json error_body; std::string base_model_path; std::string base_model_config_path; +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) std::list config_file_paths; if (access("/sys/devices/platform/soc/4851000.i2c/i2c-1/1-0043", F_OK) == 0) { config_file_paths = get_config_file_paths(base_model_path, base_model_config_path, "audio_kit"); } else { config_file_paths = get_config_file_paths(base_model_path, base_model_config_path, "audio"); } - +#else + std::list config_file_paths = + get_config_file_paths(base_model_path, base_model_config_path, "audio"); +#endif try { config_body = nlohmann::json::parse(data); for (auto file_name : config_file_paths) { @@ -201,6 +210,7 @@ public: CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.enAgcMode); CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.s16TargetLevel); CONFIG_AUTO_SET(file_body["play_param"], stVqeAttr.stAgcCfg.s16Gain); +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.bEnable); CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.s32GainDb); CONFIG_AUTO_SET(file_body["play_param"], stHpfAttr.s32Freq); @@ -215,6 +225,7 @@ public: CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[3]); CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32GainDb[4]); CONFIG_AUTO_SET(file_body["play_param"], stEqAttr.s32Samplerate); +#endif CONFIG_AUTO_SET(file_body["play_param"], gResample); CONFIG_AUTO_SET(file_body["play_param"], enInSampleRate); CONFIG_AUTO_SET(file_body["play_param"], gInstant); @@ -247,7 +258,17 @@ public: CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enBitwidth); CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enLinkMode); CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enSamplerate); - CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enLayoutMode); + // CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.enLayoutMode); + if (config_body.contains("aistAttr.enLayoutMode")) + mode_config_.aistAttr.enLayoutMode = config_body["aistAttr.enLayoutMode"]; + else if (file_body["cap_param"].contains("aistAttr.enLayoutMode")) { + mode_config_.aistAttr.enLayoutMode = file_body["cap_param"]["aistAttr.enLayoutMode"]; + if (access("/sys/devices/platform/soc/4851000.i2c/i2c-1/1-0043", F_OK) == 0) { + if (mode_config_.aistAttr.enLayoutMode == AX_AI_REF_MIC) { + mode_config_.aistAttr.enLayoutMode = AX_AI_MIC_REF; + } + } + } CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.U32Depth); CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.u32PeriodSize); CONFIG_AUTO_SET(file_body["cap_param"], aistAttr.u32PeriodCount); @@ -261,6 +282,7 @@ public: CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAgcCfg.s16TargetLevel); CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAgcCfg.s16Gain); CONFIG_AUTO_SET(file_body["cap_param"], aistVqeAttr.stAecCfg.enAecMode); +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.bEnable); CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32GainDb); CONFIG_AUTO_SET(file_body["cap_param"], stHpfAttr.s32Samplerate); @@ -276,6 +298,7 @@ public: CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[3]); CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32GainDb[4]); CONFIG_AUTO_SET(file_body["cap_param"], stEqAttr.s32Samplerate); +#endif CONFIG_AUTO_SET(file_body["cap_param"], gResample); CONFIG_AUTO_SET(file_body["cap_param"], enOutSampleRate); CONFIG_AUTO_SET(file_body["cap_param"], gDbDetection); diff --git a/projects/llm_framework/main_audio/src/sample_audio.c b/projects/llm_framework/main_audio/src/sample_audio.c index 400d46b..ab78b6c 100644 --- a/projects/llm_framework/main_audio/src/sample_audio.c +++ b/projects/llm_framework/main_audio/src/sample_audio.c @@ -116,6 +116,7 @@ void ax_play(unsigned int card, unsigned int device, float Volume, int channel, goto AO_DEINIT; } } +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) if (play_config.stHpfAttr.bEnable) { ret = AX_ACODEC_TxHpfSetAttr(card, &play_config.stHpfAttr); if (ret) { @@ -152,10 +153,14 @@ void ax_play(unsigned int card, unsigned int device, float Volume, int channel, goto DIS_LPF; } } +#endif ret = AX_AO_EnableDev(card, device); if (ret) { printf("AX_AO_EnableDev failed! ret = %x \n", ret); +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) goto DIS_EQ; +#endif + goto DIS_AO_DEVICE; } if (play_config.gResample) { ret = AX_AO_EnableResample(card, device, play_config.enInSampleRate); @@ -184,7 +189,11 @@ void ax_play(unsigned int card, unsigned int device, float Volume, int channel, AX_POOL_ReleaseBlock(stFrmInfo.u32BlkId); break; } +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) ret = AX_AO_SendFrame(card, device, &stFrmInfo, NULL, 0.f, -1); +#else + ret = AX_AO_SendFrame(card, device, &stFrmInfo, -1); +#endif if (ret != AX_SUCCESS) { AX_POOL_ReleaseBlock(stFrmInfo.u32BlkId); printf("AX_AO_SendFrame error, ret: %x\n", ret); @@ -205,6 +214,7 @@ void ax_play(unsigned int card, unsigned int device, float Volume, int channel, while (1) { ret = AX_AO_QueryDevStat(card, device, &stStatus); if (stStatus.u32DevBusyNum == 0) { +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) if (play_config.gInsertSilence) { break; } else { @@ -212,11 +222,15 @@ void ax_play(unsigned int card, unsigned int device, float Volume, int channel, break; } } +#else + break; +#endif } usleep(10 * 1000); } } printf("ao success.\n"); +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) DIS_EQ: if (play_config.stEqAttr.bEnable) { ret = AX_ACODEC_TxEqDisable(card); @@ -238,7 +252,7 @@ DIS_HPF: printf("AX_ACODEC_TxHpfDisable failed! ret= %x\n", ret); } } - +#endif DIS_AO_DEVICE: ret = AX_AO_DisableDev(card, device); if (ret) { @@ -329,7 +343,7 @@ void ax_cap_start(unsigned int card, unsigned int device, float Volume, int chan } outRate = rate; } - +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) if (cap_config.stHpfAttr.bEnable) { ret = AX_ACODEC_RxHpfSetAttr(card, &cap_config.stHpfAttr); if (ret) { @@ -366,11 +380,14 @@ void ax_cap_start(unsigned int card, unsigned int device, float Volume, int chan goto DIS_LPF; } } - +#endif ret = AX_AI_EnableDev(card, device); if (ret) { printf("AX_AI_EnableDev failed! ret = %x \n", ret); +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) goto DIS_EQ; +#endif + goto AI_DETACHPOOL; } if (cap_config.gResample) { @@ -415,7 +432,7 @@ DIS_AI_DEVICE: printf("AX_AI_DisableDev failed! ret= %x\n", ret); goto FREE_SYS; } - +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) DIS_EQ: if (cap_config.stEqAttr.bEnable) { ret = AX_ACODEC_RxEqDisable(card); @@ -437,7 +454,7 @@ DIS_HPF: printf("AX_ACODEC_RxHpfDisable failed! ret= %x\n", ret); } } - +#endif AI_DETACHPOOL: ret = AX_AI_DetachPool(card, device); if (AX_SUCCESS != ret) { diff --git a/projects/llm_framework/main_audio/src/sample_audio.h b/projects/llm_framework/main_audio/src/sample_audio.h index 037b8b9..29833fb 100644 --- a/projects/llm_framework/main_audio/src/sample_audio.h +++ b/projects/llm_framework/main_audio/src/sample_audio.h @@ -7,8 +7,11 @@ #include "ax_ai_api.h" #include "ax_ao_api.h" #include "ax_global_type.h" +#include +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) #include "ax_acodec_api.h" #include "ax_aac.h" +#endif typedef struct { unsigned int card; @@ -26,9 +29,11 @@ typedef struct { AX_AP_DNVQE_ATTR_T stVqeAttr; AX_AP_UPTALKVQE_ATTR_T aistVqeAttr; }; +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) AX_ACODEC_FREQ_ATTR_T stHpfAttr; AX_ACODEC_FREQ_ATTR_T stLpfAttr; AX_ACODEC_EQ_ATTR_T stEqAttr; +#endif int gResample; union { AX_AUDIO_SAMPLE_RATE_E enInSampleRate; diff --git a/projects/llm_framework/main_camera/SConstruct b/projects/llm_framework/main_camera/SConstruct index 92f525e..a68311d 100644 --- a/projects/llm_framework/main_camera/SConstruct +++ b/projects/llm_framework/main_camera/SConstruct @@ -41,11 +41,11 @@ SRCS += Glob(os.path.join(env["MSP_PATH"], 'sample/rtsp/*.c*') ) # INCLUDE.append(ADir('ax_sample')) # SRCS += Glob('ax_sample/*.c*') -REQUIREMENTS += ['ax_sys', 'ax_interpreter'] -REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys', 'ax_vo'] -REQUIREMENTS += ['ax_ae', 'ax_skel', 'ax_venc', 'ax_mipi', 'ax_ives', 'ax_ivps', 'ax_proton', 'ax_audio', 'tinyalsa'] -REQUIREMENTS += ['ax_audio_3a','samplerate', 'ax_fdk', 'fdk-aac', 'ax_awb', 'ax_af', 'ax_nt_stream', 'ax_nt_ctrl'] -REQUIREMENTS += ['liveMedia', 'UsageEnvironment', 'groupsock', 'BasicUsageEnvironment'] +REQUIREMENTS += ['ax_sys', 'ax_interpreter', 'ax_engine'] +if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: + REQUIREMENTS += ['ax_ae', 'ax_vo', 'ax_skel', 'ax_venc', 'ax_mipi', 'ax_ives', 'ax_ivps', 'ax_proton', 'ax_audio', 'tinyalsa'] + REQUIREMENTS += ['ax_audio_3a','samplerate', 'ax_fdk', 'fdk-aac', 'ax_awb', 'ax_af', 'ax_nt_stream', 'ax_nt_ctrl'] + REQUIREMENTS += ['liveMedia', 'UsageEnvironment', 'groupsock', 'BasicUsageEnvironment'] INCLUDE += [ADir('../include/opencv4')] static_file = [] diff --git a/projects/llm_framework/main_camera/src/axera_camera.c b/projects/llm_framework/main_camera/src/axera_camera.c index 0017288..f6a02cb 100644 --- a/projects/llm_framework/main_camera/src/axera_camera.c +++ b/projects/llm_framework/main_camera/src/axera_camera.c @@ -4,7 +4,6 @@ * SPDX-License-Identifier: MIT */ #include "camera.h" -#include "axera_camera.h" #include #include #include @@ -18,6 +17,9 @@ #include #include #include +#include +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) +#include "axera_camera.h" #include "AXRtspWrapper.h" #include "ax_venc_api.h" #include "ax_global_type.h" @@ -29,6 +31,9 @@ #include "common_vin.h" #include "ax_sys_api.h" #include "ax_ivps_api.h" +#endif + +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) #ifndef ALIGN_UP #define ALIGN_UP(x, a) ((((x) + ((a) - 1)) / a) * a) @@ -115,8 +120,6 @@ AX_VIN_CHN_ATTR_T gSc850slChn0Attr = { .tFrameRateCtrl = {AX_INVALID_FRMRATE, AX_INVALID_FRMRATE}, }; - - struct axera_camera_index_t { char name[48]; SAMPLE_VIN_CASE_E index; @@ -137,8 +140,6 @@ struct axera_camera_index_t { {"axera_single_os04a10_online", SYS_CASE_SINGLE_OS04A10_ONLINE}, {"axera_single_sc850sl", SAMPLE_VIN_SINGLE_SC850SL}}; - - /* comm pool */ COMMON_SYS_POOL_CFG_T gtSysCommPoolSingleDummySdr[] = { {2688, 1520, 2688, AX_FORMAT_BAYER_RAW_16BPP, 6}, /*vin raw16 use */ @@ -1611,9 +1612,9 @@ int axera_camera_open_from(camera_t *camera) return -10; } - axera_obj.VinParam.eSysMode = axera_config->VinParam.eSysMode; // COMMON_VIN_SENSOR; - axera_obj.VinParam.eHdrMode = axera_config->VinParam.eHdrMode; // AX_SNS_LINEAR_MODE; - axera_obj.VinParam.bAiispEnable = axera_config->VinParam.bAiispEnable; // AX_TRUE; + axera_obj.VinParam.eSysMode = axera_config->VinParam.eSysMode; // COMMON_VIN_SENSOR; + axera_obj.VinParam.eHdrMode = axera_config->VinParam.eHdrMode; // AX_SNS_LINEAR_MODE; + axera_obj.VinParam.bAiispEnable = axera_config->VinParam.bAiispEnable; // AX_TRUE; // axera_obj.gCams.tChnAttr __sample_case_config(&axera_obj.gCams, &axera_obj.VinParam, &axera_obj.tCommonArgs, &axera_obj.tPrivArgs); COMMON_SYS_Init(&axera_obj.tCommonArgs); @@ -1720,4 +1721,5 @@ int axera_camera_close(camera_t *camera) SLOGI("camera closed"); return 0; -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/projects/llm_framework/main_camera/src/main.cpp b/projects/llm_framework/main_camera/src/main.cpp index 3565b01..45f730b 100644 --- a/projects/llm_framework/main_camera/src/main.cpp +++ b/projects/llm_framework/main_camera/src/main.cpp @@ -13,7 +13,10 @@ #include #include "../../../../SDK/components/utilities/include/sample_log.h" #include "camera.h" +#include +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) #include "axera_camera.h" +#endif #include #include #include "hv/TcpServer.h" @@ -172,7 +175,7 @@ public: { out_callback_ = out_callback; } - +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) static bool parse_axera_config(const nlohmann::json &config_body, const nlohmann::json &file_body, void **custom_config) { @@ -473,11 +476,13 @@ public: (stVencChnAttr.stVencAttr.u32PicHeightSrc < frame_height)) { return true; } +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) init_rtsp(&stVencChnAttr); +#endif } return false; } - +#endif bool parse_config(const nlohmann::json &config_body) { try { @@ -502,11 +507,15 @@ public: hal_camera_open = camera_open; hal_camera_close = camera_close; hal_parse_config = NULL; - } else if (devname_.find("axera_") != std::string::npos) { + } +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) + else if (devname_.find("axera_") != std::string::npos) { hal_camera_open = axera_camera_open; hal_camera_close = axera_camera_close; hal_parse_config = llm_task::parse_axera_config; - } else { + } +#endif + else { return true; } { diff --git a/projects/llm_framework/main_depth_anything/mode_depth-anything-ax6350.json b/projects/llm_framework/main_depth_anything/mode_depth-anything-ax6350.json new file mode 100644 index 0000000..447f6f4 --- /dev/null +++ b/projects/llm_framework/main_depth_anything/mode_depth-anything-ax6350.json @@ -0,0 +1,24 @@ +{ + "mode":"depth-anything-ax650", + "type":"cv", + "homepage":"https://huggingface.co/AXERA-TECH/Depth-Anything-V2", + "compile_flage":"pulsar2 build --input depth_anything_v2_vits.onnx --config config.json --output_dir build-output --output_name depth_anything_v2_vits.axmodel --target_hardware AX650 --compiler.check 0", + "pulsar_version":"3.3-ae03a08f", + "capabilities":[ + "Segmentation" + ], + "input_type":[ + "cv.jpeg.base64" + ], + "output_type":[ + "cv.jpeg.base64" + ], + "mode_param":{ + "depth_anything_model":"depth_anything.axmodel", + "model_type":"segment", + "img_h":518, + "img_w":518 + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_depth_anything/src/EngineWrapper.cpp b/projects/llm_framework/main_depth_anything/src/EngineWrapper.cpp index 21c51e4..1aad967 100644 --- a/projects/llm_framework/main_depth_anything/src/EngineWrapper.cpp +++ b/projects/llm_framework/main_depth_anything/src/EngineWrapper.cpp @@ -32,7 +32,7 @@ typedef enum axNPU_TYPE_E { AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ } AX_NPU_TYPE_E; -#if defined(CHIP_AX650) +#if defined(CONFIG_AX_650N_MSP_ENABLED) static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_TYPE_T &eModelType, const AX_S32 &nNpuType, AX_U32 &nNpuSet) { diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-en-au-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-en-au-ax650.json new file mode 100644 index 0000000..8b94e7e --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-en-au-ax650.json @@ -0,0 +1,31 @@ +{ + "mode": "melotts-en-au-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-en.onnx --config config_decoder_u16_en-au.json --output_dir decoder-au-ax650 --output_name decoder-en-au.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "English" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-en-au.ort", + "decoder": "decoder-en-au.axmodel", + "gbin": "g-en-au.bin", + "tokens": "tokens-en.txt", + "lexicon": "lexicon-en.txt", + "tagger": "en_tn_tagger.fst", + "verbalizer": "en_tn_verbalizer.fst", + "spacker_speed": 1.2, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-en-br-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-en-br-ax650.json new file mode 100644 index 0000000..ef43e58 --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-en-br-ax650.json @@ -0,0 +1,31 @@ +{ + "mode": "melotts-en-br-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-en.onnx --config config_decoder_u16_en-br.json --output_dir decoder-br-ax650 --output_name decoder-en-br.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "English" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-en-br.ort", + "decoder": "decoder-en-br.axmodel", + "gbin": "g-en-br.bin", + "tokens": "tokens-en.txt", + "lexicon": "lexicon-en.txt", + "tagger": "en_tn_tagger.fst", + "verbalizer": "en_tn_verbalizer.fst", + "spacker_speed": 1.2, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-en-default-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-en-default-ax650.json new file mode 100644 index 0000000..3e19b40 --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-en-default-ax650.json @@ -0,0 +1,31 @@ +{ + "mode": "melotts-en-default-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-en.onnx --config config_decoder_u16_en-default.json --output_dir decoder-default-ax650 --output_name decoder-en-default.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "English" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-en-default.ort", + "decoder": "decoder-en-default.axmodel", + "gbin": "g-en-default.bin", + "tokens": "tokens-en.txt", + "lexicon": "lexicon-en.txt", + "tagger": "en_tn_tagger.fst", + "verbalizer": "en_tn_verbalizer.fst", + "spacker_speed": 1.2, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-en-india-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-en-india-ax650.json new file mode 100644 index 0000000..ee0cab6 --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-en-india-ax650.json @@ -0,0 +1,31 @@ +{ + "mode": "melotts-en-india-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-en.onnx --config config_decoder_u16_en-india.json --output_dir decoder-india-ax650 --output_name decoder-en-india.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "English" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-en-india.ort", + "decoder": "decoder-en-india.axmodel", + "gbin": "g-en-india.bin", + "tokens": "tokens-en.txt", + "lexicon": "lexicon-en.txt", + "tagger": "en_tn_tagger.fst", + "verbalizer": "en_tn_verbalizer.fst", + "spacker_speed": 1.2, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-en-us-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-en-us-ax650.json new file mode 100644 index 0000000..6ca7181 --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-en-us-ax650.json @@ -0,0 +1,31 @@ +{ + "mode": "melotts-en-us-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-en.onnx --config config_decoder_u16_en-us.json --output_dir decoder-us-ax650 --output_name decoder-en-us.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "English" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-en-us.ort", + "decoder": "decoder-en-us.axmodel", + "gbin": "g-en-us.bin", + "tokens": "tokens-en.txt", + "lexicon": "lexicon-en.txt", + "tagger": "en_tn_tagger.fst", + "verbalizer": "en_tn_verbalizer.fst", + "spacker_speed": 1.2, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-es-es-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-es-es-ax650.json new file mode 100644 index 0000000..7012574 --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-es-es-ax650.json @@ -0,0 +1,29 @@ +{ + "mode": "melotts-es-es-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-es.onnx --config config_decoder_u16_es-es.json --output_dir decoder-es-ax650 --output_name decoder-es-es.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "Spanish" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-es-es.ort", + "decoder": "decoder-es-es.axmodel", + "gbin": "g-es.bin", + "tokens": "tokens-es.txt", + "lexicon": "lexicon-es.txt", + "spacker_speed": 1.2, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-ja-jp-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-ja-jp-ax650.json new file mode 100644 index 0000000..49f26aa --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-ja-jp-ax650.json @@ -0,0 +1,31 @@ +{ + "mode": "melotts-ja-jp-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-jp.onnx --config config_decoder_u16_ja-jp.json --output_dir decoder-ja-ax650 --output_name decoder-ja-jp.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "Japanese" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-ja-jp.ort", + "decoder": "decoder-ja-jp.axmodel", + "gbin": "g-jp.bin", + "tokens": "tokens-jp.txt", + "lexicon": "lexicon-jp.txt", + "tagger": "ja_tn_tagger.fst", + "verbalizer": "ja_tn_verbalizer.fst", + "spacker_speed": 1.1, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/models/mode_melotts-zh-cn-ax650.json b/projects/llm_framework/main_melotts/models/mode_melotts-zh-cn-ax650.json new file mode 100644 index 0000000..4106097 --- /dev/null +++ b/projects/llm_framework/main_melotts/models/mode_melotts-zh-cn-ax650.json @@ -0,0 +1,31 @@ +{ + "mode": "melotts-zh-cn-ax650", + "type": "tts", + "homepage": "https://github.com/ml-inory/melotts.axera/tree/main/model_convert", + "compile_flage": "pulsar2 build --input decoder-zh.onnx --config config_decoder_u16_zh-cn.json --output_dir decoder-zh-ax650 --output_name decoder-zh-cn.axmodel --target_hardware AX650 --npu_mode NPU3 --compiler.check 0", + "pulsar_version": "4.0-64a0e58f", + "capabilities": [ + "tts", + "Chinese" + ], + "input_type": [ + "tts.utf-8" + ], + "output_type": [ + "tts.wav", + "sys.play.0_1" + ], + "mode_param": { + "encoder": "encoder-zh-cn.ort", + "decoder": "decoder-zh-cn.axmodel", + "gbin": "g-zh_mix_en.bin", + "tokens": "tokens.txt", + "lexicon": "lexicon.txt", + "tagger": "zh_tn_tagger.fst", + "verbalizer": "zh_tn_verbalizer.fst", + "spacker_speed": 1.1, + "mode_rate": 44100, + "audio_rate": 16000, + "awake_delay": 1000 + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_melotts/src/runner/EngineWrapper.cpp b/projects/llm_framework/main_melotts/src/runner/EngineWrapper.cpp index 0394ee4..bc7590a 100644 --- a/projects/llm_framework/main_melotts/src/runner/EngineWrapper.cpp +++ b/projects/llm_framework/main_melotts/src/runner/EngineWrapper.cpp @@ -32,7 +32,7 @@ typedef enum axNPU_TYPE_E { AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ } AX_NPU_TYPE_E; -#if defined(CHIP_AX650) +#if defined(CONFIG_AX_650N_MSP_ENABLED) static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_TYPE_T &eModelType, const AX_S32 &nNpuType, AX_U32 &nNpuSet) { AX_ENGINE_NPU_ATTR_T stNpuAttr; memset(&stNpuAttr, 0x00, sizeof(stNpuAttr)); diff --git a/projects/llm_framework/main_skel/SConstruct b/projects/llm_framework/main_skel/SConstruct index bae9b43..55e508b 100644 --- a/projects/llm_framework/main_skel/SConstruct +++ b/projects/llm_framework/main_skel/SConstruct @@ -3,8 +3,10 @@ import os Import('env') with open(env['PROJECT_TOOL_S']) as f: exec(f.read()) - -SRCS = append_srcs_dir(ADir('src')) +if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: + SRCS = append_srcs_dir(ADir('src')) +else: + SRCS = Glob('src/main.c') INCLUDE = [ADir('include'), ADir('.')] PRIVATE_INCLUDE = [] REQUIREMENTS = ['pthread', 'utilities', 'ax_msp', 'eventpp', 'StackFlow', 'ax-samples'] @@ -21,7 +23,8 @@ DEFINITIONS += ['-std=c++17'] LDFLAGS+=['-Wl,-rpath=/opt/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib', '-Wl,-rpath=/usr/local/m5stack/lib/gcc-10.3', '-Wl,-rpath=/opt/lib', '-Wl,-rpath=/opt/usr/lib', '-Wl,-rpath=./'] LINK_SEARCH_PATH += [ADir('../static_lib')] REQUIREMENTS += ['ax_engine', 'ax_interpreter', 'ax_sys'] -REQUIREMENTS += ['ax_skel', 'ax_engine', 'ax_venc', 'ax_vdec', 'ax_ivps', 'ax_proton'] +if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ: + REQUIREMENTS += ['ax_skel', 'ax_engine', 'ax_venc', 'ax_vdec', 'ax_ivps', 'ax_proton'] INCLUDE += [ADir('../include'), ADir('src/runner'), ADir('../include/opencv4')] diff --git a/projects/llm_framework/main_skel/src/main.c b/projects/llm_framework/main_skel/src/main.c index 144d3a4..cd835ca 100644 --- a/projects/llm_framework/main_skel/src/main.c +++ b/projects/llm_framework/main_skel/src/main.c @@ -7,7 +7,8 @@ * written consent of Axera Semiconductor Co., Ltd. * **************************************************************************************************/ - +#include +#if defined(CONFIG_AX_620E_MSP_ENABLED) || defined(CONFIG_AX_620Q_MSP_ENABLED) #include #include #include @@ -2251,3 +2252,8 @@ EXIT0: return (0 != nRet) ? -1 : 0; } +#else +int main(){ + return 0; +} +#endif \ No newline at end of file diff --git a/projects/llm_framework/main_whisper/src/runner/EngineWrapper.cpp b/projects/llm_framework/main_whisper/src/runner/EngineWrapper.cpp index 6d21964..bb0e7f8 100644 --- a/projects/llm_framework/main_whisper/src/runner/EngineWrapper.cpp +++ b/projects/llm_framework/main_whisper/src/runner/EngineWrapper.cpp @@ -33,7 +33,7 @@ typedef enum axNPU_TYPE_E { AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ } AX_NPU_TYPE_E; -#if defined(CHIP_AX650) +#if defined(CONFIG_AX_650N_MSP_ENABLED) static AX_S32 CheckModelVNpu(const std::string &strModel, const AX_ENGINE_MODEL_TYPE_T &eModelType, const AX_S32 &nNpuType, AX_U32 &nNpuSet) { AX_ENGINE_NPU_ATTR_T stNpuAttr; memset(&stNpuAttr, 0x00, sizeof(stNpuAttr)); diff --git a/projects/llm_framework/main_yolo/models/mode_yolo11n-ax650.json b/projects/llm_framework/main_yolo/models/mode_yolo11n-ax650.json new file mode 100644 index 0000000..cad0e05 --- /dev/null +++ b/projects/llm_framework/main_yolo/models/mode_yolo11n-ax650.json @@ -0,0 +1,110 @@ +{ + "mode":"yolo11n-ax650", + "type":"cv", + "homepage":"https://github.com/ultralytics/ultralytics", + "compile_flage":"pulsar2 build --target_hardware AX650 --input Workspace/cv/model/yolo11n.onnx --output_dir yolo11n --config Workspace/cv/config/yolo11_config.json --output_name yolo11n.axmodel --npu_mode NPU3", + "pulsar_version":"4.1-patch1-c37957c7", + "capabilities":[ + "Detection" + ], + "input_type":[ + "yolo.jpeg.base64" + ], + "output_type":[ + "yolo.box", + "yolo.boxV2" + ], + "mode_param":{ + "yolo_model":"yolo11n.axmodel", + "model_type":"detect", + "img_h":320, + "img_w":320, + "cls_num":80, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/models/mode_yolo11n-hand-pose-ax650.json b/projects/llm_framework/main_yolo/models/mode_yolo11n-hand-pose-ax650.json new file mode 100644 index 0000000..0ab718e --- /dev/null +++ b/projects/llm_framework/main_yolo/models/mode_yolo11n-hand-pose-ax650.json @@ -0,0 +1,32 @@ +{ + "mode":"yolo11n-hand-pose-ax650", + "type":"cv", + "homepage":"https://github.com/ultralytics/ultralytics", + "compile_flage":"pulsar2 build --target_hardware AX650 --input Workspace/cv/model/yolo11n-hand-pose-cut.onnx --output_dir yolo11n-hand-pose-ax650 --config Workspace/cv/config/yolo11_hand_pose_config.json --output_name yolo11n-hand-pose.axmodel --npu_mode NPU3", + "pulsar_version":"4.1-patch1-c37957c7", + "capabilities":[ + "Pose" + ], + "input_type":[ + "yolo.jpeg.base64" + ], + "output_type":[ + "yolo.box", + "yolo.boxV2" + ], + "mode_param":{ + "yolo_model":"yolo11n-hand-pose.axmodel", + "model_type":"pose", + "img_h":320, + "img_w":320, + "cls_num":1, + "point_num":21, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "hand" + ] + }, + "mode_param_bak":{ + } +} diff --git a/projects/llm_framework/main_yolo/models/mode_yolo11n-pose-ax650.json b/projects/llm_framework/main_yolo/models/mode_yolo11n-pose-ax650.json new file mode 100644 index 0000000..bc7d009 --- /dev/null +++ b/projects/llm_framework/main_yolo/models/mode_yolo11n-pose-ax650.json @@ -0,0 +1,32 @@ +{ + "mode":"yolo11n-pose-ax650", + "type":"cv", + "homepage":"https://github.com/ultralytics/ultralytics", + "compile_flage":"pulsar2 build --target_hardware AX650 --input Workspace/cv/model/yolo11n-seg-cut.onnx --output_dir yolo11n-seg-ax650 --config Workspace/cv/config/yolo11_seg_config.json --output_name yolo11n-seg.axmodel --npu_mode NPU3", + "pulsar_version":"4.1-patch1-c37957c7", + "capabilities":[ + "Pose" + ], + "input_type":[ + "yolo.jpeg.base64" + ], + "output_type":[ + "yolo.box", + "yolo.boxV2" + ], + "mode_param":{ + "yolo_model":"yolo11n-pose.axmodel", + "model_type":"pose", + "img_h":320, + "img_w":320, + "cls_num":1, + "point_num":17, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/models/mode_yolo11n-seg-ax650.json b/projects/llm_framework/main_yolo/models/mode_yolo11n-seg-ax650.json new file mode 100644 index 0000000..50e21d0 --- /dev/null +++ b/projects/llm_framework/main_yolo/models/mode_yolo11n-seg-ax650.json @@ -0,0 +1,110 @@ +{ + "mode":"yolo11n-seg-ax650", + "type":"cv", + "homepage":"https://github.com/ultralytics/ultralytics", + "compile_flage":"pulsar2 build --target_hardware AX650 --input Workspace/cv/model/yolo11n-pose-cut.onnx --output_dir yolo11n-pose-ax650 --config Workspace/cv/config/yolo11_pose_config.json --output_name yolo11n-pose.axmodel --npu_mode NPU3", + "pulsar_version":"4.1-patch1-c37957c7", + "capabilities":[ + "Segmentation" + ], + "input_type":[ + "yolo.jpeg.base64" + ], + "output_type":[ + "yolo.box", + "yolo.boxV2" + ], + "mode_param":{ + "yolo_model":"yolo11n-seg.axmodel", + "model_type":"segment", + "img_h":320, + "img_w":320, + "cls_num":80, + "pron_threshold":0.45, + "nms_threshold":0.45, + "cls_name":[ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" + ] + }, + "mode_param_bak":{ + } +} \ No newline at end of file diff --git a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp index 99e448d..72f16a6 100644 --- a/projects/llm_framework/main_yolo/src/EngineWrapper.cpp +++ b/projects/llm_framework/main_yolo/src/EngineWrapper.cpp @@ -33,7 +33,7 @@ typedef enum axNPU_TYPE_E { AX_BL_VNPU_2 = (1 << 4) /* running under BIG-LITTLE VNPU2 */ } AX_NPU_TYPE_E; -#if defined(CHIP_AX650) +#if defined(CONFIG_AX_650N_MSP_ENABLED) static AX_S32 CheckModelVNpu(const std::string& strModel, const AX_ENGINE_MODEL_TYPE_T& eModelType, const AX_S32& nNpuType, AX_U32& nNpuSet) { diff --git a/projects/llm_framework/tools/llm_pack.py b/projects/llm_framework/tools/llm_pack.py index 1783e53..fc0097d 100755 --- a/projects/llm_framework/tools/llm_pack.py +++ b/projects/llm_framework/tools/llm_pack.py @@ -352,6 +352,7 @@ if __name__ == "__main__": cpu_count = cpu_count - 2 # cpu_count = 50 Tasks = { + # Unit 'lib-llm':[create_lib_deb,'lib-llm', '1.8', src_folder, revision], 'llm-sys':[create_bin_deb,'llm-sys', '1.6', src_folder, revision], 'llm-audio':[create_bin_deb,'llm-audio', '1.6', src_folder, revision], @@ -368,14 +369,21 @@ if __name__ == "__main__": 'llm-vad':[create_bin_deb,'llm-vad', '1.7', src_folder, revision], 'llm-whisper':[create_bin_deb,'llm-whisper', '1.7', src_folder, revision], 'llm-openai-api':[create_bin_deb,'llm-openai-api', '1.7', src_folder, revision], + # keyword spotting Audio file 'llm-model-audio-en-us':[create_data_deb,'llm-model-audio-en-us', data_version, src_folder, revision], 'llm-model-audio-zh-cn':[create_data_deb,'llm-model-audio-zh-cn', data_version, src_folder, revision], + # Kaldi model 'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-20M-2023-02-17', data_version, src_folder, revision], 'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23':[create_data_deb,'llm-model-sherpa-ncnn-streaming-zipformer-zh-14M-2023-02-23', data_version, src_folder, revision], 'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01', '0.3', src_folder, revision], 'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01':[create_data_deb,'llm-model-sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01', '0.3', src_folder, revision], + # TTS model 'llm-model-single-speaker-english-fast':[create_data_deb,'llm-model-single-speaker-english-fast', '0.3', src_folder, revision], 'llm-model-single-speaker-fast':[create_data_deb,'llm-model-single-speaker-fast', '0.3', src_folder, revision], + # VAD model + 'llm-model-silero-vad':[create_data_deb,'llm-model-silero-vad', '0.4', src_folder, revision], + # MeloTTS model + ## AX630C 'llm-model-melotts-zh-cn':[create_data_deb,'llm-model-melotts-zh-cn', '0.6', src_folder, revision], 'llm-model-melotts-en-au':[create_data_deb,'llm-model-melotts-en-au', '0.6', src_folder, revision], 'llm-model-melotts-en-br':[create_data_deb,'llm-model-melotts-en-br', '0.6', src_folder, revision], @@ -384,20 +392,47 @@ if __name__ == "__main__": 'llm-model-melotts-en-us':[create_data_deb,'llm-model-melotts-en-us', '0.6', src_folder, revision], 'llm-model-melotts-ja-jp':[create_data_deb,'llm-model-melotts-ja-jp', '0.6', src_folder, revision], 'llm-model-melotts-es-es':[create_data_deb,'llm-model-melotts-es-es', '0.5', src_folder, revision], + ## AX650 + 'llm-model-melotts-zh-cn-ax650':[create_data_deb,'llm-model-melotts-zh-cn-ax650', '0.6', src_folder, revision], + 'llm-model-melotts-en-au-ax650':[create_data_deb,'llm-model-melotts-en-au-ax650', '0.6', src_folder, revision], + 'llm-model-melotts-en-br-ax650':[create_data_deb,'llm-model-melotts-en-br-ax650', '0.6', src_folder, revision], + 'llm-model-melotts-en-default-ax650':[create_data_deb,'llm-model-melotts-en-default-ax650', '0.6', src_folder, revision], + 'llm-model-melotts-en-india-ax650':[create_data_deb,'llm-model-melotts-en-india-ax650', '0.6', src_folder, revision], + 'llm-model-melotts-en-us-ax650':[create_data_deb,'llm-model-melotts-en-us-ax650', '0.6', src_folder, revision], + 'llm-model-melotts-ja-jp-ax650':[create_data_deb,'llm-model-melotts-ja-jp-ax650', '0.6', src_folder, revision], + 'llm-model-melotts-es-es-ax650':[create_data_deb,'llm-model-melotts-es-es-ax650', '0.5', src_folder, revision], + # Yolo model + ## AX630C 'llm-model-yolo11n':[create_data_deb,'llm-model-yolo11n', data_version, src_folder, revision], - 'llm-model-yolo11n-npu1':[create_data_deb,'llm-model-yolo11n-npu1', '0.4', src_folder, revision], 'llm-model-yolo11n-pose':[create_data_deb,'llm-model-yolo11n-pose', '0.3', src_folder, revision], - 'llm-model-yolo11n-pose-npu1':[create_data_deb,'llm-model-yolo11n-pose-npu1', '0.4', src_folder, revision], 'llm-model-yolo11n-hand-pose':[create_data_deb,'llm-model-yolo11n-hand-pose', '0.3', src_folder, revision], - 'llm-model-yolo11n-hand-pose-npu1':[create_data_deb,'llm-model-yolo11n-hand-pose-npu1', '0.4', src_folder, revision], 'llm-model-yolo11n-seg':[create_data_deb,'llm-model-yolo11n-seg', '0.3', src_folder, revision], + 'llm-model-yolo11n-npu1':[create_data_deb,'llm-model-yolo11n-npu1', '0.4', src_folder, revision], + 'llm-model-yolo11n-pose-npu1':[create_data_deb,'llm-model-yolo11n-pose-npu1', '0.4', src_folder, revision], + 'llm-model-yolo11n-hand-pose-npu1':[create_data_deb,'llm-model-yolo11n-hand-pose-npu1', '0.4', src_folder, revision], 'llm-model-yolo11n-seg-npu1':[create_data_deb,'llm-model-yolo11n-seg-npu1', '0.4', src_folder, revision], + ## AX650 + 'llm-model-yolo11n-ax650':[create_data_deb,'llm-model-yolo11n-ax650', '0.4', src_folder, revision], + 'llm-model-yolo11n-pose-ax650':[create_data_deb,'llm-model-yolo11n-pose-ax650', '0.4', src_folder, revision], + 'llm-model-yolo11n-hand-pose-ax650':[create_data_deb,'llm-model-yolo11n-hand-pose-ax650', '0.4', src_folder, revision], + 'llm-model-yolo11n-seg-ax650':[create_data_deb,'llm-model-yolo11n-seg-ax650', '0.4', src_folder, revision], + # DepthAnything model + ## AX630C 'llm-model-depth-anything-ax630c':[create_data_deb,'llm-model-depth-anything-ax630c', '0.4', src_folder, revision], 'llm-model-depth-anything-npu1-ax630c':[create_data_deb,'llm-model-depth-anything-npu1-ax630c', '0.4', src_folder, revision], + ## AX650 + 'llm-model-depth-anything-ax650':[create_data_deb,'llm-model-depth-anything-ax650', '0.4', src_folder, revision], + # Whisper model + ## AX630C 'llm-model-whisper-tiny':[create_data_deb,'llm-model-whisper-tiny', '0.4', src_folder, revision], 'llm-model-whisper-base':[create_data_deb,'llm-model-whisper-base', '0.4', src_folder, revision], 'llm-model-whisper-small':[create_data_deb,'llm-model-whisper-small', '0.4', src_folder, revision], - 'llm-model-silero-vad':[create_data_deb,'llm-model-silero-vad', '0.4', src_folder, revision], + ## AX650 + 'llm-model-whisper-tiny-ax650':[create_data_deb,'llm-model-whisper-tiny-ax650', '0.4', src_folder, revision], + 'llm-model-whisper-base-ax650':[create_data_deb,'llm-model-whisper-base-ax650', '0.4', src_folder, revision], + 'llm-model-whisper-small-ax650':[create_data_deb,'llm-model-whisper-small-ax650', '0.4', src_folder, revision], + # Qwen model + ## AX630C 'llm-model-qwen2.5-0.5B-prefill-20e':[create_data_deb,'llm-model-qwen2.5-0.5B-prefill-20e', data_version, src_folder, revision], 'llm-model-qwen2.5-0.5B-p256-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-p256-ax630c', '0.4', src_folder, revision], 'llm-model-qwen2.5-0.5B-Int4-ax630c':[create_data_deb,'llm-model-qwen2.5-0.5B-Int4-ax630c', '0.4', src_folder, revision], @@ -407,14 +442,32 @@ if __name__ == "__main__": 'llm-model-qwen2.5-1.5B-Int4-ax630c':[create_data_deb,'llm-model-qwen2.5-1.5B-Int4-ax630c', '0.4', src_folder, revision], 'llm-model-qwen2.5-coder-0.5B-ax630c':[create_data_deb,'llm-model-qwen2.5-coder-0.5B-ax630c', data_version, src_folder, revision], 'llm-model-qwen3-0.6B-ax630c':[create_data_deb,'llm-model-qwen3-0.6B-ax630c', '0.4', src_folder, revision], + ## AX650 + 'llm-model-qwen2.5-0.5B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-0.5B-Int4-ax650', '0.4', src_folder, revision], + 'llm-model-qwen2.5-HA-0.5B-ctx-ax650':[create_data_deb,'llm-model-qwen2.5-HA-0.5B-ctx-ax650', '0.5', src_folder, revision], + 'llm-model-qwen2.5-1.5B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-1.5B-Int4-ax650', '0.4', src_folder, revision], + 'llm-model-qwen2.5-3B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-3B-Int4-ax650', '0.4', src_folder, revision], + 'llm-model-qwen2.5-7B-Int4-ax650':[create_data_deb,'llm-model-qwen2.5-7B-Int4-ax650', '0.4', src_folder, revision], + # Llama model 'llm-model-llama3.2-1B-prefill-ax630c':[create_data_deb,'llm-model-llama3.2-1B-prefill-ax630c', data_version, src_folder, revision], 'llm-model-llama3.2-1B-p256-ax630c':[create_data_deb,'llm-model-llama3.2-1B-p256-ax630c', '0.4', src_folder, revision], 'llm-model-openbuddy-llama3.2-1B-ax630c':[create_data_deb,'llm-model-openbuddy-llama3.2-1B-ax630c', data_version, src_folder, revision], + # InternVL model 'llm-model-internvl2.5-1B-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-ax630c', '0.4', src_folder, revision], 'llm-model-internvl2.5-1B-364-ax630c':[create_data_deb,'llm-model-internvl2.5-1B-364-ax630c', '0.4', src_folder, revision], + 'llm-model-internvl3-1B-448-ax650':[create_data_deb,'llm-model-internvl3-1B-448-ax650', '0.6', src_folder, revision], + # DeepSeek model + ## AX630C 'llm-model-deepseek-r1-1.5B-ax630c':[create_data_deb,'llm-model-deepseek-r1-1.5B-ax630c', '0.3', src_folder, revision], 'llm-model-deepseek-r1-1.5B-p256-ax630c':[create_data_deb,'llm-model-deepseek-r1-1.5B-p256-ax630c', '0.4', src_folder, revision], + ## AX650 + 'llm-model-deepseek-r1-1.5B-Int4-ax650':[create_data_deb,'llm-model-deepseek-r1-1.5B-Int4-ax650', '0.4', src_folder, revision],# + 'llm-model-deepseek-r1-7B-Int4-ax650':[create_data_deb,'llm-model-deepseek-r1-7B-Int4-ax650', '0.4', src_folder, revision],# + # Smolvlm model + ## AX630C 'llm-model-smolvlm-256M-ax630c':[create_data_deb,'llm-model-smolvlm-256M-ax630c', '0.4', src_folder, revision], + 'llm-model-smolvlm-256M-ax650':[create_data_deb,'llm-model-smolvlm-256M-ax650', '0.4', src_folder, revision], + ## AX650 'llm-model-smolvlm-500M-ax630c':[create_data_deb,'llm-model-smolvlm-500M-ax630c', '0.4', src_folder, revision], # 'llm-model-qwen2-0.5B-prefill-20e':[create_data_deb,'llm-model-qwen2-0.5B-prefill-20e', data_version, src_folder, revision], # 'llm-model-qwen2-1.5B-prefill-20e':[create_data_deb,'llm-model-qwen2-1.5B-prefill-20e', data_version, src_folder, revision]