m5/StackFlow
0
0
Fork 0
mirror of https://github.com/m5stack/StackFlow.git synced 2026-05-20 11:32:11 -07:00
Code Issues Packages Projects Releases Wiki Activity

[update] perf llm backend & add c tokenizer

Browse Source
This commit is contained in:
LittleMouse
2025-12-18 10:53:57 +08:00
parent a674665537
commit cc9d1bc6bc
16 changed files with 1316 additions and 1410 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/projects_llm_framework_doc/llm_cosyvoice2.md
+223
View File
@@ -0,0 +1,223 @@
# llm_cosy_voice
使用 npu 加速的文字转语音单元，用于提供文字转语音服务，可使用语音克隆，用于提供多语言转语音服务。
## setup
配置单元工作。
发送 json：
```json
cosy_voice
{
"request_id": "2",
"work_id": "cosy_voice",
"action": "setup",
"object": "cosy_voice.setup",
"data": {
"model": "CosyVoice2-0.5B-ax650",
"response_format": "file",
"input": "tts.utf-8",
"enoutput": false
}
}
```
- request_id：参考基本数据解释。
- work_id：配置单元时，为 `cosy_voice`。
- action：调用的方法为 `setup`。
- object：传输的数据类型为 `cosy_voice.setup`。
- model：使用的模型为 `CosyVoice2-0.5B-ax650` 模型。
- prompt_files：要克隆的音频信息文件。
- response_format：返回结果为 `sys.pcm`, 系统音频数据，并直接发送到 llm-audio 模块进行播放。返回结果为 `file`, 生成的音频写 wav 文件，可用 `prompt_data` 指定路径或文件名。
- input：输入的为 `tts.utf-8`,代表的是从用户输入。
- enoutput：是否起用用户结果输出。
响应 json：
```json
{
"created": 1761791627,
"data": "None",
"error": {
"code": 0,
"message": ""
},
"object": "None",
"request_id": "2",
"work_id": "cosy_voice.1000"
}
```
- created：消息创建时间，unix 时间。
- work_id：返回成功创建的 work_id 单元。
## inference
### 流式输入
```json
{
"request_id": "2",
"work_id": "cosy_voice.1000",
"action": "inference",
"object": "cosy_voice.utf-8.stream",
"data": {
"delta": "今天天气真好！",
"index": 0,
"finish": true
}
}
```
- object：传输的数据类型为 `cosy_voice.utf-8.stream` 代表的是从用户 utf-8 的流式输入
- delta：流式输入的分段数据
- index：流式输入的分段索引
- finish:流式输入是否完成的标志位
### 非流式输入
```json
{
"request_id": "2",
"work_id": "cosy_voice.1000",
"action": "inference",
"object": "cosy_voice.utf-8",
"data": "今天天气真好！"
}
```
- object：传输的数据类型为 `cosy_voice.utf-8` 代表的是从用户 utf-8 的非流式输入
- data：非流式输入的数据
## pause
暂停单元工作。
发送 json：
```json
{
"request_id": "5",
"work_id": "cosy_voice.1000",
"action": "pause"
}
```
响应 json：
```json
{
"created": 1761791706,
"data": "None",
"error": {
"code": 0,
"message": ""
},
"object": "None",
"request_id": "5",
"work_id": "cosy_voice.1000"
}
```
error::code 为 0 表示执行成功。
## exit
单元退出。
发送 json：
```json
{
"request_id": "7",
"work_id": "cosy_voice.1000",
"action": "exit"
}
```
响应 json：
```json
{
"created": 1761791854,
"data": "None",
"error": {
"code": 0,
"message": ""
},
"object": "None",
"request_id": "7",
"work_id": "cosy_voice.1000"
}
```
error::code 为 0 表示执行成功。
## taskinfo
获取任务列表。
发送 json：
```json
{
"request_id": "2",
"work_id": "cosy_voice",
"action": "taskinfo"
}
```
响应 json：
```json
{
"created": 1761791739,
"data": [
"cosy_voice.1000"
],
"error": {
"code": 0,
"message": ""
},
"object": "llm.tasklist",
"request_id": "2",
"work_id": "cosy_voice"
}
```
获取任务运行参数。
```json
{
"request_id": "2",
"work_id": "cosy_voice.1000",
"action": "taskinfo"
}
```
响应 json：
```json
{
"created": 1761791761,
"data": {
"enoutput": false,
"inputs": [
"tts.utf-8"
],
"model": "CosyVoice2-0.5B-ax650",
"response_format": "sys.pcm"
},
"error": {
"code": 0,
"message": ""
},
"object": "cosy_voice.taskinfo",
"request_id": "2",
"work_id": "cosy_voice.1000"
}
```
> **注意：work_id 是按照单元的初始化注册顺序增加的，并不是固定的索引值。**
> **同类型单元不能配置多个单元同时工作，否则会产生未知错误。例如 tts 和 melo tts 不能同时拍起用工作。**
ext_components/tokenizer/Kconfig
+5
View File
@@ -0,0 +1,5 @@
menuconfig AX_TOKENIZER_ENABLED
bool "Enable tokenizer support"
default n
help
enable tokenizer support
ext_components/tokenizer/SConstruct
+52
View File
@@ -0,0 +1,52 @@
# component2/SConscript
Import("env")
import os
from pathlib import Path
with open(env["PROJECT_TOOL_S"]) as f:
exec(f.read())
_SDK_PATH = os.path.normpath(
os.environ.get("SDK_PATH", str(Path(os.getcwd()) / ".." / ".."))
)
env["GIT_REPO_LISTS"]["tokenizer"] = {
"url": "https://github.com/ZHEQIUSHUI/tokenizer.git",
"commit": "83f41d4b5b9a135c167d44fcdf2a0c56ebacca6d",
"path": str(Path(_SDK_PATH) / "github_source" / "tokenizer"),
}
if "CONFIG_AX_TOKENIZER_ENABLED" in os.environ:
check_component("tokenizer")
SRCS = []
INCLUDE = []
PRIVATE_INCLUDE = []
REQUIREMENTS = []
STATIC_LIB = []
DYNAMIC_LIB = []
DEFINITIONS = []
DEFINITIONS_PRIVATE = []
LDFLAGS = []
LINK_SEARCH_PATH = []
INCLUDE += [
os.path.join(env["GIT_REPO_LISTS"]["tokenizer"]["path"], "include"),
]
print("AX-TOKENIZER INCLUDE:", INCLUDE)
env["COMPONENTS"].append(
{
"target": os.path.basename(env["component_dir"]),
"SRCS": SRCS,
"INCLUDE": INCLUDE,
"PRIVATE_INCLUDE": PRIVATE_INCLUDE,
"REQUIREMENTS": REQUIREMENTS,
"STATIC_LIB": STATIC_LIB,
"DYNAMIC_LIB": DYNAMIC_LIB,
"DEFINITIONS": DEFINITIONS,
"DEFINITIONS_PRIVATE": DEFINITIONS_PRIVATE,
"LDFLAGS": LDFLAGS,
"LINK_SEARCH_PATH": LINK_SEARCH_PATH,
"REGISTER": "static",
}
)
projects/llm_framework/main_cosy_voice/src/runner/LLM.hpp
+3 -3
View File
@@ -246,10 +246,10 @@ public:
void Deinit()
{
for (int i = 0; i < _attr.axmodel_num; i++) {
llama_layers[i].layer.release();
llama_layers[i].layer.deinit();
}
llama_post.release();
llm_decoder.release();
llama_post.deinit();
llm_decoder.deinit();
embed_selector.Deinit();
llm_embed_selector.Deinit();
speech_embed_selector.Deinit();
projects/llm_framework/main_cosy_voice/src/runner/Token2wav.hpp
+9 -9
View File
@@ -145,15 +145,15 @@ public:
void Deinit()
{
flow_encoder_28.release();
flow_encoder_53.release();
flow_encoder_78.release();
flow_encoder_50_final.release();
flow_estimator_200.release();
flow_estimator_250.release();
flow_estimator_300.release();
hift_p2_50_first.release();
hift_p2_58.release();
flow_encoder_28.deinit();
flow_encoder_53.deinit();
flow_encoder_78.deinit();
flow_encoder_50_final.deinit();
flow_estimator_200.deinit();
flow_estimator_250.deinit();
flow_estimator_300.deinit();
hift_p2_50_first.deinit();
hift_p2_58.deinit();
flow_embed_selector.Deinit();
}
projects/llm_framework/main_cosy_voice/src/runner/ax_model_runner/ax_model_runner.hpp
+101 -108
View File
@@ -4,31 +4,7 @@
#include <map>
#include <stdexcept>
typedef enum _color_space_e
{
axdl_color_space_unknown,
axdl_color_space_nv12,
axdl_color_space_nv21,
axdl_color_space_bgr,
axdl_color_space_rgb,
} ax_color_space_e;
typedef struct _image_t
{
unsigned long long int pPhy;
void *pVir;
unsigned int nSize;
unsigned int nWidth;
unsigned int nHeight;
ax_color_space_e eDtype;
union
{
int tStride_H, tStride_W, tStride_C;
};
} ax_image_t;
typedef struct
{
typedef struct {
std::string sName;
unsigned int nIdx;
std::vector<unsigned int> vShape;
@@ -37,8 +13,7 @@ typedef struct
void *pVirAddr;
} ax_runner_tensor_t;
class ax_runner_base
{
class ax_runner_base {
protected:
std::vector<ax_runner_tensor_t> moutput_tensors;
std::vector<ax_runner_tensor_t> minput_tensors;
@@ -52,106 +27,124 @@ protected:
std::map<std::string, std::vector<ax_runner_tensor_t>> map_group_output_tensors;
std::map<std::string, std::vector<ax_runner_tensor_t>> map_group_input_tensors;
void build_tensor_maps()
{
map_input_tensors.clear();
for (const auto &t : minput_tensors) map_input_tensors[t.sName] = t;
map_output_tensors.clear();
for (const auto &t : moutput_tensors) map_output_tensors[t.sName] = t;
map_group_input_tensors.clear();
for (const auto &grp : mgroup_input_tensors) {
for (const auto &t : grp) map_group_input_tensors[t.sName].push_back(t);
}
map_group_output_tensors.clear();
for (const auto &grp : mgroup_output_tensors) {
for (const auto &t : grp) map_group_output_tensors[t.sName].push_back(t);
}
}
public:
virtual ~ax_runner_base()
{
}
virtual int init(const char *model_file, bool use_mmap = false) = 0;
virtual int init(char *model_buffer, size_t model_size) = 0;
virtual int init(char *model_buffer, size_t model_size) = 0;
virtual void deinit() = 0;
virtual void deinit() = 0;
int get_num_inputs() { return minput_tensors.size(); };
int get_num_outputs() { return moutput_tensors.size(); };
int get_num_input_groups() { return mgroup_input_tensors.size(); };
int get_num_output_groups() { return mgroup_output_tensors.size(); };
const ax_runner_tensor_t &get_input(int idx) { return minput_tensors[idx]; }
const ax_runner_tensor_t *get_inputs_ptr() { return minput_tensors.data(); }
const ax_runner_tensor_t &get_input(std::string name)
int get_num_inputs()
{
if (map_input_tensors.size() == 0)
{
for (size_t i = 0; i < minput_tensors.size(); i++)
{
map_input_tensors[minput_tensors[i].sName] = minput_tensors[i];
}
}
if (map_input_tensors.find(name) == map_input_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return minput_tensors.size();
};
int get_num_outputs()
{
return moutput_tensors.size();
};
int get_num_input_groups()
{
return mgroup_input_tensors.size();
};
int get_num_output_groups()
{
return mgroup_output_tensors.size();
};
return map_input_tensors[name];
const ax_runner_tensor_t &get_input(int idx)
{
return minput_tensors[idx];
}
const ax_runner_tensor_t *get_inputs_ptr()
{
return minput_tensors.data();
}
const ax_runner_tensor_t &get_input(int grpid, int idx) { return mgroup_input_tensors[grpid][idx]; }
const ax_runner_tensor_t *get_inputs_ptr(int grpid) { return mgroup_input_tensors[grpid].data(); }
const ax_runner_tensor_t &get_input(int grpid, std::string name)
const ax_runner_tensor_t &get_input(const std::string &name)
{
if (map_group_input_tensors.size() == 0)
{
for (size_t i = 0; i < mgroup_input_tensors.size(); i++)
{
for (size_t j = 0; j < mgroup_input_tensors[i].size(); j++)
{
map_group_input_tensors[mgroup_input_tensors[i][j].sName].push_back(mgroup_input_tensors[i][j]);
}
}
}
if (map_group_input_tensors.find(name) == map_group_input_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return map_group_input_tensors[name][grpid];
// return map_input_tensors[name];
auto it = map_input_tensors.find(name);
if (it == map_input_tensors.end()) throw std::runtime_error("input tensor not found: " + name);
return it->second;
}
const ax_runner_tensor_t &get_output(int idx) { return moutput_tensors[idx]; }
const ax_runner_tensor_t *get_outputs_ptr() { return moutput_tensors.data(); }
const ax_runner_tensor_t &get_output(std::string name)
const ax_runner_tensor_t &get_input(int grpid, int idx)
{
if (map_output_tensors.size() == 0)
{
for (size_t i = 0; i < moutput_tensors.size(); i++)
{
map_output_tensors[moutput_tensors[i].sName] = moutput_tensors[i];
}
}
if (map_output_tensors.find(name) == map_output_tensors.end())
{
throw std::runtime_error("output tensor not found: " + name);
}
return map_output_tensors[name];
return mgroup_input_tensors[grpid][idx];
}
const ax_runner_tensor_t *get_inputs_ptr(int grpid)
{
return mgroup_input_tensors[grpid].data();
}
const ax_runner_tensor_t &get_output(int grpid, int idx) { return mgroup_output_tensors[grpid][idx]; }
const ax_runner_tensor_t *get_outputs_ptr(int grpid) { return mgroup_output_tensors[grpid].data(); }
const ax_runner_tensor_t &get_output(int grpid, std::string name)
const ax_runner_tensor_t &get_input(int grpid, const std::string &name)
{
if (map_group_output_tensors.size() == 0)
{
for (size_t i = 0; i < mgroup_output_tensors.size(); i++)
{
for (size_t j = 0; j < mgroup_output_tensors[i].size(); j++)
{
map_group_output_tensors[mgroup_output_tensors[i][j].sName].push_back(mgroup_output_tensors[i][j]);
}
}
}
if (map_group_output_tensors.find(name) == map_group_output_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return map_group_output_tensors[name][grpid];
auto it = map_group_input_tensors.find(name);
if (it == map_group_input_tensors.end()) throw std::runtime_error("input tensor not found: " + name);
if (grpid < 0 || grpid >= (int)it->second.size())
throw std::runtime_error("group id out of range for: " + name);
return it->second[grpid];
}
virtual int inference() = 0;
const ax_runner_tensor_t &get_output(int idx)
{
return moutput_tensors[idx];
}
const ax_runner_tensor_t *get_outputs_ptr()
{
return moutput_tensors.data();
}
const ax_runner_tensor_t &get_output(const std::string &name)
{
auto it = map_output_tensors.find(name);
if (it == map_output_tensors.end()) throw std::runtime_error("output tensor not found: " + name);
return it->second;
}
const ax_runner_tensor_t &get_output(int grpid, int idx)
{
return mgroup_output_tensors[grpid][idx];
}
const ax_runner_tensor_t *get_outputs_ptr(int grpid)
{
return mgroup_output_tensors[grpid].data();
}
const ax_runner_tensor_t &get_output(int grpid, const std::string &name)
{
auto it = map_group_output_tensors.find(name);
if (it == map_group_output_tensors.end()) throw std::runtime_error("output tensor not found: " + name);
if (grpid < 0 || grpid >= (int)it->second.size())
throw std::runtime_error("group id out of range for: " + name);
return it->second[grpid];
}
virtual int inference() = 0;
virtual int inference(int grpid) = 0;
int operator()()
{
return inference();
}
};
// int ax_cmmcpy(unsigned long long int dst, unsigned long long int src, int size);
};
projects/llm_framework/main_cosy_voice/src/runner/ax_model_runner/ax_model_runner_ax650.cpp
+222 -347
View File
File diff suppressed because it is too large Load Diff
projects/llm_framework/main_cosy_voice/src/runner/ax_model_runner/ax_model_runner_ax650.hpp
+10 -7
View File
@@ -1,20 +1,23 @@
#pragma once
#include "ax_model_runner.hpp"
class ax_runner_ax650 : public ax_runner_base
{
struct ax_runner_ax650_handle_t;
class ax_runner_ax650 : public ax_runner_base {
protected:
struct ax_joint_runner_ax650_handle_t *m_handle = nullptr;
bool _parepare_io = false;
struct ax_runner_ax650_handle_t *m_handle = nullptr;
int sub_init();
public:
ax_runner_ax650() = default;
virtual ~ax_runner_ax650()
{
deinit();
}
int init(const char *model_file, bool use_mmap = false) override;
int init(char *model_buffer, size_t model_size) override;
void release();
void deinit() override;
int inference() override;
projects/llm_framework/main_llm/src/runner/LLM.hpp
+4 -4
View File
@@ -243,9 +243,9 @@ public:
void Deinit()
{
for (int i = 0; i < _attr.axmodel_num; i++) {
llama_layers[i].layer.release();
llama_layers[i].layer.deinit();
}
llama_post.release();
llama_post.deinit();
embed_selector.Deinit();
}
@@ -686,9 +686,9 @@ public:
void Deinit()
{
for (int i = 0; i < _attr.axmodel_num; i++) {
llama_layers[i].layer.release();
llama_layers[i].layer.deinit();
}
llama_post.release();
llama_post.deinit();
embed_selector.Deinit();
}
projects/llm_framework/main_llm/src/runner/ax_model_runner/ax_model_runner.hpp
+101 -108
View File
@@ -4,31 +4,7 @@
#include <map>
#include <stdexcept>
typedef enum _color_space_e
{
axdl_color_space_unknown,
axdl_color_space_nv12,
axdl_color_space_nv21,
axdl_color_space_bgr,
axdl_color_space_rgb,
} ax_color_space_e;
typedef struct _image_t
{
unsigned long long int pPhy;
void *pVir;
unsigned int nSize;
unsigned int nWidth;
unsigned int nHeight;
ax_color_space_e eDtype;
union
{
int tStride_H, tStride_W, tStride_C;
};
} ax_image_t;
typedef struct
{
typedef struct {
std::string sName;
unsigned int nIdx;
std::vector<unsigned int> vShape;
@@ -37,8 +13,7 @@ typedef struct
void *pVirAddr;
} ax_runner_tensor_t;
class ax_runner_base
{
class ax_runner_base {
protected:
std::vector<ax_runner_tensor_t> moutput_tensors;
std::vector<ax_runner_tensor_t> minput_tensors;
@@ -52,106 +27,124 @@ protected:
std::map<std::string, std::vector<ax_runner_tensor_t>> map_group_output_tensors;
std::map<std::string, std::vector<ax_runner_tensor_t>> map_group_input_tensors;
void build_tensor_maps()
{
map_input_tensors.clear();
for (const auto &t : minput_tensors) map_input_tensors[t.sName] = t;
map_output_tensors.clear();
for (const auto &t : moutput_tensors) map_output_tensors[t.sName] = t;
map_group_input_tensors.clear();
for (const auto &grp : mgroup_input_tensors) {
for (const auto &t : grp) map_group_input_tensors[t.sName].push_back(t);
}
map_group_output_tensors.clear();
for (const auto &grp : mgroup_output_tensors) {
for (const auto &t : grp) map_group_output_tensors[t.sName].push_back(t);
}
}
public:
virtual ~ax_runner_base()
{
}
virtual int init(const char *model_file, bool use_mmap = false) = 0;
virtual int init(char *model_buffer, size_t model_size) = 0;
virtual int init(char *model_buffer, size_t model_size) = 0;
virtual void deinit() = 0;
virtual void deinit() = 0;
int get_num_inputs() { return minput_tensors.size(); };
int get_num_outputs() { return moutput_tensors.size(); };
int get_num_input_groups() { return mgroup_input_tensors.size(); };
int get_num_output_groups() { return mgroup_output_tensors.size(); };
const ax_runner_tensor_t &get_input(int idx) { return minput_tensors[idx]; }
const ax_runner_tensor_t *get_inputs_ptr() { return minput_tensors.data(); }
const ax_runner_tensor_t &get_input(std::string name)
int get_num_inputs()
{
if (map_input_tensors.size() == 0)
{
for (size_t i = 0; i < minput_tensors.size(); i++)
{
map_input_tensors[minput_tensors[i].sName] = minput_tensors[i];
}
}
if (map_input_tensors.find(name) == map_input_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return minput_tensors.size();
};
int get_num_outputs()
{
return moutput_tensors.size();
};
int get_num_input_groups()
{
return mgroup_input_tensors.size();
};
int get_num_output_groups()
{
return mgroup_output_tensors.size();
};
return map_input_tensors[name];
const ax_runner_tensor_t &get_input(int idx)
{
return minput_tensors[idx];
}
const ax_runner_tensor_t *get_inputs_ptr()
{
return minput_tensors.data();
}
const ax_runner_tensor_t &get_input(int grpid, int idx) { return mgroup_input_tensors[grpid][idx]; }
const ax_runner_tensor_t *get_inputs_ptr(int grpid) { return mgroup_input_tensors[grpid].data(); }
const ax_runner_tensor_t &get_input(int grpid, std::string name)
const ax_runner_tensor_t &get_input(const std::string &name)
{
if (map_group_input_tensors.size() == 0)
{
for (size_t i = 0; i < mgroup_input_tensors.size(); i++)
{
for (size_t j = 0; j < mgroup_input_tensors[i].size(); j++)
{
map_group_input_tensors[mgroup_input_tensors[i][j].sName].push_back(mgroup_input_tensors[i][j]);
}
}
}
if (map_group_input_tensors.find(name) == map_group_input_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return map_group_input_tensors[name][grpid];
// return map_input_tensors[name];
auto it = map_input_tensors.find(name);
if (it == map_input_tensors.end()) throw std::runtime_error("input tensor not found: " + name);
return it->second;
}
const ax_runner_tensor_t &get_output(int idx) { return moutput_tensors[idx]; }
const ax_runner_tensor_t *get_outputs_ptr() { return moutput_tensors.data(); }
const ax_runner_tensor_t &get_output(std::string name)
const ax_runner_tensor_t &get_input(int grpid, int idx)
{
if (map_output_tensors.size() == 0)
{
for (size_t i = 0; i < moutput_tensors.size(); i++)
{
map_output_tensors[moutput_tensors[i].sName] = moutput_tensors[i];
}
}
if (map_output_tensors.find(name) == map_output_tensors.end())
{
throw std::runtime_error("output tensor not found: " + name);
}
return map_output_tensors[name];
return mgroup_input_tensors[grpid][idx];
}
const ax_runner_tensor_t *get_inputs_ptr(int grpid)
{
return mgroup_input_tensors[grpid].data();
}
const ax_runner_tensor_t &get_output(int grpid, int idx) { return mgroup_output_tensors[grpid][idx]; }
const ax_runner_tensor_t *get_outputs_ptr(int grpid) { return mgroup_output_tensors[grpid].data(); }
const ax_runner_tensor_t &get_output(int grpid, std::string name)
const ax_runner_tensor_t &get_input(int grpid, const std::string &name)
{
if (map_group_output_tensors.size() == 0)
{
for (size_t i = 0; i < mgroup_output_tensors.size(); i++)
{
for (size_t j = 0; j < mgroup_output_tensors[i].size(); j++)
{
map_group_output_tensors[mgroup_output_tensors[i][j].sName].push_back(mgroup_output_tensors[i][j]);
}
}
}
if (map_group_output_tensors.find(name) == map_group_output_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return map_group_output_tensors[name][grpid];
auto it = map_group_input_tensors.find(name);
if (it == map_group_input_tensors.end()) throw std::runtime_error("input tensor not found: " + name);
if (grpid < 0 || grpid >= (int)it->second.size())
throw std::runtime_error("group id out of range for: " + name);
return it->second[grpid];
}
virtual int inference() = 0;
const ax_runner_tensor_t &get_output(int idx)
{
return moutput_tensors[idx];
}
const ax_runner_tensor_t *get_outputs_ptr()
{
return moutput_tensors.data();
}
const ax_runner_tensor_t &get_output(const std::string &name)
{
auto it = map_output_tensors.find(name);
if (it == map_output_tensors.end()) throw std::runtime_error("output tensor not found: " + name);
return it->second;
}
const ax_runner_tensor_t &get_output(int grpid, int idx)
{
return mgroup_output_tensors[grpid][idx];
}
const ax_runner_tensor_t *get_outputs_ptr(int grpid)
{
return mgroup_output_tensors[grpid].data();
}
const ax_runner_tensor_t &get_output(int grpid, const std::string &name)
{
auto it = map_group_output_tensors.find(name);
if (it == map_group_output_tensors.end()) throw std::runtime_error("output tensor not found: " + name);
if (grpid < 0 || grpid >= (int)it->second.size())
throw std::runtime_error("group id out of range for: " + name);
return it->second[grpid];
}
virtual int inference() = 0;
virtual int inference(int grpid) = 0;
int operator()()
{
return inference();
}
};
// int ax_cmmcpy(unsigned long long int dst, unsigned long long int src, int size);
};
projects/llm_framework/main_llm/src/runner/ax_model_runner/ax_model_runner_ax650.cpp
+223 -348
View File
File diff suppressed because it is too large Load Diff
projects/llm_framework/main_llm/src/runner/ax_model_runner/ax_model_runner_ax650.hpp
+10 -7
View File
@@ -1,20 +1,23 @@
#pragma once
#include "ax_model_runner.hpp"
class ax_runner_ax650 : public ax_runner_base
{
struct ax_runner_ax650_handle_t;
class ax_runner_ax650 : public ax_runner_base {
protected:
struct ax_joint_runner_ax650_handle_t *m_handle = nullptr;
bool _parepare_io = false;
struct ax_runner_ax650_handle_t *m_handle = nullptr;
int sub_init();
public:
ax_runner_ax650() = default;
virtual ~ax_runner_ax650()
{
deinit();
}
int init(const char *model_file, bool use_mmap = false) override;
int init(char *model_buffer, size_t model_size) override;
void release();
void deinit() override;
int inference() override;
projects/llm_framework/main_vlm/src/runner/LLM.hpp
+10 -10
View File
@@ -285,11 +285,11 @@ public:
void Deinit()
{
for (int i = 0; i < _attr.axmodel_num; i++) {
llama_layers[i].layer.release();
llama_layers[i].layer.deinit();
}
llama_post.release();
vpm_encoder.release();
vpm_resampler.release();
llama_post.deinit();
vpm_encoder.deinit();
vpm_resampler.deinit();
embed_selector.Deinit();
}
@@ -856,10 +856,10 @@ public:
void Deinit()
{
for (int i = 0; i < _attr.axmodel_num; i++) {
llama_layers[i].layer.release();
llama_layers[i].layer.deinit();
}
llama_post.release();
image_encoder.release();
llama_post.deinit();
image_encoder.deinit();
embed_selector.Deinit();
}
@@ -1958,10 +1958,10 @@ public:
void Deinit()
{
for (int i = 0; i < _attr.axmodel_num; i++) {
llama_layers[i].layer.release();
llama_layers[i].layer.deinit();
}
llama_post.release();
image_encoder.release();
llama_post.deinit();
image_encoder.deinit();
embed_selector.Deinit();
}
projects/llm_framework/main_vlm/src/runner/ax_model_runner/ax_model_runner.hpp
+101 -108
View File
@@ -4,31 +4,7 @@
#include <map>
#include <stdexcept>
typedef enum _color_space_e
{
axdl_color_space_unknown,
axdl_color_space_nv12,
axdl_color_space_nv21,
axdl_color_space_bgr,
axdl_color_space_rgb,
} ax_color_space_e;
typedef struct _image_t
{
unsigned long long int pPhy;
void *pVir;
unsigned int nSize;
unsigned int nWidth;
unsigned int nHeight;
ax_color_space_e eDtype;
union
{
int tStride_H, tStride_W, tStride_C;
};
} ax_image_t;
typedef struct
{
typedef struct {
std::string sName;
unsigned int nIdx;
std::vector<unsigned int> vShape;
@@ -37,8 +13,7 @@ typedef struct
void *pVirAddr;
} ax_runner_tensor_t;
class ax_runner_base
{
class ax_runner_base {
protected:
std::vector<ax_runner_tensor_t> moutput_tensors;
std::vector<ax_runner_tensor_t> minput_tensors;
@@ -52,106 +27,124 @@ protected:
std::map<std::string, std::vector<ax_runner_tensor_t>> map_group_output_tensors;
std::map<std::string, std::vector<ax_runner_tensor_t>> map_group_input_tensors;
void build_tensor_maps()
{
map_input_tensors.clear();
for (const auto &t : minput_tensors) map_input_tensors[t.sName] = t;
map_output_tensors.clear();
for (const auto &t : moutput_tensors) map_output_tensors[t.sName] = t;
map_group_input_tensors.clear();
for (const auto &grp : mgroup_input_tensors) {
for (const auto &t : grp) map_group_input_tensors[t.sName].push_back(t);
}
map_group_output_tensors.clear();
for (const auto &grp : mgroup_output_tensors) {
for (const auto &t : grp) map_group_output_tensors[t.sName].push_back(t);
}
}
public:
virtual ~ax_runner_base()
{
}
virtual int init(const char *model_file, bool use_mmap = false) = 0;
virtual int init(char *model_buffer, size_t model_size) = 0;
virtual int init(char *model_buffer, size_t model_size) = 0;
virtual void deinit() = 0;
virtual void deinit() = 0;
int get_num_inputs() { return minput_tensors.size(); };
int get_num_outputs() { return moutput_tensors.size(); };
int get_num_input_groups() { return mgroup_input_tensors.size(); };
int get_num_output_groups() { return mgroup_output_tensors.size(); };
const ax_runner_tensor_t &get_input(int idx) { return minput_tensors[idx]; }
const ax_runner_tensor_t *get_inputs_ptr() { return minput_tensors.data(); }
const ax_runner_tensor_t &get_input(std::string name)
int get_num_inputs()
{
if (map_input_tensors.size() == 0)
{
for (size_t i = 0; i < minput_tensors.size(); i++)
{
map_input_tensors[minput_tensors[i].sName] = minput_tensors[i];
}
}
if (map_input_tensors.find(name) == map_input_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return minput_tensors.size();
};
int get_num_outputs()
{
return moutput_tensors.size();
};
int get_num_input_groups()
{
return mgroup_input_tensors.size();
};
int get_num_output_groups()
{
return mgroup_output_tensors.size();
};
return map_input_tensors[name];
const ax_runner_tensor_t &get_input(int idx)
{
return minput_tensors[idx];
}
const ax_runner_tensor_t *get_inputs_ptr()
{
return minput_tensors.data();
}
const ax_runner_tensor_t &get_input(int grpid, int idx) { return mgroup_input_tensors[grpid][idx]; }
const ax_runner_tensor_t *get_inputs_ptr(int grpid) { return mgroup_input_tensors[grpid].data(); }
const ax_runner_tensor_t &get_input(int grpid, std::string name)
const ax_runner_tensor_t &get_input(const std::string &name)
{
if (map_group_input_tensors.size() == 0)
{
for (size_t i = 0; i < mgroup_input_tensors.size(); i++)
{
for (size_t j = 0; j < mgroup_input_tensors[i].size(); j++)
{
map_group_input_tensors[mgroup_input_tensors[i][j].sName].push_back(mgroup_input_tensors[i][j]);
}
}
}
if (map_group_input_tensors.find(name) == map_group_input_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return map_group_input_tensors[name][grpid];
// return map_input_tensors[name];
auto it = map_input_tensors.find(name);
if (it == map_input_tensors.end()) throw std::runtime_error("input tensor not found: " + name);
return it->second;
}
const ax_runner_tensor_t &get_output(int idx) { return moutput_tensors[idx]; }
const ax_runner_tensor_t *get_outputs_ptr() { return moutput_tensors.data(); }
const ax_runner_tensor_t &get_output(std::string name)
const ax_runner_tensor_t &get_input(int grpid, int idx)
{
if (map_output_tensors.size() == 0)
{
for (size_t i = 0; i < moutput_tensors.size(); i++)
{
map_output_tensors[moutput_tensors[i].sName] = moutput_tensors[i];
}
}
if (map_output_tensors.find(name) == map_output_tensors.end())
{
throw std::runtime_error("output tensor not found: " + name);
}
return map_output_tensors[name];
return mgroup_input_tensors[grpid][idx];
}
const ax_runner_tensor_t *get_inputs_ptr(int grpid)
{
return mgroup_input_tensors[grpid].data();
}
const ax_runner_tensor_t &get_output(int grpid, int idx) { return mgroup_output_tensors[grpid][idx]; }
const ax_runner_tensor_t *get_outputs_ptr(int grpid) { return mgroup_output_tensors[grpid].data(); }
const ax_runner_tensor_t &get_output(int grpid, std::string name)
const ax_runner_tensor_t &get_input(int grpid, const std::string &name)
{
if (map_group_output_tensors.size() == 0)
{
for (size_t i = 0; i < mgroup_output_tensors.size(); i++)
{
for (size_t j = 0; j < mgroup_output_tensors[i].size(); j++)
{
map_group_output_tensors[mgroup_output_tensors[i][j].sName].push_back(mgroup_output_tensors[i][j]);
}
}
}
if (map_group_output_tensors.find(name) == map_group_output_tensors.end())
{
throw std::runtime_error("input tensor not found: " + name);
}
return map_group_output_tensors[name][grpid];
auto it = map_group_input_tensors.find(name);
if (it == map_group_input_tensors.end()) throw std::runtime_error("input tensor not found: " + name);
if (grpid < 0 || grpid >= (int)it->second.size())
throw std::runtime_error("group id out of range for: " + name);
return it->second[grpid];
}
virtual int inference() = 0;
const ax_runner_tensor_t &get_output(int idx)
{
return moutput_tensors[idx];
}
const ax_runner_tensor_t *get_outputs_ptr()
{
return moutput_tensors.data();
}
const ax_runner_tensor_t &get_output(const std::string &name)
{
auto it = map_output_tensors.find(name);
if (it == map_output_tensors.end()) throw std::runtime_error("output tensor not found: " + name);
return it->second;
}
const ax_runner_tensor_t &get_output(int grpid, int idx)
{
return mgroup_output_tensors[grpid][idx];
}
const ax_runner_tensor_t *get_outputs_ptr(int grpid)
{
return mgroup_output_tensors[grpid].data();
}
const ax_runner_tensor_t &get_output(int grpid, const std::string &name)
{
auto it = map_group_output_tensors.find(name);
if (it == map_group_output_tensors.end()) throw std::runtime_error("output tensor not found: " + name);
if (grpid < 0 || grpid >= (int)it->second.size())
throw std::runtime_error("group id out of range for: " + name);
return it->second[grpid];
}
virtual int inference() = 0;
virtual int inference(int grpid) = 0;
int operator()()
{
return inference();
}
};
// int ax_cmmcpy(unsigned long long int dst, unsigned long long int src, int size);
};
projects/llm_framework/main_vlm/src/runner/ax_model_runner/ax_model_runner_ax650.cpp
+232 -344
View File
File diff suppressed because it is too large Load Diff
projects/llm_framework/main_vlm/src/runner/ax_model_runner/ax_model_runner_ax650.hpp
+10 -7
View File
@@ -1,20 +1,23 @@
#pragma once
#include "ax_model_runner.hpp"
class ax_runner_ax650 : public ax_runner_base
{
struct ax_runner_ax650_handle_t;
class ax_runner_ax650 : public ax_runner_base {
protected:
struct ax_joint_runner_ax650_handle_t *m_handle = nullptr;
bool _parepare_io = false;
struct ax_runner_ax650_handle_t *m_handle = nullptr;
int sub_init();
public:
ax_runner_ax650() = default;
virtual ~ax_runner_ax650()
{
deinit();
}
int init(const char *model_file, bool use_mmap = false) override;
int init(char *model_buffer, size_t model_size) override;
void release();
void deinit() override;
int inference() override;