dasharo/linux
0
0
Fork 0
mirror of https://github.com/Dasharo/linux.git synced 2026-03-06 15:25:10 -08:00
Code Issues Packages Projects Releases Wiki Activity

accel/amdxdna: Support hardware mailbox

Browse Source 
The hardware mailboxes are used by the driver to submit requests to
firmware and receive the completion notices from hardware.

Initially, a management mailbox channel is up and running. The driver may
request firmware to create/destroy more channels dynamically through
management channel.

Add driver internal mailbox interfaces.
  - create/destroy a mailbox channel instance
  - send a message to the firmware through a specific channel
  - wait for a notification from the specific channel

Co-developed-by: George Yang <George.Yang@amd.com>
Signed-off-by: George Yang <George.Yang@amd.com>
Co-developed-by: Min Ma <min.ma@amd.com>
Signed-off-by: Min Ma <min.ma@amd.com>
Reviewed-by: Jeffrey Hugo <quic_jhugo@quicinc.com>
Signed-off-by: Lizhi Hou <lizhi.hou@amd.com>
Signed-off-by: Jeffrey Hugo <quic_jhugo@quicinc.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20241118172942.2014541-4-lizhi.hou@amd.com
This commit is contained in:
Lizhi Hou
2024-11-18 09:29:35 -08:00
committed by Jeffrey Hugo
parent 8c9ff1b181
commit b87f920b93
18 changed files with 1768 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
MAINTAINERS
View File

@@ -1186,6 +1186,7 @@ S: Supported
T: git https://gitlab.freedesktop.org/drm/misc/kernel.git
F: Documentation/accel/amdxdna/
F: drivers/accel/amdxdna/
F: include/trace/events/amdxdna.h
F: include/uapi/drm/amdxdna_accel.h
AMD XGBE DRIVER

3
drivers/accel/amdxdna/Makefile
View File

@@ -1,9 +1,12 @@
# SPDX-License-Identifier: GPL-2.0-only
amdxdna-y := \
aie2_message.o \
aie2_pci.o \
aie2_psp.o \
aie2_smu.o \
amdxdna_mailbox.o \
amdxdna_mailbox_helper.o \
amdxdna_pci_drv.o \
amdxdna_sysfs.o \
npu1_regs.o \

194
drivers/accel/amdxdna/aie2_message.c Normal file
View File

@@ -0,0 +1,194 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
*/
#include <drm/drm_device.h>
#include <drm/drm_print.h>
#include <linux/errno.h>
#include <linux/types.h>
#include "aie2_msg_priv.h"
#include "aie2_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_mailbox_helper.h"
#include "amdxdna_pci_drv.h"
#define DECLARE_AIE2_MSG(name, op) \
DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE)
static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev,
struct xdna_mailbox_msg *msg)
{
struct amdxdna_dev *xdna = ndev->xdna;
struct xdna_notify *hdl = msg->handle;
int ret;
if (!ndev->mgmt_chann)
return -ENODEV;
drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg);
if (ret == -ETIME) {
xdna_mailbox_stop_channel(ndev->mgmt_chann);
xdna_mailbox_destroy_channel(ndev->mgmt_chann);
ndev->mgmt_chann = NULL;
}
if (!ret && *hdl->data != AIE2_STATUS_SUCCESS) {
XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x",
msg->opcode, *hdl->data);
ret = -EINVAL;
}
return ret;
}
int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
{
DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND);
return aie2_send_mgmt_msg_wait(ndev, &msg);
}
int aie2_resume_fw(struct amdxdna_dev_hdl *ndev)
{
DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME);
return aie2_send_mgmt_msg_wait(ndev, &msg);
}
int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value)
{
DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG);
req.type = type;
req.value = value;
return aie2_send_mgmt_msg_wait(ndev, &msg);
}
int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value)
{
DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG);
int ret;
req.type = type;
ret = aie2_send_mgmt_msg_wait(ndev, &msg);
if (ret) {
XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret);
return ret;
}
*value = resp.value;
return 0;
}
int aie2_check_protocol_version(struct amdxdna_dev_hdl *ndev)
{
DECLARE_AIE2_MSG(protocol_version, MSG_OP_GET_PROTOCOL_VERSION);
struct amdxdna_dev *xdna = ndev->xdna;
int ret;
ret = aie2_send_mgmt_msg_wait(ndev, &msg);
if (ret) {
XDNA_ERR(xdna, "Failed to get protocol version, ret %d", ret);
return ret;
}
if (resp.major != ndev->priv->protocol_major) {
XDNA_ERR(xdna, "Incompatible firmware protocol version major %d minor %d",
resp.major, resp.minor);
return -EINVAL;
}
if (resp.minor < ndev->priv->protocol_minor) {
XDNA_ERR(xdna, "Firmware minor version smaller than supported");
return -EINVAL;
}
return 0;
}
int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid)
{
DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID);
req.pasid = pasid;
return aie2_send_mgmt_msg_wait(ndev, &msg);
}
int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version)
{
DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION);
struct amdxdna_dev *xdna = ndev->xdna;
int ret;
ret = aie2_send_mgmt_msg_wait(ndev, &msg);
if (ret)
return ret;
XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed",
resp.major, resp.minor);
version->major = resp.major;
version->minor = resp.minor;
return 0;
}
int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata)
{
DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO);
int ret;
ret = aie2_send_mgmt_msg_wait(ndev, &msg);
if (ret)
return ret;
metadata->size = resp.info.size;
metadata->cols = resp.info.cols;
metadata->rows = resp.info.rows;
metadata->version.major = resp.info.major;
metadata->version.minor = resp.info.minor;
metadata->core.row_count = resp.info.core_rows;
metadata->core.row_start = resp.info.core_row_start;
metadata->core.dma_channel_count = resp.info.core_dma_channels;
metadata->core.lock_count = resp.info.core_locks;
metadata->core.event_reg_count = resp.info.core_events;
metadata->mem.row_count = resp.info.mem_rows;
metadata->mem.row_start = resp.info.mem_row_start;
metadata->mem.dma_channel_count = resp.info.mem_dma_channels;
metadata->mem.lock_count = resp.info.mem_locks;
metadata->mem.event_reg_count = resp.info.mem_events;
metadata->shim.row_count = resp.info.shim_rows;
metadata->shim.row_start = resp.info.shim_row_start;
metadata->shim.dma_channel_count = resp.info.shim_dma_channels;
metadata->shim.lock_count = resp.info.shim_locks;
metadata->shim.event_reg_count = resp.info.shim_events;
return 0;
}
int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
struct amdxdna_fw_ver *fw_ver)
{
DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION);
int ret;
ret = aie2_send_mgmt_msg_wait(ndev, &msg);
if (ret)
return ret;
fw_ver->major = resp.major;
fw_ver->minor = resp.minor;
fw_ver->sub = resp.sub;
fw_ver->build = resp.build;
return 0;
}

370
drivers/accel/amdxdna/aie2_msg_priv.h Normal file
View File

@@ -0,0 +1,370 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
*/
#ifndef _AIE2_MSG_PRIV_H_
#define _AIE2_MSG_PRIV_H_
enum aie2_msg_opcode {
MSG_OP_CREATE_CONTEXT = 0x2,
MSG_OP_DESTROY_CONTEXT = 0x3,
MSG_OP_SYNC_BO = 0x7,
MSG_OP_EXECUTE_BUFFER_CF = 0xC,
MSG_OP_QUERY_COL_STATUS = 0xD,
MSG_OP_QUERY_AIE_TILE_INFO = 0xE,
MSG_OP_QUERY_AIE_VERSION = 0xF,
MSG_OP_EXEC_DPU = 0x10,
MSG_OP_CONFIG_CU = 0x11,
MSG_OP_CHAIN_EXEC_BUFFER_CF = 0x12,
MSG_OP_CHAIN_EXEC_DPU = 0x13,
MSG_OP_MAX_XRT_OPCODE,
MSG_OP_SUSPEND = 0x101,
MSG_OP_RESUME = 0x102,
MSG_OP_ASSIGN_MGMT_PASID = 0x103,
MSG_OP_INVOKE_SELF_TEST = 0x104,
MSG_OP_MAP_HOST_BUFFER = 0x106,
MSG_OP_GET_FIRMWARE_VERSION = 0x108,
MSG_OP_SET_RUNTIME_CONFIG = 0x10A,
MSG_OP_GET_RUNTIME_CONFIG = 0x10B,
MSG_OP_REGISTER_ASYNC_EVENT_MSG = 0x10C,
MSG_OP_MAX_DRV_OPCODE,
MSG_OP_GET_PROTOCOL_VERSION = 0x301,
MSG_OP_MAX_OPCODE
};
enum aie2_msg_status {
AIE2_STATUS_SUCCESS = 0x0,
/* AIE Error codes */
AIE2_STATUS_AIE_SATURATION_ERROR = 0x1000001,
AIE2_STATUS_AIE_FP_ERROR = 0x1000002,
AIE2_STATUS_AIE_STREAM_ERROR = 0x1000003,
AIE2_STATUS_AIE_ACCESS_ERROR = 0x1000004,
AIE2_STATUS_AIE_BUS_ERROR = 0x1000005,
AIE2_STATUS_AIE_INSTRUCTION_ERROR = 0x1000006,
AIE2_STATUS_AIE_ECC_ERROR = 0x1000007,
AIE2_STATUS_AIE_LOCK_ERROR = 0x1000008,
AIE2_STATUS_AIE_DMA_ERROR = 0x1000009,
AIE2_STATUS_AIE_MEM_PARITY_ERROR = 0x100000a,
AIE2_STATUS_AIE_PWR_CFG_ERROR = 0x100000b,
AIE2_STATUS_AIE_BACKTRACK_ERROR = 0x100000c,
AIE2_STATUS_MAX_AIE_STATUS_CODE,
/* MGMT ERT Error codes */
AIE2_STATUS_MGMT_ERT_SELF_TEST_FAILURE = 0x2000001,
AIE2_STATUS_MGMT_ERT_HASH_MISMATCH,
AIE2_STATUS_MGMT_ERT_NOAVAIL,
AIE2_STATUS_MGMT_ERT_INVALID_PARAM,
AIE2_STATUS_MGMT_ERT_ENTER_SUSPEND_FAILURE,
AIE2_STATUS_MGMT_ERT_BUSY,
AIE2_STATUS_MGMT_ERT_APPLICATION_ACTIVE,
MAX_MGMT_ERT_STATUS_CODE,
/* APP ERT Error codes */
AIE2_STATUS_APP_ERT_FIRST_ERROR = 0x3000001,
AIE2_STATUS_APP_INVALID_INSTR,
AIE2_STATUS_APP_LOAD_PDI_FAIL,
MAX_APP_ERT_STATUS_CODE,
/* NPU RTOS Error Codes */
AIE2_STATUS_INVALID_INPUT_BUFFER = 0x4000001,
AIE2_STATUS_INVALID_COMMAND,
AIE2_STATUS_INVALID_PARAM,
AIE2_STATUS_INVALID_OPERATION = 0x4000006,
AIE2_STATUS_ASYNC_EVENT_MSGS_FULL,
AIE2_STATUS_MAX_RTOS_STATUS_CODE,
MAX_AIE2_STATUS_CODE
};
struct assign_mgmt_pasid_req {
__u16 pasid;
__u16 reserved;
} __packed;
struct assign_mgmt_pasid_resp {
enum aie2_msg_status status;
} __packed;
struct map_host_buffer_req {
__u32 context_id;
__u64 buf_addr;
__u64 buf_size;
} __packed;
struct map_host_buffer_resp {
enum aie2_msg_status status;
} __packed;
#define MAX_CQ_PAIRS 2
struct cq_info {
__u32 head_addr;
__u32 tail_addr;
__u32 buf_addr;
__u32 buf_size;
};
struct cq_pair {
struct cq_info x2i_q;
struct cq_info i2x_q;
};
struct create_ctx_req {
__u32 aie_type;
__u8 start_col;
__u8 num_col;
__u16 reserved;
__u8 num_cq_pairs_requested;
__u8 reserved1;
__u16 pasid;
__u32 pad[2];
__u32 sec_comm_target_type;
__u32 context_priority;
} __packed;
struct create_ctx_resp {
enum aie2_msg_status status;
__u32 context_id;
__u16 msix_id;
__u8 num_cq_pairs_allocated;
__u8 reserved;
struct cq_pair cq_pair[MAX_CQ_PAIRS];
} __packed;
struct destroy_ctx_req {
__u32 context_id;
} __packed;
struct destroy_ctx_resp {
enum aie2_msg_status status;
} __packed;
struct execute_buffer_req {
__u32 cu_idx;
__u32 payload[19];
} __packed;
struct exec_dpu_req {
__u64 inst_buf_addr;
__u32 inst_size;
__u32 inst_prop_cnt;
__u32 cu_idx;
__u32 payload[35];
} __packed;
struct execute_buffer_resp {
enum aie2_msg_status status;
} __packed;
struct aie_tile_info {
__u32 size;
__u16 major;
__u16 minor;
__u16 cols;
__u16 rows;
__u16 core_rows;
__u16 mem_rows;
__u16 shim_rows;
__u16 core_row_start;
__u16 mem_row_start;
__u16 shim_row_start;
__u16 core_dma_channels;
__u16 mem_dma_channels;
__u16 shim_dma_channels;
__u16 core_locks;
__u16 mem_locks;
__u16 shim_locks;
__u16 core_events;
__u16 mem_events;
__u16 shim_events;
__u16 reserved;
};
struct aie_tile_info_req {
__u32 reserved;
} __packed;
struct aie_tile_info_resp {
enum aie2_msg_status status;
struct aie_tile_info info;
} __packed;
struct aie_version_info_req {
__u32 reserved;
} __packed;
struct aie_version_info_resp {
enum aie2_msg_status status;
__u16 major;
__u16 minor;
} __packed;
struct aie_column_info_req {
__u64 dump_buff_addr;
__u32 dump_buff_size;
__u32 num_cols;
__u32 aie_bitmap;
} __packed;
struct aie_column_info_resp {
enum aie2_msg_status status;
__u32 size;
} __packed;
struct suspend_req {
__u32 place_holder;
} __packed;
struct suspend_resp {
enum aie2_msg_status status;
} __packed;
struct resume_req {
__u32 place_holder;
} __packed;
struct resume_resp {
enum aie2_msg_status status;
} __packed;
struct check_header_hash_req {
__u64 hash_high;
__u64 hash_low;
} __packed;
struct check_header_hash_resp {
enum aie2_msg_status status;
} __packed;
struct query_error_req {
__u64 buf_addr;
__u32 buf_size;
__u32 next_row;
__u32 next_column;
__u32 next_module;
} __packed;
struct query_error_resp {
enum aie2_msg_status status;
__u32 num_err;
__u32 has_next_err;
__u32 next_row;
__u32 next_column;
__u32 next_module;
} __packed;
struct protocol_version_req {
__u32 reserved;
} __packed;
struct protocol_version_resp {
enum aie2_msg_status status;
__u32 major;
__u32 minor;
} __packed;
struct firmware_version_req {
__u32 reserved;
} __packed;
struct firmware_version_resp {
enum aie2_msg_status status;
__u32 major;
__u32 minor;
__u32 sub;
__u32 build;
} __packed;
#define MAX_NUM_CUS 32
#define AIE2_MSG_CFG_CU_PDI_ADDR GENMASK(16, 0)
#define AIE2_MSG_CFG_CU_FUNC GENMASK(24, 17)
struct config_cu_req {
__u32 num_cus;
__u32 cfgs[MAX_NUM_CUS];
} __packed;
struct config_cu_resp {
enum aie2_msg_status status;
} __packed;
struct set_runtime_cfg_req {
__u32 type;
__u64 value;
} __packed;
struct set_runtime_cfg_resp {
enum aie2_msg_status status;
} __packed;
struct get_runtime_cfg_req {
__u32 type;
} __packed;
struct get_runtime_cfg_resp {
enum aie2_msg_status status;
__u64 value;
} __packed;
enum async_event_type {
ASYNC_EVENT_TYPE_AIE_ERROR,
ASYNC_EVENT_TYPE_EXCEPTION,
MAX_ASYNC_EVENT_TYPE
};
#define ASYNC_BUF_SIZE SZ_8K
struct async_event_msg_req {
__u64 buf_addr;
__u32 buf_size;
} __packed;
struct async_event_msg_resp {
enum aie2_msg_status status;
enum async_event_type type;
} __packed;
#define MAX_CHAIN_CMDBUF_SIZE SZ_4K
#define slot_cf_has_space(offset, payload_size) \
(MAX_CHAIN_CMDBUF_SIZE - ((offset) + (payload_size)) > \
offsetof(struct cmd_chain_slot_execbuf_cf, args[0]))
struct cmd_chain_slot_execbuf_cf {
__u32 cu_idx;
__u32 arg_cnt;
__u32 args[] __counted_by(arg_cnt);
};
#define slot_dpu_has_space(offset, payload_size) \
(MAX_CHAIN_CMDBUF_SIZE - ((offset) + (payload_size)) > \
offsetof(struct cmd_chain_slot_dpu, args[0]))
struct cmd_chain_slot_dpu {
__u64 inst_buf_addr;
__u32 inst_size;
__u32 inst_prop_cnt;
__u32 cu_idx;
__u32 arg_cnt;
#define MAX_DPU_ARGS_SIZE (34 * sizeof(__u32))
__u32 args[] __counted_by(arg_cnt);
};
struct cmd_chain_req {
__u64 buf_addr;
__u32 buf_size;
__u32 count;
} __packed;
struct cmd_chain_resp {
enum aie2_msg_status status;
__u32 fail_cmd_idx;
enum aie2_msg_status fail_cmd_status;
} __packed;
#define AIE2_MSG_SYNC_BO_SRC_TYPE GENMASK(3, 0)
#define AIE2_MSG_SYNC_BO_DST_TYPE GENMASK(7, 4)
struct sync_bo_req {
__u64 src_addr;
__u64 dst_addr;
__u32 size;
#define SYNC_BO_DEV_MEM 0
#define SYNC_BO_HOST_MEM 2
__u32 type;
} __packed;
struct sync_bo_resp {
enum aie2_msg_status status;
} __packed;
#endif /* _AIE2_MSG_PRIV_H_ */

256
drivers/accel/amdxdna/aie2_pci.c
View File

@@ -9,16 +9,210 @@
#include <linux/errno.h>
#include <linux/firmware.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/pci.h>
#include "aie2_msg_priv.h"
#include "aie2_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_pci_drv.h"
/*
* The management mailbox channel is allocated by firmware.
* The related register and ring buffer information is on SRAM BAR.
* This struct is the register layout.
*/
struct mgmt_mbox_chann_info {
u32 x2i_tail;
u32 x2i_head;
u32 x2i_buf;
u32 x2i_buf_sz;
u32 i2x_tail;
u32 i2x_head;
u32 i2x_buf;
u32 i2x_buf_sz;
};
static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev)
{
struct amdxdna_dev *xdna = ndev->xdna;
XDNA_DBG(xdna, "i2x tail 0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg);
XDNA_DBG(xdna, "i2x head 0x%x", ndev->mgmt_i2x.mb_head_ptr_reg);
XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr);
XDNA_DBG(xdna, "i2x rsize 0x%x", ndev->mgmt_i2x.rb_size);
XDNA_DBG(xdna, "x2i tail 0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg);
XDNA_DBG(xdna, "x2i head 0x%x", ndev->mgmt_x2i.mb_head_ptr_reg);
XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr);
XDNA_DBG(xdna, "x2i rsize 0x%x", ndev->mgmt_x2i.rb_size);
XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx);
}
static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
{
struct mgmt_mbox_chann_info info_regs;
struct xdna_mailbox_chann_res *i2x;
struct xdna_mailbox_chann_res *x2i;
u32 addr, off;
u32 *reg;
int ret;
int i;
/*
* Once firmware is alive, it will write management channel
* information in SRAM BAR and write the address of that information
* at FW_ALIVE_OFF offset in SRMA BAR.
*
* Read a non-zero value from FW_ALIVE_OFF implies that firmware
* is alive.
*/
ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF),
addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT);
if (ret || !addr)
return -ETIME;
off = AIE2_SRAM_OFF(ndev, addr);
reg = (u32 *)&info_regs;
for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++)
reg[i] = readl(ndev->sram_base + off + i * sizeof(u32));
i2x = &ndev->mgmt_i2x;
x2i = &ndev->mgmt_x2i;
i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head);
i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail);
i2x->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf);
i2x->rb_size = info_regs.i2x_buf_sz;
x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head);
x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail);
x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf);
x2i->rb_size = info_regs.x2i_buf_sz;
ndev->mgmt_chan_idx = CHANN_INDEX(ndev, x2i->rb_start_addr);
aie2_dump_chann_info_debug(ndev);
/* Must clear address at FW_ALIVE_OFF */
writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF));
return 0;
}
static int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev)
{
const struct rt_config *cfg = &ndev->priv->rt_config;
u64 value;
int ret;
ret = aie2_set_runtime_cfg(ndev, cfg->type, cfg->value);
if (ret) {
XDNA_ERR(ndev->xdna, "Set runtime type %d value %d failed",
cfg->type, cfg->value);
return ret;
}
ret = aie2_get_runtime_cfg(ndev, cfg->type, &value);
if (ret) {
XDNA_ERR(ndev->xdna, "Get runtime cfg failed");
return ret;
}
if (value != cfg->value)
return -EINVAL;
return 0;
}
static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev)
{
int ret;
ret = aie2_suspend_fw(ndev);
if (ret) {
XDNA_ERR(ndev->xdna, "Suspend firmware failed");
return ret;
}
ret = aie2_resume_fw(ndev);
if (ret) {
XDNA_ERR(ndev->xdna, "Resume firmware failed");
return ret;
}
return 0;
}
static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev)
{
int ret;
ret = aie2_check_protocol_version(ndev);
if (ret) {
XDNA_ERR(ndev->xdna, "Check header hash failed");
return ret;
}
ret = aie2_runtime_cfg(ndev);
if (ret) {
XDNA_ERR(ndev->xdna, "Runtime config failed");
return ret;
}
ret = aie2_assign_mgmt_pasid(ndev, 0);
if (ret) {
XDNA_ERR(ndev->xdna, "Can not assign PASID");
return ret;
}
ret = aie2_xdna_reset(ndev);
if (ret) {
XDNA_ERR(ndev->xdna, "Reset firmware failed");
return ret;
}
return 0;
}
static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
{
int ret;
ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver);
if (ret) {
XDNA_ERR(ndev->xdna, "query firmware version failed");
return ret;
}
ret = aie2_query_aie_version(ndev, &ndev->version);
if (ret) {
XDNA_ERR(ndev->xdna, "Query AIE version failed");
return ret;
}
ret = aie2_query_aie_metadata(ndev, &ndev->metadata);
if (ret) {
XDNA_ERR(ndev->xdna, "Query AIE metadata failed");
return ret;
}
return 0;
}
static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev)
{
if (aie2_suspend_fw(ndev))
XDNA_ERR(ndev->xdna, "Suspend_fw failed");
XDNA_DBG(ndev->xdna, "Firmware suspended");
}
static void aie2_hw_stop(struct amdxdna_dev *xdna)
{
struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
aie2_mgmt_fw_fini(ndev);
xdna_mailbox_stop_channel(ndev->mgmt_chann);
xdna_mailbox_destroy_channel(ndev->mgmt_chann);
aie2_psp_stop(ndev->psp_hdl);
aie2_smu_fini(ndev);
pci_disable_device(pdev);
@@ -28,7 +222,9 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
{
struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
int ret;
struct xdna_mailbox_res mbox_res;
u32 xdna_mailbox_intr_reg;
int mgmt_mb_irq, ret;
ret = pci_enable_device(pdev);
if (ret) {
@@ -49,8 +245,56 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
goto fini_smu;
}
ret = aie2_get_mgmt_chann_info(ndev);
if (ret) {
XDNA_ERR(xdna, "firmware is not alive");
goto stop_psp;
}
mbox_res.ringbuf_base = (u64)ndev->sram_base;
mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar);
mbox_res.mbox_base = (u64)ndev->mbox_base;
mbox_res.mbox_size = MBOX_SIZE(ndev);
mbox_res.name = "xdna_mailbox";
ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res);
if (!ndev->mbox) {
XDNA_ERR(xdna, "failed to create mailbox device");
ret = -ENODEV;
goto stop_psp;
}
mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx);
if (mgmt_mb_irq < 0) {
ret = mgmt_mb_irq;
XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret);
goto stop_psp;
}
xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4;
ndev->mgmt_chann = xdna_mailbox_create_channel(ndev->mbox,
&ndev->mgmt_x2i,
&ndev->mgmt_i2x,
xdna_mailbox_intr_reg,
mgmt_mb_irq);
if (!ndev->mgmt_chann) {
XDNA_ERR(xdna, "failed to create management mailbox channel");
ret = -EINVAL;
goto stop_psp;
}
ret = aie2_mgmt_fw_init(ndev);
if (ret) {
XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret);
goto destroy_mgmt_chann;
}
return 0;
destroy_mgmt_chann:
xdna_mailbox_stop_channel(ndev->mgmt_chann);
xdna_mailbox_destroy_channel(ndev->mgmt_chann);
stop_psp:
aie2_psp_stop(ndev->psp_hdl);
fini_smu:
aie2_smu_fini(ndev);
disable_dev:
@@ -109,6 +353,7 @@ static int aie2_init(struct amdxdna_dev *xdna)
ndev->sram_base = tbl[xdna->dev_info->sram_bar];
ndev->smu_base = tbl[xdna->dev_info->smu_bar];
ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
if (ret) {
@@ -153,9 +398,18 @@ static int aie2_init(struct amdxdna_dev *xdna)
goto disable_sva;
}
ret = aie2_mgmt_fw_query(ndev);
if (ret) {
XDNA_ERR(xdna, "Query firmware failed, ret %d", ret);
goto stop_hw;
}
ndev->total_col = ndev->metadata.cols;
release_firmware(fw);
return 0;
stop_hw:
aie2_hw_stop(xdna);
disable_sva:
iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
free_irq:

62
drivers/accel/amdxdna/aie2_pci.h
View File

@@ -6,6 +6,8 @@
#ifndef _AIE2_PCI_H_
#define _AIE2_PCI_H_
#include "amdxdna_mailbox.h"
#define AIE2_INTERVAL 20000 /* us */
#define AIE2_TIMEOUT 1000000 /* us */
@@ -33,6 +35,17 @@
((_ndev)->sram_base + SRAM_REG_OFF((_ndev), (idx))); \
})
#define CHAN_SLOT_SZ SZ_8K
#define CHANN_INDEX(ndev, rbuf_off) \
(((rbuf_off) - SRAM_REG_OFF((ndev), MBOX_CHANN_OFF)) / CHAN_SLOT_SZ)
#define MBOX_SIZE(ndev) \
({ \
typeof(ndev) _ndev = (ndev); \
((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \
pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->xdna->dev_info->mbox_bar); \
})
#define SMU_MPNPUCLK_FREQ_MAX(ndev) ((ndev)->priv->smu_mpnpuclk_freq_max)
#define SMU_HCLK_FREQ_MAX(ndev) ((ndev)->priv->smu_hclk_freq_max)
@@ -63,12 +76,37 @@ enum psp_reg_idx {
PSP_MAX_REGS /* Keep this at the end */
};
struct amdxdna_fw_ver;
struct psp_config {
const void *fw_buf;
u32 fw_size;
void __iomem *psp_regs[PSP_MAX_REGS];
};
struct aie_version {
u16 major;
u16 minor;
};
struct aie_tile_metadata {
u16 row_count;
u16 row_start;
u16 dma_channel_count;
u16 lock_count;
u16 event_reg_count;
};
struct aie_metadata {
u32 size;
u16 cols;
u16 rows;
struct aie_version version;
struct aie_tile_metadata core;
struct aie_tile_metadata mem;
struct aie_tile_metadata shim;
};
struct clock_entry {
char name[16];
u32 freq_mhz;
@@ -84,9 +122,22 @@ struct amdxdna_dev_hdl {
const struct amdxdna_dev_priv *priv;
void __iomem *sram_base;
void __iomem *smu_base;
void __iomem *mbox_base;
struct psp_device *psp_hdl;
struct xdna_mailbox_chann_res mgmt_x2i;
struct xdna_mailbox_chann_res mgmt_i2x;
u32 mgmt_chan_idx;
u32 total_col;
struct aie_version version;
struct aie_metadata metadata;
struct clock_entry mp_npu_clock;
struct clock_entry h_clock;
/* Mailbox and the management channel */
struct mailbox *mbox;
struct mailbox_channel *mgmt_chann;
};
#define DEFINE_BAR_OFFSET(reg_name, bar, reg_addr) \
@@ -127,4 +178,15 @@ struct psp_device *aie2m_psp_create(struct drm_device *ddev, struct psp_config *
int aie2_psp_start(struct psp_device *psp);
void aie2_psp_stop(struct psp_device *psp);
/* aie2_message.c */
int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev);
int aie2_resume_fw(struct amdxdna_dev_hdl *ndev);
int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value);
int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value);
int aie2_check_protocol_version(struct amdxdna_dev_hdl *ndev);
int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid);
int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version);
int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata);
int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
struct amdxdna_fw_ver *fw_ver);
#endif /* _AIE2_PCI_H_ */

2
drivers/accel/amdxdna/aie2_psp.c
View File

@@ -10,6 +10,8 @@
#include <linux/iopoll.h>
#include "aie2_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_pci_drv.h"
#define PSP_STATUS_READY BIT(31)

576
drivers/accel/amdxdna/amdxdna_mailbox.c Normal file
View File

File diff suppressed because it is too large Load Diff

124
drivers/accel/amdxdna/amdxdna_mailbox.h Normal file
View File

@@ -0,0 +1,124 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
*/
#ifndef _AIE2_MAILBOX_H_
#define _AIE2_MAILBOX_H_
struct mailbox;
struct mailbox_channel;
/*
* xdna_mailbox_msg - message struct
*
* @opcode: opcode for firmware
* @handle: handle used for the notify callback
* @notify_cb: callback function to notify the sender when there is response
* @send_data: pointing to sending data
* @send_size: size of the sending data
*
* The mailbox will split the sending data in to multiple firmware message if
* the size of the data is too big. This is transparent to the sender. The
* sender will receive one notification.
*/
struct xdna_mailbox_msg {
u32 opcode;
void *handle;
int (*notify_cb)(void *handle, const u32 *data, size_t size);
u8 *send_data;
size_t send_size;
};
/*
* xdna_mailbox_res - mailbox hardware resource
*
* @ringbuf_base: ring buffer base address
* @ringbuf_size: ring buffer size
* @mbox_base: mailbox base address
* @mbox_size: mailbox size
*/
struct xdna_mailbox_res {
u64 ringbuf_base;
size_t ringbuf_size;
u64 mbox_base;
size_t mbox_size;
const char *name;
};
/*
* xdna_mailbox_chann_res - resources
*
* @rb_start_addr: ring buffer start address
* @rb_size: ring buffer size
* @mb_head_ptr_reg: mailbox head pointer register
* @mb_tail_ptr_reg: mailbox tail pointer register
*/
struct xdna_mailbox_chann_res {
u32 rb_start_addr;
u32 rb_size;
u32 mb_head_ptr_reg;
u32 mb_tail_ptr_reg;
};
/*
* xdna_mailbox_create() -- create mailbox subsystem and initialize
*
* @ddev: device pointer
* @res: SRAM and mailbox resources
*
* Return: If success, return a handle of mailbox subsystem.
* Otherwise, return NULL pointer.
*/
struct mailbox *xdnam_mailbox_create(struct drm_device *ddev,
const struct xdna_mailbox_res *res);
/*
* xdna_mailbox_create_channel() -- Create a mailbox channel instance
*
* @mailbox: the handle return from xdna_mailbox_create()
* @x2i: host to firmware mailbox resources
* @i2x: firmware to host mailbox resources
* @xdna_mailbox_intr_reg: register addr of MSI-X interrupt
* @mb_irq: Linux IRQ number associated with mailbox MSI-X interrupt vector index
*
* Return: If success, return a handle of mailbox channel. Otherwise, return NULL.
*/
struct mailbox_channel *
xdna_mailbox_create_channel(struct mailbox *mailbox,
const struct xdna_mailbox_chann_res *x2i,
const struct xdna_mailbox_chann_res *i2x,
u32 xdna_mailbox_intr_reg,
int mb_irq);
/*
* xdna_mailbox_destroy_channel() -- destroy mailbox channel
*
* @mailbox_chann: the handle return from xdna_mailbox_create_channel()
*
* Return: if success, return 0. otherwise return error code
*/
int xdna_mailbox_destroy_channel(struct mailbox_channel *mailbox_chann);
/*
* xdna_mailbox_stop_channel() -- stop mailbox channel
*
* @mailbox_chann: the handle return from xdna_mailbox_create_channel()
*
* Return: if success, return 0. otherwise return error code
*/
void xdna_mailbox_stop_channel(struct mailbox_channel *mailbox_chann);
/*
* xdna_mailbox_send_msg() -- Send a message
*
* @mailbox_chann: Mailbox channel handle
* @msg: message struct for message information
* @tx_timeout: the timeout value for sending the message in ms.
*
* Return: If success return 0, otherwise, return error code
*/
int xdna_mailbox_send_msg(struct mailbox_channel *mailbox_chann,
const struct xdna_mailbox_msg *msg, u64 tx_timeout);
#endif /* _AIE2_MAILBOX_ */

56
drivers/accel/amdxdna/amdxdna_mailbox_helper.c Normal file
View File

@@ -0,0 +1,56 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2024, Advanced Micro Devices, Inc.
*/
#include <drm/drm_device.h>
#include <drm/drm_print.h>
#include <linux/completion.h>
#include "amdxdna_mailbox.h"
#include "amdxdna_mailbox_helper.h"
#include "amdxdna_pci_drv.h"
int xdna_msg_cb(void *handle, const u32 *data, size_t size)
{
struct xdna_notify *cb_arg = handle;
int ret;
if (unlikely(!data))
goto out;
if (unlikely(cb_arg->size != size)) {
cb_arg->error = -EINVAL;
goto out;
}
print_hex_dump_debug("resp data: ", DUMP_PREFIX_OFFSET,
16, 4, data, cb_arg->size, true);
memcpy(cb_arg->data, data, cb_arg->size);
out:
ret = cb_arg->error;
complete(&cb_arg->comp);
return ret;
}
int xdna_send_msg_wait(struct amdxdna_dev *xdna, struct mailbox_channel *chann,
struct xdna_mailbox_msg *msg)
{
struct xdna_notify *hdl = msg->handle;
int ret;
ret = xdna_mailbox_send_msg(chann, msg, TX_TIMEOUT);
if (ret) {
XDNA_ERR(xdna, "Send message failed, ret %d", ret);
return ret;
}
ret = wait_for_completion_timeout(&hdl->comp,
msecs_to_jiffies(RX_TIMEOUT));
if (!ret) {
XDNA_ERR(xdna, "Wait for completion timeout");
return -ETIME;
}
return hdl->error;
}

42
drivers/accel/amdxdna/amdxdna_mailbox_helper.h Normal file
View File

@@ -0,0 +1,42 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
*/
#ifndef _AMDXDNA_MAILBOX_HELPER_H
#define _AMDXDNA_MAILBOX_HELPER_H
#define TX_TIMEOUT 2000 /* milliseconds */
#define RX_TIMEOUT 5000 /* milliseconds */
struct amdxdna_dev;
struct xdna_notify {
struct completion comp;
u32 *data;
size_t size;
int error;
};
#define DECLARE_XDNA_MSG_COMMON(name, op, status) \
struct name##_req req = { 0 }; \
struct name##_resp resp = { status }; \
struct xdna_notify hdl = { \
.error = 0, \
.data = (u32 *)&resp, \
.size = sizeof(resp), \
.comp = COMPLETION_INITIALIZER_ONSTACK(hdl.comp), \
}; \
struct xdna_mailbox_msg msg = { \
.send_data = (u8 *)&req, \
.send_size = sizeof(req), \
.handle = &hdl, \
.opcode = op, \
.notify_cb = xdna_msg_cb, \
}
int xdna_msg_cb(void *handle, const u32 *data, size_t size);
int xdna_send_msg_wait(struct amdxdna_dev *xdna, struct mailbox_channel *chann,
struct xdna_mailbox_msg *msg);
#endif /* _AMDXDNA_MAILBOX_HELPER_H */

8
drivers/accel/amdxdna/amdxdna_pci_drv.h
View File

@@ -47,12 +47,20 @@ struct amdxdna_dev_info {
const struct amdxdna_dev_ops *ops;
};
struct amdxdna_fw_ver {
u32 major;
u32 minor;
u32 sub;
u32 build;
};
struct amdxdna_dev {
struct drm_device ddev;
struct amdxdna_dev_hdl *dev_handle;
const struct amdxdna_dev_info *dev_info;
struct mutex dev_lock; /* per device lock */
struct amdxdna_fw_ver fw_ver;
};
/*

11
drivers/accel/amdxdna/amdxdna_sysfs.c
View File

@@ -24,9 +24,20 @@ static ssize_t device_type_show(struct device *dev, struct device_attribute *att
}
static DEVICE_ATTR_RO(device_type);
static ssize_t fw_version_show(struct device *dev, struct device_attribute *attr, char *buf)
{
struct amdxdna_dev *xdna = dev_get_drvdata(dev);
return sprintf(buf, "%d.%d.%d.%d\n", xdna->fw_ver.major,
xdna->fw_ver.minor, xdna->fw_ver.sub,
xdna->fw_ver.build);
}
static DEVICE_ATTR_RO(fw_version);
static struct attribute *amdxdna_attrs[] = {
&dev_attr_device_type.attr,
&dev_attr_vbnv.attr,
&dev_attr_fw_version.attr,
NULL,
};

1
drivers/accel/amdxdna/npu1_regs.c
View File

@@ -8,6 +8,7 @@
#include <linux/sizes.h>
#include "aie2_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_pci_drv.h"
/* Address definition from NPU1 docs */

1
drivers/accel/amdxdna/npu2_regs.c
View File

@@ -8,6 +8,7 @@
#include <linux/sizes.h>
#include "aie2_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_pci_drv.h"
/* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */

1
drivers/accel/amdxdna/npu4_regs.c
View File

@@ -8,6 +8,7 @@
#include <linux/sizes.h>
#include "aie2_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_pci_drv.h"
/* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */

1
drivers/accel/amdxdna/npu5_regs.c
View File

@@ -8,6 +8,7 @@
#include <linux/sizes.h>
#include "aie2_pci.h"
#include "amdxdna_mailbox.h"
#include "amdxdna_pci_drv.h"
/* NPU Public Registers on MpNPUAxiXbar (refer to Diag npu_registers.h) */

60
include/trace/events/amdxdna.h Normal file
View File

@@ -0,0 +1,60 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
*/
#undef TRACE_SYSTEM
#define TRACE_SYSTEM amdxdna
#if !defined(_TRACE_AMDXDNA_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_AMDXDNA_H
#include <linux/tracepoint.h>
DECLARE_EVENT_CLASS(xdna_mbox_msg,
TP_PROTO(char *name, u8 chann_id, u32 opcode, u32 msg_id),
TP_ARGS(name, chann_id, opcode, msg_id),
TP_STRUCT__entry(__string(name, name)
__field(u32, chann_id)
__field(u32, opcode)
__field(u32, msg_id)),
TP_fast_assign(__assign_str(name);
__entry->chann_id = chann_id;
__entry->opcode = opcode;
__entry->msg_id = msg_id;),
TP_printk("%s.%d id 0x%x opcode 0x%x", __get_str(name),
__entry->chann_id, __entry->msg_id, __entry->opcode)
);
DEFINE_EVENT(xdna_mbox_msg, mbox_set_tail,
TP_PROTO(char *name, u8 chann_id, u32 opcode, u32 id),
TP_ARGS(name, chann_id, opcode, id)
);
DEFINE_EVENT(xdna_mbox_msg, mbox_set_head,
TP_PROTO(char *name, u8 chann_id, u32 opcode, u32 id),
TP_ARGS(name, chann_id, opcode, id)
);
TRACE_EVENT(mbox_irq_handle,
TP_PROTO(char *name, int irq),
TP_ARGS(name, irq),
TP_STRUCT__entry(__string(name, name)
__field(int, irq)),
TP_fast_assign(__assign_str(name);
__entry->irq = irq;),
TP_printk("%s.%d", __get_str(name), __entry->irq)
);
#endif /* !defined(_TRACE_AMDXDNA_H) || defined(TRACE_HEADER_MULTI_READ) */
/* This part must be outside protection */
#include <trace/define_trace.h>