From cd4a1dca2ffaa07ffcbce54abde88cf6913d64e4 Mon Sep 17 00:00:00 2001 From: David Wu Date: Wed, 31 Jan 2024 17:10:56 +0800 Subject: [PATCH] i3c: master: Add driver for Rockchip IP Add driver for Rockchip I3C master IP Change-Id: I73ca38117c0e0e603da23586c7b5c93f80917b2e Signed-off-by: David Wu --- .../bindings/i3c/rockchip,i3c-master.yaml | 52 + drivers/i3c/master/Kconfig | 8 + drivers/i3c/master/Makefile | 1 + drivers/i3c/master/i3c-rockchip-master.c | 2383 +++++++++++++++++ 4 files changed, 2444 insertions(+) create mode 100644 Documentation/devicetree/bindings/i3c/rockchip,i3c-master.yaml create mode 100644 drivers/i3c/master/i3c-rockchip-master.c diff --git a/Documentation/devicetree/bindings/i3c/rockchip,i3c-master.yaml b/Documentation/devicetree/bindings/i3c/rockchip,i3c-master.yaml new file mode 100644 index 000000000000..7aaeaf457563 --- /dev/null +++ b/Documentation/devicetree/bindings/i3c/rockchip,i3c-master.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i3c/rockchip,i3c-master.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip I3C master block + +allOf: + - $ref: i3c.yaml# + +properties: + compatible: + const: rockchip,i3c-master + + reg: + maxItems: 1 + + clocks: + maxItems: 2 + + clock-names: + minItems: 2 + items: + - const: i3c + - const: pclk + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - clock-names + - interrupts + +unevaluatedProperties: false + +examples: + - | + i3c0: i3c-master@2abe0000 { + compatible = "rockchip,rk3576-i3c-master"; + reg = <0x0 0x2abe0000 0x0 0x1000>; + interrupts = ; + #address-cells = <3>; + #size-cells = <0>; + clocks = <&cru HCLK_I3C>, <&cru i3C>; + clock-names = "hclk", "i3c"; + dmas = <&dmac0 22>, <&dmac0 23>; + dma-names = "rx", "tx"; + }; diff --git a/drivers/i3c/master/Kconfig b/drivers/i3c/master/Kconfig index 3b8f95916f46..273c9b7e9de8 100644 --- a/drivers/i3c/master/Kconfig +++ b/drivers/i3c/master/Kconfig @@ -22,6 +22,14 @@ config DW_I3C_MASTER This driver can also be built as a module. If so, the module will be called dw-i3c-master. +config ROCKCHIP_I3C_MASTER + tristate "Rockchip I3C master driver" + depends on I3C + depends on HAS_IOMEM + depends on ARCH_ROCKCHIP + help + Support for Rockchip I3C Controller. + config SVC_I3C_MASTER tristate "Silvaco I3C Dual-Role Master driver" depends on I3C diff --git a/drivers/i3c/master/Makefile b/drivers/i3c/master/Makefile index b3fee0f690b2..f783e9e53a7e 100644 --- a/drivers/i3c/master/Makefile +++ b/drivers/i3c/master/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CDNS_I3C_MASTER) += i3c-master-cdns.o obj-$(CONFIG_DW_I3C_MASTER) += dw-i3c-master.o +obj-$(CONFIG_ROCKCHIP_I3C_MASTER) += i3c-rockchip-master.o obj-$(CONFIG_SVC_I3C_MASTER) += svc-i3c-master.o obj-$(CONFIG_MIPI_I3C_HCI) += mipi-i3c-hci/ diff --git a/drivers/i3c/master/i3c-rockchip-master.c b/drivers/i3c/master/i3c-rockchip-master.c new file mode 100644 index 000000000000..6d1974341382 --- /dev/null +++ b/drivers/i3c/master/i3c-rockchip-master.c @@ -0,0 +1,2383 @@ +// SPDX-License-Identifier: (GPL-2.0-only) +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * Driver for I3C master in Rockchip SoC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define I3C_BIT(nr) (BIT(nr) | BIT(nr + 16)) +#define I3C_CLR_BIT(nr) (BIT(nr + 16)) + +#define I3C_HIWORD_UPDATE(val, mask, shift) \ + ((val) << (shift) | (mask) << ((shift) + 16)) + +#define MODULE_CTL 0x0000 +#define MODULE_CONF 0x0004 +#define TOUT_CONF_I3C 0x0008 +#define CLKSTALL_CONF_I3C 0x000C +#define SLVSCL_CONF_I2C 0x0010 +#define WAIT_THLD_I2C 0x0014 +#define CMD 0x0020 +#define TXDATA 0x0024 +#define RXDATA 0x0028 +#define IBIDATA 0x002C +#define FIFO_TRIG_DMA 0x0030 +#define TXFIFO_NUM_DMA 0x0034 +#define RXFIFO_NUM_DMA 0x0038 +#define IBIFIFO_NUM_DMA 0x003C +#define FIFO_TRIG_CPU 0x0040 +#define TXFIFO_NUM_CPU 0x0044 +#define RXFIFO_NUM_CPU 0x0048 +#define IBIFIFO_NUM_CPU 0x004C +#define DMA_CONF 0x0050 +#define CLKDIV_OD 0x0054 +#define CLKDIV_PP 0x0058 +#define CLKDIV_FM 0x005C +#define TIME_CONF_PPOD 0x0060 +#define TIME_CONF_FM 0x0064 +#define DAA_CONF 0x0068 +#define BUS_FREE_CONF 0x006C +#define BUS_FLT_CONF 0x0070 +#define REG_READY_CONF 0x0074 +#define FLUSH_CTL 0x0078 +#define INT_EN 0x0090 +#define INT_STATUS 0x0094 +#define IBI_CONF 0x009C +#define IBI_MAP1 0x0100 +#define IBI_MAP2 0x0104 +#define IBI_MAP3 0x0108 +#define IBI_MAP4 0x010C +#define IBI_MAP5 0x0110 +#define IBI_MAP6 0x0114 +#define DEV_ID1_RR0 0x0120 +#define DEV_ID1_RR1 0x0124 +#define DEV_ID1_RR2 0x0128 +#define VERSION 0x0200 +#define CMD1_STATUS 0x0230 +#define ASYNC_REF_CNT 0x0260 +#define FIFO_STATUS 0x0264 +#define IBI_STATUS 0x0268 +#define DAA_STATUS 0x026C +#define RXDAT_OVER 0x0270 +#define IBIRX_OVER 0x0274 + +#define MODULE_CTL_START_EN_SHIFT (0) +#define MODULE_CTL_START_EN (0x1 << MODULE_CTL_START_EN_SHIFT) + +#define MODULE_CTL_FM_START_EN_SHIFT (1) +#define MODULE_CTL_FM_START_EN (0x1 << MODULE_CTL_FM_START_EN_SHIFT) + +#define MODULE_CONF_MODULE_EN_SHIFT (0) +#define MODULE_CONF_MODULE_EN I3C_BIT(MODULE_CONF_MODULE_EN_SHIFT) +#define MODULE_CONF_MODULE_DIS I3C_CLR_BIT(MODULE_CONF_MODULE_EN_SHIFT) + +#define MODULE_CONF_BUS_TYPE_SHIFT (1) +#define MODULE_CONF_PURE_I3C (I3C_CLR_BIT(1) | I3C_CLR_BIT(2)) +#define MODULE_CONF_MIX_FAST_I3C (I3C_BIT(1) | I3C_CLR_BIT(2)) +#define MODULE_CONF_MIX_SLOW_I3C (I3C_CLR_BIT(1) | I3C_BIT(2)) +#define MODULE_CONF_PURE_I2C (I3C_BIT(1) | I3C_BIT(2)) + +#define MODULE_CONF_ODDAA_EN_SHIFT (4) +#define MODULE_CONF_ODDAA_EN I3C_BIT(MODULE_CONF_ODDAA_EN_SHIFT) + +#define MODULE_ADDRACK_PPOD_RST_SHIFT (6) +#define MODULE_ADDRACK_PPOD_RST (0x1 << MODULE_ADDRACK_PPOD_RST_SHIFT) + +#define CLKSTALL_SHIFT (0) +#define CLKSTALL_ENABLE (0x1 << CLKSTALL_SHIFT) +#define CLKSTALL_BEFORE_ACK_SHIFT (1) +#define CLKSTALL_BEFORE_ACK_ENABLE (0x1 << CLKSTALL_BEFORE_ACK_SHIFT) +#define CLKSTALL_BEFORE_READ_SHIFT (3) +#define CLKSTALL_BEFORE_READ_ENABLE (0x1 << CLKSTALL_BEFORE_READ_SHIFT) +#define CLKSTALL_NUM_SHIFT (8) + +#define CMD_RNW_SHIFT (0) +#define CMD_RNW (0x1 << CMD_RNW_SHIFT) + +#define CMD_ADDR_SHIFT (1) +#define CMD_ADDR_MASK (0x7F << CMD_ADDR_SHIFT) +#define CMD_DEV_ADDR(a) ((a) << CMD_ADDR_SHIFT) + +#define CMD_TRAN_MODE_SHIFT (8) +#define CMD_I3C_SDR_TX_MODE (0x0 << CMD_TRAN_MODE_SHIFT) +#define CMD_I3C_SDR_RX_MODE (0x1 << CMD_TRAN_MODE_SHIFT) +#define CMD_I3C_ENTDAA_MODE (0x2 << CMD_TRAN_MODE_SHIFT) +#define CMD_I3C_ADDR_ONLY_MODE (0x3 << CMD_TRAN_MODE_SHIFT) +#define CMD_I2C_TX (0x4 << CMD_TRAN_MODE_SHIFT) +#define CMD_I2C_RX (0x5 << CMD_TRAN_MODE_SHIFT) + +#define CMD_IS_SYNC_DT_TRAN_SHIFT (11) +#define CMD_IS_SYNC_DT_TRAN (0x1 << CMD_IS_SYNC_DT_TRAN_SHIFT) + +#define CMD_IS_HDREXIT_SHIFT (12) +#define CMD_HDREXIT_PATTERN (0x1 << CMD_IS_HDREXIT_SHIFT) + +#define CMD_IS_RESET_SHIFT (13) +#define CMD_TARGET_RESET_PATTERN (0x1 << CMD_IS_RESET_SHIFT) + +#define CMD_I2C_ACK2LAST_SHIFT (14) +#define CMD_I2C_ACK2LAST (0x1 << CMD_I2C_ACK2LAST_SHIFT) + +#define CMD_I2C_ACK2NAK_SHIFT (15) +#define CMD_I2C_ACK2NAK (0x1 << CMD_I2C_ACK2NAK_SHIFT) + +#define CMD_I3C_ACK2EARLY_SHIFT (16) +#define CMD_I3C_ACK2EARLY (0x1 << CMD_I3C_ACK2EARLY_SHIFT) + +#define CMD_I3C_ACK2NAK_SHIFT (17) +#define CMD_I3C_ACK2NAK_INNORE (0x1 << CMD_I3C_ACK2NAK_SHIFT) + +#define CMD_I3C_END2IBI_SHIFT (18) +#define CMD_I3C_END2IBI (0x1 << CMD_I3C_END2IBI_SHIFT) + +#define CMD_ACK2NEXT_SHIFT (19) +#define CMD_ACK2STOP (0x0 << CMD_ACK2NEXT_SHIFT) +#define CMD_ACK2NEXT (0x1 << CMD_ACK2NEXT_SHIFT) + +#define CMD_BYTE_LEN_SHIFT (20) +#define CMD_FIFO_PL_LEN(a) ((a) << CMD_BYTE_LEN_SHIFT) + +#define REG_READY_CONF_SHIFT (0) +#define REG_READY_CONF_EN I3C_BIT(REG_READY_CONF_SHIFT) +#define REG_READY_CONF_DIS I3C_CLR_BIT(REG_READY_CONF_SHIFT) + +#define REG_READY_CONF_IBI_READY_SHIFT (1) +#define IBI_REG_READY_CONF_EN I3C_BIT(REG_READY_CONF_IBI_READY_SHIFT) +#define IBI_REG_READY_CONF_DIS I3C_CLR_BIT(REG_READY_CONF_IBI_READY_SHIFT) + +#define FLUSH_CTL_CMDFIFO_FLUSH_SHIFT (0) +#define FLUSH_CTL_CMDFIFO (0x1 << FLUSH_CTL_CMDFIFO_FLUSH_SHIFT) + +#define FLUSH_CTL_TXFIFO_FLUSH_SHIFT (1) +#define FLUSH_CTL_TXFIFO (0x1 << FLUSH_CTL_TXFIFO_FLUSH_SHIFT) + +#define FLUSH_CTL_RXFIFO_FLUSH_SHIFT (2) +#define FLUSH_CTL_RXFIFO (0x1 << FLUSH_CTL_RXFIFO_FLUSH_SHIFT) + +#define FLUSH_CTL_IBIFIFO_FLUSH_SHIFT (3) +#define FLUSH_CTL_IBIFIFO (0x1 << FLUSH_CTL_IBIFIFO_FLUSH_SHIFT) + +#define FLUSH_CTL_CMDSTATUS_FLUSH_SHIFT (4) +#define FLUSH_CTL_CMDSTATUS (0x1 << FLUSH_CTL_CMDSTATUS_FLUSH_SHIFT) + +#define FLUSH_CTL_DEVUID_FLUSH_SHIFT (5) +#define FLUSH_CTL_DEVUID (0x1 << FLUSH_CTL_DEVUID_FLUSH_SHIFT) + +#define FLUSH_CTL_IBIMAP_FLUSH_SHIFT (6) +#define FLUSH_CTL_IBIMAP (0x1 << FLUSH_CTL_IBIMAP_FLUSH_SHIFT) + +#define FLUSH_CTL_FIFO_NUM_TXRX_SHIFT (8) +#define FLUSH_CTL_FIFO_NUM_TXRX (0x1 << FLUSH_CTL_FIFO_NUM_TXRX_SHIFT) + +#define FLUSH_CTL_FIFO_NUM_IBI_SHIFT (9) +#define FLUSH_CTL_FIFO_NUM_IBI (0x1 << FLUSH_CTL_FIFO_NUM_IBI_SHIFT) + +#define FLUSH_CTL_AHB_RTX_STATUS_SHIFT (10) +#define FLUSH_CTL_AHB_RTX_STATUS (0x1 << FLUSH_CTL_AHB_RTX_STATUS_SHIFT) + +#define FLUSH_CTRL_ALL (FLUSH_CTL_CMDFIFO | FLUSH_CTL_TXFIFO | \ + FLUSH_CTL_RXFIFO | FLUSH_CTL_CMDSTATUS | \ + FLUSH_CTL_FIFO_NUM_TXRX | FLUSH_CTL_AHB_RTX_STATUS) + +#define FLUSH_IBI_ALL (FLUSH_CTL_IBIFIFO | FLUSH_CTL_FIFO_NUM_IBI) + +#define INT_EN_END_IEN_SHIFT (0) +#define IRQ_END_IEN I3C_BIT(INT_EN_END_IEN_SHIFT) +#define IRQ_END_CLR I3C_CLR_BIT(INT_EN_END_IEN_SHIFT) +#define IRQ_END_STS (0x1 << INT_EN_END_IEN_SHIFT) + +#define INT_EN_SLVREQ_START_IEN_SHIFT (1) +#define IRQ_SLVREQ_START_IEN I3C_BIT(INT_EN_SLVREQ_START_IEN_SHIFT) +#define IRQ_SLVREQ_START_DIS I3C_CLR_BIT(INT_EN_SLVREQ_START_IEN_SHIFT) +#define SLVREQ_START_STS (0x1 << INT_EN_SLVREQ_START_IEN_SHIFT) + +#define INT_EN_OWNIBI_IEN_SHIFT (2) +#define OWNIBI_IRQPD_IEN I3C_BIT(INT_EN_OWNIBI_IEN_SHIFT) +#define OWNIBI_IRQPD_DIS I3C_CLR_BIT(INT_EN_OWNIBI_IEN_SHIFT) +#define OWNIBI_IRQPD_STS (0x1 << INT_EN_OWNIBI_IEN_SHIFT) + +#define INT_EN_DATAERR_IRQPD_SHIFT (4) +#define INT_DATAERR_IRQPD_STS (0x1 << INT_EN_DATAERR_IRQPD_SHIFT) + +#define INT_EN_I2CIBI_IRQPD_SHIFT (9) +#define I2CIBI_IRQPD_STS (0x1 << INT_EN_I2CIBI_IRQPD_SHIFT) + +#define INT_EN_CPU_FIFOTRIG_IEN_SHIFT (15) +#define IRQ_CPU_FIFOTRIG_IEN I3C_BIT(INT_EN_CPU_FIFOTRIG_IEN_SHIFT) +#define IRQ_CPU_FIFOTRIG_CLR I3C_CLR_BIT(INT_EN_CPU_FIFOTRIG_IEN_SHIFT) +#define CPU_FIFOTRIG_IRQ_STS (0x1 << INT_EN_CPU_FIFOTRIG_IEN_SHIFT) + +#define INT_STATUS_LASTCMD_ID_SHIFT (16) +#define FINISHED_CMD_ID_MASK (0xF << INT_STATUS_LASTCMD_ID_SHIFT) +#define FINISHED_CMD_ID(isr) (((isr) & FINISHED_CMD_ID_MASK) >> INT_STATUS_LASTCMD_ID_SHIFT) + +#define INT_STATUS_ERRCMD_ID_SHIFT (24) +#define ERR_CMD_ID_MASK (0xF << INT_STATUS_ERRCMD_ID_SHIFT) +#define ERR_CMD_ID(isr) (((isr) & ERR_CMD_ID_MASK) >> INT_STATUS_ERRCMD_ID_SHIFT) + +#define IBI_CONF_AUTOIBI_EN_SHIFT (0) +#define IBI_CONF_AUTOIBI_EN I3C_BIT(IBI_CONF_AUTOIBI_EN_SHIFT) +#define IBI_CONF_AUTOIBI_DIS I3C_CLR_BIT(IBI_CONF_AUTOIBI_EN_SHIFT) + +#define DMA_CONF_NORDMA_RX_EN_SHIFT (1) +#define DMA_CONF_NORDMA_RX_EN (0x1 << DMA_CONF_NORDMA_RX_EN_SHIFT) + +#define DMA_CONF_NORDMA_IBI_EN_SHIFT (2) +#define DMA_CONF_NORDMA_IBI_EN (0x1 << DMA_CONF_NORDMA_IBI_EN_SHIFT) + +#define DMA_CONF_NORDMA_TX_EN_SHIFT (3) +#define DMA_CONF_NORDMA_TX_EN (0x1 << DMA_CONF_NORDMA_TX_EN_SHIFT) + +#define DMA_CONF_RX_RESP_ERR_EN_SHIFT (4) +#define DMA_CONF_RX_RESP_ERROR_EN (0x1 << DMA_CONF_RX_RESP_ERR_EN_SHIFT) + +#define DMA_CONF_IBI_RESP_ERR_EN_SHIFT (5) +#define DMA_CONF_IBI_RESP_ERROR_EN (0x1 << DMA_CONF_IBI_RESP_ERR_EN_SHIFT) + +#define DMA_CONF_TX_RESP_ERR_EN_SHIFT (6) +#define DMA_CONF_TX_RESP_ERROR_EN (0x1 << DMA_CONF_TX_RESP_ERR_EN_SHIFT) + +#define DMA_CONF_RX_HSIZE_ERR_EN_SHIFT (8) +#define DMA_CONF_RX_HSIZE_ERR_IRQEN (0x1 << DMA_CONF_RX_HSIZE_ERR_EN_SHIFT) + +#define DMA_CONF_IBI_HSIZE_ERR_EN_SHIFT (9) +#define DMA_CONF_IBI_HSIZE_ERR_IRQEN (0x1 << DMA_CONF_IBI_HSIZE_ERR_EN_SHIFT) + +#define DMA_CONF_TX_HSIZE_ERR_EN_SHIFT (10) +#define DMA_CONF_TX_HSIZE_ERR_IRQEN (0x1 << DMA_CONF_TX_HSIZE_ERR_EN_SHIFT) + +#define DMA_CONF_RX_PAD_EN_SHIFT (12) +#define DMA_CONF_RX_PAD_EN (0x1 << DMA_CONF_RX_PAD_EN_SHIFT) + +#define DMA_CONF_ENABLED (DMA_CONF_NORDMA_RX_EN | DMA_CONF_NORDMA_TX_EN | \ + DMA_CONF_TX_HSIZE_ERR_IRQEN | DMA_CONF_RX_HSIZE_ERR_IRQEN | \ + DMA_CONF_RX_PAD_EN) + +#define DMA_TXFIFO_TRIG_SHIFT (0) +#define DMA_TXFIFO_TRIG_MASK (0xF << DMA_TXFIFO_TRIG_SHIFT) +#define DMA_RXFIFO_TRIG_SHIFT (4) +#define DMA_RXFIFO_TRIG_MASK (0xF << DMA_RXFIFO_TRIG_SHIFT) +#define DMA_TXFIFO_TRIG_MASK_SHIFT (16) +#define DMA_TXFIFO_TRIG_MASK_MASK (0xF << DMA_TXFIFO_TRIG_MASK_SHIFT) +#define DMA_RXFIFO_TRIG_MASK_SHIFT (20) +#define DMA_RXFIFO_TRIG_MASK_MASK (0xF << DMA_RXFIFO_TRIG_MASK_SHIFT) + +#define DMA_TXFIFO_TRIG(trig) ((((trig) - 1) << DMA_TXFIFO_TRIG_SHIFT) | DMA_TXFIFO_TRIG_MASK_MASK) +#define DMA_RXFIFO_TRIG(trig) ((((trig) - 1) << DMA_RXFIFO_TRIG_SHIFT) | DMA_RXFIFO_TRIG_MASK_MASK) + +#define CPU_TXFIFO_TRIG_CPU_SHIFT (0) +#define CPU_TXFIFO_TRIG_CPU_MASK (0xF << CPU_TXFIFO_TRIG_CPU_SHIFT) +#define CPU_RXFIFO_TRIG_CPU_SHIFT (4) +#define CPU_RXFIFO_TRIG_CPU_MASK (0xF << CPU_RXFIFO_TRIG_CPU_SHIFT) +#define CPU_TXFIFO_TRIG_MASK_SHIFT (16) +#define CPU_TXFIFO_TRIG_MASK_MASK (0xF << CPU_TXFIFO_TRIG_MASK_SHIFT) +#define CPU_RXFIFO_TRIG_MASK_SHIFT (20) +#define CPU_RXFIFO_TRIG_MASK_MASK (0xF << CPU_RXFIFO_TRIG_MASK_SHIFT) + +#define IRQ_TXFIFO_TRIG(x) ((((x) - 1) << CPU_TXFIFO_TRIG_CPU_SHIFT) | CPU_TXFIFO_TRIG_MASK_MASK) +#define IRQ_RXFIFO_TRIG(x) ((((x) - 1) << CPU_RXFIFO_TRIG_CPU_SHIFT) | CPU_RXFIFO_TRIG_MASK_MASK) + +#define TXFIFO_SPACE2FULL_SHIFT (8) +#define TXFIFO_SPACE2FULL_MASK (0x1F << TXFIFO_SPACE2FULL_SHIFT) +#define TXFIFO_SPACE2FULL_NUMWORD(s) (((s) & TXFIFO_SPACE2FULL_MASK) >> TXFIFO_SPACE2FULL_SHIFT) + +#define RXFIFO_SPACE2EMPTY_SHIFT (16) +#define RXFIFO_SPACE2EMPTY_MASK (0x1F << RXFIFO_SPACE2EMPTY_SHIFT) +#define RXFIFO_SPACE2EMPTY_NUMWORD(s) (((s) & RXFIFO_SPACE2EMPTY_MASK) >> RXFIFO_SPACE2EMPTY_SHIFT) + +#define DAA_STATUS_ASSIGNED_CNT_SHIFT (0) +#define DAA_ASSIGNED_CNT_MASK (0xF << DAA_STATUS_ASSIGNED_CNT_SHIFT) +#define DAANAK_CNT_SHIFT (4) +#define DAANAK_CNT_MASK (0x3 << DAA_STATUS_DAANAK_CNT_SHIFT) + +#define STATUS_CMD1_ADDRNAK_SHIFT (0) +#define STATUS_ADDRNAK (0x1 << STATUS_CMD1_ADDRNAK_SHIFT) + +#define STATUS_CMD1_RXEARLY_I3C_SHIFT (1) +#define STATUS_RXEARLY_I3C (0x1 << STATUS_CMD1_RXEARLY_I3C_SHIFT) + +#define STATUS_CMD1_TXEARLY_I2C_SHIFT (2) +#define STATUS_TXEARLY_I2C (0x1 << STATUS_CMD1_TXEARLY_I2C_SHIFT) + +#define STATUS_CMD1_FORCE_STOP_SHIFT (3) +#define STATUS_FORCE_STOP (0x1 << STATUS_CMD1_FORCE_STOP_SHIFT) + +#define STATUS_CMD1_DAANAK_SHIFT (4) +#define STATUS_DAANAK (0x1 << STATUS_CMD1_DAANAK_SHIFT) + +#define STATUS_CMD1_FINISH_SHIFT (6) +#define STATUS_FINISH (0x1 << STATUS_CMD1_FINISH_SHIFT) + +#define STATUS_CMD1_RNW_SHIFT (7) +#define STATUS_RNW (0x1 << STATUS_CMD1_RNW_SHIFT) + +#define STATUS_ERR GENMASK(STATUS_CMD1_DAANAK_SHIFT + 1, 0) + +#define I3C_BUS_THIGH_MAX_NS 41 +#define SCL_I3C_TIMING_CNT_MIN 12 +#define I3C_BUS_I2C_FM_TLOW_MIN_NS 1300 +#define I3C_BUS_I2C_FM_SCL_RATE 400000 +#define I3C_BUS_I2C_FMP_TLOW_MIN_NS 500 +#define I3C_BUS_I2C_FMP_SCL_RAISE_NS 120 + +#define SCL_I3C_CLKDIV_HCNT(x) ((x) & GENMASK(15, 0)) +#define SCL_I3C_CLKDIV_LCNT(x) (((x) << 16) & GENMASK(31, 16)) +#define SCL_I2C_CLKDIV_HCNT(x) ((x) & GENMASK(15, 0)) +#define SCL_I2C_CLKDIV_LCNT(x) (((x) << 16) & GENMASK(31, 16)) + +#define I2C_SLV_HOLD_SCL_CONF 0xfffffff0 + +#define IBI_TYPE_IBI 0 +#define IBI_TYPE_HJ 1 +#define IBI_TYPE_MR 2 +#define IBI_TYPE(x) ((x) & GENMASK(1, 0)) +#define IBI_NACK BIT(2) +#define IBI_RXEARLY BIT(3) +#define IBI_XFER_BYTES(x) (((x) & GENMASK(15, 8)) >> 8) +#define IBI_SLVID(x) (((x) & GENMASK(20, 16)) >> 16) +#define I2C_IBIADDR(x) (((x) & GENMASK(32, 24)) >> 24) + +#define IBI_MAP(d) (IBI_MAP1 + (((d) - 1) / 2) * 0x4) +#define IBI_MAP_DEV_ADDR(d, addr) (((addr) & GENMASK(6, 0)) << ((((d) - 1) % 2) * 0x10)) +#define IBI_MAP_DEV_PLLEN(d, len) (((len) & GENMASK(7, 0)) << ((((d) - 1) % 2) * 0x10 + 0x8)) + +#define INTR_ALL 0xffffffff +#define STATUS_LAST_CMDID (((x) & GENMASK(19, 16)) >> 16) + +#define ROCKCHIP_I3C_MAX_DEVS 12 +#define DEVS_CTRL_DEVS_ACTIVE_MASK GENMASK(11, 0) +#define DEV_ID_RR0(d) (DEV_ID1_RR0 + (d) * 0x10) +#define DEV_ID_RR0_GET_DEV_ADDR(x) (((x) >> 1) & GENMASK(6, 0)) + +#define CMD_FIFO_CCC(x) ((x) & GENMASK(6, 0)) +#define CMDR_XFER_BYTES(x) (((x) & GENMASK(19, 8)) >> 8) + +#define DATAFIFO_DEPTH 16 +#define CMDFIFO_DEPTH 11 +#define IBIFIFO_DEPTH 16 +#define CMD_FIFO_PL_LEN_MAX 4096 + +#define CMDR_NO_ERROR 0 +#define CMDR_DDR_PREAMBLE_ERROR 1 +#define CMDR_DDR_PARITY_ERROR 2 +#define CMDR_DDR_RX_FIFO_OVF 3 +#define CMDR_DDR_TX_FIFO_UNF 4 +#define CMDR_M0_ERROR 5 +#define CMDR_M1_ERROR 6 +#define CMDR_M2_ERROR 7 +#define CMDR_MST_ABORT 8 +#define CMDR_NACK_RESP 9 + +#define DMA_TRIG_LEVEL 0x8 +#define DMA_RX_MAX_BURST_SIZE 0x4 +#define DMA_RX_MAX_BURST_LEN 0x8 + +#define CLKSTALL_NUM (0x4 << CLKSTALL_NUM_SHIFT) + +#define WAIT_TIMEOUT 1000 /* ms */ +#define TX_IDLE_WAIT_TIMEOUT 10 /* ms */ + +enum rockchip_i3c_xfer_mode { + ROCKCHIP_I3C_POLL, + ROCKCHIP_I3C_DMA, + ROCKCHIP_I2C_IRQ +}; + +/* + * If it was direct ccc, daa, or with7e xfer, there are two fifo + * cmd at least, first fifo cmd's mode is brocast 7E; the next fifo + * cmd is xfer's cmd. + */ +enum rockchip_i3c_cmd_mode { + PRIVATE_TRANSFER_WITHOUT7E, + PRIVATE_TRANSFER_WITH7E, + BROCAST_CCC, + DIRECT_CCC, + DO_DAA +}; + +struct rockchip_i3c_cmd { + u32 fifo_cmd; + u8 target_addr; + u16 tx_len; + u8 *tx_buf; + u16 rx_len; + u8 *rx_buf; + u8 error; +}; + +struct rockchip_i3c_xfer { + struct list_head node; + struct completion comp; + int ret; + u8 mode; + u8 ccc_cmd; + unsigned int ncmds; + + /* total word */ + unsigned int tx_num_word; + unsigned int rx_num_word; + + /* dma */ + int tx_sg_len; + int rx_sg_len; + u8 rx_trig; + u8 tx_trig; + u8 rx_burst_size; + u8 rx_burst_len; + bool xferred; + + /* irq */ + unsigned int cur_tx_cmd; + unsigned int cur_tx_len; + + unsigned int cur_rx_cmd; + unsigned int cur_rx_len; + + enum rockchip_i3c_xfer_mode xfer_mode; + struct rockchip_i3c_cmd cmds[]; +}; + +struct rockchip_i3c_master_caps { + u8 cmdfifodepth; + u8 datafifodepth; + u8 ibififodepth; +}; + +struct rockchip_i3c_dat_entry { + u8 addr; + struct i3c_dev_desc *ibi_dev; +}; + +struct rockchip_i3c_i2c_dev_data { + u16 id; + s16 ibi; + struct i3c_generic_ibi_pool *ibi_pool; +}; + +struct rockchip_i3c_master { + struct i3c_master_controller base; + struct device *dev; + + void __iomem *regs; + struct clk *clk; + struct clk *hclk; + struct reset_control *reset; + struct reset_control *reset_ahb; + int irq; + + dma_addr_t dma_addr_rx; + dma_addr_t dma_addr_tx; + struct dma_chan *dma_rx; + struct dma_chan *dma_tx; + struct scatterlist tx_sg[CMDFIFO_DEPTH]; + struct scatterlist rx_sg[CMDFIFO_DEPTH]; + + struct { + struct list_head list; + struct rockchip_i3c_xfer *cur; + spinlock_t lock; + } xferqueue; + + struct { + unsigned int num_slots; + struct i3c_dev_desc **slots; + spinlock_t lock; + } ibi; + struct work_struct hj_work; + struct rockchip_i3c_master_caps caps; + + u32 free_pos; + bool daa_done; + unsigned int maxdevs; + struct rockchip_i3c_dat_entry devs[ROCKCHIP_I3C_MAX_DEVS]; + + struct kmem_cache *aligned_cache; + bool suspended; +}; + +static void rockchip_i3c_master_start_xfer_locked(struct rockchip_i3c_master *master); +static void rockchip_i3c_master_end_xfer_locked(struct rockchip_i3c_master *master, + u32 isr); + +static inline struct rockchip_i3c_master * +to_rockchip_i3c_master(struct i3c_master_controller *master) +{ + return container_of(master, struct rockchip_i3c_master, base); +} + +static unsigned int +rockchip_i3c_master_wr_to_tx_fifo(struct rockchip_i3c_master *master, + const u8 *bytes, int nbytes) +{ + writesl(master->regs + TXDATA, bytes, nbytes / 4); + if (nbytes & 3) { + u32 tmp = 0; + + memcpy(&tmp, bytes + (nbytes & ~3), nbytes & 3); + writesl(master->regs + TXDATA, &tmp, 1); + } + + return DIV_ROUND_UP(nbytes, 4); +} + +static void +rockchip_i3c_master_wr_ccc_to_tx_fifo(struct rockchip_i3c_master *master, + const u8 *bytes, int nbytes, u8 cmd) +{ + u32 val = 0; + int i, n; + + val = cmd; + if (nbytes <= 3) { + for (i = 0; i < nbytes; i++) + val |= bytes[i] << (8 * (i + 1)); + writel_relaxed(val, master->regs + TXDATA); + } else { + n = nbytes - 3; + writesl(master->regs + TXDATA, bytes + 3, n / 4); + if ((nbytes - 3) & 3) { + u32 tmp = 0; + + memcpy(&tmp, bytes + (n & ~3), n & 3); + writesl(master->regs + TXDATA, &tmp, 1); + } + } +} + +static unsigned int +rockchip_i3c_master_rd_from_rx_fifo(struct rockchip_i3c_master *master, + u8 *bytes, int nbytes) +{ + readsl(master->regs + RXDATA, bytes, nbytes / 4); + if (nbytes & 3) { + u32 tmp; + + readsl(master->regs + RXDATA, &tmp, 1); + memcpy(bytes + (nbytes & ~3), &tmp, nbytes & 3); + } + + return DIV_ROUND_UP(nbytes, 4); +} + +static bool rockchip_i3c_master_supports_ccc_cmd(struct i3c_master_controller *m, + const struct i3c_ccc_cmd *cmd) +{ + if (cmd->ndests > 1) + return false; + + switch (cmd->id) { + case I3C_CCC_ENEC(true): + case I3C_CCC_ENEC(false): + case I3C_CCC_DISEC(true): + case I3C_CCC_DISEC(false): + case I3C_CCC_ENTAS(0, true): + case I3C_CCC_ENTAS(0, false): + case I3C_CCC_RSTDAA(true): + case I3C_CCC_RSTDAA(false): + case I3C_CCC_ENTDAA: + case I3C_CCC_SETMWL(true): + case I3C_CCC_SETMWL(false): + case I3C_CCC_SETMRL(true): + case I3C_CCC_SETMRL(false): + case I3C_CCC_ENTHDR(0): + case I3C_CCC_SETDASA: + case I3C_CCC_SETNEWDA: + case I3C_CCC_GETMWL: + case I3C_CCC_GETMRL: + case I3C_CCC_GETPID: + case I3C_CCC_GETBCR: + case I3C_CCC_GETDCR: + case I3C_CCC_GETSTATUS: + case I3C_CCC_GETMXDS: + case I3C_CCC_GETHDRCAP: + return true; + default: + return false; + } +} + +static void rockchip_i3c_master_disable(struct rockchip_i3c_master *master) +{ + writel_relaxed(MODULE_CONF_MODULE_DIS, master->regs + MODULE_CONF); +} + +static void rockchip_i3c_master_enable(struct rockchip_i3c_master *master) +{ + writel_relaxed(MODULE_CONF_MODULE_EN | MODULE_CONF_ODDAA_EN | + MODULE_ADDRACK_PPOD_RST, + master->regs + MODULE_CONF); +} + +static void rockchip_i3c_master_reset_controller(struct rockchip_i3c_master *i3c) +{ + if (!IS_ERR_OR_NULL(i3c->reset)) { + reset_control_assert(i3c->reset); + udelay(10); + reset_control_deassert(i3c->reset); + } + + if (!IS_ERR_OR_NULL(i3c->reset_ahb)) { + reset_control_assert(i3c->reset_ahb); + udelay(10); + reset_control_deassert(i3c->reset_ahb); + } +} + +static struct rockchip_i3c_xfer * +rockchip_i3c_master_alloc_xfer(struct rockchip_i3c_master *master, unsigned int ncmds) +{ + struct rockchip_i3c_xfer *xfer; + + xfer = kzalloc(struct_size(xfer, cmds, ncmds), GFP_KERNEL); + if (!xfer) + return NULL; + + INIT_LIST_HEAD(&xfer->node); + xfer->ncmds = ncmds; + xfer->ret = -ETIMEDOUT; + + return xfer; +} + +static void rockchip_i3c_master_free_xfer(struct rockchip_i3c_xfer *xfer) +{ + kfree(xfer); +} + +static void rockchip_i3c_master_queue_xfer(struct rockchip_i3c_master *master, + struct rockchip_i3c_xfer *xfer) +{ + unsigned long flags; + + init_completion(&xfer->comp); + spin_lock_irqsave(&master->xferqueue.lock, flags); + if (master->xferqueue.cur) { + list_add_tail(&xfer->node, &master->xferqueue.list); + } else { + master->xferqueue.cur = xfer; + rockchip_i3c_master_start_xfer_locked(master); + } + spin_unlock_irqrestore(&master->xferqueue.lock, flags); +} + +static void rockchip_i3c_master_unqueue_xfer(struct rockchip_i3c_master *master, + struct rockchip_i3c_xfer *xfer) +{ + unsigned long flags; + + spin_lock_irqsave(&master->xferqueue.lock, flags); + if (master->xferqueue.cur == xfer) { + writel_relaxed(REG_READY_CONF_DIS, master->regs + REG_READY_CONF); + master->xferqueue.cur = NULL; + writel_relaxed(IRQ_END_CLR, master->regs + INT_EN); + } else { + list_del_init(&xfer->node); + } + spin_unlock_irqrestore(&master->xferqueue.lock, flags); +} + +static void rockchip_i3c_master_put_dma_safe_buf(struct rockchip_i3c_master *master, + u8 *buf, u8 *target, unsigned int len, + bool xferred, bool rx) +{ + if (!buf || buf == target) + return; + + if (xferred && rx) + memcpy(target, buf, len); + + kmem_cache_free(master->aligned_cache, buf); +} + +static u8 *rockchip_i3c_master_get_dma_safe_buf(struct rockchip_i3c_master *master, + u8 *buf, unsigned int len, bool rx) +{ + u8 *mem; + + if (len == 0) + return NULL; + + mem = kmem_cache_zalloc(master->aligned_cache, GFP_KERNEL); + if (!mem) + return NULL; + + if (!rx) + memcpy(mem, buf, len); + + return mem; +} + +static void rockchip_i3c_maste_cleanup_dma(struct rockchip_i3c_master *master) +{ + struct rockchip_i3c_xfer *xfer = master->xferqueue.cur; + int i; + + if (xfer->rx_sg_len) + dmaengine_terminate_all(master->dma_rx); + + if (xfer->tx_sg_len) + dmaengine_terminate_all(master->dma_tx); + + for (i = 0; i < xfer->rx_sg_len; i++) + dma_unmap_single(master->dma_rx->device->dev, + sg_dma_address(&master->rx_sg[i]), + sg_dma_len(&master->rx_sg[i]), DMA_FROM_DEVICE); + + for (i = 0; i < xfer->tx_sg_len; i++) + dma_unmap_single(master->dma_tx->device->dev, + sg_dma_address(&master->tx_sg[i]), + sg_dma_len(&master->tx_sg[i]), DMA_TO_DEVICE); + + /* DMA conf */ + writel_relaxed(0x0, master->regs + DMA_CONF); +} + +static inline u32 +rockchip_i3c_master_wait_for_tx_idle(struct rockchip_i3c_master *master) +{ + u32 status; + + if (!readl_relaxed_poll_timeout(master->regs + INT_STATUS, status, + status & IRQ_END_STS, 100, + TX_IDLE_WAIT_TIMEOUT * 1000)) + return status; + + dev_warn(master->dev, "i3c controller is in busy state!\n"); + return status; +} + +static void rockchip_i3c_master_dma_irq_callback(void *data) +{ + struct rockchip_i3c_master *master = data; + struct rockchip_i3c_xfer *xfer; + + rockchip_i3c_maste_cleanup_dma(master); + xfer = master->xferqueue.cur; + xfer->xferred = true; + complete(&xfer->comp); +} + +static inline u8 rockchip_i3c_calc_rx_burst(u32 data_len, bool size_calced) +{ + u32 i, level; + + /* If burst size calculated, rx burst size is byte, half_word or word. + * tx burst size is fixed word, and for burst len calculated, it is + * guaranteed to be no larger than 1/2 RX FIFO size. + */ + if (size_calced) + level = 4; + else + level = 8; + + /* burst size: 1, 2, 4, or 8 */ + for (i = 1; i < level; i <<= 1) { + if (data_len & i) + break; + } + + return i; +} + +static inline u8 rockchip_i3c_master_calc_trig_level(u32 data_len, bool rx) +{ + /* for rx, if data len cannot divisible by 4, best trig is 1 word, + * and ensure (burst_size * burst_len) <= (trig * 4). + */ + if ((data_len % 4) && rx) + return 1; + + if (data_len % 4) + return data_len / 4 + 1; + else + return data_len / 4; +} + +static int rockchip_i3c_master_prepate_dma_sg(struct rockchip_i3c_master *master, + struct rockchip_i3c_xfer *xfer) +{ + struct dma_async_tx_descriptor *rxdesc = NULL, *txdesc = NULL; + unsigned int rx_num_word = 0, tx_num_word = 0, i; + struct rockchip_i3c_cmd *cmd; + dma_cookie_t cookie; + dma_addr_t dma_addr; + + /* DMA conf */ + writel_relaxed(DMA_CONF_ENABLED, master->regs + DMA_CONF); + + xfer->tx_sg_len = xfer->rx_sg_len = 0; + for (i = 0; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + if (cmd->fifo_cmd & CMD_RNW) { + u8 rx_burst = xfer->rx_burst_size * xfer->rx_burst_len; + + rx_num_word += DIV_ROUND_UP(cmd->rx_len, 4); + dma_addr = dma_map_single(master->dma_rx->device->dev, + cmd->rx_buf, cmd->rx_len, + DMA_FROM_DEVICE); + if (dma_mapping_error(master->dma_rx->device->dev, dma_addr)) { + dev_err(master->dev, "dma rx map failed\n"); + return -EINVAL; + } + + sg_dma_len(&master->rx_sg[xfer->rx_sg_len]) = + DIV_ROUND_UP(cmd->rx_len, rx_burst) * rx_burst; + sg_dma_address(&master->rx_sg[xfer->rx_sg_len]) = dma_addr; + xfer->rx_sg_len++; + } else { + tx_num_word += DIV_ROUND_UP(cmd->tx_len, 4); + dma_addr = dma_map_single(master->dma_tx->device->dev, + cmd->tx_buf, cmd->tx_len, DMA_TO_DEVICE); + if (dma_mapping_error(master->dma_tx->device->dev, dma_addr)) { + dev_err(master->dev, "dma tx map failed\n"); + return -EINVAL; + } + + sg_dma_len(&master->tx_sg[xfer->tx_sg_len]) = + DIV_ROUND_UP(cmd->tx_len, xfer->tx_trig * 4) * 4 * xfer->tx_trig; + sg_dma_address(&master->tx_sg[xfer->tx_sg_len]) = dma_addr; + xfer->tx_sg_len++; + } + writel_relaxed(cmd->fifo_cmd, master->regs + CMD); + } + + if (xfer->rx_sg_len > 0) { + struct dma_slave_config rxconf = { + .direction = DMA_DEV_TO_MEM, + .src_addr = master->dma_addr_rx, + .src_addr_width = xfer->rx_burst_size, + .src_maxburst = xfer->rx_burst_len, + }; + + dmaengine_slave_config(master->dma_rx, &rxconf); + writel_relaxed(DMA_RXFIFO_TRIG(xfer->rx_trig), + master->regs + FIFO_TRIG_DMA); + writel_relaxed(rx_num_word, master->regs + RXFIFO_NUM_DMA); + + rxdesc = dmaengine_prep_slave_sg(master->dma_rx, master->rx_sg, + xfer->rx_sg_len, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT); + if (!rxdesc) { + if (txdesc) + dmaengine_terminate_sync(master->dma_tx); + return -EINVAL; + } + + /* last cmd gets the callback */ + if (xfer->cmds[xfer->ncmds - 1].fifo_cmd & CMD_RNW) + rxdesc->callback = rockchip_i3c_master_dma_irq_callback; + else + rxdesc->callback = NULL; + rxdesc->callback_param = master; + + cookie = dmaengine_submit(rxdesc); + if (dma_submit_error(cookie)) { + dev_err(master->dev, "submitting dma rx failed\n"); + rockchip_i3c_maste_cleanup_dma(master); + return -EINVAL; + } + dma_async_issue_pending(master->dma_rx); + } + + if (xfer->tx_sg_len > 0) { + struct dma_slave_config txconf = { + .direction = DMA_MEM_TO_DEV, + .dst_addr = master->dma_addr_tx, + .dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES, + .dst_maxburst = xfer->tx_trig, + }; + + dmaengine_slave_config(master->dma_tx, &txconf); + /* should match the DMA burst size */ + writel_relaxed(DMA_TXFIFO_TRIG(xfer->tx_trig), + master->regs + FIFO_TRIG_DMA); + writel_relaxed(tx_num_word, master->regs + TXFIFO_NUM_DMA); + + txdesc = dmaengine_prep_slave_sg(master->dma_tx, master->tx_sg, + xfer->tx_sg_len, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT); + if (!txdesc) { + if (rxdesc) + dmaengine_terminate_sync(master->dma_rx); + return -EINVAL; + } + + /* last cmd gets the callback */ + if (!(xfer->cmds[xfer->ncmds - 1].fifo_cmd & CMD_RNW)) + txdesc->callback = rockchip_i3c_master_dma_irq_callback; + else + txdesc->callback = NULL; + txdesc->callback_param = master; + + cookie = dmaengine_submit(txdesc); + if (dma_submit_error(cookie)) { + dev_err(master->dev, "submitting dma tx failed\n"); + rockchip_i3c_maste_cleanup_dma(master); + return -EINVAL; + } + dma_async_issue_pending(master->dma_tx); + } + + return 0; +} + +static void rockchip_i3c_master_data_isr_write(struct rockchip_i3c_master *master, + struct rockchip_i3c_xfer *xfer, + unsigned int tx_num_word) +{ + unsigned int tx_space = tx_num_word; + struct rockchip_i3c_cmd *cmd; + int i; + + for (i = xfer->cur_tx_cmd; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + if (!(cmd->fifo_cmd & CMD_RNW)) { + u16 min_tx_len, word; + + /* tx fifo is full or tx data was written to fifo */ + if (tx_space <= 0 || xfer->tx_num_word <= 0) + break; + min_tx_len = min(4 * tx_space, cmd->tx_len - xfer->cur_tx_len); + word = rockchip_i3c_master_wr_to_tx_fifo(master, + (u8 *)&cmd->tx_buf[xfer->cur_tx_len], min_tx_len); + + tx_space -= word; + xfer->tx_num_word = xfer->tx_num_word - word; + xfer->cur_tx_len += min_tx_len; + if (xfer->cur_tx_len >= cmd->tx_len) { + xfer->cur_tx_cmd = i + 1; + xfer->cur_tx_len = 0; + } + } + } +} + +static int rockchip_i3c_master_data_isr_read(struct rockchip_i3c_master *master, + struct rockchip_i3c_xfer *xfer, + unsigned int rx_num_word) +{ + unsigned int rx_left = rx_num_word; + struct rockchip_i3c_cmd *cmd; + int i; + + for (i = xfer->cur_rx_cmd; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + if (cmd->fifo_cmd & CMD_RNW) { + u16 min_rx_len, word; + + /* rx fifo is empty or rx data was read to memory */ + if (rx_left <= 0 || xfer->rx_num_word <= 0) + break; + min_rx_len = min(4 * rx_left, cmd->rx_len - xfer->cur_rx_len); + word = rockchip_i3c_master_rd_from_rx_fifo(master, + (u8 *)&cmd->rx_buf[xfer->cur_rx_len], min_rx_len); + rx_left -= word; + xfer->rx_num_word = xfer->rx_num_word - word; + xfer->cur_rx_len += min_rx_len; + if (xfer->cur_rx_len >= cmd->rx_len) { + xfer->cur_rx_cmd = i + 1; + xfer->cur_rx_len = 0; + } + } + } + + return 0; +} + +static int rockchip_i3c_master_data_isr(struct rockchip_i3c_master *master, u32 isr) +{ + unsigned int fifo_status = readl_relaxed(master->regs + FIFO_STATUS); + unsigned int rx_num_word = RXFIFO_SPACE2EMPTY_NUMWORD(fifo_status); + unsigned int tx_num_word = TXFIFO_SPACE2FULL_NUMWORD(fifo_status); + + struct rockchip_i3c_xfer *xfer = master->xferqueue.cur; + + if (xfer->rx_num_word > 0 && rx_num_word > 0) + rockchip_i3c_master_data_isr_read(master, xfer, rx_num_word); + + if (xfer->tx_num_word > 0 && tx_num_word > 0) + rockchip_i3c_master_data_isr_write(master, xfer, tx_num_word); + + return 0; +} + +static int rockchip_i3c_master_prepare_irq(struct rockchip_i3c_master *master, + struct rockchip_i3c_xfer *xfer) +{ + /* At first, init global values with 0 */ + xfer->cur_tx_cmd = xfer->cur_rx_cmd = 0; + xfer->cur_tx_len = xfer->cur_rx_len = 0; + + writel_relaxed(I2C_SLV_HOLD_SCL_CONF, master->regs + SLVSCL_CONF_I2C); + /* tx fifo trig */ + writel_relaxed(IRQ_TXFIFO_TRIG(DATAFIFO_DEPTH), master->regs + FIFO_TRIG_CPU); + /* rx fifo trig */ + writel_relaxed(IRQ_RXFIFO_TRIG(DATAFIFO_DEPTH), master->regs + FIFO_TRIG_CPU); + + /* clean DMA conf */ + writel_relaxed(0x0, master->regs + DMA_CONF); + + writel_relaxed(xfer->tx_num_word, master->regs + TXFIFO_NUM_CPU); + writel_relaxed(xfer->rx_num_word, master->regs + RXFIFO_NUM_CPU); + + if (xfer->tx_num_word > 0) + rockchip_i3c_master_data_isr_write(master, xfer, DATAFIFO_DEPTH); + + return 0; +} + +static void rockchip_i3c_master_start_xfer_locked(struct rockchip_i3c_master *master) +{ + struct rockchip_i3c_xfer *xfer = master->xferqueue.cur; + struct rockchip_i3c_cmd *cmd; + u32 fifo_cmd; + int i; + + if (!xfer) + return; + + writel_relaxed(FLUSH_CTRL_ALL, master->regs + FLUSH_CTL); + writel_relaxed(INTR_ALL, master->regs + INT_STATUS); + + switch (xfer->mode) { + case BROCAST_CCC: + /* only one cmd, rnw = 0, length = cmd->tx_len + 1. */ + for (i = 0; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + writel_relaxed(cmd->fifo_cmd, master->regs + CMD); + rockchip_i3c_master_wr_ccc_to_tx_fifo(master, + cmd->tx_buf, cmd->tx_len, xfer->ccc_cmd); + } + break; + case DIRECT_CCC: + /* more than one cmd, first cmd's tx fifo write ccc cmd to tx fifo, + * write tx data to txfifo if rnw = 0. + */ + writel_relaxed(xfer->ccc_cmd, master->regs + TXDATA); + fifo_cmd = CMD_DEV_ADDR(0x7e) | CMD_FIFO_PL_LEN(1) | CMD_ACK2NEXT; + writel_relaxed(fifo_cmd, master->regs + CMD); + for (i = 0; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + rockchip_i3c_master_wr_to_tx_fifo(master, cmd->tx_buf, + cmd->tx_len); + writel_relaxed(cmd->fifo_cmd, master->regs + CMD); + } + break; + case DO_DAA: + /* Send ENTDAA cmd first, then send is_entdaa 7E/R cmd */ + writel_relaxed(xfer->ccc_cmd, master->regs + TXDATA); + fifo_cmd = CMD_DEV_ADDR(0x7e) | CMD_FIFO_PL_LEN(1) | CMD_ACK2NEXT; + writel_relaxed(fifo_cmd, master->regs + CMD); + for (i = 0; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + cmd->fifo_cmd |= CMD_DEV_ADDR(0x7e) | CMD_I3C_ENTDAA_MODE | + CMD_RNW | CMD_ACK2NEXT; + if (i == xfer->ncmds - 1) + cmd->fifo_cmd &= ~CMD_ACK2NEXT; + + writel_relaxed(cmd->fifo_cmd, master->regs + CMD); + } + break; + case PRIVATE_TRANSFER_WITH7E: + /* more than one cmd, first cmd, write 0x7e to cmd */ + fifo_cmd = CMD_DEV_ADDR(0x7e) | CMD_I3C_ADDR_ONLY_MODE | CMD_ACK2NEXT; + writel_relaxed(fifo_cmd, master->regs + CMD); + fallthrough; + case PRIVATE_TRANSFER_WITHOUT7E: + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + rockchip_i3c_master_prepate_dma_sg(master, xfer); + } else if (xfer->xfer_mode == ROCKCHIP_I2C_IRQ) { + rockchip_i3c_master_prepare_irq(master, xfer); + for (i = 0; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + writel_relaxed(cmd->fifo_cmd, master->regs + CMD); + } + } else { + for (i = 0; i < xfer->ncmds; i++) { + cmd = &xfer->cmds[i]; + rockchip_i3c_master_wr_to_tx_fifo(master, cmd->tx_buf, + cmd->tx_len); + writel_relaxed(cmd->fifo_cmd, master->regs + CMD); + } + } + + break; + default: + break; + } + + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA) + writel_relaxed(IRQ_END_CLR | IRQ_CPU_FIFOTRIG_CLR, master->regs + INT_EN); + else if (xfer->xfer_mode == ROCKCHIP_I2C_IRQ) + writel_relaxed(IRQ_END_IEN | IRQ_CPU_FIFOTRIG_IEN, master->regs + INT_EN); + else + writel_relaxed(IRQ_END_IEN | IRQ_CPU_FIFOTRIG_CLR, master->regs + INT_EN); + + /* Use FM speed before daa done */ + if (!master->daa_done) + writel_relaxed(MODULE_CTL_FM_START_EN, master->regs + MODULE_CTL); + else + writel_relaxed(MODULE_CTL_START_EN, master->regs + MODULE_CTL); + + writel_relaxed(REG_READY_CONF_EN, master->regs + REG_READY_CONF); +} + +static void rockchip_i3c_master_end_xfer_locked(struct rockchip_i3c_master *master, + u32 isr) +{ + struct rockchip_i3c_xfer *xfer = master->xferqueue.cur; + u32 finish = 0, cmd_status_offset = 0; + int i, ret = 0; + + if (!xfer) + return; + + if (!(isr & IRQ_END_STS)) { + dev_warn_ratelimited(master->dev, "xfer finish not end.\n"); + return; + } + + finish = FINISHED_CMD_ID(isr); + /* last cmd is 0, maybe ibi finish or nack happened for this transfer */ + if (!finish) { + xfer->cmds[0].error = CMDR_M2_ERROR; + goto done; + } + + /* The three modes should do a extra cmd, now we parsed the cmds[0] status + * at first. + */ + if (xfer->mode == DIRECT_CCC || xfer->mode == PRIVATE_TRANSFER_WITH7E || + xfer->mode == DO_DAA) { + u32 cmdr; + + cmd_status_offset = 1; + cmdr = readl_relaxed(master->regs + CMD1_STATUS); + if (cmdr & STATUS_ADDRNAK) { + xfer->cmds[0].error = CMDR_M2_ERROR; + goto done; + } else if (cmdr & STATUS_TXEARLY_I2C) { + xfer->cmds[0].error = CMDR_M0_ERROR; + goto done; + } + if (finish > 0) + finish -= 1; + } + + /* if it was i2c irq mode, read the left rx data in rx fifo by fifostatus. + * if it was i3c, the device "t = 0" should be cared, the following handles + * is wrong for this case. + */ + if (xfer->xfer_mode == ROCKCHIP_I2C_IRQ) { + unsigned int fifo_status = readl_relaxed(master->regs + FIFO_STATUS); + unsigned int rx_num_word = RXFIFO_SPACE2EMPTY_NUMWORD(fifo_status); + + if (rx_num_word > 0) + rockchip_i3c_master_data_isr_read(master, xfer, rx_num_word); + + writel_relaxed(0, master->regs + TXFIFO_NUM_CPU); + writel_relaxed(0, master->regs + RXFIFO_NUM_CPU); + } + + for (i = 0; i < finish; i++) { + struct rockchip_i3c_cmd *ccmd; + u32 cmdr, error; + + xfer->cmds[i].error = 0; + cmdr = readl_relaxed(master->regs + CMD1_STATUS + 0x4 * cmd_status_offset); + ccmd = &xfer->cmds[i]; + error = (cmdr & STATUS_ADDRNAK) | (cmdr & STATUS_TXEARLY_I2C); + + if (cmdr & STATUS_ERR) { + if (cmdr & STATUS_TXEARLY_I2C) { + xfer->cmds[i].error = CMDR_M0_ERROR; + break; + } else if (cmdr & STATUS_ADDRNAK) { + xfer->cmds[i].error = CMDR_M2_ERROR; + break; + } + } else { + xfer->cmds[i].error = CMDR_NO_ERROR; + } + + if (isr & INT_DATAERR_IRQPD_STS) { + xfer->cmds[i].error = CMDR_M1_ERROR; + break; + } + + /* rnw = 1 */ + if ((cmdr & STATUS_RNW) && !error) { + ccmd->rx_len = CMDR_XFER_BYTES(cmdr); + if (xfer->xfer_mode == ROCKCHIP_I3C_POLL) + rockchip_i3c_master_rd_from_rx_fifo(master, ccmd->rx_buf, + ccmd->rx_len); + } + + cmd_status_offset++; + } + +done: + for (i = 0; i < xfer->ncmds; i++) { + switch (xfer->cmds[i].error) { + case CMDR_NO_ERROR: + break; + case CMDR_M0_ERROR: + case CMDR_M1_ERROR: + case CMDR_M2_ERROR: + case CMDR_MST_ABORT: + case CMDR_NACK_RESP: + ret = -EIO; + break; + default: + ret = -EINVAL; + break; + } + } + + xfer->ret = ret; + if (xfer->xfer_mode != ROCKCHIP_I3C_DMA) + complete(&xfer->comp); + + xfer = list_first_entry_or_null(&master->xferqueue.list, + struct rockchip_i3c_xfer, node); + if (xfer) + list_del_init(&xfer->node); + + master->xferqueue.cur = xfer; + rockchip_i3c_master_start_xfer_locked(master); +} + +static enum i3c_error_code rockchip_i3c_master_cmd_get_err(struct rockchip_i3c_cmd *cmd) +{ + switch (cmd->error) { + case CMDR_M0_ERROR: + return I3C_ERROR_M0; + + case CMDR_M1_ERROR: + return I3C_ERROR_M1; + + case CMDR_M2_ERROR: + case CMDR_NACK_RESP: + return I3C_ERROR_M2; + + default: + break; + } + + return I3C_ERROR_UNKNOWN; +} + +static int rockchip_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *cmd) +{ + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_xfer *xfer; + struct rockchip_i3c_cmd *ccmd; + int ret; + + xfer = rockchip_i3c_master_alloc_xfer(master, 1); + if (!xfer) + return -ENOMEM; + + ccmd = xfer->cmds; + if (cmd->id & I3C_CCC_DIRECT) { + xfer->mode = DIRECT_CCC; + ccmd->fifo_cmd = CMD_FIFO_PL_LEN(cmd->dests[0].payload.len); + xfer->ccc_cmd = cmd->id; + } else { + xfer->mode = BROCAST_CCC; + ccmd->fifo_cmd = CMD_FIFO_PL_LEN(cmd->dests[0].payload.len + 1); + xfer->ccc_cmd = cmd->id & GENMASK(6, 0); + } + + ccmd->fifo_cmd |= CMD_DEV_ADDR(cmd->dests[0].addr); + if (cmd->rnw) { + ccmd->fifo_cmd |= CMD_RNW | CMD_I3C_SDR_RX_MODE; + ccmd->rx_buf = cmd->dests[0].payload.data; + ccmd->rx_len = cmd->dests[0].payload.len; + } else { + ccmd->tx_buf = cmd->dests[0].payload.data; + ccmd->tx_len = cmd->dests[0].payload.len; + } + + /* when ibi nack happen, continue repeat start for current transfer */ + ccmd->fifo_cmd |= CMD_I3C_END2IBI; + + rockchip_i3c_master_queue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(WAIT_TIMEOUT))) + rockchip_i3c_master_unqueue_xfer(master, xfer); + + ret = xfer->ret; + if (ret != -ETIMEDOUT) { + cmd->err = rockchip_i3c_master_cmd_get_err(&xfer->cmds[0]); + if (cmd->rnw) + cmd->dests[0].payload.len = ccmd->rx_len; + } else { + /* timeout error, maybe reset target */ + dev_err(master->dev, "timeout, ipd: 0x%x for ccc cmd\n", + readl_relaxed(master->regs + INT_STATUS)); + } + + rockchip_i3c_master_free_xfer(xfer); + + return ret; +} + +static int rockchip_i3c_master_priv_xfers(struct i3c_dev_desc *dev, + struct i3c_priv_xfer *xfers, + int nxfers) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_xfer *rockchip_xfer; + int txslots = 0, rxslots = 0, i, ret; + unsigned long timeleft; + + for (i = 0; i < nxfers; i++) { + if (xfers[i].len > CMD_FIFO_PL_LEN_MAX) + return -EOPNOTSUPP; + } + + if (!nxfers) + return -EINVAL; + + if (nxfers > master->caps.cmdfifodepth) + return -EOPNOTSUPP; + + rockchip_xfer = rockchip_i3c_master_alloc_xfer(master, nxfers); + if (!rockchip_xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + if (xfers[i].rnw) + rxslots += DIV_ROUND_UP(xfers[i].len, 4); + else + txslots += DIV_ROUND_UP(xfers[i].len, 4); + } + + if ((rxslots > master->caps.datafifodepth || + txslots > master->caps.datafifodepth) && + (master->dma_rx && master->dma_tx)) + rockchip_xfer->xfer_mode = ROCKCHIP_I3C_DMA; + else if (rxslots <= master->caps.datafifodepth && + txslots <= master->caps.datafifodepth) + rockchip_xfer->xfer_mode = ROCKCHIP_I3C_POLL; + else + return -EOPNOTSUPP; + + /* Use without 7E defaultly, if use PRIVATE_TRANSFER_WITH7E, + * the nxfers should less than "cmdfifodepth - 1". + */ + rockchip_xfer->mode = PRIVATE_TRANSFER_WITHOUT7E; + if (rockchip_xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + rockchip_xfer->rx_trig = DMA_TRIG_LEVEL; + rockchip_xfer->tx_trig = DMA_TRIG_LEVEL; + rockchip_xfer->rx_burst_size = DMA_RX_MAX_BURST_SIZE; + rockchip_xfer->rx_burst_len = DMA_RX_MAX_BURST_LEN; + } + + for (i = 0; i < nxfers; i++) { + struct rockchip_i3c_cmd *ccmd = &rockchip_xfer->cmds[i]; + u32 pl_len; + + pl_len = xfers[i].len; + ccmd->fifo_cmd = CMD_DEV_ADDR(dev->info.dyn_addr); + + if (xfers[i].rnw) { + ccmd->fifo_cmd |= CMD_RNW | CMD_I3C_SDR_RX_MODE | CMD_I3C_ACK2EARLY; + if (rockchip_xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + u8 rx_trig, rx_burst_size, rx_burst_len, *buf; + + rx_burst_size = rockchip_i3c_calc_rx_burst(xfers[i].len, false); + if (rockchip_xfer->rx_burst_size > rx_burst_size) + rockchip_xfer->rx_burst_size = rx_burst_size; + + rx_burst_len = rockchip_i3c_calc_rx_burst(xfers[i].len, true); + if (rockchip_xfer->rx_burst_len > rx_burst_len) + rockchip_xfer->rx_burst_len = rx_burst_len; + + rx_trig = rockchip_i3c_master_calc_trig_level(xfers[i].len, true); + if (rockchip_xfer->rx_trig > rx_trig) + rockchip_xfer->rx_trig = rx_trig; + + buf = rockchip_i3c_master_get_dma_safe_buf(master, xfers[i].data.in, + xfers[i].len, true); + ccmd->rx_buf = buf; + } else { + ccmd->rx_buf = xfers[i].data.in; + } + ccmd->rx_len = xfers[i].len; + } else { + ccmd->fifo_cmd |= CMD_I3C_SDR_TX_MODE; + if (rockchip_xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + u8 tx_trig, *buf; + + tx_trig = rockchip_i3c_master_calc_trig_level(xfers[i].len, false); + if (rockchip_xfer->tx_trig > tx_trig) + rockchip_xfer->tx_trig = tx_trig; + + buf = rockchip_i3c_master_get_dma_safe_buf(master, + (u8 *)xfers[i].data.out, + xfers[i].len, false); + ccmd->tx_buf = buf; + } else { + ccmd->tx_buf = (void *)xfers[i].data.out; + } + ccmd->tx_len = xfers[i].len; + } + + /* when ibi nack happen, continue repeat start for current transfer */ + ccmd->fifo_cmd |= CMD_FIFO_PL_LEN(pl_len) | CMD_I3C_END2IBI; + if (i < (nxfers - 1)) + ccmd->fifo_cmd |= CMD_ACK2NEXT; + } + + rockchip_i3c_master_queue_xfer(master, rockchip_xfer); + timeleft = wait_for_completion_timeout(&rockchip_xfer->comp, + msecs_to_jiffies(WAIT_TIMEOUT)); + if (!timeleft) { + if (rockchip_xfer->xfer_mode == ROCKCHIP_I3C_DMA) + rockchip_i3c_maste_cleanup_dma(master); + rockchip_i3c_master_unqueue_xfer(master, rockchip_xfer); + } else { + if (rockchip_xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + unsigned int status; + + /* if the last is tx, make sure tx is completed for controller */ + status = rockchip_i3c_master_wait_for_tx_idle(master); + if (status & IRQ_END_STS) + rockchip_i3c_master_end_xfer_locked(master, status); + } + } + + for (i = 0; i < nxfers; i++) { + struct rockchip_i3c_cmd *cmd = &rockchip_xfer->cmds[i]; + + if (rockchip_xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + u8 *buf, *target; + + target = xfers[i].data.in; + buf = xfers[i].rnw ? cmd->rx_buf : cmd->tx_buf; + rockchip_i3c_master_put_dma_safe_buf(master, buf, target, xfers[i].len, + rockchip_xfer->xferred, + xfers[i].rnw); + } + xfers[i].err = rockchip_i3c_master_cmd_get_err(&rockchip_xfer->cmds[i]); + if (xfers[i].rnw) + xfers[i].len = cmd->rx_len; + } + + ret = rockchip_xfer->ret; + if (ret == -ETIMEDOUT) + /* timeout error, maybe reset target */ + dev_err(master->dev, "timeout, ipd: 0x%x for i3c transfer\n", + readl_relaxed(master->regs + INT_STATUS)); + + rockchip_i3c_master_free_xfer(rockchip_xfer); + + return ret; +} + +static int rockchip_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, + const struct i2c_msg *xfers, + int nxfers) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + unsigned int nrxwords = 0, ntxwords = 0; + struct rockchip_i3c_xfer *xfer; + unsigned long timeleft; + int i, ret = 0; + + if (nxfers > master->caps.cmdfifodepth) + return -EOPNOTSUPP; + + for (i = 0; i < nxfers; i++) { + if (xfers[i].len > CMD_FIFO_PL_LEN_MAX) + return -EOPNOTSUPP; + if (xfers[i].flags & I2C_M_RD) + nrxwords += DIV_ROUND_UP(xfers[i].len, 4); + else + ntxwords += DIV_ROUND_UP(xfers[i].len, 4); + } + + xfer = rockchip_i3c_master_alloc_xfer(master, nxfers); + if (!xfer) + return -ENOMEM; + + if ((nrxwords > master->caps.datafifodepth || + ntxwords > master->caps.datafifodepth) && + (master->dma_rx && master->dma_tx)) + xfer->xfer_mode = ROCKCHIP_I3C_DMA; + else if ((nrxwords > master->caps.datafifodepth || + ntxwords > master->caps.datafifodepth) && + (!master->dma_rx || !master->dma_tx)) + xfer->xfer_mode = ROCKCHIP_I2C_IRQ; + else + xfer->xfer_mode = ROCKCHIP_I3C_POLL; + + xfer->rx_num_word = nrxwords; + xfer->tx_num_word = ntxwords; + + /* Use without 7E defaultly, if use PRIVATE_TRANSFER_WITH7E, + * the nxfers should less than "cmdfifodepth - 1". + */ + xfer->mode = PRIVATE_TRANSFER_WITHOUT7E; + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA || + xfer->xfer_mode == ROCKCHIP_I2C_IRQ) { + xfer->rx_trig = DMA_TRIG_LEVEL; + xfer->tx_trig = DMA_TRIG_LEVEL; + xfer->rx_burst_size = DMA_RX_MAX_BURST_SIZE; + xfer->rx_burst_len = DMA_RX_MAX_BURST_LEN; + } + + for (i = 0; i < nxfers; i++) { + struct rockchip_i3c_cmd *ccmd = &xfer->cmds[i]; + + ccmd->fifo_cmd = CMD_DEV_ADDR(xfers[i].addr) | + CMD_FIFO_PL_LEN(xfers[i].len); + + if (xfers[i].flags & I2C_M_TEN) + return -EOPNOTSUPP; + + if (xfers[i].flags & I2C_M_RD) { + ccmd->fifo_cmd |= CMD_RNW | CMD_I2C_RX | CMD_I2C_ACK2LAST; + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + u8 rx_trig, rx_burst_size, rx_burst_len, *buf; + + rx_burst_size = rockchip_i3c_calc_rx_burst(xfers[i].len, false); + if (xfer->rx_burst_size > rx_burst_size) + xfer->rx_burst_size = rx_burst_size; + + rx_burst_len = rockchip_i3c_calc_rx_burst(xfers[i].len, true); + if (xfer->rx_burst_len > rx_burst_len) + xfer->rx_burst_len = rx_burst_len; + + rx_trig = rockchip_i3c_master_calc_trig_level(xfers[i].len, true); + if (xfer->rx_trig > rx_trig) + xfer->rx_trig = rx_trig; + + buf = rockchip_i3c_master_get_dma_safe_buf(master, xfers[i].buf, + xfers[i].len, true); + ccmd->rx_buf = buf; + } else { + ccmd->rx_buf = xfers[i].buf; + } + ccmd->rx_len = xfers[i].len; + } else { + ccmd->fifo_cmd |= CMD_I2C_TX; + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + u8 tx_size, *buf; + + tx_size = rockchip_i3c_master_calc_trig_level(xfers[i].len, false); + if (xfer->tx_trig > tx_size) + xfer->tx_trig = tx_size; + buf = rockchip_i3c_master_get_dma_safe_buf(master, + (u8 *)xfers[i].buf, + xfers[i].len, false); + ccmd->tx_buf = buf; + } else { + ccmd->tx_buf = xfers[i].buf; + } + ccmd->tx_len = xfers[i].len; + } + + if (i < (nxfers - 1)) + ccmd->fifo_cmd |= CMD_ACK2NEXT; + } + + rockchip_i3c_master_queue_xfer(master, xfer); + timeleft = wait_for_completion_timeout(&xfer->comp, + msecs_to_jiffies(WAIT_TIMEOUT)); + if (!timeleft) { + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA) + rockchip_i3c_maste_cleanup_dma(master); + rockchip_i3c_master_unqueue_xfer(master, xfer); + } else { + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + unsigned int status; + + /* if the last is tx, make sure tx is completed for controller */ + status = rockchip_i3c_master_wait_for_tx_idle(master); + if (status & IRQ_END_STS) + rockchip_i3c_master_end_xfer_locked(master, status); + } + } + + for (i = 0; i < nxfers; i++) { + struct rockchip_i3c_cmd *ccmd = &xfer->cmds[i]; + + if (xfer->xfer_mode == ROCKCHIP_I3C_DMA) { + u8 *buf, *target; + + buf = (xfers[i].flags & I2C_M_RD) ? ccmd->rx_buf : ccmd->tx_buf; + target = xfers[i].buf; + rockchip_i3c_master_put_dma_safe_buf(master, buf, target, xfers[i].len, + xfer->xferred, + xfers[i].flags & I2C_M_RD); + } + } + + ret = xfer->ret; + if (ret == -ETIMEDOUT) + /* timeout error, maybe reset target */ + dev_err(master->dev, "timeout, ipd: 0x%x for i3c transfer\n", + readl_relaxed(master->regs + INT_STATUS)); + + rockchip_i3c_master_free_xfer(xfer); + + return ret; +} + +static inline int rockchip_i3c_master_get_addr_pos(struct rockchip_i3c_master *master, + u8 addr) +{ + int pos; + + for (pos = 0; pos < master->maxdevs; pos++) { + if (addr == master->devs[pos].addr) + return pos; + } + + return -EINVAL; +} + +static inline int rockchip_i3c_master_get_free_pos(struct rockchip_i3c_master *master) +{ + if (!(master->free_pos & GENMASK(master->maxdevs - 1, 0))) + return -ENOSPC; + + return ffs(master->free_pos) - 1; +} + +static int rockchip_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, + unsigned char old_dyn_addr) +{ + return 0; +} + +static int rockchip_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_i2c_dev_data *data; + int slot; + + slot = rockchip_i3c_master_get_free_pos(master); + if (slot < 0) + return slot; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->ibi = -1; + data->id = slot; + master->devs[slot].addr = dev->info.dyn_addr ? : dev->info.static_addr; + master->free_pos &= ~BIT(slot); + i3c_dev_set_master_data(dev, data); + + return 0; +} + +static void rockchip_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + i3c_dev_set_master_data(dev, NULL); + master->devs[data->id].addr = 0; + master->free_pos |= BIT(data->id); + kfree(data); +} + +static int rockchip_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_i2c_dev_data *data; + int slot; + + slot = rockchip_i3c_master_get_free_pos(master); + if (slot < 0) + return slot; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->id = slot; + master->devs[slot].addr = dev->addr; + master->free_pos &= ~BIT(slot); + + i2c_dev_set_master_data(dev, data); + + return 0; +} + +static void rockchip_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + + master->devs[data->id].addr = 0; + master->free_pos |= BIT(data->id); + i2c_dev_set_master_data(dev, NULL); + kfree(data); +} + +static void rockchip_i3c_master_bus_cleanup(struct i3c_master_controller *m) +{ + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + + rockchip_i3c_master_disable(master); +} + +static int rockchip_i3c_master_do_daa(struct i3c_master_controller *m) +{ + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_xfer *xfer; + int ret = 0, pos, n = 0, last, i; + u32 olddevs, newdevs; + u8 last_addr = 0; + u32 daa_status; + + olddevs = ~(master->free_pos); + /* Prepare DAT before launching DAA */ + for (pos = 1; pos < master->maxdevs; pos++) { + if (olddevs & BIT(pos)) + continue; + + ret = i3c_master_get_free_addr(m, last_addr + 1); + if (ret < 0) + return -ENOSPC; + + last_addr = ret; + master->devs[pos].addr = ret; + writel_relaxed(ret, master->regs + DEV_ID_RR0(n)); + n++; + } + + xfer = rockchip_i3c_master_alloc_xfer(master, n); + if (!xfer) + return -ENOMEM; + + xfer->mode = DO_DAA; + xfer->ccc_cmd = I3C_CCC_ENTDAA; + + rockchip_i3c_master_queue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(WAIT_TIMEOUT))) + rockchip_i3c_master_unqueue_xfer(master, xfer); + + for (i = 0; i < n; i++) + xfer->cmds[i].error = rockchip_i3c_master_cmd_get_err(&xfer->cmds[i]); + + daa_status = readl_relaxed(master->regs + DAA_STATUS); + last = daa_status & DAA_ASSIGNED_CNT_MASK; + if (last >= 1) + ret = 0; + else + ret = xfer->ret; + if (!ret) + master->daa_done = true; + + newdevs = GENMASK(master->maxdevs - last, 0); + newdevs &= ~olddevs; + + for (pos = 1, n = 0; pos < master->maxdevs; pos++) { + if (n >= last) + break; + + if (newdevs & BIT(pos)) { + if (!xfer->cmds[n].error) + i3c_master_add_i3c_dev_locked(m, master->devs[pos].addr); + n++; + } + } + + rockchip_i3c_master_free_xfer(xfer); + + return ret; +} + +static void rockchip_i3c_master_handle_ibi(struct rockchip_i3c_master *master, + u32 ibir) +{ + struct rockchip_i3c_i2c_dev_data *data; + bool data_consumed = false; + struct i3c_ibi_slot *slot; + u32 id = IBI_SLVID(ibir); + struct i3c_dev_desc *dev; + size_t nbytes; + u8 *buf; + + if (id >= master->ibi.num_slots || (ibir & IBI_NACK)) + goto out; + + dev = master->ibi.slots[id]; + spin_lock(&master->ibi.lock); + + data = i3c_dev_get_master_data(dev); + slot = i3c_generic_ibi_get_free_slot(data->ibi_pool); + if (!slot) + goto out_unlock; + + buf = slot->data; + nbytes = IBI_XFER_BYTES(ibir); + readsl(master->regs + IBIDATA, buf, nbytes / 4); + if (nbytes % 3) { + u32 tmp = readl_relaxed(master->regs + IBIDATA); + + memcpy(buf + (nbytes & ~3), &tmp, nbytes & 3); + } + + slot->len = min_t(unsigned int, IBI_XFER_BYTES(ibir), + dev->ibi->max_payload_len); + i3c_master_queue_ibi(dev, slot); + data_consumed = true; + +out_unlock: + spin_unlock(&master->ibi.lock); + +out: + /* Consume data from the FIFO if it's not been done already. */ + if (!data_consumed) { + int i; + + for (i = 0; i < IBI_XFER_BYTES(ibir); i += 4) + readl_relaxed(master->regs + IBIDATA); + } + /* At last, flush ibi status */ + writel_relaxed(FLUSH_IBI_ALL, master->regs + FLUSH_CTL); +} + +static void rockchip_i3c_master_demux_ibis(struct rockchip_i3c_master *master, + u32 isr) +{ + u32 ibir = readl_relaxed(master->regs + IBI_STATUS); + + if (!(isr & I2CIBI_IRQPD_STS)) { + switch (IBI_TYPE(ibir)) { + case IBI_TYPE_IBI: + rockchip_i3c_master_handle_ibi(master, ibir); + break; + + case IBI_TYPE_HJ: + queue_work(master->base.wq, &master->hj_work); + break; + + case IBI_TYPE_MR: + break; + + default: + break; + } + } else { + dev_err(&master->base.dev, "Get IBI request from 0x%lx at i2c transfer\n", + I2C_IBIADDR(ibir)); + } +} + +static irqreturn_t rockchip_i3c_master_interrupt(int irq, void *data) +{ + struct rockchip_i3c_master *master = data; + u32 status, ien; + + ien = readl_relaxed(master->regs + INT_EN); + status = readl_relaxed(master->regs + INT_STATUS); + writel_relaxed(status | ERR_CMD_ID_MASK | FINISHED_CMD_ID_MASK, + master->regs + INT_STATUS); + if (!(status & ien)) + return IRQ_NONE; + + spin_lock(&master->xferqueue.lock); + if (status & IRQ_END_STS) + rockchip_i3c_master_end_xfer_locked(master, status); + else if (status & CPU_FIFOTRIG_IRQ_STS) + rockchip_i3c_master_data_isr(master, status); + + if ((status & OWNIBI_IRQPD_STS) && (ien & OWNIBI_IRQPD_IEN)) + rockchip_i3c_master_demux_ibis(master, status); + spin_unlock(&master->xferqueue.lock); + + return IRQ_HANDLED; +} + +static int rockchip_i3c_master_request_ibi(struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + + data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req); + if (IS_ERR(data->ibi_pool)) + return PTR_ERR(data->ibi_pool); + + spin_lock_irqsave(&master->ibi.lock, flags); + data->ibi = data->id; + master->ibi.slots[data->id] = dev; + spin_unlock_irqrestore(&master->ibi.lock, flags); + + if (data->ibi < master->ibi.num_slots) + return 0; + + i3c_generic_ibi_free_pool(data->ibi_pool); + data->ibi_pool = NULL; + + return -ENOSPC; +} + +static void rockchip_i3c_master_free_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + + spin_lock_irqsave(&master->ibi.lock, flags); + master->ibi.slots[data->ibi] = NULL; + data->ibi = -1; + spin_unlock_irqrestore(&master->ibi.lock, flags); + + i3c_generic_ibi_free_pool(data->ibi_pool); +} + +static int rockchip_i3c_master_disable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + unsigned long flags; + int ret; + + ret = i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); + if (ret) + return ret; + + spin_lock_irqsave(&master->ibi.lock, flags); + writel_relaxed(IBI_CONF_AUTOIBI_DIS, master->regs + IBI_CONF); + writel_relaxed(OWNIBI_IRQPD_DIS, master->regs + INT_EN); + spin_unlock_irqrestore(&master->ibi.lock, flags); + + return ret; +} + +static int rockchip_i3c_master_enable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct rockchip_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + unsigned int map; + int ret; + + spin_lock_irqsave(&master->ibi.lock, flags); + map = readl_relaxed(master->regs + IBI_MAP(data->id)); + map |= IBI_MAP_DEV_ADDR(data->id, dev->info.dyn_addr) | + IBI_MAP_DEV_PLLEN(data->id, dev->info.max_ibi_len); + writel_relaxed(map, master->regs + IBI_MAP(data->id)); + + writel_relaxed(OWNIBI_IRQPD_IEN, master->regs + INT_EN); + writel_relaxed(IBI_CONF_AUTOIBI_EN, master->regs + IBI_CONF); + writel_relaxed(IBI_REG_READY_CONF_EN, master->regs + REG_READY_CONF); + spin_unlock_irqrestore(&master->ibi.lock, flags); + + ret = i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); + if (ret) { + spin_lock_irqsave(&master->ibi.lock, flags); + writel_relaxed(IBI_CONF_AUTOIBI_DIS, master->regs + IBI_CONF); + writel_relaxed(OWNIBI_IRQPD_DIS, master->regs + INT_EN); + spin_unlock_irqrestore(&master->ibi.lock, flags); + } + + return ret; +} + +static void rockchip_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct rockchip_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + i3c_generic_ibi_recycle_slot(data->ibi_pool, slot); +} + +static int rockchip_i3c_clk_cfg(struct rockchip_i3c_master *master) +{ + unsigned long rate, period; + u32 scl_timing; + int hcnt, lcnt, od_lcnt; + + rate = clk_get_rate(master->clk); + if (!rate) + return -EINVAL; + + period = DIV_ROUND_UP(1000000000, rate); + hcnt = DIV_ROUND_UP(I3C_BUS_THIGH_MAX_NS, period); + if (hcnt < SCL_I3C_TIMING_CNT_MIN) + hcnt = SCL_I3C_TIMING_CNT_MIN; + + lcnt = DIV_ROUND_UP(rate, master->base.bus.scl_rate.i3c) - hcnt; + if (lcnt < SCL_I3C_TIMING_CNT_MIN) + lcnt = SCL_I3C_TIMING_CNT_MIN; + + od_lcnt = lcnt; + hcnt = DIV_ROUND_UP(hcnt, 4) - 1; + lcnt = DIV_ROUND_UP(lcnt, 4) - 1; + + scl_timing = SCL_I3C_CLKDIV_HCNT(hcnt) | SCL_I3C_CLKDIV_LCNT(lcnt); + writel_relaxed(scl_timing, master->regs + CLKDIV_PP); + + od_lcnt = max_t(u8, DIV_ROUND_UP(I3C_BUS_TLOW_OD_MIN_NS * rate, 1000000000), + od_lcnt); + /* The scl raise time must be calculated for OD mode, as default + * FM+ mode's max raise time is 120ns, can't exceed 1/2 scl_low + * time for default. + */ + od_lcnt = max_t(u8, DIV_ROUND_UP(2 * I3C_BUS_I2C_FMP_SCL_RAISE_NS * rate, 1000000000), + od_lcnt); + od_lcnt = DIV_ROUND_UP(od_lcnt, 4) - 1; + scl_timing = SCL_I3C_CLKDIV_HCNT(hcnt) | SCL_I3C_CLKDIV_LCNT(od_lcnt); + writel_relaxed(scl_timing, master->regs + CLKDIV_OD); + + return 0; +} + +static int rockchip_i2c_clk_cfg(struct rockchip_i3c_master *master) +{ + unsigned long rate, period; + int hcnt, lcnt; + u32 scl_timing; + + rate = clk_get_rate(master->clk); + if (!rate) + return -EINVAL; + + period = DIV_ROUND_UP(1000000000, rate); + lcnt = DIV_ROUND_UP(DIV_ROUND_UP(I3C_BUS_I2C_FM_TLOW_MIN_NS, period), 4) - 1; + /* Use FM mode for default */ + hcnt = DIV_ROUND_UP(DIV_ROUND_UP(rate, I3C_BUS_I2C_FM_SCL_RATE), 4) - lcnt - 1; + scl_timing = SCL_I2C_CLKDIV_HCNT(hcnt) | SCL_I2C_CLKDIV_LCNT(lcnt); + writel_relaxed(scl_timing, master->regs + CLKDIV_FM); + + return 0; +} + +static int rockchip_i3c_master_bus_init(struct i3c_master_controller *m) +{ + struct rockchip_i3c_master *master = to_rockchip_i3c_master(m); + struct i3c_bus *bus = i3c_master_get_bus(m); + struct i3c_device_info info; + int ret; + + switch (bus->mode) { + case I3C_BUS_MODE_MIXED_FAST: + writel_relaxed(MODULE_CONF_MIX_FAST_I3C, master->regs + MODULE_CONF); + ret = rockchip_i2c_clk_cfg(master); + if (ret) + return ret; + ret = rockchip_i3c_clk_cfg(master); + if (ret) + return ret; + break; + case I3C_BUS_MODE_MIXED_LIMITED: + case I3C_BUS_MODE_MIXED_SLOW: + writel_relaxed(MODULE_CONF_MIX_SLOW_I3C, master->regs + MODULE_CONF); + ret = rockchip_i2c_clk_cfg(master); + if (ret) + return ret; + break; + case I3C_BUS_MODE_PURE: + writel_relaxed(MODULE_CONF_PURE_I3C, master->regs + MODULE_CONF); + /* FM Start will be used, so still need to configure i2c clk cfg. */ + ret = rockchip_i2c_clk_cfg(master); + if (ret) + return ret; + ret = rockchip_i3c_clk_cfg(master); + if (ret) + return ret; + break; + default: + writel_relaxed(MODULE_CONF_PURE_I2C, master->regs + MODULE_CONF); + ret = rockchip_i2c_clk_cfg(master); + if (ret) + return ret; + break; + } + + if (!master->suspended) { + /* Get an address for the master. */ + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + return ret; + + memset(&info, 0, sizeof(info)); + info.dyn_addr = ret; + + ret = i3c_master_set_info(&master->base, &info); + if (ret) + return ret; + } + + /* Give max timing for controller holding scl by default */ + writel_relaxed(0xffffffff, master->regs + TOUT_CONF_I3C); + writel_relaxed(0xffffffff, master->regs + WAIT_THLD_I2C); + + /* Workaround to Give 4 clock cycles delay before ack and T bit for 12.5M */ + writel_relaxed(CLKSTALL_ENABLE | CLKSTALL_BEFORE_ACK_ENABLE | + CLKSTALL_BEFORE_READ_ENABLE | CLKSTALL_NUM, + master->regs + CLKSTALL_CONF_I3C); + + rockchip_i3c_master_enable(master); + + return 0; +} + +static const struct i3c_master_controller_ops rockchip_i3c_master_ops = { + .bus_init = rockchip_i3c_master_bus_init, + .bus_cleanup = rockchip_i3c_master_bus_cleanup, + .attach_i3c_dev = rockchip_i3c_master_attach_i3c_dev, + .reattach_i3c_dev = rockchip_i3c_master_reattach_i3c_dev, + .detach_i3c_dev = rockchip_i3c_master_detach_i3c_dev, + .do_daa = rockchip_i3c_master_do_daa, + .supports_ccc_cmd = rockchip_i3c_master_supports_ccc_cmd, + .send_ccc_cmd = rockchip_i3c_master_send_ccc_cmd, + .priv_xfers = rockchip_i3c_master_priv_xfers, + .attach_i2c_dev = rockchip_i3c_master_attach_i2c_dev, + .detach_i2c_dev = rockchip_i3c_master_detach_i2c_dev, + .i2c_xfers = rockchip_i3c_master_i2c_xfers, + .request_ibi = rockchip_i3c_master_request_ibi, + .free_ibi = rockchip_i3c_master_free_ibi, + .recycle_ibi_slot = rockchip_i3c_master_recycle_ibi_slot, + .enable_ibi = rockchip_i3c_master_enable_ibi, + .disable_ibi = rockchip_i3c_master_disable_ibi, +}; + +static void rockchip_i3c_master_hj(struct work_struct *work) +{ + struct rockchip_i3c_master *master = + container_of(work, struct rockchip_i3c_master, hj_work); + + i3c_master_do_daa(&master->base); +} + +static int rockchip_i3c_probe(struct platform_device *pdev) +{ + struct rockchip_i3c_master *i3c; + struct resource *res; + int ret, irq; + + i3c = devm_kzalloc(&pdev->dev, sizeof(*i3c), GFP_KERNEL); + if (!i3c) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + i3c->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(i3c->regs)) + return PTR_ERR(i3c->regs); + + irq = platform_get_irq(pdev, 0); + if (i3c->irq < 0) + return i3c->irq; + + i3c->hclk = devm_clk_get(&pdev->dev, "hclk"); + if (IS_ERR(i3c->hclk)) + return PTR_ERR(i3c->hclk); + + i3c->clk = devm_clk_get(&pdev->dev, "i3c"); + if (IS_ERR(i3c->clk)) + return PTR_ERR(i3c->clk); + + ret = clk_prepare_enable(i3c->hclk); + if (ret < 0) { + dev_err(&pdev->dev, "Can't prepare i3c hclk: %d\n", ret); + return ret; + } + + ret = clk_prepare_enable(i3c->clk); + if (ret < 0) { + dev_err(&pdev->dev, "Can't prepare i3c bus clock: %d\n", ret); + goto err_hclk; + } + + i3c->dma_tx = dma_request_chan(&pdev->dev, "tx"); + if (IS_ERR(i3c->dma_tx)) { + /* Check tx to see if we need defer probing driver */ + if (PTR_ERR(i3c->dma_tx) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_clk; + } + dev_warn(&pdev->dev, "Failed to request TX DMA channel\n"); + i3c->dma_tx = NULL; + } + + i3c->dma_rx = dma_request_chan(&pdev->dev, "rx"); + if (IS_ERR(i3c->dma_rx)) { + if (PTR_ERR(i3c->dma_rx) == -EPROBE_DEFER) { + ret = -EPROBE_DEFER; + goto err_free_dma_tx; + } + dev_warn(&pdev->dev, "Failed to request RX DMA channel\n"); + i3c->dma_rx = NULL; + } + + if (i3c->dma_tx) + i3c->dma_addr_tx = res->start + TXDATA; + if (i3c->dma_rx) + i3c->dma_addr_rx = res->start + RXDATA; + + i3c->reset = devm_reset_control_get(&pdev->dev, "i3c"); + i3c->reset_ahb = devm_reset_control_get(&pdev->dev, "ahb"); + rockchip_i3c_master_reset_controller(i3c); + + spin_lock_init(&i3c->xferqueue.lock); + INIT_LIST_HEAD(&i3c->xferqueue.list); + + /* Information regarding the FIFOs depth */ + i3c->caps.cmdfifodepth = CMDFIFO_DEPTH; + i3c->caps.datafifodepth = DATAFIFO_DEPTH; + i3c->caps.ibififodepth = IBIFIFO_DEPTH; + + /* Device ID0 is reserved to describe this master. */ + i3c->maxdevs = ROCKCHIP_I3C_MAX_DEVS; + i3c->free_pos = GENMASK(ROCKCHIP_I3C_MAX_DEVS, 1); + + i3c->dev = &pdev->dev; + dev_set_drvdata(&pdev->dev, i3c); + + INIT_WORK(&i3c->hj_work, rockchip_i3c_master_hj); + spin_lock_init(&i3c->ibi.lock); + + i3c->ibi.num_slots = ROCKCHIP_I3C_MAX_DEVS; + i3c->ibi.slots = devm_kcalloc(&pdev->dev, i3c->ibi.num_slots, + sizeof(*i3c->ibi.slots), + GFP_KERNEL); + if (!i3c->ibi.slots) { + ret = -ENOMEM; + goto err_free_dma_rx; + } + + ret = devm_request_irq(&pdev->dev, irq, + rockchip_i3c_master_interrupt, 0, + dev_name(&pdev->dev), i3c); + if (ret) + goto err_free_dma_rx; + + i3c->aligned_cache = kmem_cache_create("rockchip-i3c", CMD_FIFO_PL_LEN_MAX, 0, + SLAB_CACHE_DMA | SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA32, + NULL); + if (!i3c->aligned_cache) { + ret = -ENOMEM; + dev_err(&pdev->dev, "unable to create rockchip i3c aligned cache\n"); + goto err_free_dma_rx; + } + + ret = i3c_master_register(&i3c->base, &pdev->dev, + &rockchip_i3c_master_ops, false); + if (ret) + goto err_destroy_cache; + + return 0; + +err_destroy_cache: + kmem_cache_destroy(i3c->aligned_cache); + +err_free_dma_rx: + if (i3c->dma_rx) + dma_release_channel(i3c->dma_rx); +err_free_dma_tx: + if (i3c->dma_tx) + dma_release_channel(i3c->dma_tx); +err_clk: + clk_disable_unprepare(i3c->clk); +err_hclk: + clk_disable_unprepare(i3c->hclk); + return ret; +} + +static int rockchip_i3c_remove(struct platform_device *pdev) +{ + struct rockchip_i3c_master *i3c = dev_get_drvdata(&pdev->dev); + + if (i3c->dma_rx) + dma_release_channel(i3c->dma_rx); + if (i3c->dma_tx) + dma_release_channel(i3c->dma_tx); + + kmem_cache_destroy(i3c->aligned_cache); + i3c_master_unregister(&i3c->base); + clk_disable_unprepare(i3c->clk); + clk_disable_unprepare(i3c->hclk); + + return 0; +} + +static __maybe_unused int rockchip_i3c_suspend_noirq(struct device *dev) +{ + struct rockchip_i3c_master *i3c = dev_get_drvdata(dev); + + i3c->suspended = true; + rockchip_i3c_master_bus_cleanup(&i3c->base); + clk_disable_unprepare(i3c->clk); + clk_disable_unprepare(i3c->hclk); + + return 0; +} + +static __maybe_unused int rockchip_i3c_resume_noirq(struct device *dev) +{ + struct rockchip_i3c_master *i3c = dev_get_drvdata(dev); + + clk_prepare_enable(i3c->clk); + clk_prepare_enable(i3c->hclk); + rockchip_i3c_master_bus_init(&i3c->base); + i3c->suspended = false; + + return 0; +} + +static const struct dev_pm_ops rockchip_i3c_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_i3c_suspend_noirq, + rockchip_i3c_resume_noirq) +}; + +static const struct of_device_id rockchip_i3c_of_match[] = { + { .compatible = "rockchip,i3c-master" }, + { } +}; +MODULE_DEVICE_TABLE(of, rockchip_i3c_of_match); + +static struct platform_driver rockchip_i3c_driver = { + .probe = rockchip_i3c_probe, + .remove = rockchip_i3c_remove, + .driver = { + .name = "rockchip-i3c", + .of_match_table = rockchip_i3c_of_match, + .pm = &rockchip_i3c_pm_ops, + }, +}; + +module_platform_driver(rockchip_i3c_driver); + +MODULE_AUTHOR("David Wu "); +MODULE_DESCRIPTION("Rockchip I3C Master Controller Driver"); +MODULE_LICENSE("GPL");