diff --git a/drivers/ufs/core/Makefile b/drivers/ufs/core/Makefile index 51e1867e524e..ce7d16d2cf35 100644 --- a/drivers/ufs/core/Makefile +++ b/drivers/ufs/core/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_SCSI_UFSHCD) += ufshcd-core.o -ufshcd-core-y += ufshcd.o ufs-sysfs.o ufs-mcq.o +ufshcd-core-y += ufshcd.o ufs-sysfs.o ufs-mcq.o ufs-txeq.o ufshcd-core-$(CONFIG_RPMB) += ufs-rpmb.o ufshcd-core-$(CONFIG_DEBUG_FS) += ufs-debugfs.o ufshcd-core-$(CONFIG_SCSI_UFS_BSG) += ufs_bsg.o diff --git a/drivers/ufs/core/ufs-txeq.c b/drivers/ufs/core/ufs-txeq.c new file mode 100644 index 000000000000..04f9f1ffa43e --- /dev/null +++ b/drivers/ufs/core/ufs-txeq.c @@ -0,0 +1,1186 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2026 Qualcomm Technologies, Inc. + * + * Author: + * Can Guo + */ + +#include +#include +#include +#include +#include +#include +#include "ufshcd-priv.h" + +static bool use_adaptive_txeq; +module_param(use_adaptive_txeq, bool, 0644); +MODULE_PARM_DESC(use_adaptive_txeq, "Find and apply optimal TX Equalization settings before changing Power Mode (default: false)"); + +static int txeq_gear_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, UFS_HS_G1, UFS_HS_GEAR_MAX); +} + +static const struct kernel_param_ops txeq_gear_ops = { + .set = txeq_gear_set, + .get = param_get_uint, +}; + +static unsigned int adaptive_txeq_gear = UFS_HS_G6; +module_param_cb(adaptive_txeq_gear, &txeq_gear_ops, &adaptive_txeq_gear, 0644); +MODULE_PARM_DESC(adaptive_txeq_gear, "For HS-Gear[n] and above, adaptive txeq shall be used"); + +static bool use_txeq_presets; +module_param(use_txeq_presets, bool, 0644); +MODULE_PARM_DESC(use_txeq_presets, "Use only the 8 TX Equalization Presets (pre-defined Pre-Shoot & De-Emphasis combinations) for TX EQTR (default: false)"); + +static bool txeq_presets_selected[UFS_TX_EQ_PRESET_MAX] = {[0 ... (UFS_TX_EQ_PRESET_MAX - 1)] = 1}; +module_param_array(txeq_presets_selected, bool, NULL, 0644); +MODULE_PARM_DESC(txeq_presets_selected, "Use only the selected Presets out of the 8 TX Equalization Presets for TX EQTR"); + +/* + * ufs_tx_eq_preset - Table of minimum required list of presets. + * + * A HS-G6 capable M-TX shall support the presets defined in M-PHY v6.0 spec. + * Preset Pre-Shoot(dB) De-Emphasis(dB) + * P0 0.0 0.0 + * P1 0.0 0.8 + * P2 0.0 1.6 + * P3 0.8 0.0 + * P4 1.6 0.0 + * P5 0.8 0.8 + * P6 0.8 1.6 + * P7 1.6 0.8 + */ +static const struct __ufs_tx_eq_preset { + u8 preshoot; + u8 deemphasis; +} ufs_tx_eq_preset[UFS_TX_EQ_PRESET_MAX] = { + [UFS_TX_EQ_PRESET_P0] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P0}, + [UFS_TX_EQ_PRESET_P1] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_0P8}, + [UFS_TX_EQ_PRESET_P2] = {UFS_TX_HS_PRESHOOT_DB_0P0, UFS_TX_HS_DEEMPHASIS_DB_1P6}, + [UFS_TX_EQ_PRESET_P3] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P0}, + [UFS_TX_EQ_PRESET_P4] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P0}, + [UFS_TX_EQ_PRESET_P5] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_0P8}, + [UFS_TX_EQ_PRESET_P6] = {UFS_TX_HS_PRESHOOT_DB_0P8, UFS_TX_HS_DEEMPHASIS_DB_1P6}, + [UFS_TX_EQ_PRESET_P7] = {UFS_TX_HS_PRESHOOT_DB_1P6, UFS_TX_HS_DEEMPHASIS_DB_0P8}, +}; + +/* + * pa_peer_rx_adapt_initial - Table of UniPro PA_PeerRxHSGnAdaptInitial + * attribute IDs for High Speed (HS) Gears. + * + * This table maps HS Gears to their respective UniPro PA_PeerRxHSGnAdaptInitial + * attribute IDs. Entries for Gears 1-3 are 0 (unsupported). + */ +static const u32 pa_peer_rx_adapt_initial[UFS_HS_GEAR_MAX] = { + 0, + 0, + 0, + PA_PEERRXHSG4ADAPTINITIAL, + PA_PEERRXHSG5ADAPTINITIAL, + PA_PEERRXHSG6ADAPTINITIALL0L3 +}; + +/* + * rx_adapt_initial_cap - Table of M-PHY RX_HS_Gn_ADAPT_INITIAL_Capability + * attribute IDs for High Speed (HS) Gears. + * + * This table maps HS Gears to their respective M-PHY + * RX_HS_Gn_ADAPT_INITIAL_Capability attribute IDs. Entries for Gears 1-3 are 0 + * (unsupported). + */ +static const u32 rx_adapt_initial_cap[UFS_HS_GEAR_MAX] = { + 0, + 0, + 0, + RX_HS_G4_ADAPT_INITIAL_CAP, + RX_HS_G5_ADAPT_INITIAL_CAP, + RX_HS_G6_ADAPT_INITIAL_CAP +}; + +/* + * pa_tx_eq_setting - Table of UniPro PA_TxEQGnSetting attribute IDs for High + * Speed (HS) Gears. + * + * This table maps HS Gears to their respective UniPro PA_TxEQGnSetting + * attribute IDs. + */ +static const u32 pa_tx_eq_setting[UFS_HS_GEAR_MAX] = { + PA_TXEQG1SETTING, + PA_TXEQG2SETTING, + PA_TXEQG3SETTING, + PA_TXEQG4SETTING, + PA_TXEQG5SETTING, + PA_TXEQG6SETTING +}; + +/** + * ufshcd_configure_precoding - Configure Pre-Coding for all active lanes + * @hba: per adapter instance + * @params: TX EQ parameters data structure + * + * Bit[7] in RX_FOM indicates that the receiver needs to enable Pre-Coding when + * set. Pre-Coding must be enabled on both the transmitter and receiver to + * ensure proper operation. + * + * Returns 0 on success, non-zero error code otherwise + */ +static int ufshcd_configure_precoding(struct ufs_hba *hba, + struct ufshcd_tx_eq_params *params) +{ + struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; + u32 local_precode_en = 0; + u32 peer_precode_en = 0; + int lane, ret; + + /* Enable Pre-Coding for Host's TX & Device's RX pair */ + for (lane = 0; lane < pwr_info->lane_tx; lane++) { + if (params->host[lane].precode_en) { + local_precode_en |= PRECODEEN_TX_BIT(lane); + peer_precode_en |= PRECODEEN_RX_BIT(lane); + } + } + + /* Enable Pre-Coding for Device's TX & Host's RX pair */ + for (lane = 0; lane < pwr_info->lane_rx; lane++) { + if (params->device[lane].precode_en) { + peer_precode_en |= PRECODEEN_TX_BIT(lane); + local_precode_en |= PRECODEEN_RX_BIT(lane); + } + } + + if (!local_precode_en && !peer_precode_en) { + dev_dbg(hba->dev, "Pre-Coding is not required for Host and Device\n"); + return 0; + } + + /* Set local PA_PreCodeEn */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PRECODEEN), local_precode_en); + if (ret) { + dev_err(hba->dev, "Failed to set local PA_PreCodeEn: %d\n", ret); + return ret; + } + + /* Set peer PA_PreCodeEn */ + ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_PRECODEEN), peer_precode_en); + if (ret) { + dev_err(hba->dev, "Failed to set peer PA_PreCodeEn: %d\n", ret); + return ret; + } + + dev_dbg(hba->dev, "Local PA_PreCodeEn: 0x%02x, Peer PA_PreCodeEn: 0x%02x\n", + local_precode_en, peer_precode_en); + + return 0; +} + +void ufshcd_print_tx_eq_params(struct ufs_hba *hba) +{ + struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; + struct ufshcd_tx_eq_params *params; + u32 gear = hba->pwr_info.gear_tx; + int lane; + + if (!ufshcd_is_tx_eq_supported(hba)) + return; + + if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) + return; + + params = &hba->tx_eq_params[gear - 1]; + if (!params->is_valid || !params->is_applied) + return; + + for (lane = 0; lane < pwr_info->lane_tx; lane++) + dev_dbg(hba->dev, "Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n", + lane, params->host[lane].preshoot, + params->host[lane].deemphasis, + params->host[lane].fom_val, + params->host[lane].precode_en); + + for (lane = 0; lane < pwr_info->lane_rx; lane++) + dev_dbg(hba->dev, "Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM %u, PreCodeEn %d\n", + lane, params->device[lane].preshoot, + params->device[lane].deemphasis, + params->device[lane].fom_val, + params->device[lane].precode_en); +} + +static inline u32 +ufshcd_compose_tx_eq_setting(struct ufshcd_tx_eq_settings *settings, + int num_lanes) +{ + u32 setting = 0; + int lane; + + for (lane = 0; lane < num_lanes; lane++, settings++) { + setting |= TX_HS_PRESHOOT_BITS(lane, settings->preshoot); + setting |= TX_HS_DEEMPHASIS_BITS(lane, settings->deemphasis); + } + + return setting; +} + +/** + * ufshcd_apply_tx_eq_settings - Apply TX Equalization settings for target gear + * @hba: per adapter instance + * @params: TX EQ parameters data structure + * @gear: target gear + * + * Returns 0 on success, negative error code otherwise + */ +static int ufshcd_apply_tx_eq_settings(struct ufs_hba *hba, + struct ufshcd_tx_eq_params *params, + u32 gear) +{ + struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; + u32 setting; + int ret; + + /* Compose settings for Host's TX Lanes */ + setting = ufshcd_compose_tx_eq_setting(params->host, pwr_info->lane_tx); + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting); + if (ret) + return ret; + + /* Compose settings for Device's TX Lanes */ + setting = ufshcd_compose_tx_eq_setting(params->device, pwr_info->lane_rx); + ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(pa_tx_eq_setting[gear - 1]), setting); + if (ret) + return ret; + + /* Configure Pre-Coding */ + if (gear >= UFS_HS_G6) { + ret = ufshcd_configure_precoding(hba, params); + if (ret) { + dev_err(hba->dev, "Failed to configure pre-coding: %d\n", ret); + return ret; + } + } + + return 0; +} + +/** + * ufshcd_evaluate_tx_eqtr_fom - Evaluate TX EQTR FOM results + * @hba: per adapter instance + * @pwr_mode: target power mode containing gear and rate information + * @eqtr_data: TX EQTR data structure + * @h_iter: host TX EQTR iterator data structure + * @d_iter: device TX EQTR iterator data structure + * + * Evaluate TX EQTR FOM results, update host and device TX EQTR data accordingy + * if FOM have been improved compared to previous iteration, and record TX EQTR + * FOM results. + */ +static void ufshcd_evaluate_tx_eqtr_fom(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct ufshcd_tx_eqtr_data *eqtr_data, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + u8 preshoot, deemphasis, fom_value; + bool precode_en; + int lane; + + for (lane = 0; h_iter->is_updated && lane < pwr_mode->lane_tx; lane++) { + preshoot = h_iter->preshoot; + deemphasis = h_iter->deemphasis; + fom_value = h_iter->fom[lane] & RX_FOM_VALUE_MASK; + precode_en = h_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT; + + /* Record host TX EQTR FOM */ + eqtr_data->host_fom[lane][preshoot][deemphasis] = h_iter->fom[lane]; + + /* Check if FOM has been improved for host's TX Lanes */ + if (fom_value > eqtr_data->host[lane].fom_val) { + eqtr_data->host[lane].preshoot = preshoot; + eqtr_data->host[lane].deemphasis = deemphasis; + eqtr_data->host[lane].fom_val = fom_value; + eqtr_data->host[lane].precode_en = precode_en; + } + + dev_dbg(hba->dev, "TX EQTR: Host TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n", + lane, preshoot, deemphasis, fom_value, precode_en); + } + + for (lane = 0; d_iter->is_updated && lane < pwr_mode->lane_rx; lane++) { + preshoot = d_iter->preshoot; + deemphasis = d_iter->deemphasis; + fom_value = d_iter->fom[lane] & RX_FOM_VALUE_MASK; + precode_en = d_iter->fom[lane] & RX_FOM_PRECODING_EN_BIT; + + /* Record device TX EQTR FOM */ + eqtr_data->device_fom[lane][preshoot][deemphasis] = d_iter->fom[lane]; + + /* Check if FOM has been improved for Device's TX Lanes */ + if (fom_value > eqtr_data->device[lane].fom_val) { + eqtr_data->device[lane].preshoot = preshoot; + eqtr_data->device[lane].deemphasis = deemphasis; + eqtr_data->device[lane].fom_val = fom_value; + eqtr_data->device[lane].precode_en = precode_en; + } + + dev_dbg(hba->dev, "TX EQTR: Device TX Lane %d: PreShoot %u, DeEmphasis %u, FOM value %u, PreCodeEn %d\n", + lane, preshoot, deemphasis, fom_value, precode_en); + } +} + +/** + * ufshcd_get_rx_fom - Get Figure of Merit (FOM) for both sides + * @hba: per adapter instance + * @pwr_mode: target power mode containing gear and rate information + * @h_iter: host TX EQTR iterator data structure + * @d_iter: device TX EQTR iterator data structure + * + * Returns 0 on success, negative error code otherwise + */ +static int ufshcd_get_rx_fom(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + int lane, ret; + u32 fom; + + /* Get FOM of host's TX lanes from device's RX_FOM. */ + for (lane = 0; lane < pwr_mode->lane_tx; lane++) { + ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB_SEL(RX_FOM, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + &fom); + if (ret) + return ret; + + h_iter->fom[lane] = (u8)fom; + } + + /* Get FOM of device's TX lanes from host's RX_FOM. */ + for (lane = 0; lane < pwr_mode->lane_rx; lane++) { + ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_FOM, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + &fom); + if (ret) + return ret; + + d_iter->fom[lane] = (u8)fom; + } + + ret = ufshcd_vops_get_rx_fom(hba, pwr_mode, h_iter, d_iter); + if (ret) + dev_err(hba->dev, "Failed to get FOM via vops: %d\n", ret); + + return ret; +} + +static bool ufshcd_is_txeq_preset_selected(u8 preshoot, u8 deemphasis) +{ + int i; + + for (i = 0; i < UFS_TX_EQ_PRESET_MAX; i++) { + if (!txeq_presets_selected[i]) + continue; + + if (preshoot == ufs_tx_eq_preset[i].preshoot && + deemphasis == ufs_tx_eq_preset[i].deemphasis) + return true; + } + + return false; +} + +/** + * tx_eqtr_iter_try_update - Try to update a TX EQTR iterator + * @iter: TX EQTR iterator data structure + * @preshoot: PreShoot value + * @deemphasis: DeEmphasis value + * + * This function validates whether the provided PreShoot and DeEmphasis + * combination can be used or not. If yes, it updates the TX EQTR iterator with + * the provided PreShoot and DeEmphasis, it also sets the is_updated flag + * to indicate the iterator has been updated. + */ +static void tx_eqtr_iter_try_update(struct tx_eqtr_iter *iter, + u8 preshoot, u8 deemphasis) +{ + if (!test_bit(preshoot, &iter->preshoot_bitmap) || + !test_bit(deemphasis, &iter->deemphasis_bitmap) || + (use_txeq_presets && !ufshcd_is_txeq_preset_selected(preshoot, deemphasis))) { + iter->is_updated = false; + return; + } + + iter->preshoot = preshoot; + iter->deemphasis = deemphasis; + iter->is_updated = true; +} + +/** + * tx_eqtr_iter_update() - Update host and deviceTX EQTR iterators + * @preshoot: PreShoot value + * @deemphasis: DeEmphasis value + * @h_iter: Host TX EQTR iterator data structure + * @d_iter: Device TX EQTR iterator data structure + * + * Updates host and device TX Equalization training iterators with the + * provided PreShoot and DeEmphasis. + * + * Return: true if host and/or device TX Equalization training iterator has + * been updated to the provided PreShoot and DeEmphasis, false otherwise. + */ +static bool tx_eqtr_iter_update(u8 preshoot, u8 deemphasis, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + tx_eqtr_iter_try_update(h_iter, preshoot, deemphasis); + tx_eqtr_iter_try_update(d_iter, preshoot, deemphasis); + + return h_iter->is_updated || d_iter->is_updated; +} + +/** + * ufshcd_tx_eqtr_iter_init - Initialize host and device TX EQTR iterators + * @hba: per adapter instance + * @h_iter: host TX EQTR iterator data structure + * @d_iter: device TX EQTR iterator data structure + * + * This function initializes the TX EQTR iterator structures for both host and + * device by reading their TX equalization capabilities. The capabilities are + * cached in the hba structure to avoid redundant DME operations in subsequent + * calls. In the TX EQTR procedure, the iterator structures are updated by + * tx_eqtr_iter_update() to systematically iterate through supported TX + * Equalization setting combinations. + * + * Returns 0 on success, negative error code otherwise + */ +static int ufshcd_tx_eqtr_iter_init(struct ufs_hba *hba, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + u32 cap; + int ret; + + if (!hba->host_preshoot_cap) { + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap); + if (ret) + return ret; + + hba->host_preshoot_cap = cap & TX_EQTR_CAP_MASK; + } + + if (!hba->host_deemphasis_cap) { + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap); + if (ret) + return ret; + + hba->host_deemphasis_cap = cap & TX_EQTR_CAP_MASK; + } + + if (!hba->device_preshoot_cap) { + ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_PRESHOOT_SETTING_CAP), &cap); + if (ret) + return ret; + + hba->device_preshoot_cap = cap & TX_EQTR_CAP_MASK; + } + + if (!hba->device_deemphasis_cap) { + ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(TX_HS_DEEMPHASIS_SETTING_CAP), &cap); + if (ret) + return ret; + + hba->device_deemphasis_cap = cap & TX_EQTR_CAP_MASK; + } + + /* + * Support PreShoot & DeEmphasis of value 0 is mandatory, hence they are + * not reflected in PreShoot/DeEmphasis capabilities. Left shift the + * capability bitmap by 1 and set bit[0] to reflect value 0 is + * supported, such that test_bit() can be used later for convenience. + */ + h_iter->preshoot_bitmap = (hba->host_preshoot_cap << 0x1) | 0x1; + h_iter->deemphasis_bitmap = (hba->host_deemphasis_cap << 0x1) | 0x1; + d_iter->preshoot_bitmap = (hba->device_preshoot_cap << 0x1) | 0x1; + d_iter->deemphasis_bitmap = (hba->device_deemphasis_cap << 0x1) | 0x1; + + return 0; +} + +/** + * adapt_cap_to_t_adapt - Calculate TAdapt from adapt capability + * @adapt_cap: Adapt capability + * + * For NRZ: + * IF (ADAPT_range = FINE) + * TADAPT = 650 x (ADAPT_length + 1) + * ELSE (IF ADAPT_range = COARSE) + * TADAPT = 650 x 2^ADAPT_length + * + * Returns calculated TAdapt value in term of Unit Intervals (UI) + */ +static inline u64 adapt_cap_to_t_adapt(u32 adapt_cap) +{ + u64 tadapt; + u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; + + if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) + tadapt = TADAPT_FACTOR * (adapt_length + 1); + else + tadapt = TADAPT_FACTOR * (1 << adapt_length); + + return tadapt; +} + +/** + * adapt_cap_to_t_adapt_l0l3 - Calculate TAdapt_L0_L3 from adapt capability + * @adapt_cap: Adapt capability + * + * For PAM-4: + * IF (ADAPT_range = FINE) + * TADAPT_L0_L3 = 2^9 x ADAPT_length + * ELSE IF (ADAPT_range = COARSE) + * TADAPT_L0_L3 = 2^9 x (2^ADAPT_length) + * + * Returns calculated TAdapt value in term of Unit Intervals (UI) + */ +static inline u64 adapt_cap_to_t_adapt_l0l3(u32 adapt_cap) +{ + u64 tadapt; + u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; + + if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) + tadapt = TADAPT_L0L3_FACTOR * adapt_length; + else + tadapt = TADAPT_L0L3_FACTOR * (1 << adapt_length); + + return tadapt; +} + +/** + * adapt_cap_to_t_adapt_l0l1l2l3 - Calculate TAdapt_L0_L1_L2_L3 from adapt capability + * @adapt_cap: Adapt capability + * + * For PAM-4: + * IF (ADAPT_range_L0_L1_L2_L3 = FINE) + * TADAPT_L0_L1_L2_L3 = 2^15 x (ADAPT_length_L0_L1_L2_L3 + 1) + * ELSE IF (ADAPT_range_L0_L1_L2_L3 = COARSE) + * TADAPT_L0_L1_L2_L3 = 2^15 x 2^ADAPT_length_L0_L1_L2_L3 + * + * Returns calculated TAdapt value in term of Unit Intervals (UI) + */ +static inline u64 adapt_cap_to_t_adapt_l0l1l2l3(u32 adapt_cap) +{ + u64 tadapt; + u8 adapt_length = adapt_cap & ADAPT_LENGTH_MASK; + + if (!IS_ADAPT_RANGE_COARSE(adapt_cap)) + tadapt = TADAPT_L0L1L2L3_FACTOR * (adapt_length + 1); + else + tadapt = TADAPT_L0L1L2L3_FACTOR * (1 << adapt_length); + + return tadapt; +} + +/** + * ufshcd_setup_tx_eqtr_adapt_length - Setup TX adapt length for EQTR + * @hba: per adapter instance + * @params: TX EQ parameters data structure + * @gear: target gear for EQTR + * + * This function determines and configures the proper TX adapt length (TAdapt) + * for the TX EQTR procedure based on the target gear and RX adapt capabilities + * of both host and device. + * + * Guidelines from MIPI UniPro v3.0 spec - select the minimum Adapt Length for + * the Equalization Training procedure based on the following conditions: + * + * If the target High-Speed Gear n is HS-G4 or HS-G5: + * PA_TxAdaptLength_EQTR[7:0] >= Max (10us, RX_HS_Gn_ADAPT_INITIAL_Capability, + * PA_PeerRxHsGnAdaptInitial) + * PA_TxAdaptLength_EQTR[7:0] shall be shorter than PACP_REQUEST_TIMER (10ms) + * PA_TxAdaptLength_EQTR[15:8] is not relevant for HS-G4 and HS-G5. This field + * is set to 255 (reserved value). + * + * If the target High-Speed Gear n is HS-G6: + * PA_TxAdapthLength_EQTR >= 10us + * PA_TxAdapthLength_EQTR[7:0] >= Max (RX_HS_G6_ADAPT_INITIAL_Capability, + * PA_PeerRxHsG6AdaptInitialL0L3) + * PA_TxAdapthLength_EQTR[15:8] >= Max (RX_HS_G6_ADAPT_INITIAL_L0_L1_L2_L3_Capability, + * PA_PeerRxHsG6AdaptInitialL0L1L2L3) + * PA_TxAdaptLength_EQTR shall be shorter than PACP_REQUEST_TIMER value of 10ms. + * + * Since adapt capabilities encode both range (fine/coarse) and length values, + * direct comparison is not possible. This function converts adapt capabilities + * to actual time durations in Unit Intervals (UI) using the Adapt time + * calculation formular in M-PHY v6.0 spec (Table 8), then selects the maximum + * to ensure both host and device use adequate TX adapt length. + * + * Returns 0 on success, negative error code otherwise + */ +static int ufshcd_setup_tx_eqtr_adapt_length(struct ufs_hba *hba, + struct ufshcd_tx_eq_params *params, + u32 gear) +{ + u32 adapt_eqtr; + int ret; + + if (gear == UFS_HS_G4 || gear == UFS_HS_G5) { + u64 t_adapt, t_adapt_local, t_adapt_peer; + u32 adapt_cap_local, adapt_cap_peer, adapt_length; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1], + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), + &adapt_cap_local); + if (ret) + return ret; + + if (adapt_cap_local > ADAPT_LENGTH_MAX) { + dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", + gear, adapt_cap_local); + return -EINVAL; + } + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]), + &adapt_cap_peer); + if (ret) + return ret; + + if (adapt_cap_peer > ADAPT_LENGTH_MAX) { + dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", + gear, adapt_cap_peer); + return -EINVAL; + } + + t_adapt_local = adapt_cap_to_t_adapt(adapt_cap_local); + t_adapt_peer = adapt_cap_to_t_adapt(adapt_cap_peer); + t_adapt = max(t_adapt_local, t_adapt_peer); + + dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", + gear, adapt_cap_local); + dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", + gear, adapt_cap_peer); + dev_dbg(hba->dev, "t_adapt_local = %llu UI, t_adapt_peer = %llu UI\n", + t_adapt_local, t_adapt_peer); + dev_dbg(hba->dev, "TAdapt %llu UI selected for TX EQTR\n", + t_adapt); + + adapt_length = (t_adapt_local >= t_adapt_peer) ? + adapt_cap_local : adapt_cap_peer; + + if (gear == UFS_HS_G4 && t_adapt < TX_EQTR_HS_G4_MIN_T_ADAPT) { + dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", + t_adapt, gear, TX_EQTR_HS_G4_ADAPT_DEFAULT); + adapt_length = TX_EQTR_HS_G4_ADAPT_DEFAULT; + } else if (gear == UFS_HS_G5 && t_adapt < TX_EQTR_HS_G5_MIN_T_ADAPT) { + dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", + t_adapt, gear, TX_EQTR_HS_G5_ADAPT_DEFAULT); + adapt_length = TX_EQTR_HS_G5_ADAPT_DEFAULT; + } + + adapt_eqtr = adapt_length | + (TX_EQTR_ADAPT_RESERVED << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT); + } else if (gear == UFS_HS_G6) { + u64 t_adapt, t_adapt_l0l3, t_adapt_l0l3_local, t_adapt_l0l3_peer; + u64 t_adapt_l0l1l2l3, t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer; + u32 adapt_l0l3_cap_local, adapt_l0l3_cap_peer, adapt_length_l0l3; + u32 adapt_l0l1l2l3_cap_local, adapt_l0l1l2l3_cap_peer, adapt_length_l0l1l2l3; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(rx_adapt_initial_cap[gear - 1], + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), + &adapt_l0l3_cap_local); + if (ret) + return ret; + + if (adapt_l0l3_cap_local > ADAPT_L0L3_LENGTH_MAX) { + dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", + gear, adapt_l0l3_cap_local); + return -EINVAL; + } + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(pa_peer_rx_adapt_initial[gear - 1]), + &adapt_l0l3_cap_peer); + if (ret) + return ret; + + if (adapt_l0l3_cap_peer > ADAPT_L0L3_LENGTH_MAX) { + dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP (0x%x) exceeds MAX\n", + gear, adapt_l0l3_cap_peer); + return -EINVAL; + } + + t_adapt_l0l3_local = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_local); + t_adapt_l0l3_peer = adapt_cap_to_t_adapt_l0l3(adapt_l0l3_cap_peer); + + dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", + gear, adapt_l0l3_cap_local); + dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_CAP = 0x%x\n", + gear, adapt_l0l3_cap_peer); + dev_dbg(hba->dev, "t_adapt_l0l3_local = %llu UI, t_adapt_l0l3_peer = %llu UI\n", + t_adapt_l0l3_local, t_adapt_l0l3_peer); + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_HS_G6_ADAPT_INITIAL_L0L1L2L3_CAP, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), + &adapt_l0l1l2l3_cap_local); + if (ret) + return ret; + + if (adapt_l0l1l2l3_cap_local > ADAPT_L0L1L2L3_LENGTH_MAX) { + dev_err(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n", + gear, adapt_l0l1l2l3_cap_local); + return -EINVAL; + } + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), + &adapt_l0l1l2l3_cap_peer); + if (ret) + return ret; + + if (adapt_l0l1l2l3_cap_peer > ADAPT_L0L1L2L3_LENGTH_MAX) { + dev_err(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP (0x%x) exceeds MAX\n", + gear, adapt_l0l1l2l3_cap_peer); + return -EINVAL; + } + + t_adapt_l0l1l2l3_local = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_local); + t_adapt_l0l1l2l3_peer = adapt_cap_to_t_adapt_l0l1l2l3(adapt_l0l1l2l3_cap_peer); + + dev_dbg(hba->dev, "local RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n", + gear, adapt_l0l1l2l3_cap_local); + dev_dbg(hba->dev, "peer RX_HS_G%u_ADAPT_INITIAL_L0L1L2L3_CAP = 0x%x\n", + gear, adapt_l0l1l2l3_cap_peer); + dev_dbg(hba->dev, "t_adapt_l0l1l2l3_local = %llu UI, t_adapt_l0l1l2l3_peer = %llu UI\n", + t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer); + + t_adapt_l0l1l2l3 = max(t_adapt_l0l1l2l3_local, t_adapt_l0l1l2l3_peer); + t_adapt_l0l3 = max(t_adapt_l0l3_local, t_adapt_l0l3_peer); + t_adapt = t_adapt_l0l3 + t_adapt_l0l1l2l3; + + dev_dbg(hba->dev, "TAdapt %llu PAM-4 UI selected for TX EQTR\n", + t_adapt); + + adapt_length_l0l3 = (t_adapt_l0l3_local >= t_adapt_l0l3_peer) ? + adapt_l0l3_cap_local : adapt_l0l3_cap_peer; + adapt_length_l0l1l2l3 = (t_adapt_l0l1l2l3_local >= t_adapt_l0l1l2l3_peer) ? + adapt_l0l1l2l3_cap_local : adapt_l0l1l2l3_cap_peer; + + if (t_adapt < TX_EQTR_HS_G6_MIN_T_ADAPT) { + dev_dbg(hba->dev, "TAdapt %llu UI is too short for TX EQTR for HS-G%u, use default Adapt 0x%x\n", + t_adapt, gear, TX_EQTR_HS_G6_ADAPT_DEFAULT); + adapt_length_l0l3 = TX_EQTR_HS_G6_ADAPT_DEFAULT; + } + + adapt_eqtr = adapt_length_l0l3 | + (adapt_length_l0l1l2l3 << TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT); + } else { + return -EINVAL; + } + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXADAPTLENGTH_EQTR), adapt_eqtr); + if (ret) + dev_err(hba->dev, "Failed to set adapt length for TX EQTR: %d\n", ret); + else + dev_dbg(hba->dev, "PA_TXADAPTLENGTH_EQTR configured to 0x%08x\n", adapt_eqtr); + + return ret; +} + +/** + * ufshcd_compose_tx_eqtr_setting - Compose TX EQTR setting + * @iter: TX EQTR iterator data structure + * @num_lanes: number of active lanes + * + * Returns composed TX EQTR setting, same setting is used for all active lanes + */ +static inline u32 ufshcd_compose_tx_eqtr_setting(struct tx_eqtr_iter *iter, + int num_lanes) +{ + u32 setting = 0; + int lane; + + for (lane = 0; lane < num_lanes; lane++) { + setting |= TX_HS_PRESHOOT_BITS(lane, iter->preshoot); + setting |= TX_HS_DEEMPHASIS_BITS(lane, iter->deemphasis); + } + + return setting; +} + +/** + * ufshcd_apply_tx_eqtr_settings - Apply TX EQTR setting + * @hba: per adapter instance + * @pwr_mode: target power mode containing gear and rate information + * @h_iter: host TX EQTR iterator data structure + * @d_iter: device TX EQTR iterator data structure + * + * Returns 0 on success, negative error code otherwise + */ +static int ufshcd_apply_tx_eqtr_settings(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + u32 setting; + int ret; + + setting = ufshcd_compose_tx_eqtr_setting(h_iter, pwr_mode->lane_tx); + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQTRSETTING), setting); + if (ret) + return ret; + + setting = ufshcd_compose_tx_eqtr_setting(d_iter, pwr_mode->lane_rx); + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERTXEQTRSETTING), setting); + if (ret) + return ret; + + ret = ufshcd_vops_apply_tx_eqtr_settings(hba, pwr_mode, h_iter, d_iter); + + return ret; +} + +/** + * ufshcd_update_tx_eq_params - Update TX Equalization params + * @params: TX EQ parameters data structure + * @eqtr_data: TX EQTR data structure + * + * Update TX Equalization params using results from TX EQTR data. + */ +static inline void +ufshcd_update_tx_eq_params(struct ufshcd_tx_eq_params *params, + struct ufshcd_tx_eqtr_data *eqtr_data) +{ + struct ufshcd_tx_eqtr_record *rec = params->eqtr_record; + + memcpy(params->host, eqtr_data->host, sizeof(params->host)); + memcpy(params->device, eqtr_data->device, sizeof(params->device)); + + if (!rec) + return; + + memcpy(rec->host_fom, eqtr_data->host_fom, sizeof(rec->host_fom)); + memcpy(rec->device_fom, eqtr_data->device_fom, sizeof(rec->device_fom)); + rec->last_record_ts = ktime_get(); + rec->last_record_index++; +} + +/** + * __ufshcd_tx_eqtr - TX Equalization Training (EQTR) procedure + * @hba: per adapter instance + * @params: TX EQ parameters data structure + * @pwr_mode: target power mode containing gear and rate information + * + * This function implements the complete TX EQTR procedure as defined in UFSHCI + * v5.0 specification. It iterates through all possible combinations of PreShoot + * and DeEmphasis settings to find the optimal TX Equalization settings for all + * active lanes. + * + * Returns 0 on success, negative error code otherwise + */ +static int __ufshcd_tx_eqtr(struct ufs_hba *hba, + struct ufshcd_tx_eq_params *params, + struct ufs_pa_layer_attr *pwr_mode) +{ + struct ufshcd_tx_eqtr_data *eqtr_data __free(kfree) = + kzalloc(sizeof(*eqtr_data), GFP_KERNEL); + struct tx_eqtr_iter h_iter = {}; + struct tx_eqtr_iter d_iter = {}; + u32 gear = pwr_mode->gear_tx; + u8 preshoot, deemphasis; + ktime_t start; + int ret; + + if (!eqtr_data) + return -ENOMEM; + + dev_info(hba->dev, "Start TX EQTR procedure for HS-G%u, Rate-%s, RX Lanes: %u, TX Lanes: %u\n", + gear, ufs_hs_rate_to_str(pwr_mode->hs_rate), + pwr_mode->lane_rx, pwr_mode->lane_tx); + + start = ktime_get(); + + /* Step 1 - Determine the TX Adapt Length for EQTR */ + ret = ufshcd_setup_tx_eqtr_adapt_length(hba, params, gear); + if (ret) { + dev_err(hba->dev, "Failed to setup TX EQTR Adaptation length: %d\n", ret); + return ret; + } + + /* Step 2 - Determine TX Equalization setting capabilities */ + ret = ufshcd_tx_eqtr_iter_init(hba, &h_iter, &d_iter); + if (ret) { + dev_err(hba->dev, "Failed to init TX EQTR data: %d\n", ret); + return ret; + } + + /* TX EQTR main loop */ + for (preshoot = 0; preshoot < TX_HS_NUM_PRESHOOT; preshoot++) { + for (deemphasis = 0; deemphasis < TX_HS_NUM_DEEMPHASIS; deemphasis++) { + if (!tx_eqtr_iter_update(preshoot, deemphasis, &h_iter, &d_iter)) + continue; + + /* Step 3 - Apply TX EQTR settings */ + ret = ufshcd_apply_tx_eqtr_settings(hba, pwr_mode, &h_iter, &d_iter); + if (ret) { + dev_err(hba->dev, "Failed to apply TX EQTR settings (PreShoot %u, DeEmphasis %u): %d\n", + preshoot, deemphasis, ret); + return ret; + } + + /* Step 4 - Trigger UIC TX EQTR */ + ret = ufshcd_uic_tx_eqtr(hba, gear); + if (ret) { + dev_err(hba->dev, "Failed to trigger UIC TX EQTR for target gear %u: %d\n", + gear, ret); + return ret; + } + + /* Step 5 - Get FOM */ + ret = ufshcd_get_rx_fom(hba, pwr_mode, &h_iter, &d_iter); + if (ret) { + dev_err(hba->dev, "Failed to get RX_FOM: %d\n", + ret); + return ret; + } + + ufshcd_evaluate_tx_eqtr_fom(hba, pwr_mode, eqtr_data, &h_iter, &d_iter); + } + } + + dev_info(hba->dev, "TX EQTR procedure completed! Time elapsed: %llu ms\n", + ktime_to_ms(ktime_sub(ktime_get(), start))); + + ufshcd_update_tx_eq_params(params, eqtr_data); + + return ret; +} + +/** + * ufshcd_tx_eqtr_prepare - Prepare UFS link for TX EQTR procedure + * @hba: per adapter instance + * @pwr_mode: target power mode containing gear and rate + * + * This function prepares the UFS link for TX Equalization Training (EQTR) by + * establishing the proper initial conditions required by the EQTR procedure. + * It ensures that EQTR starts from the most reliable Power Mode (HS-G1) with + * all connected lanes activated and sets host TX HS Adapt Type to INITIAL. + * + * Returns 0 on successful preparation, negative error code on failure + */ +static int ufshcd_tx_eqtr_prepare(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode) +{ + struct ufs_pa_layer_attr pwr_mode_hs_g1 = { + /* TX EQTR shall be initiated from the most reliable HS-G1 */ + .gear_rx = UFS_HS_G1, + .gear_tx = UFS_HS_G1, + .lane_rx = pwr_mode->lane_rx, + .lane_tx = pwr_mode->lane_tx, + .pwr_rx = FAST_MODE, + .pwr_tx = FAST_MODE, + /* Use the target power mode's HS rate */ + .hs_rate = pwr_mode->hs_rate, + }; + u32 rate = pwr_mode->hs_rate; + int ret; + + /* Change power mode to HS-G1, activate all connected lanes. */ + ret = ufshcd_change_power_mode(hba, &pwr_mode_hs_g1, + UFSHCD_PMC_POLICY_DONT_FORCE); + if (ret) { + dev_err(hba->dev, "TX EQTR: Failed to change power mode to HS-G1, Rate-%s: %d\n", + ufs_hs_rate_to_str(rate), ret); + return ret; + } + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), + PA_INITIAL_ADAPT); + if (ret) + dev_err(hba->dev, "TX EQTR: Failed to set Host Adapt type to INITIAL: %d\n", + ret); + + return ret; +} + +static void ufshcd_tx_eqtr_unprepare(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode) +{ + int err; + + if (pwr_mode->pwr_rx == SLOWAUTO_MODE || pwr_mode->hs_rate == 0) + return; + + err = ufshcd_change_power_mode(hba, pwr_mode, + UFSHCD_PMC_POLICY_DONT_FORCE); + if (err) + dev_err(hba->dev, "%s: Failed to restore Power Mode: %d\n", + __func__, err); +} + +/** + * ufshcd_tx_eqtr - Perform TX EQTR procedures with vops callbacks + * @hba: per adapter instance + * @params: TX EQ parameters data structure to populate + * @pwr_mode: target power mode containing gear and rate information + * + * This is the main entry point for performing TX Equalization Training (EQTR) + * procedure as defined in UFSCHI v5.0 specification. It serves as a wrapper + * around __ufshcd_tx_eqtr() to provide vops support through the variant + * operations framework. + * + * Returns 0 on success, negative error code on failure + */ +static int ufshcd_tx_eqtr(struct ufs_hba *hba, + struct ufshcd_tx_eq_params *params, + struct ufs_pa_layer_attr *pwr_mode) +{ + struct ufs_pa_layer_attr old_pwr_info; + int ret; + + if (!params->eqtr_record) { + params->eqtr_record = devm_kzalloc(hba->dev, + sizeof(*params->eqtr_record), + GFP_KERNEL); + if (!params->eqtr_record) + return -ENOMEM; + } + + memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); + + ret = ufshcd_tx_eqtr_prepare(hba, pwr_mode); + if (ret) { + dev_err(hba->dev, "Failed to prepare TX EQTR: %d\n", ret); + goto out; + } + + ret = ufshcd_vops_tx_eqtr_notify(hba, PRE_CHANGE, pwr_mode); + if (ret) + goto out; + + ret = __ufshcd_tx_eqtr(hba, params, pwr_mode); + if (ret) + goto out; + + ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode); + +out: + if (ret) + ufshcd_tx_eqtr_unprepare(hba, &old_pwr_info); + + return ret; +} + +/** + * ufshcd_config_tx_eq_settings - Configure TX Equalization settings + * @hba: per adapter instance + * @pwr_mode: target power mode containing gear and rate information + * + * This function finds and sets the TX Equalization settings for the given + * target power mode. + * + * Returns 0 on success, error code otherwise + */ +int ufshcd_config_tx_eq_settings(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode) +{ + struct ufshcd_tx_eq_params *params; + u32 gear, rate; + + if (!ufshcd_is_tx_eq_supported(hba) || !use_adaptive_txeq) + return 0; + + if (!hba->max_pwr_info.is_valid) { + dev_err(hba->dev, "Max power info is invalid\n"); + return -EINVAL; + } + + if (!pwr_mode) { + dev_err(hba->dev, "Target power mode is NULL\n"); + return -EINVAL; + } + + gear = pwr_mode->gear_tx; + rate = pwr_mode->hs_rate; + + if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) { + dev_err(hba->dev, "Invalid HS-Gear (%u) for TX Equalization\n", + gear); + return -EINVAL; + } else if (gear < max_t(u32, adaptive_txeq_gear, UFS_HS_G4)) { + /* TX EQTR is supported for HS-G4 and higher Gears */ + return 0; + } + + if (rate != PA_HS_MODE_A && rate != PA_HS_MODE_B) { + dev_err(hba->dev, "Invalid HS-Rate (%u) for TX Equalization\n", + rate); + return -EINVAL; + } + + params = &hba->tx_eq_params[gear - 1]; + if (!params->is_valid) { + int ret; + + ret = ufshcd_tx_eqtr(hba, params, pwr_mode); + if (ret) { + dev_err(hba->dev, "Failed to train TX Equalization for HS-G%u, Rate-%s: %d\n", + gear, ufs_hs_rate_to_str(rate), ret); + return ret; + } + + /* Mark TX Equalization settings as valid */ + params->is_valid = true; + params->is_applied = false; + } + + if (params->is_valid && !params->is_applied) { + int ret; + + ret = ufshcd_apply_tx_eq_settings(hba, params, gear); + if (ret) { + dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u, Rate-%s: %d\n", + gear, ufs_hs_rate_to_str(rate), ret); + return ret; + } + + params->is_applied = true; + } + + return 0; +} + +/** + * ufshcd_apply_valid_tx_eq_settings - Apply valid TX Equalization settings + * @hba: per-adapter instance + * + * This function iterates through all supported High-Speed (HS) gears and + * applies valid TX Equalization settings to both Host and Device. + */ +void ufshcd_apply_valid_tx_eq_settings(struct ufs_hba *hba) +{ + struct ufshcd_tx_eq_params *params; + int gear, err; + + if (!ufshcd_is_tx_eq_supported(hba)) + return; + + if (!hba->max_pwr_info.is_valid) { + dev_err(hba->dev, "Max power info is invalid, cannot apply TX Equalization settings\n"); + return; + } + + for (gear = UFS_HS_G1; gear <= UFS_HS_GEAR_MAX; gear++) { + params = &hba->tx_eq_params[gear - 1]; + + if (params->is_valid) { + err = ufshcd_apply_tx_eq_settings(hba, params, gear); + if (err) { + params->is_applied = false; + dev_err(hba->dev, "Failed to apply TX Equalization settings for HS-G%u: %d\n", + gear, err); + } else { + params->is_applied = true; + } + } + } +} diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index f1cec1cd01d2..c164caa9a825 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -103,6 +103,12 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable); int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id); +int ufshcd_uic_tx_eqtr(struct ufs_hba *hba, int gear); +void ufshcd_apply_valid_tx_eq_settings(struct ufs_hba *hba); +int ufshcd_config_tx_eq_settings(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode); +void ufshcd_print_tx_eq_params(struct ufs_hba *hba); + /* Wrapper functions for safely calling variant operations */ static inline const char *ufshcd_get_var_name(struct ufs_hba *hba) { @@ -297,6 +303,38 @@ static inline u32 ufshcd_vops_freq_to_gear_speed(struct ufs_hba *hba, unsigned l return 0; } +static inline int ufshcd_vops_get_rx_fom(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + if (hba->vops && hba->vops->get_rx_fom) + return hba->vops->get_rx_fom(hba, pwr_mode, h_iter, d_iter); + + return 0; +} + +static inline int ufshcd_vops_apply_tx_eqtr_settings(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + if (hba->vops && hba->vops->apply_tx_eqtr_settings) + return hba->vops->apply_tx_eqtr_settings(hba, pwr_mode, h_iter, d_iter); + + return 0; +} + +static inline int ufshcd_vops_tx_eqtr_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *pwr_mode) +{ + if (hba->vops && hba->vops->tx_eqtr_notify) + return hba->vops->tx_eqtr_notify(hba, status, pwr_mode); + + return 0; +} + extern const struct ufs_pm_lvl_states ufs_pm_lvl_states[]; /** diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 44faab8b1770..d78723dea951 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -4343,16 +4343,18 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) ret = __ufshcd_send_uic_cmd(hba, cmd); if (ret) { dev_err(hba->dev, - "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", - cmd->command, cmd->argument3, ret); + "pwr ctrl cmd 0x%x with (MIBattribute 0x%x, mode 0x%x) uic error %d\n", + cmd->command, UIC_GET_ATTR_ID(cmd->argument1), + cmd->argument3, ret); goto out; } if (!wait_for_completion_timeout(hba->uic_async_done, msecs_to_jiffies(uic_cmd_timeout))) { dev_err(hba->dev, - "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", - cmd->command, cmd->argument3); + "pwr ctrl cmd 0x%x with (MIBattribute 0x%x, mode 0x%x) completion timeout\n", + cmd->command, UIC_GET_ATTR_ID(cmd->argument1), + cmd->argument3); if (!cmd->cmd_active) { dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n", @@ -4368,14 +4370,16 @@ check_upmcrs: status = ufshcd_get_upmcrs(hba); if (status != PWR_LOCAL) { dev_err(hba->dev, - "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n", - cmd->command, status); + "pwr ctrl cmd 0x%x with (MIBattribute 0x%x, mode 0x%x) failed, host upmcrs:0x%x\n", + cmd->command, UIC_GET_ATTR_ID(cmd->argument1), + cmd->argument3, status); ret = (status != PWR_OK) ? status : -1; } out: if (ret) { ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); + ufshcd_print_tx_eq_params(hba); ufshcd_print_evt_hist(hba); } @@ -4401,6 +4405,29 @@ out_unlock: return ret; } +/** + * ufshcd_uic_tx_eqtr - Perform UIC TX Equalization Training + * @hba: per adapter instance + * @gear: target gear for EQTR + * + * Returns 0 on success, negative error code otherwise + */ +int ufshcd_uic_tx_eqtr(struct ufs_hba *hba, int gear) +{ + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_SET, + .argument1 = UIC_ARG_MIB(PA_EQTR_GEAR), + .argument3 = gear, + }; + int ret; + + ufshcd_hold(hba); + ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); + ufshcd_release(hba); + + return ret; +} + /** * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result * @hba: per adapter instance @@ -4824,6 +4851,12 @@ int ufshcd_config_pwr_mode(struct ufs_hba *hba, memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); } + ret = ufshcd_config_tx_eq_settings(hba, &final_params); + if (ret) + dev_warn(hba->dev, "Failed to configure TX Equalization for HS-G%u, Rate-%s: %d\n", + final_params.gear_tx, + ufs_hs_rate_to_str(final_params.hs_rate), ret); + return ufshcd_change_power_mode(hba, &final_params, pmc_policy); } EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); @@ -6823,6 +6856,7 @@ again: spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); + ufshcd_print_tx_eq_params(hba); ufshcd_print_evt_hist(hba); ufshcd_print_tmrs(hba, hba->outstanding_tasks); ufshcd_print_trs_all(hba, pr_prdt); @@ -7086,6 +7120,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); ufshcd_print_pwr_info(hba); + ufshcd_print_tx_eq_params(hba); } ufshcd_schedule_eh_work(hba); retval |= IRQ_HANDLED; @@ -7867,6 +7902,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); + ufshcd_print_tx_eq_params(hba); ufshcd_print_tr(hba, cmd, true); } else { ufshcd_print_tr(hba, cmd, false); @@ -8844,6 +8880,8 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) ufshcd_quirk_override_pa_h8time(hba); + + ufshcd_apply_valid_tx_eq_settings(hba); } static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 16facaee3e77..35b1288327d0 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -287,6 +287,84 @@ struct ufs_pwr_mode_info { struct ufs_pa_layer_attr info; }; +#define UFS_MAX_LANES 2 + +/** + * struct tx_eqtr_iter - TX Equalization Training iterator + * @preshoot_bitmap: PreShoot bitmap + * @deemphasis_bitmap: DeEmphasis bitmap + * @preshoot: PreShoot value + * @deemphasis: DeEmphasis value + * @fom: Figure-of-Merit read out from RX_FOM + * @is_updated: Flag to indicate if updated since previous iteration + */ +struct tx_eqtr_iter { + unsigned long preshoot_bitmap; + unsigned long deemphasis_bitmap; + u8 preshoot; + u8 deemphasis; + u8 fom[UFS_MAX_LANES]; + bool is_updated; +}; + +/** + * struct ufshcd_tx_eq_settings - TX Equalization settings + * @preshoot: PreShoot value + * @deemphasis: DeEmphasis value + * @fom_val: Figure-of-Merit value read out from RX_FOM (Bit[6:0]) + * @precode_en: Flag to indicate whether need to enable pre-coding + */ +struct ufshcd_tx_eq_settings { + u8 preshoot; + u8 deemphasis; + u8 fom_val; + bool precode_en; +}; + +/** + * struct ufshcd_tx_eqtr_data - Data used during TX Equalization Training procedure + * @host: Optimal TX EQ settings identified for host TX Lanes during TX EQTR + * @device: Optimal TX EQ settings identified for device TX Lanes during TX EQTR + * @host_fom: Host TX EQTR FOM record + * @device_fom: Device TX EQTR FOM record + */ +struct ufshcd_tx_eqtr_data { + struct ufshcd_tx_eq_settings host[UFS_MAX_LANES]; + struct ufshcd_tx_eq_settings device[UFS_MAX_LANES]; + u8 host_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS]; + u8 device_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS]; +}; + +/** + * struct ufshcd_tx_eqtr_record - TX Equalization Training record + * @host_fom: Host TX EQTR FOM record + * @device_fom: Device TX EQTR FOM record + * @last_record_ts: Timestamp of the most recent TX EQTR record + * @last_record_index: Index of the most recent TX EQTR record + */ +struct ufshcd_tx_eqtr_record { + u8 host_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS]; + u8 device_fom[UFS_MAX_LANES][TX_HS_NUM_PRESHOOT][TX_HS_NUM_DEEMPHASIS]; + ktime_t last_record_ts; + u16 last_record_index; +}; + +/** + * struct ufshcd_tx_eq_params - TX Equalization parameters structure + * @host: TX EQ settings for host TX Lanes + * @device: TX EQ settings for device TX Lanes + * @eqtr_record: Pointer to TX EQTR record + * @is_valid: True if parameter contains valid TX Equalization settings + * @is_applied: True if settings have been applied to UniPro of both sides + */ +struct ufshcd_tx_eq_params { + struct ufshcd_tx_eq_settings host[UFS_MAX_LANES]; + struct ufshcd_tx_eq_settings device[UFS_MAX_LANES]; + struct ufshcd_tx_eqtr_record *eqtr_record; + bool is_valid; + bool is_applied; +}; + /** * struct ufs_hba_variant_ops - variant specific callbacks * @name: variant name @@ -330,6 +408,11 @@ struct ufs_pwr_mode_info { * @config_esi: called to config Event Specific Interrupt * @config_scsi_dev: called to configure SCSI device parameters * @freq_to_gear_speed: called to map clock frequency to the max supported gear speed + * @apply_tx_eqtr_settings: called to apply settings for TX Equalization + * Training settings. + * @get_rx_fom: called to get Figure of Merit (FOM) value. + * @tx_eqtr_notify: called before and after TX Equalization Training procedure + * to allow platform vendor specific configs to take place. */ struct ufs_hba_variant_ops { const char *name; @@ -381,6 +464,17 @@ struct ufs_hba_variant_ops { int (*config_esi)(struct ufs_hba *hba); void (*config_scsi_dev)(struct scsi_device *sdev); u32 (*freq_to_gear_speed)(struct ufs_hba *hba, unsigned long freq); + int (*get_rx_fom)(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter); + int (*apply_tx_eqtr_settings)(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter); + int (*tx_eqtr_notify)(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *pwr_mode); }; /* clock gating state */ @@ -779,6 +873,13 @@ enum ufshcd_caps { * WriteBooster when scaling the clock down. */ UFSHCD_CAP_WB_WITH_CLK_SCALING = 1 << 12, + + /* + * This capability allows the host controller driver to apply TX + * Equalization settings discovered from UFS attributes, variant + * specific operations and TX Equaliztion Training procedure. + */ + UFSHCD_CAP_TX_EQUALIZATION = 1 << 13, }; struct ufs_hba_variant_params { @@ -955,6 +1056,15 @@ enum ufshcd_mcq_opr { * @dev_lvl_exception_count: count of device level exceptions since last reset * @dev_lvl_exception_id: vendor specific information about the device level exception event. * @rpmbs: list of OP-TEE RPMB devices (one per RPMB region) + * @host_preshoot_cap: a bitfield to indicate supported PreShoot dBs of host's TX lanes, cache of + * host M-PHY TX_HS_PreShoot_Setting_Capability Attribute (ID 0x15) + * @host_deemphasis_cap: a bitfield to indicate supported DeEmphasis dBs of host's TX lanes, cache + * of host M-PHY TX_HS_DeEmphasis_Setting_Capability Attribute (ID 0x12) + * @device_preshoot_cap: a bitfield to indicate supported PreShoot dBs of device's TX lanes, cache + * of device M-PHY TX_HS_PreShoot_Setting_Capability Attribute (ID 0x15) + * @device_deemphasis_cap: a bitfield to indicate supported DeEmphasis dBs of device's TX lanes, + * cache of device M-PHY TX_HS_DeEmphasis_Setting_Capability Attribute (ID 0x12) + * @tx_eq_params: TX Equalization settings */ struct ufs_hba { void __iomem *mmio_base; @@ -1128,6 +1238,12 @@ struct ufs_hba { u64 dev_lvl_exception_id; u32 vcc_off_delay_us; struct list_head rpmbs; + + u8 host_preshoot_cap; + u8 host_deemphasis_cap; + u8 device_preshoot_cap; + u8 device_deemphasis_cap; + struct ufshcd_tx_eq_params tx_eq_params[UFS_HS_GEAR_MAX]; }; /** @@ -1272,6 +1388,13 @@ static inline bool ufshcd_enable_wb_if_scaling_up(struct ufs_hba *hba) return hba->caps & UFSHCD_CAP_WB_WITH_CLK_SCALING; } +static inline bool ufshcd_is_tx_eq_supported(struct ufs_hba *hba) +{ + return hba->caps & UFSHCD_CAP_TX_EQUALIZATION && + hba->ufs_version >= ufshci_version(5, 0) && + hba->dev_info.wspecversion >= 0x500; +} + #define ufsmcq_writel(hba, val, reg) \ writel((val), (hba)->mcq_base + (reg)) #define ufsmcq_readl(hba, reg) \ @@ -1287,6 +1410,18 @@ static inline bool ufshcd_enable_wb_if_scaling_up(struct ufs_hba *hba) #define ufshcd_readl(hba, reg) \ readl((hba)->mmio_base + (reg)) +static inline const char *ufs_hs_rate_to_str(enum ufs_hs_gear_rate rate) +{ + switch (rate) { + case PA_HS_MODE_A: + return "A"; + case PA_HS_MODE_B: + return "B"; + default: + return "Unknown"; + } +} + /** * ufshcd_rmwl - perform read/modify/write for a controller register * @hba: per adapter instance diff --git a/include/ufs/unipro.h b/include/ufs/unipro.h index 71a5f643400c..4aa592130b4e 100644 --- a/include/ufs/unipro.h +++ b/include/ufs/unipro.h @@ -10,6 +10,8 @@ * M-TX Configuration Attributes */ #define TX_HIBERN8TIME_CAPABILITY 0x000F +#define TX_HS_DEEMPHASIS_SETTING_CAP 0x0012 +#define TX_HS_PRESHOOT_SETTING_CAP 0x0015 #define TX_MODE 0x0021 #define TX_HSRATE_SERIES 0x0022 #define TX_HSGEAR 0x0023 @@ -38,6 +40,9 @@ /* * M-RX Configuration Attributes */ +#define RX_HS_G5_ADAPT_INITIAL_CAP 0x0074 +#define RX_HS_G6_ADAPT_INITIAL_CAP 0x007B +#define RX_HS_G6_ADAPT_INITIAL_L0L1L2L3_CAP 0x007D #define RX_HS_G1_SYNC_LENGTH_CAP 0x008B #define RX_HS_G1_PREP_LENGTH_CAP 0x008C #define RX_MIN_ACTIVATETIME_CAPABILITY 0x008F @@ -50,6 +55,7 @@ #define RX_HIBERN8TIME_CAP 0x0092 #define RX_ADV_HIBERN8TIME_CAP 0x0099 #define RX_ADV_MIN_ACTIVATETIME_CAP 0x009A +#define RX_HS_G4_ADAPT_INITIAL_CAP 0x009F #define RX_MODE 0x00A1 #define RX_HSRATE_SERIES 0x00A2 #define RX_HSGEAR 0x00A3 @@ -64,6 +70,7 @@ #define CFGRXCDR8 0x00BA #define CFGRXOVR8 0x00BD #define CFGRXOVR6 0x00BF +#define RX_FOM 0x00C2 #define RXDIRECTCTRL2 0x00C7 #define CFGRXOVR4 0x00E9 #define RX_REFCLKFREQ 0x00EB @@ -73,7 +80,6 @@ #define ENARXDIRECTCFG3 0x00F3 #define ENARXDIRECTCFG2 0x00F4 - #define is_mphy_tx_attr(attr) (attr < RX_MODE) #define RX_ADV_FINE_GRAN_STEP(x) ((((x) & 0x3) << 1) | 0x1) #define SYNC_LEN_FINE(x) ((x) & 0x3F) @@ -99,6 +105,18 @@ #define UNIPRO_CB_OFFSET(x) (0x8000 | x) +#define ADAPT_LENGTH_MASK 0x7F +#define ADAPT_RANGE_BIT BIT(7) +#define IS_ADAPT_RANGE_COARSE(x) ((x) & ADAPT_RANGE_BIT) + +/* Adapt definitions */ +#define ADAPT_LENGTH_MAX 0x91 +#define ADAPT_L0L3_LENGTH_MAX 0x90 +#define ADAPT_L0L1L2L3_LENGTH_MAX 0x8C +#define TADAPT_FACTOR 650 +#define TADAPT_L0L3_FACTOR (1 << 9) +#define TADAPT_L0L1L2L3_FACTOR (1 << 15) + /* * PHY Adapter attributes */ @@ -164,10 +182,26 @@ #define PA_PACPERRORCOUNT 0x15C1 #define PA_PHYTESTCONTROL 0x15C2 #define PA_TXHSG4SYNCLENGTH 0x15D0 +#define PA_PEERRXHSG4ADAPTINITIAL 0x15D3 #define PA_TXHSADAPTTYPE 0x15D4 #define PA_TXHSG5SYNCLENGTH 0x15D6 +#define PA_PEERRXHSG5ADAPTINITIAL 0x15D9 +#define PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3 0x15DE +#define PA_PEERRXHSG6ADAPTINITIALL0L3 0x15DF +#define PA_PEERRXHSG6ADAPTINITIALL0L1L2L3 0x15E0 +#define PA_TXEQG1SETTING 0x15E1 +#define PA_TXEQG2SETTING 0x15E2 +#define PA_TXEQG3SETTING 0x15E3 +#define PA_TXEQG4SETTING 0x15E4 +#define PA_TXEQG5SETTING 0x15E5 +#define PA_TXEQG6SETTING 0x15E6 +#define PA_TXEQTRSETTING 0x15E7 +#define PA_PEERTXEQTRSETTING 0x15E8 +#define PA_PRECODEEN 0x15E9 +#define PA_EQTR_GEAR 0x15EA +#define PA_TXADAPTLENGTH_EQTR 0x15EB -/* Adpat type for PA_TXHSADAPTTYPE attribute */ +/* Adapt type for PA_TXHSADAPTTYPE attribute */ #define PA_REFRESH_ADAPT 0x00 #define PA_INITIAL_ADAPT 0x01 #define PA_NO_ADAPT 0x03 @@ -187,6 +221,82 @@ /* PHY Adapter Protocol Constants */ #define PA_MAXDATALANES 4 +/* + * TX EQTR's minimum TAdapt should not be less than 10us. + * This value is rounded up into the nearest Unit Intervals (UI) + */ +#define TX_EQTR_HS_G4_MIN_T_ADAPT 166400 +#define TX_EQTR_HS_G5_MIN_T_ADAPT 332800 +#define TX_EQTR_HS_G6_MIN_T_ADAPT 262144 + +#define TX_EQTR_HS_G4_ADAPT_DEFAULT 0x88 +#define TX_EQTR_HS_G5_ADAPT_DEFAULT 0x89 +#define TX_EQTR_HS_G6_ADAPT_DEFAULT 0x89 + +#define TX_EQTR_CAP_MASK 0x7F + +#define TX_EQTR_ADAPT_LENGTH_L0L1L2L3_SHIFT 8 +#define TX_EQTR_ADAPT_RESERVED 0xFF + +#define TX_HS_NUM_PRESHOOT 8 +#define TX_HS_NUM_DEEMPHASIS 8 +#define TX_HS_PRESHOOT_SHIFT 4 +#define TX_HS_DEEMPHASIS_SHIFT 4 +#define TX_HS_PRESHOOT_OFFSET 0 +#define TX_HS_DEEMPHASIS_OFFSET 16 + +#define TX_HS_PRESHOOT_LANE_SHIFT(lane) \ + (TX_HS_PRESHOOT_OFFSET + (lane) * TX_HS_PRESHOOT_SHIFT) +#define TX_HS_DEEMPHASIS_LANE_SHIFT(lane) \ + (TX_HS_DEEMPHASIS_OFFSET + (lane) * TX_HS_DEEMPHASIS_SHIFT) + +#define TX_HS_PRESHOOT_BITS(lane, val) \ + ((val) << TX_HS_PRESHOOT_LANE_SHIFT(lane)) +#define TX_HS_DEEMPHASIS_BITS(lane, val) \ + ((val) << TX_HS_DEEMPHASIS_LANE_SHIFT(lane)) + +#define RX_FOM_VALUE_MASK 0x7F +#define RX_FOM_PRECODING_EN_BIT BIT(7) + +#define PRECODEEN_TX_OFFSET 0 +#define PRECODEEN_RX_OFFSET 4 +#define PRECODEEN_TX_BIT(lane) (1 << (PRECODEEN_TX_OFFSET + (lane))) +#define PRECODEEN_RX_BIT(lane) (1 << (PRECODEEN_RX_OFFSET + (lane))) + +enum ufs_tx_eq_preset { + UFS_TX_EQ_PRESET_P0, + UFS_TX_EQ_PRESET_P1, + UFS_TX_EQ_PRESET_P2, + UFS_TX_EQ_PRESET_P3, + UFS_TX_EQ_PRESET_P4, + UFS_TX_EQ_PRESET_P5, + UFS_TX_EQ_PRESET_P6, + UFS_TX_EQ_PRESET_P7, + UFS_TX_EQ_PRESET_MAX, +}; + +enum ufs_tx_hs_preshoot { + UFS_TX_HS_PRESHOOT_DB_0P0, + UFS_TX_HS_PRESHOOT_DB_0P4, + UFS_TX_HS_PRESHOOT_DB_0P8, + UFS_TX_HS_PRESHOOT_DB_1P2, + UFS_TX_HS_PRESHOOT_DB_1P6, + UFS_TX_HS_PRESHOOT_DB_2P5, + UFS_TX_HS_PRESHOOT_DB_3P5, + UFS_TX_HS_PRESHOOT_DB_4P7, +}; + +enum ufs_tx_hs_deemphasis { + UFS_TX_HS_DEEMPHASIS_DB_0P0, + UFS_TX_HS_DEEMPHASIS_DB_0P8, + UFS_TX_HS_DEEMPHASIS_DB_1P6, + UFS_TX_HS_DEEMPHASIS_DB_2P5, + UFS_TX_HS_DEEMPHASIS_DB_3P5, + UFS_TX_HS_DEEMPHASIS_DB_4P7, + UFS_TX_HS_DEEMPHASIS_DB_6P0, + UFS_TX_HS_DEEMPHASIS_DB_7P6, +}; + #define DL_FC0ProtectionTimeOutVal_Default 8191 #define DL_TC0ReplayTimeOutVal_Default 65535 #define DL_AFC0ReqTimeOutVal_Default 32767