armbian/linux-cix
0
0
Fork 0
mirror of https://github.com/armbian/linux-cix.git synced 2026-01-06 12:30:45 -08:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'cxl-for-5.19' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl

Browse Source 
Pull cxl updates from Dan Williams:
 "Compute Express Link (CXL) updates for this cycle.

  The highlight is new driver-core infrastructure and CXL subsystem
  changes for allowing lockdep to validate device_lock() usage. Thanks
  to PeterZ for setting me straight on the current capabilities of the
  lockdep API, and Greg acked it as well.

  On the CXL ACPI side this update adds support for CXL _OSC so that
  platform firmware knows that it is safe to still grant Linux native
  control of PCIe hotplug and error handling in the presence of CXL
  devices. A circular dependency problem was discovered between suspend
  and CXL memory for cases where the suspend image might be stored in
  CXL memory where that image also contains the PCI register state to
  restore to re-enable the device. Disable suspend for now until an
  architecture is defined to clarify that conflict.

  Lastly a collection of reworks, fixes, and cleanups to the CXL
  subsystem where support for snooping mailbox commands and properly
  handling the "mem_enable" flow are the highlights.

  Summary:

   - Add driver-core infrastructure for lockdep validation of
     device_lock(), and fixup a deadlock report that was previously
     hidden behind the 'lockdep no validate' policy.

   - Add CXL _OSC support for claiming native control of CXL hotplug and
     error handling.

   - Disable suspend in the presence of CXL memory unless and until a
     protocol is identified for restoring PCI device context from memory
     hosted on CXL PCI devices.

   - Add support for snooping CXL mailbox commands to protect against
     inopportune changes, like set-partition with the 'immediate' flag
     set.

   - Rework how the driver detects legacy CXL 1.1 configurations (CXL
     DVSEC / 'mem_enable') before enabling new CXL 2.0 decode
     configurations (CXL HDM Capability).

   - Miscellaneous cleanups and fixes from -next exposure"

* tag 'cxl-for-5.19' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl: (47 commits)
  cxl/port: Enable HDM Capability after validating DVSEC Ranges
  cxl/port: Reuse 'struct cxl_hdm' context for hdm init
  cxl/port: Move endpoint HDM Decoder Capability init to port driver
  cxl/pci: Drop @info argument to cxl_hdm_decode_init()
  cxl/mem: Merge cxl_dvsec_ranges() and cxl_hdm_decode_init()
  cxl/mem: Skip range enumeration if mem_enable clear
  cxl/mem: Consolidate CXL DVSEC Range enumeration in the core
  cxl/pci: Move cxl_await_media_ready() to the core
  cxl/mem: Validate port connectivity before dvsec ranges
  cxl/mem: Fix cxl_mem_probe() error exit
  cxl/pci: Drop wait_for_valid() from cxl_await_media_ready()
  cxl/pci: Consolidate wait_for_media() and wait_for_media_ready()
  cxl/mem: Drop mem_enabled check from wait_for_media()
  nvdimm: Fix firmware activation deadlock scenarios
  device-core: Kill the lockdep_mutex
  nvdimm: Drop nd_device_lock()
  ACPI: NFIT: Drop nfit_device_lock()
  nvdimm: Replace lockdep_mutex with local lock classes
  cxl: Drop cxl_device_lock()
  cxl/acpi: Add root device lockdep validation
  ...
This commit is contained in:
Linus Torvalds
2022-05-27 21:24:19 -07:00
parent a9f94826e4 34e37b4c43
commit 9d004b2f4f
48 changed files with 1282 additions and 879 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/Makefile
View File

@@ -72,9 +72,9 @@ obj-$(CONFIG_PARPORT) += parport/
obj-y += base/ block/ misc/ mfd/ nfc/
obj-$(CONFIG_LIBNVDIMM) += nvdimm/
obj-$(CONFIG_DAX) += dax/
obj-$(CONFIG_CXL_BUS) += cxl/
obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/
obj-$(CONFIG_NUBUS) += nubus/
obj-y += cxl/
obj-y += macintosh/
obj-y += scsi/
obj-y += nvme/

2
drivers/acpi/bus.c
View File

@@ -443,7 +443,7 @@ static void acpi_bus_osc_negotiate_usb_control(void)
}
osc_sb_native_usb4_control =
control & ((u32 *)context.ret.pointer)[OSC_CONTROL_DWORD];
control & acpi_osc_ctx_get_pci_control(&context);
acpi_bus_decode_usb_osc("USB4 _OSC: OS supports", control);
acpi_bus_decode_usb_osc("USB4 _OSC: OS controls",

30
drivers/acpi/nfit/core.c
View File

@@ -1230,7 +1230,7 @@ static ssize_t hw_error_scrub_store(struct device *dev,
if (rc)
return rc;
nfit_device_lock(dev);
device_lock(dev);
nd_desc = dev_get_drvdata(dev);
if (nd_desc) {
struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
@@ -1247,7 +1247,7 @@ static ssize_t hw_error_scrub_store(struct device *dev,
break;
}
}
nfit_device_unlock(dev);
device_unlock(dev);
if (rc)
return rc;
return size;
@@ -1267,10 +1267,10 @@ static ssize_t scrub_show(struct device *dev,
ssize_t rc = -ENXIO;
bool busy;
nfit_device_lock(dev);
device_lock(dev);
nd_desc = dev_get_drvdata(dev);
if (!nd_desc) {
nfit_device_unlock(dev);
device_unlock(dev);
return rc;
}
acpi_desc = to_acpi_desc(nd_desc);
@@ -1287,7 +1287,7 @@ static ssize_t scrub_show(struct device *dev,
}
mutex_unlock(&acpi_desc->init_mutex);
nfit_device_unlock(dev);
device_unlock(dev);
return rc;
}
@@ -1304,14 +1304,14 @@ static ssize_t scrub_store(struct device *dev,
if (val != 1)
return -EINVAL;
nfit_device_lock(dev);
device_lock(dev);
nd_desc = dev_get_drvdata(dev);
if (nd_desc) {
struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
rc = acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
}
nfit_device_unlock(dev);
device_unlock(dev);
if (rc)
return rc;
return size;
@@ -1697,9 +1697,9 @@ static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
struct acpi_device *adev = data;
struct device *dev = &adev->dev;
nfit_device_lock(dev->parent);
device_lock(dev->parent);
__acpi_nvdimm_notify(dev, event);
nfit_device_unlock(dev->parent);
device_unlock(dev->parent);
}
static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method)
@@ -3152,8 +3152,8 @@ static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
struct device *dev = acpi_desc->dev;
/* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
nfit_device_lock(dev);
nfit_device_unlock(dev);
device_lock(dev);
device_unlock(dev);
/* Bounce the init_mutex to complete initial registration */
mutex_lock(&acpi_desc->init_mutex);
@@ -3305,8 +3305,8 @@ void acpi_nfit_shutdown(void *data)
* acpi_nfit_ars_rescan() submissions have had a chance to
* either submit or see ->cancel set.
*/
nfit_device_lock(bus_dev);
nfit_device_unlock(bus_dev);
device_lock(bus_dev);
device_unlock(bus_dev);
flush_workqueue(nfit_wq);
}
@@ -3449,9 +3449,9 @@ EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
{
nfit_device_lock(&adev->dev);
device_lock(&adev->dev);
__acpi_nfit_notify(&adev->dev, adev->handle, event);
nfit_device_unlock(&adev->dev);
device_unlock(&adev->dev);
}
static const struct acpi_device_id acpi_nfit_ids[] = {

24
drivers/acpi/nfit/nfit.h
View File

@@ -337,30 +337,6 @@ static inline struct acpi_nfit_desc *to_acpi_desc(
return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
}
#ifdef CONFIG_PROVE_LOCKING
static inline void nfit_device_lock(struct device *dev)
{
device_lock(dev);
mutex_lock(&dev->lockdep_mutex);
}
static inline void nfit_device_unlock(struct device *dev)
{
mutex_unlock(&dev->lockdep_mutex);
device_unlock(dev);
}
#else
static inline void nfit_device_lock(struct device *dev)
{
device_lock(dev);
}
static inline void nfit_device_unlock(struct device *dev)
{
device_unlock(dev);
}
#endif
const guid_t *to_nfit_uuid(enum nfit_uuids id);
int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *nfit, acpi_size sz);
void acpi_nfit_shutdown(void *data);

240
drivers/acpi/pci_root.c
View File

@@ -140,6 +140,17 @@ static struct pci_osc_bit_struct pci_osc_control_bit[] = {
{ OSC_PCI_EXPRESS_DPC_CONTROL, "DPC" },
};
static struct pci_osc_bit_struct cxl_osc_support_bit[] = {
{ OSC_CXL_1_1_PORT_REG_ACCESS_SUPPORT, "CXL11PortRegAccess" },
{ OSC_CXL_2_0_PORT_DEV_REG_ACCESS_SUPPORT, "CXL20PortDevRegAccess" },
{ OSC_CXL_PROTOCOL_ERR_REPORTING_SUPPORT, "CXLProtocolErrorReporting" },
{ OSC_CXL_NATIVE_HP_SUPPORT, "CXLNativeHotPlug" },
};
static struct pci_osc_bit_struct cxl_osc_control_bit[] = {
{ OSC_CXL_ERROR_REPORTING_CONTROL, "CXLMemErrorReporting" },
};
static void decode_osc_bits(struct acpi_pci_root *root, char *msg, u32 word,
struct pci_osc_bit_struct *table, int size)
{
@@ -168,33 +179,73 @@ static void decode_osc_control(struct acpi_pci_root *root, char *msg, u32 word)
ARRAY_SIZE(pci_osc_control_bit));
}
static u8 pci_osc_uuid_str[] = "33DB4D5B-1FF7-401C-9657-7441C03DD766";
static void decode_cxl_osc_support(struct acpi_pci_root *root, char *msg, u32 word)
{
decode_osc_bits(root, msg, word, cxl_osc_support_bit,
ARRAY_SIZE(cxl_osc_support_bit));
}
static acpi_status acpi_pci_run_osc(acpi_handle handle,
const u32 *capbuf, u32 *retval)
static void decode_cxl_osc_control(struct acpi_pci_root *root, char *msg, u32 word)
{
decode_osc_bits(root, msg, word, cxl_osc_control_bit,
ARRAY_SIZE(cxl_osc_control_bit));
}
static inline bool is_pcie(struct acpi_pci_root *root)
{
return root->bridge_type == ACPI_BRIDGE_TYPE_PCIE;
}
static inline bool is_cxl(struct acpi_pci_root *root)
{
return root->bridge_type == ACPI_BRIDGE_TYPE_CXL;
}
static u8 pci_osc_uuid_str[] = "33DB4D5B-1FF7-401C-9657-7441C03DD766";
static u8 cxl_osc_uuid_str[] = "68F2D50B-C469-4d8A-BD3D-941A103FD3FC";
static char *to_uuid(struct acpi_pci_root *root)
{
if (is_cxl(root))
return cxl_osc_uuid_str;
return pci_osc_uuid_str;
}
static int cap_length(struct acpi_pci_root *root)
{
if (is_cxl(root))
return sizeof(u32) * OSC_CXL_CAPABILITY_DWORDS;
return sizeof(u32) * OSC_PCI_CAPABILITY_DWORDS;
}
static acpi_status acpi_pci_run_osc(struct acpi_pci_root *root,
const u32 *capbuf, u32 *pci_control,
u32 *cxl_control)
{
struct acpi_osc_context context = {
.uuid_str = pci_osc_uuid_str,
.uuid_str = to_uuid(root),
.rev = 1,
.cap.length = 12,
.cap.length = cap_length(root),
.cap.pointer = (void *)capbuf,
};
acpi_status status;
status = acpi_run_osc(handle, &context);
status = acpi_run_osc(root->device->handle, &context);
if (ACPI_SUCCESS(status)) {
*retval = *((u32 *)(context.ret.pointer + 8));
*pci_control = acpi_osc_ctx_get_pci_control(&context);
if (is_cxl(root))
*cxl_control = acpi_osc_ctx_get_cxl_control(&context);
kfree(context.ret.pointer);
}
return status;
}
static acpi_status acpi_pci_query_osc(struct acpi_pci_root *root,
u32 support,
u32 *control)
static acpi_status acpi_pci_query_osc(struct acpi_pci_root *root, u32 support,
u32 *control, u32 cxl_support,
u32 *cxl_control)
{
acpi_status status;
u32 result, capbuf[3];
u32 pci_result, cxl_result, capbuf[OSC_CXL_CAPABILITY_DWORDS];
support |= root->osc_support_set;
@@ -202,10 +253,28 @@ static acpi_status acpi_pci_query_osc(struct acpi_pci_root *root,
capbuf[OSC_SUPPORT_DWORD] = support;
capbuf[OSC_CONTROL_DWORD] = *control | root->osc_control_set;
status = acpi_pci_run_osc(root->device->handle, capbuf, &result);
if (is_cxl(root)) {
cxl_support |= root->osc_ext_support_set;
capbuf[OSC_EXT_SUPPORT_DWORD] = cxl_support;
capbuf[OSC_EXT_CONTROL_DWORD] = *cxl_control | root->osc_ext_control_set;
}
retry:
status = acpi_pci_run_osc(root, capbuf, &pci_result, &cxl_result);
if (ACPI_SUCCESS(status)) {
root->osc_support_set = support;
*control = result;
*control = pci_result;
if (is_cxl(root)) {
root->osc_ext_support_set = cxl_support;
*cxl_control = cxl_result;
}
} else if (is_cxl(root)) {
/*
* CXL _OSC is optional on CXL 1.1 hosts. Fall back to PCIe _OSC
* upon any failure using CXL _OSC.
*/
root->bridge_type = ACPI_BRIDGE_TYPE_PCIE;
goto retry;
}
return status;
}
@@ -321,6 +390,8 @@ EXPORT_SYMBOL_GPL(acpi_get_pci_dev);
* @handle: ACPI handle of a PCI root bridge (or PCIe Root Complex).
* @mask: Mask of _OSC bits to request control of, place to store control mask.
* @support: _OSC supported capability.
* @cxl_mask: Mask of CXL _OSC control bits, place to store control mask.
* @cxl_support: CXL _OSC supported capability.
*
* Run _OSC query for @mask and if that is successful, compare the returned
* mask of control bits with @req. If all of the @req bits are set in the
@@ -331,12 +402,14 @@ EXPORT_SYMBOL_GPL(acpi_get_pci_dev);
* _OSC bits the BIOS has granted control of, but its contents are meaningless
* on failure.
**/
static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 support)
static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask,
u32 support, u32 *cxl_mask,
u32 cxl_support)
{
u32 req = OSC_PCI_EXPRESS_CAPABILITY_CONTROL;
struct acpi_pci_root *root;
acpi_status status;
u32 ctrl, capbuf[3];
u32 ctrl, cxl_ctrl = 0, capbuf[OSC_CXL_CAPABILITY_DWORDS];
if (!mask)
return AE_BAD_PARAMETER;
@@ -348,20 +421,42 @@ static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 s
ctrl = *mask;
*mask |= root->osc_control_set;
if (is_cxl(root)) {
cxl_ctrl = *cxl_mask;
*cxl_mask |= root->osc_ext_control_set;
}
/* Need to check the available controls bits before requesting them. */
do {
status = acpi_pci_query_osc(root, support, mask);
u32 pci_missing = 0, cxl_missing = 0;
status = acpi_pci_query_osc(root, support, mask, cxl_support,
cxl_mask);
if (ACPI_FAILURE(status))
return status;
if (ctrl == *mask)
break;
decode_osc_control(root, "platform does not support",
ctrl & ~(*mask));
if (is_cxl(root)) {
if (ctrl == *mask && cxl_ctrl == *cxl_mask)
break;
pci_missing = ctrl & ~(*mask);
cxl_missing = cxl_ctrl & ~(*cxl_mask);
} else {
if (ctrl == *mask)
break;
pci_missing = ctrl & ~(*mask);
}
if (pci_missing)
decode_osc_control(root, "platform does not support",
pci_missing);
if (cxl_missing)
decode_cxl_osc_control(root, "CXL platform does not support",
cxl_missing);
ctrl = *mask;
} while (*mask);
cxl_ctrl = *cxl_mask;
} while (*mask || *cxl_mask);
/* No need to request _OSC if the control was already granted. */
if ((root->osc_control_set & ctrl) == ctrl)
if ((root->osc_control_set & ctrl) == ctrl &&
(root->osc_ext_control_set & cxl_ctrl) == cxl_ctrl)
return AE_OK;
if ((ctrl & req) != req) {
@@ -373,11 +468,17 @@ static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 s
capbuf[OSC_QUERY_DWORD] = 0;
capbuf[OSC_SUPPORT_DWORD] = root->osc_support_set;
capbuf[OSC_CONTROL_DWORD] = ctrl;
status = acpi_pci_run_osc(handle, capbuf, mask);
if (is_cxl(root)) {
capbuf[OSC_EXT_SUPPORT_DWORD] = root->osc_ext_support_set;
capbuf[OSC_EXT_CONTROL_DWORD] = cxl_ctrl;
}
status = acpi_pci_run_osc(root, capbuf, mask, cxl_mask);
if (ACPI_FAILURE(status))
return status;
root->osc_control_set = *mask;
root->osc_ext_control_set = *cxl_mask;
return AE_OK;
}
@@ -403,6 +504,53 @@ static u32 calculate_support(void)
return support;
}
/*
* Background on hotplug support, and making it depend on only
* CONFIG_HOTPLUG_PCI_PCIE vs. also considering CONFIG_MEMORY_HOTPLUG:
*
* CONFIG_ACPI_HOTPLUG_MEMORY does depend on CONFIG_MEMORY_HOTPLUG, but
* there is no existing _OSC for memory hotplug support. The reason is that
* ACPI memory hotplug requires the OS to acknowledge / coordinate with
* memory plug events via a scan handler. On the CXL side the equivalent
* would be if Linux supported the Mechanical Retention Lock [1], or
* otherwise had some coordination for the driver of a PCI device
* undergoing hotplug to be consulted on whether the hotplug should
* proceed or not.
*
* The concern is that if Linux says no to supporting CXL hotplug then
* the BIOS may say no to giving the OS hotplug control of any other PCIe
* device. So the question here is not whether hotplug is enabled, it's
* whether it is handled natively by the at all OS, and if
* CONFIG_HOTPLUG_PCI_PCIE is enabled then the answer is "yes".
*
* Otherwise, the plan for CXL coordinated remove, since the kernel does
* not support blocking hotplug, is to require the memory device to be
* disabled before hotplug is attempted. When CONFIG_MEMORY_HOTPLUG is
* disabled that step will fail and the remove attempt cancelled by the
* user. If that is not honored and the card is removed anyway then it
* does not matter if CONFIG_MEMORY_HOTPLUG is enabled or not, it will
* cause a crash and other badness.
*
* Therefore, just say yes to CXL hotplug and require removal to
* be coordinated by userspace unless and until the kernel grows better
* mechanisms for doing "managed" removal of devices in consultation with
* the driver.
*
* [1]: https://lore.kernel.org/all/20201122014203.4706-1-ashok.raj@intel.com/
*/
static u32 calculate_cxl_support(void)
{
u32 support;
support = OSC_CXL_2_0_PORT_DEV_REG_ACCESS_SUPPORT;
if (pci_aer_available())
support |= OSC_CXL_PROTOCOL_ERR_REPORTING_SUPPORT;
if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
support |= OSC_CXL_NATIVE_HP_SUPPORT;
return support;
}
static u32 calculate_control(void)
{
u32 control;
@@ -434,6 +582,16 @@ static u32 calculate_control(void)
return control;
}
static u32 calculate_cxl_control(void)
{
u32 control = 0;
if (IS_ENABLED(CONFIG_MEMORY_FAILURE))
control |= OSC_CXL_ERROR_REPORTING_CONTROL;
return control;
}
static bool os_control_query_checks(struct acpi_pci_root *root, u32 support)
{
struct acpi_device *device = root->device;
@@ -452,10 +610,10 @@ static bool os_control_query_checks(struct acpi_pci_root *root, u32 support)
return true;
}
static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm,
bool is_pcie)
static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm)
{
u32 support, control = 0, requested = 0;
u32 cxl_support = 0, cxl_control = 0, cxl_requested = 0;
acpi_status status;
struct acpi_device *device = root->device;
acpi_handle handle = device->handle;
@@ -479,10 +637,20 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm,
if (os_control_query_checks(root, support))
requested = control = calculate_control();
status = acpi_pci_osc_control_set(handle, &control, support);
if (is_cxl(root)) {
cxl_support = calculate_cxl_support();
decode_cxl_osc_support(root, "OS supports", cxl_support);
cxl_requested = cxl_control = calculate_cxl_control();
}
status = acpi_pci_osc_control_set(handle, &control, support,
&cxl_control, cxl_support);
if (ACPI_SUCCESS(status)) {
if (control)
decode_osc_control(root, "OS now controls", control);
if (cxl_control)
decode_cxl_osc_control(root, "OS now controls",
cxl_control);
if (acpi_gbl_FADT.boot_flags & ACPI_FADT_NO_ASPM) {
/*
@@ -504,13 +672,18 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm,
*no_aspm = 1;
/* _OSC is optional for PCI host bridges */
if ((status == AE_NOT_FOUND) && !is_pcie)
if (status == AE_NOT_FOUND && !is_pcie(root))
return;
if (control) {
decode_osc_control(root, "OS requested", requested);
decode_osc_control(root, "platform willing to grant", control);
}
if (cxl_control) {
decode_cxl_osc_control(root, "OS requested", cxl_requested);
decode_cxl_osc_control(root, "platform willing to grant",
cxl_control);
}
dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n",
acpi_format_exception(status));
@@ -527,7 +700,7 @@ static int acpi_pci_root_add(struct acpi_device *device,
acpi_handle handle = device->handle;
int no_aspm = 0;
bool hotadd = system_state == SYSTEM_RUNNING;
bool is_pcie;
const char *acpi_hid;
root = kzalloc(sizeof(struct acpi_pci_root), GFP_KERNEL);
if (!root)
@@ -585,8 +758,15 @@ static int acpi_pci_root_add(struct acpi_device *device,
root->mcfg_addr = acpi_pci_root_get_mcfg_addr(handle);
is_pcie = strcmp(acpi_device_hid(device), "PNP0A08") == 0;
negotiate_os_control(root, &no_aspm, is_pcie);
acpi_hid = acpi_device_hid(root->device);
if (strcmp(acpi_hid, "PNP0A08") == 0)
root->bridge_type = ACPI_BRIDGE_TYPE_PCIE;
else if (strcmp(acpi_hid, "ACPI0016") == 0)
root->bridge_type = ACPI_BRIDGE_TYPE_CXL;
else
dev_dbg(&device->dev, "Assuming non-PCIe host bridge\n");
negotiate_os_control(root, &no_aspm);
/*
* TBD: Need PCI interface for enumeration/configuration of roots.

3
drivers/base/core.c
View File

@@ -2864,9 +2864,6 @@ void device_initialize(struct device *dev)
kobject_init(&dev->kobj, &device_ktype);
INIT_LIST_HEAD(&dev->dma_pools);
mutex_init(&dev->mutex);
#ifdef CONFIG_PROVE_LOCKING
mutex_init(&dev->lockdep_mutex);
#endif
lockdep_set_novalidate_class(&dev->mutex);
spin_lock_init(&dev->devres_lock);
INIT_LIST_HEAD(&dev->devres_head);

4
drivers/cxl/Kconfig
View File

@@ -98,4 +98,8 @@ config CXL_PORT
default CXL_BUS
tristate
config CXL_SUSPEND
def_bool y
depends on SUSPEND && CXL_MEM
endif

2
drivers/cxl/Makefile
View File

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CXL_BUS) += core/
obj-y += core/
obj-$(CONFIG_CXL_PCI) += cxl_pci.o
obj-$(CONFIG_CXL_MEM) += cxl_mem.o
obj-$(CONFIG_CXL_ACPI) += cxl_acpi.o

13
drivers/cxl/acpi.c
View File

@@ -275,6 +275,13 @@ static int add_root_nvdimm_bridge(struct device *match, void *data)
return 1;
}
static struct lock_class_key cxl_root_key;
static void cxl_acpi_lock_reset_class(void *dev)
{
device_lock_reset_class(dev);
}
static int cxl_acpi_probe(struct platform_device *pdev)
{
int rc;
@@ -283,6 +290,12 @@ static int cxl_acpi_probe(struct platform_device *pdev)
struct acpi_device *adev = ACPI_COMPANION(host);
struct cxl_cfmws_context ctx;
device_lock_set_class(&pdev->dev, &cxl_root_key);
rc = devm_add_action_or_reset(&pdev->dev, cxl_acpi_lock_reset_class,
&pdev->dev);
if (rc)
return rc;
root_port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL);
if (IS_ERR(root_port))
return PTR_ERR(root_port);

1
drivers/cxl/core/Makefile
View File

@@ -1,5 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CXL_BUS) += cxl_core.o
obj-$(CONFIG_CXL_SUSPEND) += suspend.o
ccflags-y += -I$(srctree)/drivers/cxl
cxl_core-y := port.o

364
drivers/cxl/core/mbox.c
View File

@@ -35,6 +35,7 @@ static bool cxl_raw_allow_all;
.flags = _flags, \
}
#define CXL_VARIABLE_PAYLOAD ~0U
/*
* This table defines the supported mailbox commands for the driver. This table
* is made up of a UAPI structure. Non-negative values as parameters in the
@@ -44,26 +45,26 @@ static bool cxl_raw_allow_all;
static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
#ifdef CONFIG_CXL_MEM_RAW_COMMANDS
CXL_CMD(RAW, ~0, ~0, 0),
CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),
#endif
CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE),
CXL_CMD(GET_SUPPORTED_LOGS, 0, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
CXL_CMD(GET_LSA, 0x8, ~0, 0),
CXL_CMD(GET_LSA, 0x8, CXL_VARIABLE_PAYLOAD, 0),
CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE),
CXL_CMD(GET_LOG, 0x18, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
CXL_CMD(SET_LSA, ~0, 0, 0),
CXL_CMD(SET_LSA, CXL_VARIABLE_PAYLOAD, 0, 0),
CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
CXL_CMD(GET_POISON, 0x10, ~0, 0),
CXL_CMD(GET_POISON, 0x10, CXL_VARIABLE_PAYLOAD, 0),
CXL_CMD(INJECT_POISON, 0x8, 0, 0),
CXL_CMD(CLEAR_POISON, 0x48, 0, 0),
CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
CXL_CMD(SCAN_MEDIA, 0x11, 0, 0),
CXL_CMD(GET_SCAN_MEDIA, 0, ~0, 0),
CXL_CMD(GET_SCAN_MEDIA, 0, CXL_VARIABLE_PAYLOAD, 0),
};
/*
@@ -127,6 +128,17 @@ static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
return NULL;
}
static const char *cxl_mem_opcode_to_name(u16 opcode)
{
struct cxl_mem_command *c;
c = cxl_mem_find_command(opcode);
if (!c)
return NULL;
return cxl_command_names[c->info.id].name;
}
/**
* cxl_mbox_send_cmd() - Send a mailbox command to a device.
* @cxlds: The device data for the operation
@@ -136,7 +148,7 @@ static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
* @out: Caller allocated buffer for the output.
* @out_size: Expected size of output.
*
* Context: Any context. Will acquire and release mbox_mutex.
* Context: Any context.
* Return:
* * %>=0 - Number of bytes returned in @out.
* * %-E2BIG - Payload is too large for hardware.
@@ -169,17 +181,17 @@ int cxl_mbox_send_cmd(struct cxl_dev_state *cxlds, u16 opcode, void *in,
if (rc)
return rc;
/* TODO: Map return code to proper kernel style errno */
if (mbox_cmd.return_code != CXL_MBOX_SUCCESS)
return -ENXIO;
if (mbox_cmd.return_code != CXL_MBOX_CMD_RC_SUCCESS)
return cxl_mbox_cmd_rc2errno(&mbox_cmd);
/*
* Variable sized commands can't be validated and so it's up to the
* caller to do that if they wish.
*/
if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size)
return -EIO;
if (cmd->info.size_out != CXL_VARIABLE_PAYLOAD) {
if (mbox_cmd.size_out != out_size)
return -EIO;
}
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_mbox_send_cmd, CXL);
@@ -207,11 +219,167 @@ static bool cxl_mem_raw_command_allowed(u16 opcode)
return true;
}
/**
* cxl_payload_from_user_allowed() - Check contents of in_payload.
* @opcode: The mailbox command opcode.
* @payload_in: Pointer to the input payload passed in from user space.
*
* Return:
* * true - payload_in passes check for @opcode.
* * false - payload_in contains invalid or unsupported values.
*
* The driver may inspect payload contents before sending a mailbox
* command from user space to the device. The intent is to reject
* commands with input payloads that are known to be unsafe. This
* check is not intended to replace the users careful selection of
* mailbox command parameters and makes no guarantee that the user
* command will succeed, nor that it is appropriate.
*
* The specific checks are determined by the opcode.
*/
static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in)
{
switch (opcode) {
case CXL_MBOX_OP_SET_PARTITION_INFO: {
struct cxl_mbox_set_partition_info *pi = payload_in;
if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG)
return false;
break;
}
default:
break;
}
return true;
}
static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
struct cxl_dev_state *cxlds, u16 opcode,
size_t in_size, size_t out_size, u64 in_payload)
{
*mbox = (struct cxl_mbox_cmd) {
.opcode = opcode,
.size_in = in_size,
};
if (in_size) {
mbox->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
in_size);
if (IS_ERR(mbox->payload_in))
return PTR_ERR(mbox->payload_in);
if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) {
dev_dbg(cxlds->dev, "%s: input payload not allowed\n",
cxl_mem_opcode_to_name(opcode));
kvfree(mbox->payload_in);
return -EBUSY;
}
}
/* Prepare to handle a full payload for variable sized output */
if (out_size == CXL_VARIABLE_PAYLOAD)
mbox->size_out = cxlds->payload_size;
else
mbox->size_out = out_size;
if (mbox->size_out) {
mbox->payload_out = kvzalloc(mbox->size_out, GFP_KERNEL);
if (!mbox->payload_out) {
kvfree(mbox->payload_in);
return -ENOMEM;
}
}
return 0;
}
static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
{
kvfree(mbox->payload_in);
kvfree(mbox->payload_out);
}
static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
const struct cxl_send_command *send_cmd,
struct cxl_dev_state *cxlds)
{
if (send_cmd->raw.rsvd)
return -EINVAL;
/*
* Unlike supported commands, the output size of RAW commands
* gets passed along without further checking, so it must be
* validated here.
*/
if (send_cmd->out.size > cxlds->payload_size)
return -EINVAL;
if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
return -EPERM;
dev_WARN_ONCE(cxlds->dev, true, "raw command path used\n");
*mem_cmd = (struct cxl_mem_command) {
.info = {
.id = CXL_MEM_COMMAND_ID_RAW,
.size_in = send_cmd->in.size,
.size_out = send_cmd->out.size,
},
.opcode = send_cmd->raw.opcode
};
return 0;
}
static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
const struct cxl_send_command *send_cmd,
struct cxl_dev_state *cxlds)
{
struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
const struct cxl_command_info *info = &c->info;
if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
return -EINVAL;
if (send_cmd->rsvd)
return -EINVAL;
if (send_cmd->in.rsvd || send_cmd->out.rsvd)
return -EINVAL;
/* Check that the command is enabled for hardware */
if (!test_bit(info->id, cxlds->enabled_cmds))
return -ENOTTY;
/* Check that the command is not claimed for exclusive kernel use */
if (test_bit(info->id, cxlds->exclusive_cmds))
return -EBUSY;
/* Check the input buffer is the expected size */
if (info->size_in != send_cmd->in.size)
return -ENOMEM;
/* Check the output buffer is at least large enough */
if (send_cmd->out.size < info->size_out)
return -ENOMEM;
*mem_cmd = (struct cxl_mem_command) {
.info = {
.id = info->id,
.flags = info->flags,
.size_in = send_cmd->in.size,
.size_out = send_cmd->out.size,
},
.opcode = c->opcode
};
return 0;
}
/**
* cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
* @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
* @cxlds: The device data for the operation
* @send_cmd: &struct cxl_send_command copied in from userspace.
* @out_cmd: Sanitized and populated &struct cxl_mem_command.
*
* Return:
* * %0 - @out_cmd is ready to send.
@@ -221,17 +389,15 @@ static bool cxl_mem_raw_command_allowed(u16 opcode)
* * %-EPERM - Attempted to use a protected command.
* * %-EBUSY - Kernel has claimed exclusive access to this opcode
*
* The result of this command is a fully validated command in @out_cmd that is
* The result of this command is a fully validated command in @mbox_cmd that is
* safe to send to the hardware.
*
* See handle_mailbox_cmd_from_user()
*/
static int cxl_validate_cmd_from_user(struct cxl_dev_state *cxlds,
const struct cxl_send_command *send_cmd,
struct cxl_mem_command *out_cmd)
static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
struct cxl_dev_state *cxlds,
const struct cxl_send_command *send_cmd)
{
const struct cxl_command_info *info;
struct cxl_mem_command *c;
struct cxl_mem_command mem_cmd;
int rc;
if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
return -ENOTTY;
@@ -244,78 +410,19 @@ static int cxl_validate_cmd_from_user(struct cxl_dev_state *cxlds,
if (send_cmd->in.size > cxlds->payload_size)
return -EINVAL;
/*
* Checks are bypassed for raw commands but a WARN/taint will occur
* later in the callchain
*/
if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) {
const struct cxl_mem_command temp = {
.info = {
.id = CXL_MEM_COMMAND_ID_RAW,
.flags = 0,
.size_in = send_cmd->in.size,
.size_out = send_cmd->out.size,
},
.opcode = send_cmd->raw.opcode
};
/* Sanitize and construct a cxl_mem_command */
if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, cxlds);
else
rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, cxlds);
if (send_cmd->raw.rsvd)
return -EINVAL;
if (rc)
return rc;
/*
* Unlike supported commands, the output size of RAW commands
* gets passed along without further checking, so it must be
* validated here.
*/
if (send_cmd->out.size > cxlds->payload_size)
return -EINVAL;
if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
return -EPERM;
memcpy(out_cmd, &temp, sizeof(temp));
return 0;
}
if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
return -EINVAL;
if (send_cmd->rsvd)
return -EINVAL;
if (send_cmd->in.rsvd || send_cmd->out.rsvd)
return -EINVAL;
/* Convert user's command into the internal representation */
c = &cxl_mem_commands[send_cmd->id];
info = &c->info;
/* Check that the command is enabled for hardware */
if (!test_bit(info->id, cxlds->enabled_cmds))
return -ENOTTY;
/* Check that the command is not claimed for exclusive kernel use */
if (test_bit(info->id, cxlds->exclusive_cmds))
return -EBUSY;
/* Check the input buffer is the expected size */
if (info->size_in >= 0 && info->size_in != send_cmd->in.size)
return -ENOMEM;
/* Check the output buffer is at least large enough */
if (info->size_out >= 0 && send_cmd->out.size < info->size_out)
return -ENOMEM;
memcpy(out_cmd, c, sizeof(*c));
out_cmd->info.size_in = send_cmd->in.size;
/*
* XXX: out_cmd->info.size_out will be controlled by the driver, and the
* specified number of bytes @send_cmd->out.size will be copied back out
* to userspace.
*/
return 0;
/* Sanitize and construct a cxl_mbox_cmd */
return cxl_mbox_cmd_ctor(mbox_cmd, cxlds, mem_cmd.opcode,
mem_cmd.info.size_in, mem_cmd.info.size_out,
send_cmd->in.payload);
}
int cxl_query_cmd(struct cxl_memdev *cxlmd,
@@ -355,8 +462,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
/**
* handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
* @cxlds: The device data for the operation
* @cmd: The validated command.
* @in_payload: Pointer to userspace's input payload.
* @mbox_cmd: The validated mailbox command.
* @out_payload: Pointer to userspace's output payload.
* @size_out: (Input) Max payload size to copy out.
* (Output) Payload size hardware generated.
@@ -371,51 +477,27 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd,
* * %-EINTR - Mailbox acquisition interrupted.
* * %-EXXX - Transaction level failures.
*
* Creates the appropriate mailbox command and dispatches it on behalf of a
* userspace request. The input and output payloads are copied between
* userspace.
* Dispatches a mailbox command on behalf of a userspace request.
* The output payload is copied to userspace.
*
* See cxl_send_cmd().
*/
static int handle_mailbox_cmd_from_user(struct cxl_dev_state *cxlds,
const struct cxl_mem_command *cmd,
u64 in_payload, u64 out_payload,
s32 *size_out, u32 *retval)
struct cxl_mbox_cmd *mbox_cmd,
u64 out_payload, s32 *size_out,
u32 *retval)
{
struct device *dev = cxlds->dev;
struct cxl_mbox_cmd mbox_cmd = {
.opcode = cmd->opcode,
.size_in = cmd->info.size_in,
.size_out = cmd->info.size_out,
};
int rc;
if (cmd->info.size_out) {
mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL);
if (!mbox_cmd.payload_out)
return -ENOMEM;
}
if (cmd->info.size_in) {
mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
cmd->info.size_in);
if (IS_ERR(mbox_cmd.payload_in)) {
kvfree(mbox_cmd.payload_out);
return PTR_ERR(mbox_cmd.payload_in);
}
}
dev_dbg(dev,
"Submitting %s command for user\n"
"\topcode: %x\n"
"\tsize: %ub\n",
cxl_command_names[cmd->info.id].name, mbox_cmd.opcode,
cmd->info.size_in);
"\tsize: %zx\n",
cxl_mem_opcode_to_name(mbox_cmd->opcode),
mbox_cmd->opcode, mbox_cmd->size_in);
dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW,
"raw command path used\n");
rc = cxlds->mbox_send(cxlds, &mbox_cmd);
rc = cxlds->mbox_send(cxlds, mbox_cmd);
if (rc)
goto out;
@@ -424,22 +506,21 @@ static int handle_mailbox_cmd_from_user(struct cxl_dev_state *cxlds,
* to userspace. While the payload may have written more output than
* this it will have to be ignored.
*/
if (mbox_cmd.size_out) {
dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out,
if (mbox_cmd->size_out) {
dev_WARN_ONCE(dev, mbox_cmd->size_out > *size_out,
"Invalid return size\n");
if (copy_to_user(u64_to_user_ptr(out_payload),
mbox_cmd.payload_out, mbox_cmd.size_out)) {
mbox_cmd->payload_out, mbox_cmd->size_out)) {
rc = -EFAULT;
goto out;
}
}
*size_out = mbox_cmd.size_out;
*retval = mbox_cmd.return_code;
*size_out = mbox_cmd->size_out;
*retval = mbox_cmd->return_code;
out:
kvfree(mbox_cmd.payload_in);
kvfree(mbox_cmd.payload_out);
cxl_mbox_cmd_dtor(mbox_cmd);
return rc;
}
@@ -448,7 +529,7 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct device *dev = &cxlmd->dev;
struct cxl_send_command send;
struct cxl_mem_command c;
struct cxl_mbox_cmd mbox_cmd;
int rc;
dev_dbg(dev, "Send IOCTL\n");
@@ -456,17 +537,12 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
if (copy_from_user(&send, s, sizeof(send)))
return -EFAULT;
rc = cxl_validate_cmd_from_user(cxlmd->cxlds, &send, &c);
rc = cxl_validate_cmd_from_user(&mbox_cmd, cxlmd->cxlds, &send);
if (rc)
return rc;
/* Prepare to handle a full payload for variable sized output */
if (c.info.size_out < 0)
c.info.size_out = cxlds->payload_size;
rc = handle_mailbox_cmd_from_user(cxlds, &c, send.in.payload,
send.out.payload, &send.out.size,
&send.retval);
rc = handle_mailbox_cmd_from_user(cxlds, &mbox_cmd, send.out.payload,
&send.out.size, &send.retval);
if (rc)
return rc;

3
drivers/cxl/core/memdev.c
View File

@@ -228,6 +228,8 @@ static void detach_memdev(struct work_struct *work)
put_device(&cxlmd->dev);
}
static struct lock_class_key cxl_memdev_key;
static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
const struct file_operations *fops)
{
@@ -247,6 +249,7 @@ static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
dev = &cxlmd->dev;
device_initialize(dev);
lockdep_set_class(&dev->mutex, &cxl_memdev_key);
dev->parent = cxlds->dev;
dev->bus = &cxl_bus_type;
dev->devt = MKDEV(cxl_mem_major, cxlmd->id);

364
drivers/cxl/core/pci.c
View File

@@ -1,8 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/delay.h>
#include <linux/pci.h>
#include <cxlpci.h>
#include <cxlmem.h>
#include <cxl.h>
#include "core.h"
@@ -13,6 +16,10 @@
* a set of helpers for CXL interactions which occur via PCIe.
*/
static unsigned short media_ready_timeout = 60;
module_param(media_ready_timeout, ushort, 0644);
MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready");
struct cxl_walk_context {
struct pci_bus *bus;
struct cxl_port *port;
@@ -94,3 +101,360 @@ int devm_cxl_port_enumerate_dports(struct cxl_port *port)
return ctx.count;
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL);
/*
* Wait up to @media_ready_timeout for the device to report memory
* active.
*/
int cxl_await_media_ready(struct cxl_dev_state *cxlds)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
int d = cxlds->cxl_dvsec;
bool active = false;
u64 md_status;
int rc, i;
for (i = media_ready_timeout; i; i--) {
u32 temp;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp);
if (rc)
return rc;
active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp);
if (active)
break;
msleep(1000);
}
if (!active) {
dev_err(&pdev->dev,
"timeout awaiting memory active after %d seconds\n",
media_ready_timeout);
return -ETIMEDOUT;
}
md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
if (!CXLMDEV_READY(md_status))
return -EIO;
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, CXL);
static int wait_for_valid(struct cxl_dev_state *cxlds)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
int d = cxlds->cxl_dvsec, rc;
u32 val;
/*
* Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high
* and Size Low registers are valid. Must be set within 1 second of
* deassertion of reset to CXL device. Likely it is already set by the
* time this runs, but otherwise give a 1.5 second timeout in case of
* clock skew.
*/
rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
if (rc)
return rc;
if (val & CXL_DVSEC_MEM_INFO_VALID)
return 0;
msleep(1500);
rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val);
if (rc)
return rc;
if (val & CXL_DVSEC_MEM_INFO_VALID)
return 0;
return -ETIMEDOUT;
}
static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
int d = cxlds->cxl_dvsec;
u16 ctrl;
int rc;
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
if (rc < 0)
return rc;
if ((ctrl & CXL_DVSEC_MEM_ENABLE) == val)
return 1;
ctrl &= ~CXL_DVSEC_MEM_ENABLE;
ctrl |= val;
rc = pci_write_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, ctrl);
if (rc < 0)
return rc;
return 0;
}
static void clear_mem_enable(void *cxlds)
{
cxl_set_mem_enable(cxlds, 0);
}
static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds)
{
int rc;
rc = cxl_set_mem_enable(cxlds, CXL_DVSEC_MEM_ENABLE);
if (rc < 0)
return rc;
if (rc > 0)
return 0;
return devm_add_action_or_reset(host, clear_mem_enable, cxlds);
}
static bool range_contains(struct range *r1, struct range *r2)
{
return r1->start <= r2->start && r1->end >= r2->end;
}
/* require dvsec ranges to be covered by a locked platform window */
static int dvsec_range_allowed(struct device *dev, void *arg)
{
struct range *dev_range = arg;
struct cxl_decoder *cxld;
struct range root_range;
if (!is_root_decoder(dev))
return 0;
cxld = to_cxl_decoder(dev);
if (!(cxld->flags & CXL_DECODER_F_LOCK))
return 0;
if (!(cxld->flags & CXL_DECODER_F_RAM))
return 0;
root_range = (struct range) {
.start = cxld->platform_res.start,
.end = cxld->platform_res.end,
};
return range_contains(&root_range, dev_range);
}
static void disable_hdm(void *_cxlhdm)
{
u32 global_ctrl;
struct cxl_hdm *cxlhdm = _cxlhdm;
void __iomem *hdm = cxlhdm->regs.hdm_decoder;
global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
writel(global_ctrl & ~CXL_HDM_DECODER_ENABLE,
hdm + CXL_HDM_DECODER_CTRL_OFFSET);
}
static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm)
{
void __iomem *hdm = cxlhdm->regs.hdm_decoder;
u32 global_ctrl;
global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
writel(global_ctrl | CXL_HDM_DECODER_ENABLE,
hdm + CXL_HDM_DECODER_CTRL_OFFSET);
return devm_add_action_or_reset(host, disable_hdm, cxlhdm);
}
static bool __cxl_hdm_decode_init(struct cxl_dev_state *cxlds,
struct cxl_hdm *cxlhdm,
struct cxl_endpoint_dvsec_info *info)
{
void __iomem *hdm = cxlhdm->regs.hdm_decoder;
struct cxl_port *port = cxlhdm->port;
struct device *dev = cxlds->dev;
struct cxl_port *root;
int i, rc, allowed;
u32 global_ctrl;
global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);
/*
* If the HDM Decoder Capability is already enabled then assume
* that some other agent like platform firmware set it up.
*/
if (global_ctrl & CXL_HDM_DECODER_ENABLE) {
rc = devm_cxl_enable_mem(&port->dev, cxlds);
if (rc)
return false;
return true;
}
root = to_cxl_port(port->dev.parent);
while (!is_cxl_root(root) && is_cxl_port(root->dev.parent))
root = to_cxl_port(root->dev.parent);
if (!is_cxl_root(root)) {
dev_err(dev, "Failed to acquire root port for HDM enable\n");
return false;
}
for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) {
struct device *cxld_dev;
cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i],
dvsec_range_allowed);
if (!cxld_dev) {
dev_dbg(dev, "DVSEC Range%d denied by platform\n", i);
continue;
}
dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i);
put_device(cxld_dev);
allowed++;
}
if (!allowed) {
cxl_set_mem_enable(cxlds, 0);
info->mem_enabled = 0;
}
/*
* Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base
* [High,Low] when HDM operation is enabled the range register values
* are ignored by the device, but the spec also recommends matching the
* DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges
* are expected even though Linux does not require or maintain that
* match. If at least one DVSEC range is enabled and allowed, skip HDM
* Decoder Capability Enable.
*/
if (info->mem_enabled)
return false;
rc = devm_cxl_enable_hdm(&port->dev, cxlhdm);
if (rc)
return false;
rc = devm_cxl_enable_mem(&port->dev, cxlds);
if (rc)
return false;
return true;
}
/**
* cxl_hdm_decode_init() - Setup HDM decoding for the endpoint
* @cxlds: Device state
* @cxlhdm: Mapped HDM decoder Capability
*
* Try to enable the endpoint's HDM Decoder Capability
*/
int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
struct cxl_endpoint_dvsec_info info = { 0 };
int hdm_count, rc, i, ranges = 0;
struct device *dev = &pdev->dev;
int d = cxlds->cxl_dvsec;
u16 cap, ctrl;
if (!d) {
dev_dbg(dev, "No DVSEC Capability\n");
return -ENXIO;
}
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap);
if (rc)
return rc;
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
if (rc)
return rc;
if (!(cap & CXL_DVSEC_MEM_CAPABLE)) {
dev_dbg(dev, "Not MEM Capable\n");
return -ENXIO;
}
/*
* It is not allowed by spec for MEM.capable to be set and have 0 legacy
* HDM decoders (values > 2 are also undefined as of CXL 2.0). As this
* driver is for a spec defined class code which must be CXL.mem
* capable, there is no point in continuing to enable CXL.mem.
*/
hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
if (!hdm_count || hdm_count > 2)
return -EINVAL;
rc = wait_for_valid(cxlds);
if (rc) {
dev_dbg(dev, "Failure awaiting MEM_INFO_VALID (%d)\n", rc);
return rc;
}
/*
* The current DVSEC values are moot if the memory capability is
* disabled, and they will remain moot after the HDM Decoder
* capability is enabled.
*/
info.mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);
if (!info.mem_enabled)
goto hdm_init;
for (i = 0; i < hdm_count; i++) {
u64 base, size;
u32 temp;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp);
if (rc)
return rc;
size = (u64)temp << 32;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp);
if (rc)
return rc;
size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp);
if (rc)
return rc;
base = (u64)temp << 32;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp);
if (rc)
return rc;
base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK;
info.dvsec_range[i] = (struct range) {
.start = base,
.end = base + size - 1
};
if (size)
ranges++;
}
info.ranges = ranges;
/*
* If DVSEC ranges are being used instead of HDM decoder registers there
* is no use in trying to manage those.
*/
hdm_init:
if (!__cxl_hdm_decode_init(cxlds, cxlhdm, &info)) {
dev_err(dev,
"Legacy range registers configuration prevents HDM operation.\n");
return -EBUSY;
}
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL);

10
drivers/cxl/core/pmem.c
View File

@@ -80,6 +80,8 @@ struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd)
}
EXPORT_SYMBOL_NS_GPL(cxl_find_nvdimm_bridge, CXL);
static struct lock_class_key cxl_nvdimm_bridge_key;
static struct cxl_nvdimm_bridge *cxl_nvdimm_bridge_alloc(struct cxl_port *port)
{
struct cxl_nvdimm_bridge *cxl_nvb;
@@ -99,6 +101,7 @@ static struct cxl_nvdimm_bridge *cxl_nvdimm_bridge_alloc(struct cxl_port *port)
cxl_nvb->port = port;
cxl_nvb->state = CXL_NVB_NEW;
device_initialize(dev);
lockdep_set_class(&dev->mutex, &cxl_nvdimm_bridge_key);
device_set_pm_not_required(dev);
dev->parent = &port->dev;
dev->bus = &cxl_bus_type;
@@ -121,10 +124,10 @@ static void unregister_nvb(void *_cxl_nvb)
* work to flush. Once the state has been changed to 'dead' then no new
* work can be queued by user-triggered bind.
*/
cxl_device_lock(&cxl_nvb->dev);
device_lock(&cxl_nvb->dev);
flush = cxl_nvb->state != CXL_NVB_NEW;
cxl_nvb->state = CXL_NVB_DEAD;
cxl_device_unlock(&cxl_nvb->dev);
device_unlock(&cxl_nvb->dev);
/*
* Even though the device core will trigger device_release_driver()
@@ -214,6 +217,8 @@ struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev)
}
EXPORT_SYMBOL_NS_GPL(to_cxl_nvdimm, CXL);
static struct lock_class_key cxl_nvdimm_key;
static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_memdev *cxlmd)
{
struct cxl_nvdimm *cxl_nvd;
@@ -226,6 +231,7 @@ static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_memdev *cxlmd)
dev = &cxl_nvd->dev;
cxl_nvd->cxlmd = cxlmd;
device_initialize(dev);
lockdep_set_class(&dev->mutex, &cxl_nvdimm_key);
device_set_pm_not_required(dev);
dev->parent = &cxlmd->dev;
dev->bus = &cxl_bus_type;

68
drivers/cxl/core/port.c
View File

@@ -312,10 +312,10 @@ static void cxl_port_release(struct device *dev)
struct cxl_port *port = to_cxl_port(dev);
struct cxl_ep *ep, *_e;
cxl_device_lock(dev);
device_lock(dev);
list_for_each_entry_safe(ep, _e, &port->endpoints, list)
cxl_ep_release(ep);
cxl_device_unlock(dev);
device_unlock(dev);
ida_free(&cxl_port_ida, port->id);
kfree(port);
}
@@ -391,6 +391,8 @@ static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
return devm_add_action_or_reset(host, cxl_unlink_uport, port);
}
static struct lock_class_key cxl_port_key;
static struct cxl_port *cxl_port_alloc(struct device *uport,
resource_size_t component_reg_phys,
struct cxl_port *parent_port)
@@ -415,9 +417,10 @@ static struct cxl_port *cxl_port_alloc(struct device *uport,
* description.
*/
dev = &port->dev;
if (parent_port)
if (parent_port) {
dev->parent = &parent_port->dev;
else
port->depth = parent_port->depth + 1;
} else
dev->parent = uport;
port->uport = uport;
@@ -427,6 +430,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport,
INIT_LIST_HEAD(&port->endpoints);
device_initialize(dev);
lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
device_set_pm_not_required(dev);
dev->bus = &cxl_bus_type;
dev->type = &cxl_port_type;
@@ -457,8 +461,6 @@ struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
if (IS_ERR(port))
return port;
if (parent_port)
port->depth = parent_port->depth + 1;
dev = &port->dev;
if (is_cxl_memdev(uport))
rc = dev_set_name(dev, "endpoint%d", port->id);
@@ -554,7 +556,7 @@ static int match_root_child(struct device *dev, const void *match)
return 0;
port = to_cxl_port(dev);
cxl_device_lock(dev);
device_lock(dev);
list_for_each_entry(dport, &port->dports, list) {
iter = match;
while (iter) {
@@ -564,7 +566,7 @@ static int match_root_child(struct device *dev, const void *match)
}
}
out:
cxl_device_unlock(dev);
device_unlock(dev);
return !!iter;
}
@@ -623,13 +625,13 @@ static int add_dport(struct cxl_port *port, struct cxl_dport *new)
static void cond_cxl_root_lock(struct cxl_port *port)
{
if (is_cxl_root(port))
cxl_device_lock(&port->dev);
device_lock(&port->dev);
}
static void cond_cxl_root_unlock(struct cxl_port *port)
{
if (is_cxl_root(port))
cxl_device_unlock(&port->dev);
device_unlock(&port->dev);
}
static void cxl_dport_remove(void *data)
@@ -736,15 +738,15 @@ static int add_ep(struct cxl_port *port, struct cxl_ep *new)
{
struct cxl_ep *dup;
cxl_device_lock(&port->dev);
device_lock(&port->dev);
if (port->dead) {
cxl_device_unlock(&port->dev);
device_unlock(&port->dev);
return -ENXIO;
}
dup = find_ep(port, new->ep);
if (!dup)
list_add_tail(&new->list, &port->endpoints);
cxl_device_unlock(&port->dev);
device_unlock(&port->dev);
return dup ? -EEXIST : 0;
}
@@ -854,12 +856,12 @@ static void delete_endpoint(void *data)
goto out;
parent = &parent_port->dev;
cxl_device_lock(parent);
device_lock(parent);
if (parent->driver && endpoint->uport) {
devm_release_action(parent, cxl_unlink_uport, endpoint);
devm_release_action(parent, unregister_port, endpoint);
}
cxl_device_unlock(parent);
device_unlock(parent);
put_device(parent);
out:
put_device(&endpoint->dev);
@@ -920,7 +922,7 @@ static void cxl_detach_ep(void *data)
}
parent_port = to_cxl_port(port->dev.parent);
cxl_device_lock(&parent_port->dev);
device_lock(&parent_port->dev);
if (!parent_port->dev.driver) {
/*
* The bottom-up race to delete the port lost to a
@@ -928,12 +930,12 @@ static void cxl_detach_ep(void *data)
* parent_port ->remove() will have cleaned up all
* descendants.
*/
cxl_device_unlock(&parent_port->dev);
device_unlock(&parent_port->dev);
put_device(&port->dev);
continue;
}
cxl_device_lock(&port->dev);
device_lock(&port->dev);
ep = find_ep(port, &cxlmd->dev);
dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
@@ -948,7 +950,7 @@ static void cxl_detach_ep(void *data)
port->dead = true;
list_splice_init(&port->dports, &reap_dports);
}
cxl_device_unlock(&port->dev);
device_unlock(&port->dev);
if (!list_empty(&reap_dports)) {
dev_dbg(&cxlmd->dev, "delete %s\n",
@@ -956,7 +958,7 @@ static void cxl_detach_ep(void *data)
delete_switch_port(port, &reap_dports);
}
put_device(&port->dev);
cxl_device_unlock(&parent_port->dev);
device_unlock(&parent_port->dev);
}
}
@@ -1004,7 +1006,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
return -EAGAIN;
}
cxl_device_lock(&parent_port->dev);
device_lock(&parent_port->dev);
if (!parent_port->dev.driver) {
dev_warn(&cxlmd->dev,
"port %s:%s disabled, failed to enumerate CXL.mem\n",
@@ -1022,7 +1024,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
get_device(&port->dev);
}
out:
cxl_device_unlock(&parent_port->dev);
device_unlock(&parent_port->dev);
if (IS_ERR(port))
rc = PTR_ERR(port);
@@ -1133,14 +1135,14 @@ struct cxl_dport *cxl_find_dport_by_dev(struct cxl_port *port,
{
struct cxl_dport *dport;
cxl_device_lock(&port->dev);
device_lock(&port->dev);
list_for_each_entry(dport, &port->dports, list)
if (dport->dport == dev) {
cxl_device_unlock(&port->dev);
device_unlock(&port->dev);
return dport;
}
cxl_device_unlock(&port->dev);
device_unlock(&port->dev);
return NULL;
}
EXPORT_SYMBOL_NS_GPL(cxl_find_dport_by_dev, CXL);
@@ -1173,6 +1175,8 @@ static int decoder_populate_targets(struct cxl_decoder *cxld,
return rc;
}
static struct lock_class_key cxl_decoder_key;
/**
* cxl_decoder_alloc - Allocate a new CXL decoder
* @port: owning port of this decoder
@@ -1214,6 +1218,7 @@ static struct cxl_decoder *cxl_decoder_alloc(struct cxl_port *port,
seqlock_init(&cxld->target_lock);
dev = &cxld->dev;
device_initialize(dev);
lockdep_set_class(&dev->mutex, &cxl_decoder_key);
device_set_pm_not_required(dev);
dev->parent = &port->dev;
dev->bus = &cxl_bus_type;
@@ -1379,9 +1384,9 @@ int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
port = to_cxl_port(cxld->dev.parent);
cxl_device_lock(&port->dev);
device_lock(&port->dev);
rc = cxl_decoder_add_locked(cxld, target_map);
cxl_device_unlock(&port->dev);
device_unlock(&port->dev);
return rc;
}
@@ -1452,14 +1457,7 @@ static int cxl_bus_probe(struct device *dev)
{
int rc;
/*
* Take the CXL nested lock since the driver core only holds
* @dev->mutex and not @dev->lockdep_mutex.
*/
cxl_nested_lock(dev);
rc = to_cxl_drv(dev->driver)->probe(dev);
cxl_nested_unlock(dev);
dev_dbg(dev, "probe: %d\n", rc);
return rc;
}
@@ -1468,10 +1466,8 @@ static void cxl_bus_remove(struct device *dev)
{
struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
cxl_nested_lock(dev);
if (cxl_drv->remove)
cxl_drv->remove(dev);
cxl_nested_unlock(dev);
}
static struct workqueue_struct *cxl_bus_wq;

24
drivers/cxl/core/suspend.c Normal file
View File

@@ -0,0 +1,24 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
#include <linux/atomic.h>
#include <linux/export.h>
#include "cxlmem.h"
static atomic_t mem_active;
bool cxl_mem_active(void)
{
return atomic_read(&mem_active) != 0;
}
void cxl_mem_active_inc(void)
{
atomic_inc(&mem_active);
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_active_inc, CXL);
void cxl_mem_active_dec(void)
{
atomic_dec(&mem_active);
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_active_dec, CXL);

78
drivers/cxl/cxl.h
View File

@@ -405,82 +405,4 @@ struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_nvdimm *cxl_nvd);
#define __mock static
#endif
#ifdef CONFIG_PROVE_CXL_LOCKING
enum cxl_lock_class {
CXL_ANON_LOCK,
CXL_NVDIMM_LOCK,
CXL_NVDIMM_BRIDGE_LOCK,
CXL_PORT_LOCK,
/*
* Be careful to add new lock classes here, CXL_PORT_LOCK is
* extended by the port depth, so a maximum CXL port topology
* depth would need to be defined first.
*/
};
static inline void cxl_nested_lock(struct device *dev)
{
if (is_cxl_port(dev)) {
struct cxl_port *port = to_cxl_port(dev);
mutex_lock_nested(&dev->lockdep_mutex,
CXL_PORT_LOCK + port->depth);
} else if (is_cxl_decoder(dev)) {
struct cxl_port *port = to_cxl_port(dev->parent);
/*
* A decoder is the immediate child of a port, so set
* its lock class equal to other child device siblings.
*/
mutex_lock_nested(&dev->lockdep_mutex,
CXL_PORT_LOCK + port->depth + 1);
} else if (is_cxl_nvdimm_bridge(dev))
mutex_lock_nested(&dev->lockdep_mutex, CXL_NVDIMM_BRIDGE_LOCK);
else if (is_cxl_nvdimm(dev))
mutex_lock_nested(&dev->lockdep_mutex, CXL_NVDIMM_LOCK);
else
mutex_lock_nested(&dev->lockdep_mutex, CXL_ANON_LOCK);
}
static inline void cxl_nested_unlock(struct device *dev)
{
mutex_unlock(&dev->lockdep_mutex);
}
static inline void cxl_device_lock(struct device *dev)
{
/*
* For double lock errors the lockup will happen before lockdep
* warns at cxl_nested_lock(), so assert explicitly.
*/
lockdep_assert_not_held(&dev->lockdep_mutex);
device_lock(dev);
cxl_nested_lock(dev);
}
static inline void cxl_device_unlock(struct device *dev)
{
cxl_nested_unlock(dev);
device_unlock(dev);
}
#else
static inline void cxl_nested_lock(struct device *dev)
{
}
static inline void cxl_nested_unlock(struct device *dev)
{
}
static inline void cxl_device_lock(struct device *dev)
{
device_lock(dev);
}
static inline void cxl_device_unlock(struct device *dev)
{
device_unlock(dev);
}
#endif
#endif /* __CXL_H__ */

75
drivers/cxl/cxlmem.h
View File

@@ -85,9 +85,60 @@ struct cxl_mbox_cmd {
size_t size_in;
size_t size_out;
u16 return_code;
#define CXL_MBOX_SUCCESS 0
};
/*
* Per CXL 2.0 Section 8.2.8.4.5.1
*/
#define CMD_CMD_RC_TABLE \
C(SUCCESS, 0, NULL), \
C(BACKGROUND, -ENXIO, "background cmd started successfully"), \
C(INPUT, -ENXIO, "cmd input was invalid"), \
C(UNSUPPORTED, -ENXIO, "cmd is not supported"), \
C(INTERNAL, -ENXIO, "internal device error"), \
C(RETRY, -ENXIO, "temporary error, retry once"), \
C(BUSY, -ENXIO, "ongoing background operation"), \
C(MEDIADISABLED, -ENXIO, "media access is disabled"), \
C(FWINPROGRESS, -ENXIO, "one FW package can be transferred at a time"), \
C(FWOOO, -ENXIO, "FW package content was transferred out of order"), \
C(FWAUTH, -ENXIO, "FW package authentication failed"), \
C(FWSLOT, -ENXIO, "FW slot is not supported for requested operation"), \
C(FWROLLBACK, -ENXIO, "rolled back to the previous active FW"), \
C(FWRESET, -ENXIO, "FW failed to activate, needs cold reset"), \
C(HANDLE, -ENXIO, "one or more Event Record Handles were invalid"), \
C(PADDR, -ENXIO, "physical address specified is invalid"), \
C(POISONLMT, -ENXIO, "poison injection limit has been reached"), \
C(MEDIAFAILURE, -ENXIO, "permanent issue with the media"), \
C(ABORT, -ENXIO, "background cmd was aborted by device"), \
C(SECURITY, -ENXIO, "not valid in the current security state"), \
C(PASSPHRASE, -ENXIO, "phrase doesn't match current set passphrase"), \
C(MBUNSUPPORTED, -ENXIO, "unsupported on the mailbox it was issued on"),\
C(PAYLOADLEN, -ENXIO, "invalid payload length")
#undef C
#define C(a, b, c) CXL_MBOX_CMD_RC_##a
enum { CMD_CMD_RC_TABLE };
#undef C
#define C(a, b, c) { b, c }
struct cxl_mbox_cmd_rc {
int err;
const char *desc;
};
static const
struct cxl_mbox_cmd_rc cxl_mbox_cmd_rctable[] ={ CMD_CMD_RC_TABLE };
#undef C
static inline const char *cxl_mbox_cmd_rc2str(struct cxl_mbox_cmd *mbox_cmd)
{
return cxl_mbox_cmd_rctable[mbox_cmd->return_code].desc;
}
static inline int cxl_mbox_cmd_rc2errno(struct cxl_mbox_cmd *mbox_cmd)
{
return cxl_mbox_cmd_rctable[mbox_cmd->return_code].err;
}
/*
* CXL 2.0 - Memory capacity multiplier
* See Section 8.2.9.5
@@ -141,7 +192,6 @@ struct cxl_endpoint_dvsec_info {
* @info: Cached DVSEC information about the device.
* @serial: PCIe Device Serial Number
* @mbox_send: @dev specific transport for transmitting mailbox commands
* @wait_media_ready: @dev specific method to await media ready
*
* See section 8.2.9.5.2 Capacity Configuration and Label Storage for
* details on capacity parameters.
@@ -172,11 +222,9 @@ struct cxl_dev_state {
u64 next_persistent_bytes;
resource_size_t component_reg_phys;
struct cxl_endpoint_dvsec_info info;
u64 serial;
int (*mbox_send)(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd);
int (*wait_media_ready)(struct cxl_dev_state *cxlds);
};
enum cxl_opcode {
@@ -262,6 +310,13 @@ struct cxl_mbox_set_lsa {
u8 data[];
} __packed;
struct cxl_mbox_set_partition_info {
__le64 volatile_capacity;
u8 flags;
} __packed;
#define CXL_SET_PARTITION_IMMEDIATE_FLAG BIT(0)
/**
* struct cxl_mem_command - Driver representation of a memory device command
* @info: Command information as it exists for the UAPI
@@ -290,11 +345,23 @@ struct cxl_mem_command {
int cxl_mbox_send_cmd(struct cxl_dev_state *cxlds, u16 opcode, void *in,
size_t in_size, void *out, size_t out_size);
int cxl_dev_state_identify(struct cxl_dev_state *cxlds);
int cxl_await_media_ready(struct cxl_dev_state *cxlds);
int cxl_enumerate_cmds(struct cxl_dev_state *cxlds);
int cxl_mem_create_range_info(struct cxl_dev_state *cxlds);
struct cxl_dev_state *cxl_dev_state_create(struct device *dev);
void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds);
void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds);
#ifdef CONFIG_CXL_SUSPEND
void cxl_mem_active_inc(void);
void cxl_mem_active_dec(void);
#else
static inline void cxl_mem_active_inc(void)
{
}
static inline void cxl_mem_active_dec(void)
{
}
#endif
struct cxl_hdm {
struct cxl_component_regs regs;

2
drivers/cxl/cxlpci.h
View File

@@ -72,4 +72,6 @@ static inline resource_size_t cxl_regmap_to_base(struct pci_dev *pdev,
}
int devm_cxl_port_enumerate_dports(struct cxl_port *port);
struct cxl_dev_state;
int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm);
#endif /* __CXL_PCI_H__ */

148
drivers/cxl/mem.c
View File

@@ -24,27 +24,6 @@
* in higher level operations.
*/
static int wait_for_media(struct cxl_memdev *cxlmd)
{
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
int rc;
if (!info->mem_enabled)
return -EBUSY;
rc = cxlds->wait_media_ready(cxlds);
if (rc)
return rc;
/*
* We know the device is active, and enabled, if any ranges are non-zero
* we'll need to check later before adding the port since that owns the
* HDM decoder registers.
*/
return 0;
}
static int create_endpoint(struct cxl_memdev *cxlmd,
struct cxl_port *parent_port)
{
@@ -67,72 +46,14 @@ static int create_endpoint(struct cxl_memdev *cxlmd,
return cxl_endpoint_autoremove(cxlmd, endpoint);
}
/**
* cxl_dvsec_decode_init() - Setup HDM decoding for the endpoint
* @cxlds: Device state
*
* Additionally, enables global HDM decoding. Warning: don't call this outside
* of probe. Once probe is complete, the port driver owns all access to the HDM
* decoder registers.
*
* Returns: false if DVSEC Ranges are being used instead of HDM
* decoders, or if it can not be determined if DVSEC Ranges are in use.
* Otherwise, returns true.
*/
__mock bool cxl_dvsec_decode_init(struct cxl_dev_state *cxlds)
static void enable_suspend(void *data)
{
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
struct cxl_register_map map;
struct cxl_component_reg_map *cmap = &map.component_map;
bool global_enable, do_hdm_init = false;
void __iomem *crb;
u32 global_ctrl;
/* map hdm decoder */
crb = ioremap(cxlds->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);
if (!crb) {
dev_dbg(cxlds->dev, "Failed to map component registers\n");
return false;
}
cxl_probe_component_regs(cxlds->dev, crb, cmap);
if (!cmap->hdm_decoder.valid) {
dev_dbg(cxlds->dev, "Invalid HDM decoder registers\n");
goto out;
}
global_ctrl = readl(crb + cmap->hdm_decoder.offset +
CXL_HDM_DECODER_CTRL_OFFSET);
global_enable = global_ctrl & CXL_HDM_DECODER_ENABLE;
if (!global_enable && info->ranges) {
dev_dbg(cxlds->dev,
"DVSEC ranges already programmed and HDM decoders not enabled.\n");
goto out;
}
do_hdm_init = true;
/*
* Permanently (for this boot at least) opt the device into HDM
* operation. Individual HDM decoders still need to be enabled after
* this point.
*/
if (!global_enable) {
dev_dbg(cxlds->dev, "Enabling HDM decode\n");
writel(global_ctrl | CXL_HDM_DECODER_ENABLE,
crb + cmap->hdm_decoder.offset +
CXL_HDM_DECODER_CTRL_OFFSET);
}
out:
iounmap(crb);
return do_hdm_init;
cxl_mem_active_dec();
}
static int cxl_mem_probe(struct device *dev)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct cxl_port *parent_port;
int rc;
@@ -147,44 +68,6 @@ static int cxl_mem_probe(struct device *dev)
if (work_pending(&cxlmd->detach_work))
return -EBUSY;
rc = wait_for_media(cxlmd);
if (rc) {
dev_err(dev, "Media not active (%d)\n", rc);
return rc;
}
/*
* If DVSEC ranges are being used instead of HDM decoder registers there
* is no use in trying to manage those.
*/
if (!cxl_dvsec_decode_init(cxlds)) {
struct cxl_endpoint_dvsec_info *info = &cxlds->info;
int i;
/* */
for (i = 0; i < 2; i++) {
u64 base, size;
/*
* Give a nice warning to the user that BIOS has really
* botched things for them if it didn't place DVSEC
* ranges in the memory map.
*/
base = info->dvsec_range[i].start;
size = range_len(&info->dvsec_range[i]);
if (size && !region_intersects(base, size,
IORESOURCE_SYSTEM_RAM,
IORES_DESC_NONE)) {
dev_err(dev,
"DVSEC range %#llx-%#llx must be reserved by BIOS, but isn't\n",
base, base + size - 1);
}
}
dev_err(dev,
"Active DVSEC range registers in use. Will not bind.\n");
return -EBUSY;
}
rc = devm_cxl_enumerate_ports(cxlmd);
if (rc)
return rc;
@@ -195,19 +78,36 @@ static int cxl_mem_probe(struct device *dev)
return -ENXIO;
}
cxl_device_lock(&parent_port->dev);
device_lock(&parent_port->dev);
if (!parent_port->dev.driver) {
dev_err(dev, "CXL port topology %s not enabled\n",
dev_name(&parent_port->dev));
rc = -ENXIO;
goto out;
goto unlock;
}
rc = create_endpoint(cxlmd, parent_port);
out:
cxl_device_unlock(&parent_port->dev);
unlock:
device_unlock(&parent_port->dev);
put_device(&parent_port->dev);
return rc;
if (rc)
return rc;
/*
* The kernel may be operating out of CXL memory on this device,
* there is no spec defined way to determine whether this device
* preserves contents over suspend, and there is no simple way
* to arrange for the suspend image to avoid CXL memory which
* would setup a circular dependency between PCI resume and save
* state restoration.
*
* TODO: support suspend when all the regions this device is
* hosting are locked and covered by the system address map,
* i.e. platform firmware owns restoring the HDM configuration
* that it locked.
*/
cxl_mem_active_inc();
return devm_add_action_or_reset(dev, enable_suspend, NULL);
}
static struct cxl_driver cxl_mem_driver = {

Some files were not shown because too many files have changed in this diff Show More