mirror of
https://github.com/armbian/linux-cix.git
synced 2026-01-06 12:30:45 -08:00
Merge tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd
Pull iommufd implementation from Jason Gunthorpe: "iommufd is the user API to control the IOMMU subsystem as it relates to managing IO page tables that point at user space memory. It takes over from drivers/vfio/vfio_iommu_type1.c (aka the VFIO container) which is the VFIO specific interface for a similar idea. We see a broad need for extended features, some being highly IOMMU device specific: - Binding iommu_domain's to PASID/SSID - Userspace IO page tables, for ARM, x86 and S390 - Kernel bypassed invalidation of user page tables - Re-use of the KVM page table in the IOMMU - Dirty page tracking in the IOMMU - Runtime Increase/Decrease of IOPTE size - PRI support with faults resolved in userspace Many of these HW features exist to support VM use cases - for instance the combination of PASID, PRI and Userspace IO Page Tables allows an implementation of DMA Shared Virtual Addressing (vSVA) within a guest. Dirty tracking enables VM live migration with SRIOV devices and PASID support allow creating "scalable IOV" devices, among other things. As these features are fundamental to a VM platform they need to be uniformly exposed to all the driver families that do DMA into VMs, which is currently VFIO and VDPA" For more background, see the extended explanations in Jason's pull request: https://lore.kernel.org/lkml/Y5dzTU8dlmXTbzoJ@nvidia.com/ * tag 'for-linus-iommufd' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd: (62 commits) iommufd: Change the order of MSI setup iommufd: Improve a few unclear bits of code iommufd: Fix comment typos vfio: Move vfio group specific code into group.c vfio: Refactor dma APIs for emulated devices vfio: Wrap vfio group module init/clean code into helpers vfio: Refactor vfio_device open and close vfio: Make vfio_device_open() truly device specific vfio: Swap order of vfio_device_container_register() and open_device() vfio: Set device->group in helper function vfio: Create wrappers for group register/unregister vfio: Move the sanity check of the group to vfio_create_group() vfio: Simplify vfio_create_group() iommufd: Allow iommufd to supply /dev/vfio/vfio vfio: Make vfio_container optionally compiled vfio: Move container related MODULE_ALIAS statements into container.c vfio-iommufd: Support iommufd for emulated VFIO devices vfio-iommufd: Support iommufd for physical VFIO devices vfio-iommufd: Allow iommufd to be used in place of a container fd vfio: Use IOMMU_CAP_ENFORCE_CACHE_COHERENCY for vfio_file_enforced_coherent() ...
This commit is contained in:
Linus Torvalds
@@ -441,8 +441,11 @@ ForEachMacros:
|
||||
- 'inet_lhash2_for_each_icsk'
|
||||
- 'inet_lhash2_for_each_icsk_continue'
|
||||
- 'inet_lhash2_for_each_icsk_rcu'
|
||||
- 'interval_tree_for_each_double_span'
|
||||
- 'interval_tree_for_each_span'
|
||||
- 'intlist__for_each_entry'
|
||||
- 'intlist__for_each_entry_safe'
|
||||
- 'iopt_for_each_contig_area'
|
||||
- 'kcore_copy__for_each_phdr'
|
||||
- 'key_for_each'
|
||||
- 'key_for_each_safe'
|
||||
|
||||
@@ -25,6 +25,7 @@ place where this information is gathered.
|
||||
ebpf/index
|
||||
ioctl/index
|
||||
iommu
|
||||
iommufd
|
||||
media/index
|
||||
netlink/index
|
||||
sysfs-platform_profile
|
||||
|
||||
@@ -105,6 +105,7 @@ Code Seq# Include File Comments
|
||||
'8' all SNP8023 advanced NIC card
|
||||
<mailto:mcr@solidum.com>
|
||||
';' 64-7F linux/vfio.h
|
||||
';' 80-FF linux/iommufd.h
|
||||
'=' 00-3f uapi/linux/ptp_clock.h <mailto:richardcochran@gmail.com>
|
||||
'@' 00-0F linux/radeonfb.h conflict!
|
||||
'@' 00-0F drivers/video/aty/aty128fb.c conflict!
|
||||
|
||||
223
Documentation/userspace-api/iommufd.rst
Normal file
223
Documentation/userspace-api/iommufd.rst
Normal file
@@ -0,0 +1,223 @@
|
||||
.. SPDX-License-Identifier: GPL-2.0+
|
||||
|
||||
=======
|
||||
IOMMUFD
|
||||
=======
|
||||
|
||||
:Author: Jason Gunthorpe
|
||||
:Author: Kevin Tian
|
||||
|
||||
Overview
|
||||
========
|
||||
|
||||
IOMMUFD is the user API to control the IOMMU subsystem as it relates to managing
|
||||
IO page tables from userspace using file descriptors. It intends to be general
|
||||
and consumable by any driver that wants to expose DMA to userspace. These
|
||||
drivers are eventually expected to deprecate any internal IOMMU logic
|
||||
they may already/historically implement (e.g. vfio_iommu_type1.c).
|
||||
|
||||
At minimum iommufd provides universal support of managing I/O address spaces and
|
||||
I/O page tables for all IOMMUs, with room in the design to add non-generic
|
||||
features to cater to specific hardware functionality.
|
||||
|
||||
In this context the capital letter (IOMMUFD) refers to the subsystem while the
|
||||
small letter (iommufd) refers to the file descriptors created via /dev/iommu for
|
||||
use by userspace.
|
||||
|
||||
Key Concepts
|
||||
============
|
||||
|
||||
User Visible Objects
|
||||
--------------------
|
||||
|
||||
Following IOMMUFD objects are exposed to userspace:
|
||||
|
||||
- IOMMUFD_OBJ_IOAS, representing an I/O address space (IOAS), allowing map/unmap
|
||||
of user space memory into ranges of I/O Virtual Address (IOVA).
|
||||
|
||||
The IOAS is a functional replacement for the VFIO container, and like the VFIO
|
||||
container it copies an IOVA map to a list of iommu_domains held within it.
|
||||
|
||||
- IOMMUFD_OBJ_DEVICE, representing a device that is bound to iommufd by an
|
||||
external driver.
|
||||
|
||||
- IOMMUFD_OBJ_HW_PAGETABLE, representing an actual hardware I/O page table
|
||||
(i.e. a single struct iommu_domain) managed by the iommu driver.
|
||||
|
||||
The IOAS has a list of HW_PAGETABLES that share the same IOVA mapping and
|
||||
it will synchronize its mapping with each member HW_PAGETABLE.
|
||||
|
||||
All user-visible objects are destroyed via the IOMMU_DESTROY uAPI.
|
||||
|
||||
The diagram below shows relationship between user-visible objects and kernel
|
||||
datastructures (external to iommufd), with numbers referred to operations
|
||||
creating the objects and links::
|
||||
|
||||
_________________________________________________________
|
||||
| iommufd |
|
||||
| [1] |
|
||||
| _________________ |
|
||||
| | | |
|
||||
| | | |
|
||||
| | | |
|
||||
| | | |
|
||||
| | | |
|
||||
| | | |
|
||||
| | | [3] [2] |
|
||||
| | | ____________ __________ |
|
||||
| | IOAS |<--| |<------| | |
|
||||
| | | |HW_PAGETABLE| | DEVICE | |
|
||||
| | | |____________| |__________| |
|
||||
| | | | | |
|
||||
| | | | | |
|
||||
| | | | | |
|
||||
| | | | | |
|
||||
| | | | | |
|
||||
| |_________________| | | |
|
||||
| | | | |
|
||||
|_________|___________________|___________________|_______|
|
||||
| | |
|
||||
| _____v______ _______v_____
|
||||
| PFN storage | | | |
|
||||
|------------>|iommu_domain| |struct device|
|
||||
|____________| |_____________|
|
||||
|
||||
1. IOMMUFD_OBJ_IOAS is created via the IOMMU_IOAS_ALLOC uAPI. An iommufd can
|
||||
hold multiple IOAS objects. IOAS is the most generic object and does not
|
||||
expose interfaces that are specific to single IOMMU drivers. All operations
|
||||
on the IOAS must operate equally on each of the iommu_domains inside of it.
|
||||
|
||||
2. IOMMUFD_OBJ_DEVICE is created when an external driver calls the IOMMUFD kAPI
|
||||
to bind a device to an iommufd. The driver is expected to implement a set of
|
||||
ioctls to allow userspace to initiate the binding operation. Successful
|
||||
completion of this operation establishes the desired DMA ownership over the
|
||||
device. The driver must also set the driver_managed_dma flag and must not
|
||||
touch the device until this operation succeeds.
|
||||
|
||||
3. IOMMUFD_OBJ_HW_PAGETABLE is created when an external driver calls the IOMMUFD
|
||||
kAPI to attach a bound device to an IOAS. Similarly the external driver uAPI
|
||||
allows userspace to initiate the attaching operation. If a compatible
|
||||
pagetable already exists then it is reused for the attachment. Otherwise a
|
||||
new pagetable object and iommu_domain is created. Successful completion of
|
||||
this operation sets up the linkages among IOAS, device and iommu_domain. Once
|
||||
this completes the device could do DMA.
|
||||
|
||||
Every iommu_domain inside the IOAS is also represented to userspace as a
|
||||
HW_PAGETABLE object.
|
||||
|
||||
.. note::
|
||||
|
||||
Future IOMMUFD updates will provide an API to create and manipulate the
|
||||
HW_PAGETABLE directly.
|
||||
|
||||
A device can only bind to an iommufd due to DMA ownership claim and attach to at
|
||||
most one IOAS object (no support of PASID yet).
|
||||
|
||||
Kernel Datastructure
|
||||
--------------------
|
||||
|
||||
User visible objects are backed by following datastructures:
|
||||
|
||||
- iommufd_ioas for IOMMUFD_OBJ_IOAS.
|
||||
- iommufd_device for IOMMUFD_OBJ_DEVICE.
|
||||
- iommufd_hw_pagetable for IOMMUFD_OBJ_HW_PAGETABLE.
|
||||
|
||||
Several terminologies when looking at these datastructures:
|
||||
|
||||
- Automatic domain - refers to an iommu domain created automatically when
|
||||
attaching a device to an IOAS object. This is compatible to the semantics of
|
||||
VFIO type1.
|
||||
|
||||
- Manual domain - refers to an iommu domain designated by the user as the
|
||||
target pagetable to be attached to by a device. Though currently there are
|
||||
no uAPIs to directly create such domain, the datastructure and algorithms
|
||||
are ready for handling that use case.
|
||||
|
||||
- In-kernel user - refers to something like a VFIO mdev that is using the
|
||||
IOMMUFD access interface to access the IOAS. This starts by creating an
|
||||
iommufd_access object that is similar to the domain binding a physical device
|
||||
would do. The access object will then allow converting IOVA ranges into struct
|
||||
page * lists, or doing direct read/write to an IOVA.
|
||||
|
||||
iommufd_ioas serves as the metadata datastructure to manage how IOVA ranges are
|
||||
mapped to memory pages, composed of:
|
||||
|
||||
- struct io_pagetable holding the IOVA map
|
||||
- struct iopt_area's representing populated portions of IOVA
|
||||
- struct iopt_pages representing the storage of PFNs
|
||||
- struct iommu_domain representing the IO page table in the IOMMU
|
||||
- struct iopt_pages_access representing in-kernel users of PFNs
|
||||
- struct xarray pinned_pfns holding a list of pages pinned by in-kernel users
|
||||
|
||||
Each iopt_pages represents a logical linear array of full PFNs. The PFNs are
|
||||
ultimately derived from userspace VAs via an mm_struct. Once they have been
|
||||
pinned the PFNs are stored in IOPTEs of an iommu_domain or inside the pinned_pfns
|
||||
xarray if they have been pinned through an iommufd_access.
|
||||
|
||||
PFN have to be copied between all combinations of storage locations, depending
|
||||
on what domains are present and what kinds of in-kernel "software access" users
|
||||
exist. The mechanism ensures that a page is pinned only once.
|
||||
|
||||
An io_pagetable is composed of iopt_areas pointing at iopt_pages, along with a
|
||||
list of iommu_domains that mirror the IOVA to PFN map.
|
||||
|
||||
Multiple io_pagetable-s, through their iopt_area-s, can share a single
|
||||
iopt_pages which avoids multi-pinning and double accounting of page
|
||||
consumption.
|
||||
|
||||
iommufd_ioas is sharable between subsystems, e.g. VFIO and VDPA, as long as
|
||||
devices managed by different subsystems are bound to a same iommufd.
|
||||
|
||||
IOMMUFD User API
|
||||
================
|
||||
|
||||
.. kernel-doc:: include/uapi/linux/iommufd.h
|
||||
|
||||
IOMMUFD Kernel API
|
||||
==================
|
||||
|
||||
The IOMMUFD kAPI is device-centric with group-related tricks managed behind the
|
||||
scene. This allows the external drivers calling such kAPI to implement a simple
|
||||
device-centric uAPI for connecting its device to an iommufd, instead of
|
||||
explicitly imposing the group semantics in its uAPI as VFIO does.
|
||||
|
||||
.. kernel-doc:: drivers/iommu/iommufd/device.c
|
||||
:export:
|
||||
|
||||
.. kernel-doc:: drivers/iommu/iommufd/main.c
|
||||
:export:
|
||||
|
||||
VFIO and IOMMUFD
|
||||
----------------
|
||||
|
||||
Connecting a VFIO device to iommufd can be done in two ways.
|
||||
|
||||
First is a VFIO compatible way by directly implementing the /dev/vfio/vfio
|
||||
container IOCTLs by mapping them into io_pagetable operations. Doing so allows
|
||||
the use of iommufd in legacy VFIO applications by symlinking /dev/vfio/vfio to
|
||||
/dev/iommufd or extending VFIO to SET_CONTAINER using an iommufd instead of a
|
||||
container fd.
|
||||
|
||||
The second approach directly extends VFIO to support a new set of device-centric
|
||||
user API based on aforementioned IOMMUFD kernel API. It requires userspace
|
||||
change but better matches the IOMMUFD API semantics and easier to support new
|
||||
iommufd features when comparing it to the first approach.
|
||||
|
||||
Currently both approaches are still work-in-progress.
|
||||
|
||||
There are still a few gaps to be resolved to catch up with VFIO type1, as
|
||||
documented in iommufd_vfio_check_extension().
|
||||
|
||||
Future TODOs
|
||||
============
|
||||
|
||||
Currently IOMMUFD supports only kernel-managed I/O page table, similar to VFIO
|
||||
type1. New features on the radar include:
|
||||
|
||||
- Binding iommu_domain's to PASID/SSID
|
||||
- Userspace page tables, for ARM, x86 and S390
|
||||
- Kernel bypass'd invalidation of user page tables
|
||||
- Re-use of the KVM page table in the IOMMU
|
||||
- Dirty page tracking in the IOMMU
|
||||
- Runtime Increase/Decrease of IOPTE size
|
||||
- PRI support with faults resolved in userspace
|
||||
12
MAINTAINERS
12
MAINTAINERS
@@ -10793,6 +10793,18 @@ F: drivers/iommu/dma-iommu.h
|
||||
F: drivers/iommu/iova.c
|
||||
F: include/linux/iova.h
|
||||
|
||||
IOMMUFD
|
||||
M: Jason Gunthorpe <jgg@nvidia.com>
|
||||
M: Kevin Tian <kevin.tian@intel.com>
|
||||
L: iommu@lists.linux.dev
|
||||
S: Maintained
|
||||
T: git git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git
|
||||
F: Documentation/userspace-api/iommufd.rst
|
||||
F: drivers/iommu/iommufd/
|
||||
F: include/linux/iommufd.h
|
||||
F: include/uapi/linux/iommufd.h
|
||||
F: tools/testing/selftests/iommu/
|
||||
|
||||
IOMMU SUBSYSTEM
|
||||
M: Joerg Roedel <joro@8bytes.org>
|
||||
M: Will Deacon <will@kernel.org>
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#include <linux/pci.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/sched/task.h>
|
||||
#include <linux/intel-svm.h>
|
||||
#include <linux/io-64-nonatomic-lo-hi.h>
|
||||
#include <linux/cdev.h>
|
||||
#include <linux/fs.h>
|
||||
@@ -100,7 +99,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
|
||||
filp->private_data = ctx;
|
||||
|
||||
if (device_user_pasid_enabled(idxd)) {
|
||||
sva = iommu_sva_bind_device(dev, current->mm, NULL);
|
||||
sva = iommu_sva_bind_device(dev, current->mm);
|
||||
if (IS_ERR(sva)) {
|
||||
rc = PTR_ERR(sva);
|
||||
dev_err(dev, "pasid allocation failed: %d\n", rc);
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
#include <linux/io-64-nonatomic-lo-hi.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/idr.h>
|
||||
#include <linux/intel-svm.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <uapi/linux/idxd.h>
|
||||
#include <linux/dmaengine.h>
|
||||
@@ -502,29 +501,7 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d
|
||||
|
||||
static int idxd_enable_system_pasid(struct idxd_device *idxd)
|
||||
{
|
||||
int flags;
|
||||
unsigned int pasid;
|
||||
struct iommu_sva *sva;
|
||||
|
||||
flags = SVM_FLAG_SUPERVISOR_MODE;
|
||||
|
||||
sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
|
||||
if (IS_ERR(sva)) {
|
||||
dev_warn(&idxd->pdev->dev,
|
||||
"iommu sva bind failed: %ld\n", PTR_ERR(sva));
|
||||
return PTR_ERR(sva);
|
||||
}
|
||||
|
||||
pasid = iommu_sva_get_pasid(sva);
|
||||
if (pasid == IOMMU_PASID_INVALID) {
|
||||
iommu_sva_unbind_device(sva);
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
idxd->sva = sva;
|
||||
idxd->pasid = pasid;
|
||||
dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
|
||||
return 0;
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
||||
static void idxd_disable_system_pasid(struct idxd_device *idxd)
|
||||
|
||||
@@ -669,9 +669,6 @@ static int intel_vgpu_open_device(struct vfio_device *vfio_dev)
|
||||
|
||||
vgpu->attached = true;
|
||||
|
||||
kvmgt_protect_table_init(vgpu);
|
||||
gvt_cache_init(vgpu);
|
||||
|
||||
vgpu->track_node.track_write = kvmgt_page_track_write;
|
||||
vgpu->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
|
||||
kvm_get_kvm(vgpu->vfio_device.kvm);
|
||||
@@ -715,6 +712,11 @@ static void intel_vgpu_close_device(struct vfio_device *vfio_dev)
|
||||
kvmgt_protect_table_destroy(vgpu);
|
||||
gvt_cache_destroy(vgpu);
|
||||
|
||||
WARN_ON(vgpu->nr_cache_entries);
|
||||
|
||||
vgpu->gfn_cache = RB_ROOT;
|
||||
vgpu->dma_addr_cache = RB_ROOT;
|
||||
|
||||
intel_vgpu_release_msi_eventfd_ctx(vgpu);
|
||||
|
||||
vgpu->attached = false;
|
||||
@@ -1445,9 +1447,17 @@ static int intel_vgpu_init_dev(struct vfio_device *vfio_dev)
|
||||
struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);
|
||||
struct intel_vgpu_type *type =
|
||||
container_of(mdev->type, struct intel_vgpu_type, type);
|
||||
int ret;
|
||||
|
||||
vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt;
|
||||
return intel_gvt_create_vgpu(vgpu, type->conf);
|
||||
ret = intel_gvt_create_vgpu(vgpu, type->conf);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
kvmgt_protect_table_init(vgpu);
|
||||
gvt_cache_init(vgpu);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)
|
||||
@@ -1468,6 +1478,9 @@ static const struct vfio_device_ops intel_vgpu_dev_ops = {
|
||||
.mmap = intel_vgpu_mmap,
|
||||
.ioctl = intel_vgpu_ioctl,
|
||||
.dma_unmap = intel_vgpu_dma_unmap,
|
||||
.bind_iommufd = vfio_iommufd_emulated_bind,
|
||||
.unbind_iommufd = vfio_iommufd_emulated_unbind,
|
||||
.attach_ioas = vfio_iommufd_emulated_attach_ioas,
|
||||
};
|
||||
|
||||
static int intel_vgpu_probe(struct mdev_device *mdev)
|
||||
|
||||
@@ -188,6 +188,7 @@ config MSM_IOMMU
|
||||
|
||||
source "drivers/iommu/amd/Kconfig"
|
||||
source "drivers/iommu/intel/Kconfig"
|
||||
source "drivers/iommu/iommufd/Kconfig"
|
||||
|
||||
config IRQ_REMAP
|
||||
bool "Support for Interrupt Remapping"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
obj-y += amd/ intel/ arm/
|
||||
obj-y += amd/ intel/ arm/ iommufd/
|
||||
obj-$(CONFIG_IOMMU_API) += iommu.o
|
||||
obj-$(CONFIG_IOMMU_API) += iommu-traces.o
|
||||
obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
|
||||
@@ -28,6 +28,6 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
|
||||
obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
|
||||
obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o
|
||||
obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o
|
||||
obj-$(CONFIG_IOMMU_SVA) += iommu-sva-lib.o io-pgfault.o
|
||||
obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o
|
||||
obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
|
||||
obj-$(CONFIG_APPLE_DART) += apple-dart.o
|
||||
|
||||
@@ -2155,21 +2155,13 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
|
||||
static int amd_iommu_attach_device(struct iommu_domain *dom,
|
||||
struct device *dev)
|
||||
{
|
||||
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
|
||||
struct protection_domain *domain = to_pdomain(dom);
|
||||
struct iommu_dev_data *dev_data;
|
||||
struct amd_iommu *iommu;
|
||||
struct amd_iommu *iommu = rlookup_amd_iommu(dev);
|
||||
int ret;
|
||||
|
||||
if (!check_device(dev))
|
||||
return -EINVAL;
|
||||
|
||||
dev_data = dev_iommu_priv_get(dev);
|
||||
dev_data->defer_attach = false;
|
||||
|
||||
iommu = rlookup_amd_iommu(dev);
|
||||
if (!iommu)
|
||||
return -EINVAL;
|
||||
|
||||
if (dev_data->domain)
|
||||
detach_device(dev);
|
||||
|
||||
@@ -2286,6 +2278,8 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap)
|
||||
return false;
|
||||
case IOMMU_CAP_PRE_BOOT_PROTECTION:
|
||||
return amdr_ivrs_remap_support;
|
||||
case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
|
||||
return true;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
#include <linux/slab.h>
|
||||
|
||||
#include "arm-smmu-v3.h"
|
||||
#include "../../iommu-sva-lib.h"
|
||||
#include "../../iommu-sva.h"
|
||||
#include "../../io-pgtable-arm.h"
|
||||
|
||||
struct arm_smmu_mmu_notifier {
|
||||
@@ -344,11 +344,6 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
|
||||
if (!bond)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
/* Allocate a PASID for this mm if necessary */
|
||||
ret = iommu_sva_alloc_pasid(mm, 1, (1U << master->ssid_bits) - 1);
|
||||
if (ret)
|
||||
goto err_free_bond;
|
||||
|
||||
bond->mm = mm;
|
||||
bond->sva.dev = dev;
|
||||
refcount_set(&bond->refs, 1);
|
||||
@@ -367,42 +362,6 @@ err_free_bond:
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
struct iommu_sva *
|
||||
arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
|
||||
{
|
||||
struct iommu_sva *handle;
|
||||
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
|
||||
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
||||
|
||||
if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
mutex_lock(&sva_lock);
|
||||
handle = __arm_smmu_sva_bind(dev, mm);
|
||||
mutex_unlock(&sva_lock);
|
||||
return handle;
|
||||
}
|
||||
|
||||
void arm_smmu_sva_unbind(struct iommu_sva *handle)
|
||||
{
|
||||
struct arm_smmu_bond *bond = sva_to_bond(handle);
|
||||
|
||||
mutex_lock(&sva_lock);
|
||||
if (refcount_dec_and_test(&bond->refs)) {
|
||||
list_del(&bond->list);
|
||||
arm_smmu_mmu_notifier_put(bond->smmu_mn);
|
||||
kfree(bond);
|
||||
}
|
||||
mutex_unlock(&sva_lock);
|
||||
}
|
||||
|
||||
u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
|
||||
{
|
||||
struct arm_smmu_bond *bond = sva_to_bond(handle);
|
||||
|
||||
return bond->mm->pasid;
|
||||
}
|
||||
|
||||
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
|
||||
{
|
||||
unsigned long reg, fld;
|
||||
@@ -550,3 +509,64 @@ void arm_smmu_sva_notifier_synchronize(void)
|
||||
*/
|
||||
mmu_notifier_synchronize();
|
||||
}
|
||||
|
||||
void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
|
||||
struct device *dev, ioasid_t id)
|
||||
{
|
||||
struct mm_struct *mm = domain->mm;
|
||||
struct arm_smmu_bond *bond = NULL, *t;
|
||||
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
|
||||
|
||||
mutex_lock(&sva_lock);
|
||||
list_for_each_entry(t, &master->bonds, list) {
|
||||
if (t->mm == mm) {
|
||||
bond = t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!WARN_ON(!bond) && refcount_dec_and_test(&bond->refs)) {
|
||||
list_del(&bond->list);
|
||||
arm_smmu_mmu_notifier_put(bond->smmu_mn);
|
||||
kfree(bond);
|
||||
}
|
||||
mutex_unlock(&sva_lock);
|
||||
}
|
||||
|
||||
static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
|
||||
struct device *dev, ioasid_t id)
|
||||
{
|
||||
int ret = 0;
|
||||
struct iommu_sva *handle;
|
||||
struct mm_struct *mm = domain->mm;
|
||||
|
||||
mutex_lock(&sva_lock);
|
||||
handle = __arm_smmu_sva_bind(dev, mm);
|
||||
if (IS_ERR(handle))
|
||||
ret = PTR_ERR(handle);
|
||||
mutex_unlock(&sva_lock);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
|
||||
{
|
||||
kfree(domain);
|
||||
}
|
||||
|
||||
static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
|
||||
.set_dev_pasid = arm_smmu_sva_set_dev_pasid,
|
||||
.free = arm_smmu_sva_domain_free
|
||||
};
|
||||
|
||||
struct iommu_domain *arm_smmu_sva_domain_alloc(void)
|
||||
{
|
||||
struct iommu_domain *domain;
|
||||
|
||||
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
|
||||
if (!domain)
|
||||
return NULL;
|
||||
domain->ops = &arm_smmu_sva_domain_ops;
|
||||
|
||||
return domain;
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
|
||||
#include "arm-smmu-v3.h"
|
||||
#include "../../dma-iommu.h"
|
||||
#include "../../iommu-sva-lib.h"
|
||||
#include "../../iommu-sva.h"
|
||||
|
||||
static bool disable_bypass = true;
|
||||
module_param(disable_bypass, bool, 0444);
|
||||
@@ -2009,6 +2009,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
|
||||
{
|
||||
struct arm_smmu_domain *smmu_domain;
|
||||
|
||||
if (type == IOMMU_DOMAIN_SVA)
|
||||
return arm_smmu_sva_domain_alloc();
|
||||
|
||||
if (type != IOMMU_DOMAIN_UNMANAGED &&
|
||||
type != IOMMU_DOMAIN_DMA &&
|
||||
type != IOMMU_DOMAIN_DMA_FQ &&
|
||||
@@ -2430,23 +2433,14 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
|
||||
goto out_unlock;
|
||||
}
|
||||
} else if (smmu_domain->smmu != smmu) {
|
||||
dev_err(dev,
|
||||
"cannot attach to SMMU %s (upstream of %s)\n",
|
||||
dev_name(smmu_domain->smmu->dev),
|
||||
dev_name(smmu->dev));
|
||||
ret = -ENXIO;
|
||||
ret = -EINVAL;
|
||||
goto out_unlock;
|
||||
} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
|
||||
master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
|
||||
dev_err(dev,
|
||||
"cannot attach to incompatible domain (%u SSID bits != %u)\n",
|
||||
smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
|
||||
ret = -EINVAL;
|
||||
goto out_unlock;
|
||||
} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
|
||||
smmu_domain->stall_enabled != master->stall_enabled) {
|
||||
dev_err(dev, "cannot attach to stall-%s domain\n",
|
||||
smmu_domain->stall_enabled ? "enabled" : "disabled");
|
||||
ret = -EINVAL;
|
||||
goto out_unlock;
|
||||
}
|
||||
@@ -2838,6 +2832,17 @@ static int arm_smmu_def_domain_type(struct device *dev)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
|
||||
{
|
||||
struct iommu_domain *domain;
|
||||
|
||||
domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
|
||||
if (WARN_ON(IS_ERR(domain)) || !domain)
|
||||
return;
|
||||
|
||||
arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
|
||||
}
|
||||
|
||||
static struct iommu_ops arm_smmu_ops = {
|
||||
.capable = arm_smmu_capable,
|
||||
.domain_alloc = arm_smmu_domain_alloc,
|
||||
@@ -2846,11 +2851,9 @@ static struct iommu_ops arm_smmu_ops = {
|
||||
.device_group = arm_smmu_device_group,
|
||||
.of_xlate = arm_smmu_of_xlate,
|
||||
.get_resv_regions = arm_smmu_get_resv_regions,
|
||||
.remove_dev_pasid = arm_smmu_remove_dev_pasid,
|
||||
.dev_enable_feat = arm_smmu_dev_enable_feature,
|
||||
.dev_disable_feat = arm_smmu_dev_disable_feature,
|
||||
.sva_bind = arm_smmu_sva_bind,
|
||||
.sva_unbind = arm_smmu_sva_unbind,
|
||||
.sva_get_pasid = arm_smmu_sva_get_pasid,
|
||||
.page_response = arm_smmu_page_response,
|
||||
.def_domain_type = arm_smmu_def_domain_type,
|
||||
.pgsize_bitmap = -1UL, /* Restricted during device attach */
|
||||
@@ -3543,6 +3546,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
|
||||
/* SID/SSID sizes */
|
||||
smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
|
||||
smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
|
||||
smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
|
||||
|
||||
/*
|
||||
* If the SMMU supports fewer bits than would fill a single L2 stream
|
||||
|
||||
@@ -754,11 +754,10 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
|
||||
int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
|
||||
int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
|
||||
bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
|
||||
struct iommu_sva *arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm,
|
||||
void *drvdata);
|
||||
void arm_smmu_sva_unbind(struct iommu_sva *handle);
|
||||
u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle);
|
||||
void arm_smmu_sva_notifier_synchronize(void);
|
||||
struct iommu_domain *arm_smmu_sva_domain_alloc(void);
|
||||
void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
|
||||
struct device *dev, ioasid_t id);
|
||||
#else /* CONFIG_ARM_SMMU_V3_SVA */
|
||||
static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
|
||||
{
|
||||
@@ -790,19 +789,17 @@ static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline struct iommu_sva *
|
||||
arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
|
||||
{
|
||||
return ERR_PTR(-ENODEV);
|
||||
}
|
||||
|
||||
static inline void arm_smmu_sva_unbind(struct iommu_sva *handle) {}
|
||||
|
||||
static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
|
||||
{
|
||||
return IOMMU_PASID_INVALID;
|
||||
}
|
||||
|
||||
static inline void arm_smmu_sva_notifier_synchronize(void) {}
|
||||
|
||||
static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
|
||||
struct device *dev,
|
||||
ioasid_t id)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_ARM_SMMU_V3_SVA */
|
||||
#endif /* _ARM_SMMU_V3_H */
|
||||
|
||||
@@ -1150,9 +1150,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
|
||||
* different SMMUs.
|
||||
*/
|
||||
if (smmu_domain->smmu != smmu) {
|
||||
dev_err(dev,
|
||||
"cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
|
||||
dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
|
||||
ret = -EINVAL;
|
||||
goto rpm_put;
|
||||
}
|
||||
|
||||
@@ -381,13 +381,8 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev
|
||||
* Sanity check the domain. We don't support domains across
|
||||
* different IOMMUs.
|
||||
*/
|
||||
if (qcom_domain->iommu != qcom_iommu) {
|
||||
dev_err(dev, "cannot attach to IOMMU %s while already "
|
||||
"attached to domain on IOMMU %s\n",
|
||||
dev_name(qcom_domain->iommu->dev),
|
||||
dev_name(qcom_iommu->dev));
|
||||
if (qcom_domain->iommu != qcom_iommu)
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -211,7 +211,7 @@ int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot)
|
||||
ppaace->op_encode.index_ot.omi = omi;
|
||||
} else if (~omi != 0) {
|
||||
pr_debug("bad operation mapping index: %d\n", omi);
|
||||
return -EINVAL;
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
/* configure stash id */
|
||||
|
||||
@@ -258,7 +258,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
|
||||
liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
|
||||
if (!liodn) {
|
||||
pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
|
||||
return -EINVAL;
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
spin_lock_irqsave(&dma_domain->domain_lock, flags);
|
||||
@@ -267,7 +267,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
|
||||
if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
|
||||
pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
|
||||
liodn[i], dev->of_node);
|
||||
ret = -EINVAL;
|
||||
ret = -ENODEV;
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -1105,6 +1105,13 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
|
||||
|
||||
raw_spin_lock_init(&iommu->register_lock);
|
||||
|
||||
/*
|
||||
* A value of N in PSS field of eCap register indicates hardware
|
||||
* supports PASID field of N+1 bits.
|
||||
*/
|
||||
if (pasid_supported(iommu))
|
||||
iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
|
||||
|
||||
/*
|
||||
* This is only for hotplug; at boot time intel_iommu_enabled won't
|
||||
* be set yet. When intel_iommu_init() runs, it registers the units
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
#include "iommu.h"
|
||||
#include "../dma-iommu.h"
|
||||
#include "../irq_remapping.h"
|
||||
#include "../iommu-sva-lib.h"
|
||||
#include "../iommu-sva.h"
|
||||
#include "pasid.h"
|
||||
#include "cap_audit.h"
|
||||
|
||||
@@ -4188,6 +4188,8 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
|
||||
return domain;
|
||||
case IOMMU_DOMAIN_IDENTITY:
|
||||
return &si_domain->domain;
|
||||
case IOMMU_DOMAIN_SVA:
|
||||
return intel_svm_domain_alloc();
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
@@ -4213,19 +4215,15 @@ static int prepare_domain_attach_device(struct iommu_domain *domain,
|
||||
return -ENODEV;
|
||||
|
||||
if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
|
||||
return -EOPNOTSUPP;
|
||||
return -EINVAL;
|
||||
|
||||
/* check if this iommu agaw is sufficient for max mapped address */
|
||||
addr_width = agaw_to_width(iommu->agaw);
|
||||
if (addr_width > cap_mgaw(iommu->cap))
|
||||
addr_width = cap_mgaw(iommu->cap);
|
||||
|
||||
if (dmar_domain->max_addr > (1LL << addr_width)) {
|
||||
dev_err(dev, "%s: iommu width (%d) is not "
|
||||
"sufficient for the mapped address (%llx)\n",
|
||||
__func__, addr_width, dmar_domain->max_addr);
|
||||
return -EFAULT;
|
||||
}
|
||||
if (dmar_domain->max_addr > (1LL << addr_width))
|
||||
return -EINVAL;
|
||||
dmar_domain->gaw = addr_width;
|
||||
|
||||
/*
|
||||
@@ -4471,14 +4469,20 @@ static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain)
|
||||
|
||||
static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
|
||||
{
|
||||
if (cap == IOMMU_CAP_CACHE_COHERENCY)
|
||||
return true;
|
||||
if (cap == IOMMU_CAP_INTR_REMAP)
|
||||
return irq_remapping_enabled == 1;
|
||||
if (cap == IOMMU_CAP_PRE_BOOT_PROTECTION)
|
||||
return dmar_platform_optin();
|
||||
struct device_domain_info *info = dev_iommu_priv_get(dev);
|
||||
|
||||
return false;
|
||||
switch (cap) {
|
||||
case IOMMU_CAP_CACHE_COHERENCY:
|
||||
return true;
|
||||
case IOMMU_CAP_INTR_REMAP:
|
||||
return irq_remapping_enabled == 1;
|
||||
case IOMMU_CAP_PRE_BOOT_PROTECTION:
|
||||
return dmar_platform_optin();
|
||||
case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
|
||||
return ecap_sc_support(info->iommu->ecap);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static struct iommu_device *intel_iommu_probe_device(struct device *dev)
|
||||
@@ -4732,6 +4736,28 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
|
||||
__mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
|
||||
}
|
||||
|
||||
static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
|
||||
{
|
||||
struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
|
||||
struct iommu_domain *domain;
|
||||
|
||||
/* Domain type specific cleanup: */
|
||||
domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
|
||||
if (domain) {
|
||||
switch (domain->type) {
|
||||
case IOMMU_DOMAIN_SVA:
|
||||
intel_svm_remove_dev_pasid(dev, pasid);
|
||||
break;
|
||||
default:
|
||||
/* should never reach here */
|
||||
WARN_ON(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
intel_pasid_tear_down_entry(iommu, dev, pasid, false);
|
||||
}
|
||||
|
||||
const struct iommu_ops intel_iommu_ops = {
|
||||
.capable = intel_iommu_capable,
|
||||
.domain_alloc = intel_iommu_domain_alloc,
|
||||
@@ -4744,11 +4770,9 @@ const struct iommu_ops intel_iommu_ops = {
|
||||
.dev_disable_feat = intel_iommu_dev_disable_feat,
|
||||
.is_attach_deferred = intel_iommu_is_attach_deferred,
|
||||
.def_domain_type = device_def_domain_type,
|
||||
.remove_dev_pasid = intel_iommu_remove_dev_pasid,
|
||||
.pgsize_bitmap = SZ_4K,
|
||||
#ifdef CONFIG_INTEL_IOMMU_SVM
|
||||
.sva_bind = intel_svm_bind,
|
||||
.sva_unbind = intel_svm_unbind,
|
||||
.sva_get_pasid = intel_svm_get_pasid,
|
||||
.page_response = intel_svm_page_response,
|
||||
#endif
|
||||
.default_domain_ops = &(const struct iommu_domain_ops) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user