A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-28 18:51:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge branches 'arm/rockchip', 'arm/exynos', 'arm/smmu', 'x86/vt-d', 'x86/amd', 'default-domains' and 'core' into next

Browse source
This commit is contained in:
Joerg Roedel 2015-06-19 17:17:47 +02:00
commit 5ffde2f671
1078 changed files with 15514 additions and 8689 deletions
Download patch file Download diff file Expand all files Collapse all files

13
drivers/iommu/Kconfig
View file

@ -339,6 +339,7 @@ config SPAPR_TCE_IOMMU
Enables bits of IOMMU API required by VFIO. The iommu_ops
is not implemented as it is not necessary for VFIO.
# ARM IOMMU support
config ARM_SMMU
bool "ARM Ltd. System MMU (SMMU) Support"
depends on (ARM64 || ARM) && MMU
@ -352,4 +353,16 @@ config ARM_SMMU
Say Y here if your SoC includes an IOMMU device implementing
the ARM SMMU architecture.
config ARM_SMMU_V3
bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support"
depends on ARM64 && PCI
select IOMMU_API
select IOMMU_IO_PGTABLE_LPAE
help
Support for implementations of the ARM System MMU architecture
version 3 providing translation support to a PCIe root complex.
Say Y here if your system includes an IOMMU device implementing
the ARM SMMUv3 architecture.
endif # IOMMU_SUPPORT

1
drivers/iommu/Makefile
View file

@ -9,6 +9,7 @@ obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
obj-$(CONFIG_DMAR_TABLE) += dmar.o
obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o
obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o

602
drivers/iommu/amd_iommu.c
View file

@ -64,10 +64,6 @@
static DEFINE_RWLOCK(amd_iommu_devtable_lock);
/* A list of preallocated protection domains */
static LIST_HEAD(iommu_pd_list);
static DEFINE_SPINLOCK(iommu_pd_list_lock);
/* List of all available dev_data structures */
static LIST_HEAD(dev_data_list);
static DEFINE_SPINLOCK(dev_data_list_lock);
@ -119,7 +115,7 @@ struct iommu_cmd {
struct kmem_cache *amd_iommu_irq_cache;
static void update_domain(struct protection_domain *domain);
static int __init alloc_passthrough_domain(void);
static int alloc_passthrough_domain(void);
/****************************************************************************
*
@ -234,31 +230,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
}
/*
* In this function the list of preallocated protection domains is traversed to
* find the domain for a specific device
* This function actually applies the mapping to the page table of the
* dma_ops domain.
*/
static struct dma_ops_domain *find_protection_domain(u16 devid)
static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,
struct unity_map_entry *e)
{
struct dma_ops_domain *entry, *ret = NULL;
unsigned long flags;
u16 alias = amd_iommu_alias_table[devid];
u64 addr;
if (list_empty(&iommu_pd_list))
return NULL;
spin_lock_irqsave(&iommu_pd_list_lock, flags);
list_for_each_entry(entry, &iommu_pd_list, list) {
if (entry->target_dev == devid ||
entry->target_dev == alias) {
ret = entry;
break;
}
for (addr = e->address_start; addr < e->address_end;
addr += PAGE_SIZE) {
if (addr < dma_dom->aperture_size)
__set_bit(addr >> PAGE_SHIFT,
dma_dom->aperture[0]->bitmap);
}
}
spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
/*
* Inits the unity mappings required for a specific device
*/
static void init_unity_mappings_for_device(struct device *dev,
struct dma_ops_domain *dma_dom)
{
struct unity_map_entry *e;
u16 devid;
return ret;
devid = get_device_id(dev);
list_for_each_entry(e, &amd_iommu_unity_map, list) {
if (!(devid >= e->devid_start && devid <= e->devid_end))
continue;
alloc_unity_mapping(dma_dom, e);
}
}
/*
@ -290,11 +293,23 @@ static bool check_device(struct device *dev)
static void init_iommu_group(struct device *dev)
{
struct dma_ops_domain *dma_domain;
struct iommu_domain *domain;
struct iommu_group *group;
group = iommu_group_get_for_dev(dev);
if (!IS_ERR(group))
iommu_group_put(group);
if (IS_ERR(group))
return;
domain = iommu_group_default_domain(group);
if (!domain)
goto out;
dma_domain = to_pdomain(domain)->priv;
init_unity_mappings_for_device(dev, dma_domain);
out:
iommu_group_put(group);
}
static int __last_alias(struct pci_dev *pdev, u16 alias, void *data)
@ -434,64 +449,15 @@ static void iommu_uninit_device(struct device *dev)
/* Unlink from alias, it may change if another device is re-plugged */
dev_data->alias_data = NULL;
/* Remove dma-ops */
dev->archdata.dma_ops = NULL;
/*
* We keep dev_data around for unplugged devices and reuse it when the
* device is re-plugged - not doing so would introduce a ton of races.
*/
}
void __init amd_iommu_uninit_devices(void)
{
struct iommu_dev_data *dev_data, *n;
struct pci_dev *pdev = NULL;
for_each_pci_dev(pdev) {
if (!check_device(&pdev->dev))
continue;
iommu_uninit_device(&pdev->dev);
}
/* Free all of our dev_data structures */
list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
free_dev_data(dev_data);
}
int __init amd_iommu_init_devices(void)
{
struct pci_dev *pdev = NULL;
int ret = 0;
for_each_pci_dev(pdev) {
if (!check_device(&pdev->dev))
continue;
ret = iommu_init_device(&pdev->dev);
if (ret == -ENOTSUPP)
iommu_ignore_device(&pdev->dev);
else if (ret)
goto out_free;
}
/*
* Initialize IOMMU groups only after iommu_init_device() has
* had a chance to populate any IVRS defined aliases.
*/
for_each_pci_dev(pdev) {
if (check_device(&pdev->dev))
init_iommu_group(&pdev->dev);
}
return 0;
out_free:
amd_iommu_uninit_devices();
return ret;
}
#ifdef CONFIG_AMD_IOMMU_STATS
/*
@ -1463,94 +1429,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,
return unmapped;
}
/*
* This function checks if a specific unity mapping entry is needed for
* this specific IOMMU.
*/
static int iommu_for_unity_map(struct amd_iommu *iommu,
struct unity_map_entry *entry)
{
u16 bdf, i;
for (i = entry->devid_start; i <= entry->devid_end; ++i) {
bdf = amd_iommu_alias_table[i];
if (amd_iommu_rlookup_table[bdf] == iommu)
return 1;
}
return 0;
}
/*
* This function actually applies the mapping to the page table of the
* dma_ops domain.
*/
static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
struct unity_map_entry *e)
{
u64 addr;
int ret;
for (addr = e->address_start; addr < e->address_end;
addr += PAGE_SIZE) {
ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
PAGE_SIZE);
if (ret)
return ret;
/*
* if unity mapping is in aperture range mark the page
* as allocated in the aperture
*/
if (addr < dma_dom->aperture_size)
__set_bit(addr >> PAGE_SHIFT,
dma_dom->aperture[0]->bitmap);
}
return 0;
}
/*
* Init the unity mappings for a specific IOMMU in the system
*
* Basically iterates over all unity mapping entries and applies them to
* the default domain DMA of that IOMMU if necessary.
*/
static int iommu_init_unity_mappings(struct amd_iommu *iommu)
{
struct unity_map_entry *entry;
int ret;
list_for_each_entry(entry, &amd_iommu_unity_map, list) {
if (!iommu_for_unity_map(iommu, entry))
continue;
ret = dma_ops_unity_map(iommu->default_dom, entry);
if (ret)
return ret;
}
return 0;
}
/*
* Inits the unity mappings required for a specific device
*/
static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
u16 devid)
{
struct unity_map_entry *e;
int ret;
list_for_each_entry(e, &amd_iommu_unity_map, list) {
if (!(devid >= e->devid_start && devid <= e->devid_end))
continue;
ret = dma_ops_unity_map(dma_dom, e);
if (ret)
return ret;
}
return 0;
}
/****************************************************************************
*
* The next functions belong to the address allocator for the dma_ops
@ -1704,14 +1582,16 @@ static unsigned long dma_ops_area_alloc(struct device *dev,
unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
int i = start >> APERTURE_RANGE_SHIFT;
unsigned long boundary_size;
unsigned long boundary_size, mask;
unsigned long address = -1;
unsigned long limit;
next_bit >>= PAGE_SHIFT;
boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
PAGE_SIZE) >> PAGE_SHIFT;
mask = dma_get_seg_boundary(dev);
boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
1UL << (BITS_PER_LONG - PAGE_SHIFT);
for (;i < max_index; ++i) {
unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
@ -1869,9 +1749,15 @@ static void free_pt_##LVL (unsigned long __pt) \
pt = (u64 *)__pt; \
\
for (i = 0; i < 512; ++i) { \
/* PTE present? */ \
if (!IOMMU_PTE_PRESENT(pt[i])) \
continue; \
\
/* Large PTE? */ \
if (PM_PTE_LEVEL(pt[i]) == 0 || \
PM_PTE_LEVEL(pt[i]) == 7) \
continue; \
\
p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \
FN(p); \
} \
@ -2008,7 +1894,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
goto free_dma_dom;
dma_dom->need_flush = false;
dma_dom->target_dev = 0xffff;
add_domain_to_list(&dma_dom->domain);
@ -2373,110 +2258,67 @@ static void detach_device(struct device *dev)
dev_data->ats.enabled = false;
}
/*
* Find out the protection domain structure for a given PCI device. This
* will give us the pointer to the page table root for example.
*/
static struct protection_domain *domain_for_device(struct device *dev)
static int amd_iommu_add_device(struct device *dev)
{
struct iommu_dev_data *dev_data;
struct protection_domain *dom = NULL;
unsigned long flags;
dev_data = get_dev_data(dev);
if (dev_data->domain)
return dev_data->domain;
if (dev_data->alias_data != NULL) {
struct iommu_dev_data *alias_data = dev_data->alias_data;
read_lock_irqsave(&amd_iommu_devtable_lock, flags);
if (alias_data->domain != NULL) {
__attach_device(dev_data, alias_data->domain);
dom = alias_data->domain;
}
read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}
return dom;
}
static int device_change_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct dma_ops_domain *dma_domain;
struct protection_domain *domain;
struct iommu_dev_data *dev_data;
struct device *dev = data;
struct iommu_domain *domain;
struct amd_iommu *iommu;
unsigned long flags;
u16 devid;
int ret;
if (!check_device(dev))
if (!check_device(dev) || get_dev_data(dev))
return 0;
devid = get_device_id(dev);
iommu = amd_iommu_rlookup_table[devid];
dev_data = get_dev_data(dev);
devid = get_device_id(dev);
iommu = amd_iommu_rlookup_table[devid];
switch (action) {
case BUS_NOTIFY_ADD_DEVICE:
ret = iommu_init_device(dev);
if (ret) {
if (ret != -ENOTSUPP)
pr_err("Failed to initialize device %s - trying to proceed anyway\n",
dev_name(dev));
iommu_init_device(dev);
init_iommu_group(dev);
/*
* dev_data is still NULL and
* got initialized in iommu_init_device
*/
dev_data = get_dev_data(dev);
if (iommu_pass_through || dev_data->iommu_v2) {
dev_data->passthrough = true;
attach_device(dev, pt_domain);
break;
}
domain = domain_for_device(dev);
/* allocate a protection domain if a device is added */
dma_domain = find_protection_domain(devid);
if (!dma_domain) {
dma_domain = dma_ops_domain_alloc();
if (!dma_domain)
goto out;
dma_domain->target_dev = devid;
spin_lock_irqsave(&iommu_pd_list_lock, flags);
list_add_tail(&dma_domain->list, &iommu_pd_list);
spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
}
dev->archdata.dma_ops = &amd_iommu_dma_ops;
break;
case BUS_NOTIFY_REMOVED_DEVICE:
iommu_uninit_device(dev);
default:
iommu_ignore_device(dev);
dev->archdata.dma_ops = &nommu_dma_ops;
goto out;
}
init_iommu_group(dev);
iommu_completion_wait(iommu);
dev_data = get_dev_data(dev);
BUG_ON(!dev_data);
if (dev_data->iommu_v2)
iommu_request_dm_for_dev(dev);
/* Domains are initialized for this device - have a look what we ended up with */
domain = iommu_get_domain_for_dev(dev);
if (domain->type == IOMMU_DOMAIN_IDENTITY) {
dev_data->passthrough = true;
dev->archdata.dma_ops = &nommu_dma_ops;
} else {
dev->archdata.dma_ops = &amd_iommu_dma_ops;
}
out:
iommu_completion_wait(iommu);
return 0;
}
static struct notifier_block device_nb = {
.notifier_call = device_change_notifier,
};
void amd_iommu_init_notifier(void)
static void amd_iommu_remove_device(struct device *dev)
{
bus_register_notifier(&pci_bus_type, &device_nb);
struct amd_iommu *iommu;
u16 devid;
if (!check_device(dev))
return;
devid = get_device_id(dev);
iommu = amd_iommu_rlookup_table[devid];
iommu_uninit_device(dev);
iommu_completion_wait(iommu);
}
/*****************************************************************************
@ -2495,28 +2337,20 @@ void amd_iommu_init_notifier(void)
static struct protection_domain *get_domain(struct device *dev)
{
struct protection_domain *domain;
struct dma_ops_domain *dma_dom;
u16 devid = get_device_id(dev);
struct iommu_domain *io_domain;
if (!check_device(dev))
return ERR_PTR(-EINVAL);
domain = domain_for_device(dev);
if (domain != NULL && !dma_ops_domain(domain))
io_domain = iommu_get_domain_for_dev(dev);
if (!io_domain)
return NULL;
domain = to_pdomain(io_domain);
if (!dma_ops_domain(domain))
return ERR_PTR(-EBUSY);
if (domain != NULL)
return domain;
/* Device not bound yet - bind it */
dma_dom = find_protection_domain(devid);
if (!dma_dom)
dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
attach_device(dev, &dma_dom->domain);
DUMP_printk("Using protection domain %d for device %s\n",
dma_dom->domain.id, dev_name(dev));
return &dma_dom->domain;
return domain;
}
static void update_device_table(struct protection_domain *domain)
@ -2930,6 +2764,7 @@ static void *alloc_coherent(struct device *dev, size_t size,
size = PAGE_ALIGN(size);
dma_mask = dev->coherent_dma_mask;
flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
flag |= __GFP_ZERO;
page = alloc_pages(flag | __GFP_NOWARN, get_order(size));
if (!page) {
@ -3011,54 +2846,6 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
return check_device(dev);
}
/*
* The function for pre-allocating protection domains.
*
* If the driver core informs the DMA layer if a driver grabs a device
* we don't need to preallocate the protection domains anymore.
* For now we have to.
*/
static void __init prealloc_protection_domains(void)
{
struct iommu_dev_data *dev_data;
struct dma_ops_domain *dma_dom;
struct pci_dev *dev = NULL;
u16 devid;
for_each_pci_dev(dev) {
/* Do we handle this device? */
if (!check_device(&dev->dev))
continue;
dev_data = get_dev_data(&dev->dev);
if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
/* Make sure passthrough domain is allocated */
alloc_passthrough_domain();
dev_data->passthrough = true;
attach_device(&dev->dev, pt_domain);
pr_info("AMD-Vi: Using passthrough domain for device %s\n",
dev_name(&dev->dev));
}
/* Is there already any domain for it? */
if (domain_for_device(&dev->dev))
continue;
devid = get_device_id(&dev->dev);
dma_dom = dma_ops_domain_alloc();
if (!dma_dom)
continue;
init_unity_mappings_for_device(dma_dom, devid);
dma_dom->target_dev = devid;
attach_device(&dev->dev, &dma_dom->domain);
list_add_tail(&dma_dom->list, &iommu_pd_list);
}
}
static struct dma_map_ops amd_iommu_dma_ops = {
.alloc = alloc_coherent,
.free = free_coherent,
@ -3069,76 +2856,16 @@ static struct dma_map_ops amd_iommu_dma_ops = {
.dma_supported = amd_iommu_dma_supported,
};
static unsigned device_dma_ops_init(void)
int __init amd_iommu_init_api(void)
{
struct iommu_dev_data *dev_data;
struct pci_dev *pdev = NULL;
unsigned unhandled = 0;
for_each_pci_dev(pdev) {
if (!check_device(&pdev->dev)) {
iommu_ignore_device(&pdev->dev);
unhandled += 1;
continue;
}
dev_data = get_dev_data(&pdev->dev);
if (!dev_data->passthrough)
pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
else
pdev->dev.archdata.dma_ops = &nommu_dma_ops;
}
return unhandled;
}
/*
* The function which clues the AMD IOMMU driver into dma_ops.
*/
void __init amd_iommu_init_api(void)
{
bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
return bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
}
int __init amd_iommu_init_dma_ops(void)
{
struct amd_iommu *iommu;
int ret, unhandled;
/*
* first allocate a default protection domain for every IOMMU we
* found in the system. Devices not assigned to any other
* protection domain will be assigned to the default one.
*/
for_each_iommu(iommu) {
iommu->default_dom = dma_ops_domain_alloc();
if (iommu->default_dom == NULL)
return -ENOMEM;
iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
ret = iommu_init_unity_mappings(iommu);
if (ret)
goto free_domains;
}
/*
* Pre-allocate the protection domains for each device.
*/
prealloc_protection_domains();
iommu_detected = 1;
swiotlb = 0;
/* Make the driver finally visible to the drivers */
unhandled = device_dma_ops_init();
if (unhandled && max_pfn > MAX_DMA32_PFN) {
/* There are unhandled devices - initialize swiotlb for them */
swiotlb = 1;
}
amd_iommu_stats_init();
if (amd_iommu_unmap_flush)
@ -3147,14 +2874,6 @@ int __init amd_iommu_init_dma_ops(void)
pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
return 0;
free_domains:
for_each_iommu(iommu) {
dma_ops_domain_free(iommu->default_dom);
}
return ret;
}
/*****************************************************************************
@ -3221,7 +2940,7 @@ out_err:
return NULL;
}
static int __init alloc_passthrough_domain(void)
static int alloc_passthrough_domain(void)
{
if (pt_domain != NULL)
return 0;
@ -3239,30 +2958,46 @@ static int __init alloc_passthrough_domain(void)
static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
{
struct protection_domain *pdomain;
struct dma_ops_domain *dma_domain;
/* We only support unmanaged domains for now */
if (type != IOMMU_DOMAIN_UNMANAGED)
switch (type) {
case IOMMU_DOMAIN_UNMANAGED:
pdomain = protection_domain_alloc();
if (!pdomain)
return NULL;
pdomain->mode = PAGE_MODE_3_LEVEL;
pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
if (!pdomain->pt_root) {
protection_domain_free(pdomain);
return NULL;
}
pdomain->domain.geometry.aperture_start = 0;
pdomain->domain.geometry.aperture_end = ~0ULL;
pdomain->domain.geometry.force_aperture = true;
break;
case IOMMU_DOMAIN_DMA:
dma_domain = dma_ops_domain_alloc();
if (!dma_domain) {
pr_err("AMD-Vi: Failed to allocate\n");
return NULL;
}
pdomain = &dma_domain->domain;
break;
case IOMMU_DOMAIN_IDENTITY:
pdomain = protection_domain_alloc();
if (!pdomain)
return NULL;
pdomain->mode = PAGE_MODE_NONE;
break;
default:
return NULL;
pdomain = protection_domain_alloc();
if (!pdomain)
goto out_free;
pdomain->mode = PAGE_MODE_3_LEVEL;
pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
if (!pdomain->pt_root)
goto out_free;
pdomain->domain.geometry.aperture_start = 0;
pdomain->domain.geometry.aperture_end = ~0ULL;
pdomain->domain.geometry.force_aperture = true;
}
return &pdomain->domain;
out_free:
protection_domain_free(pdomain);
return NULL;
}
static void amd_iommu_domain_free(struct iommu_domain *dom)
@ -3412,6 +3147,47 @@ static bool amd_iommu_capable(enum iommu_cap cap)
return false;
}
static void amd_iommu_get_dm_regions(struct device *dev,
struct list_head *head)
{
struct unity_map_entry *entry;
u16 devid;
devid = get_device_id(dev);
list_for_each_entry(entry, &amd_iommu_unity_map, list) {
struct iommu_dm_region *region;
if (devid < entry->devid_start || devid > entry->devid_end)
continue;
region = kzalloc(sizeof(*region), GFP_KERNEL);
if (!region) {
pr_err("Out of memory allocating dm-regions for %s\n",
dev_name(dev));
return;
}
region->start = entry->address_start;
region->length = entry->address_end - entry->address_start;
if (entry->prot & IOMMU_PROT_IR)
region->prot |= IOMMU_READ;
if (entry->prot & IOMMU_PROT_IW)
region->prot |= IOMMU_WRITE;
list_add_tail(&region->list, head);
}
}
static void amd_iommu_put_dm_regions(struct device *dev,
struct list_head *head)
{
struct iommu_dm_region *entry, *next;
list_for_each_entry_safe(entry, next, head, list)
kfree(entry);
}
static const struct iommu_ops amd_iommu_ops = {
.capable = amd_iommu_capable,
.domain_alloc = amd_iommu_domain_alloc,
@ -3422,6 +3198,10 @@ static const struct iommu_ops amd_iommu_ops = {
.unmap = amd_iommu_unmap,
.map_sg = default_iommu_map_sg,
.iova_to_phys = amd_iommu_iova_to_phys,
.add_device = amd_iommu_add_device,
.remove_device = amd_iommu_remove_device,
.get_dm_regions = amd_iommu_get_dm_regions,
.put_dm_regions = amd_iommu_put_dm_regions,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
};

34
drivers/iommu/amd_iommu_init.c
View file

@ -226,6 +226,7 @@ static enum iommu_init_state init_state = IOMMU_START_STATE;
static int amd_iommu_enable_interrupts(void);
static int __init iommu_go_to_state(enum iommu_init_state state);
static void init_device_table_dma(void);
static inline void update_last_devid(u16 devid)
{
@ -1385,9 +1386,15 @@ static int __init amd_iommu_init_pci(void)
break;
}
ret = amd_iommu_init_devices();
init_device_table_dma();
print_iommu_info();
for_each_iommu(iommu)
iommu_flush_all_caches(iommu);
ret = amd_iommu_init_api();
if (!ret)
print_iommu_info();
return ret;
}
@ -1825,8 +1832,6 @@ static bool __init check_ioapic_information(void)
static void __init free_dma_resources(void)
{
amd_iommu_uninit_devices();
free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
get_order(MAX_DOMAIN_ID/8));
@ -2019,27 +2024,10 @@ static bool detect_ivrs(void)
static int amd_iommu_init_dma(void)
{
struct amd_iommu *iommu;
int ret;
if (iommu_pass_through)
ret = amd_iommu_init_passthrough();
return amd_iommu_init_passthrough();
else
ret = amd_iommu_init_dma_ops();
if (ret)
return ret;
init_device_table_dma();
for_each_iommu(iommu)
iommu_flush_all_caches(iommu);
amd_iommu_init_api();
amd_iommu_init_notifier();
return 0;
return amd_iommu_init_dma_ops();
}
/****************************************************************************

2
drivers/iommu/amd_iommu_proto.h
View file

@ -30,7 +30,7 @@ extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu);
extern int amd_iommu_init_devices(void);
extern void amd_iommu_uninit_devices(void);
extern void amd_iommu_init_notifier(void);
extern void amd_iommu_init_api(void);
extern int amd_iommu_init_api(void);
/* Needed for interrupt remapping */
extern int amd_iommu_prepare(void);

11
drivers/iommu/amd_iommu_types.h
View file

@ -446,8 +446,6 @@ struct aperture_range {
* Data container for a dma_ops specific protection domain
*/
struct dma_ops_domain {
struct list_head list;
/* generic protection domain information */
struct protection_domain domain;
@ -462,12 +460,6 @@ struct dma_ops_domain {
/* This will be set to true when TLB needs to be flushed */
bool need_flush;
/*
* if this is a preallocated domain, keep the device for which it was
* preallocated in this variable
*/
u16 target_dev;
};
/*
@ -552,9 +544,6 @@ struct amd_iommu {
/* if one, we need to send a completion wait command */
bool need_sync;
/* default dma_ops domain for that IOMMU */
struct dma_ops_domain *default_dom;
/* IOMMU sysfs device */
struct device *iommu_dev;

1
drivers/iommu/amd_iommu_v2.c
View file

@ -266,6 +266,7 @@ static void put_pasid_state(struct pasid_state *pasid_state)
static void put_pasid_state_wait(struct pasid_state *pasid_state)
{
atomic_dec(&pasid_state->count);
wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));
free_pasid_state(pasid_state);
}

2670
drivers/iommu/arm-smmu-v3.c Normal file
View file

File diff suppressed because it is too large Load diff

53
drivers/iommu/arm-smmu.c
View file

@ -202,8 +202,7 @@
#define ARM_SMMU_CB_S1_TLBIVAL 0x620
#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
#define ARM_SMMU_CB_ATS1PR_LO 0x800
#define ARM_SMMU_CB_ATS1PR_HI 0x804
#define ARM_SMMU_CB_ATS1PR 0x800
#define ARM_SMMU_CB_ATSR 0x8f0
#define SCTLR_S1_ASIDPNE (1 << 12)
@ -224,14 +223,7 @@
#define RESUME_TERMINATE (1 << 0)
#define TTBCR2_SEP_SHIFT 15
#define TTBCR2_SEP_MASK 0x7
#define TTBCR2_ADDR_32 0
#define TTBCR2_ADDR_36 1
#define TTBCR2_ADDR_40 2
#define TTBCR2_ADDR_42 3
#define TTBCR2_ADDR_44 4
#define TTBCR2_ADDR_48 5
#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
#define TTBRn_HI_ASID_SHIFT 16
@ -254,7 +246,7 @@
#define FSYNR0_WNR (1 << 4)
static int force_stage;
module_param_named(force_stage, force_stage, int, S_IRUGO | S_IWUSR);
module_param_named(force_stage, force_stage, int, S_IRUGO);
MODULE_PARM_DESC(force_stage,
"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
@ -793,26 +785,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
if (smmu->version > ARM_SMMU_V1) {
reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
switch (smmu->va_size) {
case 32:
reg |= (TTBCR2_ADDR_32 << TTBCR2_SEP_SHIFT);
break;
case 36:
reg |= (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT);
break;
case 40:
reg |= (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT);
break;
case 42:
reg |= (TTBCR2_ADDR_42 << TTBCR2_SEP_SHIFT);
break;
case 44:
reg |= (TTBCR2_ADDR_44 << TTBCR2_SEP_SHIFT);
break;
case 48:
reg |= (TTBCR2_ADDR_48 << TTBCR2_SEP_SHIFT);
break;
}
reg |= TTBCR2_SEP_UPSTREAM;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
}
} else {
@ -1255,18 +1228,18 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
void __iomem *cb_base;
u32 tmp;
u64 phys;
unsigned long va;
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
if (smmu->version == 1) {
u32 reg = iova & ~0xfff;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO);
} else {
u32 reg = iova & ~0xfff;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO);
reg = ((u64)iova & ~0xfff) >> 32;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_HI);
}
/* ATS1 registers can only be written atomically */
va = iova & ~0xfffUL;
#ifdef CONFIG_64BIT
if (smmu->version == ARM_SMMU_V2)
writeq_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
else
#endif
writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
!(tmp & ATSR_ACTIVE), 5, 50)) {

28
drivers/iommu/dmar.c
View file

@ -26,7 +26,7 @@
* These routines are used by both DMA-remapping and Interrupt-remapping
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
#define pr_fmt(fmt) "DMAR: " fmt
#include <linux/pci.h>
#include <linux/dmar.h>
@ -555,7 +555,7 @@ static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
break;
} else if (next > end) {
/* Avoid passing table end */
pr_warn(FW_BUG "record passes table end\n");
pr_warn(FW_BUG "Record passes table end\n");
ret = -EINVAL;
break;
}
@ -802,7 +802,7 @@ int __init dmar_table_init(void)
ret = parse_dmar_table();
if (ret < 0) {
if (ret != -ENODEV)
pr_info("parse DMAR table failure.\n");
pr_info("Parse DMAR table failure.\n");
} else if (list_empty(&dmar_drhd_units)) {
pr_info("No DMAR devices found\n");
ret = -ENODEV;
@ -847,7 +847,7 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
else
addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
if (!addr) {
pr_warn("IOMMU: can't validate: %llx\n", drhd->address);
pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
return -EINVAL;
}
@ -921,14 +921,14 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
iommu->reg_size = VTD_PAGE_SIZE;
if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
pr_err("IOMMU: can't reserve memory\n");
pr_err("Can't reserve memory\n");
err = -EBUSY;
goto out;
}
iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
if (!iommu->reg) {
pr_err("IOMMU: can't map the region\n");
pr_err("Can't map the region\n");
err = -ENOMEM;
goto release;
}
@ -952,13 +952,13 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
iommu->reg_size = map_size;
if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
iommu->name)) {
pr_err("IOMMU: can't reserve memory\n");
pr_err("Can't reserve memory\n");
err = -EBUSY;
goto out;
}
iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
if (!iommu->reg) {
pr_err("IOMMU: can't map the region\n");
pr_err("Can't map the region\n");
err = -ENOMEM;
goto release;
}
@ -1014,14 +1014,14 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
return -ENOMEM;
if (dmar_alloc_seq_id(iommu) < 0) {
pr_err("IOMMU: failed to allocate seq_id\n");
pr_err("Failed to allocate seq_id\n");
err = -ENOSPC;
goto error;
}
err = map_iommu(iommu, drhd->reg_base_addr);
if (err) {
pr_err("IOMMU: failed to map %s\n", iommu->name);
pr_err("Failed to map %s\n", iommu->name);
goto error_free_seq_id;
}
@ -1045,8 +1045,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
iommu->node = -1;
ver = readl(iommu->reg + DMAR_VER_REG);
pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
iommu->seq_id,
pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
iommu->name,
(unsigned long long)drhd->reg_base_addr,
DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
(unsigned long long)iommu->cap,
@ -1644,7 +1644,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu)
irq = dmar_alloc_hwirq();
if (irq <= 0) {
pr_err("IOMMU: no free vectors\n");
pr_err("No free IRQ vectors\n");
return -EINVAL;
}
@ -1661,7 +1661,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu)
ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
if (ret)
pr_err("IOMMU: can't request irq\n");
pr_err("Can't request irq\n");
return ret;
}

527
drivers/iommu/exynos-iommu.c
View file

@ -12,21 +12,22 @@
#define DEBUG
#endif
#include <linux/io.h>
#include <linux/interrupt.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/pm_runtime.h>
#include <linux/clk.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/mm.h>
#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/memblock.h>
#include <linux/export.h>
#include <linux/of.h>
#include <linux/of_iommu.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/slab.h>
#include <asm/cacheflush.h>
#include <asm/dma-iommu.h>
#include <asm/pgtable.h>
typedef u32 sysmmu_iova_t;
@ -184,35 +185,50 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
"UNKNOWN FAULT"
};
/* attached to dev.archdata.iommu of the master device */
/*
* This structure is attached to dev.archdata.iommu of the master device
* on device add, contains a list of SYSMMU controllers defined by device tree,
* which are bound to given master device. It is usually referenced by 'owner'
* pointer.
*/
struct exynos_iommu_owner {
struct list_head client; /* entry of exynos_iommu_domain.clients */
struct device *dev;
struct device *sysmmu;
struct iommu_domain *domain;
void *vmm_data; /* IO virtual memory manager's data */
spinlock_t lock; /* Lock to preserve consistency of System MMU */
struct list_head controllers; /* list of sysmmu_drvdata.owner_node */
};
/*
* This structure exynos specific generalization of struct iommu_domain.
* It contains list of SYSMMU controllers from all master devices, which has
* been attached to this domain and page tables of IO address space defined by
* it. It is usually referenced by 'domain' pointer.
*/
struct exynos_iommu_domain {
struct list_head clients; /* list of sysmmu_drvdata.node */
sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */
short *lv2entcnt; /* free lv2 entry counter for each section */
spinlock_t lock; /* lock for this structure */
spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
struct list_head clients; /* list of sysmmu_drvdata.domain_node */
sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */
short *lv2entcnt; /* free lv2 entry counter for each section */
spinlock_t lock; /* lock for modyfying list of clients */
spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
struct iommu_domain domain; /* generic domain data structure */
};
/*
* This structure hold all data of a single SYSMMU controller, this includes
* hw resources like registers and clocks, pointers and list nodes to connect
* it to all other structures, internal state and parameters read from device
* tree. It is usually referenced by 'data' pointer.
*/
struct sysmmu_drvdata {
struct device *sysmmu; /* System MMU's device descriptor */
struct device *master; /* Owner of system MMU */
void __iomem *sfrbase;
struct clk *clk;
struct clk *clk_master;
int activations;
spinlock_t lock;
struct iommu_domain *domain;
phys_addr_t pgtable;
struct device *sysmmu; /* SYSMMU controller device */
struct device *master; /* master device (owner) */
void __iomem *sfrbase; /* our registers */
struct clk *clk; /* SYSMMU's clock */
struct clk *clk_master; /* master's device clock */
int activations; /* number of calls to sysmmu_enable */
spinlock_t lock; /* lock for modyfying state */
struct exynos_iommu_domain *domain; /* domain we belong to */
struct list_head domain_node; /* node for domain clients list */
struct list_head owner_node; /* node for owner controllers list */
phys_addr_t pgtable; /* assigned page table structure */
unsigned int version; /* our version */
};
static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom)
@ -244,11 +260,6 @@ static void sysmmu_unblock(void __iomem *sfrbase)
__raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);
}
static unsigned int __raw_sysmmu_version(struct sysmmu_drvdata *data)
{
return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));
}
static bool sysmmu_block(void __iomem *sfrbase)
{
int i = 120;
@ -345,7 +356,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
show_fault_information(dev_name(data->sysmmu),
itype, base, addr);
if (data->domain)
ret = report_iommu_fault(data->domain,
ret = report_iommu_fault(&data->domain->domain,
data->master, addr, itype);
}
@ -408,7 +419,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data)
unsigned int cfg = CFG_LRU | CFG_QOS(15);
unsigned int ver;
ver = __raw_sysmmu_version(data);
ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));
if (MMU_MAJ_VER(ver) == 3) {
if (MMU_MIN_VER(ver) >= 2) {
cfg |= CFG_FLPDCACHE;
@ -422,6 +433,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data)
}
__raw_writel(cfg, data->sfrbase + REG_MMU_CFG);
data->version = ver;
}
static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
@ -442,8 +454,8 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
clk_disable(data->clk_master);
}
static int __sysmmu_enable(struct sysmmu_drvdata *data,
phys_addr_t pgtable, struct iommu_domain *domain)
static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable,
struct exynos_iommu_domain *domain)
{
int ret = 0;
unsigned long flags;
@ -470,77 +482,17 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data,
return ret;
}
/* __exynos_sysmmu_enable: Enables System MMU
*
* returns -error if an error occurred and System MMU is not enabled,
* 0 if the System MMU has been just enabled and 1 if System MMU was already
* enabled before.
*/
static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable,
struct iommu_domain *domain)
{
int ret = 0;
unsigned long flags;
struct exynos_iommu_owner *owner = dev->archdata.iommu;
struct sysmmu_drvdata *data;
BUG_ON(!has_sysmmu(dev));
spin_lock_irqsave(&owner->lock, flags);
data = dev_get_drvdata(owner->sysmmu);
ret = __sysmmu_enable(data, pgtable, domain);
if (ret >= 0)
data->master = dev;
spin_unlock_irqrestore(&owner->lock, flags);
return ret;
}
int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable)
{
BUG_ON(!memblock_is_memory(pgtable));
return __exynos_sysmmu_enable(dev, pgtable, NULL);
}
static bool exynos_sysmmu_disable(struct device *dev)
{
unsigned long flags;
bool disabled = true;
struct exynos_iommu_owner *owner = dev->archdata.iommu;
struct sysmmu_drvdata *data;
BUG_ON(!has_sysmmu(dev));
spin_lock_irqsave(&owner->lock, flags);
data = dev_get_drvdata(owner->sysmmu);
disabled = __sysmmu_disable(data);
if (disabled)
data->master = NULL;
spin_unlock_irqrestore(&owner->lock, flags);
return disabled;
}
static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
sysmmu_iova_t iova)
{
if (__raw_sysmmu_version(data) == MAKE_MMU_VER(3, 3))
if (data->version == MAKE_MMU_VER(3, 3))
__raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY);
}
static void sysmmu_tlb_invalidate_flpdcache(struct device *dev,
static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
sysmmu_iova_t iova)
{
unsigned long flags;
struct exynos_iommu_owner *owner = dev->archdata.iommu;
struct sysmmu_drvdata *data = dev_get_drvdata(owner->sysmmu);
if (!IS_ERR(data->clk_master))
clk_enable(data->clk_master);
@ -554,14 +506,10 @@ static void sysmmu_tlb_invalidate_flpdcache(struct device *dev,
clk_disable(data->clk_master);
}
static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
size_t size)
static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
sysmmu_iova_t iova, size_t size)
{
struct exynos_iommu_owner *owner = dev->archdata.iommu;
unsigned long flags;
struct sysmmu_drvdata *data;
data = dev_get_drvdata(owner->sysmmu);
spin_lock_irqsave(&data->lock, flags);
if (is_sysmmu_active(data)) {
@ -580,7 +528,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
* 1MB page can be cached in one of all sets.
* 64KB page can be one of 16 consecutive sets.
*/
if (MMU_MAJ_VER(__raw_sysmmu_version(data)) == 2)
if (MMU_MAJ_VER(data->version) == 2)
num_inv = min_t(unsigned int, size / PAGE_SIZE, 64);
if (sysmmu_block(data->sfrbase)) {
@ -591,32 +539,8 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
if (!IS_ERR(data->clk_master))
clk_disable(data->clk_master);
} else {
dev_dbg(dev, "disabled. Skipping TLB invalidation @ %#x\n",
iova);
}
spin_unlock_irqrestore(&data->lock, flags);
}
void exynos_sysmmu_tlb_invalidate(struct device *dev)
{
struct exynos_iommu_owner *owner = dev->archdata.iommu;
unsigned long flags;
struct sysmmu_drvdata *data;
data = dev_get_drvdata(owner->sysmmu);
spin_lock_irqsave(&data->lock, flags);
if (is_sysmmu_active(data)) {
if (!IS_ERR(data->clk_master))
clk_enable(data->clk_master);
if (sysmmu_block(data->sfrbase)) {
__sysmmu_tlb_invalidate(data->sfrbase);
sysmmu_unblock(data->sfrbase);
}
if (!IS_ERR(data->clk_master))
clk_disable(data->clk_master);
} else {
dev_dbg(dev, "disabled. Skipping TLB invalidation\n");
dev_dbg(data->master,
"disabled. Skipping TLB invalidation @ %#x\n", iova);
}
spin_unlock_irqrestore(&data->lock, flags);
}
@ -682,6 +606,36 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
return 0;
}
#ifdef CONFIG_PM_SLEEP
static int exynos_sysmmu_suspend(struct device *dev)
{
struct sysmmu_drvdata *data = dev_get_drvdata(dev);
dev_dbg(dev, "suspend\n");
if (is_sysmmu_active(data)) {
__sysmmu_disable_nocount(data);
pm_runtime_put(dev);
}
return 0;
}
static int exynos_sysmmu_resume(struct device *dev)
{
struct sysmmu_drvdata *data = dev_get_drvdata(dev);
dev_dbg(dev, "resume\n");
if (is_sysmmu_active(data)) {
pm_runtime_get_sync(dev);
__sysmmu_enable_nocount(data);
}
return 0;
}
#endif
static const struct dev_pm_ops sysmmu_pm_ops = {
SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume)
};
static const struct of_device_id sysmmu_of_match[] __initconst = {
{ .compatible = "samsung,exynos-sysmmu", },
{ },
@ -692,6 +646,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = {
.driver = {
.name = "exynos-sysmmu",
.of_match_table = sysmmu_of_match,
.pm = &sysmmu_pm_ops,
}
};
@ -704,105 +659,109 @@ static inline void pgtable_flush(void *vastart, void *vaend)
static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
{
struct exynos_iommu_domain *exynos_domain;
struct exynos_iommu_domain *domain;
int i;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
exynos_domain = kzalloc(sizeof(*exynos_domain), GFP_KERNEL);
if (!exynos_domain)
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
exynos_domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
if (!exynos_domain->pgtable)
domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
if (!domain->pgtable)
goto err_pgtable;
exynos_domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
if (!exynos_domain->lv2entcnt)
domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
if (!domain->lv2entcnt)
goto err_counter;
/* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
exynos_domain->pgtable[i + 0] = ZERO_LV2LINK;
exynos_domain->pgtable[i + 1] = ZERO_LV2LINK;
exynos_domain->pgtable[i + 2] = ZERO_LV2LINK;
exynos_domain->pgtable[i + 3] = ZERO_LV2LINK;
exynos_domain->pgtable[i + 4] = ZERO_LV2LINK;
exynos_domain->pgtable[i + 5] = ZERO_LV2LINK;
exynos_domain->pgtable[i + 6] = ZERO_LV2LINK;
exynos_domain->pgtable[i + 7] = ZERO_LV2LINK;
domain->pgtable[i + 0] = ZERO_LV2LINK;
domain->pgtable[i + 1] = ZERO_LV2LINK;
domain->pgtable[i + 2] = ZERO_LV2LINK;
domain->pgtable[i + 3] = ZERO_LV2LINK;
domain->pgtable[i + 4] = ZERO_LV2LINK;
domain->pgtable[i + 5] = ZERO_LV2LINK;
domain->pgtable[i + 6] = ZERO_LV2LINK;
domain->pgtable[i + 7] = ZERO_LV2LINK;
}
pgtable_flush(exynos_domain->pgtable, exynos_domain->pgtable + NUM_LV1ENTRIES);
pgtable_flush(domain->pgtable, domain->pgtable + NUM_LV1ENTRIES);
spin_lock_init(&exynos_domain->lock);
spin_lock_init(&exynos_domain->pgtablelock);
INIT_LIST_HEAD(&exynos_domain->clients);
spin_lock_init(&domain->lock);
spin_lock_init(&domain->pgtablelock);
INIT_LIST_HEAD(&domain->clients);
exynos_domain->domain.geometry.aperture_start = 0;
exynos_domain->domain.geometry.aperture_end = ~0UL;
exynos_domain->domain.geometry.force_aperture = true;
domain->domain.geometry.aperture_start = 0;
domain->domain.geometry.aperture_end = ~0UL;
domain->domain.geometry.force_aperture = true;
return &exynos_domain->domain;
return &domain->domain;
err_counter:
free_pages((unsigned long)exynos_domain->pgtable, 2);
free_pages((unsigned long)domain->pgtable, 2);
err_pgtable:
kfree(exynos_domain);
kfree(domain);
return NULL;
}
static void exynos_iommu_domain_free(struct iommu_domain *domain)
static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain)
{
struct exynos_iommu_domain *priv = to_exynos_domain(domain);
struct exynos_iommu_owner *owner;
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
struct sysmmu_drvdata *data, *next;
unsigned long flags;
int i;
WARN_ON(!list_empty(&priv->clients));
WARN_ON(!list_empty(&domain->clients));
spin_lock_irqsave(&priv->lock, flags);
spin_lock_irqsave(&domain->lock, flags);
list_for_each_entry(owner, &priv->clients, client) {
while (!exynos_sysmmu_disable(owner->dev))
; /* until System MMU is actually disabled */
list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
if (__sysmmu_disable(data))
data->master = NULL;
list_del_init(&data->domain_node);
}
while (!list_empty(&priv->clients))
list_del_init(priv->clients.next);
spin_unlock_irqrestore(&priv->lock, flags);
spin_unlock_irqrestore(&domain->lock, flags);
for (i = 0; i < NUM_LV1ENTRIES; i++)
if (lv1ent_page(priv->pgtable + i))
if (lv1ent_page(domain->pgtable + i))
kmem_cache_free(lv2table_kmem_cache,
phys_to_virt(lv2table_base(priv->pgtable + i)));
phys_to_virt(lv2table_base(domain->pgtable + i)));
free_pages((unsigned long)priv->pgtable, 2);
free_pages((unsigned long)priv->lv2entcnt, 1);
kfree(priv);
free_pages((unsigned long)domain->pgtable, 2);
free_pages((unsigned long)domain->lv2entcnt, 1);
kfree(domain);
}
static int exynos_iommu_attach_device(struct iommu_domain *domain,
static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
struct device *dev)
{
struct exynos_iommu_owner *owner = dev->archdata.iommu;
struct exynos_iommu_domain *priv = to_exynos_domain(domain);
phys_addr_t pagetable = virt_to_phys(priv->pgtable);
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
struct sysmmu_drvdata *data;
phys_addr_t pagetable = virt_to_phys(domain->pgtable);
unsigned long flags;
int ret;
int ret = -ENODEV;
spin_lock_irqsave(&priv->lock, flags);
if (!has_sysmmu(dev))
return -ENODEV;
ret = __exynos_sysmmu_enable(dev, pagetable, domain);
if (ret == 0) {
list_add_tail(&owner->client, &priv->clients);
owner->domain = domain;
list_for_each_entry(data, &owner->controllers, owner_node) {
pm_runtime_get_sync(data->sysmmu);
ret = __sysmmu_enable(data, pagetable, domain);
if (ret >= 0) {
data->master = dev;
spin_lock_irqsave(&domain->lock, flags);
list_add_tail(&data->domain_node, &domain->clients);
spin_unlock_irqrestore(&domain->lock, flags);
}
}
spin_unlock_irqrestore(&priv->lock, flags);
if (ret < 0) {
dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n",
__func__, &pagetable);
@ -815,36 +774,39 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain,
return ret;
}
static void exynos_iommu_detach_device(struct iommu_domain *domain,
static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain,
struct device *dev)
{
struct exynos_iommu_owner *owner;
struct exynos_iommu_domain *priv = to_exynos_domain(domain);
phys_addr_t pagetable = virt_to_phys(priv->pgtable);
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
phys_addr_t pagetable = virt_to_phys(domain->pgtable);
struct sysmmu_drvdata *data, *next;
unsigned long flags;
bool found = false;
spin_lock_irqsave(&priv->lock, flags);
if (!has_sysmmu(dev))
return;
list_for_each_entry(owner, &priv->clients, client) {
if (owner == dev->archdata.iommu) {
if (exynos_sysmmu_disable(dev)) {
list_del_init(&owner->client);
owner->domain = NULL;
spin_lock_irqsave(&domain->lock, flags);
list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
if (data->master == dev) {
if (__sysmmu_disable(data)) {
data->master = NULL;
list_del_init(&data->domain_node);
}
break;
pm_runtime_put(data->sysmmu);
found = true;
}
}
spin_unlock_irqrestore(&domain->lock, flags);
spin_unlock_irqrestore(&priv->lock, flags);
if (owner == dev->archdata.iommu)
if (found)
dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n",
__func__, &pagetable);
else
dev_err(dev, "%s: No IOMMU is attached\n", __func__);
}
static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain,
sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter)
{
if (lv1ent_section(sent)) {
@ -862,6 +824,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
return ERR_PTR(-ENOMEM);
*sent = mk_lv1ent_page(virt_to_phys(pent));
kmemleak_ignore(pent);
*pgcounter = NUM_LV2ENTRIES;
pgtable_flush(pent, pent + NUM_LV2ENTRIES);
pgtable_flush(sent, sent + 1);
@ -884,20 +847,19 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
* not currently mapped.
*/
if (need_flush_flpd_cache) {
struct exynos_iommu_owner *owner;
struct sysmmu_drvdata *data;
spin_lock(&priv->lock);
list_for_each_entry(owner, &priv->clients, client)
sysmmu_tlb_invalidate_flpdcache(
owner->dev, iova);
spin_unlock(&priv->lock);
spin_lock(&domain->lock);
list_for_each_entry(data, &domain->clients, domain_node)
sysmmu_tlb_invalidate_flpdcache(data, iova);
spin_unlock(&domain->lock);
}
}
return page_entry(sent, iova);
}
static int lv1set_section(struct exynos_iommu_domain *priv,
static int lv1set_section(struct exynos_iommu_domain *domain,
sysmmu_pte_t *sent, sysmmu_iova_t iova,
phys_addr_t paddr, short *pgcnt)
{
@ -922,17 +884,17 @@ static int lv1set_section(struct exynos_iommu_domain *priv,
pgtable_flush(sent, sent + 1);
spin_lock(&priv->lock);
spin_lock(&domain->lock);
if (lv1ent_page_zero(sent)) {
struct exynos_iommu_owner *owner;
struct sysmmu_drvdata *data;
/*
* Flushing FLPD cache in System MMU v3.3 that may cache a FLPD
* entry by speculative prefetch of SLPD which has no mapping.
*/
list_for_each_entry(owner, &priv->clients, client)
sysmmu_tlb_invalidate_flpdcache(owner->dev, iova);
list_for_each_entry(data, &domain->clients, domain_node)
sysmmu_tlb_invalidate_flpdcache(data, iova);
}
spin_unlock(&priv->lock);
spin_unlock(&domain->lock);
return 0;
}
@ -992,74 +954,75 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
* than or equal to 128KiB.
* - Start address of an I/O virtual region must be aligned by 128KiB.
*/
static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova,
phys_addr_t paddr, size_t size, int prot)
static int exynos_iommu_map(struct iommu_domain *iommu_domain,
unsigned long l_iova, phys_addr_t paddr, size_t size,
int prot)
{
struct exynos_iommu_domain *priv = to_exynos_domain(domain);
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
sysmmu_pte_t *entry;
sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
unsigned long flags;
int ret = -ENOMEM;
BUG_ON(priv->pgtable == NULL);
BUG_ON(domain->pgtable == NULL);
spin_lock_irqsave(&priv->pgtablelock, flags);
spin_lock_irqsave(&domain->pgtablelock, flags);
entry = section_entry(priv->pgtable, iova);
entry = section_entry(domain->pgtable, iova);
if (size == SECT_SIZE) {
ret = lv1set_section(priv, entry, iova, paddr,
&priv->lv2entcnt[lv1ent_offset(iova)]);
ret = lv1set_section(domain, entry, iova, paddr,
&domain->lv2entcnt[lv1ent_offset(iova)]);
} else {
sysmmu_pte_t *pent;
pent = alloc_lv2entry(priv, entry, iova,
&priv->lv2entcnt[lv1ent_offset(iova)]);
pent = alloc_lv2entry(domain, entry, iova,
&domain->lv2entcnt[lv1ent_offset(iova)]);
if (IS_ERR(pent))
ret = PTR_ERR(pent);
else
ret = lv2set_page(pent, paddr, size,
&priv->lv2entcnt[lv1ent_offset(iova)]);
&domain->lv2entcnt[lv1ent_offset(iova)]);
}
if (ret)
pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
__func__, ret, size, iova);
spin_unlock_irqrestore(&priv->pgtablelock, flags);
spin_unlock_irqrestore(&domain->pgtablelock, flags);
return ret;
}
static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv,
sysmmu_iova_t iova, size_t size)
static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain,
sysmmu_iova_t iova, size_t size)
{
struct exynos_iommu_owner *owner;
struct sysmmu_drvdata *data;
unsigned long flags;
spin_lock_irqsave(&priv->lock, flags);
spin_lock_irqsave(&domain->lock, flags);
list_for_each_entry(owner, &priv->clients, client)
sysmmu_tlb_invalidate_entry(owner->dev, iova, size);
list_for_each_entry(data, &domain->clients, domain_node)
sysmmu_tlb_invalidate_entry(data, iova, size);
spin_unlock_irqrestore(&priv->lock, flags);
spin_unlock_irqrestore(&domain->lock, flags);
}
static size_t exynos_iommu_unmap(struct iommu_domain *domain,
unsigned long l_iova, size_t size)
static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
unsigned long l_iova, size_t size)
{
struct exynos_iommu_domain *priv = to_exynos_domain(domain);
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
sysmmu_pte_t *ent;
size_t err_pgsize;
unsigned long flags;
BUG_ON(priv->pgtable == NULL);
BUG_ON(domain->pgtable == NULL);
spin_lock_irqsave(&priv->pgtablelock, flags);
spin_lock_irqsave(&domain->pgtablelock, flags);
ent = section_entry(priv->pgtable, iova);
ent = section_entry(domain->pgtable, iova);
if (lv1ent_section(ent)) {
if (WARN_ON(size < SECT_SIZE)) {
@ -1093,7 +1056,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
*ent = 0;
size = SPAGE_SIZE;
pgtable_flush(ent, ent + 1);
priv->lv2entcnt[lv1ent_offset(iova)] += 1;
domain->lv2entcnt[lv1ent_offset(iova)] += 1;
goto done;
}
@ -1107,15 +1070,15 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
pgtable_flush(ent, ent + SPAGES_PER_LPAGE);
size = LPAGE_SIZE;
priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
done:
spin_unlock_irqrestore(&priv->pgtablelock, flags);
spin_unlock_irqrestore(&domain->pgtablelock, flags);
exynos_iommu_tlb_invalidate_entry(priv, iova, size);
exynos_iommu_tlb_invalidate_entry(domain, iova, size);
return size;
err:
spin_unlock_irqrestore(&priv->pgtablelock, flags);
spin_unlock_irqrestore(&domain->pgtablelock, flags);
pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n",
__func__, size, iova, err_pgsize);
@ -1123,17 +1086,17 @@ err:
return 0;
}
static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain,
dma_addr_t iova)
{
struct exynos_iommu_domain *priv = to_exynos_domain(domain);
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
sysmmu_pte_t *entry;
unsigned long flags;
phys_addr_t phys = 0;
spin_lock_irqsave(&priv->pgtablelock, flags);
spin_lock_irqsave(&domain->pgtablelock, flags);
entry = section_entry(priv->pgtable, iova);
entry = section_entry(domain->pgtable, iova);
if (lv1ent_section(entry)) {
phys = section_phys(entry) + section_offs(iova);
@ -1146,7 +1109,7 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
phys = spage_phys(entry) + spage_offs(iova);
}
spin_unlock_irqrestore(&priv->pgtablelock, flags);
spin_unlock_irqrestore(&domain->pgtablelock, flags);
return phys;
}
@ -1156,6 +1119,9 @@ static int exynos_iommu_add_device(struct device *dev)
struct iommu_group *group;
int ret;
if (!has_sysmmu(dev))
return -ENODEV;
group = iommu_group_get(dev);
if (!group) {
@ -1174,10 +1140,40 @@ static int exynos_iommu_add_device(struct device *dev)
static void exynos_iommu_remove_device(struct device *dev)
{
if (!has_sysmmu(dev))
return;
iommu_group_remove_device(dev);
}
static const struct iommu_ops exynos_iommu_ops = {
static int exynos_iommu_of_xlate(struct device *dev,
struct of_phandle_args *spec)
{
struct exynos_iommu_owner *owner = dev->archdata.iommu;
struct platform_device *sysmmu = of_find_device_by_node(spec->np);
struct sysmmu_drvdata *data;
if (!sysmmu)
return -ENODEV;
data = platform_get_drvdata(sysmmu);
if (!data)
return -ENODEV;
if (!owner) {
owner = kzalloc(sizeof(*owner), GFP_KERNEL);
if (!owner)
return -ENOMEM;
INIT_LIST_HEAD(&owner->controllers);
dev->archdata.iommu = owner;
}
list_add_tail(&data->owner_node, &owner->controllers);
return 0;
}
static struct iommu_ops exynos_iommu_ops = {
.domain_alloc = exynos_iommu_domain_alloc,
.domain_free = exynos_iommu_domain_free,
.attach_dev = exynos_iommu_attach_device,
@ -1189,19 +1185,15 @@ static const struct iommu_ops exynos_iommu_ops = {
.add_device = exynos_iommu_add_device,
.remove_device = exynos_iommu_remove_device,
.pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
.of_xlate = exynos_iommu_of_xlate,
};
static bool init_done;
static int __init exynos_iommu_init(void)
{
struct device_node *np;
int ret;
np = of_find_matching_node(NULL, sysmmu_of_match);
if (!np)
return 0;
of_node_put(np);
lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table",
LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL);
if (!lv2table_kmem_cache) {
@ -1230,6 +1222,8 @@ static int __init exynos_iommu_init(void)
goto err_set_iommu;
}
init_done = true;
return 0;
err_set_iommu:
kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
@ -1239,4 +1233,21 @@ err_reg_driver:
kmem_cache_destroy(lv2table_kmem_cache);
return ret;
}
subsys_initcall(exynos_iommu_init);
static int __init exynos_iommu_of_setup(struct device_node *np)
{
struct platform_device *pdev;
if (!init_done)
exynos_iommu_init();
pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
if (IS_ERR(pdev))
return PTR_ERR(pdev);
of_iommu_set_ops(np, &exynos_iommu_ops);
return 0;
}
IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu",
exynos_iommu_of_setup);

526
drivers/iommu/intel-iommu.c
View file

@ -15,8 +15,11 @@
* Shaohua Li <shaohua.li@intel.com>,
* Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
* Fenghua Yu <fenghua.yu@intel.com>
* Joerg Roedel <jroedel@suse.de>
*/
#define pr_fmt(fmt) "DMAR: " fmt
#include <linux/init.h>
#include <linux/bitmap.h>
#include <linux/debugfs.h>
@ -40,6 +43,7 @@
#include <linux/pci-ats.h>
#include <linux/memblock.h>
#include <linux/dma-contiguous.h>
#include <linux/crash_dump.h>
#include <asm/irq_remapping.h>
#include <asm/cacheflush.h>
#include <asm/iommu.h>
@ -190,7 +194,29 @@ struct root_entry {
};
#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
/*
* Take a root_entry and return the Lower Context Table Pointer (LCTP)
* if marked present.
*/
static phys_addr_t root_entry_lctp(struct root_entry *re)
{
if (!(re->lo & 1))
return 0;
return re->lo & VTD_PAGE_MASK;
}
/*
* Take a root_entry and return the Upper Context Table Pointer (UCTP)
* if marked present.
*/
static phys_addr_t root_entry_uctp(struct root_entry *re)
{
if (!(re->hi & 1))
return 0;
return re->hi & VTD_PAGE_MASK;
}
/*
* low 64 bits:
* 0: present
@ -207,10 +233,38 @@ struct context_entry {
u64 hi;
};
static inline bool context_present(struct context_entry *context)
static inline void context_clear_pasid_enable(struct context_entry *context)
{
context->lo &= ~(1ULL << 11);
}
static inline bool context_pasid_enabled(struct context_entry *context)
{
return !!(context->lo & (1ULL << 11));
}
static inline void context_set_copied(struct context_entry *context)
{
context->hi |= (1ull << 3);
}
static inline bool context_copied(struct context_entry *context)
{
return !!(context->hi & (1ULL << 3));
}
static inline bool __context_present(struct context_entry *context)
{
return (context->lo & 1);
}
static inline bool context_present(struct context_entry *context)
{
return context_pasid_enabled(context) ?
__context_present(context) :
__context_present(context) && !context_copied(context);
}
static inline void context_set_present(struct context_entry *context)
{
context->lo |= 1;
@ -247,6 +301,11 @@ static inline void context_set_domain_id(struct context_entry *context,
context->hi |= (value & ((1 << 16) - 1)) << 8;
}
static inline int context_domain_id(struct context_entry *c)
{
return((c->hi >> 8) & 0xffff);
}
static inline void context_clear_entry(struct context_entry *context)
{
context->lo = 0;
@ -422,6 +481,14 @@ static int dmar_map_gfx = 1;
static int dmar_forcedac;
static int intel_iommu_strict;
static int intel_iommu_superpage = 1;
static int intel_iommu_ecs = 1;
/* We only actually use ECS when PASID support (on the new bit 40)
* is also advertised. Some early implementations — the ones with
* PASID support on bit 28 — have issues even when we *only* use
* extended root/context tables. */
#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
ecap_pasid(iommu->ecap))
int intel_iommu_gfx_mapped;
EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
@ -432,6 +499,25 @@ static LIST_HEAD(device_domain_list);
static const struct iommu_ops intel_iommu_ops;
static bool translation_pre_enabled(struct intel_iommu *iommu)
{
return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
}
static void clear_translation_pre_enabled(struct intel_iommu *iommu)
{
iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
}
static void init_translation_status(struct intel_iommu *iommu)
{
u32 gsts;
gsts = readl(iommu->reg + DMAR_GSTS_REG);
if (gsts & DMA_GSTS_TES)
iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
}
/* Convert generic 'struct iommu_domain to private struct dmar_domain */
static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
{
@ -445,26 +531,26 @@ static int __init intel_iommu_setup(char *str)
while (*str) {
if (!strncmp(str, "on", 2)) {
dmar_disabled = 0;
printk(KERN_INFO "Intel-IOMMU: enabled\n");
pr_info("IOMMU enabled\n");
} else if (!strncmp(str, "off", 3)) {
dmar_disabled = 1;
printk(KERN_INFO "Intel-IOMMU: disabled\n");
pr_info("IOMMU disabled\n");
} else if (!strncmp(str, "igfx_off", 8)) {
dmar_map_gfx = 0;
printk(KERN_INFO
"Intel-IOMMU: disable GFX device mapping\n");
pr_info("Disable GFX device mapping\n");
} else if (!strncmp(str, "forcedac", 8)) {
printk(KERN_INFO
"Intel-IOMMU: Forcing DAC for PCI devices\n");
pr_info("Forcing DAC for PCI devices\n");
dmar_forcedac = 1;
} else if (!strncmp(str, "strict", 6)) {
printk(KERN_INFO
"Intel-IOMMU: disable batched IOTLB flush\n");
pr_info("Disable batched IOTLB flush\n");
intel_iommu_strict = 1;
} else if (!strncmp(str, "sp_off", 6)) {
printk(KERN_INFO
"Intel-IOMMU: disable supported super page\n");
pr_info("Disable supported super page\n");
intel_iommu_superpage = 0;
} else if (!strncmp(str, "ecs_off", 7)) {
printk(KERN_INFO
"Intel-IOMMU: disable extended context table support\n");
intel_iommu_ecs = 0;
}
str += strcspn(str, ",");
@ -669,7 +755,7 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu
struct context_entry *context;
u64 *entry;
if (ecap_ecs(iommu->ecap)) {
if (ecs_enabled(iommu)) {
if (devfn >= 0x80) {
devfn -= 0x80;
entry = &root->hi;
@ -696,6 +782,11 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu
return &context[devfn];
}
static int iommu_dummy(struct device *dev)
{
return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
}
static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
{
struct dmar_drhd_unit *drhd = NULL;
@ -705,6 +796,9 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf
u16 segment = 0;
int i;
if (iommu_dummy(dev))
return NULL;
if (dev_is_pci(dev)) {
pdev = to_pci_dev(dev);
segment = pci_domain_nr(pdev->bus);
@ -798,7 +892,7 @@ static void free_context_table(struct intel_iommu *iommu)
if (context)
free_pgtable_page(context);
if (!ecap_ecs(iommu->ecap))
if (!ecs_enabled(iommu))
continue;
context = iommu_context_addr(iommu, i, 0x80, 0);
@ -1112,7 +1206,7 @@ static int iommu_alloc_root_entry(struct intel_iommu *iommu)
root = (struct root_entry *)alloc_pgtable_page(iommu->node);
if (!root) {
pr_err("IOMMU: allocating root entry for %s failed\n",
pr_err("Allocating root entry for %s failed\n",
iommu->name);
return -ENOMEM;
}
@ -1133,7 +1227,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu)
unsigned long flag;
addr = virt_to_phys(iommu->root_entry);
if (ecap_ecs(iommu->ecap))
if (ecs_enabled(iommu))
addr |= DMA_RTADDR_RTT;
raw_spin_lock_irqsave(&iommu->register_lock, flag);
@ -1250,9 +1344,9 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
/* check IOTLB invalidation granularity */
if (DMA_TLB_IAIG(val) == 0)
printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
pr_err("Flush IOTLB failed\n");
if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
pr_debug("TLB flush request %Lx, actual %Lx\n",
(unsigned long long)DMA_TLB_IIRG(type),
(unsigned long long)DMA_TLB_IAIG(val));
}
@ -1423,8 +1517,8 @@ static int iommu_init_domains(struct intel_iommu *iommu)
unsigned long nlongs;
ndomains = cap_ndoms(iommu->cap);
pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
iommu->seq_id, ndomains);
pr_debug("%s: Number of Domains supported <%ld>\n",
iommu->name, ndomains);
nlongs = BITS_TO_LONGS(ndomains);
spin_lock_init(&iommu->lock);
@ -1434,15 +1528,15 @@ static int iommu_init_domains(struct intel_iommu *iommu)
*/
iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
if (!iommu->domain_ids) {
pr_err("IOMMU%d: allocating domain id array failed\n",
iommu->seq_id);
pr_err("%s: Allocating domain id array failed\n",
iommu->name);
return -ENOMEM;
}
iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
GFP_KERNEL);
if (!iommu->domains) {
pr_err("IOMMU%d: allocating domain array failed\n",
iommu->seq_id);
pr_err("%s: Allocating domain array failed\n",
iommu->name);
kfree(iommu->domain_ids);
iommu->domain_ids = NULL;
return -ENOMEM;
@ -1547,7 +1641,7 @@ static int iommu_attach_domain(struct dmar_domain *domain,
num = __iommu_attach_domain(domain, iommu);
spin_unlock_irqrestore(&iommu->lock, flags);
if (num < 0)
pr_err("IOMMU: no free domain ids\n");
pr_err("%s: No free domain ids\n", iommu->name);
return num;
}
@ -1639,7 +1733,7 @@ static int dmar_init_reserved_ranges(void)
iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
IOVA_PFN(IOAPIC_RANGE_END));
if (!iova) {
printk(KERN_ERR "Reserve IOAPIC range failed\n");
pr_err("Reserve IOAPIC range failed\n");
return -ENODEV;
}
@ -1655,7 +1749,7 @@ static int dmar_init_reserved_ranges(void)
IOVA_PFN(r->start),
IOVA_PFN(r->end));
if (!iova) {
printk(KERN_ERR "Reserve iova failed\n");
pr_err("Reserve iova failed\n");
return -ENODEV;
}
}
@ -1702,7 +1796,7 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
sagaw = cap_sagaw(iommu->cap);
if (!test_bit(agaw, &sagaw)) {
/* hardware doesn't support it, choose a bigger one */
pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
pr_debug("Hardware doesn't support agaw %d\n", agaw);
agaw = find_next_bit(&sagaw, 5, agaw);
if (agaw >= 5)
return -ENODEV;
@ -1795,6 +1889,8 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
return 0;
}
context_clear_entry(context);
id = domain->id;
pgd = domain->pgd;
@ -1803,7 +1899,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
id = iommu_attach_vm_domain(domain, iommu);
if (id < 0) {
spin_unlock_irqrestore(&iommu->lock, flags);
pr_err("IOMMU: no free domain ids\n");
pr_err("%s: No free domain ids\n", iommu->name);
return -EFAULT;
}
}
@ -2030,8 +2126,8 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
if (tmp) {
static int dumps = 5;
printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
iov_pfn, tmp, (unsigned long long)pteval);
pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
iov_pfn, tmp, (unsigned long long)pteval);
if (dumps) {
dumps--;
debug_dma_dump_mappings(NULL);
@ -2303,7 +2399,7 @@ static int iommu_domain_identity_map(struct dmar_domain *domain,
if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
dma_to_mm_pfn(last_vpfn))) {
printk(KERN_ERR "IOMMU: reserve iova failed\n");
pr_err("Reserving iova failed\n");
return -ENOMEM;
}
@ -2336,15 +2432,14 @@ static int iommu_prepare_identity_map(struct device *dev,
range which is reserved in E820, so which didn't get set
up to start with in si_domain */
if (domain == si_domain && hw_pass_through) {
printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
dev_name(dev), start, end);
pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
dev_name(dev), start, end);
return 0;
}
printk(KERN_INFO
"IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
dev_name(dev), start, end);
pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
dev_name(dev), start, end);
if (end < start) {
WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
@ -2401,12 +2496,11 @@ static inline void iommu_prepare_isa(void)
if (!pdev)
return;
printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
pr_info("Prepare 0-16MiB unity mapping for LPC\n");
ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
if (ret)
printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
"floppy might not work\n");
pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
pci_dev_put(pdev);
}
@ -2450,7 +2544,7 @@ static int __init si_domain_init(int hw)
return -EFAULT;
}
pr_debug("IOMMU: identity mapping domain is domain %d\n",
pr_debug("Identity mapping domain is domain %d\n",
si_domain->id);
if (hw)
@ -2650,8 +2744,8 @@ static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw
hw ? CONTEXT_TT_PASS_THROUGH :
CONTEXT_TT_MULTI_LEVEL);
if (!ret)
pr_info("IOMMU: %s identity mapping for device %s\n",
hw ? "hardware" : "software", dev_name(dev));
pr_info("%s identity mapping for device %s\n",
hw ? "Hardware" : "Software", dev_name(dev));
else if (ret == -ENODEV)
/* device not associated with an iommu */
ret = 0;
@ -2669,10 +2763,6 @@ static int __init iommu_prepare_static_identity_mapping(int hw)
int i;
int ret = 0;
ret = si_domain_init(hw);
if (ret)
return -EFAULT;
for_each_pci_dev(pdev) {
ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
if (ret)
@ -2686,7 +2776,7 @@ static int __init iommu_prepare_static_identity_mapping(int hw)
if (dev->bus != &acpi_bus_type)
continue;
adev= to_acpi_device(dev);
mutex_lock(&adev->physical_node_lock);
list_for_each_entry(pn, &adev->physical_node_list, node) {
@ -2728,19 +2818,200 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu)
*/
iommu->flush.flush_context = __iommu_flush_context;
iommu->flush.flush_iotlb = __iommu_flush_iotlb;
pr_info("IOMMU: %s using Register based invalidation\n",
pr_info("%s: Using Register based invalidation\n",
iommu->name);
} else {
iommu->flush.flush_context = qi_flush_context;
iommu->flush.flush_iotlb = qi_flush_iotlb;
pr_info("IOMMU: %s using Queued invalidation\n", iommu->name);
pr_info("%s: Using Queued invalidation\n", iommu->name);
}
}
static int copy_context_table(struct intel_iommu *iommu,
struct root_entry *old_re,
struct context_entry **tbl,
int bus, bool ext)
{
struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
phys_addr_t old_ce_phys;
tbl_idx = ext ? bus * 2 : bus;
for (devfn = 0; devfn < 256; devfn++) {
/* First calculate the correct index */
idx = (ext ? devfn * 2 : devfn) % 256;
if (idx == 0) {
/* First save what we may have and clean up */
if (new_ce) {
tbl[tbl_idx] = new_ce;
__iommu_flush_cache(iommu, new_ce,
VTD_PAGE_SIZE);
pos = 1;
}
if (old_ce)
iounmap(old_ce);
ret = 0;
if (devfn < 0x80)
old_ce_phys = root_entry_lctp(old_re);
else
old_ce_phys = root_entry_uctp(old_re);
if (!old_ce_phys) {
if (ext && devfn == 0) {
/* No LCTP, try UCTP */
devfn = 0x7f;
continue;
} else {
goto out;
}
}
ret = -ENOMEM;
old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
if (!old_ce)
goto out;
new_ce = alloc_pgtable_page(iommu->node);
if (!new_ce)
goto out_unmap;
ret = 0;
}
/* Now copy the context entry */
ce = old_ce[idx];
if (!__context_present(&ce))
continue;
did = context_domain_id(&ce);
if (did >= 0 && did < cap_ndoms(iommu->cap))
set_bit(did, iommu->domain_ids);
/*
* We need a marker for copied context entries. This
* marker needs to work for the old format as well as
* for extended context entries.
*
* Bit 67 of the context entry is used. In the old
* format this bit is available to software, in the
* extended format it is the PGE bit, but PGE is ignored
* by HW if PASIDs are disabled (and thus still
* available).
*
* So disable PASIDs first and then mark the entry
* copied. This means that we don't copy PASID
* translations from the old kernel, but this is fine as
* faults there are not fatal.
*/
context_clear_pasid_enable(&ce);
context_set_copied(&ce);
new_ce[idx] = ce;
}
tbl[tbl_idx + pos] = new_ce;
__iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
out_unmap:
iounmap(old_ce);
out:
return ret;
}
static int copy_translation_tables(struct intel_iommu *iommu)
{
struct context_entry **ctxt_tbls;
struct root_entry *old_rt;
phys_addr_t old_rt_phys;
int ctxt_table_entries;
unsigned long flags;
u64 rtaddr_reg;
int bus, ret;
bool new_ext, ext;
rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
new_ext = !!ecap_ecs(iommu->ecap);
/*
* The RTT bit can only be changed when translation is disabled,
* but disabling translation means to open a window for data
* corruption. So bail out and don't copy anything if we would
* have to change the bit.
*/
if (new_ext != ext)
return -EINVAL;
old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
if (!old_rt_phys)
return -EINVAL;
old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
if (!old_rt)
return -ENOMEM;
/* This is too big for the stack - allocate it from slab */
ctxt_table_entries = ext ? 512 : 256;
ret = -ENOMEM;
ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
if (!ctxt_tbls)
goto out_unmap;
for (bus = 0; bus < 256; bus++) {
ret = copy_context_table(iommu, &old_rt[bus],
ctxt_tbls, bus, ext);
if (ret) {
pr_err("%s: Failed to copy context table for bus %d\n",
iommu->name, bus);
continue;
}
}
spin_lock_irqsave(&iommu->lock, flags);
/* Context tables are copied, now write them to the root_entry table */
for (bus = 0; bus < 256; bus++) {
int idx = ext ? bus * 2 : bus;
u64 val;
if (ctxt_tbls[idx]) {
val = virt_to_phys(ctxt_tbls[idx]) | 1;
iommu->root_entry[bus].lo = val;
}
if (!ext || !ctxt_tbls[idx + 1])
continue;
val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
iommu->root_entry[bus].hi = val;
}
spin_unlock_irqrestore(&iommu->lock, flags);
kfree(ctxt_tbls);
__iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
ret = 0;
out_unmap:
iounmap(old_rt);
return ret;
}
static int __init init_dmars(void)
{
struct dmar_drhd_unit *drhd;
struct dmar_rmrr_unit *rmrr;
bool copied_tables = false;
struct device *dev;
struct intel_iommu *iommu;
int i, ret;
@ -2761,8 +3032,7 @@ static int __init init_dmars(void)
g_num_of_iommus++;
continue;
}
printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
DMAR_UNITS_SUPPORTED);
pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
}
/* Preallocate enough resources for IOMMU hot-addition */
@ -2772,7 +3042,7 @@ static int __init init_dmars(void)
g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
GFP_KERNEL);
if (!g_iommus) {
printk(KERN_ERR "Allocating global iommu array failed\n");
pr_err("Allocating global iommu array failed\n");
ret = -ENOMEM;
goto error;
}
@ -2787,10 +3057,21 @@ static int __init init_dmars(void)
for_each_active_iommu(iommu, drhd) {
g_iommus[iommu->seq_id] = iommu;
intel_iommu_init_qi(iommu);
ret = iommu_init_domains(iommu);
if (ret)
goto free_iommu;
init_translation_status(iommu);
if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
iommu_disable_translation(iommu);
clear_translation_pre_enabled(iommu);
pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
iommu->name);
}
/*
* TBD:
* we could share the same root & context tables
@ -2799,13 +3080,41 @@ static int __init init_dmars(void)
ret = iommu_alloc_root_entry(iommu);
if (ret)
goto free_iommu;
if (translation_pre_enabled(iommu)) {
pr_info("Translation already enabled - trying to copy translation structures\n");
ret = copy_translation_tables(iommu);
if (ret) {
/*
* We found the IOMMU with translation
* enabled - but failed to copy over the
* old root-entry table. Try to proceed
* by disabling translation now and
* allocating a clean root-entry table.
* This might cause DMAR faults, but
* probably the dump will still succeed.
*/
pr_err("Failed to copy translation tables from previous kernel for %s\n",
iommu->name);
iommu_disable_translation(iommu);
clear_translation_pre_enabled(iommu);
} else {
pr_info("Copied translation tables from previous kernel for %s\n",
iommu->name);
copied_tables = true;
}
}
iommu_flush_write_buffer(iommu);
iommu_set_root_entry(iommu);
iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
if (!ecap_pass_through(iommu->ecap))
hw_pass_through = 0;
}
for_each_active_iommu(iommu, drhd)
intel_iommu_init_qi(iommu);
if (iommu_pass_through)
iommu_identity_mapping |= IDENTMAP_ALL;
@ -2813,8 +3122,23 @@ static int __init init_dmars(void)
iommu_identity_mapping |= IDENTMAP_GFX;
#endif
if (iommu_identity_mapping) {
ret = si_domain_init(hw_pass_through);
if (ret)
goto free_iommu;
}
check_tylersburg_isoch();
/*
* If we copied translations from a previous kernel in the kdump
* case, we can not assign the devices to domains now, as that
* would eliminate the old mappings. So skip this part and defer
* the assignment to device driver initialization time.
*/
if (copied_tables)
goto domains_done;
/*
* If pass through is not set or not enabled, setup context entries for
* identity mappings for rmrr, gfx, and isa and may fall back to static
@ -2823,7 +3147,7 @@ static int __init init_dmars(void)
if (iommu_identity_mapping) {
ret = iommu_prepare_static_identity_mapping(hw_pass_through);
if (ret) {
printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
pr_crit("Failed to setup IOMMU pass-through\n");
goto free_iommu;
}
}
@ -2841,20 +3165,21 @@ static int __init init_dmars(void)
* endfor
* endfor
*/
printk(KERN_INFO "IOMMU: Setting RMRR:\n");
pr_info("Setting RMRR:\n");
for_each_rmrr_units(rmrr) {
/* some BIOS lists non-exist devices in DMAR table. */
for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
i, dev) {
ret = iommu_prepare_rmrr_dev(rmrr, dev);
if (ret)
printk(KERN_ERR
"IOMMU: mapping reserved region failed\n");
pr_err("Mapping reserved region failed\n");
}
}
iommu_prepare_isa();
domains_done:
/*
* for each drhd
* enable fault log
@ -2879,11 +3204,9 @@ static int __init init_dmars(void)
if (ret)
goto free_iommu;
iommu_set_root_entry(iommu);
if (!translation_pre_enabled(iommu))
iommu_enable_translation(iommu);
iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
iommu_enable_translation(iommu);
iommu_disable_protect_mem_regions(iommu);
}
@ -2924,7 +3247,7 @@ static struct iova *intel_alloc_iova(struct device *dev,
}
iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
if (unlikely(!iova)) {
printk(KERN_ERR "Allocating %ld-page iova for %s failed",
pr_err("Allocating %ld-page iova for %s failed",
nrpages, dev_name(dev));
return NULL;
}
@ -2939,7 +3262,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
if (!domain) {
printk(KERN_ERR "Allocating domain for %s failed",
pr_err("Allocating domain for %s failed\n",
dev_name(dev));
return NULL;
}
@ -2948,7 +3271,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
if (unlikely(!domain_context_mapped(dev))) {
ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
if (ret) {
printk(KERN_ERR "Domain context map for %s failed",
pr_err("Domain context map for %s failed\n",
dev_name(dev));
return NULL;
}
@ -2969,11 +3292,6 @@ static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
return __get_valid_domain_for_dev(dev);
}
static int iommu_dummy(struct device *dev)
{
return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
}
/* Check if the dev needs to go through non-identity map and unmap process.*/
static int iommu_no_mapping(struct device *dev)
{
@ -2995,8 +3313,8 @@ static int iommu_no_mapping(struct device *dev)
* to non-identity mapping.
*/
domain_remove_one_dev_info(si_domain, dev);
printk(KERN_INFO "32bit %s uses non-identity mapping\n",
dev_name(dev));
pr_info("32bit %s uses non-identity mapping\n",
dev_name(dev));
return 0;
}
} else {
@ -3011,8 +3329,8 @@ static int iommu_no_mapping(struct device *dev)
CONTEXT_TT_PASS_THROUGH :
CONTEXT_TT_MULTI_LEVEL);
if (!ret) {
printk(KERN_INFO "64bit %s uses identity mapping\n",
dev_name(dev));
pr_info("64bit %s uses identity mapping\n",
dev_name(dev));
return 1;
}
}
@ -3081,7 +3399,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
error:
if (iova)
__free_iova(&domain->iovad, iova);
printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
dev_name(dev), size, (unsigned long long)paddr, dir);
return 0;
}
@ -3396,7 +3714,7 @@ static inline int iommu_domain_cache_init(void)
NULL);
if (!iommu_domain_cache) {
printk(KERN_ERR "Couldn't create iommu_domain cache\n");
pr_err("Couldn't create iommu_domain cache\n");
ret = -ENOMEM;
}
@ -3413,7 +3731,7 @@ static inline int iommu_devinfo_cache_init(void)
SLAB_HWCACHE_ALIGN,
NULL);
if (!iommu_devinfo_cache) {
printk(KERN_ERR "Couldn't create devinfo cache\n");
pr_err("Couldn't create devinfo cache\n");
ret = -ENOMEM;
}
@ -3790,19 +4108,19 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
return 0;
if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
pr_warn("IOMMU: %s doesn't support hardware pass through.\n",
pr_warn("%s: Doesn't support hardware pass through.\n",
iommu->name);
return -ENXIO;
}
if (!ecap_sc_support(iommu->ecap) &&
domain_update_iommu_snooping(iommu)) {
pr_warn("IOMMU: %s doesn't support snooping.\n",
pr_warn("%s: Doesn't support snooping.\n",
iommu->name);
return -ENXIO;
}
sp = domain_update_iommu_superpage(iommu) - 1;
if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
pr_warn("IOMMU: %s doesn't support large page.\n",
pr_warn("%s: Doesn't support large page.\n",
iommu->name);
return -ENXIO;
}
@ -4033,7 +4351,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
start = mhp->start_pfn << PAGE_SHIFT;
end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
if (iommu_domain_identity_map(si_domain, start, end)) {
pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
pr_warn("Failed to build identity map for [%llx-%llx]\n",
start, end);
return NOTIFY_BAD;
}
@ -4051,7 +4369,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
iova = find_iova(&si_domain->iovad, start_vpfn);
if (iova == NULL) {
pr_debug("dmar: failed get IOVA for PFN %lx\n",
pr_debug("Failed get IOVA for PFN %lx\n",
start_vpfn);
break;
}
@ -4059,7 +4377,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
iova = split_and_remove_iova(&si_domain->iovad, iova,
start_vpfn, last_vpfn);
if (iova == NULL) {
pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
start_vpfn, last_vpfn);
return NOTIFY_BAD;
}
@ -4168,13 +4486,6 @@ int __init intel_iommu_init(void)
goto out_free_dmar;
}
/*
* Disable translation if already enabled prior to OS handover.
*/
for_each_active_iommu(iommu, drhd)
if (iommu->gcmd & DMA_GCMD_TE)
iommu_disable_translation(iommu);
if (dmar_dev_scope_init() < 0) {
if (force_on)
panic("tboot: Failed to initialize DMAR device scope\n");
@ -4185,10 +4496,10 @@ int __init intel_iommu_init(void)
goto out_free_dmar;
if (list_empty(&dmar_rmrr_units))
printk(KERN_INFO "DMAR: No RMRR found\n");
pr_info("No RMRR found\n");
if (list_empty(&dmar_atsr_units))
printk(KERN_INFO "DMAR: No ATSR found\n");
pr_info("No ATSR found\n");
if (dmar_init_reserved_ranges()) {
if (force_on)
@ -4202,12 +4513,11 @@ int __init intel_iommu_init(void)
if (ret) {
if (force_on)
panic("tboot: Failed to initialize DMARs\n");
printk(KERN_ERR "IOMMU: dmar init failed\n");
pr_err("Initialization failed\n");
goto out_free_reserved_range;
}
up_write(&dmar_global_lock);
printk(KERN_INFO
"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
init_timer(&unmap_timer);
#ifdef CONFIG_SWIOTLB
@ -4349,13 +4659,11 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
if (!dmar_domain) {
printk(KERN_ERR
"intel_iommu_domain_init: dmar_domain == NULL\n");
pr_err("Can't allocate dmar_domain\n");
return NULL;
}
if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
printk(KERN_ERR
"intel_iommu_domain_init() failed\n");
pr_err("Domain initialization failed\n");
domain_exit(dmar_domain);
return NULL;
}
@ -4414,7 +4722,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
addr_width = cap_mgaw(iommu->cap);
if (dmar_domain->max_addr > (1LL << addr_width)) {
printk(KERN_ERR "%s: iommu width (%d) is not "
pr_err("%s: iommu width (%d) is not "
"sufficient for the mapped address (%llx)\n",
__func__, addr_width, dmar_domain->max_addr);
return -EFAULT;
@ -4468,7 +4776,7 @@ static int intel_iommu_map(struct iommu_domain *domain,
/* check if minimum agaw is sufficient for mapped address */
end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
if (end < max_addr) {
printk(KERN_ERR "%s: iommu width (%d) is not "
pr_err("%s: iommu width (%d) is not "
"sufficient for the mapped address (%llx)\n",
__func__, dmar_domain->gaw, max_addr);
return -EFAULT;
@ -4609,7 +4917,7 @@ static const struct iommu_ops intel_iommu_ops = {
static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
{
/* G4x/GM45 integrated gfx dmar support is totally busted. */
printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
pr_info("Disabling IOMMU for graphics on this chipset\n");
dmar_map_gfx = 0;
}
@ -4627,7 +4935,7 @@ static void quirk_iommu_rwbf(struct pci_dev *dev)
* Mobile 4 Series Chipset neglects to set RWBF capability,
* but needs it. Same seems to hold for the desktop versions.
*/
printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
pr_info("Forcing write-buffer flush capability\n");
rwbf_quirk = 1;
}
@ -4657,11 +4965,11 @@ static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
return;
if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
dmar_map_gfx = 0;
} else if (dmar_map_gfx) {
/* we have to ensure the gfx device is idle before we flush */
printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
pr_info("Disabling batched IOTLB flush on Ironlake\n");
intel_iommu_strict = 1;
}
}
@ -4723,7 +5031,7 @@ static void __init check_tylersburg_isoch(void)
iommu_identity_mapping |= IDENTMAP_AZALIA;
return;
}
printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
vtisochctrl);
}

259
drivers/iommu/intel_irq_remapping.c
View file

@ -1,3 +1,6 @@
#define pr_fmt(fmt) "DMAR-IR: " fmt
#include <linux/interrupt.h>
#include <linux/dmar.h>
#include <linux/spinlock.h>
@ -8,6 +11,7 @@
#include <linux/irq.h>
#include <linux/intel-iommu.h>
#include <linux/acpi.h>
#include <linux/crash_dump.h>
#include <asm/io_apic.h>
#include <asm/smp.h>
#include <asm/cpu.h>
@ -51,8 +55,28 @@ static struct hpet_scope ir_hpet[MAX_HPET_TBS];
*/
static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
static int __init parse_ioapics_under_ir(void);
static bool ir_pre_enabled(struct intel_iommu *iommu)
{
return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);
}
static void clear_ir_pre_enabled(struct intel_iommu *iommu)
{
iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;
}
static void init_ir_status(struct intel_iommu *iommu)
{
u32 gsts;
gsts = readl(iommu->reg + DMAR_GSTS_REG);
if (gsts & DMA_GSTS_IRES)
iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;
}
static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
{
struct irq_cfg *cfg = irq_cfg(irq);
@ -100,8 +124,7 @@ static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
}
if (mask > ecap_max_handle_mask(iommu->ecap)) {
printk(KERN_ERR
"Requested mask %x exceeds the max invalidation handle"
pr_err("Requested mask %x exceeds the max invalidation handle"
" mask value %Lx\n", mask,
ecap_max_handle_mask(iommu->ecap));
return -1;
@ -333,7 +356,7 @@ static int set_ioapic_sid(struct irte *irte, int apic)
up_read(&dmar_global_lock);
if (sid == 0) {
pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic);
return -1;
}
@ -360,7 +383,7 @@ static int set_hpet_sid(struct irte *irte, u8 id)
up_read(&dmar_global_lock);
if (sid == 0) {
pr_warning("Failed to set source-id of HPET block (%d)\n", id);
pr_warn("Failed to set source-id of HPET block (%d)\n", id);
return -1;
}
@ -424,11 +447,59 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev)
return 0;
}
static int iommu_load_old_irte(struct intel_iommu *iommu)
{
struct irte *old_ir_table;
phys_addr_t irt_phys;
unsigned int i;
size_t size;
u64 irta;
if (!is_kdump_kernel()) {
pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n",
iommu->name);
clear_ir_pre_enabled(iommu);
iommu_disable_irq_remapping(iommu);
return -EINVAL;
}
/* Check whether the old ir-table has the same size as ours */
irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);
if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK)
!= INTR_REMAP_TABLE_REG_SIZE)
return -EINVAL;
irt_phys = irta & VTD_PAGE_MASK;
size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte);
/* Map the old IR table */
old_ir_table = ioremap_cache(irt_phys, size);
if (!old_ir_table)
return -ENOMEM;
/* Copy data over */
memcpy(iommu->ir_table->base, old_ir_table, size);
__iommu_flush_cache(iommu, iommu->ir_table->base, size);
/*
* Now check the table for used entries and mark those as
* allocated in the bitmap
*/
for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) {
if (iommu->ir_table->base[i].present)
bitmap_set(iommu->ir_table->bitmap, i, 1);
}
return 0;
}
static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
{
unsigned long flags;
u64 addr;
u32 sts;
unsigned long flags;
addr = virt_to_phys((void *)iommu->ir_table->base);
@ -445,10 +516,16 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
/*
* global invalidation of interrupt entry cache before enabling
* interrupt-remapping.
* Global invalidation of interrupt entry cache to make sure the
* hardware uses the new irq remapping table.
*/
qi_global_iec(iommu);
}
static void iommu_enable_irq_remapping(struct intel_iommu *iommu)
{
unsigned long flags;
u32 sts;
raw_spin_lock_irqsave(&iommu->register_lock, flags);
@ -505,12 +582,48 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
ir_table->base = page_address(pages);
ir_table->bitmap = bitmap;
iommu->ir_table = ir_table;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (!iommu->qi) {
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
dmar_disable_qi(iommu);
if (dmar_enable_qi(iommu)) {
pr_err("Failed to enable queued invalidation\n");
goto out_free_bitmap;
}
}
init_ir_status(iommu);
if (ir_pre_enabled(iommu)) {
if (iommu_load_old_irte(iommu))
pr_err("Failed to copy IR table for %s from previous kernel\n",
iommu->name);
else
pr_info("Copied IR table for %s from previous kernel\n",
iommu->name);
}
iommu_set_irq_remapping(iommu, eim_mode);
return 0;
out_free_bitmap:
kfree(bitmap);
out_free_pages:
__free_pages(pages, INTR_REMAP_PAGE_ORDER);
out_free_table:
kfree(ir_table);
iommu->ir_table = NULL;
return -ENOMEM;
}
@ -580,17 +693,17 @@ static void __init intel_cleanup_irq_remapping(void)
}
if (x2apic_supported())
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
}
static int __init intel_prepare_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
int eim = 0;
if (irq_remap_broken) {
printk(KERN_WARNING
"This system BIOS has enabled interrupt remapping\n"
pr_warn("This system BIOS has enabled interrupt remapping\n"
"on a chipset that contains an erratum making that\n"
"feature unstable. To maintain system stability\n"
"interrupt remapping is being disabled. Please\n"
@ -606,7 +719,7 @@ static int __init intel_prepare_irq_remapping(void)
return -ENODEV;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enabling interrupt remapping\n");
pr_info("Not enabling interrupt remapping\n");
goto error;
}
@ -615,10 +728,34 @@ static int __init intel_prepare_irq_remapping(void)
if (!ecap_ir_support(iommu->ecap))
goto error;
/* Do the allocations early */
for_each_iommu(iommu, drhd)
if (intel_setup_irq_remapping(iommu))
/* Detect remapping mode: lapic or x2apic */
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
if (!eim) {
pr_info("x2apic is disabled because BIOS sets x2apic opt out bit.");
pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
}
}
for_each_iommu(iommu, drhd) {
if (eim && !ecap_eim_support(iommu->ecap)) {
pr_info("%s does not support EIM\n", iommu->name);
eim = 0;
}
}
eim_mode = eim;
if (eim)
pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
/* Do the initializations early */
for_each_iommu(iommu, drhd) {
if (intel_setup_irq_remapping(iommu)) {
pr_err("Failed to setup irq remapping for %s\n",
iommu->name);
goto error;
}
}
return 0;
@ -632,68 +769,13 @@ static int __init intel_enable_irq_remapping(void)
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
bool setup = false;
int eim = 0;
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
if (!eim)
pr_info("x2apic is disabled because BIOS sets x2apic opt out bit. You can use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
}
for_each_iommu(iommu, drhd) {
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_iommu(iommu, drhd)
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
eim = 0;
}
eim_mode = eim;
if (eim)
pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_iommu(iommu, drhd) {
int ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
goto error;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_iommu(iommu, drhd) {
iommu_set_irq_remapping(iommu, eim);
if (!ir_pre_enabled(iommu))
iommu_enable_irq_remapping(iommu);
setup = true;
}
@ -709,9 +791,9 @@ static int __init intel_enable_irq_remapping(void)
*/
x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
intel_cleanup_irq_remapping();
@ -930,6 +1012,7 @@ static int reenable_irq_remapping(int eim)
/* Set up interrupt remapping for iommu.*/
iommu_set_irq_remapping(iommu, eim);
iommu_enable_irq_remapping(iommu);
setup = true;
}
@ -1145,14 +1228,12 @@ static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec)
down_read(&dmar_global_lock);
iommu = map_dev_to_ir(dev);
if (!iommu) {
printk(KERN_ERR
"Unable to map PCI %s to iommu\n", pci_name(dev));
pr_err("Unable to map PCI %s to iommu\n", pci_name(dev));
index = -ENOENT;
} else {
index = alloc_irte(iommu, irq, nvec);
if (index < 0) {
printk(KERN_ERR
"Unable to allocate %d IRTE for PCI %s\n",
pr_err("Unable to allocate %d IRTE for PCI %s\n",
nvec, pci_name(dev));
index = -ENOSPC;
}
@ -1242,28 +1323,12 @@ static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)
/* Setup Interrupt-remapping now. */
ret = intel_setup_irq_remapping(iommu);
if (ret) {
pr_err("DRHD %Lx: failed to allocate resource\n",
iommu->reg_phys);
ir_remove_ioapic_hpet_scope(iommu);
return ret;
}
if (!iommu->qi) {
/* Clear previous faults. */
dmar_fault(-1, iommu);
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/* Enable queued invalidation */
ret = dmar_enable_qi(iommu);
if (!ret) {
iommu_set_irq_remapping(iommu, eim);
} else {
pr_err("DRHD %Lx: failed to enable queued invalidation, ecap %Lx, ret %d\n",
iommu->reg_phys, iommu->ecap, ret);
pr_err("Failed to setup irq remapping for %s\n",
iommu->name);
intel_teardown_irq_remapping(iommu);
ir_remove_ioapic_hpet_scope(iommu);
} else {
iommu_enable_irq_remapping(iommu);
}
return ret;

373
drivers/iommu/iommu.c
View file

@ -16,7 +16,7 @@
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#define pr_fmt(fmt) "iommu: " fmt
#include <linux/device.h>
#include <linux/kernel.h>
@ -51,6 +51,8 @@ struct iommu_group {
void (*iommu_data_release)(void *iommu_data);
char *name;
int id;
struct iommu_domain *default_domain;
struct iommu_domain *domain;
};
struct iommu_device {
@ -75,6 +77,15 @@ struct iommu_group_attribute iommu_group_attr_##_name = \
#define to_iommu_group(_kobj) \
container_of(_kobj, struct iommu_group, kobj)
static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
unsigned type);
static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev);
static int __iommu_attach_group(struct iommu_domain *domain,
struct iommu_group *group);
static void __iommu_detach_group(struct iommu_domain *domain,
struct iommu_group *group);
static ssize_t iommu_group_attr_show(struct kobject *kobj,
struct attribute *__attr, char *buf)
{
@ -128,6 +139,8 @@ static void iommu_group_release(struct kobject *kobj)
{
struct iommu_group *group = to_iommu_group(kobj);
pr_debug("Releasing group %d\n", group->id);
if (group->iommu_data_release)
group->iommu_data_release(group->iommu_data);
@ -135,6 +148,9 @@ static void iommu_group_release(struct kobject *kobj)
ida_remove(&iommu_group_ida, group->id);
mutex_unlock(&iommu_group_mutex);
if (group->default_domain)
iommu_domain_free(group->default_domain);
kfree(group->name);
kfree(group);
}
@ -207,6 +223,8 @@ again:
*/
kobject_put(&group->kobj);
pr_debug("Allocated group %d\n", group->id);
return group;
}
EXPORT_SYMBOL_GPL(iommu_group_alloc);
@ -307,6 +325,52 @@ int iommu_group_set_name(struct iommu_group *group, const char *name)
}
EXPORT_SYMBOL_GPL(iommu_group_set_name);
static int iommu_group_create_direct_mappings(struct iommu_group *group,
struct device *dev)
{
struct iommu_domain *domain = group->default_domain;
struct iommu_dm_region *entry;
struct list_head mappings;
unsigned long pg_size;
int ret = 0;
if (!domain || domain->type != IOMMU_DOMAIN_DMA)
return 0;
BUG_ON(!domain->ops->pgsize_bitmap);
pg_size = 1UL << __ffs(domain->ops->pgsize_bitmap);
INIT_LIST_HEAD(&mappings);
iommu_get_dm_regions(dev, &mappings);
/* We need to consider overlapping regions for different devices */
list_for_each_entry(entry, &mappings, list) {
dma_addr_t start, end, addr;
start = ALIGN(entry->start, pg_size);
end = ALIGN(entry->start + entry->length, pg_size);
for (addr = start; addr < end; addr += pg_size) {
phys_addr_t phys_addr;
phys_addr = iommu_iova_to_phys(domain, addr);
if (phys_addr)
continue;
ret = iommu_map(domain, addr, addr, pg_size, entry->prot);
if (ret)
goto out;
}
}
out:
iommu_put_dm_regions(dev, &mappings);
return ret;
}
/**
* iommu_group_add_device - add a device to an iommu group
* @group: the group into which to add the device (reference should be held)
@ -363,8 +427,12 @@ rename:
dev->iommu_group = group;
iommu_group_create_direct_mappings(group, dev);
mutex_lock(&group->mutex);
list_add_tail(&device->list, &group->devices);
if (group->domain)
__iommu_attach_device(group->domain, dev);
mutex_unlock(&group->mutex);
/* Notify any listeners about change to group. */
@ -372,6 +440,9 @@ rename:
IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev);
trace_add_device_to_group(group->id, dev);
pr_info("Adding device %s to group %d\n", dev_name(dev), group->id);
return 0;
}
EXPORT_SYMBOL_GPL(iommu_group_add_device);
@ -388,6 +459,8 @@ void iommu_group_remove_device(struct device *dev)
struct iommu_group *group = dev->iommu_group;
struct iommu_device *tmp_device, *device = NULL;
pr_info("Removing device %s from group %d\n", dev_name(dev), group->id);
/* Pre-notify listeners that a device is being removed. */
blocking_notifier_call_chain(&group->notifier,
IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev);
@ -417,6 +490,17 @@ void iommu_group_remove_device(struct device *dev)
}
EXPORT_SYMBOL_GPL(iommu_group_remove_device);
static int iommu_group_device_count(struct iommu_group *group)
{
struct iommu_device *entry;
int ret = 0;
list_for_each_entry(entry, &group->devices, list)
ret++;
return ret;
}
/**
* iommu_group_for_each_dev - iterate over each device in the group
* @group: the group
@ -428,19 +512,30 @@ EXPORT_SYMBOL_GPL(iommu_group_remove_device);
* The group->mutex is held across callbacks, which will block calls to
* iommu_group_add/remove_device.
*/
int iommu_group_for_each_dev(struct iommu_group *group, void *data,
int (*fn)(struct device *, void *))
static int __iommu_group_for_each_dev(struct iommu_group *group, void *data,
int (*fn)(struct device *, void *))
{
struct iommu_device *device;
int ret = 0;
mutex_lock(&group->mutex);
list_for_each_entry(device, &group->devices, list) {
ret = fn(device->dev, data);
if (ret)
break;
}
return ret;
}
int iommu_group_for_each_dev(struct iommu_group *group, void *data,
int (*fn)(struct device *, void *))
{
int ret;
mutex_lock(&group->mutex);
ret = __iommu_group_for_each_dev(group, data, fn);
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_group_for_each_dev);
@ -692,7 +787,19 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
return group;
/* No shared group found, allocate new */
return iommu_group_alloc();
group = iommu_group_alloc();
if (IS_ERR(group))
return NULL;
/*
* Try to allocate a default domain - needs support from the
* IOMMU driver.
*/
group->default_domain = __iommu_domain_alloc(pdev->dev.bus,
IOMMU_DOMAIN_DMA);
group->domain = group->default_domain;
return group;
}
/**
@ -731,6 +838,11 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
return group;
}
struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
{
return group->default_domain;
}
static int add_iommu_group(struct device *dev, void *data)
{
struct iommu_callback_data *cb = data;
@ -741,7 +853,16 @@ static int add_iommu_group(struct device *dev, void *data)
WARN_ON(dev->iommu_group);
ops->add_device(dev);
return ops->add_device(dev);
}
static int remove_iommu_group(struct device *dev, void *data)
{
struct iommu_callback_data *cb = data;
const struct iommu_ops *ops = cb->ops;
if (ops->remove_device && dev->iommu_group)
ops->remove_device(dev);
return 0;
}
@ -761,7 +882,7 @@ static int iommu_bus_notifier(struct notifier_block *nb,
if (action == BUS_NOTIFY_ADD_DEVICE) {
if (ops->add_device)
return ops->add_device(dev);
} else if (action == BUS_NOTIFY_DEL_DEVICE) {
} else if (action == BUS_NOTIFY_REMOVED_DEVICE) {
if (ops->remove_device && dev->iommu_group) {
ops->remove_device(dev);
return 0;
@ -814,19 +935,25 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
nb->notifier_call = iommu_bus_notifier;
err = bus_register_notifier(bus, nb);
if (err) {
kfree(nb);
return err;
}
if (err)
goto out_free;
err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
if (err) {
bus_unregister_notifier(bus, nb);
kfree(nb);
return err;
}
if (err)
goto out_err;
return 0;
out_err:
/* Clean up */
bus_for_each_dev(bus, NULL, &cb, remove_iommu_group);
bus_unregister_notifier(bus, nb);
out_free:
kfree(nb);
return err;
}
/**
@ -898,22 +1025,28 @@ void iommu_set_fault_handler(struct iommu_domain *domain,
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
unsigned type)
{
struct iommu_domain *domain;
if (bus == NULL || bus->iommu_ops == NULL)
return NULL;
domain = bus->iommu_ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED);
domain = bus->iommu_ops->domain_alloc(type);
if (!domain)
return NULL;
domain->ops = bus->iommu_ops;
domain->type = IOMMU_DOMAIN_UNMANAGED;
domain->type = type;
return domain;
}
struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
{
return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED);
}
EXPORT_SYMBOL_GPL(iommu_domain_alloc);
void iommu_domain_free(struct iommu_domain *domain)
@ -922,7 +1055,8 @@ void iommu_domain_free(struct iommu_domain *domain)
}
EXPORT_SYMBOL_GPL(iommu_domain_free);
int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
int ret;
if (unlikely(domain->ops->attach_dev == NULL))
@ -933,9 +1067,38 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
trace_attach_device_to_domain(dev);
return ret;
}
int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
struct iommu_group *group;
int ret;
group = iommu_group_get(dev);
/* FIXME: Remove this when groups a mandatory for iommu drivers */
if (group == NULL)
return __iommu_attach_device(domain, dev);
/*
* We have a group - lock it to make sure the device-count doesn't
* change while we are attaching
*/
mutex_lock(&group->mutex);
ret = -EINVAL;
if (iommu_group_device_count(group) != 1)
goto out_unlock;
ret = __iommu_attach_group(domain, group);
out_unlock:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_device);
void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
static void __iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
if (unlikely(domain->ops->detach_dev == NULL))
return;
@ -943,8 +1106,48 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
domain->ops->detach_dev(domain, dev);
trace_detach_device_from_domain(dev);
}
void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
{
struct iommu_group *group;
group = iommu_group_get(dev);
/* FIXME: Remove this when groups a mandatory for iommu drivers */
if (group == NULL)
return __iommu_detach_device(domain, dev);
mutex_lock(&group->mutex);
if (iommu_group_device_count(group) != 1) {
WARN_ON(1);
goto out_unlock;
}
__iommu_detach_group(domain, group);
out_unlock:
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
EXPORT_SYMBOL_GPL(iommu_detach_device);
struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
{
struct iommu_domain *domain;
struct iommu_group *group;
group = iommu_group_get(dev);
/* FIXME: Remove this when groups a mandatory for iommu drivers */
if (group == NULL)
return NULL;
domain = group->domain;
iommu_group_put(group);
return domain;
}
EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev);
/*
* IOMMU groups are really the natrual working unit of the IOMMU, but
* the IOMMU API works on domains and devices. Bridge that gap by
@ -959,13 +1162,34 @@ static int iommu_group_do_attach_device(struct device *dev, void *data)
{
struct iommu_domain *domain = data;
return iommu_attach_device(domain, dev);
return __iommu_attach_device(domain, dev);
}
static int __iommu_attach_group(struct iommu_domain *domain,
struct iommu_group *group)
{
int ret;
if (group->default_domain && group->domain != group->default_domain)
return -EBUSY;
ret = __iommu_group_for_each_dev(group, domain,
iommu_group_do_attach_device);
if (ret == 0)
group->domain = domain;
return ret;
}
int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
{
return iommu_group_for_each_dev(group, domain,
iommu_group_do_attach_device);
int ret;
mutex_lock(&group->mutex);
ret = __iommu_attach_group(domain, group);
mutex_unlock(&group->mutex);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_group);
@ -973,14 +1197,40 @@ static int iommu_group_do_detach_device(struct device *dev, void *data)
{
struct iommu_domain *domain = data;
iommu_detach_device(domain, dev);
__iommu_detach_device(domain, dev);
return 0;
}
static void __iommu_detach_group(struct iommu_domain *domain,
struct iommu_group *group)
{
int ret;
if (!group->default_domain) {
__iommu_group_for_each_dev(group, domain,
iommu_group_do_detach_device);
group->domain = NULL;
return;
}
if (group->domain == group->default_domain)
return;
/* Detach by re-attaching to the default domain */
ret = __iommu_group_for_each_dev(group, group->default_domain,
iommu_group_do_attach_device);
if (ret != 0)
WARN_ON(1);
else
group->domain = group->default_domain;
}
void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
{
iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device);
mutex_lock(&group->mutex);
__iommu_detach_group(domain, group);
mutex_unlock(&group->mutex);
}
EXPORT_SYMBOL_GPL(iommu_detach_group);
@ -1207,7 +1457,7 @@ static int __init iommu_init(void)
return 0;
}
arch_initcall(iommu_init);
core_initcall(iommu_init);
int iommu_domain_get_attr(struct iommu_domain *domain,
enum iommu_attr attr, void *data)
@ -1273,3 +1523,72 @@ int iommu_domain_set_attr(struct iommu_domain *domain,
return ret;
}
EXPORT_SYMBOL_GPL(iommu_domain_set_attr);
void iommu_get_dm_regions(struct device *dev, struct list_head *list)
{
const struct iommu_ops *ops = dev->bus->iommu_ops;
if (ops && ops->get_dm_regions)
ops->get_dm_regions(dev, list);
}
void iommu_put_dm_regions(struct device *dev, struct list_head *list)
{
const struct iommu_ops *ops = dev->bus->iommu_ops;
if (ops && ops->put_dm_regions)
ops->put_dm_regions(dev, list);
}
/* Request that a device is direct mapped by the IOMMU */
int iommu_request_dm_for_dev(struct device *dev)
{
struct iommu_domain *dm_domain;
struct iommu_group *group;
int ret;
/* Device must already be in a group before calling this function */
group = iommu_group_get_for_dev(dev);
if (IS_ERR(group))
return PTR_ERR(group);
mutex_lock(&group->mutex);
/* Check if the default domain is already direct mapped */
ret = 0;
if (group->default_domain &&
group->default_domain->type == IOMMU_DOMAIN_IDENTITY)
goto out;
/* Don't change mappings of existing devices */
ret = -EBUSY;
if (iommu_group_device_count(group) != 1)
goto out;
/* Allocate a direct mapped domain */
ret = -ENOMEM;
dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY);
if (!dm_domain)
goto out;
/* Attach the device to the domain */
ret = __iommu_attach_group(dm_domain, group);
if (ret) {
iommu_domain_free(dm_domain);
goto out;
}
/* Make the direct mapped domain the default for this group */
if (group->default_domain)
iommu_domain_free(group->default_domain);
group->default_domain = dm_domain;
pr_info("Using direct mapping for device %s\n", dev_name(dev));
ret = 0;
out:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}

4
drivers/iommu/iova.c
View file

@ -227,6 +227,7 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova)
/* Figure out where to put new node */
while (*new) {
struct iova *this = container_of(*new, struct iova, node);
parent = *new;
if (iova->pfn_lo < this->pfn_lo)
@ -350,6 +351,7 @@ void
free_iova(struct iova_domain *iovad, unsigned long pfn)
{
struct iova *iova = find_iova(iovad, pfn);
if (iova)
__free_iova(iovad, iova);
@ -369,6 +371,7 @@ void put_iova_domain(struct iova_domain *iovad)
node = rb_first(&iovad->rbroot);
while (node) {
struct iova *iova = container_of(node, struct iova, node);
rb_erase(node, &iovad->rbroot);
free_iova_mem(iova);
node = rb_first(&iovad->rbroot);
@ -482,6 +485,7 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
for (node = rb_first(&from->rbroot); node; node = rb_next(node)) {
struct iova *iova = container_of(node, struct iova, node);
struct iova *new_iova;
new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi);
if (!new_iova)
printk(KERN_ERR "Reserve iova range %lx@%lx failed\n",

4
drivers/iommu/rockchip-iommu.c
View file

@ -1015,20 +1015,18 @@ static int rk_iommu_remove(struct platform_device *pdev)
return 0;
}
#ifdef CONFIG_OF
static const struct of_device_id rk_iommu_dt_ids[] = {
{ .compatible = "rockchip,iommu" },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids);
#endif
static struct platform_driver rk_iommu_driver = {
.probe = rk_iommu_probe,
.remove = rk_iommu_remove,
.driver = {
.name = "rk_iommu",
.of_match_table = of_match_ptr(rk_iommu_dt_ids),
.of_match_table = rk_iommu_dt_ids,
},
};