mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-08 04:42:04 -04:00
iommu/arm-smmu: Merge all IOMMU changes from msm-4.19 to msm-lahaina
This patch merges all of the IOMMU/SMMU, DMA mapping, fast, and lazy mapping changes from msm-4.19 into msm-lahaina. Change-Id: If7c1f641a8c836dbb799e2f3439f443ff299b299 Signed-off-by: Isaac J. Manjarres <isaacm@codeaurora.org>
This commit is contained in:
Isaac J. Manjarres
parent
b18bea500c
commit
fb3f251037
31 changed files with 11748 additions and 1791 deletions
|
|
@ -907,6 +907,27 @@ config ARCH_WANT_HUGE_PMD_SHARE
|
|||
config ARCH_HAS_CACHE_LINE_SIZE
|
||||
def_bool y
|
||||
|
||||
if ARM64 && IOMMU_DMA
|
||||
|
||||
config ARM64_DMA_IOMMU_ALIGNMENT
|
||||
int "Maximum PAGE_SIZE order of alignment for DMA IOMMU buffers"
|
||||
range 4 9
|
||||
default 9
|
||||
help
|
||||
DMA mapping framework by default aligns all buffers to the smallest
|
||||
PAGE_SIZE order which is greater than or equal to the requested buffer
|
||||
size. This works well for buffers up to a few hundreds kilobytes, but
|
||||
for larger buffers it just a waste of address space. Drivers which has
|
||||
relatively small addressing window (like 64Mib) might run out of
|
||||
virtual space with just a few allocations.
|
||||
|
||||
With this parameter you can specify the maximum PAGE_SIZE order for
|
||||
DMA IOMMU buffers. Larger buffers will be aligned only to this
|
||||
specified order. The order is expressed as a power of two multiplied
|
||||
by the PAGE_SIZE.
|
||||
|
||||
endif
|
||||
|
||||
config ARCH_ENABLE_SPLIT_PMD_PTLOCK
|
||||
def_bool y if PGTABLE_LEVELS > 2
|
||||
|
||||
|
|
|
|||
|
|
@ -16,16 +16,32 @@
|
|||
#include <linux/dma-direct.h>
|
||||
#include <linux/dma-noncoherent.h>
|
||||
#include <linux/dma-contiguous.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/swiotlb.h>
|
||||
#include <linux/pci.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <linux/of_address.h>
|
||||
#include <linux/dma-mapping-fast.h>
|
||||
|
||||
|
||||
static bool is_dma_coherent(struct device *dev, unsigned long attrs)
|
||||
{
|
||||
if (attrs & DMA_ATTR_FORCE_COHERENT)
|
||||
return true;
|
||||
else if (attrs & DMA_ATTR_FORCE_NON_COHERENT)
|
||||
return false;
|
||||
else if (dev_is_dma_coherent(dev))
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot,
|
||||
unsigned long attrs)
|
||||
{
|
||||
if (!dev_is_dma_coherent(dev) || (attrs & DMA_ATTR_WRITE_COMBINE))
|
||||
if (!is_dma_coherent(dev, attrs) || (attrs & DMA_ATTR_WRITE_COMBINE))
|
||||
return pgprot_writecombine(prot);
|
||||
return prot;
|
||||
}
|
||||
|
|
@ -103,7 +119,7 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size,
|
|||
dma_addr_t *handle, gfp_t gfp,
|
||||
unsigned long attrs)
|
||||
{
|
||||
bool coherent = dev_is_dma_coherent(dev);
|
||||
bool coherent = is_dma_coherent(dev, attrs);
|
||||
int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
|
||||
size_t iosize = size;
|
||||
void *addr;
|
||||
|
|
@ -117,7 +133,8 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size,
|
|||
* Some drivers rely on this, and we probably don't want the
|
||||
* possibility of stale kernel data being read by devices anyway.
|
||||
*/
|
||||
gfp |= __GFP_ZERO;
|
||||
if (!(attrs & DMA_ATTR_SKIP_ZEROING))
|
||||
gfp |= __GFP_ZERO;
|
||||
|
||||
if (!gfpflags_allow_blocking(gfp)) {
|
||||
struct page *page;
|
||||
|
|
@ -232,31 +249,30 @@ static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
|
|||
{
|
||||
struct vm_struct *area;
|
||||
int ret;
|
||||
unsigned long pfn = 0;
|
||||
|
||||
vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs);
|
||||
|
||||
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
|
||||
return ret;
|
||||
|
||||
if (!is_vmalloc_addr(cpu_addr)) {
|
||||
unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));
|
||||
return __swiotlb_mmap_pfn(vma, pfn, size);
|
||||
}
|
||||
|
||||
if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
|
||||
/*
|
||||
* DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,
|
||||
* hence in the vmalloc space.
|
||||
*/
|
||||
unsigned long pfn = vmalloc_to_pfn(cpu_addr);
|
||||
return __swiotlb_mmap_pfn(vma, pfn, size);
|
||||
}
|
||||
|
||||
area = find_vm_area(cpu_addr);
|
||||
if (WARN_ON(!area || !area->pages))
|
||||
return -ENXIO;
|
||||
|
||||
return iommu_dma_mmap(area->pages, size, vma);
|
||||
if (area && area->pages)
|
||||
return iommu_dma_mmap(area->pages, size, vma);
|
||||
else if (!is_vmalloc_addr(cpu_addr))
|
||||
pfn = page_to_pfn(virt_to_page(cpu_addr));
|
||||
else if (is_vmalloc_addr(cpu_addr))
|
||||
/*
|
||||
* DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations are
|
||||
* always remapped, hence in the vmalloc space.
|
||||
*/
|
||||
pfn = vmalloc_to_pfn(cpu_addr);
|
||||
|
||||
if (pfn)
|
||||
return __swiotlb_mmap_pfn(vma, pfn, size);
|
||||
|
||||
return -ENXIO;
|
||||
}
|
||||
|
||||
static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
|
||||
|
|
@ -264,27 +280,24 @@ static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
|
|||
size_t size, unsigned long attrs)
|
||||
{
|
||||
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
||||
struct page *page = NULL;
|
||||
struct vm_struct *area = find_vm_area(cpu_addr);
|
||||
|
||||
if (!is_vmalloc_addr(cpu_addr)) {
|
||||
struct page *page = virt_to_page(cpu_addr);
|
||||
return __swiotlb_get_sgtable_page(sgt, page, size);
|
||||
}
|
||||
|
||||
if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
|
||||
if (area && area->pages)
|
||||
return sg_alloc_table_from_pages(sgt, area->pages, count, 0,
|
||||
size, GFP_KERNEL);
|
||||
else if (!is_vmalloc_addr(cpu_addr))
|
||||
page = virt_to_page(cpu_addr);
|
||||
else if (is_vmalloc_addr(cpu_addr))
|
||||
/*
|
||||
* DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,
|
||||
* hence in the vmalloc space.
|
||||
* DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations
|
||||
* are always remapped, hence in the vmalloc space.
|
||||
*/
|
||||
struct page *page = vmalloc_to_page(cpu_addr);
|
||||
page = vmalloc_to_page(cpu_addr);
|
||||
|
||||
if (page)
|
||||
return __swiotlb_get_sgtable_page(sgt, page, size);
|
||||
}
|
||||
|
||||
if (WARN_ON(!area || !area->pages))
|
||||
return -ENXIO;
|
||||
|
||||
return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size,
|
||||
GFP_KERNEL);
|
||||
return -ENXIO;
|
||||
}
|
||||
|
||||
static void __iommu_sync_single_for_cpu(struct device *dev,
|
||||
|
|
@ -292,11 +305,12 @@ static void __iommu_sync_single_for_cpu(struct device *dev,
|
|||
enum dma_data_direction dir)
|
||||
{
|
||||
phys_addr_t phys;
|
||||
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
|
||||
|
||||
if (dev_is_dma_coherent(dev))
|
||||
if (!domain || iommu_is_iova_coherent(domain, dev_addr))
|
||||
return;
|
||||
|
||||
phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr);
|
||||
phys = iommu_iova_to_phys(domain, dev_addr);
|
||||
arch_sync_dma_for_cpu(dev, phys, size, dir);
|
||||
}
|
||||
|
||||
|
|
@ -305,11 +319,12 @@ static void __iommu_sync_single_for_device(struct device *dev,
|
|||
enum dma_data_direction dir)
|
||||
{
|
||||
phys_addr_t phys;
|
||||
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
|
||||
|
||||
if (dev_is_dma_coherent(dev))
|
||||
if (!domain || iommu_is_iova_coherent(domain, dev_addr))
|
||||
return;
|
||||
|
||||
phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr);
|
||||
phys = iommu_iova_to_phys(domain, dev_addr);
|
||||
arch_sync_dma_for_device(dev, phys, size, dir);
|
||||
}
|
||||
|
||||
|
|
@ -318,7 +333,7 @@ static dma_addr_t __iommu_map_page(struct device *dev, struct page *page,
|
|||
enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
bool coherent = dev_is_dma_coherent(dev);
|
||||
bool coherent = is_dma_coherent(dev, attrs);
|
||||
int prot = dma_info_to_prot(dir, coherent, attrs);
|
||||
dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);
|
||||
|
||||
|
|
@ -344,9 +359,11 @@ static void __iommu_sync_sg_for_cpu(struct device *dev,
|
|||
enum dma_data_direction dir)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
dma_addr_t iova = sg_dma_address(sgl);
|
||||
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
|
||||
int i;
|
||||
|
||||
if (dev_is_dma_coherent(dev))
|
||||
if (!domain || iommu_is_iova_coherent(domain, iova))
|
||||
return;
|
||||
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
|
|
@ -358,9 +375,11 @@ static void __iommu_sync_sg_for_device(struct device *dev,
|
|||
enum dma_data_direction dir)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
dma_addr_t iova = sg_dma_address(sgl);
|
||||
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
|
||||
int i;
|
||||
|
||||
if (dev_is_dma_coherent(dev))
|
||||
if (!domain || iommu_is_iova_coherent(domain, iova))
|
||||
return;
|
||||
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
|
|
@ -371,13 +390,18 @@ static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
|||
int nelems, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
bool coherent = dev_is_dma_coherent(dev);
|
||||
bool coherent = is_dma_coherent(dev, attrs);
|
||||
int ret;
|
||||
|
||||
ret = iommu_dma_map_sg(dev, sgl, nelems,
|
||||
dma_info_to_prot(dir, coherent, attrs));
|
||||
if (!ret)
|
||||
return ret;
|
||||
|
||||
if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
|
||||
__iommu_sync_sg_for_device(dev, sgl, nelems, dir);
|
||||
|
||||
return iommu_dma_map_sg(dev, sgl, nelems,
|
||||
dma_info_to_prot(dir, coherent, attrs));
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void __iommu_unmap_sg_attrs(struct device *dev,
|
||||
|
|
@ -414,10 +438,30 @@ static int __init __iommu_dma_init(void)
|
|||
}
|
||||
arch_initcall(__iommu_dma_init);
|
||||
|
||||
static int __iommu_init_dma_resources(struct device *dev,
|
||||
struct iommu_domain *domain, u64 dma_base,
|
||||
u64 size)
|
||||
{
|
||||
int is_fast, ret = 0;
|
||||
|
||||
iommu_domain_get_attr(domain, DOMAIN_ATTR_FAST, &is_fast);
|
||||
|
||||
if (is_fast) {
|
||||
dev->dma_ops = fast_smmu_get_dma_ops();
|
||||
} else {
|
||||
ret = iommu_dma_init_domain(domain, dma_base, size, dev);
|
||||
if (!ret)
|
||||
dev->dma_ops = &iommu_dma_ops;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
|
||||
const struct iommu_ops *ops)
|
||||
{
|
||||
struct iommu_domain *domain;
|
||||
int s1_bypass;
|
||||
|
||||
if (!ops)
|
||||
return;
|
||||
|
|
@ -431,13 +475,20 @@ static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
|
|||
if (!domain)
|
||||
goto out_err;
|
||||
|
||||
if (domain->type == IOMMU_DOMAIN_DMA) {
|
||||
if (iommu_dma_init_domain(domain, dma_base, size, dev))
|
||||
goto out_err;
|
||||
iommu_domain_get_attr(domain, DOMAIN_ATTR_S1_BYPASS, &s1_bypass);
|
||||
if (s1_bypass)
|
||||
return;
|
||||
|
||||
dev->dma_ops = &iommu_dma_ops;
|
||||
/* Allow iommu-debug to call arch_setup_dma_ops to reconfigure itself */
|
||||
if (domain->type != IOMMU_DOMAIN_DMA &&
|
||||
!of_device_is_compatible(dev->of_node, "iommu-debug-test")) {
|
||||
dev_err(dev, "Invalid iommu domain type!\n");
|
||||
return;
|
||||
}
|
||||
|
||||
if (__iommu_init_dma_resources(dev, domain, dma_base, size))
|
||||
goto out_err;
|
||||
|
||||
return;
|
||||
|
||||
out_err:
|
||||
|
|
|
|||
|
|
@ -63,6 +63,58 @@ config IOMMU_IO_PGTABLE_ARMV7S_SELFTEST
|
|||
|
||||
If unsure, say N here.
|
||||
|
||||
config IOMMU_IO_PGTABLE_FAST
|
||||
bool "Fast ARMv7/v8 Long Descriptor Format"
|
||||
depends on (ARM || ARM64) && IOMMU_DMA
|
||||
help
|
||||
Enable support for a subset of the ARM long descriptor pagetable
|
||||
format. This allocator achieves fast performance by
|
||||
pre-allocating and pre-populating page table memory up front.
|
||||
only supports a 32 bit virtual address space.
|
||||
|
||||
This implementation is mainly optimized for use cases where the
|
||||
buffers are small (<= 64K) since it only supports 4K page sizes.
|
||||
|
||||
config IOMMU_IO_PGTABLE_FAST_SELFTEST
|
||||
bool "Fast IO pgtable selftests"
|
||||
depends on IOMMU_IO_PGTABLE_FAST
|
||||
help
|
||||
Enable self-tests for "fast" page table allocator.
|
||||
This performs a series of page-table consistency checks
|
||||
during boot.
|
||||
|
||||
If unsure, say N here.
|
||||
|
||||
config IOMMU_IO_PGTABLE_FAST_PROVE_TLB
|
||||
bool "Prove correctness of TLB maintenance in the Fast DMA mapper"
|
||||
depends on IOMMU_IO_PGTABLE_FAST
|
||||
help
|
||||
Enables some debug features that help prove correctness of TLB
|
||||
maintenance routines in the Fast DMA mapper. This option will
|
||||
slow things down considerably, so should only be used in a debug
|
||||
configuration. This relies on the ability to set bits in an
|
||||
invalid page table entry, which is disallowed on some hardware
|
||||
due to errata. If you're running on such a platform then this
|
||||
option can only be used with unit tests. It will break real use
|
||||
cases.
|
||||
|
||||
If unsure, say N here.
|
||||
|
||||
config QCOM_IOMMU_IO_PGTABLE_QUIRKS
|
||||
bool "IO Pagetable quirks for performance"
|
||||
depends on ARM || ARM64
|
||||
depends on IOMMU_IO_PGTABLE_FAST || IOMMU_IO_PGTABLE_LPAE
|
||||
depends on ARM_SMMU
|
||||
help
|
||||
Enables some quirks that are used when creating the IOMMU's
|
||||
page tables for a particular domain for faster translations.
|
||||
The quirks that are supported deal with allowing for page
|
||||
tables to be IO-coherent, allowing for page tables to be
|
||||
saved in the system cache, and disabling the write-allocate
|
||||
hint when saving page tables in the system cache.
|
||||
|
||||
If unsure, say Y here.
|
||||
|
||||
endmenu
|
||||
|
||||
config IOMMU_DEBUGFS
|
||||
|
|
@ -399,6 +451,38 @@ config ARM_SMMU_V3
|
|||
Say Y here if your system includes an IOMMU device implementing
|
||||
the ARM SMMUv3 architecture.
|
||||
|
||||
config ARM_SMMU_SELFTEST
|
||||
bool "ARM SMMU self test support"
|
||||
depends on ARM_SMMU
|
||||
help
|
||||
Enables self tests for arm smmu. Tests basic hardware
|
||||
configurations like interrupts. Note that enabling this
|
||||
option can marginally increase the boot time.
|
||||
|
||||
If unsure, say N here.
|
||||
|
||||
config IOMMU_TLBSYNC_DEBUG
|
||||
bool "TLB sync timeout debug"
|
||||
depends on ARM_SMMU
|
||||
help
|
||||
Enables to collect the SMMU system state information right
|
||||
after the first TLB sync timeout failure by calling BUG().
|
||||
Note to use this only on debug builds.
|
||||
|
||||
If unsure, say N here.
|
||||
|
||||
config QCOM_LAZY_MAPPING
|
||||
tristate "Reference counted iommu-mapping support"
|
||||
depends on ION
|
||||
depends on IOMMU_API
|
||||
help
|
||||
ION buffers may be shared between several software clients.
|
||||
Reference counting the mapping may simplify coordination between
|
||||
these clients, and decrease latency by preventing multiple
|
||||
map/unmaps of the same region.
|
||||
|
||||
If unsure, say N here.
|
||||
|
||||
config S390_IOMMU
|
||||
def_bool y if S390 && PCI
|
||||
depends on S390 && PCI
|
||||
|
|
@ -454,6 +538,38 @@ config MTK_IOMMU_V1
|
|||
|
||||
if unsure, say N here.
|
||||
|
||||
menuconfig IOMMU_DEBUG
|
||||
bool "IOMMU Profiling and Debugging"
|
||||
help
|
||||
This option is used to enable profiling and debugging in
|
||||
the IOMMU framework code. IOMMU profiling and debugging
|
||||
can be done through the debugfs nodes which this option
|
||||
makes available.
|
||||
|
||||
if IOMMU_DEBUG
|
||||
|
||||
config IOMMU_DEBUG_TRACKING
|
||||
bool "Track key IOMMU events"
|
||||
select IOMMU_API
|
||||
help
|
||||
Enables additional debug tracking in the IOMMU framework code.
|
||||
Tracking information and tests can be accessed through various
|
||||
debugfs files.
|
||||
|
||||
Say Y here if you need to debug IOMMU issues and are okay with
|
||||
the performance penalty of the tracking.
|
||||
|
||||
config IOMMU_TESTS
|
||||
bool "Interactive IOMMU performance/functional tests"
|
||||
select IOMMU_API
|
||||
help
|
||||
Enables a suite of IOMMU unit tests. The tests are runnable
|
||||
through debugfs. Unlike the IOMMU_DEBUG_TRACKING option, the
|
||||
impact of enabling this option to overal system performance
|
||||
should be minimal.
|
||||
|
||||
endif # IOMMU_DEBUG
|
||||
|
||||
config QCOM_IOMMU
|
||||
# Note: iommu drivers cannot (yet?) be built as modules
|
||||
bool "Qualcomm IOMMU Support"
|
||||
|
|
|
|||
|
|
@ -4,11 +4,14 @@ obj-$(CONFIG_IOMMU_API) += iommu-traces.o
|
|||
obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
|
||||
obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o
|
||||
obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o
|
||||
obj-$(CONFIG_QCOM_LAZY_MAPPING) += msm_dma_iommu_mapping.o
|
||||
obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o
|
||||
obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o
|
||||
obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o
|
||||
obj-$(CONFIG_IOMMU_IOVA) += iova.o
|
||||
obj-$(CONFIG_IOMMU_IO_PGTABLE_FAST) += io-pgtable-fast.o dma-mapping-fast.o
|
||||
obj-$(CONFIG_OF_IOMMU) += of_iommu.o
|
||||
obj-$(CONFIG_IOMMU_DEBUG) += iommu-debug.o
|
||||
obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o
|
||||
obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
|
||||
obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@
|
|||
#define sCR0_VMID16EN (1 << 31)
|
||||
#define sCR0_BSU_SHIFT 14
|
||||
#define sCR0_BSU_MASK 0x3
|
||||
#define sCR0_SHCFG_SHIFT 22
|
||||
#define sCR0_SHCFG_MASK 0x3
|
||||
#define sCR0_SHCFG_NSH 3
|
||||
|
||||
/* Auxiliary Configuration register */
|
||||
#define ARM_SMMU_GR0_sACR 0x10
|
||||
|
|
@ -93,6 +96,8 @@
|
|||
#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
|
||||
#define SMR_VALID (1 << 31)
|
||||
#define SMR_MASK_SHIFT 16
|
||||
#define SMR_MASK_MASK 0x7FFF
|
||||
#define SID_MASK 0x7FFF
|
||||
#define SMR_ID_SHIFT 0
|
||||
|
||||
#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
|
||||
|
|
@ -101,6 +106,9 @@
|
|||
#define S2CR_EXIDVALID (1 << 10)
|
||||
#define S2CR_TYPE_SHIFT 16
|
||||
#define S2CR_TYPE_MASK 0x3
|
||||
#define S2CR_SHCFG_SHIFT 8
|
||||
#define S2CR_SHCFG_MASK 0x3
|
||||
#define S2CR_SHCFG_NSH 0x3
|
||||
enum arm_smmu_s2cr_type {
|
||||
S2CR_TYPE_TRANS,
|
||||
S2CR_TYPE_BYPASS,
|
||||
|
|
@ -136,6 +144,7 @@ enum arm_smmu_s2cr_privcfg {
|
|||
#define CBAR_IRPTNDX_MASK 0xff
|
||||
|
||||
#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
|
||||
#define CBFRSYNRA_SID_MASK (0xffff)
|
||||
|
||||
#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
|
||||
#define CBA2R_RW64_32BIT (0 << 0)
|
||||
|
|
@ -155,20 +164,38 @@ enum arm_smmu_s2cr_privcfg {
|
|||
#define ARM_SMMU_CB_S1_MAIR1 0x3c
|
||||
#define ARM_SMMU_CB_PAR 0x50
|
||||
#define ARM_SMMU_CB_FSR 0x58
|
||||
#define ARM_SMMU_CB_FSRRESTORE 0x5c
|
||||
#define ARM_SMMU_CB_FAR 0x60
|
||||
#define ARM_SMMU_CB_FSYNR0 0x68
|
||||
#define ARM_SMMU_CB_FSYNR1 0x6c
|
||||
#define ARM_SMMU_CB_S1_TLBIVA 0x600
|
||||
#define ARM_SMMU_CB_S1_TLBIASID 0x610
|
||||
#define ARM_SMMU_CB_S1_TLBIALL 0x618
|
||||
#define ARM_SMMU_CB_S1_TLBIVAL 0x620
|
||||
#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
|
||||
#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
|
||||
#define ARM_SMMU_CB_TLBSYNC 0x7f0
|
||||
#define ARM_SMMU_CB_TLBSTATUS 0x7f4
|
||||
#define TLBSTATUS_SACTIVE (1 << 0)
|
||||
#define ARM_SMMU_CB_ATS1PR 0x800
|
||||
#define ARM_SMMU_CB_ATSR 0x8f0
|
||||
#define ARM_SMMU_STATS_SYNC_INV_TBU_ACK 0x25dc
|
||||
#define ARM_SMMU_TBU_PWR_STATUS 0x2204
|
||||
#define ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR 0x2670
|
||||
|
||||
#define SCTLR_MEM_ATTR_SHIFT 16
|
||||
#define SCTLR_SHCFG_SHIFT 22
|
||||
#define SCTLR_RACFG_SHIFT 24
|
||||
#define SCTLR_WACFG_SHIFT 26
|
||||
#define SCTLR_SHCFG_MASK 0x3
|
||||
#define SCTLR_SHCFG_NSH 0x3
|
||||
#define SCTLR_RACFG_RA 0x2
|
||||
#define SCTLR_WACFG_WA 0x2
|
||||
#define SCTLR_MEM_ATTR_OISH_WB_CACHE 0xf
|
||||
#define SCTLR_MTCFG (1 << 20)
|
||||
#define SCTLR_S1_ASIDPNE (1 << 12)
|
||||
#define SCTLR_CFCFG (1 << 7)
|
||||
#define SCTLR_HUPCF (1 << 8)
|
||||
#define SCTLR_CFIE (1 << 6)
|
||||
#define SCTLR_CFRE (1 << 5)
|
||||
#define SCTLR_E (1 << 4)
|
||||
|
|
|
|||
84
drivers/iommu/arm-smmu-trace.h
Normal file
84
drivers/iommu/arm-smmu-trace.h
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#undef TRACE_SYSTEM
|
||||
#define TRACE_SYSTEM arm_smmu
|
||||
|
||||
#if !defined(_TRACE_ARM_SMMU_H) || defined(TRACE_HEADER_MULTI_READ)
|
||||
#define _TRACE_ARM_SMMU_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/tracepoint.h>
|
||||
|
||||
struct device;
|
||||
|
||||
DECLARE_EVENT_CLASS(iommu_tlbi,
|
||||
|
||||
TP_PROTO(struct device *dev, u64 time),
|
||||
|
||||
TP_ARGS(dev, time),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(device, dev_name(dev))
|
||||
__field(u64, time)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(device, dev_name(dev));
|
||||
__entry->time = time;
|
||||
),
|
||||
|
||||
TP_printk("IOMMU:%s %lld us",
|
||||
__get_str(device), __entry->time
|
||||
)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_tlbi, tlbi_start,
|
||||
|
||||
TP_PROTO(struct device *dev, u64 time),
|
||||
|
||||
TP_ARGS(dev, time)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_tlbi, tlbi_end,
|
||||
|
||||
TP_PROTO(struct device *dev, u64 time),
|
||||
|
||||
TP_ARGS(dev, time)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(iommu_tlbi, tlbsync_timeout,
|
||||
|
||||
TP_PROTO(struct device *dev, u64 time),
|
||||
|
||||
TP_ARGS(dev, time)
|
||||
);
|
||||
|
||||
TRACE_EVENT(smmu_init,
|
||||
|
||||
TP_PROTO(u64 time),
|
||||
|
||||
TP_ARGS(time),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(u64, time)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->time = time;
|
||||
),
|
||||
|
||||
TP_printk("ARM SMMU init latency: %lld us", __entry->time)
|
||||
);
|
||||
#endif /* _TRACE_ARM_SMMU_H */
|
||||
|
||||
#undef TRACE_INCLUDE_PATH
|
||||
#define TRACE_INCLUDE_PATH .
|
||||
|
||||
#undef TRACE_INCLUDE_FILE
|
||||
#define TRACE_INCLUDE_FILE arm-smmu-trace
|
||||
|
||||
/* This part must be outside protection */
|
||||
#include <trace/define_trace.h>
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -372,6 +372,15 @@ int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
|
|||
if (attrs & DMA_ATTR_PRIVILEGED)
|
||||
prot |= IOMMU_PRIV;
|
||||
|
||||
if (!(attrs & DMA_ATTR_EXEC_MAPPING))
|
||||
prot |= IOMMU_NOEXEC;
|
||||
|
||||
if (attrs & DMA_ATTR_IOMMU_USE_UPSTREAM_HINT)
|
||||
prot |= IOMMU_USE_UPSTREAM_HINT;
|
||||
|
||||
if (attrs & DMA_ATTR_IOMMU_USE_LLC_NWA)
|
||||
prot |= IOMMU_USE_LLC_NWA;
|
||||
|
||||
switch (dir) {
|
||||
case DMA_BIDIRECTIONAL:
|
||||
return prot | IOMMU_READ | IOMMU_WRITE;
|
||||
|
|
@ -468,7 +477,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev,
|
|||
unsigned int count, unsigned long order_mask, gfp_t gfp)
|
||||
{
|
||||
struct page **pages;
|
||||
unsigned int i = 0, nid = dev_to_node(dev);
|
||||
unsigned int i = 0;
|
||||
|
||||
order_mask &= (2U << MAX_ORDER) - 1;
|
||||
if (!order_mask)
|
||||
|
|
@ -493,12 +502,11 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev,
|
|||
for (order_mask &= (2U << __fls(count)) - 1;
|
||||
order_mask; order_mask &= ~order_size) {
|
||||
unsigned int order = __fls(order_mask);
|
||||
gfp_t alloc_flags = gfp;
|
||||
|
||||
order_size = 1U << order;
|
||||
if (order_mask > order_size)
|
||||
alloc_flags |= __GFP_NORETRY;
|
||||
page = alloc_pages_node(nid, alloc_flags, order);
|
||||
page = alloc_pages(order ?
|
||||
(gfp | __GFP_NORETRY) &
|
||||
~__GFP_RECLAIM : gfp, order);
|
||||
if (!page)
|
||||
continue;
|
||||
if (!order)
|
||||
|
|
@ -535,7 +543,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev,
|
|||
void iommu_dma_free(struct device *dev, struct page **pages, size_t size,
|
||||
dma_addr_t *handle)
|
||||
{
|
||||
__iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size);
|
||||
__iommu_dma_unmap(iommu_get_domain_for_dev(dev), *handle, size);
|
||||
__iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
|
||||
*handle = DMA_MAPPING_ERROR;
|
||||
}
|
||||
|
|
@ -562,7 +570,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,
|
|||
unsigned long attrs, int prot, dma_addr_t *handle,
|
||||
void (*flush_page)(struct device *, const void *, phys_addr_t))
|
||||
{
|
||||
struct iommu_domain *domain = iommu_get_dma_domain(dev);
|
||||
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
|
||||
struct iommu_dma_cookie *cookie = domain->iova_cookie;
|
||||
struct iova_domain *iovad = &cookie->iovad;
|
||||
struct page **pages;
|
||||
|
|
@ -667,13 +675,13 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
|
|||
unsigned long offset, size_t size, int prot)
|
||||
{
|
||||
return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot,
|
||||
iommu_get_dma_domain(dev));
|
||||
iommu_get_domain_for_dev(dev));
|
||||
}
|
||||
|
||||
void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,
|
||||
enum dma_data_direction dir, unsigned long attrs)
|
||||
{
|
||||
__iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size);
|
||||
__iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle, size);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -683,7 +691,7 @@ void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,
|
|||
* avoid individually crossing any boundaries, so we merely need to check a
|
||||
* segment's start address to avoid concatenating across one.
|
||||
*/
|
||||
static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
|
||||
int iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
|
||||
dma_addr_t dma_addr)
|
||||
{
|
||||
struct scatterlist *s, *cur = sg;
|
||||
|
|
@ -736,7 +744,7 @@ static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
|
|||
* If mapping failed, then just restore the original list,
|
||||
* but making sure the DMA fields are invalidated.
|
||||
*/
|
||||
static void __invalidate_sg(struct scatterlist *sg, int nents)
|
||||
void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents)
|
||||
{
|
||||
struct scatterlist *s;
|
||||
int i;
|
||||
|
|
@ -758,14 +766,10 @@ static void __invalidate_sg(struct scatterlist *sg, int nents)
|
|||
* impedance-matching, to be able to hand off a suitably-aligned list,
|
||||
* but still preserve the original offsets and sizes for the caller.
|
||||
*/
|
||||
int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, int prot)
|
||||
size_t iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad,
|
||||
struct scatterlist *sg, int nents)
|
||||
{
|
||||
struct iommu_domain *domain = iommu_get_dma_domain(dev);
|
||||
struct iommu_dma_cookie *cookie = domain->iova_cookie;
|
||||
struct iova_domain *iovad = &cookie->iovad;
|
||||
struct scatterlist *s, *prev = NULL;
|
||||
dma_addr_t iova;
|
||||
size_t iova_len = 0;
|
||||
unsigned long mask = dma_get_seg_boundary(dev);
|
||||
int i;
|
||||
|
|
@ -809,6 +813,26 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
|||
prev = s;
|
||||
}
|
||||
|
||||
return iova_len;
|
||||
}
|
||||
|
||||
int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, int prot)
|
||||
{
|
||||
struct iommu_domain *domain;
|
||||
struct iommu_dma_cookie *cookie;
|
||||
struct iova_domain *iovad;
|
||||
dma_addr_t iova;
|
||||
size_t iova_len;
|
||||
|
||||
domain = iommu_get_domain_for_dev(dev);
|
||||
if (!domain)
|
||||
return 0;
|
||||
cookie = domain->iova_cookie;
|
||||
iovad = &cookie->iovad;
|
||||
|
||||
iova_len = iommu_dma_prepare_map_sg(dev, iovad, sg, nents);
|
||||
|
||||
iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
|
||||
if (!iova)
|
||||
goto out_restore_sg;
|
||||
|
|
@ -820,12 +844,12 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
|||
if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len)
|
||||
goto out_free_iova;
|
||||
|
||||
return __finalise_sg(dev, sg, nents, iova);
|
||||
return iommu_dma_finalise_sg(dev, sg, nents, iova);
|
||||
|
||||
out_free_iova:
|
||||
iommu_dma_free_iova(cookie, iova, iova_len);
|
||||
out_restore_sg:
|
||||
__invalidate_sg(sg, nents);
|
||||
iommu_dma_invalidate_sg(sg, nents);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -846,7 +870,7 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
|
|||
sg = tmp;
|
||||
}
|
||||
end = sg_dma_address(sg) + sg_dma_len(sg);
|
||||
__iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start);
|
||||
__iommu_dma_unmap(iommu_get_domain_for_dev(dev), start, end - start);
|
||||
}
|
||||
|
||||
dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
|
||||
|
|
@ -854,13 +878,13 @@ dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
|
|||
{
|
||||
return __iommu_dma_map(dev, phys, size,
|
||||
dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,
|
||||
iommu_get_dma_domain(dev));
|
||||
iommu_get_domain_for_dev(dev));
|
||||
}
|
||||
|
||||
void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
|
||||
size_t size, enum dma_data_direction dir, unsigned long attrs)
|
||||
{
|
||||
__iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size);
|
||||
__iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle, size);
|
||||
}
|
||||
|
||||
static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
|
||||
|
|
|
|||
969
drivers/iommu/dma-mapping-fast.c
Normal file
969
drivers/iommu/dma-mapping-fast.c
Normal file
|
|
@ -0,0 +1,969 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/dma-contiguous.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/dma-mapping-fast.h>
|
||||
#include <linux/io-pgtable-fast.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/dma-iommu.h>
|
||||
#include <linux/iova.h>
|
||||
#include <trace/events/iommu.h>
|
||||
#include <linux/io-pgtable.h>
|
||||
|
||||
/* some redundant definitions... :( TODO: move to io-pgtable-fast.h */
|
||||
#define FAST_PAGE_SHIFT 12
|
||||
#define FAST_PAGE_SIZE (1UL << FAST_PAGE_SHIFT)
|
||||
#define FAST_PAGE_MASK (~(PAGE_SIZE - 1))
|
||||
|
||||
static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot,
|
||||
bool coherent)
|
||||
{
|
||||
if (!coherent || (attrs & DMA_ATTR_WRITE_COMBINE))
|
||||
return pgprot_writecombine(prot);
|
||||
return prot;
|
||||
}
|
||||
|
||||
static bool is_dma_coherent(struct device *dev, unsigned long attrs)
|
||||
{
|
||||
bool is_coherent;
|
||||
|
||||
if (attrs & DMA_ATTR_FORCE_COHERENT)
|
||||
is_coherent = true;
|
||||
else if (attrs & DMA_ATTR_FORCE_NON_COHERENT)
|
||||
is_coherent = false;
|
||||
else if (is_device_dma_coherent(dev))
|
||||
is_coherent = true;
|
||||
else
|
||||
is_coherent = false;
|
||||
|
||||
return is_coherent;
|
||||
}
|
||||
|
||||
static struct dma_fast_smmu_mapping *dev_get_mapping(struct device *dev)
|
||||
{
|
||||
struct iommu_domain *domain;
|
||||
|
||||
domain = iommu_get_domain_for_dev(dev);
|
||||
if (!domain)
|
||||
return ERR_PTR(-EINVAL);
|
||||
return domain->iova_cookie;
|
||||
}
|
||||
|
||||
/*
|
||||
* Checks if the allocated range (ending at @end) covered the upcoming
|
||||
* stale bit. We don't need to know exactly where the range starts since
|
||||
* we already know where the candidate search range started. If, starting
|
||||
* from the beginning of the candidate search range, we had to step over
|
||||
* (or landed directly on top of) the upcoming stale bit, then we return
|
||||
* true.
|
||||
*
|
||||
* Due to wrapping, there are two scenarios we'll need to check: (1) if the
|
||||
* range [search_start, upcoming_stale] spans 0 (i.e. search_start >
|
||||
* upcoming_stale), and, (2) if the range: [search_start, upcoming_stale]
|
||||
* does *not* span 0 (i.e. search_start <= upcoming_stale). And for each
|
||||
* of those two scenarios we need to handle three cases: (1) the bit was
|
||||
* found before wrapping or
|
||||
*/
|
||||
static bool __bit_covered_stale(unsigned long upcoming_stale,
|
||||
unsigned long search_start,
|
||||
unsigned long end)
|
||||
{
|
||||
if (search_start > upcoming_stale) {
|
||||
if (end >= search_start) {
|
||||
/*
|
||||
* We started searching above upcoming_stale and we
|
||||
* didn't wrap, so we couldn't have crossed
|
||||
* upcoming_stale.
|
||||
*/
|
||||
return false;
|
||||
}
|
||||
/*
|
||||
* We wrapped. Did we cross (or land on top of)
|
||||
* upcoming_stale?
|
||||
*/
|
||||
return end >= upcoming_stale;
|
||||
}
|
||||
|
||||
if (search_start <= upcoming_stale) {
|
||||
if (end >= search_start) {
|
||||
/*
|
||||
* We didn't wrap. Did we cross (or land on top
|
||||
* of) upcoming_stale?
|
||||
*/
|
||||
return end >= upcoming_stale;
|
||||
}
|
||||
/*
|
||||
* We wrapped. So we must have crossed upcoming_stale
|
||||
* (since we started searching below it).
|
||||
*/
|
||||
return true;
|
||||
}
|
||||
|
||||
/* we should have covered all logical combinations... */
|
||||
WARN_ON(1);
|
||||
return true;
|
||||
}
|
||||
|
||||
static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping,
|
||||
unsigned long attrs,
|
||||
size_t size)
|
||||
{
|
||||
unsigned long bit, prev_search_start, nbits = size >> FAST_PAGE_SHIFT;
|
||||
unsigned long align = (1 << get_order(size)) - 1;
|
||||
|
||||
bit = bitmap_find_next_zero_area(
|
||||
mapping->bitmap, mapping->num_4k_pages, mapping->next_start,
|
||||
nbits, align);
|
||||
if (unlikely(bit > mapping->num_4k_pages)) {
|
||||
/* try wrapping */
|
||||
mapping->next_start = 0; /* TODO: SHOULD I REALLY DO THIS?!? */
|
||||
bit = bitmap_find_next_zero_area(
|
||||
mapping->bitmap, mapping->num_4k_pages, 0, nbits,
|
||||
align);
|
||||
if (unlikely(bit > mapping->num_4k_pages))
|
||||
return DMA_ERROR_CODE;
|
||||
}
|
||||
|
||||
bitmap_set(mapping->bitmap, bit, nbits);
|
||||
prev_search_start = mapping->next_start;
|
||||
mapping->next_start = bit + nbits;
|
||||
if (unlikely(mapping->next_start >= mapping->num_4k_pages))
|
||||
mapping->next_start = 0;
|
||||
|
||||
/*
|
||||
* If we just re-allocated a VA whose TLB hasn't been invalidated
|
||||
* since it was last used and unmapped, we need to invalidate it
|
||||
* here. We actually invalidate the entire TLB so that we don't
|
||||
* have to invalidate the TLB again until we wrap back around.
|
||||
*/
|
||||
if (mapping->have_stale_tlbs &&
|
||||
__bit_covered_stale(mapping->upcoming_stale_bit,
|
||||
prev_search_start,
|
||||
bit + nbits - 1)) {
|
||||
bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC);
|
||||
|
||||
iommu_tlbiall(mapping->domain);
|
||||
mapping->have_stale_tlbs = false;
|
||||
av8l_fast_clear_stale_ptes(mapping->pgtbl_ops, skip_sync);
|
||||
}
|
||||
|
||||
return (bit << FAST_PAGE_SHIFT) + mapping->base;
|
||||
}
|
||||
|
||||
/*
|
||||
* Checks whether the candidate bit will be allocated sooner than the
|
||||
* current upcoming stale bit. We can say candidate will be upcoming
|
||||
* sooner than the current upcoming stale bit if it lies between the
|
||||
* starting bit of the next search range and the upcoming stale bit
|
||||
* (allowing for wrap-around).
|
||||
*
|
||||
* Stated differently, we're checking the relative ordering of three
|
||||
* unsigned numbers. So we need to check all 6 (i.e. 3!) permutations,
|
||||
* namely:
|
||||
*
|
||||
* 0 |---A---B---C---| TOP (Case 1)
|
||||
* 0 |---A---C---B---| TOP (Case 2)
|
||||
* 0 |---B---A---C---| TOP (Case 3)
|
||||
* 0 |---B---C---A---| TOP (Case 4)
|
||||
* 0 |---C---A---B---| TOP (Case 5)
|
||||
* 0 |---C---B---A---| TOP (Case 6)
|
||||
*
|
||||
* Note that since we're allowing numbers to wrap, the following three
|
||||
* scenarios are all equivalent for Case 1:
|
||||
*
|
||||
* 0 |---A---B---C---| TOP
|
||||
* 0 |---C---A---B---| TOP (C has wrapped. This is Case 5.)
|
||||
* 0 |---B---C---A---| TOP (C and B have wrapped. This is Case 4.)
|
||||
*
|
||||
* In any of these cases, if we start searching from A, we will find B
|
||||
* before we find C.
|
||||
*
|
||||
* We can also find two equivalent cases for Case 2:
|
||||
*
|
||||
* 0 |---A---C---B---| TOP
|
||||
* 0 |---B---A---C---| TOP (B has wrapped. This is Case 3.)
|
||||
* 0 |---C---B---A---| TOP (B and C have wrapped. This is Case 6.)
|
||||
*
|
||||
* In any of these cases, if we start searching from A, we will find C
|
||||
* before we find B.
|
||||
*/
|
||||
static bool __bit_is_sooner(unsigned long candidate,
|
||||
struct dma_fast_smmu_mapping *mapping)
|
||||
{
|
||||
unsigned long A = mapping->next_start;
|
||||
unsigned long B = candidate;
|
||||
unsigned long C = mapping->upcoming_stale_bit;
|
||||
|
||||
if ((A < B && B < C) || /* Case 1 */
|
||||
(C < A && A < B) || /* Case 5 */
|
||||
(B < C && C < A)) /* Case 4 */
|
||||
return true;
|
||||
|
||||
if ((A < C && C < B) || /* Case 2 */
|
||||
(B < A && A < C) || /* Case 3 */
|
||||
(C < B && B < A)) /* Case 6 */
|
||||
return false;
|
||||
|
||||
/*
|
||||
* For simplicity, we've been ignoring the possibility of any of
|
||||
* our three numbers being equal. Handle those cases here (they
|
||||
* shouldn't happen very often, (I think?)).
|
||||
*/
|
||||
|
||||
/*
|
||||
* If candidate is the next bit to be searched then it's definitely
|
||||
* sooner.
|
||||
*/
|
||||
if (A == B)
|
||||
return true;
|
||||
|
||||
/*
|
||||
* If candidate is the next upcoming stale bit we'll return false
|
||||
* to avoid doing `upcoming = candidate' in the caller (which would
|
||||
* be useless since they're already equal)
|
||||
*/
|
||||
if (B == C)
|
||||
return false;
|
||||
|
||||
/*
|
||||
* If next start is the upcoming stale bit then candidate can't
|
||||
* possibly be sooner. The "soonest" bit is already selected.
|
||||
*/
|
||||
if (A == C)
|
||||
return false;
|
||||
|
||||
/* We should have covered all logical combinations. */
|
||||
WARN(1, "Well, that's awkward. A=%ld, B=%ld, C=%ld\n", A, B, C);
|
||||
return true;
|
||||
}
|
||||
|
||||
static void __fast_smmu_free_iova(struct dma_fast_smmu_mapping *mapping,
|
||||
dma_addr_t iova, size_t size)
|
||||
{
|
||||
unsigned long start_bit = (iova - mapping->base) >> FAST_PAGE_SHIFT;
|
||||
unsigned long nbits = size >> FAST_PAGE_SHIFT;
|
||||
|
||||
/*
|
||||
* We don't invalidate TLBs on unmap. We invalidate TLBs on map
|
||||
* when we're about to re-allocate a VA that was previously
|
||||
* unmapped but hasn't yet been invalidated. So we need to keep
|
||||
* track of which bit is the closest to being re-allocated here.
|
||||
*/
|
||||
if (__bit_is_sooner(start_bit, mapping))
|
||||
mapping->upcoming_stale_bit = start_bit;
|
||||
|
||||
bitmap_clear(mapping->bitmap, start_bit, nbits);
|
||||
mapping->have_stale_tlbs = true;
|
||||
}
|
||||
|
||||
|
||||
static void __fast_dma_page_cpu_to_dev(struct page *page, unsigned long off,
|
||||
size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
__dma_map_area(page_address(page) + off, size, dir);
|
||||
}
|
||||
|
||||
static void __fast_dma_page_dev_to_cpu(struct page *page, unsigned long off,
|
||||
size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
__dma_unmap_area(page_address(page) + off, size, dir);
|
||||
|
||||
/* TODO: WHAT IS THIS? */
|
||||
/*
|
||||
* Mark the D-cache clean for this page to avoid extra flushing.
|
||||
*/
|
||||
if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
|
||||
set_bit(PG_dcache_clean, &page->flags);
|
||||
}
|
||||
|
||||
static dma_addr_t fast_smmu_map_page(struct device *dev, struct page *page,
|
||||
unsigned long offset, size_t size,
|
||||
enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
dma_addr_t iova;
|
||||
unsigned long flags;
|
||||
phys_addr_t phys_plus_off = page_to_phys(page) + offset;
|
||||
phys_addr_t phys_to_map = round_down(phys_plus_off, FAST_PAGE_SIZE);
|
||||
unsigned long offset_from_phys_to_map = phys_plus_off & ~FAST_PAGE_MASK;
|
||||
size_t len = ALIGN(size + offset_from_phys_to_map, FAST_PAGE_SIZE);
|
||||
bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC);
|
||||
bool is_coherent = is_dma_coherent(dev, attrs);
|
||||
int prot = dma_info_to_prot(dir, is_coherent, attrs);
|
||||
|
||||
if (!skip_sync && !is_coherent)
|
||||
__fast_dma_page_cpu_to_dev(phys_to_page(phys_to_map),
|
||||
offset_from_phys_to_map, size, dir);
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
|
||||
iova = __fast_smmu_alloc_iova(mapping, attrs, len);
|
||||
|
||||
if (unlikely(iova == DMA_ERROR_CODE))
|
||||
goto fail;
|
||||
|
||||
if (unlikely(av8l_fast_map_public(mapping->pgtbl_ops, iova,
|
||||
phys_to_map, len, prot)))
|
||||
goto fail_free_iova;
|
||||
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
|
||||
trace_map(to_msm_iommu_domain(mapping->domain), iova, phys_to_map, len,
|
||||
prot);
|
||||
return iova + offset_from_phys_to_map;
|
||||
|
||||
fail_free_iova:
|
||||
__fast_smmu_free_iova(mapping, iova, size);
|
||||
fail:
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
return DMA_ERROR_CODE;
|
||||
}
|
||||
|
||||
static void fast_smmu_unmap_page(struct device *dev, dma_addr_t iova,
|
||||
size_t size, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
unsigned long flags;
|
||||
unsigned long offset = iova & ~FAST_PAGE_MASK;
|
||||
size_t len = ALIGN(size + offset, FAST_PAGE_SIZE);
|
||||
bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC);
|
||||
bool is_coherent = is_dma_coherent(dev, attrs);
|
||||
|
||||
if (!skip_sync && !is_coherent) {
|
||||
phys_addr_t phys;
|
||||
|
||||
phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova);
|
||||
WARN_ON(!phys);
|
||||
|
||||
__fast_dma_page_dev_to_cpu(phys_to_page(phys), offset,
|
||||
size, dir);
|
||||
}
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
av8l_fast_unmap_public(mapping->pgtbl_ops, iova, len);
|
||||
__fast_smmu_free_iova(mapping, iova, len);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
|
||||
trace_unmap(to_msm_iommu_domain(mapping->domain), iova - offset, len,
|
||||
len);
|
||||
}
|
||||
|
||||
static void fast_smmu_sync_single_for_cpu(struct device *dev,
|
||||
dma_addr_t iova, size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
unsigned long offset = iova & ~FAST_PAGE_MASK;
|
||||
|
||||
if (!av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) {
|
||||
phys_addr_t phys;
|
||||
|
||||
phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova);
|
||||
WARN_ON(!phys);
|
||||
|
||||
__fast_dma_page_dev_to_cpu(phys_to_page(phys), offset,
|
||||
size, dir);
|
||||
}
|
||||
}
|
||||
|
||||
static void fast_smmu_sync_single_for_device(struct device *dev,
|
||||
dma_addr_t iova, size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
unsigned long offset = iova & ~FAST_PAGE_MASK;
|
||||
|
||||
if (!av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) {
|
||||
phys_addr_t phys;
|
||||
|
||||
phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova);
|
||||
WARN_ON(!phys);
|
||||
|
||||
__fast_dma_page_cpu_to_dev(phys_to_page(phys), offset,
|
||||
size, dir);
|
||||
}
|
||||
}
|
||||
|
||||
static void fast_smmu_sync_sg_for_cpu(struct device *dev,
|
||||
struct scatterlist *sgl, int nelems,
|
||||
enum dma_data_direction dir)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
dma_addr_t iova = sg_dma_address(sgl);
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
int i;
|
||||
|
||||
if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova))
|
||||
return;
|
||||
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_unmap_area(sg_virt(sg), sg->length, dir);
|
||||
}
|
||||
|
||||
static void fast_smmu_sync_sg_for_device(struct device *dev,
|
||||
struct scatterlist *sgl, int nelems,
|
||||
enum dma_data_direction dir)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
dma_addr_t iova = sg_dma_address(sgl);
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
int i;
|
||||
|
||||
if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova))
|
||||
return;
|
||||
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_map_area(sg_virt(sg), sg->length, dir);
|
||||
}
|
||||
|
||||
static int fast_smmu_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
size_t iova_len;
|
||||
bool is_coherent = is_dma_coherent(dev, attrs);
|
||||
int prot = dma_info_to_prot(dir, is_coherent, attrs);
|
||||
int ret;
|
||||
dma_addr_t iova;
|
||||
unsigned long flags;
|
||||
size_t unused;
|
||||
|
||||
iova_len = iommu_dma_prepare_map_sg(dev, mapping->iovad, sg, nents);
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
iova = __fast_smmu_alloc_iova(mapping, attrs, iova_len);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
|
||||
if (unlikely(iova == DMA_ERROR_CODE))
|
||||
goto fail;
|
||||
|
||||
av8l_fast_map_sg_public(mapping->pgtbl_ops, iova, sg, nents, prot,
|
||||
&unused);
|
||||
|
||||
ret = iommu_dma_finalise_sg(dev, sg, nents, iova);
|
||||
|
||||
if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
|
||||
fast_smmu_sync_sg_for_device(dev, sg, nents, dir);
|
||||
|
||||
return ret;
|
||||
fail:
|
||||
iommu_dma_invalidate_sg(sg, nents);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void fast_smmu_unmap_sg(struct device *dev,
|
||||
struct scatterlist *sg, int nelems,
|
||||
enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
unsigned long flags;
|
||||
dma_addr_t start;
|
||||
size_t len;
|
||||
struct scatterlist *tmp;
|
||||
int i;
|
||||
|
||||
if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
|
||||
fast_smmu_sync_sg_for_cpu(dev, sg, nelems, dir);
|
||||
|
||||
/*
|
||||
* The scatterlist segments are mapped into a single
|
||||
* contiguous IOVA allocation, so this is incredibly easy.
|
||||
*/
|
||||
start = sg_dma_address(sg);
|
||||
for_each_sg(sg_next(sg), tmp, nelems - 1, i) {
|
||||
if (sg_dma_len(tmp) == 0)
|
||||
break;
|
||||
sg = tmp;
|
||||
}
|
||||
len = sg_dma_address(sg) + sg_dma_len(sg) - start;
|
||||
|
||||
av8l_fast_unmap_public(mapping->pgtbl_ops, start, len);
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
__fast_smmu_free_iova(mapping, start, len);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
}
|
||||
|
||||
static void __fast_smmu_free_pages(struct page **pages, int count)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < count; i++)
|
||||
__free_page(pages[i]);
|
||||
kvfree(pages);
|
||||
}
|
||||
|
||||
static struct page **__fast_smmu_alloc_pages(unsigned int count, gfp_t gfp)
|
||||
{
|
||||
struct page **pages;
|
||||
unsigned int i = 0, array_size = count * sizeof(*pages);
|
||||
|
||||
if (array_size <= PAGE_SIZE)
|
||||
pages = kzalloc(array_size, GFP_KERNEL);
|
||||
else
|
||||
pages = vzalloc(array_size);
|
||||
if (!pages)
|
||||
return NULL;
|
||||
|
||||
/* IOMMU can map any pages, so himem can also be used here */
|
||||
gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
|
||||
|
||||
for (i = 0; i < count; ++i) {
|
||||
struct page *page = alloc_page(gfp);
|
||||
|
||||
if (!page) {
|
||||
__fast_smmu_free_pages(pages, i);
|
||||
return NULL;
|
||||
}
|
||||
pages[i] = page;
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
static void *fast_smmu_alloc(struct device *dev, size_t size,
|
||||
dma_addr_t *handle, gfp_t gfp,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
struct sg_table sgt;
|
||||
dma_addr_t dma_addr, iova_iter;
|
||||
void *addr;
|
||||
unsigned long flags;
|
||||
struct sg_mapping_iter miter;
|
||||
size_t count = ALIGN(size, SZ_4K) >> PAGE_SHIFT;
|
||||
bool is_coherent = is_dma_coherent(dev, attrs);
|
||||
int prot = dma_info_to_prot(DMA_BIDIRECTIONAL, is_coherent, attrs);
|
||||
pgprot_t remap_prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent);
|
||||
struct page **pages;
|
||||
|
||||
/*
|
||||
* sg_alloc_table_from_pages accepts unsigned int value for count
|
||||
* so check count doesn't exceed UINT_MAX.
|
||||
*/
|
||||
|
||||
if (count > UINT_MAX) {
|
||||
dev_err(dev, "count: %zx exceeds UNIT_MAX\n", count);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
*handle = DMA_ERROR_CODE;
|
||||
|
||||
pages = __fast_smmu_alloc_pages(count, gfp);
|
||||
if (!pages) {
|
||||
dev_err(dev, "no pages\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size = ALIGN(size, SZ_4K);
|
||||
if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, gfp)) {
|
||||
dev_err(dev, "no sg tablen\n");
|
||||
goto out_free_pages;
|
||||
}
|
||||
|
||||
if (!is_coherent) {
|
||||
/*
|
||||
* The CPU-centric flushing implied by SG_MITER_TO_SG isn't
|
||||
* sufficient here, so skip it by using the "wrong" direction.
|
||||
*/
|
||||
sg_miter_start(&miter, sgt.sgl, sgt.orig_nents,
|
||||
SG_MITER_FROM_SG);
|
||||
while (sg_miter_next(&miter))
|
||||
__dma_flush_area(miter.addr, miter.length);
|
||||
sg_miter_stop(&miter);
|
||||
}
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
dma_addr = __fast_smmu_alloc_iova(mapping, attrs, size);
|
||||
if (dma_addr == DMA_ERROR_CODE) {
|
||||
dev_err(dev, "no iova\n");
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
goto out_free_sg;
|
||||
}
|
||||
iova_iter = dma_addr;
|
||||
sg_miter_start(&miter, sgt.sgl, sgt.orig_nents,
|
||||
SG_MITER_FROM_SG | SG_MITER_ATOMIC);
|
||||
while (sg_miter_next(&miter)) {
|
||||
if (unlikely(av8l_fast_map_public(
|
||||
mapping->pgtbl_ops, iova_iter,
|
||||
page_to_phys(miter.page),
|
||||
miter.length, prot))) {
|
||||
dev_err(dev, "no map public\n");
|
||||
/* TODO: unwind previously successful mappings */
|
||||
goto out_free_iova;
|
||||
}
|
||||
iova_iter += miter.length;
|
||||
}
|
||||
sg_miter_stop(&miter);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
|
||||
addr = dma_common_pages_remap(pages, size, VM_USERMAP, remap_prot,
|
||||
__builtin_return_address(0));
|
||||
if (!addr) {
|
||||
dev_err(dev, "no common pages\n");
|
||||
goto out_unmap;
|
||||
}
|
||||
|
||||
*handle = dma_addr;
|
||||
sg_free_table(&sgt);
|
||||
return addr;
|
||||
|
||||
out_unmap:
|
||||
/* need to take the lock again for page tables and iova */
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
av8l_fast_unmap_public(mapping->pgtbl_ops, dma_addr, size);
|
||||
out_free_iova:
|
||||
__fast_smmu_free_iova(mapping, dma_addr, size);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
out_free_sg:
|
||||
sg_free_table(&sgt);
|
||||
out_free_pages:
|
||||
__fast_smmu_free_pages(pages, count);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void fast_smmu_free(struct device *dev, size_t size,
|
||||
void *vaddr, dma_addr_t dma_handle,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
struct vm_struct *area;
|
||||
struct page **pages;
|
||||
size_t count = ALIGN(size, SZ_4K) >> FAST_PAGE_SHIFT;
|
||||
unsigned long flags;
|
||||
|
||||
size = ALIGN(size, SZ_4K);
|
||||
|
||||
area = find_vm_area(vaddr);
|
||||
if (WARN_ON_ONCE(!area))
|
||||
return;
|
||||
|
||||
pages = area->pages;
|
||||
dma_common_free_remap(vaddr, size, VM_USERMAP);
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
av8l_fast_unmap_public(mapping->pgtbl_ops, dma_handle, size);
|
||||
__fast_smmu_free_iova(mapping, dma_handle, size);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
__fast_smmu_free_pages(pages, count);
|
||||
}
|
||||
|
||||
static int fast_smmu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
|
||||
void *cpu_addr, dma_addr_t dma_addr,
|
||||
size_t size, unsigned long attrs)
|
||||
{
|
||||
struct vm_struct *area;
|
||||
unsigned long uaddr = vma->vm_start;
|
||||
struct page **pages;
|
||||
int i, nr_pages, ret = 0;
|
||||
bool coherent = is_dma_coherent(dev, attrs);
|
||||
|
||||
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
|
||||
coherent);
|
||||
area = find_vm_area(cpu_addr);
|
||||
if (!area)
|
||||
return -EINVAL;
|
||||
|
||||
pages = area->pages;
|
||||
nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
||||
for (i = vma->vm_pgoff; i < nr_pages && uaddr < vma->vm_end; i++) {
|
||||
ret = vm_insert_page(vma, uaddr, pages[i]);
|
||||
if (ret)
|
||||
break;
|
||||
uaddr += PAGE_SIZE;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int fast_smmu_get_sgtable(struct device *dev, struct sg_table *sgt,
|
||||
void *cpu_addr, dma_addr_t dma_addr,
|
||||
size_t size, unsigned long attrs)
|
||||
{
|
||||
unsigned int n_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
||||
struct vm_struct *area;
|
||||
|
||||
area = find_vm_area(cpu_addr);
|
||||
if (!area || !area->pages)
|
||||
return -EINVAL;
|
||||
|
||||
return sg_alloc_table_from_pages(sgt, area->pages, n_pages, 0, size,
|
||||
GFP_KERNEL);
|
||||
}
|
||||
|
||||
static dma_addr_t fast_smmu_dma_map_resource(
|
||||
struct device *dev, phys_addr_t phys_addr,
|
||||
size_t size, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
size_t offset = phys_addr & ~FAST_PAGE_MASK;
|
||||
size_t len = round_up(size + offset, FAST_PAGE_SIZE);
|
||||
dma_addr_t dma_addr;
|
||||
int prot;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
dma_addr = __fast_smmu_alloc_iova(mapping, attrs, len);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
|
||||
if (dma_addr == DMA_ERROR_CODE)
|
||||
return dma_addr;
|
||||
|
||||
prot = dma_info_to_prot(dir, false, attrs);
|
||||
prot |= IOMMU_MMIO;
|
||||
|
||||
if (iommu_map(mapping->domain, dma_addr, phys_addr - offset,
|
||||
len, prot)) {
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
__fast_smmu_free_iova(mapping, dma_addr, len);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
return DMA_ERROR_CODE;
|
||||
}
|
||||
return dma_addr + offset;
|
||||
}
|
||||
|
||||
static void fast_smmu_dma_unmap_resource(
|
||||
struct device *dev, dma_addr_t addr,
|
||||
size_t size, enum dma_data_direction dir,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev);
|
||||
size_t offset = addr & ~FAST_PAGE_MASK;
|
||||
size_t len = round_up(size + offset, FAST_PAGE_SIZE);
|
||||
unsigned long flags;
|
||||
|
||||
iommu_unmap(mapping->domain, addr - offset, len);
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
__fast_smmu_free_iova(mapping, addr, len);
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
}
|
||||
|
||||
static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast,
|
||||
void *data)
|
||||
{
|
||||
av8l_fast_iopte *pmds, *ptep = data;
|
||||
dma_addr_t iova;
|
||||
unsigned long bitmap_idx;
|
||||
struct io_pgtable *tbl;
|
||||
|
||||
tbl = container_of(fast->pgtbl_ops, struct io_pgtable, ops);
|
||||
pmds = tbl->cfg.av8l_fast_cfg.pmds;
|
||||
|
||||
bitmap_idx = (unsigned long)(ptep - pmds);
|
||||
iova = bitmap_idx << FAST_PAGE_SHIFT;
|
||||
dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova);
|
||||
dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx);
|
||||
dev_err(fast->dev, "ptep: %pK pmds: %pK diff: %lu\n", ptep,
|
||||
pmds, bitmap_idx);
|
||||
print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS,
|
||||
32, 8, fast->bitmap, fast->bitmap_size, false);
|
||||
}
|
||||
|
||||
static int fast_smmu_notify(struct notifier_block *self,
|
||||
unsigned long action, void *data)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *fast = container_of(
|
||||
self, struct dma_fast_smmu_mapping, notifier);
|
||||
|
||||
switch (action) {
|
||||
case MAPPED_OVER_STALE_TLB:
|
||||
__fast_smmu_mapped_over_stale(fast, data);
|
||||
return NOTIFY_OK;
|
||||
default:
|
||||
WARN(1, "Unhandled notifier action");
|
||||
return NOTIFY_DONE;
|
||||
}
|
||||
}
|
||||
|
||||
static const struct dma_map_ops fast_smmu_dma_ops = {
|
||||
.alloc = fast_smmu_alloc,
|
||||
.free = fast_smmu_free,
|
||||
.mmap = fast_smmu_mmap_attrs,
|
||||
.get_sgtable = fast_smmu_get_sgtable,
|
||||
.map_page = fast_smmu_map_page,
|
||||
.unmap_page = fast_smmu_unmap_page,
|
||||
.sync_single_for_cpu = fast_smmu_sync_single_for_cpu,
|
||||
.sync_single_for_device = fast_smmu_sync_single_for_device,
|
||||
.map_sg = fast_smmu_map_sg,
|
||||
.unmap_sg = fast_smmu_unmap_sg,
|
||||
.sync_sg_for_cpu = fast_smmu_sync_sg_for_cpu,
|
||||
.sync_sg_for_device = fast_smmu_sync_sg_for_device,
|
||||
.map_resource = fast_smmu_dma_map_resource,
|
||||
.unmap_resource = fast_smmu_dma_unmap_resource,
|
||||
};
|
||||
|
||||
/**
|
||||
* __fast_smmu_create_mapping_sized
|
||||
* @base: bottom of the VA range
|
||||
* @size: size of the VA range in bytes
|
||||
*
|
||||
* Creates a mapping structure which holds information about used/unused IO
|
||||
* address ranges, which is required to perform mapping with IOMMU aware
|
||||
* functions. The only VA range supported is [0, 4GB).
|
||||
*
|
||||
* The client device need to be attached to the mapping with
|
||||
* fast_smmu_attach_device function.
|
||||
*/
|
||||
static struct dma_fast_smmu_mapping *__fast_smmu_create_mapping_sized(
|
||||
dma_addr_t base, u64 size)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *fast;
|
||||
|
||||
fast = kzalloc(sizeof(struct dma_fast_smmu_mapping), GFP_KERNEL);
|
||||
if (!fast)
|
||||
goto err;
|
||||
|
||||
fast->base = base;
|
||||
fast->size = size;
|
||||
fast->num_4k_pages = size >> FAST_PAGE_SHIFT;
|
||||
fast->bitmap_size = BITS_TO_LONGS(fast->num_4k_pages) * sizeof(long);
|
||||
|
||||
fast->bitmap = kzalloc(fast->bitmap_size, GFP_KERNEL | __GFP_NOWARN |
|
||||
__GFP_NORETRY);
|
||||
if (!fast->bitmap)
|
||||
fast->bitmap = vzalloc(fast->bitmap_size);
|
||||
|
||||
if (!fast->bitmap)
|
||||
goto err2;
|
||||
|
||||
spin_lock_init(&fast->lock);
|
||||
|
||||
fast->iovad = kzalloc(sizeof(*fast->iovad), GFP_KERNEL);
|
||||
if (!fast->iovad)
|
||||
goto err_free_bitmap;
|
||||
init_iova_domain(fast->iovad, FAST_PAGE_SIZE,
|
||||
base >> FAST_PAGE_SHIFT);
|
||||
|
||||
return fast;
|
||||
|
||||
err_free_bitmap:
|
||||
kvfree(fast->bitmap);
|
||||
err2:
|
||||
kfree(fast);
|
||||
err:
|
||||
return ERR_PTR(-ENOMEM);
|
||||
}
|
||||
|
||||
/*
|
||||
* Based off of similar code from dma-iommu.c, but modified to use a different
|
||||
* iova allocator
|
||||
*/
|
||||
static void fast_smmu_reserve_pci_windows(struct device *dev,
|
||||
struct dma_fast_smmu_mapping *mapping)
|
||||
{
|
||||
struct pci_host_bridge *bridge;
|
||||
struct resource_entry *window;
|
||||
phys_addr_t start, end;
|
||||
struct pci_dev *pci_dev;
|
||||
unsigned long flags;
|
||||
|
||||
if (!dev_is_pci(dev))
|
||||
return;
|
||||
|
||||
pci_dev = to_pci_dev(dev);
|
||||
bridge = pci_find_host_bridge(pci_dev->bus);
|
||||
|
||||
spin_lock_irqsave(&mapping->lock, flags);
|
||||
resource_list_for_each_entry(window, &bridge->windows) {
|
||||
if (resource_type(window->res) != IORESOURCE_MEM &&
|
||||
resource_type(window->res) != IORESOURCE_IO)
|
||||
continue;
|
||||
|
||||
start = round_down(window->res->start - window->offset,
|
||||
FAST_PAGE_SIZE);
|
||||
end = round_up(window->res->end - window->offset,
|
||||
FAST_PAGE_SIZE);
|
||||
start = max_t(unsigned long, mapping->base, start);
|
||||
end = min_t(unsigned long, mapping->base + mapping->size, end);
|
||||
if (start >= end)
|
||||
continue;
|
||||
|
||||
dev_dbg(dev, "iova allocator reserved 0x%pa-0x%pa\n",
|
||||
&start, &end);
|
||||
|
||||
start = (start - mapping->base) >> FAST_PAGE_SHIFT;
|
||||
end = (end - mapping->base) >> FAST_PAGE_SHIFT;
|
||||
bitmap_set(mapping->bitmap, start, end - start);
|
||||
}
|
||||
spin_unlock_irqrestore(&mapping->lock, flags);
|
||||
}
|
||||
|
||||
void fast_smmu_put_dma_cookie(struct iommu_domain *domain)
|
||||
{
|
||||
struct dma_fast_smmu_mapping *fast = domain->iova_cookie;
|
||||
|
||||
if (!fast)
|
||||
return;
|
||||
|
||||
if (fast->iovad) {
|
||||
put_iova_domain(fast->iovad);
|
||||
kfree(fast->iovad);
|
||||
}
|
||||
|
||||
if (fast->bitmap)
|
||||
kvfree(fast->bitmap);
|
||||
|
||||
kfree(fast);
|
||||
domain->iova_cookie = NULL;
|
||||
}
|
||||
|
||||
const struct dma_map_ops *fast_smmu_get_dma_ops(void)
|
||||
{
|
||||
return &fast_smmu_dma_ops;
|
||||
}
|
||||
|
||||
/**
|
||||
* fast_smmu_init_mapping
|
||||
* @dev: valid struct device pointer
|
||||
* @domain: valid IOMMU domain pointer
|
||||
* @pgtable_ops: The page table ops associated with this domain
|
||||
*
|
||||
* Called the first time a device is attached to this mapping.
|
||||
* Not for dma client use.
|
||||
*/
|
||||
int fast_smmu_init_mapping(struct device *dev, struct iommu_domain *domain,
|
||||
struct io_pgtable_ops *pgtable_ops)
|
||||
{
|
||||
u64 dma_base = domain->geometry.aperture_start;
|
||||
u64 dma_end = domain->geometry.aperture_end;
|
||||
u64 size = dma_end - dma_base + 1;
|
||||
struct dma_fast_smmu_mapping *fast;
|
||||
|
||||
if (domain->iova_cookie) {
|
||||
fast = domain->iova_cookie;
|
||||
goto finish;
|
||||
}
|
||||
|
||||
if (!pgtable_ops)
|
||||
return -EINVAL;
|
||||
|
||||
if (dma_base + size > (SZ_1G * 4ULL)) {
|
||||
dev_err(dev, "Iova end address too large\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
fast = __fast_smmu_create_mapping_sized(dma_base, size);
|
||||
if (IS_ERR(fast))
|
||||
return -ENOMEM;
|
||||
|
||||
fast->domain = domain;
|
||||
fast->dev = dev;
|
||||
domain->iova_cookie = fast;
|
||||
|
||||
fast->pgtbl_ops = pgtable_ops;
|
||||
|
||||
fast->notifier.notifier_call = fast_smmu_notify;
|
||||
av8l_register_notify(&fast->notifier);
|
||||
|
||||
finish:
|
||||
fast_smmu_reserve_pci_windows(dev, fast);
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
778
drivers/iommu/io-pgtable-fast.c
Normal file
778
drivers/iommu/io-pgtable-fast.c
Normal file
|
|
@ -0,0 +1,778 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#define pr_fmt(fmt) "io-pgtable-fast: " fmt
|
||||
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/sizes.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/io-pgtable.h>
|
||||
#include <linux/io-pgtable-fast.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <linux/vmalloc.h>
|
||||
|
||||
|
||||
#define AV8L_FAST_MAX_ADDR_BITS 48
|
||||
|
||||
/* Struct accessors */
|
||||
#define iof_pgtable_to_data(x) \
|
||||
container_of((x), struct av8l_fast_io_pgtable, iop)
|
||||
|
||||
#define iof_pgtable_ops_to_pgtable(x) \
|
||||
container_of((x), struct io_pgtable, ops)
|
||||
|
||||
#define iof_pgtable_ops_to_data(x) \
|
||||
iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x))
|
||||
|
||||
struct av8l_fast_io_pgtable {
|
||||
struct io_pgtable iop;
|
||||
av8l_fast_iopte *pgd;
|
||||
av8l_fast_iopte *puds[4];
|
||||
av8l_fast_iopte *pmds;
|
||||
struct page **pages; /* page table memory */
|
||||
};
|
||||
|
||||
/* Page table bits */
|
||||
#define AV8L_FAST_PTE_TYPE_SHIFT 0
|
||||
#define AV8L_FAST_PTE_TYPE_MASK 0x3
|
||||
|
||||
#define AV8L_FAST_PTE_TYPE_BLOCK 1
|
||||
#define AV8L_FAST_PTE_TYPE_TABLE 3
|
||||
#define AV8L_FAST_PTE_TYPE_PAGE 3
|
||||
|
||||
#define AV8L_FAST_PTE_NSTABLE (((av8l_fast_iopte)1) << 63)
|
||||
#define AV8L_FAST_PTE_XN (((av8l_fast_iopte)3) << 53)
|
||||
#define AV8L_FAST_PTE_AF (((av8l_fast_iopte)1) << 10)
|
||||
#define AV8L_FAST_PTE_SH_NS (((av8l_fast_iopte)0) << 8)
|
||||
#define AV8L_FAST_PTE_SH_OS (((av8l_fast_iopte)2) << 8)
|
||||
#define AV8L_FAST_PTE_SH_IS (((av8l_fast_iopte)3) << 8)
|
||||
#define AV8L_FAST_PTE_SH_MASK (((av8l_fast_iopte)3) << 8)
|
||||
#define AV8L_FAST_PTE_NS (((av8l_fast_iopte)1) << 5)
|
||||
#define AV8L_FAST_PTE_VALID (((av8l_fast_iopte)1) << 0)
|
||||
|
||||
#define AV8L_FAST_PTE_ATTR_LO_MASK (((av8l_fast_iopte)0x3ff) << 2)
|
||||
/* Ignore the contiguous bit for block splitting */
|
||||
#define AV8L_FAST_PTE_ATTR_HI_MASK (((av8l_fast_iopte)6) << 52)
|
||||
#define AV8L_FAST_PTE_ATTR_MASK (AV8L_FAST_PTE_ATTR_LO_MASK | \
|
||||
AV8L_FAST_PTE_ATTR_HI_MASK)
|
||||
#define AV8L_FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000)
|
||||
|
||||
|
||||
/* Stage-1 PTE */
|
||||
#define AV8L_FAST_PTE_AP_PRIV_RW (((av8l_fast_iopte)0) << 6)
|
||||
#define AV8L_FAST_PTE_AP_RW (((av8l_fast_iopte)1) << 6)
|
||||
#define AV8L_FAST_PTE_AP_PRIV_RO (((av8l_fast_iopte)2) << 6)
|
||||
#define AV8L_FAST_PTE_AP_RO (((av8l_fast_iopte)3) << 6)
|
||||
#define AV8L_FAST_PTE_ATTRINDX_SHIFT 2
|
||||
#define AV8L_FAST_PTE_ATTRINDX_MASK 0x7
|
||||
#define AV8L_FAST_PTE_nG (((av8l_fast_iopte)1) << 11)
|
||||
|
||||
/* Stage-2 PTE */
|
||||
#define AV8L_FAST_PTE_HAP_FAULT (((av8l_fast_iopte)0) << 6)
|
||||
#define AV8L_FAST_PTE_HAP_READ (((av8l_fast_iopte)1) << 6)
|
||||
#define AV8L_FAST_PTE_HAP_WRITE (((av8l_fast_iopte)2) << 6)
|
||||
#define AV8L_FAST_PTE_MEMATTR_OIWB (((av8l_fast_iopte)0xf) << 2)
|
||||
#define AV8L_FAST_PTE_MEMATTR_NC (((av8l_fast_iopte)0x5) << 2)
|
||||
#define AV8L_FAST_PTE_MEMATTR_DEV (((av8l_fast_iopte)0x1) << 2)
|
||||
|
||||
/* Register bits */
|
||||
#define ARM_32_LPAE_TCR_EAE (1 << 31)
|
||||
#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31)
|
||||
|
||||
#define AV8L_FAST_TCR_TG0_4K (0 << 14)
|
||||
#define AV8L_FAST_TCR_TG0_64K (1 << 14)
|
||||
#define AV8L_FAST_TCR_TG0_16K (2 << 14)
|
||||
|
||||
#define AV8L_FAST_TCR_SH0_SHIFT 12
|
||||
#define AV8L_FAST_TCR_SH0_MASK 0x3
|
||||
#define AV8L_FAST_TCR_SH_NS 0
|
||||
#define AV8L_FAST_TCR_SH_OS 2
|
||||
#define AV8L_FAST_TCR_SH_IS 3
|
||||
|
||||
#define AV8L_FAST_TCR_ORGN0_SHIFT 10
|
||||
#define AV8L_FAST_TCR_IRGN0_SHIFT 8
|
||||
#define AV8L_FAST_TCR_RGN_MASK 0x3
|
||||
#define AV8L_FAST_TCR_RGN_NC 0
|
||||
#define AV8L_FAST_TCR_RGN_WBWA 1
|
||||
#define AV8L_FAST_TCR_RGN_WT 2
|
||||
#define AV8L_FAST_TCR_RGN_WB 3
|
||||
|
||||
#define AV8L_FAST_TCR_SL0_SHIFT 6
|
||||
#define AV8L_FAST_TCR_SL0_MASK 0x3
|
||||
|
||||
#define AV8L_FAST_TCR_T0SZ_SHIFT 0
|
||||
#define AV8L_FAST_TCR_SZ_MASK 0xf
|
||||
|
||||
#define AV8L_FAST_TCR_PS_SHIFT 16
|
||||
#define AV8L_FAST_TCR_PS_MASK 0x7
|
||||
|
||||
#define AV8L_FAST_TCR_IPS_SHIFT 32
|
||||
#define AV8L_FAST_TCR_IPS_MASK 0x7
|
||||
|
||||
#define AV8L_FAST_TCR_PS_32_BIT 0x0ULL
|
||||
#define AV8L_FAST_TCR_PS_36_BIT 0x1ULL
|
||||
#define AV8L_FAST_TCR_PS_40_BIT 0x2ULL
|
||||
#define AV8L_FAST_TCR_PS_42_BIT 0x3ULL
|
||||
#define AV8L_FAST_TCR_PS_44_BIT 0x4ULL
|
||||
#define AV8L_FAST_TCR_PS_48_BIT 0x5ULL
|
||||
|
||||
#define AV8L_FAST_TCR_EPD1_SHIFT 23
|
||||
#define AV8L_FAST_TCR_EPD1_FAULT 1
|
||||
|
||||
#define AV8L_FAST_MAIR_ATTR_SHIFT(n) ((n) << 3)
|
||||
#define AV8L_FAST_MAIR_ATTR_MASK 0xff
|
||||
#define AV8L_FAST_MAIR_ATTR_DEVICE 0x04
|
||||
#define AV8L_FAST_MAIR_ATTR_NC 0x44
|
||||
#define AV8L_FAST_MAIR_ATTR_WBRWA 0xff
|
||||
#define AV8L_FAST_MAIR_ATTR_UPSTREAM 0xf4
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_NC 0
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_CACHE 1
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_DEV 2
|
||||
#define AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM 3
|
||||
|
||||
#define AV8L_FAST_PAGE_SHIFT 12
|
||||
|
||||
#define PTE_MAIR_IDX(pte) \
|
||||
((pte >> AV8L_FAST_PTE_ATTRINDX_SHIFT) & \
|
||||
AV8L_FAST_PTE_ATTRINDX_MASK)
|
||||
|
||||
#define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK)
|
||||
|
||||
#define iopte_pmd_offset(pmds, iova) (pmds + (iova >> 12))
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <linux/notifier.h>
|
||||
|
||||
static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list);
|
||||
|
||||
void av8l_register_notify(struct notifier_block *nb)
|
||||
{
|
||||
atomic_notifier_chain_register(&av8l_notifier_list, nb);
|
||||
}
|
||||
EXPORT_SYMBOL(av8l_register_notify);
|
||||
|
||||
static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
|
||||
{
|
||||
if (unlikely(*ptep)) {
|
||||
atomic_notifier_call_chain(
|
||||
&av8l_notifier_list, MAPPED_OVER_STALE_TLB,
|
||||
(void *) ptep);
|
||||
pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n",
|
||||
*ptep, ptep);
|
||||
pr_err("Nearby memory:\n");
|
||||
print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS,
|
||||
32, 8, ptep - 16, 32 * sizeof(*ptep), false);
|
||||
}
|
||||
}
|
||||
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, bool skip_sync)
|
||||
{
|
||||
int i;
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte *pmdp = data->pmds;
|
||||
|
||||
for (i = 0; i < ((SZ_1G * 4UL) >> AV8L_FAST_PAGE_SHIFT); ++i) {
|
||||
if (!(*pmdp & AV8L_FAST_PTE_VALID)) {
|
||||
*pmdp = 0;
|
||||
if (!skip_sync)
|
||||
dmac_clean_range(pmdp, pmdp + 1);
|
||||
}
|
||||
pmdp++;
|
||||
}
|
||||
}
|
||||
#else
|
||||
static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
static void av8l_clean_range(struct io_pgtable_ops *ops,
|
||||
av8l_fast_iopte *start, av8l_fast_iopte *end)
|
||||
{
|
||||
struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops);
|
||||
|
||||
if (!(iop->cfg.quirks & IO_PGTABLE_QUIRK_NO_DMA))
|
||||
dmac_clean_range(start, end);
|
||||
}
|
||||
|
||||
static av8l_fast_iopte
|
||||
av8l_fast_prot_to_pte(struct av8l_fast_io_pgtable *data, int prot)
|
||||
{
|
||||
av8l_fast_iopte pte = AV8L_FAST_PTE_XN
|
||||
| AV8L_FAST_PTE_TYPE_PAGE
|
||||
| AV8L_FAST_PTE_AF
|
||||
| AV8L_FAST_PTE_nG
|
||||
| AV8L_FAST_PTE_SH_OS;
|
||||
|
||||
if (prot & IOMMU_MMIO)
|
||||
pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV
|
||||
<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
|
||||
else if (prot & IOMMU_CACHE)
|
||||
pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE
|
||||
<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
|
||||
else if (prot & IOMMU_USE_UPSTREAM_HINT)
|
||||
pte |= (AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM
|
||||
<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
|
||||
|
||||
if (!(prot & IOMMU_WRITE))
|
||||
pte |= AV8L_FAST_PTE_AP_RO;
|
||||
else
|
||||
pte |= AV8L_FAST_PTE_AP_RW;
|
||||
|
||||
return pte;
|
||||
}
|
||||
|
||||
static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);
|
||||
unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT;
|
||||
av8l_fast_iopte pte;
|
||||
|
||||
pte = av8l_fast_prot_to_pte(data, prot);
|
||||
paddr &= AV8L_FAST_PTE_ADDR_MASK;
|
||||
for (i = 0; i < nptes; i++, paddr += SZ_4K) {
|
||||
__av8l_check_for_stale_tlb(ptep + i);
|
||||
*(ptep + i) = pte | paddr;
|
||||
}
|
||||
av8l_clean_range(ops, ptep, ptep + nptes);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot)
|
||||
{
|
||||
return av8l_fast_map(ops, iova, paddr, size, prot);
|
||||
}
|
||||
|
||||
static size_t
|
||||
__av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size, bool allow_stale_tlb)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
unsigned long nptes;
|
||||
av8l_fast_iopte *ptep;
|
||||
int val = allow_stale_tlb
|
||||
? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI
|
||||
: 0;
|
||||
|
||||
ptep = iopte_pmd_offset(data->pmds, iova);
|
||||
nptes = size >> AV8L_FAST_PAGE_SHIFT;
|
||||
|
||||
memset(ptep, val, sizeof(*ptep) * nptes);
|
||||
av8l_clean_range(ops, ptep, ptep + nptes);
|
||||
if (!allow_stale_tlb)
|
||||
io_pgtable_tlb_flush_all(&data->iop);
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
/* caller must take care of tlb cache maintenance */
|
||||
void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size)
|
||||
{
|
||||
__av8l_fast_unmap(ops, iova, size, true);
|
||||
}
|
||||
|
||||
static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size)
|
||||
{
|
||||
return __av8l_fast_unmap(ops, iova, size, false);
|
||||
}
|
||||
|
||||
static int av8l_fast_map_sg(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
int i;
|
||||
|
||||
for_each_sg(sgl, sg, nents, i) {
|
||||
av8l_fast_map(ops, iova, sg_phys(sg), sg->length, prot);
|
||||
iova += sg->length;
|
||||
}
|
||||
|
||||
return nents;
|
||||
}
|
||||
|
||||
int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size)
|
||||
{
|
||||
return av8l_fast_map_sg(ops, iova, sgl, nents, prot, size);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_ARM64)
|
||||
#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27)
|
||||
#elif defined(CONFIG_ARM)
|
||||
#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27)
|
||||
#endif
|
||||
|
||||
static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte pte, *pgdp, *pudp, *pmdp;
|
||||
unsigned long pgd;
|
||||
phys_addr_t phys;
|
||||
const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT;
|
||||
const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK;
|
||||
const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE;
|
||||
const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE;
|
||||
const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK;
|
||||
|
||||
/* TODO: clean up some of these magic numbers... */
|
||||
|
||||
pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova);
|
||||
pgdp = (av8l_fast_iopte *)pgd;
|
||||
|
||||
pte = *pgdp;
|
||||
if (((pte >> pts) & ptm) != ptt)
|
||||
return 0;
|
||||
pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18));
|
||||
|
||||
pte = *pudp;
|
||||
if (((pte >> pts) & ptm) != ptt)
|
||||
return 0;
|
||||
pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9));
|
||||
|
||||
pte = *pmdp;
|
||||
if (((pte >> pts) & ptm) != ptp)
|
||||
return 0;
|
||||
phys = pte & am;
|
||||
|
||||
return phys | (iova & 0xfff);
|
||||
}
|
||||
|
||||
phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return av8l_fast_iova_to_phys(ops, iova);
|
||||
}
|
||||
|
||||
static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);
|
||||
|
||||
return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) &&
|
||||
((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) ||
|
||||
(PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_IS)));
|
||||
}
|
||||
|
||||
bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return av8l_fast_iova_coherent(ops, iova);
|
||||
}
|
||||
|
||||
static struct av8l_fast_io_pgtable *
|
||||
av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data;
|
||||
struct msm_io_pgtable_info *pgtbl_info = to_msm_io_pgtable_info(cfg);
|
||||
|
||||
data = kmalloc(sizeof(*data), GFP_KERNEL);
|
||||
if (!data)
|
||||
return NULL;
|
||||
|
||||
data->iop.ops = (struct io_pgtable_ops) {
|
||||
.map = av8l_fast_map,
|
||||
.unmap = av8l_fast_unmap,
|
||||
.iova_to_phys = av8l_fast_iova_to_phys,
|
||||
};
|
||||
|
||||
pgtbl_info->map_sg = av8l_fast_map_sg;
|
||||
pgtbl_info->is_iova_coherent = av8l_fast_iova_coherent;
|
||||
return data;
|
||||
}
|
||||
|
||||
/*
|
||||
* We need 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and
|
||||
* 2048 pages for pmds (each pud page contains 512 table entries, each
|
||||
* pointing to a pmd).
|
||||
*/
|
||||
#define NUM_PGD_PAGES 1
|
||||
#define NUM_PUD_PAGES 4
|
||||
#define NUM_PMD_PAGES 2048
|
||||
#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES)
|
||||
|
||||
static int
|
||||
av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
|
||||
struct io_pgtable_cfg *cfg, void *cookie)
|
||||
{
|
||||
int i, j, pg = 0;
|
||||
struct page **pages, *page;
|
||||
|
||||
pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN |
|
||||
__GFP_NORETRY);
|
||||
|
||||
if (!pages)
|
||||
pages = vmalloc(sizeof(*pages) * NUM_PGTBL_PAGES);
|
||||
|
||||
if (!pages)
|
||||
return -ENOMEM;
|
||||
|
||||
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
||||
if (!page)
|
||||
goto err_free_pages_arr;
|
||||
pages[pg++] = page;
|
||||
data->pgd = page_address(page);
|
||||
|
||||
/*
|
||||
* We need 2048 entries at level 2 to map 4GB of VA space. A page
|
||||
* can hold 512 entries, so we need 4 pages.
|
||||
*/
|
||||
for (i = 0; i < 4; ++i) {
|
||||
av8l_fast_iopte pte, *ptep;
|
||||
|
||||
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
||||
if (!page)
|
||||
goto err_free_pages;
|
||||
pages[pg++] = page;
|
||||
data->puds[i] = page_address(page);
|
||||
pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
|
||||
ptep = ((av8l_fast_iopte *)data->pgd) + i;
|
||||
*ptep = pte;
|
||||
}
|
||||
dmac_clean_range(data->pgd, data->pgd + 4);
|
||||
|
||||
/*
|
||||
* We have 4 puds, each of which can point to 512 pmds, so we'll
|
||||
* have 2048 pmds, each of which can hold 512 ptes, for a grand
|
||||
* total of 2048*512=1048576 PTEs.
|
||||
*/
|
||||
for (i = 0; i < 4; ++i) {
|
||||
for (j = 0; j < 512; ++j) {
|
||||
av8l_fast_iopte pte, *pudp;
|
||||
void *addr;
|
||||
|
||||
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
||||
if (!page)
|
||||
goto err_free_pages;
|
||||
pages[pg++] = page;
|
||||
|
||||
addr = page_address(page);
|
||||
dmac_clean_range(addr, addr + SZ_4K);
|
||||
|
||||
pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
|
||||
pudp = data->puds[i] + j;
|
||||
*pudp = pte;
|
||||
}
|
||||
dmac_clean_range(data->puds[i], data->puds[i] + 512);
|
||||
}
|
||||
|
||||
if (WARN_ON(pg != NUM_PGTBL_PAGES))
|
||||
goto err_free_pages;
|
||||
|
||||
/*
|
||||
* We map the pmds into a virtually contiguous space so that we
|
||||
* don't have to traverse the first two levels of the page tables
|
||||
* to find the appropriate pud. Instead, it will be a simple
|
||||
* offset from the virtual base of the pmds.
|
||||
*/
|
||||
data->pmds = vmap(&pages[NUM_PGD_PAGES + NUM_PUD_PAGES], NUM_PMD_PAGES,
|
||||
VM_IOREMAP, PAGE_KERNEL);
|
||||
if (!data->pmds)
|
||||
goto err_free_pages;
|
||||
|
||||
data->pages = pages;
|
||||
return 0;
|
||||
|
||||
err_free_pages:
|
||||
for (i = 0; i < pg; ++i)
|
||||
__free_page(pages[i]);
|
||||
err_free_pages_arr:
|
||||
kvfree(pages);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static struct io_pgtable *
|
||||
av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
|
||||
{
|
||||
u64 reg;
|
||||
struct av8l_fast_io_pgtable *data =
|
||||
av8l_fast_alloc_pgtable_data(cfg);
|
||||
|
||||
if (!data)
|
||||
return NULL;
|
||||
|
||||
/* restrict according to the fast map requirements */
|
||||
cfg->ias = 32;
|
||||
cfg->pgsize_bitmap = SZ_4K;
|
||||
|
||||
/* TCR */
|
||||
if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT)
|
||||
reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT);
|
||||
else if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)
|
||||
reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_IRGN0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT);
|
||||
else
|
||||
reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) |
|
||||
(AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_ORGN0_SHIFT);
|
||||
|
||||
reg |= AV8L_FAST_TCR_TG0_4K;
|
||||
|
||||
switch (cfg->oas) {
|
||||
case 32:
|
||||
reg |= (AV8L_FAST_TCR_PS_32_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 36:
|
||||
reg |= (AV8L_FAST_TCR_PS_36_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 40:
|
||||
reg |= (AV8L_FAST_TCR_PS_40_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 42:
|
||||
reg |= (AV8L_FAST_TCR_PS_42_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 44:
|
||||
reg |= (AV8L_FAST_TCR_PS_44_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
case 48:
|
||||
reg |= (AV8L_FAST_TCR_PS_48_BIT << AV8L_FAST_TCR_IPS_SHIFT);
|
||||
break;
|
||||
default:
|
||||
goto out_free_data;
|
||||
}
|
||||
|
||||
reg |= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT;
|
||||
reg |= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT;
|
||||
#if defined(CONFIG_ARM)
|
||||
reg |= ARM_32_LPAE_TCR_EAE;
|
||||
#endif
|
||||
cfg->av8l_fast_cfg.tcr = reg;
|
||||
|
||||
/* MAIRs */
|
||||
reg = (AV8L_FAST_MAIR_ATTR_NC
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) |
|
||||
(AV8L_FAST_MAIR_ATTR_WBRWA
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) |
|
||||
(AV8L_FAST_MAIR_ATTR_DEVICE
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV)) |
|
||||
(AV8L_FAST_MAIR_ATTR_UPSTREAM
|
||||
<< AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM));
|
||||
|
||||
cfg->av8l_fast_cfg.mair[0] = reg;
|
||||
cfg->av8l_fast_cfg.mair[1] = 0;
|
||||
|
||||
/* Allocate all page table memory! */
|
||||
if (av8l_fast_prepopulate_pgtables(data, cfg, cookie))
|
||||
goto out_free_data;
|
||||
|
||||
cfg->av8l_fast_cfg.pmds = data->pmds;
|
||||
|
||||
/* TTBRs */
|
||||
cfg->av8l_fast_cfg.ttbr[0] = virt_to_phys(data->pgd);
|
||||
cfg->av8l_fast_cfg.ttbr[1] = 0;
|
||||
return &data->iop;
|
||||
|
||||
out_free_data:
|
||||
kfree(data);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void av8l_fast_free_pgtable(struct io_pgtable *iop)
|
||||
{
|
||||
int i;
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop);
|
||||
|
||||
vunmap(data->pmds);
|
||||
for (i = 0; i < NUM_PGTBL_PAGES; ++i)
|
||||
__free_page(data->pages[i]);
|
||||
kvfree(data->pages);
|
||||
kfree(data);
|
||||
}
|
||||
|
||||
struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = {
|
||||
.alloc = av8l_fast_alloc_pgtable,
|
||||
.free = av8l_fast_free_pgtable,
|
||||
};
|
||||
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST
|
||||
|
||||
#include <linux/dma-contiguous.h>
|
||||
|
||||
static struct io_pgtable_cfg *cfg_cookie;
|
||||
|
||||
static void dummy_tlb_flush_all(void *cookie)
|
||||
{
|
||||
WARN_ON(cookie != cfg_cookie);
|
||||
}
|
||||
|
||||
static void dummy_tlb_add_flush(unsigned long iova, size_t size, size_t granule,
|
||||
bool leaf, void *cookie)
|
||||
{
|
||||
WARN_ON(cookie != cfg_cookie);
|
||||
WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
|
||||
}
|
||||
|
||||
static void dummy_tlb_sync(void *cookie)
|
||||
{
|
||||
WARN_ON(cookie != cfg_cookie);
|
||||
}
|
||||
|
||||
static struct iommu_gather_ops dummy_tlb_ops __initdata = {
|
||||
.tlb_flush_all = dummy_tlb_flush_all,
|
||||
.tlb_add_flush = dummy_tlb_add_flush,
|
||||
.tlb_sync = dummy_tlb_sync,
|
||||
};
|
||||
|
||||
/*
|
||||
* Returns true if the iova range is successfully mapped to the contiguous
|
||||
* phys range in ops.
|
||||
*/
|
||||
static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops,
|
||||
const unsigned long iova_start,
|
||||
const phys_addr_t phys_start,
|
||||
const size_t size)
|
||||
{
|
||||
u64 iova = iova_start;
|
||||
phys_addr_t phys = phys_start;
|
||||
|
||||
while (iova < (iova_start + size)) {
|
||||
/* + 42 just to make sure offsetting is working */
|
||||
if (ops->iova_to_phys(ops, iova + 42) != (phys + 42))
|
||||
return false;
|
||||
iova += SZ_4K;
|
||||
phys += SZ_4K;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static int __init av8l_fast_positive_testing(void)
|
||||
{
|
||||
int failed = 0;
|
||||
u64 iova;
|
||||
struct io_pgtable_ops *ops;
|
||||
struct io_pgtable_cfg cfg;
|
||||
struct av8l_fast_io_pgtable *data;
|
||||
av8l_fast_iopte *pmds;
|
||||
u64 max = SZ_1G * 4ULL - 1;
|
||||
|
||||
cfg = (struct io_pgtable_cfg) {
|
||||
.quirks = 0,
|
||||
.tlb = &dummy_tlb_ops,
|
||||
.ias = 32,
|
||||
.oas = 32,
|
||||
.pgsize_bitmap = SZ_4K,
|
||||
};
|
||||
|
||||
cfg_cookie = &cfg;
|
||||
ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &cfg, &cfg);
|
||||
|
||||
if (WARN_ON(!ops))
|
||||
return 1;
|
||||
|
||||
data = iof_pgtable_ops_to_data(ops);
|
||||
pmds = data->pmds;
|
||||
|
||||
/* map the entire 4GB VA space with 4K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_4K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all */
|
||||
for (iova = 0; iova < max; iova += SZ_4K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(ops, false);
|
||||
|
||||
/* map the entire 4GB VA space with 8K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_8K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all with 8K unmap calls */
|
||||
for (iova = 0; iova < max; iova += SZ_8K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(ops, false);
|
||||
|
||||
/* map the entire 4GB VA space with 16K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_16K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all */
|
||||
for (iova = 0; iova < max; iova += SZ_16K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(ops, false);
|
||||
|
||||
/* map the entire 4GB VA space with 64K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_64K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all at once */
|
||||
if (WARN_ON(ops->unmap(ops, 0, max) != max))
|
||||
failed++;
|
||||
|
||||
free_io_pgtable_ops(ops);
|
||||
return failed;
|
||||
}
|
||||
|
||||
static int __init av8l_fast_do_selftests(void)
|
||||
{
|
||||
int failed = 0;
|
||||
|
||||
failed += av8l_fast_positive_testing();
|
||||
|
||||
pr_err("selftest: completed with %d failures\n", failed);
|
||||
|
||||
return 0;
|
||||
}
|
||||
subsys_initcall(av8l_fast_do_selftests);
|
||||
#endif
|
||||
|
|
@ -11,6 +11,10 @@
|
|||
#include <linux/io-pgtable.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/debugfs.h>
|
||||
#include <linux/atomic.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
static const struct io_pgtable_init_fns *
|
||||
io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
|
||||
|
|
@ -24,8 +28,13 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
|
|||
#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S
|
||||
[ARM_V7S] = &io_pgtable_arm_v7s_init_fns,
|
||||
#endif
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
[ARM_V8L_FAST] = &io_pgtable_av8l_fast_init_fns,
|
||||
#endif
|
||||
};
|
||||
|
||||
static struct dentry *io_pgtable_top;
|
||||
|
||||
struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
|
||||
struct io_pgtable_cfg *cfg,
|
||||
void *cookie)
|
||||
|
|
@ -68,3 +77,59 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops)
|
|||
io_pgtable_init_table[iop->fmt]->free(iop);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(free_io_pgtable_ops);
|
||||
|
||||
static atomic_t pages_allocated;
|
||||
|
||||
void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
|
||||
size_t size, gfp_t gfp_mask)
|
||||
{
|
||||
void *ret;
|
||||
struct msm_iommu_gather_ops *ops = to_msm_iommu_gather_ops(cfg->tlb);
|
||||
|
||||
if (ops->alloc_pages_exact)
|
||||
ret = ops->alloc_pages_exact(cookie, size, gfp_mask);
|
||||
else
|
||||
ret = alloc_pages_exact(size, gfp_mask);
|
||||
|
||||
if (likely(ret))
|
||||
atomic_add(1 << get_order(size), &pages_allocated);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
|
||||
void *virt, size_t size)
|
||||
{
|
||||
struct msm_iommu_gather_ops *ops = to_msm_iommu_gather_ops(cfg->tlb);
|
||||
|
||||
if (ops->free_pages_exact)
|
||||
ops->free_pages_exact(cookie, virt, size);
|
||||
else
|
||||
free_pages_exact(virt, size);
|
||||
|
||||
atomic_sub(1 << get_order(size), &pages_allocated);
|
||||
}
|
||||
|
||||
static int io_pgtable_init(void)
|
||||
{
|
||||
io_pgtable_top = debugfs_create_dir("io-pgtable", iommu_debugfs_top);
|
||||
|
||||
if (!io_pgtable_top)
|
||||
return -ENODEV;
|
||||
|
||||
if (!debugfs_create_atomic_t("pages", 0600,
|
||||
io_pgtable_top, &pages_allocated)) {
|
||||
debugfs_remove_recursive(io_pgtable_top);
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void io_pgtable_exit(void)
|
||||
{
|
||||
debugfs_remove_recursive(io_pgtable_top);
|
||||
}
|
||||
|
||||
module_init(io_pgtable_init);
|
||||
module_exit(io_pgtable_exit);
|
||||
|
|
|
|||
2378
drivers/iommu/iommu-debug.c
Normal file
2378
drivers/iommu/iommu-debug.c
Normal file
File diff suppressed because it is too large
Load diff
27
drivers/iommu/iommu-debug.h
Normal file
27
drivers/iommu/iommu-debug.h
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef IOMMU_DEBUG_H
|
||||
#define IOMMU_DEBUG_H
|
||||
|
||||
#ifdef CONFIG_IOMMU_DEBUG_TRACKING
|
||||
|
||||
void iommu_debug_attach_device(struct iommu_domain *domain, struct device *dev);
|
||||
void iommu_debug_domain_remove(struct iommu_domain *domain);
|
||||
|
||||
#else /* !CONFIG_IOMMU_DEBUG_TRACKING */
|
||||
|
||||
static inline void iommu_debug_attach_device(struct iommu_domain *domain,
|
||||
struct device *dev)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void iommu_debug_domain_remove(struct iommu_domain *domain)
|
||||
{
|
||||
}
|
||||
|
||||
#endif /* CONFIG_IOMMU_DEBUG_TRACKING */
|
||||
|
||||
#endif /* IOMMU_DEBUG_H */
|
||||
|
|
@ -32,6 +32,7 @@ void iommu_debugfs_setup(void)
|
|||
{
|
||||
if (!iommu_debugfs_dir) {
|
||||
iommu_debugfs_dir = debugfs_create_dir("iommu", NULL);
|
||||
iommu_debugfs_top = iommu_debugfs_dir;
|
||||
pr_warn("\n");
|
||||
pr_warn("*************************************************************\n");
|
||||
pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
|
||||
|
|
|
|||
|
|
@ -20,10 +20,13 @@
|
|||
#include <linux/err.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/bitops.h>
|
||||
#include <linux/debugfs.h>
|
||||
#include <linux/property.h>
|
||||
#include <linux/fsl/mc.h>
|
||||
#include <trace/events/iommu.h>
|
||||
|
||||
#include "iommu-debug.h"
|
||||
|
||||
static struct kset *iommu_group_kset;
|
||||
static DEFINE_IDA(iommu_group_ida);
|
||||
#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
|
||||
|
|
@ -1327,6 +1330,7 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc);
|
|||
|
||||
void iommu_domain_free(struct iommu_domain *domain)
|
||||
{
|
||||
iommu_debug_domain_remove(domain);
|
||||
domain->ops->domain_free(domain);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_domain_free);
|
||||
|
|
@ -1343,8 +1347,10 @@ static int __iommu_attach_device(struct iommu_domain *domain,
|
|||
return -ENODEV;
|
||||
|
||||
ret = domain->ops->attach_dev(domain, dev);
|
||||
if (!ret)
|
||||
if (!ret) {
|
||||
trace_attach_device_to_domain(dev);
|
||||
iommu_debug_attach_device(domain, dev);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -1460,9 +1466,6 @@ static int __iommu_attach_group(struct iommu_domain *domain,
|
|||
{
|
||||
int ret;
|
||||
|
||||
if (group->default_domain && group->domain != group->default_domain)
|
||||
return -EBUSY;
|
||||
|
||||
ret = __iommu_group_for_each_dev(group, domain,
|
||||
iommu_group_do_attach_device);
|
||||
if (ret == 0)
|
||||
|
|
@ -1492,28 +1495,18 @@ static int iommu_group_do_detach_device(struct device *dev, void *data)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Although upstream implements detaching the default_domain as a noop,
|
||||
* the "SID switch" secure usecase require complete removal of SIDS/SMRS
|
||||
* from HLOS iommu registers.
|
||||
*/
|
||||
static void __iommu_detach_group(struct iommu_domain *domain,
|
||||
struct iommu_group *group)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (!group->default_domain) {
|
||||
__iommu_group_for_each_dev(group, domain,
|
||||
__iommu_group_for_each_dev(group, domain,
|
||||
iommu_group_do_detach_device);
|
||||
group->domain = NULL;
|
||||
return;
|
||||
}
|
||||
|
||||
if (group->domain == group->default_domain)
|
||||
return;
|
||||
|
||||
/* Detach by re-attaching to the default domain */
|
||||
ret = __iommu_group_for_each_dev(group, group->default_domain,
|
||||
iommu_group_do_attach_device);
|
||||
if (ret != 0)
|
||||
WARN_ON(1);
|
||||
else
|
||||
group->domain = group->default_domain;
|
||||
group->domain = NULL;
|
||||
return;
|
||||
}
|
||||
|
||||
void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)
|
||||
|
|
@ -1533,8 +1526,40 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
|
||||
|
||||
static size_t iommu_pgsize(struct iommu_domain *domain,
|
||||
unsigned long addr_merge, size_t size)
|
||||
phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||
dma_addr_t iova)
|
||||
{
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (unlikely(ops->iova_to_phys_hard == NULL))
|
||||
return 0;
|
||||
|
||||
return ops->iova_to_phys_hard(domain, iova);
|
||||
}
|
||||
|
||||
uint64_t iommu_iova_to_pte(struct iommu_domain *domain,
|
||||
dma_addr_t iova)
|
||||
{
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (unlikely(ops->iova_to_pte == NULL))
|
||||
return 0;
|
||||
|
||||
return ops->iova_to_pte(domain, iova);
|
||||
}
|
||||
|
||||
bool iommu_is_iova_coherent(struct iommu_domain *domain, dma_addr_t iova)
|
||||
{
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (unlikely(ops->is_iova_coherent == NULL))
|
||||
return false;
|
||||
|
||||
return ops->is_iova_coherent(domain, iova);
|
||||
}
|
||||
|
||||
size_t iommu_pgsize(unsigned long pgsize_bitmap,
|
||||
unsigned long addr_merge, size_t size)
|
||||
{
|
||||
unsigned int pgsize_idx;
|
||||
size_t pgsize;
|
||||
|
|
@ -1553,10 +1578,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain,
|
|||
pgsize = (1UL << (pgsize_idx + 1)) - 1;
|
||||
|
||||
/* throw away page sizes not supported by the hardware */
|
||||
pgsize &= domain->pgsize_bitmap;
|
||||
pgsize &= pgsize_bitmap;
|
||||
|
||||
/* make sure we're still sane */
|
||||
BUG_ON(!pgsize);
|
||||
if (!pgsize) {
|
||||
pr_err("invalid pgsize/addr/size! 0x%lx 0x%lx 0x%zx\n",
|
||||
pgsize_bitmap, addr_merge, size);
|
||||
BUG();
|
||||
}
|
||||
|
||||
/* pick the biggest page */
|
||||
pgsize_idx = __fls(pgsize);
|
||||
|
|
@ -1599,7 +1628,8 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
|
|||
pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
|
||||
|
||||
while (size) {
|
||||
size_t pgsize = iommu_pgsize(domain, iova | paddr, size);
|
||||
size_t pgsize = iommu_pgsize(domain->pgsize_bitmap,
|
||||
iova | paddr, size);
|
||||
|
||||
pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
|
||||
iova, &paddr, pgsize);
|
||||
|
|
@ -1620,7 +1650,8 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
|
|||
if (ret)
|
||||
iommu_unmap(domain, orig_iova, orig_size - size);
|
||||
else
|
||||
trace_map(orig_iova, orig_paddr, orig_size);
|
||||
trace_map(to_msm_iommu_domain(domain), orig_iova, orig_paddr,
|
||||
orig_size, prot);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -1663,14 +1694,14 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
|
|||
* or we hit an area that isn't mapped.
|
||||
*/
|
||||
while (unmapped < size) {
|
||||
size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
|
||||
size_t left = size - unmapped;
|
||||
|
||||
unmapped_page = ops->unmap(domain, iova, pgsize);
|
||||
unmapped_page = ops->unmap(domain, iova, left);
|
||||
if (!unmapped_page)
|
||||
break;
|
||||
|
||||
if (sync && ops->iotlb_range_add)
|
||||
ops->iotlb_range_add(domain, iova, pgsize);
|
||||
ops->iotlb_range_add(domain, iova, left);
|
||||
|
||||
pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
|
||||
iova, unmapped_page);
|
||||
|
|
@ -1682,7 +1713,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
|
|||
if (sync && ops->iotlb_sync)
|
||||
ops->iotlb_sync(domain);
|
||||
|
||||
trace_unmap(orig_iova, size, unmapped);
|
||||
trace_unmap(to_msm_iommu_domain(domain), orig_iova, size, unmapped);
|
||||
return unmapped;
|
||||
}
|
||||
|
||||
|
|
@ -1702,6 +1733,19 @@ EXPORT_SYMBOL_GPL(iommu_unmap_fast);
|
|||
|
||||
size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
|
||||
struct scatterlist *sg, unsigned int nents, int prot)
|
||||
{
|
||||
size_t mapped = 0;
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (ops->map_sg)
|
||||
mapped = ops->map_sg(domain, iova, sg, nents, prot);
|
||||
trace_map_sg(to_msm_iommu_domain(domain), iova, mapped, prot);
|
||||
return mapped;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_map_sg);
|
||||
|
||||
size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
|
||||
struct scatterlist *sg, unsigned int nents, int prot)
|
||||
{
|
||||
size_t len = 0, mapped = 0;
|
||||
phys_addr_t start;
|
||||
|
|
@ -1740,7 +1784,7 @@ out_err:
|
|||
return 0;
|
||||
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(iommu_map_sg);
|
||||
EXPORT_SYMBOL(default_iommu_map_sg);
|
||||
|
||||
int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
|
||||
phys_addr_t paddr, u64 size, int prot)
|
||||
|
|
@ -1804,6 +1848,8 @@ int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(report_iommu_fault);
|
||||
|
||||
struct dentry *iommu_debugfs_top;
|
||||
|
||||
static int __init iommu_init(void)
|
||||
{
|
||||
iommu_group_kset = kset_create_and_add("iommu_groups",
|
||||
|
|
@ -1877,6 +1923,23 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list)
|
|||
ops->put_resv_regions(dev, list);
|
||||
}
|
||||
|
||||
/**
|
||||
* iommu_trigger_fault() - trigger an IOMMU fault
|
||||
* @domain: iommu domain
|
||||
*
|
||||
* Triggers a fault on the device to which this domain is attached.
|
||||
*
|
||||
* This function should only be used for debugging purposes, for obvious
|
||||
* reasons.
|
||||
*/
|
||||
void iommu_trigger_fault(struct iommu_domain *domain, unsigned long flags)
|
||||
{
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (ops->trigger_fault)
|
||||
ops->trigger_fault(domain, flags);
|
||||
}
|
||||
|
||||
struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
|
||||
size_t length, int prot,
|
||||
enum iommu_resv_type type)
|
||||
|
|
|
|||
478
drivers/iommu/msm_dma_iommu_mapping.c
Normal file
478
drivers/iommu/msm_dma_iommu_mapping.c
Normal file
|
|
@ -0,0 +1,478 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/kref.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/rbtree.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/err.h>
|
||||
#include <asm/barrier.h>
|
||||
|
||||
#include <linux/msm_dma_iommu_mapping.h>
|
||||
|
||||
/**
|
||||
* struct msm_iommu_map - represents a mapping of an ion buffer to an iommu
|
||||
* @lnode - list node to exist in the buffer's list of iommu mappings
|
||||
* @dev - Device this is mapped to. Used as key
|
||||
* @sgl - The scatterlist for this mapping
|
||||
* @nents - Number of entries in sgl
|
||||
* @dir - The direction for the map.
|
||||
* @meta - Backpointer to the meta this guy belongs to.
|
||||
* @ref - for reference counting this mapping
|
||||
* @attrs - dma mapping attributes
|
||||
* @buf_start_addr - address of start of buffer
|
||||
*
|
||||
* Represents a mapping of one dma_buf buffer to a particular device
|
||||
* and address range. There may exist other mappings of this buffer in
|
||||
* different devices. All mappings will have the same cacheability and security.
|
||||
*/
|
||||
struct msm_iommu_map {
|
||||
struct list_head lnode;
|
||||
struct rb_node node;
|
||||
struct device *dev;
|
||||
struct scatterlist *sgl;
|
||||
unsigned int nents;
|
||||
enum dma_data_direction dir;
|
||||
struct msm_iommu_meta *meta;
|
||||
struct kref ref;
|
||||
unsigned long attrs;
|
||||
dma_addr_t buf_start_addr;
|
||||
};
|
||||
|
||||
struct msm_iommu_meta {
|
||||
struct rb_node node;
|
||||
struct list_head iommu_maps;
|
||||
struct kref ref;
|
||||
struct mutex lock;
|
||||
void *buffer;
|
||||
};
|
||||
|
||||
static struct rb_root iommu_root;
|
||||
static DEFINE_MUTEX(msm_iommu_map_mutex);
|
||||
|
||||
static void msm_iommu_meta_add(struct msm_iommu_meta *meta)
|
||||
{
|
||||
struct rb_root *root = &iommu_root;
|
||||
struct rb_node **p = &root->rb_node;
|
||||
struct rb_node *parent = NULL;
|
||||
struct msm_iommu_meta *entry;
|
||||
|
||||
while (*p) {
|
||||
parent = *p;
|
||||
entry = rb_entry(parent, struct msm_iommu_meta, node);
|
||||
|
||||
if (meta->buffer < entry->buffer)
|
||||
p = &(*p)->rb_left;
|
||||
else if (meta->buffer > entry->buffer)
|
||||
p = &(*p)->rb_right;
|
||||
else
|
||||
pr_err("%s: dma_buf %pK already exists\n", __func__,
|
||||
entry->buffer);
|
||||
}
|
||||
|
||||
rb_link_node(&meta->node, parent, p);
|
||||
rb_insert_color(&meta->node, root);
|
||||
}
|
||||
|
||||
static struct msm_iommu_meta *msm_iommu_meta_lookup(void *buffer)
|
||||
{
|
||||
struct rb_root *root = &iommu_root;
|
||||
struct rb_node **p = &root->rb_node;
|
||||
struct rb_node *parent = NULL;
|
||||
struct msm_iommu_meta *entry = NULL;
|
||||
|
||||
while (*p) {
|
||||
parent = *p;
|
||||
entry = rb_entry(parent, struct msm_iommu_meta, node);
|
||||
|
||||
if (buffer < entry->buffer)
|
||||
p = &(*p)->rb_left;
|
||||
else if (buffer > entry->buffer)
|
||||
p = &(*p)->rb_right;
|
||||
else
|
||||
return entry;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void msm_iommu_add(struct msm_iommu_meta *meta,
|
||||
struct msm_iommu_map *iommu)
|
||||
{
|
||||
INIT_LIST_HEAD(&iommu->lnode);
|
||||
list_add(&iommu->lnode, &meta->iommu_maps);
|
||||
}
|
||||
|
||||
|
||||
static struct msm_iommu_map *msm_iommu_lookup(struct msm_iommu_meta *meta,
|
||||
struct device *dev)
|
||||
{
|
||||
struct msm_iommu_map *entry;
|
||||
|
||||
list_for_each_entry(entry, &meta->iommu_maps, lnode) {
|
||||
if (entry->dev == dev)
|
||||
return entry;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf)
|
||||
{
|
||||
struct msm_iommu_meta *meta;
|
||||
|
||||
meta = kzalloc(sizeof(*meta), GFP_KERNEL);
|
||||
|
||||
if (!meta)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
INIT_LIST_HEAD(&meta->iommu_maps);
|
||||
meta->buffer = dma_buf->priv;
|
||||
kref_init(&meta->ref);
|
||||
mutex_init(&meta->lock);
|
||||
msm_iommu_meta_add(meta);
|
||||
|
||||
return meta;
|
||||
}
|
||||
|
||||
static void msm_iommu_meta_put(struct msm_iommu_meta *meta);
|
||||
|
||||
static struct scatterlist *clone_sgl(struct scatterlist *sg, int nents)
|
||||
{
|
||||
struct scatterlist *next, *s;
|
||||
int i;
|
||||
struct sg_table table;
|
||||
|
||||
if (sg_alloc_table(&table, nents, GFP_KERNEL))
|
||||
return NULL;
|
||||
next = table.sgl;
|
||||
for_each_sg(sg, s, nents, i) {
|
||||
*next = *s;
|
||||
next = sg_next(next);
|
||||
}
|
||||
return table.sgl;
|
||||
}
|
||||
|
||||
static inline int __msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_meta *iommu_meta = NULL;
|
||||
int ret = 0;
|
||||
bool extra_meta_ref_taken = false;
|
||||
int late_unmap = !(attrs & DMA_ATTR_NO_DELAYED_UNMAP);
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
iommu_meta = msm_iommu_meta_lookup(dma_buf->priv);
|
||||
|
||||
if (!iommu_meta) {
|
||||
iommu_meta = msm_iommu_meta_create(dma_buf);
|
||||
|
||||
if (IS_ERR(iommu_meta)) {
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
ret = PTR_ERR(iommu_meta);
|
||||
goto out;
|
||||
}
|
||||
if (late_unmap) {
|
||||
kref_get(&iommu_meta->ref);
|
||||
extra_meta_ref_taken = true;
|
||||
}
|
||||
} else {
|
||||
kref_get(&iommu_meta->ref);
|
||||
}
|
||||
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
mutex_lock(&iommu_meta->lock);
|
||||
iommu_map = msm_iommu_lookup(iommu_meta, dev);
|
||||
if (!iommu_map) {
|
||||
iommu_map = kmalloc(sizeof(*iommu_map), GFP_KERNEL);
|
||||
|
||||
if (!iommu_map) {
|
||||
ret = -ENOMEM;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
ret = dma_map_sg_attrs(dev, sg, nents, dir, attrs);
|
||||
if (!ret) {
|
||||
kfree(iommu_map);
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
iommu_map->sgl = clone_sgl(sg, nents);
|
||||
if (!iommu_map->sgl) {
|
||||
kfree(iommu_map);
|
||||
ret = -ENOMEM;
|
||||
goto out_unlock;
|
||||
}
|
||||
iommu_map->nents = nents;
|
||||
iommu_map->dev = dev;
|
||||
iommu_map->dir = dir;
|
||||
iommu_map->attrs = attrs;
|
||||
iommu_map->buf_start_addr = sg_phys(sg);
|
||||
|
||||
kref_init(&iommu_map->ref);
|
||||
if (late_unmap)
|
||||
kref_get(&iommu_map->ref);
|
||||
iommu_map->meta = iommu_meta;
|
||||
msm_iommu_add(iommu_meta, iommu_map);
|
||||
|
||||
} else {
|
||||
if (nents == iommu_map->nents &&
|
||||
dir == iommu_map->dir &&
|
||||
(attrs & ~DMA_ATTR_SKIP_CPU_SYNC) ==
|
||||
(iommu_map->attrs & ~DMA_ATTR_SKIP_CPU_SYNC) &&
|
||||
sg_phys(sg) == iommu_map->buf_start_addr) {
|
||||
struct scatterlist *sg_tmp = sg;
|
||||
struct scatterlist *map_sg;
|
||||
int i;
|
||||
|
||||
for_each_sg(iommu_map->sgl, map_sg, nents, i) {
|
||||
sg_dma_address(sg_tmp) = sg_dma_address(map_sg);
|
||||
sg_dma_len(sg_tmp) = sg_dma_len(map_sg);
|
||||
if (sg_dma_len(map_sg) == 0)
|
||||
break;
|
||||
|
||||
sg_tmp = sg_next(sg_tmp);
|
||||
if (sg_tmp == NULL)
|
||||
break;
|
||||
}
|
||||
|
||||
kref_get(&iommu_map->ref);
|
||||
|
||||
if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
|
||||
dma_sync_sg_for_device(dev, iommu_map->sgl,
|
||||
iommu_map->nents, iommu_map->dir);
|
||||
|
||||
if (is_device_dma_coherent(dev))
|
||||
/*
|
||||
* Ensure all outstanding changes for coherent
|
||||
* buffers are applied to the cache before any
|
||||
* DMA occurs.
|
||||
*/
|
||||
dmb(ish);
|
||||
ret = nents;
|
||||
} else {
|
||||
bool start_diff = (sg_phys(sg) !=
|
||||
iommu_map->buf_start_addr);
|
||||
|
||||
dev_err(dev, "lazy map request differs:\n"
|
||||
"req dir:%d, original dir:%d\n"
|
||||
"req nents:%d, original nents:%d\n"
|
||||
"req map attrs:%lu, original map attrs:%lu\n"
|
||||
"req buffer start address differs:%d\n",
|
||||
dir, iommu_map->dir, nents,
|
||||
iommu_map->nents, attrs, iommu_map->attrs,
|
||||
start_diff);
|
||||
ret = -EINVAL;
|
||||
}
|
||||
}
|
||||
mutex_unlock(&iommu_meta->lock);
|
||||
return ret;
|
||||
|
||||
out_unlock:
|
||||
mutex_unlock(&iommu_meta->lock);
|
||||
out:
|
||||
if (!IS_ERR(iommu_meta)) {
|
||||
if (extra_meta_ref_taken)
|
||||
msm_iommu_meta_put(iommu_meta);
|
||||
msm_iommu_meta_put(iommu_meta);
|
||||
}
|
||||
return ret;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* We are not taking a reference to the dma_buf here. It is expected that
|
||||
* clients hold reference to the dma_buf until they are done with mapping and
|
||||
* unmapping.
|
||||
*/
|
||||
int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, struct dma_buf *dma_buf,
|
||||
unsigned long attrs)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (IS_ERR_OR_NULL(dev)) {
|
||||
pr_err("%s: dev pointer is invalid\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (IS_ERR_OR_NULL(sg)) {
|
||||
pr_err("%s: sg table pointer is invalid\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (IS_ERR_OR_NULL(dma_buf)) {
|
||||
pr_err("%s: dma_buf pointer is invalid\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
ret = __msm_dma_map_sg(dev, sg, nents, dir, dma_buf, attrs);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_map_sg_attrs);
|
||||
|
||||
static void msm_iommu_meta_destroy(struct kref *kref)
|
||||
{
|
||||
struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta,
|
||||
ref);
|
||||
|
||||
if (!list_empty(&meta->iommu_maps)) {
|
||||
WARN(1, "%s: DMA Buffer %pK being destroyed with outstanding iommu mappings!\n",
|
||||
__func__, meta->buffer);
|
||||
}
|
||||
rb_erase(&meta->node, &iommu_root);
|
||||
kfree(meta);
|
||||
}
|
||||
|
||||
static void msm_iommu_meta_put(struct msm_iommu_meta *meta)
|
||||
{
|
||||
/*
|
||||
* Need to lock here to prevent race against map/unmap
|
||||
*/
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
kref_put(&meta->ref, msm_iommu_meta_destroy);
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
}
|
||||
|
||||
static void msm_iommu_map_release(struct kref *kref)
|
||||
{
|
||||
struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map,
|
||||
ref);
|
||||
struct sg_table table;
|
||||
|
||||
table.nents = table.orig_nents = map->nents;
|
||||
table.sgl = map->sgl;
|
||||
list_del(&map->lnode);
|
||||
|
||||
/* Skip an additional cache maintenance on the dma unmap path */
|
||||
if (!(map->attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
map->attrs |= DMA_ATTR_SKIP_CPU_SYNC;
|
||||
dma_unmap_sg_attrs(map->dev, map->sgl, map->nents, map->dir,
|
||||
map->attrs);
|
||||
sg_free_table(&table);
|
||||
kfree(map);
|
||||
}
|
||||
|
||||
void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_meta *meta;
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
meta = msm_iommu_meta_lookup(dma_buf->priv);
|
||||
if (!meta) {
|
||||
WARN(1, "%s: (%pK) was never mapped\n", __func__, dma_buf);
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
goto out;
|
||||
|
||||
}
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
mutex_lock(&meta->lock);
|
||||
iommu_map = msm_iommu_lookup(meta, dev);
|
||||
|
||||
if (!iommu_map) {
|
||||
WARN(1, "%s: (%pK) was never mapped for device %p\n", __func__,
|
||||
dma_buf, dev);
|
||||
mutex_unlock(&meta->lock);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (dir != iommu_map->dir)
|
||||
WARN(1, "%s: (%pK) dir:%d differs from original dir:%d\n",
|
||||
__func__, dma_buf, dir, iommu_map->dir);
|
||||
|
||||
if (attrs && ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0))
|
||||
dma_sync_sg_for_cpu(dev, iommu_map->sgl, iommu_map->nents, dir);
|
||||
|
||||
iommu_map->attrs = attrs;
|
||||
kref_put(&iommu_map->ref, msm_iommu_map_release);
|
||||
mutex_unlock(&meta->lock);
|
||||
|
||||
msm_iommu_meta_put(meta);
|
||||
|
||||
out:
|
||||
return;
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_unmap_sg_attrs);
|
||||
|
||||
int msm_dma_unmap_all_for_dev(struct device *dev)
|
||||
{
|
||||
int ret = 0;
|
||||
struct msm_iommu_meta *meta;
|
||||
struct rb_root *root;
|
||||
struct rb_node *meta_node;
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
root = &iommu_root;
|
||||
meta_node = rb_first(root);
|
||||
while (meta_node) {
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_map *iommu_map_next;
|
||||
|
||||
meta = rb_entry(meta_node, struct msm_iommu_meta, node);
|
||||
mutex_lock(&meta->lock);
|
||||
list_for_each_entry_safe(iommu_map, iommu_map_next,
|
||||
&meta->iommu_maps, lnode)
|
||||
if (iommu_map->dev == dev)
|
||||
if (!kref_put(&iommu_map->ref,
|
||||
msm_iommu_map_release))
|
||||
ret = -EINVAL;
|
||||
|
||||
mutex_unlock(&meta->lock);
|
||||
meta_node = rb_next(meta_node);
|
||||
}
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_unmap_all_for_dev);
|
||||
|
||||
/*
|
||||
* Only to be called by ION code when a buffer is freed
|
||||
*/
|
||||
void msm_dma_buf_freed(void *buffer)
|
||||
{
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_map *iommu_map_next;
|
||||
struct msm_iommu_meta *meta;
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
meta = msm_iommu_meta_lookup(buffer);
|
||||
if (!meta) {
|
||||
/* Already unmapped (assuming no late unmapping) */
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
return;
|
||||
}
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
mutex_lock(&meta->lock);
|
||||
|
||||
list_for_each_entry_safe(iommu_map, iommu_map_next, &meta->iommu_maps,
|
||||
lnode)
|
||||
kref_put(&iommu_map->ref, msm_iommu_map_release);
|
||||
|
||||
if (!list_empty(&meta->iommu_maps)) {
|
||||
WARN(1, "%s: DMA buffer %pK destroyed with outstanding iommu mappings\n",
|
||||
__func__, meta->buffer);
|
||||
}
|
||||
|
||||
INIT_LIST_HEAD(&meta->iommu_maps);
|
||||
mutex_unlock(&meta->lock);
|
||||
|
||||
msm_iommu_meta_put(meta);
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_buf_freed);
|
||||
|
||||
MODULE_LICENSE("GPL v2");
|
||||
|
|
@ -104,8 +104,11 @@ static int of_iommu_xlate(struct device *dev,
|
|||
* IOMMU device we're waiting for, which will be useful if we ever get
|
||||
* a proper probe-ordering dependency mechanism in future.
|
||||
*/
|
||||
if (!ops)
|
||||
if (!ops) {
|
||||
if (IS_ENABLED(CONFIG_MODULES))
|
||||
return -EPROBE_DEFER;
|
||||
return driver_deferred_probe_check_state(dev);
|
||||
}
|
||||
|
||||
return ops->of_xlate(dev, iommu_spec);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -197,4 +197,13 @@ config QCOM_SECURE_CHAN_MANAGER
|
|||
support for Qualcomm SoC. SCM provides communication
|
||||
channel to communicate with secure world (EL2 and EL3)
|
||||
by using smc call.
|
||||
|
||||
config QCOM_SECURE_BUFFER
|
||||
tristate "Helper functions for secure buffers through TZ"
|
||||
depends on QCOM_SECURE_CHAN_MANAGER
|
||||
help
|
||||
Enable for targets that need to call into TZ to secure
|
||||
memory buffers. This ensures that only the correct clients can
|
||||
use this memory and no unauthorized access is made to the
|
||||
buffer.
|
||||
endmenu
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ obj-$(CONFIG_QCOM_SMP2P) += smp2p.o
|
|||
obj-$(CONFIG_QCOM_SMSM) += smsm.o
|
||||
CFLAGS_scm.o :=$(call as-instr,.arch_extension sec,-DREQUIRES_SEC=1, -Wno-asm-operand-widths)
|
||||
obj-$(CONFIG_QCOM_SECURE_CHAN_MANAGER) += scm.o
|
||||
obj-$(CONFIG_QCOM_SECURE_BUFFER) += secure_buffer.o
|
||||
obj-$(CONFIG_QCOM_WCNSS_CTRL) += wcnss_ctrl.o
|
||||
obj-$(CONFIG_QCOM_APR) += apr.o
|
||||
obj-$(CONFIG_QCOM_LLCC) += llcc-slice.o
|
||||
|
|
|
|||
330
drivers/soc/qcom/secure_buffer.c
Normal file
330
drivers/soc/qcom/secure_buffer.c
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (C) 2011 Google, Inc
|
||||
* Copyright (c) 2011-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/highmem.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/kref.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/of.h>
|
||||
#include <soc/qcom/scm.h>
|
||||
#include <soc/qcom/secure_buffer.h>
|
||||
|
||||
#define MEM_PROT_ASSIGN_ID 0x16
|
||||
#define BATCH_MAX_SIZE SZ_2M
|
||||
#define BATCH_MAX_SECTIONS 32
|
||||
|
||||
struct dest_vm_and_perm_info {
|
||||
u32 vm;
|
||||
u32 perm;
|
||||
u64 ctx;
|
||||
u32 ctx_size;
|
||||
};
|
||||
|
||||
struct mem_prot_info {
|
||||
phys_addr_t addr;
|
||||
u64 size;
|
||||
};
|
||||
|
||||
static struct device *qcom_secure_buffer_dev;
|
||||
|
||||
static struct dest_vm_and_perm_info *
|
||||
populate_dest_info(int *dest_vmids, int nelements, int *dest_perms,
|
||||
size_t *size_in_bytes)
|
||||
{
|
||||
struct dest_vm_and_perm_info *dest_info;
|
||||
int i;
|
||||
size_t size;
|
||||
|
||||
/* Ensure allocated size is less than PAGE_ALLOC_COSTLY_ORDER */
|
||||
size = nelements * sizeof(*dest_info);
|
||||
if (size > PAGE_SIZE)
|
||||
return NULL;
|
||||
|
||||
dest_info = kzalloc(size, GFP_KERNEL);
|
||||
if (!dest_info)
|
||||
return NULL;
|
||||
|
||||
for (i = 0; i < nelements; i++) {
|
||||
dest_info[i].vm = dest_vmids[i];
|
||||
dest_info[i].perm = dest_perms[i];
|
||||
dest_info[i].ctx = 0x0;
|
||||
dest_info[i].ctx_size = 0;
|
||||
}
|
||||
|
||||
*size_in_bytes = size;
|
||||
return dest_info;
|
||||
}
|
||||
|
||||
static unsigned int get_batches_from_sgl(struct mem_prot_info *sg_table_copy,
|
||||
struct scatterlist *sgl,
|
||||
struct scatterlist **next_sgl)
|
||||
{
|
||||
u64 batch_size = 0;
|
||||
unsigned int i = 0;
|
||||
struct scatterlist *curr_sgl = sgl;
|
||||
|
||||
/* Ensure no zero size batches */
|
||||
do {
|
||||
sg_table_copy[i].addr = page_to_phys(sg_page(curr_sgl));
|
||||
sg_table_copy[i].size = curr_sgl->length;
|
||||
batch_size += sg_table_copy[i].size;
|
||||
curr_sgl = sg_next(curr_sgl);
|
||||
i++;
|
||||
} while (curr_sgl && i < BATCH_MAX_SECTIONS &&
|
||||
curr_sgl->length + batch_size < BATCH_MAX_SIZE);
|
||||
|
||||
*next_sgl = curr_sgl;
|
||||
return i;
|
||||
}
|
||||
|
||||
static int batched_hyp_assign(struct sg_table *table, struct scm_desc *desc)
|
||||
{
|
||||
unsigned int entries_size;
|
||||
unsigned int batch_start = 0;
|
||||
unsigned int batches_processed;
|
||||
struct scatterlist *curr_sgl = table->sgl;
|
||||
struct scatterlist *next_sgl;
|
||||
int ret = 0;
|
||||
struct mem_prot_info *sg_table_copy = kcalloc(BATCH_MAX_SECTIONS,
|
||||
sizeof(*sg_table_copy),
|
||||
GFP_KERNEL);
|
||||
dma_addr_t entries_dma_addr;
|
||||
|
||||
if (!sg_table_copy)
|
||||
return -ENOMEM;
|
||||
|
||||
while (batch_start < table->nents) {
|
||||
batches_processed = get_batches_from_sgl(sg_table_copy,
|
||||
curr_sgl, &next_sgl);
|
||||
curr_sgl = next_sgl;
|
||||
entries_size = batches_processed * sizeof(*sg_table_copy);
|
||||
entries_dma_addr = dma_map_single(qcom_secure_buffer_dev,
|
||||
sg_table_copy, entries_size,
|
||||
DMA_TO_DEVICE);
|
||||
if (dma_mapping_error(qcom_secure_buffer_dev,
|
||||
entries_dma_addr)) {
|
||||
ret = -EADDRNOTAVAIL;
|
||||
break;
|
||||
}
|
||||
desc->args[0] = entries_dma_addr;
|
||||
desc->args[1] = entries_size;
|
||||
|
||||
ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP,
|
||||
MEM_PROT_ASSIGN_ID), desc);
|
||||
dma_unmap_single(qcom_secure_buffer_dev, entries_dma_addr,
|
||||
entries_size, DMA_TO_DEVICE);
|
||||
if (ret) {
|
||||
pr_info("%s: Failed to assign memory protection, ret = %d\n",
|
||||
__func__, ret);
|
||||
/*
|
||||
* Make it clear to clients that the memory may no
|
||||
* longer be in a usable state.
|
||||
*/
|
||||
ret = -EADDRNOTAVAIL;
|
||||
break;
|
||||
}
|
||||
|
||||
batch_start += batches_processed;
|
||||
}
|
||||
|
||||
kfree(sg_table_copy);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* When -EADDRNOTAVAIL is returned the memory may no longer be in
|
||||
* a usable state and should no longer be accessed by the HLOS.
|
||||
*/
|
||||
int hyp_assign_table(struct sg_table *table,
|
||||
u32 *source_vm_list, int source_nelems,
|
||||
int *dest_vmids, int *dest_perms,
|
||||
int dest_nelems)
|
||||
{
|
||||
int ret = 0;
|
||||
struct scm_desc desc = {0};
|
||||
u32 *source_vm_copy;
|
||||
size_t source_vm_copy_size;
|
||||
struct dest_vm_and_perm_info *dest_vm_copy;
|
||||
size_t dest_vm_copy_size;
|
||||
dma_addr_t source_dma_addr, dest_dma_addr;
|
||||
|
||||
if (!qcom_secure_buffer_dev)
|
||||
return -EPROBE_DEFER;
|
||||
|
||||
if (!table || !table->sgl || !source_vm_list || !source_nelems ||
|
||||
!dest_vmids || !dest_perms || !dest_nelems)
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* We can only pass cache-aligned sizes to hypervisor, so we need
|
||||
* to kmalloc and memcpy the source_vm_list here.
|
||||
*/
|
||||
source_vm_copy_size = sizeof(*source_vm_copy) * source_nelems;
|
||||
source_vm_copy = kmemdup(source_vm_list, source_vm_copy_size,
|
||||
GFP_KERNEL);
|
||||
if (!source_vm_copy)
|
||||
return -ENOMEM;
|
||||
|
||||
source_dma_addr = dma_map_single(qcom_secure_buffer_dev, source_vm_copy,
|
||||
source_vm_copy_size, DMA_TO_DEVICE);
|
||||
if (dma_mapping_error(qcom_secure_buffer_dev, source_dma_addr)) {
|
||||
ret = -ENOMEM;
|
||||
goto out_free_source;
|
||||
}
|
||||
|
||||
dest_vm_copy = populate_dest_info(dest_vmids, dest_nelems, dest_perms,
|
||||
&dest_vm_copy_size);
|
||||
if (!dest_vm_copy) {
|
||||
ret = -ENOMEM;
|
||||
goto out_unmap_source;
|
||||
}
|
||||
|
||||
dest_dma_addr = dma_map_single(qcom_secure_buffer_dev, dest_vm_copy,
|
||||
dest_vm_copy_size, DMA_TO_DEVICE);
|
||||
if (dma_mapping_error(qcom_secure_buffer_dev, dest_dma_addr)) {
|
||||
ret = -ENOMEM;
|
||||
goto out_free_dest;
|
||||
}
|
||||
|
||||
|
||||
desc.args[2] = source_dma_addr;
|
||||
desc.args[3] = source_vm_copy_size;
|
||||
desc.args[4] = dest_dma_addr;
|
||||
desc.args[5] = dest_vm_copy_size;
|
||||
desc.args[6] = 0;
|
||||
|
||||
desc.arginfo = SCM_ARGS(7, SCM_RO, SCM_VAL, SCM_RO, SCM_VAL, SCM_RO,
|
||||
SCM_VAL, SCM_VAL);
|
||||
|
||||
ret = batched_hyp_assign(table, &desc);
|
||||
|
||||
dma_unmap_single(qcom_secure_buffer_dev, dest_dma_addr,
|
||||
dest_vm_copy_size, DMA_TO_DEVICE);
|
||||
out_free_dest:
|
||||
kfree(dest_vm_copy);
|
||||
out_unmap_source:
|
||||
dma_unmap_single(qcom_secure_buffer_dev, source_dma_addr,
|
||||
source_vm_copy_size, DMA_TO_DEVICE);
|
||||
out_free_source:
|
||||
kfree(source_vm_copy);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(hyp_assign_table);
|
||||
|
||||
int hyp_assign_phys(phys_addr_t addr, u64 size, u32 *source_vm_list,
|
||||
int source_nelems, int *dest_vmids,
|
||||
int *dest_perms, int dest_nelems)
|
||||
{
|
||||
struct sg_table table;
|
||||
int ret;
|
||||
|
||||
if (!qcom_secure_buffer_dev)
|
||||
return -EPROBE_DEFER;
|
||||
|
||||
ret = sg_alloc_table(&table, 1, GFP_KERNEL);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
sg_set_page(table.sgl, phys_to_page(addr), size, 0);
|
||||
|
||||
ret = hyp_assign_table(&table, source_vm_list, source_nelems,
|
||||
dest_vmids, dest_perms, dest_nelems);
|
||||
|
||||
sg_free_table(&table);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(hyp_assign_phys);
|
||||
|
||||
const char *msm_secure_vmid_to_string(int secure_vmid)
|
||||
{
|
||||
switch (secure_vmid) {
|
||||
case VMID_HLOS:
|
||||
return "VMID_HLOS";
|
||||
case VMID_CP_TOUCH:
|
||||
return "VMID_CP_TOUCH";
|
||||
case VMID_CP_BITSTREAM:
|
||||
return "VMID_CP_BITSTREAM";
|
||||
case VMID_CP_PIXEL:
|
||||
return "VMID_CP_PIXEL";
|
||||
case VMID_CP_NON_PIXEL:
|
||||
return "VMID_CP_NON_PIXEL";
|
||||
case VMID_CP_CAMERA:
|
||||
return "VMID_CP_CAMERA";
|
||||
case VMID_HLOS_FREE:
|
||||
return "VMID_HLOS_FREE";
|
||||
case VMID_MSS_MSA:
|
||||
return "VMID_MSS_MSA";
|
||||
case VMID_MSS_NONMSA:
|
||||
return "VMID_MSS_NONMSA";
|
||||
case VMID_CP_SEC_DISPLAY:
|
||||
return "VMID_CP_SEC_DISPLAY";
|
||||
case VMID_CP_APP:
|
||||
return "VMID_CP_APP";
|
||||
case VMID_WLAN:
|
||||
return "VMID_WLAN";
|
||||
case VMID_WLAN_CE:
|
||||
return "VMID_WLAN_CE";
|
||||
case VMID_CP_CAMERA_PREVIEW:
|
||||
return "VMID_CP_CAMERA_PREVIEW";
|
||||
case VMID_CP_SPSS_SP:
|
||||
return "VMID_CP_SPSS_SP";
|
||||
case VMID_CP_SPSS_SP_SHARED:
|
||||
return "VMID_CP_SPSS_SP_SHARED";
|
||||
case VMID_CP_SPSS_HLOS_SHARED:
|
||||
return "VMID_CP_SPSS_HLOS_SHARED";
|
||||
case VMID_INVAL:
|
||||
return "VMID_INVAL";
|
||||
default:
|
||||
return "Unknown VMID";
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL(msm_secure_vmid_to_string);
|
||||
|
||||
u32 msm_secure_get_vmid_perms(u32 vmid)
|
||||
{
|
||||
if (vmid == VMID_CP_SEC_DISPLAY)
|
||||
return PERM_READ;
|
||||
else if (vmid == VMID_CP_CDSP)
|
||||
return PERM_READ | PERM_WRITE | PERM_EXEC;
|
||||
else
|
||||
return PERM_READ | PERM_WRITE;
|
||||
}
|
||||
EXPORT_SYMBOL(msm_secure_get_vmid_perms);
|
||||
|
||||
static int qcom_secure_buffer_probe(struct platform_device *pdev)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
qcom_secure_buffer_dev = &pdev->dev;
|
||||
|
||||
#ifdef CONFIG_ARM64
|
||||
ret = dma_set_mask(qcom_secure_buffer_dev, DMA_BIT_MASK(64));
|
||||
#else
|
||||
ret = dma_set_mask(qcom_secure_buffer_dev, DMA_BIT_MASK(32));
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
static const struct of_device_id qcom_secure_buffer_of_match[] = {
|
||||
{.compatible = "qcom,secure-buffer"},
|
||||
{}
|
||||
};
|
||||
MODULE_DEVICE_TABLE(of, qcom_secure_buffer_of_match);
|
||||
|
||||
static struct platform_driver qcom_secure_buffer_driver = {
|
||||
.probe = qcom_secure_buffer_probe,
|
||||
.driver = {
|
||||
.name = "qcom_secure_buffer",
|
||||
.of_match_table = qcom_secure_buffer_of_match,
|
||||
},
|
||||
};
|
||||
module_platform_driver(qcom_secure_buffer_driver);
|
||||
|
||||
MODULE_LICENSE("GPL v2");
|
||||
|
|
@ -14,6 +14,8 @@
|
|||
#include <linux/iommu.h>
|
||||
#include <linux/msi.h>
|
||||
|
||||
struct iova_domain;
|
||||
|
||||
int iommu_dma_init(void);
|
||||
|
||||
/* Domain management interface for IOMMU drivers */
|
||||
|
|
@ -45,6 +47,11 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
|
|||
unsigned long offset, size_t size, int prot);
|
||||
int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, int prot);
|
||||
size_t iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad,
|
||||
struct scatterlist *sg, int nents);
|
||||
int iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, dma_addr_t dma_addr);
|
||||
void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents);
|
||||
|
||||
/*
|
||||
* Arch code with no special attribute handling may use these
|
||||
|
|
|
|||
58
include/linux/dma-mapping-fast.h
Normal file
58
include/linux/dma-mapping-fast.h
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef __LINUX_DMA_MAPPING_FAST_H
|
||||
#define __LINUX_DMA_MAPPING_FAST_H
|
||||
|
||||
#include <linux/iommu.h>
|
||||
#include <linux/io-pgtable-fast.h>
|
||||
|
||||
struct dma_iommu_mapping;
|
||||
struct io_pgtable_ops;
|
||||
struct iova_domain;
|
||||
|
||||
struct dma_fast_smmu_mapping {
|
||||
struct device *dev;
|
||||
struct iommu_domain *domain;
|
||||
struct iova_domain *iovad;
|
||||
|
||||
dma_addr_t base;
|
||||
size_t size;
|
||||
size_t num_4k_pages;
|
||||
|
||||
unsigned int bitmap_size;
|
||||
unsigned long *bitmap;
|
||||
unsigned long next_start;
|
||||
unsigned long upcoming_stale_bit;
|
||||
bool have_stale_tlbs;
|
||||
|
||||
dma_addr_t pgtbl_dma_handle;
|
||||
struct io_pgtable_ops *pgtbl_ops;
|
||||
|
||||
spinlock_t lock;
|
||||
struct notifier_block notifier;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
int fast_smmu_init_mapping(struct device *dev, struct iommu_domain *domain,
|
||||
struct io_pgtable_ops *pgtable_ops);
|
||||
void fast_smmu_put_dma_cookie(struct iommu_domain *domain);
|
||||
const struct dma_map_ops *fast_smmu_get_dma_ops(void);
|
||||
#else
|
||||
static inline int fast_smmu_init_mapping(struct device *dev,
|
||||
struct iommu_domain *domain,
|
||||
struct io_pgtable_ops *pgtable_ops)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
static inline void fast_smmu_put_dma_cookie(struct iommu_domain *domain) {}
|
||||
static __maybe_unused const struct dma_map_ops *fast_smmu_get_dma_ops(void)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* __LINUX_DMA_MAPPING_FAST_H */
|
||||
|
|
@ -70,6 +70,52 @@
|
|||
*/
|
||||
#define DMA_ATTR_PRIVILEGED (1UL << 9)
|
||||
|
||||
/*
|
||||
* DMA_ATTR_SKIP_ZEROING: Do not zero mapping.
|
||||
*/
|
||||
#define DMA_ATTR_SKIP_ZEROING (1UL << 10)
|
||||
/*
|
||||
* DMA_ATTR_NO_DELAYED_UNMAP: Used by msm specific lazy mapping to indicate
|
||||
* that the mapping can be freed on unmap, rather than when the ion_buffer
|
||||
* is freed.
|
||||
*/
|
||||
#define DMA_ATTR_NO_DELAYED_UNMAP (1UL << 11)
|
||||
/*
|
||||
* DMA_ATTR_EXEC_MAPPING: The mapping has executable permissions.
|
||||
*/
|
||||
#define DMA_ATTR_EXEC_MAPPING (1UL << 12)
|
||||
/*
|
||||
* DMA_ATTR_IOMMU_USE_UPSTREAM_HINT: Normally an smmu will override any bus
|
||||
* attributes (i.e cacheablilty) provided by the client device. Some hardware
|
||||
* may be designed to use the original attributes instead.
|
||||
*/
|
||||
#define DMA_ATTR_IOMMU_USE_UPSTREAM_HINT (1UL << 13)
|
||||
/*
|
||||
* When passed to a DMA map call the DMA_ATTR_FORCE_COHERENT DMA
|
||||
* attribute can be used to force a buffer to be mapped as IO coherent.
|
||||
*/
|
||||
#define DMA_ATTR_FORCE_COHERENT (1UL << 14)
|
||||
/*
|
||||
* When passed to a DMA map call the DMA_ATTR_FORCE_NON_COHERENT DMA
|
||||
* attribute can be used to force a buffer to not be mapped as IO
|
||||
* coherent.
|
||||
*/
|
||||
#define DMA_ATTR_FORCE_NON_COHERENT (1UL << 15)
|
||||
/*
|
||||
* DMA_ATTR_DELAYED_UNMAP: Used by ION, it will ensure that mappings are not
|
||||
* removed on unmap but instead are removed when the ion_buffer is freed.
|
||||
*/
|
||||
#define DMA_ATTR_DELAYED_UNMAP (1UL << 16)
|
||||
|
||||
/*
|
||||
* DMA_ATTR_IOMMU_USE_LLC_NWA: Overrides the bus attributes to use the System
|
||||
* Cache(LLC) with allocation policy as Inner Non-Cacheable, Outer Cacheable:
|
||||
* Write-Back, Read-Allocate, No Write-Allocate policy.
|
||||
*/
|
||||
#define DMA_ATTR_IOMMU_USE_LLC_NWA (1UL << 17)
|
||||
|
||||
#define DMA_ERROR_CODE (~(dma_addr_t)0)
|
||||
|
||||
/*
|
||||
* A dma_addr_t can hold any valid DMA or bus address for the platform.
|
||||
* It can be given to a device to use as a DMA source or target. A CPU cannot
|
||||
|
|
|
|||
100
include/linux/io-pgtable-fast.h
Normal file
100
include/linux/io-pgtable-fast.h
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef __LINUX_IO_PGTABLE_FAST_H
|
||||
#define __LINUX_IO_PGTABLE_FAST_H
|
||||
|
||||
#include <linux/notifier.h>
|
||||
|
||||
/*
|
||||
* This ought to be private to io-pgtable-fast, but dma-mapping-fast
|
||||
* currently requires it for a debug usecase.
|
||||
*/
|
||||
typedef u64 av8l_fast_iopte;
|
||||
|
||||
struct io_pgtable_ops;
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
|
||||
int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot);
|
||||
|
||||
void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
size_t size);
|
||||
|
||||
int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size);
|
||||
|
||||
bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova);
|
||||
|
||||
phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova);
|
||||
#else
|
||||
static inline int
|
||||
av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
static inline void av8l_fast_unmap_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, size_t size)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova, struct scatterlist *sgl,
|
||||
unsigned int nents, int prot, size_t *size)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
static inline phys_addr_t
|
||||
av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
|
||||
unsigned long iova)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST */
|
||||
|
||||
|
||||
/* events for notifiers passed to av8l_register_notify */
|
||||
#define MAPPED_OVER_STALE_TLB 1
|
||||
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
|
||||
/*
|
||||
* Doesn't matter what we use as long as bit 0 is unset. The reason why we
|
||||
* need a different value at all is that there are certain hardware
|
||||
* platforms with erratum that require that a PTE actually be zero'd out
|
||||
* and not just have its valid bit unset.
|
||||
*/
|
||||
#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0xa
|
||||
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, bool skip_sync);
|
||||
void av8l_register_notify(struct notifier_block *nb);
|
||||
|
||||
#else /* !CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */
|
||||
|
||||
#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0
|
||||
|
||||
static inline void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops,
|
||||
bool skip_sync)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void av8l_register_notify(struct notifier_block *nb)
|
||||
{
|
||||
}
|
||||
|
||||
#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */
|
||||
|
||||
#endif /* __LINUX_IO_PGTABLE_FAST_H */
|
||||
|
|
@ -3,6 +3,13 @@
|
|||
#define __IO_PGTABLE_H
|
||||
#include <linux/bitops.h>
|
||||
|
||||
#include <linux/scatterlist.h>
|
||||
|
||||
#define to_msm_iommu_gather_ops(_tlb_ops) \
|
||||
container_of(_tlb_ops, struct msm_iommu_gather_ops, tlb_ops)
|
||||
#define to_msm_io_pgtable_info(_cfg) \
|
||||
container_of(_cfg, struct msm_io_pgtable_info, pgtbl_cfg)
|
||||
|
||||
/*
|
||||
* Public API for use by IOMMU drivers
|
||||
*/
|
||||
|
|
@ -13,6 +20,9 @@ enum io_pgtable_fmt {
|
|||
ARM_64_LPAE_S2,
|
||||
ARM_V7S,
|
||||
ARM_MALI_LPAE,
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
ARM_V8L_FAST,
|
||||
#endif
|
||||
IO_PGTABLE_NUM_FMTS,
|
||||
};
|
||||
|
||||
|
|
@ -35,6 +45,21 @@ struct iommu_gather_ops {
|
|||
void (*tlb_sync)(void *cookie);
|
||||
};
|
||||
|
||||
/**
|
||||
* struct msm_iommu_gather_ops - MSM and standard IOMMU callbacks for TLB and
|
||||
* page table management.
|
||||
*
|
||||
* @alloc_pages_exact: Allocate page table memory (optional, defaults to
|
||||
* alloc_pages_exact)
|
||||
* @free_pages_exact: Free page table memory (optional, defaults to
|
||||
* free_pages_exact)
|
||||
*/
|
||||
struct msm_iommu_gather_ops {
|
||||
void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask);
|
||||
void (*free_pages_exact)(void *cookie, void *virt, size_t size);
|
||||
const struct iommu_gather_ops tlb_ops;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct io_pgtable_cfg - Configuration data for a set of page tables.
|
||||
*
|
||||
|
|
@ -68,6 +93,7 @@ struct io_pgtable_cfg {
|
|||
* when the SoC is in "4GB mode" and they can only access the high
|
||||
* remap of DRAM (0x1_00000000 to 0x1_ffffffff).
|
||||
*
|
||||
|
||||
* IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever
|
||||
* be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a
|
||||
* software-emulated IOMMU), such that pagetable updates need not
|
||||
|
|
@ -76,6 +102,15 @@ struct io_pgtable_cfg {
|
|||
* IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
|
||||
* on unmap, for DMA domains using the flush queue mechanism for
|
||||
* delayed invalidation.
|
||||
|
||||
* IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT: Override the attributes
|
||||
* set in TCR for the page table walker. Use attributes specified
|
||||
* by the upstream hw instead.
|
||||
*
|
||||
* IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA: Override the attributes
|
||||
* set in TCR for the page table walker with Write-Back,
|
||||
* no Write-Allocate cacheable encoding.
|
||||
*
|
||||
*/
|
||||
#define IO_PGTABLE_QUIRK_ARM_NS BIT(0)
|
||||
#define IO_PGTABLE_QUIRK_NO_PERMS BIT(1)
|
||||
|
|
@ -83,6 +118,8 @@ struct io_pgtable_cfg {
|
|||
#define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3)
|
||||
#define IO_PGTABLE_QUIRK_NO_DMA BIT(4)
|
||||
#define IO_PGTABLE_QUIRK_NON_STRICT BIT(5)
|
||||
#define IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT BIT(6)
|
||||
#define IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA BIT(7)
|
||||
unsigned long quirks;
|
||||
unsigned long pgsize_bitmap;
|
||||
unsigned int ias;
|
||||
|
|
@ -114,6 +151,15 @@ struct io_pgtable_cfg {
|
|||
u64 transtab;
|
||||
u64 memattr;
|
||||
} arm_mali_lpae_cfg;
|
||||
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
struct {
|
||||
u64 ttbr[2];
|
||||
u64 tcr;
|
||||
u64 mair[2];
|
||||
void *pmds;
|
||||
} av8l_fast_cfg;
|
||||
#endif
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -136,6 +182,28 @@ struct io_pgtable_ops {
|
|||
unsigned long iova);
|
||||
};
|
||||
|
||||
/**
|
||||
* struct msm_io_pgtable_info - MSM specific page table manipulation API for
|
||||
* IOMMU drivers, and page table configuration.
|
||||
*
|
||||
* @map_sg: Map a scatterlist. Returns the number of bytes mapped,
|
||||
* or 0 on failure. The size parameter contains the size
|
||||
* of the partial mapping in case of failure.
|
||||
* @is_iova_coherent: Checks coherency of given IOVA. Returns True if coherent
|
||||
* and False if non-coherent.
|
||||
* @iova_to_pte: Translate iova to Page Table Entry (PTE).
|
||||
* @pgtbl_cfg: The configuration for a set of page tables.
|
||||
*/
|
||||
struct msm_io_pgtable_info {
|
||||
int (*map_sg)(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
struct scatterlist *sg, unsigned int nents, int prot,
|
||||
size_t *size);
|
||||
bool (*is_iova_coherent)(struct io_pgtable_ops *ops,
|
||||
unsigned long iova);
|
||||
uint64_t (*iova_to_pte)(struct io_pgtable_ops *ops, unsigned long iova);
|
||||
struct io_pgtable_cfg pgtbl_cfg;
|
||||
};
|
||||
|
||||
/**
|
||||
* alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU.
|
||||
*
|
||||
|
|
@ -184,17 +252,23 @@ struct io_pgtable {
|
|||
|
||||
static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
|
||||
{
|
||||
if (!iop->cfg.tlb)
|
||||
return;
|
||||
iop->cfg.tlb->tlb_flush_all(iop->cookie);
|
||||
}
|
||||
|
||||
static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop,
|
||||
unsigned long iova, size_t size, size_t granule, bool leaf)
|
||||
{
|
||||
if (!iop->cfg.tlb)
|
||||
return;
|
||||
iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie);
|
||||
}
|
||||
|
||||
static inline void io_pgtable_tlb_sync(struct io_pgtable *iop)
|
||||
{
|
||||
if (!iop->cfg.tlb)
|
||||
return;
|
||||
iop->cfg.tlb->tlb_sync(iop->cookie);
|
||||
}
|
||||
|
||||
|
|
@ -216,5 +290,32 @@ extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
|
|||
extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
|
||||
extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
|
||||
extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
|
||||
#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
|
||||
extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* io_pgtable_alloc_pages_exact:
|
||||
* allocate an exact number of physically-contiguous pages.
|
||||
* @size: the number of bytes to allocate
|
||||
* @gfp_mask: GFP flags for the allocation
|
||||
*
|
||||
* Like alloc_pages_exact(), but with some additional accounting for debug
|
||||
* purposes.
|
||||
*/
|
||||
void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
|
||||
size_t size, gfp_t gfp_mask);
|
||||
|
||||
/**
|
||||
* io_pgtable_free_pages_exact:
|
||||
* release memory allocated via io_pgtable_alloc_pages_exact()
|
||||
* @virt: the value returned by alloc_pages_exact.
|
||||
* @size: size of allocation, same value as passed to alloc_pages_exact().
|
||||
*
|
||||
* Like free_pages_exact(), but with some additional accounting for debug
|
||||
* purposes.
|
||||
*/
|
||||
void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,
|
||||
void *virt, size_t size);
|
||||
|
||||
#endif /* __IO_PGTABLE_H */
|
||||
|
|
|
|||
|
|
@ -29,6 +29,11 @@
|
|||
* if the IOMMU page table format is equivalent.
|
||||
*/
|
||||
#define IOMMU_PRIV (1 << 5)
|
||||
/* Use upstream device's bus attribute */
|
||||
#define IOMMU_USE_UPSTREAM_HINT (1 << 6)
|
||||
|
||||
/* Use upstream device's bus attribute with no write-allocate cache policy */
|
||||
#define IOMMU_USE_LLC_NWA (1 << 7)
|
||||
|
||||
struct iommu_ops;
|
||||
struct iommu_group;
|
||||
|
|
@ -39,8 +44,12 @@ struct notifier_block;
|
|||
struct iommu_sva;
|
||||
|
||||
/* iommu fault flags */
|
||||
#define IOMMU_FAULT_READ 0x0
|
||||
#define IOMMU_FAULT_WRITE 0x1
|
||||
#define IOMMU_FAULT_READ (1 << 0)
|
||||
#define IOMMU_FAULT_WRITE (1 << 1)
|
||||
#define IOMMU_FAULT_TRANSLATION (1 << 2)
|
||||
#define IOMMU_FAULT_PERMISSION (1 << 3)
|
||||
#define IOMMU_FAULT_EXTERNAL (1 << 4)
|
||||
#define IOMMU_FAULT_TRANSACTION_STALLED (1 << 5)
|
||||
|
||||
typedef int (*iommu_fault_handler_t)(struct iommu_domain *,
|
||||
struct device *, unsigned long, int, void *);
|
||||
|
|
@ -53,6 +62,10 @@ struct iommu_domain_geometry {
|
|||
bool force_aperture; /* DMA only allowed in mappable range? */
|
||||
};
|
||||
|
||||
struct iommu_pgtbl_info {
|
||||
void *ops;
|
||||
};
|
||||
|
||||
/* Domain feature flags */
|
||||
#define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */
|
||||
#define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API
|
||||
|
|
@ -77,6 +90,12 @@ struct iommu_domain_geometry {
|
|||
#define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \
|
||||
__IOMMU_DOMAIN_DMA_API)
|
||||
|
||||
#define to_msm_iommu_ops(_iommu_ops) \
|
||||
container_of(_iommu_ops, struct msm_iommu_ops, iommu_ops)
|
||||
#define to_msm_iommu_domain(_iommu_domain) \
|
||||
container_of(_iommu_domain, struct msm_iommu_domain, iommu_domain)
|
||||
|
||||
#define IOMMU_DOMAIN_NAME_LEN 32
|
||||
struct iommu_domain {
|
||||
unsigned type;
|
||||
const struct iommu_ops *ops;
|
||||
|
|
@ -87,6 +106,11 @@ struct iommu_domain {
|
|||
void *iova_cookie;
|
||||
};
|
||||
|
||||
struct msm_iommu_domain {
|
||||
char name[IOMMU_DOMAIN_NAME_LEN];
|
||||
struct iommu_domain iommu_domain;
|
||||
};
|
||||
|
||||
enum iommu_cap {
|
||||
IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA
|
||||
transactions */
|
||||
|
|
@ -105,6 +129,11 @@ enum iommu_cap {
|
|||
* DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints.
|
||||
* The caller can invoke iommu_domain_get_attr to check if the underlying
|
||||
* iommu implementation supports these constraints.
|
||||
*
|
||||
* DOMAIN_ATTR_NO_CFRE
|
||||
* Some bus implementations may enter a bad state if iommu reports an error
|
||||
* on context fault. As context faults are not always fatal, this must be
|
||||
* avoided.
|
||||
*/
|
||||
|
||||
enum iommu_attr {
|
||||
|
|
@ -119,6 +148,28 @@ enum iommu_attr {
|
|||
DOMAIN_ATTR_MAX,
|
||||
};
|
||||
|
||||
#define DOMAIN_ATTR_PT_BASE_ADDR (DOMAIN_ATTR_MAX + 1)
|
||||
#define DOMAIN_ATTR_CONTEXT_BANK (DOMAIN_ATTR_MAX + 2)
|
||||
#define DOMAIN_ATTR_DYNAMIC (DOMAIN_ATTR_MAX + 3)
|
||||
#define DOMAIN_ATTR_TTBR0 (DOMAIN_ATTR_MAX + 4)
|
||||
#define DOMAIN_ATTR_CONTEXTIDR (DOMAIN_ATTR_MAX + 5)
|
||||
#define DOMAIN_ATTR_PROCID (DOMAIN_ATTR_MAX + 6)
|
||||
#define DOMAIN_ATTR_NON_FATAL_FAULTS (DOMAIN_ATTR_MAX + 7)
|
||||
#define DOMAIN_ATTR_S1_BYPASS (DOMAIN_ATTR_MAX + 8)
|
||||
#define DOMAIN_ATTR_ATOMIC (DOMAIN_ATTR_MAX + 9)
|
||||
#define DOMAIN_ATTR_SECURE_VMID (DOMAIN_ATTR_MAX + 10)
|
||||
#define DOMAIN_ATTR_FAST (DOMAIN_ATTR_MAX + 11)
|
||||
#define DOMAIN_ATTR_PGTBL_INFO (DOMAIN_ATTR_MAX + 12)
|
||||
#define DOMAIN_ATTR_USE_UPSTREAM_HINT (DOMAIN_ATTR_MAX + 13)
|
||||
#define DOMAIN_ATTR_EARLY_MAP (DOMAIN_ATTR_MAX + 14)
|
||||
#define DOMAIN_ATTR_PAGE_TABLE_IS_COHERENT (DOMAIN_ATTR_MAX + 15)
|
||||
#define DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT (DOMAIN_ATTR_MAX + 16)
|
||||
#define DOMAIN_ATTR_CB_STALL_DISABLE (DOMAIN_ATTR_MAX + 17)
|
||||
#define DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR (DOMAIN_ATTR_MAX + 18)
|
||||
#define DOMAIN_ATTR_USE_LLC_NWA (DOMAIN_ATTR_MAX + 19)
|
||||
#define DOMAIN_ATTR_NO_CFRE (DOMAIN_ATTR_MAX + 20)
|
||||
#define DOMAIN_ATTR_DEBUG (DOMAIN_ATTR_MAX + 21)
|
||||
|
||||
/* These are the possible reserved region types */
|
||||
enum iommu_resv_type {
|
||||
/* Memory regions which must be mapped 1:1 at all times */
|
||||
|
|
@ -147,6 +198,7 @@ struct iommu_resv_region {
|
|||
enum iommu_resv_type type;
|
||||
};
|
||||
|
||||
extern struct dentry *iommu_debugfs_top;
|
||||
/* Per device IOMMU features */
|
||||
enum iommu_dev_features {
|
||||
IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */
|
||||
|
|
@ -275,6 +327,34 @@ struct iommu_ops {
|
|||
unsigned long pgsize_bitmap;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct msm_iommu_ops - standard iommu ops, as well as additional MSM
|
||||
* specific iommu ops
|
||||
* @map_sg: map a scatter-gather list of physically contiguous memory chunks
|
||||
* to an iommu domain
|
||||
* @iova_to_phys_hard: translate iova to physical address using IOMMU hardware
|
||||
* @is_iova_coherent: checks coherency of the given iova
|
||||
* @trigger_fault: trigger a fault on the device attached to an iommu domain
|
||||
* @tlbi_domain: Invalidate all TLBs covering an iommu domain
|
||||
* @enable_config_clocks: Enable all config clocks for this domain's IOMMU
|
||||
* @disable_config_clocks: Disable all config clocks for this domain's IOMMU
|
||||
* @iova_to_pte: translate iova to Page Table Entry (PTE).
|
||||
* @iommu_ops: the standard iommu ops
|
||||
*/
|
||||
struct msm_iommu_ops {
|
||||
size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova,
|
||||
struct scatterlist *sg, unsigned int nents, int prot);
|
||||
phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain,
|
||||
dma_addr_t iova);
|
||||
bool (*is_iova_coherent)(struct iommu_domain *domain, dma_addr_t iova);
|
||||
void (*trigger_fault)(struct iommu_domain *domain, unsigned long flags);
|
||||
void (*tlbi_domain)(struct iommu_domain *domain);
|
||||
int (*enable_config_clocks)(struct iommu_domain *domain);
|
||||
void (*disable_config_clocks)(struct iommu_domain *domain);
|
||||
uint64_t (*iova_to_pte)(struct iommu_domain *domain, dma_addr_t iova);
|
||||
struct iommu_ops iommu_ops;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct iommu_device - IOMMU core representation of one IOMMU hardware
|
||||
* instance
|
||||
|
|
@ -334,6 +414,8 @@ extern int iommu_attach_device(struct iommu_domain *domain,
|
|||
extern void iommu_detach_device(struct iommu_domain *domain,
|
||||
struct device *dev);
|
||||
extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
|
||||
extern size_t iommu_pgsize(unsigned long pgsize_bitmap,
|
||||
unsigned long addr_merge, size_t size);
|
||||
extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
|
||||
extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot);
|
||||
|
|
@ -343,7 +425,14 @@ extern size_t iommu_unmap_fast(struct iommu_domain *domain,
|
|||
unsigned long iova, size_t size);
|
||||
extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
|
||||
struct scatterlist *sg,unsigned int nents, int prot);
|
||||
extern size_t default_iommu_map_sg(struct iommu_domain *domain,
|
||||
unsigned long iova, struct scatterlist *sg,
|
||||
unsigned int nents, int prot);
|
||||
extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
|
||||
extern phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||
dma_addr_t iova);
|
||||
extern bool iommu_is_iova_coherent(struct iommu_domain *domain,
|
||||
dma_addr_t iova);
|
||||
extern void iommu_set_fault_handler(struct iommu_domain *domain,
|
||||
iommu_fault_handler_t handler, void *token);
|
||||
|
||||
|
|
@ -393,6 +482,9 @@ extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
|
|||
int prot);
|
||||
extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr);
|
||||
|
||||
extern uint64_t iommu_iova_to_pte(struct iommu_domain *domain,
|
||||
dma_addr_t iova);
|
||||
|
||||
extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
|
||||
unsigned long iova, int flags);
|
||||
|
||||
|
|
@ -415,6 +507,9 @@ static inline void iommu_tlb_sync(struct iommu_domain *domain)
|
|||
domain->ops->iotlb_sync(domain);
|
||||
}
|
||||
|
||||
extern void iommu_trigger_fault(struct iommu_domain *domain,
|
||||
unsigned long flags);
|
||||
|
||||
/* PCI device grouping function */
|
||||
extern struct iommu_group *pci_device_group(struct device *dev);
|
||||
/* Generic device grouping function */
|
||||
|
|
@ -422,6 +517,31 @@ extern struct iommu_group *generic_device_group(struct device *dev);
|
|||
/* FSL-MC device grouping function */
|
||||
struct iommu_group *fsl_mc_device_group(struct device *dev);
|
||||
|
||||
static inline void iommu_tlbiall(struct iommu_domain *domain)
|
||||
{
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (ops->tlbi_domain)
|
||||
ops->tlbi_domain(domain);
|
||||
}
|
||||
|
||||
static inline int iommu_enable_config_clocks(struct iommu_domain *domain)
|
||||
{
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (ops->enable_config_clocks)
|
||||
return ops->enable_config_clocks(domain);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void iommu_disable_config_clocks(struct iommu_domain *domain)
|
||||
{
|
||||
struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops);
|
||||
|
||||
if (ops->disable_config_clocks)
|
||||
ops->disable_config_clocks(domain);
|
||||
}
|
||||
|
||||
/**
|
||||
* struct iommu_fwspec - per-device IOMMU instance data
|
||||
* @ops: ops for this device's IOMMU
|
||||
|
|
@ -588,6 +708,18 @@ static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_ad
|
|||
return 0;
|
||||
}
|
||||
|
||||
static inline phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||
dma_addr_t iova)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline bool iommu_is_iova_coherent(struct iommu_domain *domain,
|
||||
dma_addr_t iova)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline void iommu_set_fault_handler(struct iommu_domain *domain,
|
||||
iommu_fault_handler_t handler, void *token)
|
||||
{
|
||||
|
|
@ -747,6 +879,24 @@ static inline void iommu_device_unlink(struct device *dev, struct device *link)
|
|||
{
|
||||
}
|
||||
|
||||
static inline void iommu_trigger_fault(struct iommu_domain *domain,
|
||||
unsigned long flags)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void iommu_tlbiall(struct iommu_domain *domain)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int iommu_enable_config_clocks(struct iommu_domain *domain)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void iommu_disable_config_clocks(struct iommu_domain *domain)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int iommu_fwspec_init(struct device *dev,
|
||||
struct fwnode_handle *iommu_fwnode,
|
||||
const struct iommu_ops *ops)
|
||||
|
|
|
|||
109
include/linux/msm_dma_iommu_mapping.h
Normal file
109
include/linux/msm_dma_iommu_mapping.h
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2015-2016, 2018 The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _LINUX_MSM_DMA_IOMMU_MAPPING_H
|
||||
#define _LINUX_MSM_DMA_IOMMU_MAPPING_H
|
||||
|
||||
#include <linux/device.h>
|
||||
#include <linux/dma-buf.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
|
||||
#if IS_ENABLED(CONFIG_QCOM_LAZY_MAPPING)
|
||||
/*
|
||||
* This function is not taking a reference to the dma_buf here. It is expected
|
||||
* that clients hold reference to the dma_buf until they are done with mapping
|
||||
* and unmapping.
|
||||
*/
|
||||
int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, struct dma_buf *dma_buf,
|
||||
unsigned long attrs);
|
||||
|
||||
/*
|
||||
* This function takes an extra reference to the dma_buf.
|
||||
* What this means is that calling msm_dma_unmap_sg will not result in buffer's
|
||||
* iommu mapping being removed, which means that subsequent calls to lazy map
|
||||
* will simply re-use the existing iommu mapping.
|
||||
* The iommu unmapping of the buffer will occur when the ION buffer is
|
||||
* destroyed.
|
||||
* Using lazy mapping can provide a performance benefit because subsequent
|
||||
* mappings are faster.
|
||||
*
|
||||
* The limitation of using this API are that all subsequent iommu mappings
|
||||
* must be the same as the original mapping, ie they must map the same part of
|
||||
* the buffer with the same dma data direction. Also there can't be multiple
|
||||
* mappings of different parts of the buffer.
|
||||
*/
|
||||
static inline int msm_dma_map_sg_lazy(struct device *dev,
|
||||
struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, 0);
|
||||
}
|
||||
|
||||
static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
unsigned long attrs;
|
||||
|
||||
attrs = DMA_ATTR_NO_DELAYED_UNMAP;
|
||||
return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, attrs);
|
||||
}
|
||||
|
||||
void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs);
|
||||
|
||||
int msm_dma_unmap_all_for_dev(struct device *dev);
|
||||
|
||||
/*
|
||||
* Below is private function only to be called by framework (ION) and not by
|
||||
* clients.
|
||||
*/
|
||||
void msm_dma_buf_freed(void *buffer);
|
||||
|
||||
#else /*CONFIG_QCOM_LAZY_MAPPING*/
|
||||
|
||||
static inline int msm_dma_map_sg_attrs(struct device *dev,
|
||||
struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, struct dma_buf *dma_buf,
|
||||
unsigned long attrs)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline void
|
||||
msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int msm_dma_map_sg_lazy(struct device *dev,
|
||||
struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline int msm_dma_unmap_all_for_dev(struct device *dev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void msm_dma_buf_freed(void *buffer) {}
|
||||
#endif /*CONFIG_QCOM_LAZY_MAPPING*/
|
||||
|
||||
#endif
|
||||
80
include/soc/qcom/secure_buffer.h
Normal file
80
include/soc/qcom/secure_buffer.h
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef __QCOM_SECURE_BUFFER_H__
|
||||
#define __QCOM_SECURE_BUFFER_H__
|
||||
|
||||
#include <linux/scatterlist.h>
|
||||
|
||||
/*
|
||||
* if you add a secure VMID here make sure you update
|
||||
* msm_secure_vmid_to_string.
|
||||
* Make sure to keep the VMID_LAST as the last entry in the enum.
|
||||
* This is needed in ion to create a list and it's sized using VMID_LAST.
|
||||
*/
|
||||
enum vmid {
|
||||
VMID_HLOS = 0x3,
|
||||
VMID_CP_TOUCH = 0x8,
|
||||
VMID_CP_BITSTREAM = 0x9,
|
||||
VMID_CP_PIXEL = 0xA,
|
||||
VMID_CP_NON_PIXEL = 0xB,
|
||||
VMID_CP_CAMERA = 0xD,
|
||||
VMID_HLOS_FREE = 0xE,
|
||||
VMID_MSS_MSA = 0xF,
|
||||
VMID_MSS_NONMSA = 0x10,
|
||||
VMID_CP_SEC_DISPLAY = 0x11,
|
||||
VMID_CP_APP = 0x12,
|
||||
VMID_WLAN = 0x18,
|
||||
VMID_WLAN_CE = 0x19,
|
||||
VMID_CP_SPSS_SP = 0x1A,
|
||||
VMID_CP_CAMERA_PREVIEW = 0x1D,
|
||||
VMID_CP_SPSS_SP_SHARED = 0x22,
|
||||
VMID_CP_SPSS_HLOS_SHARED = 0x24,
|
||||
VMID_CP_CDSP = 0x2A,
|
||||
VMID_LAST,
|
||||
VMID_INVAL = -1
|
||||
};
|
||||
|
||||
#define PERM_READ 0x4
|
||||
#define PERM_WRITE 0x2
|
||||
#define PERM_EXEC 0x1
|
||||
|
||||
#if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER)
|
||||
int hyp_assign_table(struct sg_table *table,
|
||||
u32 *source_vm_list, int source_nelems,
|
||||
int *dest_vmids, int *dest_perms,
|
||||
int dest_nelems);
|
||||
int hyp_assign_phys(phys_addr_t addr, u64 size,
|
||||
u32 *source_vmlist, int source_nelems,
|
||||
int *dest_vmids, int *dest_perms, int dest_nelems);
|
||||
const char *msm_secure_vmid_to_string(int secure_vmid);
|
||||
u32 msm_secure_get_vmid_perms(u32 vmid);
|
||||
#else
|
||||
static inline int hyp_assign_table(struct sg_table *table,
|
||||
u32 *source_vm_list, int source_nelems,
|
||||
int *dest_vmids, int *dest_perms,
|
||||
int dest_nelems)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline int hyp_assign_phys(phys_addr_t addr, u64 size,
|
||||
u32 *source_vmlist, int source_nelems,
|
||||
int *dest_vmids, int *dest_perms, int dest_nelems)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline const char *msm_secure_vmid_to_string(int secure_vmid)
|
||||
{
|
||||
return "N/A";
|
||||
}
|
||||
|
||||
static inline u32 msm_secure_get_vmid_perms(u32 vmid)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
|
@ -12,8 +12,10 @@
|
|||
#define _TRACE_IOMMU_H
|
||||
|
||||
#include <linux/tracepoint.h>
|
||||
#include <linux/iommu.h>
|
||||
|
||||
struct device;
|
||||
struct iommu_domain;
|
||||
|
||||
DECLARE_EVENT_CLASS(iommu_group_event,
|
||||
|
||||
|
|
@ -85,47 +87,84 @@ DEFINE_EVENT(iommu_device_event, detach_device_from_domain,
|
|||
|
||||
TRACE_EVENT(map,
|
||||
|
||||
TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size),
|
||||
TP_PROTO(struct msm_iommu_domain *domain, unsigned long iova,
|
||||
phys_addr_t paddr, size_t size, int prot),
|
||||
|
||||
TP_ARGS(iova, paddr, size),
|
||||
TP_ARGS(domain, iova, paddr, size, prot),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(name, domain->name)
|
||||
__field(u64, iova)
|
||||
__field(u64, paddr)
|
||||
__field(size_t, size)
|
||||
__field(int, prot)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(name, domain->name);
|
||||
__entry->iova = iova;
|
||||
__entry->paddr = paddr;
|
||||
__entry->size = size;
|
||||
__entry->prot = prot;
|
||||
),
|
||||
|
||||
TP_printk("IOMMU: iova=0x%016llx paddr=0x%016llx size=%zu",
|
||||
__entry->iova, __entry->paddr, __entry->size
|
||||
TP_printk("IOMMU:%s iova=0x%016llx paddr=0x%016llx size=0x%zx prot=0x%x",
|
||||
__get_str(name), __entry->iova, __entry->paddr,
|
||||
__entry->size, __entry->prot
|
||||
)
|
||||
);
|
||||
|
||||
TRACE_EVENT(unmap,
|
||||
|
||||
TP_PROTO(unsigned long iova, size_t size, size_t unmapped_size),
|
||||
TP_PROTO(struct msm_iommu_domain *domain, unsigned long iova,
|
||||
size_t size, size_t unmapped_size),
|
||||
|
||||
TP_ARGS(iova, size, unmapped_size),
|
||||
TP_ARGS(domain, iova, size, unmapped_size),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(name, domain->name)
|
||||
__field(u64, iova)
|
||||
__field(size_t, size)
|
||||
__field(size_t, unmapped_size)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(name, domain->name);
|
||||
__entry->iova = iova;
|
||||
__entry->size = size;
|
||||
__entry->unmapped_size = unmapped_size;
|
||||
),
|
||||
|
||||
TP_printk("IOMMU: iova=0x%016llx size=%zu unmapped_size=%zu",
|
||||
__entry->iova, __entry->size, __entry->unmapped_size
|
||||
TP_printk("IOMMU:%s iova=0x%016llx size=0x%zx unmapped_size=0x%zx",
|
||||
__get_str(name), __entry->iova, __entry->size,
|
||||
__entry->unmapped_size
|
||||
)
|
||||
);
|
||||
|
||||
TRACE_EVENT(map_sg,
|
||||
|
||||
TP_PROTO(struct msm_iommu_domain *domain, unsigned long iova,
|
||||
size_t size, int prot),
|
||||
|
||||
TP_ARGS(domain, iova, size, prot),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__string(name, domain->name)
|
||||
__field(u64, iova)
|
||||
__field(size_t, size)
|
||||
__field(int, prot)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__assign_str(name, domain->name);
|
||||
__entry->iova = iova;
|
||||
__entry->size = size;
|
||||
__entry->prot = prot;
|
||||
),
|
||||
|
||||
TP_printk("IOMMU:%s iova=0x%016llx size=0x%zx prot=0x%x",
|
||||
__get_str(name), __entry->iova, __entry->size,
|
||||
__entry->prot
|
||||
)
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -192,10 +192,22 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
|
|||
__dma_direct_free_pages(dev, size, virt_to_page(cpu_addr));
|
||||
}
|
||||
|
||||
static bool is_dma_coherent(struct device *dev, unsigned long attrs)
|
||||
{
|
||||
if (attrs & DMA_ATTR_FORCE_COHERENT)
|
||||
return true;
|
||||
else if (attrs & DMA_ATTR_FORCE_NON_COHERENT)
|
||||
return false;
|
||||
else if (dev_is_dma_coherent(dev))
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
void *dma_direct_alloc(struct device *dev, size_t size,
|
||||
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
|
||||
{
|
||||
if (!dev_is_dma_coherent(dev))
|
||||
if (!is_dma_coherent(dev, attrs))
|
||||
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
|
||||
return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
|
||||
}
|
||||
|
|
@ -203,7 +215,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
|
|||
void dma_direct_free(struct device *dev, size_t size,
|
||||
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
|
||||
{
|
||||
if (!dev_is_dma_coherent(dev))
|
||||
if (!is_dma_coherent(dev, attrs))
|
||||
arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
|
||||
else
|
||||
dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
|
||||
|
|
@ -286,7 +298,7 @@ void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
|
|||
{
|
||||
phys_addr_t phys = dma_to_phys(dev, addr);
|
||||
|
||||
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
if (!is_dma_coherent(dev, attrs) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
|
||||
|
||||
if (unlikely(is_swiotlb_buffer(phys)))
|
||||
|
|
@ -327,7 +339,7 @@ dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
|
|||
return DMA_MAPPING_ERROR;
|
||||
}
|
||||
|
||||
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
if (!is_dma_coherent(dev, attrs) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
arch_sync_dma_for_device(dev, phys, size, dir);
|
||||
return dma_addr;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue