diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index bfbbd4ce474d..21bf71870c84 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -907,6 +907,27 @@ config ARCH_WANT_HUGE_PMD_SHARE config ARCH_HAS_CACHE_LINE_SIZE def_bool y +if ARM64 && IOMMU_DMA + +config ARM64_DMA_IOMMU_ALIGNMENT + int "Maximum PAGE_SIZE order of alignment for DMA IOMMU buffers" + range 4 9 + default 9 + help + DMA mapping framework by default aligns all buffers to the smallest + PAGE_SIZE order which is greater than or equal to the requested buffer + size. This works well for buffers up to a few hundreds kilobytes, but + for larger buffers it just a waste of address space. Drivers which has + relatively small addressing window (like 64Mib) might run out of + virtual space with just a few allocations. + + With this parameter you can specify the maximum PAGE_SIZE order for + DMA IOMMU buffers. Larger buffers will be aligned only to this + specified order. The order is expressed as a power of two multiplied + by the PAGE_SIZE. + +endif + config ARCH_ENABLE_SPLIT_PMD_PTLOCK def_bool y if PGTABLE_LEVELS > 2 diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 5992eb9a9a08..b1aea6703afe 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -16,16 +16,32 @@ #include #include #include +#include #include #include #include #include +#include +#include + + +static bool is_dma_coherent(struct device *dev, unsigned long attrs) +{ + if (attrs & DMA_ATTR_FORCE_COHERENT) + return true; + else if (attrs & DMA_ATTR_FORCE_NON_COHERENT) + return false; + else if (dev_is_dma_coherent(dev)) + return true; + else + return false; +} pgprot_t arch_dma_mmap_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) { - if (!dev_is_dma_coherent(dev) || (attrs & DMA_ATTR_WRITE_COMBINE)) + if (!is_dma_coherent(dev, attrs) || (attrs & DMA_ATTR_WRITE_COMBINE)) return pgprot_writecombine(prot); return prot; } @@ -103,7 +119,7 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { - bool coherent = dev_is_dma_coherent(dev); + bool coherent = is_dma_coherent(dev, attrs); int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); size_t iosize = size; void *addr; @@ -117,7 +133,8 @@ static void *__iommu_alloc_attrs(struct device *dev, size_t size, * Some drivers rely on this, and we probably don't want the * possibility of stale kernel data being read by devices anyway. */ - gfp |= __GFP_ZERO; + if (!(attrs & DMA_ATTR_SKIP_ZEROING)) + gfp |= __GFP_ZERO; if (!gfpflags_allow_blocking(gfp)) { struct page *page; @@ -232,31 +249,30 @@ static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, { struct vm_struct *area; int ret; + unsigned long pfn = 0; vma->vm_page_prot = arch_dma_mmap_pgprot(dev, vma->vm_page_prot, attrs); if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) return ret; - if (!is_vmalloc_addr(cpu_addr)) { - unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr)); - return __swiotlb_mmap_pfn(vma, pfn, size); - } - - if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { - /* - * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, - * hence in the vmalloc space. - */ - unsigned long pfn = vmalloc_to_pfn(cpu_addr); - return __swiotlb_mmap_pfn(vma, pfn, size); - } - area = find_vm_area(cpu_addr); - if (WARN_ON(!area || !area->pages)) - return -ENXIO; - return iommu_dma_mmap(area->pages, size, vma); + if (area && area->pages) + return iommu_dma_mmap(area->pages, size, vma); + else if (!is_vmalloc_addr(cpu_addr)) + pfn = page_to_pfn(virt_to_page(cpu_addr)); + else if (is_vmalloc_addr(cpu_addr)) + /* + * DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations are + * always remapped, hence in the vmalloc space. + */ + pfn = vmalloc_to_pfn(cpu_addr); + + if (pfn) + return __swiotlb_mmap_pfn(vma, pfn, size); + + return -ENXIO; } static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt, @@ -264,27 +280,24 @@ static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt, size_t size, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + struct page *page = NULL; struct vm_struct *area = find_vm_area(cpu_addr); - if (!is_vmalloc_addr(cpu_addr)) { - struct page *page = virt_to_page(cpu_addr); - return __swiotlb_get_sgtable_page(sgt, page, size); - } - - if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { + if (area && area->pages) + return sg_alloc_table_from_pages(sgt, area->pages, count, 0, + size, GFP_KERNEL); + else if (!is_vmalloc_addr(cpu_addr)) + page = virt_to_page(cpu_addr); + else if (is_vmalloc_addr(cpu_addr)) /* - * DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped, - * hence in the vmalloc space. + * DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations + * are always remapped, hence in the vmalloc space. */ - struct page *page = vmalloc_to_page(cpu_addr); + page = vmalloc_to_page(cpu_addr); + + if (page) return __swiotlb_get_sgtable_page(sgt, page, size); - } - - if (WARN_ON(!area || !area->pages)) - return -ENXIO; - - return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size, - GFP_KERNEL); + return -ENXIO; } static void __iommu_sync_single_for_cpu(struct device *dev, @@ -292,11 +305,12 @@ static void __iommu_sync_single_for_cpu(struct device *dev, enum dma_data_direction dir) { phys_addr_t phys; + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - if (dev_is_dma_coherent(dev)) + if (!domain || iommu_is_iova_coherent(domain, dev_addr)) return; - phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr); + phys = iommu_iova_to_phys(domain, dev_addr); arch_sync_dma_for_cpu(dev, phys, size, dir); } @@ -305,11 +319,12 @@ static void __iommu_sync_single_for_device(struct device *dev, enum dma_data_direction dir) { phys_addr_t phys; + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); - if (dev_is_dma_coherent(dev)) + if (!domain || iommu_is_iova_coherent(domain, dev_addr)) return; - phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dev_addr); + phys = iommu_iova_to_phys(domain, dev_addr); arch_sync_dma_for_device(dev, phys, size, dir); } @@ -318,7 +333,7 @@ static dma_addr_t __iommu_map_page(struct device *dev, struct page *page, enum dma_data_direction dir, unsigned long attrs) { - bool coherent = dev_is_dma_coherent(dev); + bool coherent = is_dma_coherent(dev, attrs); int prot = dma_info_to_prot(dir, coherent, attrs); dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot); @@ -344,9 +359,11 @@ static void __iommu_sync_sg_for_cpu(struct device *dev, enum dma_data_direction dir) { struct scatterlist *sg; + dma_addr_t iova = sg_dma_address(sgl); + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); int i; - if (dev_is_dma_coherent(dev)) + if (!domain || iommu_is_iova_coherent(domain, iova)) return; for_each_sg(sgl, sg, nelems, i) @@ -358,9 +375,11 @@ static void __iommu_sync_sg_for_device(struct device *dev, enum dma_data_direction dir) { struct scatterlist *sg; + dma_addr_t iova = sg_dma_address(sgl); + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); int i; - if (dev_is_dma_coherent(dev)) + if (!domain || iommu_is_iova_coherent(domain, iova)) return; for_each_sg(sgl, sg, nelems, i) @@ -371,13 +390,18 @@ static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int nelems, enum dma_data_direction dir, unsigned long attrs) { - bool coherent = dev_is_dma_coherent(dev); + bool coherent = is_dma_coherent(dev, attrs); + int ret; + + ret = iommu_dma_map_sg(dev, sgl, nelems, + dma_info_to_prot(dir, coherent, attrs)); + if (!ret) + return ret; if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __iommu_sync_sg_for_device(dev, sgl, nelems, dir); - return iommu_dma_map_sg(dev, sgl, nelems, - dma_info_to_prot(dir, coherent, attrs)); + return ret; } static void __iommu_unmap_sg_attrs(struct device *dev, @@ -414,10 +438,30 @@ static int __init __iommu_dma_init(void) } arch_initcall(__iommu_dma_init); +static int __iommu_init_dma_resources(struct device *dev, + struct iommu_domain *domain, u64 dma_base, + u64 size) +{ + int is_fast, ret = 0; + + iommu_domain_get_attr(domain, DOMAIN_ATTR_FAST, &is_fast); + + if (is_fast) { + dev->dma_ops = fast_smmu_get_dma_ops(); + } else { + ret = iommu_dma_init_domain(domain, dma_base, size, dev); + if (!ret) + dev->dma_ops = &iommu_dma_ops; + } + + return ret; +} + static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, const struct iommu_ops *ops) { struct iommu_domain *domain; + int s1_bypass; if (!ops) return; @@ -431,13 +475,20 @@ static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, if (!domain) goto out_err; - if (domain->type == IOMMU_DOMAIN_DMA) { - if (iommu_dma_init_domain(domain, dma_base, size, dev)) - goto out_err; + iommu_domain_get_attr(domain, DOMAIN_ATTR_S1_BYPASS, &s1_bypass); + if (s1_bypass) + return; - dev->dma_ops = &iommu_dma_ops; + /* Allow iommu-debug to call arch_setup_dma_ops to reconfigure itself */ + if (domain->type != IOMMU_DOMAIN_DMA && + !of_device_is_compatible(dev->of_node, "iommu-debug-test")) { + dev_err(dev, "Invalid iommu domain type!\n"); + return; } + if (__iommu_init_dma_resources(dev, domain, dma_base, size)) + goto out_err; + return; out_err: diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 83664db5221d..16a317eb40e3 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -63,6 +63,58 @@ config IOMMU_IO_PGTABLE_ARMV7S_SELFTEST If unsure, say N here. +config IOMMU_IO_PGTABLE_FAST + bool "Fast ARMv7/v8 Long Descriptor Format" + depends on (ARM || ARM64) && IOMMU_DMA + help + Enable support for a subset of the ARM long descriptor pagetable + format. This allocator achieves fast performance by + pre-allocating and pre-populating page table memory up front. + only supports a 32 bit virtual address space. + + This implementation is mainly optimized for use cases where the + buffers are small (<= 64K) since it only supports 4K page sizes. + +config IOMMU_IO_PGTABLE_FAST_SELFTEST + bool "Fast IO pgtable selftests" + depends on IOMMU_IO_PGTABLE_FAST + help + Enable self-tests for "fast" page table allocator. + This performs a series of page-table consistency checks + during boot. + + If unsure, say N here. + +config IOMMU_IO_PGTABLE_FAST_PROVE_TLB + bool "Prove correctness of TLB maintenance in the Fast DMA mapper" + depends on IOMMU_IO_PGTABLE_FAST + help + Enables some debug features that help prove correctness of TLB + maintenance routines in the Fast DMA mapper. This option will + slow things down considerably, so should only be used in a debug + configuration. This relies on the ability to set bits in an + invalid page table entry, which is disallowed on some hardware + due to errata. If you're running on such a platform then this + option can only be used with unit tests. It will break real use + cases. + + If unsure, say N here. + +config QCOM_IOMMU_IO_PGTABLE_QUIRKS + bool "IO Pagetable quirks for performance" + depends on ARM || ARM64 + depends on IOMMU_IO_PGTABLE_FAST || IOMMU_IO_PGTABLE_LPAE + depends on ARM_SMMU + help + Enables some quirks that are used when creating the IOMMU's + page tables for a particular domain for faster translations. + The quirks that are supported deal with allowing for page + tables to be IO-coherent, allowing for page tables to be + saved in the system cache, and disabling the write-allocate + hint when saving page tables in the system cache. + + If unsure, say Y here. + endmenu config IOMMU_DEBUGFS @@ -399,6 +451,38 @@ config ARM_SMMU_V3 Say Y here if your system includes an IOMMU device implementing the ARM SMMUv3 architecture. +config ARM_SMMU_SELFTEST + bool "ARM SMMU self test support" + depends on ARM_SMMU + help + Enables self tests for arm smmu. Tests basic hardware + configurations like interrupts. Note that enabling this + option can marginally increase the boot time. + + If unsure, say N here. + +config IOMMU_TLBSYNC_DEBUG + bool "TLB sync timeout debug" + depends on ARM_SMMU + help + Enables to collect the SMMU system state information right + after the first TLB sync timeout failure by calling BUG(). + Note to use this only on debug builds. + + If unsure, say N here. + +config QCOM_LAZY_MAPPING + tristate "Reference counted iommu-mapping support" + depends on ION + depends on IOMMU_API + help + ION buffers may be shared between several software clients. + Reference counting the mapping may simplify coordination between + these clients, and decrease latency by preventing multiple + map/unmaps of the same region. + + If unsure, say N here. + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI @@ -454,6 +538,38 @@ config MTK_IOMMU_V1 if unsure, say N here. +menuconfig IOMMU_DEBUG + bool "IOMMU Profiling and Debugging" + help + This option is used to enable profiling and debugging in + the IOMMU framework code. IOMMU profiling and debugging + can be done through the debugfs nodes which this option + makes available. + +if IOMMU_DEBUG + +config IOMMU_DEBUG_TRACKING + bool "Track key IOMMU events" + select IOMMU_API + help + Enables additional debug tracking in the IOMMU framework code. + Tracking information and tests can be accessed through various + debugfs files. + + Say Y here if you need to debug IOMMU issues and are okay with + the performance penalty of the tracking. + +config IOMMU_TESTS + bool "Interactive IOMMU performance/functional tests" + select IOMMU_API + help + Enables a suite of IOMMU unit tests. The tests are runnable + through debugfs. Unlike the IOMMU_DEBUG_TRACKING option, the + impact of enabling this option to overal system performance + should be minimal. + +endif # IOMMU_DEBUG + config QCOM_IOMMU # Note: iommu drivers cannot (yet?) be built as modules bool "Qualcomm IOMMU Support" diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 8c71a15e986b..39e703649f88 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -4,11 +4,14 @@ obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o +obj-$(CONFIG_QCOM_LAZY_MAPPING) += msm_dma_iommu_mapping.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_IOMMU_IOVA) += iova.o +obj-$(CONFIG_IOMMU_IO_PGTABLE_FAST) += io-pgtable-fast.o dma-mapping-fast.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o +obj-$(CONFIG_IOMMU_DEBUG) += iommu-debug.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o diff --git a/drivers/iommu/arm-smmu-regs.h b/drivers/iommu/arm-smmu-regs.h index 1c278f7ae888..07b748747cfc 100644 --- a/drivers/iommu/arm-smmu-regs.h +++ b/drivers/iommu/arm-smmu-regs.h @@ -25,6 +25,9 @@ #define sCR0_VMID16EN (1 << 31) #define sCR0_BSU_SHIFT 14 #define sCR0_BSU_MASK 0x3 +#define sCR0_SHCFG_SHIFT 22 +#define sCR0_SHCFG_MASK 0x3 +#define sCR0_SHCFG_NSH 3 /* Auxiliary Configuration register */ #define ARM_SMMU_GR0_sACR 0x10 @@ -93,6 +96,8 @@ #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) #define SMR_VALID (1 << 31) #define SMR_MASK_SHIFT 16 +#define SMR_MASK_MASK 0x7FFF +#define SID_MASK 0x7FFF #define SMR_ID_SHIFT 0 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) @@ -101,6 +106,9 @@ #define S2CR_EXIDVALID (1 << 10) #define S2CR_TYPE_SHIFT 16 #define S2CR_TYPE_MASK 0x3 +#define S2CR_SHCFG_SHIFT 8 +#define S2CR_SHCFG_MASK 0x3 +#define S2CR_SHCFG_NSH 0x3 enum arm_smmu_s2cr_type { S2CR_TYPE_TRANS, S2CR_TYPE_BYPASS, @@ -136,6 +144,7 @@ enum arm_smmu_s2cr_privcfg { #define CBAR_IRPTNDX_MASK 0xff #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) +#define CBFRSYNRA_SID_MASK (0xffff) #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define CBA2R_RW64_32BIT (0 << 0) @@ -155,20 +164,38 @@ enum arm_smmu_s2cr_privcfg { #define ARM_SMMU_CB_S1_MAIR1 0x3c #define ARM_SMMU_CB_PAR 0x50 #define ARM_SMMU_CB_FSR 0x58 +#define ARM_SMMU_CB_FSRRESTORE 0x5c #define ARM_SMMU_CB_FAR 0x60 #define ARM_SMMU_CB_FSYNR0 0x68 +#define ARM_SMMU_CB_FSYNR1 0x6c #define ARM_SMMU_CB_S1_TLBIVA 0x600 #define ARM_SMMU_CB_S1_TLBIASID 0x610 +#define ARM_SMMU_CB_S1_TLBIALL 0x618 #define ARM_SMMU_CB_S1_TLBIVAL 0x620 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 #define ARM_SMMU_CB_TLBSYNC 0x7f0 #define ARM_SMMU_CB_TLBSTATUS 0x7f4 +#define TLBSTATUS_SACTIVE (1 << 0) #define ARM_SMMU_CB_ATS1PR 0x800 #define ARM_SMMU_CB_ATSR 0x8f0 +#define ARM_SMMU_STATS_SYNC_INV_TBU_ACK 0x25dc +#define ARM_SMMU_TBU_PWR_STATUS 0x2204 +#define ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR 0x2670 +#define SCTLR_MEM_ATTR_SHIFT 16 +#define SCTLR_SHCFG_SHIFT 22 +#define SCTLR_RACFG_SHIFT 24 +#define SCTLR_WACFG_SHIFT 26 +#define SCTLR_SHCFG_MASK 0x3 +#define SCTLR_SHCFG_NSH 0x3 +#define SCTLR_RACFG_RA 0x2 +#define SCTLR_WACFG_WA 0x2 +#define SCTLR_MEM_ATTR_OISH_WB_CACHE 0xf +#define SCTLR_MTCFG (1 << 20) #define SCTLR_S1_ASIDPNE (1 << 12) #define SCTLR_CFCFG (1 << 7) +#define SCTLR_HUPCF (1 << 8) #define SCTLR_CFIE (1 << 6) #define SCTLR_CFRE (1 << 5) #define SCTLR_E (1 << 4) diff --git a/drivers/iommu/arm-smmu-trace.h b/drivers/iommu/arm-smmu-trace.h new file mode 100644 index 000000000000..44cb073898db --- /dev/null +++ b/drivers/iommu/arm-smmu-trace.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM arm_smmu + +#if !defined(_TRACE_ARM_SMMU_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ARM_SMMU_H + +#include +#include + +struct device; + +DECLARE_EVENT_CLASS(iommu_tlbi, + + TP_PROTO(struct device *dev, u64 time), + + TP_ARGS(dev, time), + + TP_STRUCT__entry( + __string(device, dev_name(dev)) + __field(u64, time) + ), + + TP_fast_assign( + __assign_str(device, dev_name(dev)); + __entry->time = time; + ), + + TP_printk("IOMMU:%s %lld us", + __get_str(device), __entry->time + ) +); + +DEFINE_EVENT(iommu_tlbi, tlbi_start, + + TP_PROTO(struct device *dev, u64 time), + + TP_ARGS(dev, time) +); + +DEFINE_EVENT(iommu_tlbi, tlbi_end, + + TP_PROTO(struct device *dev, u64 time), + + TP_ARGS(dev, time) +); + +DEFINE_EVENT(iommu_tlbi, tlbsync_timeout, + + TP_PROTO(struct device *dev, u64 time), + + TP_ARGS(dev, time) +); + +TRACE_EVENT(smmu_init, + + TP_PROTO(u64 time), + + TP_ARGS(time), + + TP_STRUCT__entry( + __field(u64, time) + ), + + TP_fast_assign( + __entry->time = time; + ), + + TP_printk("ARM SMMU init latency: %lld us", __entry->time) +); +#endif /* _TRACE_ARM_SMMU_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE arm-smmu-trace + +/* This part must be outside protection */ +#include diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 586dd5a46d9f..abe122f1f74d 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -21,8 +21,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -30,7 +33,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -41,12 +45,20 @@ #include #include #include +#include +#include +#include +#include #include #include +#include #include "arm-smmu-regs.h" +#define CREATE_TRACE_POINTS +#include "arm-smmu-trace.h" + /* * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU * global register space are still, in fact, using a hypervisor to mediate it @@ -62,9 +74,14 @@ #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10) #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8) -#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ +#define TLB_LOOP_TIMEOUT 500000 /* 500ms */ #define TLB_SPIN_COUNT 10 +#define ARM_SMMU_IMPL_DEF0(smmu) \ + ((smmu)->base + (2 * (1 << (smmu)->pgshift))) +#define ARM_SMMU_IMPL_DEF1(smmu) \ + ((smmu)->base + (6 * (1 << (smmu)->pgshift))) + /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 @@ -99,6 +116,10 @@ #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 +#define ARM_SMMU_ICC_AVG_BW 0 +#define ARM_SMMU_ICC_PEAK_BW_HIGH 1000 +#define ARM_SMMU_ICC_PEAK_BW_LOW 0 + static int force_stage; /* * not really modular, but the easiest way to keep compat with existing @@ -124,18 +145,34 @@ enum arm_smmu_implementation { ARM_MMU500, CAVIUM_SMMUV2, QCOM_SMMUV2, + QCOM_SMMUV500, }; +struct arm_smmu_impl_def_reg { + u32 offset; + u32 value; +}; + +/* + * attach_count + * The SMR and S2CR registers are only programmed when the number of + * devices attached to the iommu using these registers is > 0. This + * is required for the "SID switch" use case for secure display. + * Protected by stream_map_mutex. + */ struct arm_smmu_s2cr { struct iommu_group *group; int count; + int attach_count; enum arm_smmu_s2cr_type type; enum arm_smmu_s2cr_privcfg privcfg; u8 cbndx; + bool cb_handoff; }; #define s2cr_init_val (struct arm_smmu_s2cr){ \ .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ + .cb_handoff = false, \ } struct arm_smmu_smr { @@ -163,11 +200,41 @@ struct arm_smmu_master_cfg { #define for_each_cfg_sme(fw, i, idx) \ for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) +/* + * Describes resources required for on/off power operation. + * Separate reference count is provided for atomic/nonatomic + * operations. + */ +struct arm_smmu_power_resources { + struct platform_device *pdev; + struct device *dev; + + struct clk **clocks; + int num_clocks; + + struct regulator_bulk_data *gdscs; + int num_gdscs; + + struct icc_path *icc_path; + + /* Protects power_count */ + struct mutex power_lock; + int power_count; + + /* Protects clock_refs_count */ + spinlock_t clock_refs_lock; + int clock_refs_count; + int regulator_defer; +}; + +struct arm_smmu_arch_ops; struct arm_smmu_device { struct device *dev; void __iomem *base; void __iomem *cb_base; + unsigned long size; + phys_addr_t phys_addr; unsigned long pgshift; #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) @@ -186,6 +253,11 @@ struct arm_smmu_device { u32 features; #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) +#define ARM_SMMU_OPT_FATAL_ASF (1 << 1) +#define ARM_SMMU_OPT_SKIP_INIT (1 << 2) +#define ARM_SMMU_OPT_3LVL_TABLES (1 << 4) +#define ARM_SMMU_OPT_NO_ASID_RETENTION (1 << 5) +#define ARM_SMMU_OPT_DISABLE_ATOS (1 << 6) u32 options; enum arm_smmu_arch_version version; enum arm_smmu_implementation model; @@ -202,7 +274,7 @@ struct arm_smmu_device { struct arm_smmu_smr *smrs; struct arm_smmu_s2cr *s2crs; struct mutex stream_map_mutex; - + struct mutex iommu_group_mutex; unsigned long va_size; unsigned long ipa_size; unsigned long pa_size; @@ -214,12 +286,29 @@ struct arm_smmu_device { struct clk_bulk_data *clks; int num_clks; + struct list_head list; + u32 cavium_id_base; /* Specific to Cavium */ spinlock_t global_sync_lock; /* IOMMU core code handle */ struct iommu_device iommu; + + /* Specific to QCOM */ + struct arm_smmu_impl_def_reg *impl_def_attach_registers; + unsigned int num_impl_def_attach_registers; + + struct arm_smmu_power_resources *pwr; + + spinlock_t atos_lock; + + /* protects idr */ + struct mutex idr_mutex; + struct idr asid_idr; + + struct arm_smmu_arch_ops *arch_ops; + void *archdata; }; enum arm_smmu_context_fmt { @@ -237,9 +326,21 @@ struct arm_smmu_cfg { u16 vmid; }; u32 cbar; + u32 procid; enum arm_smmu_context_fmt fmt; }; #define INVALID_IRPTNDX 0xff +#define INVALID_CBNDX 0xff +#define INVALID_ASID 0xffff +/* + * In V7L and V8L with TTBCR2.AS == 0, ASID is 8 bits. + * V8L 16 with TTBCR2.AS == 1 (16 bit ASID) isn't supported yet. + */ +#define MAX_ASID 0xff + +#define ARM_SMMU_CB_ASID(smmu, cfg) ((cfg)->asid) +#define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + \ + (cfg)->cbndx + 1) enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, @@ -248,16 +349,33 @@ enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_BYPASS, }; +struct arm_smmu_pte_info { + void *virt_addr; + size_t size; + struct list_head entry; +}; + struct arm_smmu_domain { struct arm_smmu_device *smmu; + struct device *dev; struct io_pgtable_ops *pgtbl_ops; const struct iommu_gather_ops *tlb_ops; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; bool non_strict; struct mutex init_mutex; /* Protects smmu pointer */ - spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ - struct iommu_domain domain; + spinlock_t cb_lock; /* Serialises ATS1* ops */ + spinlock_t sync_lock; /* Serialises TLB syncs */ + struct msm_io_pgtable_info pgtbl_info; + u32 attributes; + u32 secure_vmid; + struct list_head pte_info_list; + struct list_head unassign_list; + struct mutex assign_lock; + struct list_head secure_pool_list; + /* nonsecure pool protected by pgtbl_lock */ + struct list_head nonsecure_pool; + struct msm_iommu_domain domain; }; struct arm_smmu_option_prop { @@ -271,9 +389,41 @@ static bool using_legacy_binding, using_generic_binding; static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" }, + { ARM_SMMU_OPT_FATAL_ASF, "qcom,fatal-asf" }, + { ARM_SMMU_OPT_SKIP_INIT, "qcom,skip-init" }, + { ARM_SMMU_OPT_3LVL_TABLES, "qcom,use-3-lvl-tables" }, + { ARM_SMMU_OPT_NO_ASID_RETENTION, "qcom,no-asid-retention" }, + { ARM_SMMU_OPT_DISABLE_ATOS, "qcom,disable-atos" }, { 0, NULL}, }; +static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova); +static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, + dma_addr_t iova); +static void arm_smmu_destroy_domain_context(struct iommu_domain *domain); + +static int arm_smmu_prepare_pgtable(void *addr, void *cookie); +static void arm_smmu_unprepare_pgtable(void *cookie, void *addr, size_t size); +static int arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain); +static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain); + +static uint64_t arm_smmu_iova_to_pte(struct iommu_domain *domain, + dma_addr_t iova); + +static int arm_smmu_enable_s1_translations(struct arm_smmu_domain *smmu_domain); + +static int arm_smmu_alloc_cb(struct iommu_domain *domain, + struct arm_smmu_device *smmu, + struct device *dev); + +static int arm_smmu_setup_default_domain(struct device *dev, + struct iommu_domain *domain); +static int __arm_smmu_domain_set_attr(struct iommu_domain *domain, + enum iommu_attr attr, void *data); +static int arm_smmu_domain_get_attr(struct iommu_domain *domain, + enum iommu_attr attr, void *data); + static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) { if (pm_runtime_enabled(smmu->dev)) @@ -290,7 +440,14 @@ static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) { - return container_of(dom, struct arm_smmu_domain, domain); + struct msm_iommu_domain *msm_domain = to_msm_iommu_domain(dom); + + return container_of(msm_domain, struct arm_smmu_domain, domain); +} + +static struct arm_smmu_domain *cb_cfg_to_smmu_domain(struct arm_smmu_cfg *cfg) +{ + return container_of(cfg, struct arm_smmu_domain, cfg); } static void parse_driver_options(struct arm_smmu_device *smmu) @@ -301,12 +458,230 @@ static void parse_driver_options(struct arm_smmu_device *smmu) if (of_property_read_bool(smmu->dev->of_node, arm_smmu_options[i].prop)) { smmu->options |= arm_smmu_options[i].opt; - dev_notice(smmu->dev, "option %s\n", + dev_dbg(smmu->dev, "option %s\n", arm_smmu_options[i].prop); } } while (arm_smmu_options[++i].opt); } +static bool is_dynamic_domain(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + return !!(smmu_domain->attributes & (1 << DOMAIN_ATTR_DYNAMIC)); +} + +static bool is_iommu_pt_coherent(struct arm_smmu_domain *smmu_domain) +{ + if (smmu_domain->attributes & + (1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT)) + return true; + else if (smmu_domain->smmu && smmu_domain->smmu->dev) + return dev_is_dma_coherent(smmu_domain->smmu->dev); + else + return false; +} + +static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain) +{ + return (smmu_domain->secure_vmid != VMID_INVAL); +} + +static void arm_smmu_secure_domain_lock(struct arm_smmu_domain *smmu_domain) +{ + if (arm_smmu_has_secure_vmid(smmu_domain)) + mutex_lock(&smmu_domain->assign_lock); +} + +static void arm_smmu_secure_domain_unlock(struct arm_smmu_domain *smmu_domain) +{ + if (arm_smmu_has_secure_vmid(smmu_domain)) + mutex_unlock(&smmu_domain->assign_lock); +} + +#ifdef CONFIG_ARM_SMMU_SELFTEST + +static int selftest; +module_param_named(selftest, selftest, int, 0644); +static int irq_count; + +static DECLARE_WAIT_QUEUE_HEAD(wait_int); +static irqreturn_t arm_smmu_cf_selftest(int irq, void *cb_base) +{ + u32 fsr; + struct irq_data *irq_data = irq_get_irq_data(irq); + unsigned long hwirq = ULONG_MAX; + + fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); + + irq_count++; + if (irq_data) + hwirq = irq_data->hwirq; + pr_info("Interrupt (irq:%d hwirq:%ld) received, fsr:0x%x\n", + irq, hwirq, fsr); + + writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR); + + wake_up(&wait_int); + return IRQ_HANDLED; +} + +static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) +{ + int cb; + int cb_count = 0; + + if (!selftest) + return; + + cb = smmu->num_s2_context_banks; + + if (smmu->version < ARM_SMMU_V2) + return; + + for_each_clear_bit_from(cb, smmu->context_map, + smmu->num_context_banks) { + int irq; + int ret; + void *cb_base; + u32 reg; + u32 reg_orig; + int irq_cnt; + + irq = smmu->irqs[smmu->num_global_irqs + cb]; + cb_base = ARM_SMMU_CB(smmu, cb); + + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + arm_smmu_cf_selftest, + IRQF_ONESHOT | IRQF_SHARED, + "arm-smmu-context-fault", cb_base); + if (ret < 0) { + dev_err(smmu->dev, + "Failed to request cntx IRQ %d (%u)\n", + cb, irq); + continue; + } + + cb_count++; + irq_cnt = irq_count; + + reg_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR); + reg = reg_orig | SCTLR_CFIE | SCTLR_CFRE; + + writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); + dev_info(smmu->dev, "Testing cntx %d irq %d\n", cb, irq); + + /* Make sure ARM_SMMU_CB_SCTLR is configured */ + wmb(); + writel_relaxed(FSR_TF, cb_base + ARM_SMMU_CB_FSRRESTORE); + + wait_event_timeout(wait_int, (irq_count > irq_cnt), + msecs_to_jiffies(1000)); + + /* Make sure ARM_SMMU_CB_FSRRESTORE is written to */ + wmb(); + writel_relaxed(reg_orig, cb_base + ARM_SMMU_CB_SCTLR); + devm_free_irq(smmu->dev, irq, cb_base); + } + + dev_info(smmu->dev, + "Interrupt selftest completed...\n"); + dev_info(smmu->dev, + "Tested %d contexts, received %d interrupts\n", + cb_count, irq_count); + WARN_ON(cb_count != irq_count); + irq_count = 0; +} +#else +static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) +{ +} +#endif + +/* + * init() + * Hook for additional device tree parsing at probe time. + * + * device_reset() + * Hook for one-time architecture-specific register settings. + * + * iova_to_phys_hard() + * Provides debug information. May be called from the context fault irq handler. + * + * init_context_bank() + * Hook for architecture-specific settings which require knowledge of the + * dynamically allocated context bank number. + * + * device_group() + * Hook for checking whether a device is compatible with a said group. + */ +struct arm_smmu_arch_ops { + int (*init)(struct arm_smmu_device *smmu); + void (*device_reset)(struct arm_smmu_device *smmu); + phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain, + dma_addr_t iova); + void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, + struct device *dev); + int (*device_group)(struct device *dev, struct iommu_group *group); +}; + +static int arm_smmu_arch_init(struct arm_smmu_device *smmu) +{ + if (!smmu->arch_ops) + return 0; + if (!smmu->arch_ops->init) + return 0; + return smmu->arch_ops->init(smmu); +} + +static void arm_smmu_arch_device_reset(struct arm_smmu_device *smmu) +{ + if (!smmu->arch_ops) + return; + if (!smmu->arch_ops->device_reset) + return; + return smmu->arch_ops->device_reset(smmu); +} + +static void arm_smmu_arch_init_context_bank( + struct arm_smmu_domain *smmu_domain, struct device *dev) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (!smmu->arch_ops) + return; + if (!smmu->arch_ops->init_context_bank) + return; + return smmu->arch_ops->init_context_bank(smmu_domain, dev); +} + +static int arm_smmu_arch_device_group(struct device *dev, + struct iommu_group *group) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + + if (!smmu->arch_ops) + return 0; + if (!smmu->arch_ops->device_group) + return 0; + return smmu->arch_ops->device_group(dev, group); +} + +static void arm_smmu_arch_write_sync(struct arm_smmu_device *smmu) +{ + u32 id; + + if (!smmu) + return; + + /* Read to complete prior write transcations */ + id = readl_relaxed(ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_ID0); + + /* Wait for read to complete before off */ + rmb(); +} + static struct device_node *dev_get_dev_node(struct device *dev) { if (dev_is_pci(dev)) { @@ -343,7 +718,7 @@ static int __find_legacy_master_phandle(struct device *dev, void *data) } static struct platform_driver arm_smmu_driver; -static struct iommu_ops arm_smmu_ops; +static struct msm_iommu_ops arm_smmu_ops; static int arm_smmu_register_legacy_master(struct device *dev, struct arm_smmu_device **smmu) @@ -354,8 +729,9 @@ static int arm_smmu_register_legacy_master(struct device *dev, void *data = ⁢ u32 *sids; __be32 pci_sid; - int err; + int err = 0; + memset(&it, 0, sizeof(it)); np = dev_get_dev_node(dev); if (!np || !of_find_property(np, "#stream-id-cells", NULL)) { of_node_put(np); @@ -381,7 +757,7 @@ static int arm_smmu_register_legacy_master(struct device *dev, } err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, - &arm_smmu_ops); + &arm_smmu_ops.iommu_ops); if (err) return err; @@ -414,23 +790,313 @@ static void __arm_smmu_free_bitmap(unsigned long *map, int idx) clear_bit(idx, map); } +static int arm_smmu_prepare_clocks(struct arm_smmu_power_resources *pwr) +{ + int i, ret = 0; + + for (i = 0; i < pwr->num_clocks; ++i) { + ret = clk_prepare(pwr->clocks[i]); + if (ret) { + dev_err(pwr->dev, "Couldn't prepare clock #%d\n", i); + while (i--) + clk_unprepare(pwr->clocks[i]); + break; + } + } + return ret; +} + +static void arm_smmu_unprepare_clocks(struct arm_smmu_power_resources *pwr) +{ + int i; + + for (i = pwr->num_clocks; i; --i) + clk_unprepare(pwr->clocks[i - 1]); +} + +static int arm_smmu_enable_clocks(struct arm_smmu_power_resources *pwr) +{ + int i, ret = 0; + + for (i = 0; i < pwr->num_clocks; ++i) { + ret = clk_enable(pwr->clocks[i]); + if (ret) { + dev_err(pwr->dev, "Couldn't enable clock #%d\n", i); + while (i--) + clk_disable(pwr->clocks[i]); + break; + } + } + + return ret; +} + +static void arm_smmu_disable_clocks(struct arm_smmu_power_resources *pwr) +{ + int i; + + for (i = pwr->num_clocks; i; --i) + clk_disable(pwr->clocks[i - 1]); +} + +static int arm_smmu_raise_interconnect_bw(struct arm_smmu_power_resources *pwr) +{ + if (!pwr->icc_path) + return 0; + return icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW, + ARM_SMMU_ICC_PEAK_BW_HIGH); +} + +static void arm_smmu_lower_interconnect_bw(struct arm_smmu_power_resources *pwr) +{ + if (!pwr->icc_path) + return; + WARN_ON(icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW, + ARM_SMMU_ICC_PEAK_BW_LOW)); +} + +static int arm_smmu_enable_regulators(struct arm_smmu_power_resources *pwr) +{ + struct regulator_bulk_data *consumers; + int num_consumers, ret; + int i; + + num_consumers = pwr->num_gdscs; + consumers = pwr->gdscs; + for (i = 0; i < num_consumers; i++) { + ret = regulator_enable(consumers[i].consumer); + if (ret) + goto out; + } + return 0; + +out: + i -= 1; + for (; i >= 0; i--) + regulator_disable(consumers[i].consumer); + return ret; +} + +static int arm_smmu_disable_regulators(struct arm_smmu_power_resources *pwr) +{ + struct regulator_bulk_data *consumers; + int i; + int num_consumers, ret, r; + + num_consumers = pwr->num_gdscs; + consumers = pwr->gdscs; + for (i = num_consumers - 1; i >= 0; --i) { + ret = regulator_disable_deferred(consumers[i].consumer, + pwr->regulator_defer); + if (ret != 0) + goto err; + } + + return 0; + +err: + pr_err("Failed to disable %s: %d\n", consumers[i].supply, ret); + for (++i; i < num_consumers; ++i) { + r = regulator_enable(consumers[i].consumer); + if (r != 0) + pr_err("Failed to rename %s: %d\n", + consumers[i].supply, r); + } + + return ret; +} + +/* Clocks must be prepared before this (arm_smmu_prepare_clocks) */ +static int arm_smmu_power_on_atomic(struct arm_smmu_power_resources *pwr) +{ + int ret = 0; + unsigned long flags; + + spin_lock_irqsave(&pwr->clock_refs_lock, flags); + if (pwr->clock_refs_count > 0) { + pwr->clock_refs_count++; + spin_unlock_irqrestore(&pwr->clock_refs_lock, flags); + return 0; + } + + ret = arm_smmu_enable_clocks(pwr); + if (!ret) + pwr->clock_refs_count = 1; + + spin_unlock_irqrestore(&pwr->clock_refs_lock, flags); + return ret; +} + +/* Clocks should be unprepared after this (arm_smmu_unprepare_clocks) */ +static void arm_smmu_power_off_atomic(struct arm_smmu_power_resources *pwr) +{ + unsigned long flags; + struct arm_smmu_device *smmu = pwr->dev->driver_data; + + arm_smmu_arch_write_sync(smmu); + + spin_lock_irqsave(&pwr->clock_refs_lock, flags); + if (pwr->clock_refs_count == 0) { + WARN(1, "%s: bad clock_ref_count\n", dev_name(pwr->dev)); + spin_unlock_irqrestore(&pwr->clock_refs_lock, flags); + return; + + } else if (pwr->clock_refs_count > 1) { + pwr->clock_refs_count--; + spin_unlock_irqrestore(&pwr->clock_refs_lock, flags); + return; + } + + arm_smmu_disable_clocks(pwr); + + pwr->clock_refs_count = 0; + spin_unlock_irqrestore(&pwr->clock_refs_lock, flags); +} + +static int arm_smmu_power_on_slow(struct arm_smmu_power_resources *pwr) +{ + int ret; + + mutex_lock(&pwr->power_lock); + if (pwr->power_count > 0) { + pwr->power_count += 1; + mutex_unlock(&pwr->power_lock); + return 0; + } + + ret = arm_smmu_raise_interconnect_bw(pwr); + if (ret) + goto out_unlock; + + ret = arm_smmu_enable_regulators(pwr); + if (ret) + goto out_disable_bus; + + ret = arm_smmu_prepare_clocks(pwr); + if (ret) + goto out_disable_regulators; + + pwr->power_count = 1; + mutex_unlock(&pwr->power_lock); + return 0; + +out_disable_regulators: + regulator_bulk_disable(pwr->num_gdscs, pwr->gdscs); +out_disable_bus: + arm_smmu_lower_interconnect_bw(pwr); +out_unlock: + mutex_unlock(&pwr->power_lock); + return ret; +} + +static void arm_smmu_power_off_slow(struct arm_smmu_power_resources *pwr) +{ + mutex_lock(&pwr->power_lock); + if (pwr->power_count == 0) { + WARN(1, "%s: Bad power count\n", dev_name(pwr->dev)); + mutex_unlock(&pwr->power_lock); + return; + + } else if (pwr->power_count > 1) { + pwr->power_count--; + mutex_unlock(&pwr->power_lock); + return; + } + + arm_smmu_unprepare_clocks(pwr); + arm_smmu_disable_regulators(pwr); + arm_smmu_lower_interconnect_bw(pwr); + pwr->power_count = 0; + mutex_unlock(&pwr->power_lock); +} + +static int arm_smmu_power_on(struct arm_smmu_power_resources *pwr) +{ + int ret; + + ret = arm_smmu_power_on_slow(pwr); + if (ret) + return ret; + + ret = arm_smmu_power_on_atomic(pwr); + if (ret) + goto out_disable; + + return 0; + +out_disable: + arm_smmu_power_off_slow(pwr); + return ret; +} + +static void arm_smmu_power_off(struct arm_smmu_power_resources *pwr) +{ + arm_smmu_power_off_atomic(pwr); + arm_smmu_power_off_slow(pwr); +} + +/* + * Must be used instead of arm_smmu_power_on if it may be called from + * atomic context + */ +static int arm_smmu_domain_power_on(struct iommu_domain *domain, + struct arm_smmu_device *smmu) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int atomic_domain = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC); + + if (atomic_domain) + return arm_smmu_power_on_atomic(smmu->pwr); + + return arm_smmu_power_on(smmu->pwr); +} + +/* + * Must be used instead of arm_smmu_power_on if it may be called from + * atomic context + */ +static void arm_smmu_domain_power_off(struct iommu_domain *domain, + struct arm_smmu_device *smmu) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int atomic_domain = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC); + + if (atomic_domain) { + arm_smmu_power_off_atomic(smmu->pwr); + return; + } + + arm_smmu_power_off(smmu->pwr); +} + /* Wait for any pending TLB invalidations to complete */ -static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, +static int __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, void __iomem *sync, void __iomem *status) { unsigned int spin_cnt, delay; + u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress; writel_relaxed(QCOM_DUMMY_VAL, sync); for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE)) - return; + return 0; cpu_relax(); } udelay(delay); } + sync_inv_ack = scm_io_read((unsigned long)(smmu->phys_addr + + ARM_SMMU_STATS_SYNC_INV_TBU_ACK)); + tbu_pwr_status = scm_io_read((unsigned long)(smmu->phys_addr + + ARM_SMMU_TBU_PWR_STATUS)); + sync_inv_progress = scm_io_read((unsigned long)(smmu->phys_addr + + ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR)); + trace_tlbsync_timeout(smmu->dev, 0); dev_err_ratelimited(smmu->dev, - "TLB sync timed out -- SMMU may be deadlocked\n"); + "TLB sync timed out -- SMMU may be deadlocked ack 0x%x pwr 0x%x sync and invalidation progress 0x%x\n", + sync_inv_ack, tbu_pwr_status, sync_inv_progress); + BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); + return -EINVAL; } static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) @@ -439,8 +1105,10 @@ static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) unsigned long flags; spin_lock_irqsave(&smmu->global_sync_lock, flags); - __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC, - base + ARM_SMMU_GR0_sTLBGSTATUS); + if (__arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC, + base + ARM_SMMU_GR0_sTLBGSTATUS)) + dev_err_ratelimited(smmu->dev, + "TLB global sync failed!\n"); spin_unlock_irqrestore(&smmu->global_sync_lock, flags); } @@ -451,10 +1119,14 @@ static void arm_smmu_tlb_sync_context(void *cookie) void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx); unsigned long flags; - spin_lock_irqsave(&smmu_domain->cb_lock, flags); - __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC, - base + ARM_SMMU_CB_TLBSTATUS); - spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + spin_lock_irqsave(&smmu_domain->sync_lock, flags); + if (__arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC, + base + ARM_SMMU_CB_TLBSTATUS)) + dev_err_ratelimited(smmu->dev, + "TLB sync on cb%d failed for device %s\n", + smmu_domain->cfg.cbndx, + dev_name(smmu_domain->dev)); + spin_unlock_irqrestore(&smmu_domain->sync_lock, flags); } static void arm_smmu_tlb_sync_vmid(void *cookie) @@ -467,15 +1139,22 @@ static void arm_smmu_tlb_sync_vmid(void *cookie) static void arm_smmu_tlb_inv_context_s1(void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; + struct device *dev = smmu_domain->dev; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); + bool use_tlbiall = smmu->options & ARM_SMMU_OPT_NO_ASID_RETENTION; + ktime_t cur = ktime_get(); + + trace_tlbi_start(dev, 0); + + if (!use_tlbiall) + writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID); + else + writel(QCOM_DUMMY_VAL, base + ARM_SMMU_CB_S1_TLBIALL); - /* - * NOTE: this is not a relaxed write; it needs to guarantee that PTEs - * cleared by the current CPU are visible to the SMMU before the TLBI. - */ - writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID); arm_smmu_tlb_sync_context(cookie); + trace_tlbi_end(dev, ktime_us_delta(ktime_get(), cur)); } static void arm_smmu_tlb_inv_context_s2(void *cookie) @@ -484,7 +1163,6 @@ static void arm_smmu_tlb_inv_context_s2(void *cookie) struct arm_smmu_device *smmu = smmu_domain->smmu; void __iomem *base = ARM_SMMU_GR0(smmu); - /* NOTE: see above */ writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); arm_smmu_tlb_sync_global(smmu); } @@ -494,13 +1172,15 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, { struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx); + bool use_tlbiall = smmu->options & ARM_SMMU_OPT_NO_ASID_RETENTION; if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) wmb(); - if (stage1) { + if (stage1 && !use_tlbiall) { reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { @@ -518,6 +1198,9 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, iova += granule >> 12; } while (size -= granule); } + } else if (stage1 && use_tlbiall) { + reg += ARM_SMMU_CB_S1_TLBIALL; + writel_relaxed(0, reg); } else { reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2; @@ -547,51 +1230,355 @@ static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size, writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID); } -static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = { - .tlb_flush_all = arm_smmu_tlb_inv_context_s1, - .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, - .tlb_sync = arm_smmu_tlb_sync_context, +struct arm_smmu_secure_pool_chunk { + void *addr; + size_t size; + struct list_head list; }; -static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = { - .tlb_flush_all = arm_smmu_tlb_inv_context_s2, - .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, - .tlb_sync = arm_smmu_tlb_sync_context, +static void *arm_smmu_secure_pool_remove(struct arm_smmu_domain *smmu_domain, + size_t size) +{ + struct arm_smmu_secure_pool_chunk *it; + + list_for_each_entry(it, &smmu_domain->secure_pool_list, list) { + if (it->size == size) { + void *addr = it->addr; + + list_del(&it->list); + kfree(it); + return addr; + } + } + + return NULL; +} + +static int arm_smmu_secure_pool_add(struct arm_smmu_domain *smmu_domain, + void *addr, size_t size) +{ + struct arm_smmu_secure_pool_chunk *chunk; + + chunk = kmalloc(sizeof(*chunk), GFP_ATOMIC); + if (!chunk) + return -ENOMEM; + + chunk->addr = addr; + chunk->size = size; + memset(addr, 0, size); + list_add(&chunk->list, &smmu_domain->secure_pool_list); + + return 0; +} + +static void arm_smmu_secure_pool_destroy(struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_secure_pool_chunk *it, *i; + + list_for_each_entry_safe(it, i, &smmu_domain->secure_pool_list, list) { + arm_smmu_unprepare_pgtable(smmu_domain, it->addr, it->size); + /* pages will be freed later (after being unassigned) */ + list_del(&it->list); + kfree(it); + } +} + +static void *arm_smmu_alloc_pages_exact(void *cookie, + size_t size, gfp_t gfp_mask) +{ + int ret; + void *page; + struct arm_smmu_domain *smmu_domain = cookie; + + if (!arm_smmu_has_secure_vmid(smmu_domain)) { + struct page *pg; + /* size is expected to be 4K with current configuration */ + if (size == PAGE_SIZE) { + pg = list_first_entry_or_null( + &smmu_domain->nonsecure_pool, struct page, lru); + if (pg) { + list_del_init(&pg->lru); + return page_address(pg); + } + } + return alloc_pages_exact(size, gfp_mask); + } + + page = arm_smmu_secure_pool_remove(smmu_domain, size); + if (page) + return page; + + page = alloc_pages_exact(size, gfp_mask); + if (page) { + ret = arm_smmu_prepare_pgtable(page, cookie); + if (ret) { + free_pages_exact(page, size); + return NULL; + } + } + + return page; +} + +static void arm_smmu_free_pages_exact(void *cookie, void *virt, size_t size) +{ + struct arm_smmu_domain *smmu_domain = cookie; + + if (!arm_smmu_has_secure_vmid(smmu_domain)) { + free_pages_exact(virt, size); + return; + } + + if (arm_smmu_secure_pool_add(smmu_domain, virt, size)) + arm_smmu_unprepare_pgtable(smmu_domain, virt, size); +} + +static const struct msm_iommu_gather_ops arm_smmu_s1_tlb_ops = { + .alloc_pages_exact = arm_smmu_alloc_pages_exact, + .free_pages_exact = arm_smmu_free_pages_exact, + .tlb_ops = { + .tlb_flush_all = arm_smmu_tlb_inv_context_s1, + .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, + .tlb_sync = arm_smmu_tlb_sync_context, + } }; -static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = { - .tlb_flush_all = arm_smmu_tlb_inv_context_s2, - .tlb_add_flush = arm_smmu_tlb_inv_vmid_nosync, - .tlb_sync = arm_smmu_tlb_sync_vmid, +static const struct msm_iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = { + .alloc_pages_exact = arm_smmu_alloc_pages_exact, + .free_pages_exact = arm_smmu_free_pages_exact, + .tlb_ops = { + .tlb_flush_all = arm_smmu_tlb_inv_context_s2, + .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, + .tlb_sync = arm_smmu_tlb_sync_context, + } }; +static const struct msm_iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = { + .alloc_pages_exact = arm_smmu_alloc_pages_exact, + .free_pages_exact = arm_smmu_free_pages_exact, + .tlb_ops = { + .tlb_flush_all = arm_smmu_tlb_inv_context_s2, + .tlb_add_flush = arm_smmu_tlb_inv_vmid_nosync, + .tlb_sync = arm_smmu_tlb_sync_vmid, + } +}; + +static void print_ctx_regs(struct arm_smmu_device *smmu, struct arm_smmu_cfg + *cfg, unsigned int fsr) +{ + u32 fsynr0; + void __iomem *cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); + void __iomem *gr1_base = ARM_SMMU_GR1(smmu); + bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; + + fsynr0 = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); + + dev_err(smmu->dev, "FAR = 0x%016llx\n", + readq_relaxed(cb_base + ARM_SMMU_CB_FAR)); + dev_err(smmu->dev, "PAR = 0x%pK\n", + (void *) readq_relaxed(cb_base + ARM_SMMU_CB_PAR)); + + dev_err(smmu->dev, + "FSR = 0x%08x [%s%s%s%s%s%s%s%s%s%s]\n", + fsr, + (fsr & 0x02) ? (fsynr0 & 0x10 ? + "TF W " : "TF R ") : "", + (fsr & 0x04) ? "AFF " : "", + (fsr & 0x08) ? (fsynr0 & 0x10 ? + "PF W " : "PF R ") : "", + (fsr & 0x10) ? "EF " : "", + (fsr & 0x20) ? "TLBMCF " : "", + (fsr & 0x40) ? "TLBLKF " : "", + (fsr & 0x80) ? "MHF " : "", + (fsr & 0x100) ? "UUT " : "", + (fsr & 0x40000000) ? "SS " : "", + (fsr & 0x80000000) ? "MULTI " : ""); + + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { + dev_err(smmu->dev, "TTBR0 = 0x%pK\n", + (void *) (unsigned long) + readl_relaxed(cb_base + ARM_SMMU_CB_TTBR0)); + dev_err(smmu->dev, "TTBR1 = 0x%pK\n", + (void *) (unsigned long) + readl_relaxed(cb_base + ARM_SMMU_CB_TTBR1)); + } else { + dev_err(smmu->dev, "TTBR0 = 0x%pK\n", + (void *) readq_relaxed(cb_base + ARM_SMMU_CB_TTBR0)); + if (stage1) + dev_err(smmu->dev, "TTBR1 = 0x%pK\n", + (void *) readq_relaxed(cb_base + + ARM_SMMU_CB_TTBR1)); + } + + + dev_err(smmu->dev, "SCTLR = 0x%08x ACTLR = 0x%08x\n", + readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR), + readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR)); + dev_err(smmu->dev, "CBAR = 0x%08x\n", + readl_relaxed(gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx))); + dev_err(smmu->dev, "MAIR0 = 0x%08x MAIR1 = 0x%08x\n", + readl_relaxed(cb_base + ARM_SMMU_CB_S1_MAIR0), + readl_relaxed(cb_base + ARM_SMMU_CB_S1_MAIR1)); + +} + +static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain, + dma_addr_t iova, u32 fsr) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct msm_io_pgtable_info *pgtbl_info = &smmu_domain->pgtbl_info; + phys_addr_t phys; + phys_addr_t phys_post_tlbiall; + + phys = arm_smmu_iova_to_phys_hard(domain, iova); + pgtbl_info->pgtbl_cfg.tlb->tlb_flush_all(smmu_domain); + phys_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova); + + if (phys != phys_post_tlbiall) { + dev_err(smmu->dev, + "ATOS results differed across TLBIALL...\n" + "Before: %pa After: %pa\n", &phys, &phys_post_tlbiall); + } + + return (phys == 0 ? phys_post_tlbiall : phys); +} + static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { - u32 fsr, fsynr, cbfrsynra; + int flags, ret, tmp; + u32 fsr, fsynr0, fsynr1, frsynra, resume; unsigned long iova; struct iommu_domain *domain = dev; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_device *smmu = smmu_domain->smmu; - void __iomem *gr1_base = ARM_SMMU_GR1(smmu); void __iomem *cb_base; + void __iomem *gr1_base; + bool fatal_asf = smmu->options & ARM_SMMU_OPT_FATAL_ASF; + phys_addr_t phys_soft; + uint64_t pte; + bool non_fatal_fault = !!(smmu_domain->attributes & + (1 << DOMAIN_ATTR_NON_FATAL_FAULTS)); + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + ret = arm_smmu_power_on(smmu->pwr); + if (ret) + return IRQ_NONE; + + gr1_base = ARM_SMMU_GR1(smmu); cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); - if (!(fsr & FSR_FAULT)) - return IRQ_NONE; + if (!(fsr & FSR_FAULT)) { + ret = IRQ_NONE; + goto out_power_off; + } + + if (fatal_asf && (fsr & FSR_ASF)) { + dev_err(smmu->dev, + "Took an address size fault. Refusing to recover.\n"); + BUG(); + } + + fsynr0 = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); + fsynr1 = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR1); + flags = fsynr0 & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + if (fsr & FSR_TF) + flags |= IOMMU_FAULT_TRANSLATION; + if (fsr & FSR_PF) + flags |= IOMMU_FAULT_PERMISSION; + if (fsr & FSR_EF) + flags |= IOMMU_FAULT_EXTERNAL; + if (fsr & FSR_SS) + flags |= IOMMU_FAULT_TRANSACTION_STALLED; - fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); - cbfrsynra = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); + phys_soft = arm_smmu_iova_to_phys(domain, iova); + frsynra = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); + frsynra &= CBFRSYNRA_SID_MASK; + tmp = report_iommu_fault(domain, smmu->dev, iova, flags); + if (!tmp || (tmp == -EBUSY)) { + dev_dbg(smmu->dev, + "Context fault handled by client: iova=0x%08lx, cb=%d, fsr=0x%x, fsynr0=0x%x, fsynr1=0x%x\n", + iova, cfg->cbndx, fsr, fsynr0, fsynr1); + dev_dbg(smmu->dev, + "soft iova-to-phys=%pa\n", &phys_soft); + ret = IRQ_HANDLED; + resume = RESUME_TERMINATE; + } else { + if (__ratelimit(&_rs)) { + phys_addr_t phys_atos; - dev_err_ratelimited(smmu->dev, - "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", - fsr, iova, fsynr, cbfrsynra, cfg->cbndx); + print_ctx_regs(smmu, cfg, fsr); + phys_atos = arm_smmu_verify_fault(domain, iova, fsr); + dev_err(smmu->dev, + "Unhandled context fault: iova=0x%08lx, cb=%d, fsr=0x%x, fsynr0=0x%x, fsynr1=0x%x\n", + iova, cfg->cbndx, fsr, fsynr0, fsynr1); - writel(fsr, cb_base + ARM_SMMU_CB_FSR); - return IRQ_HANDLED; + + dev_err(smmu->dev, + "soft iova-to-phys=%pa\n", &phys_soft); + if (!phys_soft) + dev_err(smmu->dev, + "SOFTWARE TABLE WALK FAILED! Looks like %s accessed an unmapped address!\n", + dev_name(smmu->dev)); + else { + pte = arm_smmu_iova_to_pte(domain, iova); + dev_err(smmu->dev, "PTE = %016llx\n", pte); + } + if (phys_atos) + dev_err(smmu->dev, "hard iova-to-phys (ATOS)=%pa\n", + &phys_atos); + else + dev_err(smmu->dev, "hard iova-to-phys (ATOS) failed\n"); + dev_err(smmu->dev, "SID=0x%x\n", frsynra); + } + ret = IRQ_HANDLED; + resume = RESUME_TERMINATE; + if (!non_fatal_fault) { + dev_err(smmu->dev, + "Unhandled arm-smmu context fault!\n"); + BUG(); + } + } + + /* + * If the client returns -EBUSY, do not clear FSR and do not RESUME + * if stalled. This is required to keep the IOMMU client stalled on + * the outstanding fault. This gives the client a chance to take any + * debug action and then terminate the stalled transaction. + * So, the sequence in case of stall on fault should be: + * 1) Do not clear FSR or write to RESUME here + * 2) Client takes any debug action + * 3) Client terminates the stalled transaction and resumes the IOMMU + * 4) Client clears FSR. The FSR should only be cleared after 3) and + * not before so that the fault remains outstanding. This ensures + * SCTLR.HUPCF has the desired effect if subsequent transactions also + * need to be terminated. + */ + if (tmp != -EBUSY) { + /* Clear the faulting FSR */ + writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR); + + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation + */ + wmb(); + + /* Retry or terminate any stalled transactions */ + if (fsr & FSR_SS) + writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME); + } + +out_power_off: + arm_smmu_power_off(smmu->pwr); + + return ret; } static irqreturn_t arm_smmu_global_fault(int irq, void *dev) @@ -600,13 +1587,18 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) struct arm_smmu_device *smmu = dev; void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu); + if (arm_smmu_power_on(smmu->pwr)) + return IRQ_NONE; + gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR); gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0); gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1); gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2); - if (!gfsr) + if (!gfsr) { + arm_smmu_power_off(smmu->pwr); return IRQ_NONE; + } dev_err_ratelimited(smmu->dev, "Unexpected global fault, this could be serious\n"); @@ -615,6 +1607,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) gfsr, gfsynr0, gfsynr1, gfsynr2); writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR); + arm_smmu_power_off(smmu->pwr); return IRQ_HANDLED; } @@ -669,12 +1662,14 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, } } -static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) +static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx, + u32 attributes) { u32 reg; bool stage1; struct arm_smmu_cb *cb = &smmu->cbs[idx]; struct arm_smmu_cfg *cfg = cb->cfg; + struct arm_smmu_domain *smmu_domain = NULL; void __iomem *cb_base, *gr1_base; cb_base = ARM_SMMU_CB(smmu, idx); @@ -746,7 +1741,38 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) } /* SCTLR */ - reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M; + reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE; + + /* + * Ensure bypass transactions are Non-shareable only for clients + * who are not io-coherent. + */ + smmu_domain = cb_cfg_to_smmu_domain(cfg); + + /* + * Override cacheability, shareability, r/w allocation for + * clients who are io-coherent + */ + if (of_dma_is_coherent(smmu_domain->dev->of_node)) { + + reg |= SCTLR_RACFG_RA << SCTLR_RACFG_SHIFT; + reg |= SCTLR_WACFG_WA << SCTLR_WACFG_SHIFT; + reg |= SCTLR_MTCFG; + reg |= SCTLR_MEM_ATTR_OISH_WB_CACHE << SCTLR_MEM_ATTR_SHIFT; + } else + reg |= SCTLR_SHCFG_NSH << SCTLR_SHCFG_SHIFT; + + if (attributes & (1 << DOMAIN_ATTR_CB_STALL_DISABLE)) { + reg &= ~SCTLR_CFCFG; + reg |= SCTLR_HUPCF; + } + + if (attributes & (1 << DOMAIN_ATTR_NO_CFRE)) + reg &= ~SCTLR_CFRE; + + if ((!(attributes & (1 << DOMAIN_ATTR_S1_BYPASS)) && + !(attributes & (1 << DOMAIN_ATTR_EARLY_MAP))) || !stage1) + reg |= SCTLR_M; if (stage1) reg |= SCTLR_S1_ASIDPNE; if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) @@ -755,25 +1781,224 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); } -static int arm_smmu_init_domain_context(struct iommu_domain *domain, - struct arm_smmu_device *smmu) +static int arm_smmu_init_asid(struct iommu_domain *domain, + struct arm_smmu_device *smmu) { - int irq, start, ret = 0; - unsigned long ias, oas; - struct io_pgtable_ops *pgtbl_ops; - struct io_pgtable_cfg pgtbl_cfg; - enum io_pgtable_fmt fmt; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + bool dynamic = is_dynamic_domain(domain); + int ret; + + if (!dynamic) { + cfg->asid = cfg->cbndx + 1; + } else { + mutex_lock(&smmu->idr_mutex); + ret = idr_alloc_cyclic(&smmu->asid_idr, domain, + smmu->num_context_banks + 2, + MAX_ASID + 1, GFP_KERNEL); + + mutex_unlock(&smmu->idr_mutex); + if (ret < 0) { + dev_err(smmu->dev, "dynamic ASID allocation failed: %d\n", + ret); + return ret; + } + cfg->asid = ret; + } + return 0; +} + +static void arm_smmu_free_asid(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + bool dynamic = is_dynamic_domain(domain); + + if (cfg->asid == INVALID_ASID || !dynamic) + return; + + mutex_lock(&smmu->idr_mutex); + idr_remove(&smmu->asid_idr, cfg->asid); + mutex_unlock(&smmu->idr_mutex); +} + +/* + * Checks for "qcom,iommu-dma-addr-pool" property to specify the IOVA range + * for the domain. If not present, the domain geometry is unmodified. + */ +static int arm_smmu_adjust_domain_geometry(struct device *dev, + struct iommu_domain_geometry *geometry) +{ + struct device_node *np; + int naddr, nsize, len; + u64 dma_base, dma_size, dma_end; + const __be32 *ranges; + dma_addr_t hw_base = geometry->aperture_start; + dma_addr_t hw_end = geometry->aperture_end; + + if (!dev->of_node) + return 0; + + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + if (!np) + np = dev->of_node; + + ranges = of_get_property(np, "qcom,iommu-dma-addr-pool", &len); + if (!ranges) + return 0; + + len /= sizeof(u32); + naddr = of_n_addr_cells(np); + nsize = of_n_size_cells(np); + if (len < naddr + nsize) { + dev_err(dev, "Invalid length for qcom,iommu-dma-addr-pool, expected %d cells\n", + naddr + nsize); + return -EINVAL; + } + if (naddr == 0 || nsize == 0) { + dev_err(dev, "Invalid #address-cells %d or #size-cells %d\n", + naddr, nsize); + return -EINVAL; + } + + dma_base = of_read_number(ranges, naddr); + dma_size = of_read_number(ranges + naddr, nsize); + dma_end = dma_base + dma_size - 1; + + /* + * The original geometry describes the IOVA limitations of the hardware, + * so lets make sure that the IOVA range for this device is at least + * within those bounds. + */ + if (!((hw_base <= dma_base) && (dma_end <= hw_end))) + return -EINVAL; + + geometry->aperture_start = dma_base; + geometry->aperture_end = dma_end; + return 0; +} + +/* This function assumes that the domain's init mutex is held */ +static int arm_smmu_get_dma_cookie(struct device *dev, + struct arm_smmu_domain *smmu_domain) +{ + int is_fast = !!(smmu_domain->attributes & (1 << DOMAIN_ATTR_FAST)); + int s1_bypass = !!(smmu_domain->attributes & + (1 << DOMAIN_ATTR_S1_BYPASS)); + struct iommu_domain *domain = &smmu_domain->domain.iommu_domain; + struct io_pgtable_ops *pgtbl_ops = smmu_domain->pgtbl_ops; + + if (s1_bypass) + return 0; + + else if (is_fast) + return fast_smmu_init_mapping(dev, domain, pgtbl_ops); + + return iommu_get_dma_cookie(domain); +} + +static void arm_smmu_put_dma_cookie(struct iommu_domain *domain) +{ + int s1_bypass = 0, is_fast = 0; + + iommu_domain_get_attr(domain, DOMAIN_ATTR_S1_BYPASS, + &s1_bypass); + iommu_domain_get_attr(domain, DOMAIN_ATTR_FAST, &is_fast); + + if (is_fast && IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_FAST)) + fast_smmu_put_dma_cookie(domain); + else if (!s1_bypass) + iommu_put_dma_cookie(domain); +} + +static void arm_smmu_domain_get_qcom_quirks(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu, + unsigned long *quirks) +{ + if (smmu_domain->attributes & (1 << DOMAIN_ATTR_USE_UPSTREAM_HINT)) + *quirks |= IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT; + if (is_iommu_pt_coherent(smmu_domain)) + *quirks |= IO_PGTABLE_QUIRK_NO_DMA; + if (smmu_domain->attributes & (1 << DOMAIN_ATTR_USE_LLC_NWA)) + *quirks |= IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA; +} + +static int arm_smmu_setup_context_bank(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu, + struct device *dev) +{ + struct iommu_domain *domain = &smmu_domain->domain.iommu_domain; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct io_pgtable_cfg *pgtbl_cfg = &smmu_domain->pgtbl_info.pgtbl_cfg; + bool dynamic = is_dynamic_domain(domain); + int irq, ret = 0; + + if (!dynamic) { + /* Initialise the context bank with our page table cfg */ + arm_smmu_init_context_bank(smmu_domain, pgtbl_cfg); + arm_smmu_write_context_bank(smmu, cfg->cbndx, + smmu_domain->attributes); + + arm_smmu_arch_init_context_bank(smmu_domain, dev); + + if (smmu->version < ARM_SMMU_V2) { + cfg->irptndx = atomic_inc_return(&smmu->irptndx); + cfg->irptndx %= smmu->num_context_irqs; + } else { + cfg->irptndx = cfg->cbndx; + } + + /* + * Request context fault interrupt. Do this last to avoid the + * handler seeing a half-initialised domain state. + */ + irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + arm_smmu_context_fault, IRQF_ONESHOT | IRQF_SHARED, + "arm-smmu-context-fault", domain); + if (ret < 0) { + dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", + cfg->irptndx, irq); + cfg->irptndx = INVALID_IRPTNDX; + } + } else { + cfg->irptndx = INVALID_IRPTNDX; + } + + return ret; +} + +static int arm_smmu_init_domain_context(struct iommu_domain *domain, + struct arm_smmu_device *smmu, + struct device *dev) +{ + int start, ret = 0; + unsigned long ias, oas; + enum io_pgtable_fmt fmt; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct msm_io_pgtable_info *pgtbl_info = &smmu_domain->pgtbl_info; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + unsigned long quirks = 0; mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) goto out_unlock; + if (domain->type == IOMMU_DOMAIN_DMA) { + ret = arm_smmu_setup_default_domain(dev, domain); + if (ret) { + dev_err(dev, "%s: default domain setup failed\n", + __func__); + goto out_unlock; + } + } + if (domain->type == IOMMU_DOMAIN_IDENTITY) { smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; smmu_domain->smmu = smmu; - goto out_unlock; + smmu_domain->cfg.irptndx = INVALID_IRPTNDX; + smmu_domain->cfg.asid = INVALID_ASID; } /* @@ -833,6 +2058,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, oas = smmu->ipa_size; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S1; + if (smmu->options & ARM_SMMU_OPT_3LVL_TABLES) + ias = min(ias, 39UL); } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { fmt = ARM_32_LPAE_S1; ias = min(ias, 32UL); @@ -842,7 +2069,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, ias = min(ias, 32UL); oas = min(oas, 32UL); } - smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops; + smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops.tlb_ops; break; case ARM_SMMU_DOMAIN_NESTED: /* @@ -862,33 +2089,32 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, oas = min(oas, 40UL); } if (smmu->version == ARM_SMMU_V2) - smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2; + smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2.tlb_ops; else - smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1; + smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1.tlb_ops; break; default: ret = -EINVAL; goto out_unlock; } - ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, - smmu->num_context_banks); + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + if (smmu_domain->attributes & (1 << DOMAIN_ATTR_FAST)) + fmt = ARM_V8L_FAST; +#endif + + if (smmu_domain->non_strict) + quirks |= IO_PGTABLE_QUIRK_NON_STRICT; + arm_smmu_domain_get_qcom_quirks(smmu_domain, smmu, &quirks); + + ret = arm_smmu_alloc_cb(domain, smmu, dev); if (ret < 0) goto out_unlock; cfg->cbndx = ret; - if (smmu->version < ARM_SMMU_V2) { - cfg->irptndx = atomic_inc_return(&smmu->irptndx); - cfg->irptndx %= smmu->num_context_irqs; - } else { - cfg->irptndx = cfg->cbndx; - } - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) - cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base; - else - cfg->asid = cfg->cbndx + smmu->cavium_id_base; - - pgtbl_cfg = (struct io_pgtable_cfg) { + pgtbl_info->pgtbl_cfg = (struct io_pgtable_cfg) { + .quirks = quirks, .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, @@ -896,60 +2122,77 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, .iommu_dev = smmu->dev, }; - if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) - pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA; - - if (smmu_domain->non_strict) - pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; - smmu_domain->smmu = smmu; - pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); - if (!pgtbl_ops) { + smmu_domain->dev = dev; + smmu_domain->pgtbl_ops = alloc_io_pgtable_ops(fmt, + &pgtbl_info->pgtbl_cfg, + smmu_domain); + if (!smmu_domain->pgtbl_ops) { ret = -ENOMEM; goto out_clear_smmu; } + /* + * assign any page table memory that might have been allocated + * during alloc_io_pgtable_ops + */ + arm_smmu_secure_domain_lock(smmu_domain); + arm_smmu_assign_table(smmu_domain); + arm_smmu_secure_domain_unlock(smmu_domain); + /* Update the domain's page sizes to reflect the page table format */ - domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + domain->pgsize_bitmap = pgtbl_info->pgtbl_cfg.pgsize_bitmap; domain->geometry.aperture_end = (1UL << ias) - 1; + ret = arm_smmu_adjust_domain_geometry(dev, &domain->geometry); + if (ret) + goto out_clear_smmu; domain->geometry.force_aperture = true; - /* Initialise the context bank with our page table cfg */ - arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); - arm_smmu_write_context_bank(smmu, cfg->cbndx); - - /* - * Request context fault interrupt. Do this last to avoid the - * handler seeing a half-initialised domain state. - */ - irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; - ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, - IRQF_SHARED, "arm-smmu-context-fault", domain); - if (ret < 0) { - dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", - cfg->irptndx, irq); - cfg->irptndx = INVALID_IRPTNDX; + if (domain->type == IOMMU_DOMAIN_DMA) { + ret = arm_smmu_get_dma_cookie(dev, smmu_domain); + if (ret) + goto out_clear_smmu; } + /* Assign an asid */ + ret = arm_smmu_init_asid(domain, smmu); + if (ret) + goto out_clear_smmu; + + ret = arm_smmu_setup_context_bank(smmu_domain, smmu, dev); + if (ret) + goto out_clear_smmu; + + strlcpy(smmu_domain->domain.name, dev_name(dev), + sizeof(smmu_domain->domain.name)); mutex_unlock(&smmu_domain->init_mutex); - /* Publish page table ops for map/unmap */ - smmu_domain->pgtbl_ops = pgtbl_ops; return 0; out_clear_smmu: + arm_smmu_destroy_domain_context(domain); smmu_domain->smmu = NULL; out_unlock: mutex_unlock(&smmu_domain->init_mutex); return ret; } +static void arm_smmu_domain_reinit(struct arm_smmu_domain *smmu_domain) +{ + smmu_domain->cfg.irptndx = INVALID_IRPTNDX; + smmu_domain->cfg.cbndx = INVALID_CBNDX; + smmu_domain->secure_vmid = VMID_INVAL; +} + static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - int ret, irq; + void __iomem *cb_base; + int irq; + bool dynamic; + int ret; if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) return; @@ -958,12 +2201,35 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) if (ret < 0) return; + ret = arm_smmu_power_on(smmu->pwr); + if (ret) { + WARN_ONCE(ret, "Woops, powering on smmu %pK failed. Leaking context bank\n", + smmu); + arm_smmu_rpm_put(smmu); + return; + } + + dynamic = is_dynamic_domain(domain); + if (dynamic) { + arm_smmu_free_asid(domain); + free_io_pgtable_ops(smmu_domain->pgtbl_ops); + arm_smmu_power_off(smmu->pwr); + arm_smmu_rpm_put(smmu); + arm_smmu_secure_domain_lock(smmu_domain); + arm_smmu_secure_pool_destroy(smmu_domain); + arm_smmu_unassign_table(smmu_domain); + arm_smmu_secure_domain_unlock(smmu_domain); + arm_smmu_domain_reinit(smmu_domain); + return; + } + /* * Disable the context bank and free the page tables before freeing * it. */ smmu->cbs[cfg->cbndx].cfg = NULL; - arm_smmu_write_context_bank(smmu, cfg->cbndx); + cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); + writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); if (cfg->irptndx != INVALID_IRPTNDX) { irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; @@ -971,9 +2237,15 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) } free_io_pgtable_ops(smmu_domain->pgtbl_ops); + arm_smmu_secure_domain_lock(smmu_domain); + arm_smmu_secure_pool_destroy(smmu_domain); + arm_smmu_unassign_table(smmu_domain); + arm_smmu_secure_domain_unlock(smmu_domain); __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); + arm_smmu_power_off(smmu->pwr); arm_smmu_rpm_put(smmu); + arm_smmu_domain_reinit(smmu_domain); } static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) @@ -981,8 +2253,8 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) struct arm_smmu_domain *smmu_domain; if (type != IOMMU_DOMAIN_UNMANAGED && - type != IOMMU_DOMAIN_DMA && - type != IOMMU_DOMAIN_IDENTITY) + type != IOMMU_DOMAIN_IDENTITY && + type != IOMMU_DOMAIN_DMA) return NULL; /* * Allocate the domain and initialise some of its data structures. @@ -993,16 +2265,17 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) if (!smmu_domain) return NULL; - if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding || - iommu_get_dma_cookie(&smmu_domain->domain))) { - kfree(smmu_domain); - return NULL; - } - mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->cb_lock); + spin_lock_init(&smmu_domain->sync_lock); + INIT_LIST_HEAD(&smmu_domain->pte_info_list); + INIT_LIST_HEAD(&smmu_domain->unassign_list); + mutex_init(&smmu_domain->assign_lock); + INIT_LIST_HEAD(&smmu_domain->secure_pool_list); + INIT_LIST_HEAD(&smmu_domain->nonsecure_pool); + arm_smmu_domain_reinit(smmu_domain); - return &smmu_domain->domain; + return &smmu_domain->domain.iommu_domain; } static void arm_smmu_domain_free(struct iommu_domain *domain) @@ -1013,7 +2286,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) * Free the domain resources. We assume that all devices have * already been detached. */ - iommu_put_dma_cookie(domain); + arm_smmu_put_dma_cookie(domain); arm_smmu_destroy_domain_context(domain); kfree(smmu_domain); } @@ -1033,7 +2306,8 @@ static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT | (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT | - (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT; + (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT | + S2CR_SHCFG_NSH << S2CR_SHCFG_SHIFT; if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && smmu->smrs[idx].valid) @@ -1054,25 +2328,48 @@ static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) */ static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) { + unsigned long size; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); - u32 smr; + u32 smr, id; + int idx; + /* Check if Stream Match Register support is included */ if (!smmu->smrs) return; + /* ID0 */ + id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); + size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK; + + /* + * Few SMR registers may be inuse before the smmu driver + * probes(say by the bootloader). Find a SMR register + * which is not inuse. + */ + for (idx = 0; idx < size; idx++) { + smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(idx)); + if (!(smr & SMR_VALID)) + break; + } + if (idx == size) { + dev_err(smmu->dev, + "Unable to compute streamid_masks\n"); + return; + } + /* * SMR.ID bits may not be preserved if the corresponding MASK * bits are set, so check each one separately. We can reject * masters later if they try to claim IDs outside these masks. */ smr = smmu->streamid_mask << SMR_ID_SHIFT; - writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); - smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); + writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(idx)); + smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(idx)); smmu->streamid_mask = smr >> SMR_ID_SHIFT; smr = smmu->streamid_mask << SMR_MASK_SHIFT; - writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); - smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); + writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(idx)); + smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(idx)); smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT; } @@ -1132,13 +2429,14 @@ static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) static int arm_smmu_master_alloc_smes(struct device *dev) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; struct arm_smmu_device *smmu = cfg->smmu; struct arm_smmu_smr *smrs = smmu->smrs; struct iommu_group *group; int i, idx, ret; + mutex_lock(&smmu->iommu_group_mutex); mutex_lock(&smmu->stream_map_mutex); /* Figure out a viable stream map entry allocation */ for_each_cfg_sme(fwspec, i, idx) { @@ -1147,12 +2445,12 @@ static int arm_smmu_master_alloc_smes(struct device *dev) if (idx != INVALID_SMENDX) { ret = -EEXIST; - goto out_err; + goto sme_err; } ret = arm_smmu_find_sme(smmu, sid, mask); if (ret < 0) - goto out_err; + goto sme_err; idx = ret; if (smrs && smmu->s2crs[idx].count == 0) { @@ -1163,31 +2461,34 @@ static int arm_smmu_master_alloc_smes(struct device *dev) smmu->s2crs[idx].count++; cfg->smendx[i] = (s16)idx; } + mutex_unlock(&smmu->stream_map_mutex); group = iommu_group_get_for_dev(dev); if (!group) group = ERR_PTR(-ENOMEM); if (IS_ERR(group)) { ret = PTR_ERR(group); - goto out_err; + goto iommu_group_err; } iommu_group_put(group); - /* It worked! Now, poke the actual hardware */ - for_each_cfg_sme(fwspec, i, idx) { - arm_smmu_write_sme(smmu, idx); + /* It worked! Don't poke the actual hardware until we've attached */ + for_each_cfg_sme(fwspec, i, idx) smmu->s2crs[idx].group = group; - } - mutex_unlock(&smmu->stream_map_mutex); + mutex_unlock(&smmu->iommu_group_mutex); return 0; -out_err: +iommu_group_err: + mutex_lock(&smmu->stream_map_mutex); + +sme_err: while (i--) { arm_smmu_free_sme(smmu, cfg->smendx[i]); cfg->smendx[i] = INVALID_SMENDX; } mutex_unlock(&smmu->stream_map_mutex); + mutex_unlock(&smmu->iommu_group_mutex); return ret; } @@ -1206,6 +2507,36 @@ static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec) mutex_unlock(&smmu->stream_map_mutex); } +static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, + struct iommu_fwspec *fwspec) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_s2cr *s2cr = smmu->s2crs; + int i, idx; + const struct iommu_gather_ops *tlb; + + tlb = smmu_domain->pgtbl_info.pgtbl_cfg.tlb; + + mutex_lock(&smmu->stream_map_mutex); + for_each_cfg_sme(fwspec, i, idx) { + if (WARN_ON(s2cr[idx].attach_count == 0)) { + mutex_unlock(&smmu->stream_map_mutex); + return; + } + s2cr[idx].attach_count -= 1; + + if (s2cr[idx].attach_count > 0) + continue; + + writel_relaxed(0, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx)); + writel_relaxed(0, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx)); + } + mutex_unlock(&smmu->stream_map_mutex); + + /* Ensure there are no stale mappings for this context bank */ + tlb->tlb_flush_all(smmu_domain); +} + static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, struct iommu_fwspec *fwspec) { @@ -1220,26 +2551,338 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, else type = S2CR_TYPE_TRANS; + mutex_lock(&smmu->stream_map_mutex); for_each_cfg_sme(fwspec, i, idx) { - if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) + if (s2cr[idx].attach_count++ > 0) continue; s2cr[idx].type = type; s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; s2cr[idx].cbndx = cbndx; - arm_smmu_write_s2cr(smmu, idx); + arm_smmu_write_sme(smmu, idx); } + mutex_unlock(&smmu->stream_map_mutex); + return 0; } +static void arm_smmu_detach_dev(struct iommu_domain *domain, + struct device *dev) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + int dynamic = smmu_domain->attributes & (1 << DOMAIN_ATTR_DYNAMIC); + int atomic_domain = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC); + + if (dynamic) + return; + + if (!smmu) { + dev_err(dev, "Domain not attached; cannot detach!\n"); + return; + } + + if (atomic_domain) + arm_smmu_power_on_atomic(smmu->pwr); + else + arm_smmu_power_on(smmu->pwr); + + arm_smmu_domain_remove_master(smmu_domain, fwspec); + arm_smmu_power_off(smmu->pwr); +} + +static int arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain) +{ + int ret = 0; + int dest_vmids[2] = {VMID_HLOS, smmu_domain->secure_vmid}; + int dest_perms[2] = {PERM_READ | PERM_WRITE, PERM_READ}; + int source_vmid = VMID_HLOS; + struct arm_smmu_pte_info *pte_info, *temp; + + if (!arm_smmu_has_secure_vmid(smmu_domain)) + return ret; + + list_for_each_entry(pte_info, &smmu_domain->pte_info_list, entry) { + ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr), + PAGE_SIZE, &source_vmid, 1, + dest_vmids, dest_perms, 2); + if (WARN_ON(ret)) + break; + } + + list_for_each_entry_safe(pte_info, temp, &smmu_domain->pte_info_list, + entry) { + list_del(&pte_info->entry); + kfree(pte_info); + } + return ret; +} + +static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain) +{ + int ret; + int dest_vmids = VMID_HLOS; + int dest_perms = PERM_READ | PERM_WRITE | PERM_EXEC; + int source_vmlist[2] = {VMID_HLOS, smmu_domain->secure_vmid}; + struct arm_smmu_pte_info *pte_info, *temp; + + if (!arm_smmu_has_secure_vmid(smmu_domain)) + return; + + list_for_each_entry(pte_info, &smmu_domain->unassign_list, entry) { + ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr), + PAGE_SIZE, source_vmlist, 2, + &dest_vmids, &dest_perms, 1); + if (WARN_ON(ret)) + break; + free_pages_exact(pte_info->virt_addr, pte_info->size); + } + + list_for_each_entry_safe(pte_info, temp, &smmu_domain->unassign_list, + entry) { + list_del(&pte_info->entry); + kfree(pte_info); + } +} + +static void arm_smmu_unprepare_pgtable(void *cookie, void *addr, size_t size) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_pte_info *pte_info; + + if (!arm_smmu_has_secure_vmid(smmu_domain)) { + WARN(1, "Invalid VMID is set !!\n"); + return; + } + + pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC); + if (!pte_info) + return; + + pte_info->virt_addr = addr; + pte_info->size = size; + list_add_tail(&pte_info->entry, &smmu_domain->unassign_list); +} + +static int arm_smmu_prepare_pgtable(void *addr, void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_pte_info *pte_info; + + if (!arm_smmu_has_secure_vmid(smmu_domain)) { + WARN(1, "Invalid VMID is set !!\n"); + return -EINVAL; + } + + pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC); + if (!pte_info) + return -ENOMEM; + pte_info->virt_addr = addr; + list_add_tail(&pte_info->entry, &smmu_domain->pte_info_list); + return 0; +} + +static void arm_smmu_prealloc_memory(struct arm_smmu_domain *smmu_domain, + size_t size, struct list_head *pool) +{ + int i; + u32 nr = 0; + struct page *page; + + if ((smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC)) || + arm_smmu_has_secure_vmid(smmu_domain)) + return; + + /* number of 2nd level pagetable entries */ + nr += round_up(size, SZ_1G) >> 30; + /* number of 3rd level pagetabel entries */ + nr += round_up(size, SZ_2M) >> 21; + + /* Retry later with atomic allocation on error */ + for (i = 0; i < nr; i++) { + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, 0); + if (!page) + break; + list_add(&page->lru, pool); + } +} + +static void arm_smmu_release_prealloc_memory( + struct arm_smmu_domain *smmu_domain, struct list_head *list) +{ + struct page *page, *tmp; + + list_for_each_entry_safe(page, tmp, list, lru) { + list_del(&page->lru); + __free_pages(page, 0); + } +} + +static struct device_node *arm_iommu_get_of_node(struct device *dev) +{ + struct device_node *np; + + if (!dev->of_node) + return NULL; + + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + return np ? np : dev->of_node; +} + +static int arm_smmu_setup_default_domain(struct device *dev, + struct iommu_domain *domain) +{ + struct device_node *np; + int ret; + const char *str; + int attr = 1; + u32 val; + + np = arm_iommu_get_of_node(dev); + if (!np) + return 0; + + ret = of_property_read_string(np, "qcom,iommu-dma", &str); + if (ret) + str = "default"; + + if (!strcmp(str, "bypass")) + __arm_smmu_domain_set_attr( + domain, DOMAIN_ATTR_S1_BYPASS, &attr); + else if (!strcmp(str, "fastmap")) + __arm_smmu_domain_set_attr( + domain, DOMAIN_ATTR_FAST, &attr); + else if (!strcmp(str, "atomic")) + __arm_smmu_domain_set_attr( + domain, DOMAIN_ATTR_ATOMIC, &attr); + else if (!strcmp(str, "disabled")) { + /* + * Don't touch hw, and don't allocate irqs or other resources. + * Ensure the context bank is set to a valid value per dynamic + * attr requirement. + */ + __arm_smmu_domain_set_attr( + domain, DOMAIN_ATTR_DYNAMIC, &attr); + val = 0; + __arm_smmu_domain_set_attr( + domain, DOMAIN_ATTR_CONTEXT_BANK, &val); + } + + /* + * default value: + * Stall-on-fault + * faults trigger kernel panic + * return abort + */ + if (of_property_match_string(np, "qcom,iommu-faults", + "stall-disable") >= 0) + __arm_smmu_domain_set_attr(domain, + DOMAIN_ATTR_CB_STALL_DISABLE, &attr); + + if (of_property_match_string(np, "qcom,iommu-faults", "non-fatal") >= 0) + __arm_smmu_domain_set_attr(domain, + DOMAIN_ATTR_NON_FATAL_FAULTS, &attr); + + if (of_property_match_string(np, "qcom,iommu-faults", "no-CFRE") >= 0) + __arm_smmu_domain_set_attr( + domain, DOMAIN_ATTR_NO_CFRE, &attr); + + /* Default value: disabled */ + ret = of_property_read_u32(np, "qcom,iommu-vmid", &val); + if (!ret) { + __arm_smmu_domain_set_attr( + domain, DOMAIN_ATTR_SECURE_VMID, &val); + } + + /* Default value: disabled */ + ret = of_property_read_string(np, "qcom,iommu-pagetable", &str); + if (ret) + str = "disabled"; + if (!strcmp(str, "coherent")) + __arm_smmu_domain_set_attr(domain, + DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT, &attr); + else if (!strcmp(str, "LLC")) + __arm_smmu_domain_set_attr(domain, + DOMAIN_ATTR_USE_UPSTREAM_HINT, &attr); + else if (!strcmp(str, "LLC_NWA")) + __arm_smmu_domain_set_attr(domain, + DOMAIN_ATTR_USE_LLC_NWA, &attr); + + + /* Default value: disabled */ + if (of_property_read_bool(np, "qcom,iommu-earlymap")) + __arm_smmu_domain_set_attr(domain, + DOMAIN_ATTR_EARLY_MAP, &attr); + return 0; +} + +struct lookup_iommu_group_data { + struct device_node *np; + struct iommu_group *group; +}; + +/* This isn't a "fast lookup" since its N^2, but probably good enough */ +static int __bus_lookup_iommu_group(struct device *dev, void *priv) +{ + struct lookup_iommu_group_data *data = priv; + struct device_node *np; + struct iommu_group *group; + + group = iommu_group_get(dev); + if (!group) + return 0; + + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + if (np != data->np) { + iommu_group_put(group); + return 0; + } + + data->group = group; + iommu_group_put(group); + return 1; +} + +static struct iommu_group *of_get_device_group(struct device *dev) +{ + struct lookup_iommu_group_data data = { + .np = NULL, + .group = NULL, + }; + struct iommu_group *group; + int ret; + + data.np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + if (!data.np) + return NULL; + + ret = bus_for_each_dev(&platform_bus_type, NULL, &data, + __bus_lookup_iommu_group); + if (ret > 0) + return data.group; + + ret = bus_for_each_dev(&pci_bus_type, NULL, &data, + __bus_lookup_iommu_group); + if (ret > 0) + return data.group; + + group = generic_device_group(dev); + if (IS_ERR(group)) + return NULL; + return group; +} + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int atomic_domain = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC); + int s1_bypass = 0; - if (!fwspec || fwspec->ops != &arm_smmu_ops) { + if (!fwspec || fwspec->ops != &arm_smmu_ops.iommu_ops) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); return -ENXIO; } @@ -1260,10 +2903,28 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (ret < 0) return ret; + /* Enable Clocks and Power */ + ret = arm_smmu_power_on(smmu->pwr); + if (ret) { + arm_smmu_rpm_put(smmu); + return ret; + } + /* Ensure that the domain is finalised */ - ret = arm_smmu_init_domain_context(domain, smmu); + ret = arm_smmu_init_domain_context(domain, smmu, dev); if (ret < 0) - goto rpm_put; + goto out_power_off; + + ret = arm_smmu_domain_get_attr(domain, DOMAIN_ATTR_S1_BYPASS, + &s1_bypass); + if (s1_bypass) + domain->type = IOMMU_DOMAIN_UNMANAGED; + + /* Do not modify the SIDs, HW is still running */ + if (is_dynamic_domain(domain)) { + ret = 0; + goto out_power_off; + } /* * Sanity check the domain. We don't support domains across @@ -1274,48 +2935,122 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); ret = -EINVAL; - goto rpm_put; + goto out_power_off; } /* Looks ok, so add the device to the domain */ ret = arm_smmu_domain_add_master(smmu_domain, fwspec); -rpm_put: +out_power_off: + /* + * Keep an additional vote for non-atomic power until domain is + * detached + */ + if (!ret && atomic_domain) { + WARN_ON(arm_smmu_power_on(smmu->pwr)); + arm_smmu_power_off_atomic(smmu->pwr); + } + + arm_smmu_power_off(smmu->pwr); arm_smmu_rpm_put(smmu); + return ret; } static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { - struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; - struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; int ret; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + struct arm_smmu_device *smmu = smmu_domain->smmu; + LIST_HEAD(nonsecure_pool); if (!ops) return -ENODEV; + arm_smmu_secure_domain_lock(smmu_domain); arm_smmu_rpm_get(smmu); + spin_lock_irqsave(&smmu_domain->cb_lock, flags); ret = ops->map(ops, iova, paddr, size, prot); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); arm_smmu_rpm_put(smmu); + /* if the map call failed due to insufficient memory, + * then retry again with preallocated memory to see + * if the map call succeeds. + */ + if (ret == -ENOMEM) { + arm_smmu_prealloc_memory(smmu_domain, size, &nonsecure_pool); + arm_smmu_rpm_get(smmu); + spin_lock_irqsave(&smmu_domain->cb_lock, flags); + list_splice_init(&nonsecure_pool, &smmu_domain->nonsecure_pool); + ret = ops->map(ops, iova, paddr, size, prot); + list_splice_init(&smmu_domain->nonsecure_pool, &nonsecure_pool); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + arm_smmu_rpm_put(smmu); + arm_smmu_release_prealloc_memory(smmu_domain, &nonsecure_pool); + + } + + arm_smmu_assign_table(smmu_domain); + arm_smmu_secure_domain_unlock(smmu_domain); + + return ret; +} + +static uint64_t arm_smmu_iova_to_pte(struct iommu_domain *domain, + dma_addr_t iova) +{ + uint64_t ret; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct msm_io_pgtable_info *pgtbl_info = &smmu_domain->pgtbl_info; + + if (!pgtbl_info->iova_to_pte) + return 0; + + spin_lock_irqsave(&smmu_domain->cb_lock, flags); + ret = pgtbl_info->iova_to_pte(smmu_domain->pgtbl_ops, iova); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); return ret; } static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { - struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; - struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; size_t ret; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + unsigned long flags; if (!ops) return 0; + ret = arm_smmu_domain_power_on(domain, smmu_domain->smmu); + if (ret) + return ret; + + arm_smmu_secure_domain_lock(smmu_domain); + arm_smmu_rpm_get(smmu); + spin_lock_irqsave(&smmu_domain->cb_lock, flags); ret = ops->unmap(ops, iova, size); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); arm_smmu_rpm_put(smmu); + arm_smmu_domain_power_off(domain, smmu_domain->smmu); + /* + * While splitting up block mappings, we might allocate page table + * memory during unmap, so the vmids needs to be assigned to the + * memory here as well. + */ + arm_smmu_assign_table(smmu_domain); + /* Also unassign any pages that were free'd during unmap */ + arm_smmu_unassign_table(smmu_domain); + arm_smmu_secure_domain_unlock(smmu_domain); return ret; } @@ -1343,27 +3078,101 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain) } } -static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, +#define MAX_MAP_SG_BATCH_SIZE (SZ_4M) +static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot) +{ + int ret; + size_t size, batch_size, size_to_unmap = 0; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + struct msm_io_pgtable_info *pgtbl_info = &smmu_domain->pgtbl_info; + unsigned int idx_start, idx_end; + struct scatterlist *sg_start, *sg_end; + unsigned long __saved_iova_start; + LIST_HEAD(nonsecure_pool); + + if (!pgtbl_info->map_sg) + return -ENODEV; + + + arm_smmu_secure_domain_lock(smmu_domain); + + __saved_iova_start = iova; + idx_start = idx_end = 0; + sg_start = sg_end = sg; + while (idx_end < nents) { + batch_size = sg_end->length; + sg_end = sg_next(sg_end); + idx_end++; + while ((idx_end < nents) && + (batch_size + sg_end->length < MAX_MAP_SG_BATCH_SIZE)) { + + batch_size += sg_end->length; + sg_end = sg_next(sg_end); + idx_end++; + } + + spin_lock_irqsave(&smmu_domain->cb_lock, flags); + ret = pgtbl_info->map_sg(ops, iova, sg_start, + idx_end - idx_start, prot, &size); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + + + if (ret == -ENOMEM) { + arm_smmu_prealloc_memory(smmu_domain, + batch_size, &nonsecure_pool); + spin_lock_irqsave(&smmu_domain->cb_lock, flags); + list_splice_init(&nonsecure_pool, + &smmu_domain->nonsecure_pool); + ret = pgtbl_info->map_sg(ops, iova, sg_start, + idx_end - idx_start, prot, + &size); + list_splice_init(&smmu_domain->nonsecure_pool, + &nonsecure_pool); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + arm_smmu_release_prealloc_memory(smmu_domain, + &nonsecure_pool); + } + + /* Returns 0 on error */ + if (!ret) { + size_to_unmap = iova + size - __saved_iova_start; + goto out; + } + + iova += batch_size; + idx_start = idx_end; + sg_start = sg_end; + } + +out: + arm_smmu_assign_table(smmu_domain); + + if (size_to_unmap) { + arm_smmu_unmap(domain, __saved_iova_start, size_to_unmap); + iova = __saved_iova_start; + } + arm_smmu_secure_domain_unlock(smmu_domain); + return iova - __saved_iova_start; +} + +static phys_addr_t __arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dma_addr_t iova) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; struct device *dev = smmu->dev; void __iomem *cb_base; u32 tmp; u64 phys; - unsigned long va, flags; - int ret; - - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) - return 0; + unsigned long va; cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); - spin_lock_irqsave(&smmu_domain->cb_lock, flags); /* ATS1 registers can only be written atomically */ va = iova & ~0xfffUL; if (smmu->version == ARM_SMMU_V2) @@ -1373,29 +3182,31 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) { - spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + phys = ops->iova_to_phys(ops, iova); dev_err(dev, - "iova to phys timed out on %pad. Falling back to software table walk.\n", - &iova); - return ops->iova_to_phys(ops, iova); + "iova to phys timed out on %pad. software table walk result=%pa.\n", + &iova, &phys); + phys = 0; + return phys; } phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR); - spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); if (phys & CB_PAR_F) { dev_err(dev, "translation fault!\n"); dev_err(dev, "PAR = 0x%llx\n", phys); - return 0; + phys = 0; + } else { + phys = (phys & (PHYS_MASK & ~0xfffULL)) | (iova & 0xfff); } - arm_smmu_rpm_put(smmu); - - return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); + return phys; } static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { + phys_addr_t ret; + unsigned long flags; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; @@ -1405,11 +3216,50 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; + spin_lock_irqsave(&smmu_domain->cb_lock, flags); + ret = ops->iova_to_phys(ops, iova); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + + return ret; +} + +/* + * This function can sleep, and cannot be called from atomic context. Will + * power on register block if required. This restriction does not apply to the + * original iova_to_phys() op. + */ +static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, + dma_addr_t iova) +{ + phys_addr_t ret = 0; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (smmu->options & ARM_SMMU_OPT_DISABLE_ATOS) + return 0; + + if (arm_smmu_power_on(smmu_domain->smmu->pwr)) + return 0; + + if (smmu_domain->smmu->arch_ops && + smmu_domain->smmu->arch_ops->iova_to_phys_hard) { + ret = smmu_domain->smmu->arch_ops->iova_to_phys_hard( + domain, iova); + goto out; + } + + spin_lock_irqsave(&smmu_domain->cb_lock, flags); if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && smmu_domain->stage == ARM_SMMU_DOMAIN_S1) - return arm_smmu_iova_to_phys_hard(domain, iova); + ret = __arm_smmu_iova_to_phys_hard(domain, iova); - return ops->iova_to_phys(ops, iova); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + +out: + arm_smmu_power_off(smmu_domain->smmu->pwr); + + return ret; } static bool arm_smmu_capable(enum iommu_cap cap) @@ -1446,7 +3296,8 @@ static int arm_smmu_add_device(struct device *dev) { struct arm_smmu_device *smmu; struct arm_smmu_master_cfg *cfg; - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct device_link *link; int i, ret; if (using_legacy_binding) { @@ -1457,15 +3308,21 @@ static int arm_smmu_add_device(struct device *dev) * will allocate/initialise a new one. Thus we need to update fwspec for * later use. */ - fwspec = dev_iommu_fwspec_get(dev); + fwspec = dev->iommu_fwspec; if (ret) goto out_free; - } else if (fwspec && fwspec->ops == &arm_smmu_ops) { + } else if (fwspec && fwspec->ops == &arm_smmu_ops.iommu_ops) { smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); + if (!smmu) + return -ENODEV; } else { return -ENODEV; } + ret = arm_smmu_power_on(smmu->pwr); + if (ret) + goto out_free; + ret = -EINVAL; for (i = 0; i < fwspec->num_ids; i++) { u16 sid = fwspec->ids[i]; @@ -1474,12 +3331,12 @@ static int arm_smmu_add_device(struct device *dev) if (sid & ~smmu->streamid_mask) { dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", sid, smmu->streamid_mask); - goto out_free; + goto out_pwr_off; } if (mask & ~smmu->smr_mask_mask) { dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", mask, smmu->smr_mask_mask); - goto out_free; + goto out_pwr_off; } } @@ -1487,32 +3344,33 @@ static int arm_smmu_add_device(struct device *dev) cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), GFP_KERNEL); if (!cfg) - goto out_free; + goto out_pwr_off; cfg->smmu = smmu; fwspec->iommu_priv = cfg; while (i--) cfg->smendx[i] = INVALID_SMENDX; - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) + link = device_link_add(dev, smmu->dev, DL_FLAG_STATELESS); + if (!link) { + dev_err(dev, "error in device link creation between %s & %s\n", + dev_name(smmu->dev), dev_name(dev)); + ret = -ENODEV; goto out_cfg_free; + } ret = arm_smmu_master_alloc_smes(dev); - arm_smmu_rpm_put(smmu); - if (ret) - goto out_cfg_free; - - iommu_device_link(&smmu->iommu, dev); - - device_link_add(dev, smmu->dev, - DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); - + goto out_dev_link_free; + arm_smmu_power_off(smmu->pwr); return 0; +out_dev_link_free: + device_link_del(link); out_cfg_free: kfree(cfg); +out_pwr_off: + arm_smmu_power_off(smmu->pwr); out_free: iommu_fwspec_free(dev); return ret; @@ -1520,55 +3378,78 @@ out_free: static void arm_smmu_remove_device(struct device *dev) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_master_cfg *cfg; + struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu; + struct device_link *link; int ret; - if (!fwspec || fwspec->ops != &arm_smmu_ops) + if (!fwspec || fwspec->ops != &arm_smmu_ops.iommu_ops) return; - cfg = fwspec->iommu_priv; - smmu = cfg->smmu; + smmu = fwspec_smmu(fwspec); ret = arm_smmu_rpm_get(smmu); if (ret < 0) return; - iommu_device_unlink(&smmu->iommu, dev); + if (arm_smmu_power_on(smmu->pwr)) { + WARN_ON(1); + arm_smmu_rpm_put(smmu); + return; + } + + /* Remove the device link between dev and the smmu if any */ + list_for_each_entry(link, &smmu->dev->links.consumers, s_node) { + if (link->consumer == dev) + device_link_del(link); + } + arm_smmu_master_free_smes(fwspec); - - arm_smmu_rpm_put(smmu); - iommu_group_remove_device(dev); kfree(fwspec->iommu_priv); iommu_fwspec_free(dev); + arm_smmu_power_off(smmu->pwr); + arm_smmu_rpm_put(smmu); } static struct iommu_group *arm_smmu_device_group(struct device *dev) { - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu = fwspec_smmu(fwspec); struct iommu_group *group = NULL; int i, idx; + group = of_get_device_group(dev); for_each_cfg_sme(fwspec, i, idx) { if (group && smmu->s2crs[idx].group && - group != smmu->s2crs[idx].group) + group != smmu->s2crs[idx].group) { + dev_err(dev, "ID:%x IDX:%x is already in a group!\n", + fwspec->ids[i], idx); return ERR_PTR(-EINVAL); + } - group = smmu->s2crs[idx].group; + if (!group) + group = smmu->s2crs[idx].group; } if (group) - return iommu_group_ref_get(group); + iommu_group_ref_get(group); + else { + if (dev_is_pci(dev)) + group = pci_device_group(dev); + else if (dev_is_fsl_mc(dev)) + group = fsl_mc_device_group(dev); + else + group = generic_device_group(dev); - if (dev_is_pci(dev)) - group = pci_device_group(dev); - else if (dev_is_fsl_mc(dev)) - group = fsl_mc_device_group(dev); - else - group = generic_device_group(dev); + if (IS_ERR(group)) + return NULL; + } + + if (arm_smmu_arch_device_group(dev, group)) { + iommu_group_put(group); + return ERR_PTR(-EINVAL); + } return group; } @@ -1577,74 +3458,399 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_cfg *pgtbl_cfg = &smmu_domain->pgtbl_info.pgtbl_cfg; + int ret = 0; + unsigned long iommu_attr = (unsigned long)attr; - switch(domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - switch (attr) { - case DOMAIN_ATTR_NESTING: - *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); - return 0; - default: - return -ENODEV; - } + mutex_lock(&smmu_domain->init_mutex); + switch (iommu_attr) { + case DOMAIN_ATTR_NESTING: + *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); + ret = 0; break; - case IOMMU_DOMAIN_DMA: - switch (attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: - *(int *)data = smmu_domain->non_strict; - return 0; - default: - return -ENODEV; + case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: + *(int *)data = smmu_domain->non_strict; + ret = 0; + break; + case DOMAIN_ATTR_PT_BASE_ADDR: + *((phys_addr_t *)data) = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; + ret = 0; + break; + case DOMAIN_ATTR_CONTEXT_BANK: + /* context bank index isn't valid until we are attached */ + if (smmu_domain->smmu == NULL) { + ret = -ENODEV; + break; + } + *((unsigned int *) data) = smmu_domain->cfg.cbndx; + ret = 0; + break; + case DOMAIN_ATTR_TTBR0: { + u64 val; + struct arm_smmu_device *smmu = smmu_domain->smmu; + /* not valid until we are attached */ + if (smmu == NULL) { + ret = -ENODEV; + break; + } + val = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; + if (smmu_domain->cfg.cbar != CBAR_TYPE_S2_TRANS) + val |= (u64)ARM_SMMU_CB_ASID(smmu, &smmu_domain->cfg) + << (TTBRn_ASID_SHIFT); + *((u64 *)data) = val; + ret = 0; + break; + } + case DOMAIN_ATTR_CONTEXTIDR: + /* not valid until attached */ + if (smmu_domain->smmu == NULL) { + ret = -ENODEV; + break; + } + *((u32 *)data) = smmu_domain->cfg.procid; + ret = 0; + break; + case DOMAIN_ATTR_PROCID: + *((u32 *)data) = smmu_domain->cfg.procid; + ret = 0; + break; + case DOMAIN_ATTR_DYNAMIC: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_DYNAMIC)); + ret = 0; + break; + case DOMAIN_ATTR_NON_FATAL_FAULTS: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_NON_FATAL_FAULTS)); + ret = 0; + break; + case DOMAIN_ATTR_S1_BYPASS: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_S1_BYPASS)); + ret = 0; + break; + case DOMAIN_ATTR_SECURE_VMID: + *((int *)data) = smmu_domain->secure_vmid; + ret = 0; + break; + case DOMAIN_ATTR_PGTBL_INFO: { + struct iommu_pgtbl_info *info = data; + + if (!(smmu_domain->attributes & (1 << DOMAIN_ATTR_FAST))) { + ret = -ENODEV; + break; + } + info->ops = smmu_domain->pgtbl_ops; + ret = 0; + break; + } + case DOMAIN_ATTR_FAST: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_FAST)); + ret = 0; + break; + case DOMAIN_ATTR_USE_UPSTREAM_HINT: + *((int *)data) = !!(smmu_domain->attributes & + (1 << DOMAIN_ATTR_USE_UPSTREAM_HINT)); + ret = 0; + break; + case DOMAIN_ATTR_USE_LLC_NWA: + *((int *)data) = !!(smmu_domain->attributes & + (1 << DOMAIN_ATTR_USE_LLC_NWA)); + ret = 0; + break; + case DOMAIN_ATTR_EARLY_MAP: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_EARLY_MAP)); + ret = 0; + break; + case DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR)); + ret = 0; + break; + case DOMAIN_ATTR_PAGE_TABLE_IS_COHERENT: + if (!smmu_domain->smmu) { + ret = -ENODEV; + break; + } + *((int *)data) = is_iommu_pt_coherent(smmu_domain); + ret = 0; + break; + case DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT)); + ret = 0; + break; + case DOMAIN_ATTR_CB_STALL_DISABLE: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_CB_STALL_DISABLE)); + ret = 0; + break; + case DOMAIN_ATTR_NO_CFRE: + *((int *)data) = !!(smmu_domain->attributes + & (1 << DOMAIN_ATTR_NO_CFRE)); + ret = 0; + break; + case DOMAIN_ATTR_DEBUG: + *((int *)data) = !!(smmu_domain->attributes & + (1 << DOMAIN_ATTR_DEBUG)); + ret = 0; + break; + default: + ret = -ENODEV; + break; + } + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static int __arm_smmu_domain_set_attr2(struct iommu_domain *domain, + enum iommu_attr attr, void *data); +static int __arm_smmu_domain_set_attr(struct iommu_domain *domain, + enum iommu_attr attr, void *data) +{ + int ret = 0; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + unsigned long iommu_attr = (unsigned long)attr; + + switch (iommu_attr) { + case DOMAIN_ATTR_NESTING: + if (smmu_domain->smmu) { + ret = -EPERM; + goto out; + } + + if (*(int *)data) + smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; + else + smmu_domain->stage = ARM_SMMU_DOMAIN_S1; + + break; + case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: + smmu_domain->non_strict = *(int *)data; + break; + case DOMAIN_ATTR_PROCID: + if (smmu_domain->smmu != NULL) { + dev_err(smmu_domain->smmu->dev, + "cannot change procid attribute while attached\n"); + ret = -EBUSY; + break; + } + smmu_domain->cfg.procid = *((u32 *)data); + ret = 0; + break; + case DOMAIN_ATTR_DYNAMIC: { + int dynamic = *((int *)data); + + if (smmu_domain->smmu != NULL) { + dev_err(smmu_domain->smmu->dev, + "cannot change dynamic attribute while attached\n"); + ret = -EBUSY; + break; + } + + if (dynamic) + smmu_domain->attributes |= 1 << DOMAIN_ATTR_DYNAMIC; + else + smmu_domain->attributes &= ~(1 << DOMAIN_ATTR_DYNAMIC); + ret = 0; + break; + } + case DOMAIN_ATTR_CONTEXT_BANK: + /* context bank can't be set while attached */ + if (smmu_domain->smmu != NULL) { + ret = -EBUSY; + break; + } + /* ... and it can only be set for dynamic contexts. */ + if (!(smmu_domain->attributes & (1 << DOMAIN_ATTR_DYNAMIC))) { + ret = -EINVAL; + break; + } + + /* this will be validated during attach */ + smmu_domain->cfg.cbndx = *((unsigned int *)data); + ret = 0; + break; + case DOMAIN_ATTR_NON_FATAL_FAULTS: { + u32 non_fatal_faults = *((int *)data); + + if (non_fatal_faults) + smmu_domain->attributes |= + 1 << DOMAIN_ATTR_NON_FATAL_FAULTS; + else + smmu_domain->attributes &= + ~(1 << DOMAIN_ATTR_NON_FATAL_FAULTS); + ret = 0; + break; + } + case DOMAIN_ATTR_S1_BYPASS: { + int bypass = *((int *)data); + + /* bypass can't be changed while attached */ + if (smmu_domain->smmu != NULL) { + ret = -EBUSY; + break; + } + if (bypass) + smmu_domain->attributes |= 1 << DOMAIN_ATTR_S1_BYPASS; + else + smmu_domain->attributes &= + ~(1 << DOMAIN_ATTR_S1_BYPASS); + + ret = 0; + break; + } + case DOMAIN_ATTR_ATOMIC: + { + int atomic_ctx = *((int *)data); + + /* can't be changed while attached */ + if (smmu_domain->smmu != NULL) { + ret = -EBUSY; + break; + } + if (atomic_ctx) + smmu_domain->attributes |= (1 << DOMAIN_ATTR_ATOMIC); + else + smmu_domain->attributes &= ~(1 << DOMAIN_ATTR_ATOMIC); + break; + } + case DOMAIN_ATTR_SECURE_VMID: + if (smmu_domain->secure_vmid != VMID_INVAL) { + ret = -ENODEV; + WARN(1, "secure vmid already set!"); + break; + } + smmu_domain->secure_vmid = *((int *)data); + break; + /* + * fast_smmu_unmap_page() and fast_smmu_alloc_iova() both + * expect that the bus/clock/regulator are already on. Thus also + * force DOMAIN_ATTR_ATOMIC to bet set. + */ + case DOMAIN_ATTR_FAST: + if (*((int *)data)) { + if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_FAST)) { + smmu_domain->attributes |= + 1 << DOMAIN_ATTR_FAST; + smmu_domain->attributes |= + 1 << DOMAIN_ATTR_ATOMIC; + ret = 0; + } else { + ret = -ENOTSUPP; + } } break; default: - return -EINVAL; + ret = __arm_smmu_domain_set_attr2(domain, attr, data); } +out: + return ret; +} + +/* yeee-haw */ +static int __arm_smmu_domain_set_attr2(struct iommu_domain *domain, + enum iommu_attr attr, void *data) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret = 0; + unsigned long iommu_attr = (unsigned long)attr; + + switch (iommu_attr) { + case DOMAIN_ATTR_USE_UPSTREAM_HINT: + case DOMAIN_ATTR_USE_LLC_NWA: + if (IS_ENABLED(CONFIG_QCOM_IOMMU_IO_PGTABLE_QUIRKS)) { + + /* can't be changed while attached */ + if (smmu_domain->smmu != NULL) { + ret = -EBUSY; + } else if (*((int *)data)) { + smmu_domain->attributes |= 1 << attr; + ret = 0; + } + } else { + ret = -ENOTSUPP; + } + break; + case DOMAIN_ATTR_EARLY_MAP: { + int early_map = *((int *)data); + + ret = 0; + if (early_map) { + smmu_domain->attributes |= + 1 << DOMAIN_ATTR_EARLY_MAP; + } else { + if (smmu_domain->smmu) + ret = arm_smmu_enable_s1_translations( + smmu_domain); + + if (!ret) + smmu_domain->attributes &= + ~(1 << DOMAIN_ATTR_EARLY_MAP); + } + break; + } + case DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR: + case DOMAIN_ATTR_CB_STALL_DISABLE: + case DOMAIN_ATTR_NO_CFRE: + if (*((int *)data)) + smmu_domain->attributes |= + 1 << attr; + ret = 0; + break; + case DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT: { + int force_coherent = *((int *)data); + + if (IS_ENABLED(CONFIG_QCOM_IOMMU_IO_PGTABLE_QUIRKS)) { + if (smmu_domain->smmu != NULL) { + dev_err(smmu_domain->smmu->dev, + "cannot change force coherent attribute while attached\n"); + ret = -EBUSY; + } else if (force_coherent) { + smmu_domain->attributes |= + 1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT; + ret = 0; + } else { + smmu_domain->attributes &= + ~(1 << DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT); + ret = 0; + } + } else { + ret = -ENOTSUPP; + } + break; + } + case DOMAIN_ATTR_DEBUG: { + int is_debug_domain = *((int *)data); + + if (is_debug_domain) + smmu_domain->attributes |= 1 << DOMAIN_ATTR_DEBUG; + else + smmu_domain->attributes &= ~(1 << DOMAIN_ATTR_DEBUG); + ret = 0; + break; + } + default: + ret = -ENODEV; + } + + return ret; } static int arm_smmu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { - int ret = 0; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret; mutex_lock(&smmu_domain->init_mutex); - - switch(domain->type) { - case IOMMU_DOMAIN_UNMANAGED: - switch (attr) { - case DOMAIN_ATTR_NESTING: - if (smmu_domain->smmu) { - ret = -EPERM; - goto out_unlock; - } - - if (*(int *)data) - smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; - else - smmu_domain->stage = ARM_SMMU_DOMAIN_S1; - break; - default: - ret = -ENODEV; - } - break; - case IOMMU_DOMAIN_DMA: - switch (attr) { - case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: - smmu_domain->non_strict = *(int *)data; - break; - default: - ret = -ENODEV; - } - break; - default: - ret = -EINVAL; - } -out_unlock: + ret = __arm_smmu_domain_set_attr(domain, attr, data); mutex_unlock(&smmu_domain->init_mutex); + return ret; } - static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) { u32 mask, fwid = 0; @@ -1684,44 +3890,131 @@ static void arm_smmu_put_resv_regions(struct device *dev, list_for_each_entry_safe(entry, next, head, list) kfree(entry); } +static int arm_smmu_enable_s1_translations(struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; + void __iomem *cb_base; + u32 reg; + int ret; -static struct iommu_ops arm_smmu_ops = { - .capable = arm_smmu_capable, - .domain_alloc = arm_smmu_domain_alloc, - .domain_free = arm_smmu_domain_free, - .attach_dev = arm_smmu_attach_dev, - .map = arm_smmu_map, - .unmap = arm_smmu_unmap, - .flush_iotlb_all = arm_smmu_flush_iotlb_all, - .iotlb_sync = arm_smmu_iotlb_sync, - .iova_to_phys = arm_smmu_iova_to_phys, - .add_device = arm_smmu_add_device, - .remove_device = arm_smmu_remove_device, - .device_group = arm_smmu_device_group, - .domain_get_attr = arm_smmu_domain_get_attr, - .domain_set_attr = arm_smmu_domain_set_attr, - .of_xlate = arm_smmu_of_xlate, - .get_resv_regions = arm_smmu_get_resv_regions, - .put_resv_regions = arm_smmu_put_resv_regions, - .pgsize_bitmap = -1UL, /* Restricted during device attach */ + cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); + ret = arm_smmu_power_on(smmu->pwr); + if (ret) + return ret; + + reg = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR); + reg |= SCTLR_M; + + writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); + arm_smmu_power_off(smmu->pwr); + return ret; +} + +static bool arm_smmu_is_iova_coherent(struct iommu_domain *domain, + dma_addr_t iova) +{ + bool ret; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + struct msm_io_pgtable_info *pgtbl_info = &smmu_domain->pgtbl_info; + + if (!pgtbl_info->is_iova_coherent) + return false; + + spin_lock_irqsave(&smmu_domain->cb_lock, flags); + ret = pgtbl_info->is_iova_coherent(ops, iova); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + return ret; +} + +static void arm_smmu_trigger_fault(struct iommu_domain *domain, + unsigned long flags) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu; + void __iomem *cb_base; + + if (!smmu_domain->smmu) { + pr_err("Can't trigger faults on non-attached domains\n"); + return; + } + + smmu = smmu_domain->smmu; + if (arm_smmu_power_on(smmu->pwr)) + return; + + cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); + dev_err(smmu->dev, "Writing 0x%lx to FSRRESTORE on cb %d\n", + flags, cfg->cbndx); + writel_relaxed(flags, cb_base + ARM_SMMU_CB_FSRRESTORE); + /* give the interrupt time to fire... */ + msleep(1000); + + arm_smmu_power_off(smmu->pwr); +} + +static void arm_smmu_tlbi_domain(struct iommu_domain *domain) +{ + arm_smmu_tlb_inv_context_s1(to_smmu_domain(domain)); +} + +static int arm_smmu_enable_config_clocks(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + return arm_smmu_power_on(smmu_domain->smmu->pwr); +} + +static void arm_smmu_disable_config_clocks(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + arm_smmu_power_off(smmu_domain->smmu->pwr); +} + +static struct msm_iommu_ops arm_smmu_ops = { + .map_sg = arm_smmu_map_sg, + .iova_to_phys_hard = arm_smmu_iova_to_phys_hard, + .is_iova_coherent = arm_smmu_is_iova_coherent, + .trigger_fault = arm_smmu_trigger_fault, + .tlbi_domain = arm_smmu_tlbi_domain, + .enable_config_clocks = arm_smmu_enable_config_clocks, + .disable_config_clocks = arm_smmu_disable_config_clocks, + .iova_to_pte = arm_smmu_iova_to_pte, + .iommu_ops = { + + .capable = arm_smmu_capable, + .domain_alloc = arm_smmu_domain_alloc, + .domain_free = arm_smmu_domain_free, + .attach_dev = arm_smmu_attach_dev, + .detach_dev = arm_smmu_detach_dev, + .map = arm_smmu_map, + .unmap = arm_smmu_unmap, + .flush_iotlb_all = arm_smmu_flush_iotlb_all, + .iotlb_sync = arm_smmu_iotlb_sync, + .iova_to_phys = arm_smmu_iova_to_phys, + .add_device = arm_smmu_add_device, + .remove_device = arm_smmu_remove_device, + .device_group = arm_smmu_device_group, + .domain_get_attr = arm_smmu_domain_get_attr, + .domain_set_attr = arm_smmu_domain_set_attr, + .of_xlate = arm_smmu_of_xlate, + .get_resv_regions = arm_smmu_get_resv_regions, + .put_resv_regions = arm_smmu_put_resv_regions, + /* Restricted during device attach */ + .pgsize_bitmap = -1UL, + } }; -static void arm_smmu_device_reset(struct arm_smmu_device *smmu) +static void arm_smmu_context_bank_reset(struct arm_smmu_device *smmu) { - void __iomem *gr0_base = ARM_SMMU_GR0(smmu); int i; u32 reg, major; - - /* clear global FSR */ - reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); - writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); - - /* - * Reset stream mapping groups: Initial values mark all SMRn as - * invalid and all S2CRn as bypass unless overridden. - */ - for (i = 0; i < smmu->num_mapping_groups; ++i) - arm_smmu_write_sme(smmu, i); + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + void __iomem *cb_base; if (smmu->model == ARM_MMU500) { /* @@ -1738,15 +4031,15 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) * Allow unmatched Stream IDs to allocate bypass * TLB entries for reduced latency. */ - reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN; + reg |= ARM_MMU500_ACR_SMTNMB_TLBEN; writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR); } /* Make sure all context banks are disabled and clear CB_FSR */ for (i = 0; i < smmu->num_context_banks; ++i) { - void __iomem *cb_base = ARM_SMMU_CB(smmu, i); + cb_base = ARM_SMMU_CB(smmu, i); - arm_smmu_write_context_bank(smmu, i); + arm_smmu_write_context_bank(smmu, i, 0); writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); /* * Disable MMU-500's not-particularly-beneficial next-page @@ -1758,6 +4051,28 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR); } } +} + +static void arm_smmu_device_reset(struct arm_smmu_device *smmu) +{ + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + int i; + u32 reg; + + /* clear global FSR */ + reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); + writel_relaxed(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); + + /* + * Reset stream mapping groups: Initial values mark all SMRn as + * invalid and all S2CRn as bypass unless overridden. + */ + if (!(smmu->options & ARM_SMMU_OPT_SKIP_INIT)) { + for (i = 0; i < smmu->num_mapping_groups; ++i) + arm_smmu_write_sme(smmu, i); + + arm_smmu_context_bank_reset(smmu); + } /* Invalidate the TLB, just in case */ writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLH); @@ -1790,9 +4105,16 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_EXIDS) reg |= sCR0_EXIDENABLE; + /* Force bypass transaction to be Non-Shareable & not io-coherent */ + reg &= ~(sCR0_SHCFG_MASK << sCR0_SHCFG_SHIFT); + reg |= sCR0_SHCFG_NSH << sCR0_SHCFG_SHIFT; + /* Push the button */ arm_smmu_tlb_sync_global(smmu); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); + + /* Manage any implementation defined features */ + arm_smmu_arch_device_reset(smmu); } static int arm_smmu_id_size_to_bits(int size) @@ -1814,6 +4136,289 @@ static int arm_smmu_id_size_to_bits(int size) } } + +/* + * Some context banks needs to be transferred from bootloader to HLOS in a way + * that allows ongoing traffic. The current expectation is that these context + * banks operate in bypass mode. + * Additionally, there must be exactly one device in devicetree with stream-ids + * overlapping those used by the bootloader. + */ +static int arm_smmu_alloc_cb(struct iommu_domain *domain, + struct arm_smmu_device *smmu, + struct device *dev) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + u32 i, idx; + int cb = -EINVAL; + bool dynamic; + + /* + * Dynamic domains have already set cbndx through domain attribute. + * Verify that they picked a valid value. + */ + dynamic = is_dynamic_domain(domain); + if (dynamic) { + cb = smmu_domain->cfg.cbndx; + if (cb < smmu->num_context_banks) + return cb; + else + return -EINVAL; + } + + mutex_lock(&smmu->stream_map_mutex); + for_each_cfg_sme(fwspec, i, idx) { + if (smmu->s2crs[idx].cb_handoff) + cb = smmu->s2crs[idx].cbndx; + } + + if (cb < 0) { + mutex_unlock(&smmu->stream_map_mutex); + return __arm_smmu_alloc_bitmap(smmu->context_map, + smmu->num_s2_context_banks, + smmu->num_context_banks); + } + + for (i = 0; i < smmu->num_mapping_groups; i++) { + if (smmu->s2crs[i].cb_handoff && smmu->s2crs[i].cbndx == cb) { + smmu->s2crs[i].cb_handoff = false; + smmu->s2crs[i].count -= 1; + } + } + mutex_unlock(&smmu->stream_map_mutex); + + return cb; +} + +static int arm_smmu_handoff_cbs(struct arm_smmu_device *smmu) +{ + u32 i, raw_smr, raw_s2cr; + struct arm_smmu_smr smr; + struct arm_smmu_s2cr s2cr; + + for (i = 0; i < smmu->num_mapping_groups; i++) { + raw_smr = readl_relaxed(ARM_SMMU_GR0(smmu) + + ARM_SMMU_GR0_SMR(i)); + if (!(raw_smr & SMR_VALID)) + continue; + + smr.mask = (raw_smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK; + smr.id = (u16)raw_smr; + smr.valid = true; + + raw_s2cr = readl_relaxed(ARM_SMMU_GR0(smmu) + + ARM_SMMU_GR0_S2CR(i)); + memset(&s2cr, 0, sizeof(s2cr)); + s2cr.group = NULL; + s2cr.count = 1; + s2cr.type = (raw_s2cr >> S2CR_TYPE_SHIFT) & S2CR_TYPE_MASK; + s2cr.privcfg = (raw_s2cr >> S2CR_PRIVCFG_SHIFT) & + S2CR_PRIVCFG_MASK; + s2cr.cbndx = (u8)raw_s2cr; + s2cr.cb_handoff = true; + + if (s2cr.type != S2CR_TYPE_TRANS) + continue; + + smmu->smrs[i] = smr; + smmu->s2crs[i] = s2cr; + bitmap_set(smmu->context_map, s2cr.cbndx, 1); + dev_dbg(smmu->dev, "Handoff smr: %x s2cr: %x cb: %d\n", + raw_smr, raw_s2cr, s2cr.cbndx); + } + + return 0; +} + +static int arm_smmu_parse_impl_def_registers(struct arm_smmu_device *smmu) +{ + struct device *dev = smmu->dev; + int i, ntuples, ret; + u32 *tuples; + struct arm_smmu_impl_def_reg *regs, *regit; + + if (!of_find_property(dev->of_node, "attach-impl-defs", &ntuples)) + return 0; + + ntuples /= sizeof(u32); + if (ntuples % 2) { + dev_err(dev, + "Invalid number of attach-impl-defs registers: %d\n", + ntuples); + return -EINVAL; + } + + regs = devm_kmalloc( + dev, sizeof(*smmu->impl_def_attach_registers) * ntuples, + GFP_KERNEL); + if (!regs) + return -ENOMEM; + + tuples = devm_kmalloc(dev, sizeof(u32) * ntuples * 2, GFP_KERNEL); + if (!tuples) + return -ENOMEM; + + ret = of_property_read_u32_array(dev->of_node, "attach-impl-defs", + tuples, ntuples); + if (ret) + return ret; + + for (i = 0, regit = regs; i < ntuples; i += 2, ++regit) { + regit->offset = tuples[i]; + regit->value = tuples[i + 1]; + } + + devm_kfree(dev, tuples); + + smmu->impl_def_attach_registers = regs; + smmu->num_impl_def_attach_registers = ntuples / 2; + + return 0; +} + + +static int arm_smmu_init_clocks(struct arm_smmu_power_resources *pwr) +{ + const char *cname; + struct property *prop; + int i; + struct device *dev = pwr->dev; + + pwr->num_clocks = + of_property_count_strings(dev->of_node, "clock-names"); + + if (pwr->num_clocks < 1) { + pwr->num_clocks = 0; + return 0; + } + + pwr->clocks = devm_kzalloc( + dev, sizeof(*pwr->clocks) * pwr->num_clocks, + GFP_KERNEL); + + if (!pwr->clocks) + return -ENOMEM; + + i = 0; + of_property_for_each_string(dev->of_node, "clock-names", + prop, cname) { + struct clk *c = devm_clk_get(dev, cname); + + if (IS_ERR(c)) { + dev_err(dev, "Couldn't get clock: %s\n", + cname); + return PTR_ERR(c); + } + + if (clk_get_rate(c) == 0) { + long rate = clk_round_rate(c, 1000); + + clk_set_rate(c, rate); + } + + pwr->clocks[i] = c; + + ++i; + } + return 0; +} + +static int arm_smmu_init_regulators(struct arm_smmu_power_resources *pwr) +{ + const char *cname; + struct property *prop; + int i, ret = 0; + struct device *dev = pwr->dev; + + pwr->num_gdscs = + of_property_count_strings(dev->of_node, "qcom,regulator-names"); + + if (pwr->num_gdscs < 1) { + pwr->num_gdscs = 0; + return 0; + } + + pwr->gdscs = devm_kzalloc( + dev, sizeof(*pwr->gdscs) * pwr->num_gdscs, GFP_KERNEL); + + if (!pwr->gdscs) + return -ENOMEM; + + if (!of_property_read_u32(dev->of_node, + "qcom,deferred-regulator-disable-delay", + &(pwr->regulator_defer))) + dev_info(dev, "regulator defer delay %d\n", + pwr->regulator_defer); + + i = 0; + of_property_for_each_string(dev->of_node, "qcom,regulator-names", + prop, cname) + pwr->gdscs[i++].supply = cname; + + ret = devm_regulator_bulk_get(dev, pwr->num_gdscs, pwr->gdscs); + return ret; +} + +static int arm_smmu_init_interconnect(struct arm_smmu_power_resources *pwr) +{ + struct device *dev = pwr->dev; + + /* We don't want the interconnect APIs to print an error message */ + if (!of_find_property(dev->of_node, "interconnects", NULL)) { + dev_dbg(dev, "No interconnect info\n"); + return 0; + } + + pwr->icc_path = of_icc_get(dev, NULL); + if (IS_ERR_OR_NULL(pwr->icc_path)) { + if (PTR_ERR(pwr->icc_path) != -EPROBE_DEFER) + dev_err(dev, "Unable to read interconnect path from devicetree rc: %d\n", + PTR_ERR(pwr->icc_path)); + return pwr->icc_path ? PTR_ERR(pwr->icc_path) : -EINVAL; + } + + return 0; +} + +/* + * Cleanup done by devm. Any non-devm resources must clean up themselves. + */ +static struct arm_smmu_power_resources *arm_smmu_init_power_resources( + struct platform_device *pdev) +{ + struct arm_smmu_power_resources *pwr; + int ret; + + pwr = devm_kzalloc(&pdev->dev, sizeof(*pwr), GFP_KERNEL); + if (!pwr) + return ERR_PTR(-ENOMEM); + + pwr->dev = &pdev->dev; + pwr->pdev = pdev; + mutex_init(&pwr->power_lock); + spin_lock_init(&pwr->clock_refs_lock); + + ret = arm_smmu_init_clocks(pwr); + if (ret) + return ERR_PTR(ret); + + ret = arm_smmu_init_regulators(pwr); + if (ret) + return ERR_PTR(ret); + + ret = arm_smmu_init_interconnect(pwr); + if (ret) + return ERR_PTR(ret); + + return pwr; +} + +static void arm_smmu_exit_power_resources(struct arm_smmu_power_resources *pwr) +{ + icc_put(pwr->icc_path); +} + static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) { unsigned long size; @@ -1822,8 +4427,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; int i; - dev_notice(smmu->dev, "probing hardware configuration...\n"); - dev_notice(smmu->dev, "SMMUv%d with:\n", + dev_dbg(smmu->dev, "probing hardware configuration...\n"); + dev_dbg(smmu->dev, "SMMUv%d with:\n", smmu->version == ARM_SMMU_V2 ? 2 : 1); /* ID0 */ @@ -1837,17 +4442,17 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if (id & ID0_S1TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S1; - dev_notice(smmu->dev, "\tstage 1 translation\n"); + dev_dbg(smmu->dev, "\tstage 1 translation\n"); } if (id & ID0_S2TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S2; - dev_notice(smmu->dev, "\tstage 2 translation\n"); + dev_dbg(smmu->dev, "\tstage 2 translation\n"); } if (id & ID0_NTS) { smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; - dev_notice(smmu->dev, "\tnested translation\n"); + dev_dbg(smmu->dev, "\tnested translation\n"); } if (!(smmu->features & @@ -1859,7 +4464,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if ((id & ID0_S1TS) && ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) { smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; - dev_notice(smmu->dev, "\taddress translation ops\n"); + dev_dbg(smmu->dev, "\taddress translation ops\n"); } /* @@ -1912,6 +4517,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) smmu->num_mapping_groups = size; mutex_init(&smmu->stream_map_mutex); + mutex_init(&smmu->iommu_group_mutex); spin_lock_init(&smmu->global_sync_lock); if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { @@ -1925,20 +4531,22 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; /* Check for size mismatch of SMMU address space from mapped region */ - size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); + size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & + ID1_NUMPAGENDXB_MASK) + 1); size <<= smmu->pgshift; if (smmu->cb_base != gr0_base + size) dev_warn(smmu->dev, "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n", size * 2, (smmu->cb_base - gr0_base) * 2); - smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK; + smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) + & ID1_NUMS2CB_MASK; smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK; if (smmu->num_s2_context_banks > smmu->num_context_banks) { dev_err(smmu->dev, "impossible number of S2 context banks!\n"); return -ENODEV; } - dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", + dev_dbg(smmu->dev, "\t%u context banks (%u stage-2 only)\n", smmu->num_context_banks, smmu->num_s2_context_banks); /* * Cavium CN88xx erratum #27704. @@ -2004,21 +4612,21 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) smmu->pgsize_bitmap |= SZ_64K | SZ_512M; - if (arm_smmu_ops.pgsize_bitmap == -1UL) - arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; + if (arm_smmu_ops.iommu_ops.pgsize_bitmap == -1UL) + arm_smmu_ops.iommu_ops.pgsize_bitmap = smmu->pgsize_bitmap; else - arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; - dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", + arm_smmu_ops.iommu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; + dev_dbg(smmu->dev, "\tSupported page sizes: 0x%08lx\n", smmu->pgsize_bitmap); if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) - dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", - smmu->va_size, smmu->ipa_size); + dev_dbg(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", + smmu->va_size, smmu->ipa_size); if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) - dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", - smmu->ipa_size, smmu->pa_size); + dev_dbg(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", + smmu->ipa_size, smmu->pa_size); return 0; } @@ -2026,17 +4634,26 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) struct arm_smmu_match_data { enum arm_smmu_arch_version version; enum arm_smmu_implementation model; + struct arm_smmu_arch_ops *arch_ops; }; -#define ARM_SMMU_MATCH_DATA(name, ver, imp) \ -static const struct arm_smmu_match_data name = { .version = ver, .model = imp } +#define ARM_SMMU_MATCH_DATA(name, ver, imp, ops) \ +static struct arm_smmu_match_data name = { \ +.version = ver, \ +.model = imp, \ +.arch_ops = ops, \ +} \ -ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); -ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); -ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); -ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); -ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); -ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); +static struct arm_smmu_arch_ops qsmmuv500_arch_ops; + +ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU, NULL); +ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU, NULL); +ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU, NULL); +ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500, NULL); +ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2, NULL); +ARM_SMMU_MATCH_DATA(qcom_smmuv500, ARM_SMMU_V2, QCOM_SMMUV500, + &qsmmuv500_arch_ops); +ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2, NULL); static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, @@ -2045,9 +4662,11 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, + { .compatible = "qcom,qsmmu-v500", .data = &qcom_smmuv500 }, { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, { }, }; +MODULE_DEVICE_TABLE(of, arm_smmu_of_match); #ifdef CONFIG_ACPI static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) @@ -2115,25 +4734,37 @@ static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, } #endif -static int arm_smmu_device_dt_probe(struct platform_device *pdev, - struct arm_smmu_device *smmu) +static void arm_smmu_bus_init(void) +{ + /* Oh, for a proper bus abstraction */ + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &arm_smmu_ops.iommu_ops); +#ifdef CONFIG_ARM_AMBA + if (!iommu_present(&amba_bustype)) + bus_set_iommu(&amba_bustype, &arm_smmu_ops.iommu_ops); +#endif +#ifdef CONFIG_PCI + if (!iommu_present(&pci_bus_type)) { + pci_request_acs(); + bus_set_iommu(&pci_bus_type, &arm_smmu_ops.iommu_ops); + } +#endif +#ifdef CONFIG_FSL_MC_BUS + if (!iommu_present(&fsl_mc_bus_type)) + bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops); +#endif +} + +static int qsmmuv500_tbu_register(struct device *dev, void *data); +static int arm_smmu_device_dt_probe(struct platform_device *pdev) { const struct arm_smmu_match_data *data; + struct resource *res; + struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; + int num_irqs, i, err; bool legacy_binding; - if (of_property_read_u32(dev->of_node, "#global-interrupts", - &smmu->num_global_irqs)) { - dev_err(dev, "missing #global-interrupts property\n"); - return -ENODEV; - } - - data = of_device_get_match_data(dev); - smmu->version = data->version; - smmu->model = data->model; - - parse_driver_options(smmu); - legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); if (legacy_binding && !using_generic_binding) { if (!using_legacy_binding) @@ -2146,62 +4777,41 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev, return -ENODEV; } + smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); + if (!smmu) + return -ENOMEM; + + smmu->dev = dev; + spin_lock_init(&smmu->atos_lock); + idr_init(&smmu->asid_idr); + mutex_init(&smmu->idr_mutex); + + data = of_device_get_match_data(dev); + smmu->version = data->version; + smmu->model = data->model; + smmu->arch_ops = data->arch_ops; + if (of_dma_is_coherent(dev->of_node)) smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; - return 0; -} - -static void arm_smmu_bus_init(void) -{ - /* Oh, for a proper bus abstraction */ - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &arm_smmu_ops); -#ifdef CONFIG_ARM_AMBA - if (!iommu_present(&amba_bustype)) - bus_set_iommu(&amba_bustype, &arm_smmu_ops); -#endif -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - pci_request_acs(); - bus_set_iommu(&pci_bus_type, &arm_smmu_ops); - } -#endif -#ifdef CONFIG_FSL_MC_BUS - if (!iommu_present(&fsl_mc_bus_type)) - bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops); -#endif -} - -static int arm_smmu_device_probe(struct platform_device *pdev) -{ - struct resource *res; - resource_size_t ioaddr; - struct arm_smmu_device *smmu; - struct device *dev = &pdev->dev; - int num_irqs, i, err; - - smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); - if (!smmu) { - dev_err(dev, "failed to allocate arm_smmu_device\n"); - return -ENOMEM; - } - smmu->dev = dev; - - if (dev->of_node) - err = arm_smmu_device_dt_probe(pdev, smmu); - else - err = arm_smmu_device_acpi_probe(pdev, smmu); - - if (err) - return err; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ioaddr = res->start; + if (res == NULL) { + dev_err(dev, "no MEM resource info\n"); + return -EINVAL; + } + + smmu->phys_addr = res->start; smmu->base = devm_ioremap_resource(dev, res); if (IS_ERR(smmu->base)) return PTR_ERR(smmu->base); smmu->cb_base = smmu->base + resource_size(res) / 2; + smmu->size = resource_size(res); + + if (of_property_read_u32(dev->of_node, "#global-interrupts", + &smmu->num_global_irqs)) { + dev_err(dev, "missing #global-interrupts property\n"); + return -ENODEV; + } num_irqs = 0; while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { @@ -2218,10 +4828,8 @@ static int arm_smmu_device_probe(struct platform_device *pdev) smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs), GFP_KERNEL); - if (!smmu->irqs) { - dev_err(dev, "failed to allocate %d irqs\n", num_irqs); + if (!smmu->irqs) return -ENOMEM; - } for (i = 0; i < num_irqs; ++i) { int irq = platform_get_irq(pdev, i); @@ -2233,65 +4841,70 @@ static int arm_smmu_device_probe(struct platform_device *pdev) smmu->irqs[i] = irq; } - err = devm_clk_bulk_get_all(dev, &smmu->clks); - if (err < 0) { - dev_err(dev, "failed to get clocks %d\n", err); - return err; - } - smmu->num_clks = err; + parse_driver_options(smmu); - err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); + smmu->pwr = arm_smmu_init_power_resources(pdev); + if (IS_ERR(smmu->pwr)) + return PTR_ERR(smmu->pwr); + + err = arm_smmu_power_on(smmu->pwr); if (err) - return err; + goto out_exit_power_resources; err = arm_smmu_device_cfg_probe(smmu); if (err) - return err; + goto out_power_off; + + err = arm_smmu_handoff_cbs(smmu); + if (err) + goto out_power_off; + + err = arm_smmu_parse_impl_def_registers(smmu); + if (err) + goto out_power_off; if (smmu->version == ARM_SMMU_V2) { if (smmu->num_context_banks > smmu->num_context_irqs) { dev_err(dev, - "found only %d context irq(s) but %d required\n", - smmu->num_context_irqs, smmu->num_context_banks); + "found %d context interrupt(s) but have %d context banks. assuming %d context interrupts.\n", + smmu->num_context_irqs, smmu->num_context_banks, + smmu->num_context_banks); return -ENODEV; } - /* Ignore superfluous interrupts */ smmu->num_context_irqs = smmu->num_context_banks; } for (i = 0; i < smmu->num_global_irqs; ++i) { - err = devm_request_irq(smmu->dev, smmu->irqs[i], - arm_smmu_global_fault, - IRQF_SHARED, - "arm-smmu global fault", - smmu); + err = devm_request_threaded_irq(smmu->dev, smmu->irqs[i], + NULL, arm_smmu_global_fault, + IRQF_ONESHOT | IRQF_SHARED, + "arm-smmu global fault", smmu); if (err) { dev_err(dev, "failed to request global IRQ %d (%u)\n", i, smmu->irqs[i]); - return err; + goto out_power_off; } } - err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, - "smmu.%pa", &ioaddr); - if (err) { - dev_err(dev, "Failed to register iommu in sysfs\n"); - return err; - } + err = arm_smmu_arch_init(smmu); + if (err) + goto out_power_off; - iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops); + iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops.iommu_ops); iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); err = iommu_device_register(&smmu->iommu); + if (err) { dev_err(dev, "Failed to register iommu\n"); return err; } - platform_set_drvdata(pdev, smmu); arm_smmu_device_reset(smmu); arm_smmu_test_smr_masks(smmu); + arm_smmu_interrupt_selftest(smmu); + arm_smmu_power_off(smmu->pwr); /* * We want to avoid touching dev->power.lock in fastpaths unless @@ -2313,6 +4926,14 @@ static int arm_smmu_device_probe(struct platform_device *pdev) arm_smmu_bus_init(); return 0; + +out_power_off: + arm_smmu_power_off(smmu->pwr); + +out_exit_power_resources: + arm_smmu_exit_power_resources(smmu->pwr); + + return err; } /* @@ -2329,27 +4950,29 @@ static int arm_smmu_legacy_bus_init(void) } device_initcall_sync(arm_smmu_legacy_bus_init); -static void arm_smmu_device_shutdown(struct platform_device *pdev) +static int arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); if (!smmu) - return; + return -ENODEV; + + if (arm_smmu_power_on(smmu->pwr)) + return -EINVAL; if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_err(&pdev->dev, "removing device with active domains!\n"); - arm_smmu_rpm_get(smmu); + idr_destroy(&smmu->asid_idr); + /* Turn the thing off */ - writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); - arm_smmu_rpm_put(smmu); + writel_relaxed(sCR0_CLIENTPD, + ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); + arm_smmu_power_off(smmu->pwr); - if (pm_runtime_enabled(smmu->dev)) - pm_runtime_force_suspend(smmu->dev); - else - clk_bulk_disable(smmu->num_clks, smmu->clks); + arm_smmu_exit_power_resources(smmu->pwr); - clk_bulk_unprepare(smmu->num_clks, smmu->clks); + return 0; } static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) @@ -2361,7 +4984,12 @@ static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) if (ret) return ret; + ret = arm_smmu_power_on(smmu->pwr); + if (ret) + return ret; + arm_smmu_device_reset(smmu); + arm_smmu_power_off(smmu->pwr); return 0; } @@ -2399,12 +5027,693 @@ static const struct dev_pm_ops arm_smmu_pm_ops = { static struct platform_driver arm_smmu_driver = { .driver = { - .name = "arm-smmu", - .of_match_table = of_match_ptr(arm_smmu_of_match), - .pm = &arm_smmu_pm_ops, - .suppress_bind_attrs = true, + .name = "arm-smmu", + .of_match_table = of_match_ptr(arm_smmu_of_match), + .pm = &arm_smmu_pm_ops, + .suppress_bind_attrs = true, }, - .probe = arm_smmu_device_probe, - .shutdown = arm_smmu_device_shutdown, + .probe = arm_smmu_device_dt_probe, + .remove = arm_smmu_device_remove, +}; + +static struct platform_driver qsmmuv500_tbu_driver; +static int __init arm_smmu_init(void) +{ + static bool registered; + int ret = 0; + ktime_t cur; + + if (registered) + return 0; + + cur = ktime_get(); + ret = platform_driver_register(&qsmmuv500_tbu_driver); + if (ret) + return ret; + + ret = platform_driver_register(&arm_smmu_driver); + registered = !ret; + trace_smmu_init(ktime_us_delta(ktime_get(), cur)); + + return ret; +} + +static void __exit arm_smmu_exit(void) +{ + return platform_driver_unregister(&arm_smmu_driver); +} + +module_init(arm_smmu_init); +module_exit(arm_smmu_exit); + +#define TCU_HW_VERSION_HLOS1 (0x18) + +#define DEBUG_SID_HALT_REG 0x0 +#define DEBUG_SID_HALT_VAL (0x1 << 16) +#define DEBUG_SID_HALT_SID_MASK 0x3ff + +#define DEBUG_VA_ADDR_REG 0x8 + +#define DEBUG_TXN_TRIGG_REG 0x18 +#define DEBUG_TXN_AXPROT_SHIFT 6 +#define DEBUG_TXN_AXCACHE_SHIFT 2 +#define DEBUG_TRX_WRITE (0x1 << 1) +#define DEBUG_TXN_READ (0x0 << 1) +#define DEBUG_TXN_TRIGGER 0x1 + +#define DEBUG_SR_HALT_ACK_REG 0x20 +#define DEBUG_SR_HALT_ACK_VAL (0x1 << 1) +#define DEBUG_SR_ECATS_RUNNING_VAL (0x1 << 0) + +#define DEBUG_PAR_REG 0x28 +#define DEBUG_PAR_PA_MASK ((0x1ULL << 36) - 1) +#define DEBUG_PAR_PA_SHIFT 12 +#define DEBUG_PAR_FAULT_VAL 0x1 + +#define DEBUG_AXUSER_REG 0x30 +#define DEBUG_AXUSER_CDMID_MASK 0xff +#define DEBUG_AXUSER_CDMID_SHIFT 36 +#define DEBUG_AXUSER_CDMID_VAL 255 + +#define TBU_DBG_TIMEOUT_US 100 + +struct actlr_setting { + struct arm_smmu_smr smr; + u32 actlr; +}; + +struct qsmmuv500_archdata { + struct list_head tbus; + void __iomem *tcu_base; + u32 version; + struct actlr_setting *actlrs; + u32 actlr_tbl_size; +}; +#define get_qsmmuv500_archdata(smmu) \ + ((struct qsmmuv500_archdata *)(smmu->archdata)) + +struct qsmmuv500_tbu_device { + struct list_head list; + struct device *dev; + struct arm_smmu_device *smmu; + void __iomem *base; + void __iomem *status_reg; + + struct arm_smmu_power_resources *pwr; + u32 sid_start; + u32 num_sids; + + /* Protects halt count */ + spinlock_t halt_lock; + u32 halt_count; +}; + +struct qsmmuv500_group_iommudata { + bool has_actlr; + u32 actlr; +}; +#define to_qsmmuv500_group_iommudata(group) \ + ((struct qsmmuv500_group_iommudata *) \ + (iommu_group_get_iommudata(group))) + + +static bool arm_smmu_fwspec_match_smr(struct iommu_fwspec *fwspec, + struct arm_smmu_smr *smr) +{ + struct arm_smmu_smr *smr2; + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + int i, idx; + + for_each_cfg_sme(fwspec, i, idx) { + smr2 = &smmu->smrs[idx]; + /* Continue if table entry does not match */ + if ((smr->id ^ smr2->id) & ~(smr->mask | smr2->mask)) + continue; + return true; + } + return false; +} + +static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu, + struct arm_smmu_domain *smmu_domain) +{ + unsigned long flags; + u32 halt, fsr, sctlr_orig, sctlr, status; + void __iomem *base, *cb_base; + + if (of_property_read_bool(tbu->dev->of_node, + "qcom,opt-out-tbu-halting")) { + dev_notice(tbu->dev, "TBU opted-out for halting!\n"); + return -EBUSY; + } + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (tbu->halt_count) { + tbu->halt_count++; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return 0; + } + + cb_base = ARM_SMMU_CB(smmu_domain->smmu, smmu_domain->cfg.cbndx); + base = tbu->base; + halt = readl_relaxed(base + DEBUG_SID_HALT_REG); + halt |= DEBUG_SID_HALT_VAL; + writel_relaxed(halt, base + DEBUG_SID_HALT_REG); + + if (!readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG, status, + (status & DEBUG_SR_HALT_ACK_VAL), + 0, TBU_DBG_TIMEOUT_US)) + goto out; + + fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); + if (!(fsr & FSR_FAULT)) { + dev_err(tbu->dev, "Couldn't halt TBU!\n"); + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return -ETIMEDOUT; + } + + /* + * We are in a fault; Our request to halt the bus will not complete + * until transactions in front of us (such as the fault itself) have + * completed. Disable iommu faults and terminate any existing + * transactions. + */ + sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); + writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR); + + writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR); + writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME); + + if (readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG, status, + (status & DEBUG_SR_HALT_ACK_VAL), + 0, TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "Couldn't halt TBU from fault context!\n"); + writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR); + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return -ETIMEDOUT; + } + + writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR); +out: + tbu->halt_count = 1; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return 0; +} + +static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu) +{ + unsigned long flags; + u32 val; + void __iomem *base; + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (!tbu->halt_count) { + WARN(1, "%s: bad tbu->halt_count", dev_name(tbu->dev)); + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return; + + } else if (tbu->halt_count > 1) { + tbu->halt_count--; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return; + } + + base = tbu->base; + val = readl_relaxed(base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_VAL; + writel_relaxed(val, base + DEBUG_SID_HALT_REG); + + tbu->halt_count = 0; + spin_unlock_irqrestore(&tbu->halt_lock, flags); +} + +static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( + struct arm_smmu_device *smmu, u32 sid) +{ + struct qsmmuv500_tbu_device *tbu = NULL; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + + list_for_each_entry(tbu, &data->tbus, list) { + if (tbu->sid_start <= sid && + sid < tbu->sid_start + tbu->num_sids) + return tbu; + } + return NULL; +} + +static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain, + struct qsmmuv500_tbu_device *tbu, + unsigned long *flags) +{ + struct arm_smmu_device *smmu = tbu->smmu; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + u32 val; + + spin_lock_irqsave(&smmu->atos_lock, *flags); + /* The status register is not accessible on version 1.0 */ + if (data->version == 0x01000000) + return 0; + + if (readl_poll_timeout_atomic(tbu->status_reg, + val, (val == 0x1), 0, + TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "ECATS hw busy!\n"); + spin_unlock_irqrestore(&smmu->atos_lock, *flags); + return -ETIMEDOUT; + } + + return 0; +} + +static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, + struct qsmmuv500_tbu_device *tbu, + unsigned long *flags) +{ + struct arm_smmu_device *smmu = tbu->smmu; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + + /* The status register is not accessible on version 1.0 */ + if (data->version != 0x01000000) + writel_relaxed(0, tbu->status_reg); + spin_unlock_irqrestore(&smmu->atos_lock, *flags); +} + +/* + * Zero means failure. + */ +static phys_addr_t qsmmuv500_iova_to_phys( + struct iommu_domain *domain, dma_addr_t iova, u32 sid) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct qsmmuv500_tbu_device *tbu; + int ret; + phys_addr_t phys = 0; + u64 val, fsr; + unsigned long flags; + void __iomem *cb_base; + u32 sctlr_orig, sctlr; + int needs_redo = 0; + ktime_t timeout; + + /* only 36 bit iova is supported */ + if (iova >= (1ULL << 36)) { + dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n", + &iova); + return 0; + } + + cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); + tbu = qsmmuv500_find_tbu(smmu, sid); + if (!tbu) + return 0; + + ret = arm_smmu_power_on(tbu->pwr); + if (ret) + return 0; + + ret = qsmmuv500_tbu_halt(tbu, smmu_domain); + if (ret) + goto out_power_off; + + /* + * ECATS can trigger the fault interrupt, so disable it temporarily + * and check for an interrupt manually. + */ + sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); + writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR); + + /* Only one concurrent atos operation */ + ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags); + if (ret) + goto out_resume; + +redo: + /* Set address and stream-id */ + val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID_MASK; + val |= sid & DEBUG_SID_HALT_SID_MASK; + writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG); + val = (u64)(DEBUG_AXUSER_CDMID_VAL & DEBUG_AXUSER_CDMID_MASK) << + DEBUG_AXUSER_CDMID_SHIFT; + writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); + + /* + * Write-back Read and Write-Allocate + * Priviledged, nonsecure, data transaction + * Read operation. + */ + val = 0xF << DEBUG_TXN_AXCACHE_SHIFT; + val |= 0x3 << DEBUG_TXN_AXPROT_SHIFT; + val |= DEBUG_TXN_TRIGGER; + writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); + + ret = 0; + timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); + for (;;) { + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) + break; + val = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); + if (val & FSR_FAULT) + break; + if (ktime_compare(ktime_get(), timeout) > 0) { + dev_err(tbu->dev, "ECATS translation timed out!\n"); + ret = -ETIMEDOUT; + break; + } + } + + val = readq_relaxed(tbu->base + DEBUG_PAR_REG); + fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); + if (fsr & FSR_FAULT) { + dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx, SID=0x%x\n", + fsr, sid); + + /* Clear pending interrupts */ + writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation. + */ + wmb(); + writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME); + + /* Check if ECATS translation failed */ + if (val & DEBUG_PAR_FAULT_VAL) + dev_err(tbu->dev, "ECATS translation failed! PAR = %llx\n", + val); + ret = -EINVAL; + } + + phys = (val >> DEBUG_PAR_PA_SHIFT) & DEBUG_PAR_PA_MASK; + if (ret < 0) + phys = 0; + + /* Reset hardware */ + writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG); + writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG); + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID_MASK; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + /* + * After a failed translation, the next successful translation will + * incorrectly be reported as a failure. + */ + if (!phys && needs_redo++ < 2) + goto redo; + + writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR); + qsmmuv500_ecats_unlock(smmu_domain, tbu, &flags); + +out_resume: + qsmmuv500_tbu_resume(tbu); + +out_power_off: + /* Read to complete prior write transcations */ + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + + /* Wait for read to complete before off */ + rmb(); + + arm_smmu_power_off(tbu->pwr); + + return phys; +} + +static phys_addr_t qsmmuv500_iova_to_phys_hard( + struct iommu_domain *domain, dma_addr_t iova) +{ + u16 sid; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct iommu_fwspec *fwspec; + void __iomem *gr1_base; + u32 frsynra; + int is_debug_domain; + + arm_smmu_domain_get_attr(domain, DOMAIN_ATTR_DEBUG, &is_debug_domain); + + /* Check to see if the domain is associated with the test + * device. If the domain belongs to the test device, then + * pick the SID from fwspec. + */ + if (is_debug_domain) { + fwspec = smmu_domain->dev->iommu_fwspec; + sid = (u16)fwspec->ids[0]; + } else { + + /* If the domain belongs to an actual device, read + * SID from the corresponding frsynra register + */ + gr1_base = ARM_SMMU_GR1(smmu); + frsynra = readl_relaxed(gr1_base + + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); + frsynra &= CBFRSYNRA_SID_MASK; + sid = frsynra; + } + return qsmmuv500_iova_to_phys(domain, iova, sid); +} + +static void qsmmuv500_release_group_iommudata(void *data) +{ + kfree(data); +} + +/* If a device has a valid actlr, it must match */ +static int qsmmuv500_device_group(struct device *dev, + struct iommu_group *group) +{ + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_group_iommudata *iommudata; + u32 actlr, i; + struct arm_smmu_smr *smr; + + iommudata = to_qsmmuv500_group_iommudata(group); + if (!iommudata) { + iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); + if (!iommudata) + return -ENOMEM; + + iommu_group_set_iommudata(group, iommudata, + qsmmuv500_release_group_iommudata); + } + + for (i = 0; i < data->actlr_tbl_size; i++) { + smr = &data->actlrs[i].smr; + actlr = data->actlrs[i].actlr; + + if (!arm_smmu_fwspec_match_smr(fwspec, smr)) + continue; + + if (!iommudata->has_actlr) { + iommudata->actlr = actlr; + iommudata->has_actlr = true; + } else if (iommudata->actlr != actlr) { + return -EINVAL; + } + } + + return 0; +} + +static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, + struct device *dev) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct qsmmuv500_group_iommudata *iommudata = + to_qsmmuv500_group_iommudata(dev->iommu_group); + void __iomem *cb_base; + const struct iommu_gather_ops *tlb; + + if (!iommudata->has_actlr) + return; + + tlb = smmu_domain->pgtbl_info.pgtbl_cfg.tlb; + cb_base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx); + + writel_relaxed(iommudata->actlr, cb_base + ARM_SMMU_CB_ACTLR); + + /* + * Flush the context bank after modifying ACTLR to ensure there + * are no cache entries with stale state + */ + tlb->tlb_flush_all(smmu_domain); +} + +static int qsmmuv500_tbu_register(struct device *dev, void *cookie) +{ + struct arm_smmu_device *smmu = cookie; + struct qsmmuv500_tbu_device *tbu; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + + if (!dev->driver) { + dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); + return -EINVAL; + } + + tbu = dev_get_drvdata(dev); + + INIT_LIST_HEAD(&tbu->list); + tbu->smmu = smmu; + list_add(&tbu->list, &data->tbus); + return 0; +} + +static int qsmmuv500_read_actlr_tbl(struct arm_smmu_device *smmu) +{ + int len, i; + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct actlr_setting *actlrs; + const __be32 *cell; + + cell = of_get_property(dev->of_node, "qcom,actlr", NULL); + if (!cell) + return 0; + + len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr", + sizeof(u32) * 3); + if (len < 0) + return 0; + + actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL); + if (!actlrs) + return -ENOMEM; + + for (i = 0; i < len; i++) { + actlrs[i].smr.id = of_read_number(cell++, 1); + actlrs[i].smr.mask = of_read_number(cell++, 1); + actlrs[i].actlr = of_read_number(cell++, 1); + } + + data->actlrs = actlrs; + data->actlr_tbl_size = len; + return 0; +} + +static int qsmmuv500_arch_init(struct arm_smmu_device *smmu) +{ + struct resource *res; + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data; + struct platform_device *pdev; + int ret; + u32 val; + void __iomem *reg; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + INIT_LIST_HEAD(&data->tbus); + + pdev = container_of(dev, struct platform_device, dev); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base"); + if (!res) { + dev_err(dev, "Unable to get the tcu-base\n"); + return -EINVAL; + } + data->tcu_base = devm_ioremap(dev, res->start, resource_size(res)); + if (IS_ERR(data->tcu_base)) + return PTR_ERR(data->tcu_base); + + data->version = readl_relaxed(data->tcu_base + TCU_HW_VERSION_HLOS1); + smmu->archdata = data; + + ret = qsmmuv500_read_actlr_tbl(smmu); + if (ret) + return ret; + + reg = ARM_SMMU_GR0(smmu); + val = readl_relaxed(reg + ARM_SMMU_GR0_sACR); + val &= ~ARM_MMU500_ACR_CACHE_LOCK; + writel_relaxed(val, reg + ARM_SMMU_GR0_sACR); + val = readl_relaxed(reg + ARM_SMMU_GR0_sACR); + /* + * Modifiying the nonsecure copy of the sACR register is only + * allowed if permission is given in the secure sACR register. + * Attempt to detect if we were able to update the value. + */ + WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); + + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + return ret; + + /* Attempt to register child devices */ + ret = device_for_each_child(dev, smmu, qsmmuv500_tbu_register); + if (ret) + return -EPROBE_DEFER; + + return 0; +} + +static struct arm_smmu_arch_ops qsmmuv500_arch_ops = { + .init = qsmmuv500_arch_init, + .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, + .init_context_bank = qsmmuv500_init_cb, + .device_group = qsmmuv500_device_group, +}; + +static const struct of_device_id qsmmuv500_tbu_of_match[] = { + {.compatible = "qcom,qsmmuv500-tbu"}, + {} +}; + +static int qsmmuv500_tbu_probe(struct platform_device *pdev) +{ + struct resource *res; + struct device *dev = &pdev->dev; + struct qsmmuv500_tbu_device *tbu; + const __be32 *cell; + int len; + + tbu = devm_kzalloc(dev, sizeof(*tbu), GFP_KERNEL); + if (!tbu) + return -ENOMEM; + + INIT_LIST_HEAD(&tbu->list); + tbu->dev = dev; + spin_lock_init(&tbu->halt_lock); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base"); + tbu->base = devm_ioremap_resource(dev, res); + if (IS_ERR(tbu->base)) + return PTR_ERR(tbu->base); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "status-reg"); + tbu->status_reg = devm_ioremap_resource(dev, res); + if (IS_ERR(tbu->status_reg)) + return PTR_ERR(tbu->status_reg); + + cell = of_get_property(dev->of_node, "qcom,stream-id-range", &len); + if (!cell || len < 8) + return -EINVAL; + + tbu->sid_start = of_read_number(cell, 1); + tbu->num_sids = of_read_number(cell + 1, 1); + + tbu->pwr = arm_smmu_init_power_resources(pdev); + if (IS_ERR(tbu->pwr)) + return PTR_ERR(tbu->pwr); + + dev_set_drvdata(dev, tbu); + return 0; +} + +static struct platform_driver qsmmuv500_tbu_driver = { + .driver = { + .name = "qsmmuv500-tbu", + .of_match_table = of_match_ptr(qsmmuv500_tbu_of_match), + }, + .probe = qsmmuv500_tbu_probe, }; -builtin_platform_driver(arm_smmu_driver); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 379318266468..f29c77d89a66 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -372,6 +372,15 @@ int dma_info_to_prot(enum dma_data_direction dir, bool coherent, if (attrs & DMA_ATTR_PRIVILEGED) prot |= IOMMU_PRIV; + if (!(attrs & DMA_ATTR_EXEC_MAPPING)) + prot |= IOMMU_NOEXEC; + + if (attrs & DMA_ATTR_IOMMU_USE_UPSTREAM_HINT) + prot |= IOMMU_USE_UPSTREAM_HINT; + + if (attrs & DMA_ATTR_IOMMU_USE_LLC_NWA) + prot |= IOMMU_USE_LLC_NWA; + switch (dir) { case DMA_BIDIRECTIONAL: return prot | IOMMU_READ | IOMMU_WRITE; @@ -468,7 +477,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, unsigned int count, unsigned long order_mask, gfp_t gfp) { struct page **pages; - unsigned int i = 0, nid = dev_to_node(dev); + unsigned int i = 0; order_mask &= (2U << MAX_ORDER) - 1; if (!order_mask) @@ -493,12 +502,11 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, for (order_mask &= (2U << __fls(count)) - 1; order_mask; order_mask &= ~order_size) { unsigned int order = __fls(order_mask); - gfp_t alloc_flags = gfp; order_size = 1U << order; - if (order_mask > order_size) - alloc_flags |= __GFP_NORETRY; - page = alloc_pages_node(nid, alloc_flags, order); + page = alloc_pages(order ? + (gfp | __GFP_NORETRY) & + ~__GFP_RECLAIM : gfp, order); if (!page) continue; if (!order) @@ -535,7 +543,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, void iommu_dma_free(struct device *dev, struct page **pages, size_t size, dma_addr_t *handle) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), *handle, size); + __iommu_dma_unmap(iommu_get_domain_for_dev(dev), *handle, size); __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT); *handle = DMA_MAPPING_ERROR; } @@ -562,7 +570,7 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, unsigned long attrs, int prot, dma_addr_t *handle, void (*flush_page)(struct device *, const void *, phys_addr_t)) { - struct iommu_domain *domain = iommu_get_dma_domain(dev); + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; struct page **pages; @@ -667,13 +675,13 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, int prot) { return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot, - iommu_get_dma_domain(dev)); + iommu_get_domain_for_dev(dev)); } void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); + __iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle, size); } /* @@ -683,7 +691,7 @@ void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, * avoid individually crossing any boundaries, so we merely need to check a * segment's start address to avoid concatenating across one. */ -static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, +int iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents, dma_addr_t dma_addr) { struct scatterlist *s, *cur = sg; @@ -736,7 +744,7 @@ static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, * If mapping failed, then just restore the original list, * but making sure the DMA fields are invalidated. */ -static void __invalidate_sg(struct scatterlist *sg, int nents) +void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents) { struct scatterlist *s; int i; @@ -758,14 +766,10 @@ static void __invalidate_sg(struct scatterlist *sg, int nents) * impedance-matching, to be able to hand off a suitably-aligned list, * but still preserve the original offsets and sizes for the caller. */ -int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, - int nents, int prot) +size_t iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad, + struct scatterlist *sg, int nents) { - struct iommu_domain *domain = iommu_get_dma_domain(dev); - struct iommu_dma_cookie *cookie = domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; struct scatterlist *s, *prev = NULL; - dma_addr_t iova; size_t iova_len = 0; unsigned long mask = dma_get_seg_boundary(dev); int i; @@ -809,6 +813,26 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, prev = s; } + return iova_len; +} + +int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, int prot) +{ + struct iommu_domain *domain; + struct iommu_dma_cookie *cookie; + struct iova_domain *iovad; + dma_addr_t iova; + size_t iova_len; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return 0; + cookie = domain->iova_cookie; + iovad = &cookie->iovad; + + iova_len = iommu_dma_prepare_map_sg(dev, iovad, sg, nents); + iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev); if (!iova) goto out_restore_sg; @@ -820,12 +844,12 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len) goto out_free_iova; - return __finalise_sg(dev, sg, nents, iova); + return iommu_dma_finalise_sg(dev, sg, nents, iova); out_free_iova: iommu_dma_free_iova(cookie, iova, iova_len); out_restore_sg: - __invalidate_sg(sg, nents); + iommu_dma_invalidate_sg(sg, nents); return 0; } @@ -846,7 +870,7 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, sg = tmp; } end = sg_dma_address(sg) + sg_dma_len(sg); - __iommu_dma_unmap(iommu_get_dma_domain(dev), start, end - start); + __iommu_dma_unmap(iommu_get_domain_for_dev(dev), start, end - start); } dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, @@ -854,13 +878,13 @@ dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, { return __iommu_dma_map(dev, phys, size, dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO, - iommu_get_dma_domain(dev)); + iommu_get_domain_for_dev(dev)); } void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { - __iommu_dma_unmap(iommu_get_dma_domain(dev), handle, size); + __iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle, size); } static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, diff --git a/drivers/iommu/dma-mapping-fast.c b/drivers/iommu/dma-mapping-fast.c new file mode 100644 index 000000000000..61a8dadf4f7e --- /dev/null +++ b/drivers/iommu/dma-mapping-fast.c @@ -0,0 +1,969 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* some redundant definitions... :( TODO: move to io-pgtable-fast.h */ +#define FAST_PAGE_SHIFT 12 +#define FAST_PAGE_SIZE (1UL << FAST_PAGE_SHIFT) +#define FAST_PAGE_MASK (~(PAGE_SIZE - 1)) + +static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot, + bool coherent) +{ + if (!coherent || (attrs & DMA_ATTR_WRITE_COMBINE)) + return pgprot_writecombine(prot); + return prot; +} + +static bool is_dma_coherent(struct device *dev, unsigned long attrs) +{ + bool is_coherent; + + if (attrs & DMA_ATTR_FORCE_COHERENT) + is_coherent = true; + else if (attrs & DMA_ATTR_FORCE_NON_COHERENT) + is_coherent = false; + else if (is_device_dma_coherent(dev)) + is_coherent = true; + else + is_coherent = false; + + return is_coherent; +} + +static struct dma_fast_smmu_mapping *dev_get_mapping(struct device *dev) +{ + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return ERR_PTR(-EINVAL); + return domain->iova_cookie; +} + +/* + * Checks if the allocated range (ending at @end) covered the upcoming + * stale bit. We don't need to know exactly where the range starts since + * we already know where the candidate search range started. If, starting + * from the beginning of the candidate search range, we had to step over + * (or landed directly on top of) the upcoming stale bit, then we return + * true. + * + * Due to wrapping, there are two scenarios we'll need to check: (1) if the + * range [search_start, upcoming_stale] spans 0 (i.e. search_start > + * upcoming_stale), and, (2) if the range: [search_start, upcoming_stale] + * does *not* span 0 (i.e. search_start <= upcoming_stale). And for each + * of those two scenarios we need to handle three cases: (1) the bit was + * found before wrapping or + */ +static bool __bit_covered_stale(unsigned long upcoming_stale, + unsigned long search_start, + unsigned long end) +{ + if (search_start > upcoming_stale) { + if (end >= search_start) { + /* + * We started searching above upcoming_stale and we + * didn't wrap, so we couldn't have crossed + * upcoming_stale. + */ + return false; + } + /* + * We wrapped. Did we cross (or land on top of) + * upcoming_stale? + */ + return end >= upcoming_stale; + } + + if (search_start <= upcoming_stale) { + if (end >= search_start) { + /* + * We didn't wrap. Did we cross (or land on top + * of) upcoming_stale? + */ + return end >= upcoming_stale; + } + /* + * We wrapped. So we must have crossed upcoming_stale + * (since we started searching below it). + */ + return true; + } + + /* we should have covered all logical combinations... */ + WARN_ON(1); + return true; +} + +static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping, + unsigned long attrs, + size_t size) +{ + unsigned long bit, prev_search_start, nbits = size >> FAST_PAGE_SHIFT; + unsigned long align = (1 << get_order(size)) - 1; + + bit = bitmap_find_next_zero_area( + mapping->bitmap, mapping->num_4k_pages, mapping->next_start, + nbits, align); + if (unlikely(bit > mapping->num_4k_pages)) { + /* try wrapping */ + mapping->next_start = 0; /* TODO: SHOULD I REALLY DO THIS?!? */ + bit = bitmap_find_next_zero_area( + mapping->bitmap, mapping->num_4k_pages, 0, nbits, + align); + if (unlikely(bit > mapping->num_4k_pages)) + return DMA_ERROR_CODE; + } + + bitmap_set(mapping->bitmap, bit, nbits); + prev_search_start = mapping->next_start; + mapping->next_start = bit + nbits; + if (unlikely(mapping->next_start >= mapping->num_4k_pages)) + mapping->next_start = 0; + + /* + * If we just re-allocated a VA whose TLB hasn't been invalidated + * since it was last used and unmapped, we need to invalidate it + * here. We actually invalidate the entire TLB so that we don't + * have to invalidate the TLB again until we wrap back around. + */ + if (mapping->have_stale_tlbs && + __bit_covered_stale(mapping->upcoming_stale_bit, + prev_search_start, + bit + nbits - 1)) { + bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC); + + iommu_tlbiall(mapping->domain); + mapping->have_stale_tlbs = false; + av8l_fast_clear_stale_ptes(mapping->pgtbl_ops, skip_sync); + } + + return (bit << FAST_PAGE_SHIFT) + mapping->base; +} + +/* + * Checks whether the candidate bit will be allocated sooner than the + * current upcoming stale bit. We can say candidate will be upcoming + * sooner than the current upcoming stale bit if it lies between the + * starting bit of the next search range and the upcoming stale bit + * (allowing for wrap-around). + * + * Stated differently, we're checking the relative ordering of three + * unsigned numbers. So we need to check all 6 (i.e. 3!) permutations, + * namely: + * + * 0 |---A---B---C---| TOP (Case 1) + * 0 |---A---C---B---| TOP (Case 2) + * 0 |---B---A---C---| TOP (Case 3) + * 0 |---B---C---A---| TOP (Case 4) + * 0 |---C---A---B---| TOP (Case 5) + * 0 |---C---B---A---| TOP (Case 6) + * + * Note that since we're allowing numbers to wrap, the following three + * scenarios are all equivalent for Case 1: + * + * 0 |---A---B---C---| TOP + * 0 |---C---A---B---| TOP (C has wrapped. This is Case 5.) + * 0 |---B---C---A---| TOP (C and B have wrapped. This is Case 4.) + * + * In any of these cases, if we start searching from A, we will find B + * before we find C. + * + * We can also find two equivalent cases for Case 2: + * + * 0 |---A---C---B---| TOP + * 0 |---B---A---C---| TOP (B has wrapped. This is Case 3.) + * 0 |---C---B---A---| TOP (B and C have wrapped. This is Case 6.) + * + * In any of these cases, if we start searching from A, we will find C + * before we find B. + */ +static bool __bit_is_sooner(unsigned long candidate, + struct dma_fast_smmu_mapping *mapping) +{ + unsigned long A = mapping->next_start; + unsigned long B = candidate; + unsigned long C = mapping->upcoming_stale_bit; + + if ((A < B && B < C) || /* Case 1 */ + (C < A && A < B) || /* Case 5 */ + (B < C && C < A)) /* Case 4 */ + return true; + + if ((A < C && C < B) || /* Case 2 */ + (B < A && A < C) || /* Case 3 */ + (C < B && B < A)) /* Case 6 */ + return false; + + /* + * For simplicity, we've been ignoring the possibility of any of + * our three numbers being equal. Handle those cases here (they + * shouldn't happen very often, (I think?)). + */ + + /* + * If candidate is the next bit to be searched then it's definitely + * sooner. + */ + if (A == B) + return true; + + /* + * If candidate is the next upcoming stale bit we'll return false + * to avoid doing `upcoming = candidate' in the caller (which would + * be useless since they're already equal) + */ + if (B == C) + return false; + + /* + * If next start is the upcoming stale bit then candidate can't + * possibly be sooner. The "soonest" bit is already selected. + */ + if (A == C) + return false; + + /* We should have covered all logical combinations. */ + WARN(1, "Well, that's awkward. A=%ld, B=%ld, C=%ld\n", A, B, C); + return true; +} + +static void __fast_smmu_free_iova(struct dma_fast_smmu_mapping *mapping, + dma_addr_t iova, size_t size) +{ + unsigned long start_bit = (iova - mapping->base) >> FAST_PAGE_SHIFT; + unsigned long nbits = size >> FAST_PAGE_SHIFT; + + /* + * We don't invalidate TLBs on unmap. We invalidate TLBs on map + * when we're about to re-allocate a VA that was previously + * unmapped but hasn't yet been invalidated. So we need to keep + * track of which bit is the closest to being re-allocated here. + */ + if (__bit_is_sooner(start_bit, mapping)) + mapping->upcoming_stale_bit = start_bit; + + bitmap_clear(mapping->bitmap, start_bit, nbits); + mapping->have_stale_tlbs = true; +} + + +static void __fast_dma_page_cpu_to_dev(struct page *page, unsigned long off, + size_t size, enum dma_data_direction dir) +{ + __dma_map_area(page_address(page) + off, size, dir); +} + +static void __fast_dma_page_dev_to_cpu(struct page *page, unsigned long off, + size_t size, enum dma_data_direction dir) +{ + __dma_unmap_area(page_address(page) + off, size, dir); + + /* TODO: WHAT IS THIS? */ + /* + * Mark the D-cache clean for this page to avoid extra flushing. + */ + if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) + set_bit(PG_dcache_clean, &page->flags); +} + +static dma_addr_t fast_smmu_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + dma_addr_t iova; + unsigned long flags; + phys_addr_t phys_plus_off = page_to_phys(page) + offset; + phys_addr_t phys_to_map = round_down(phys_plus_off, FAST_PAGE_SIZE); + unsigned long offset_from_phys_to_map = phys_plus_off & ~FAST_PAGE_MASK; + size_t len = ALIGN(size + offset_from_phys_to_map, FAST_PAGE_SIZE); + bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC); + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = dma_info_to_prot(dir, is_coherent, attrs); + + if (!skip_sync && !is_coherent) + __fast_dma_page_cpu_to_dev(phys_to_page(phys_to_map), + offset_from_phys_to_map, size, dir); + + spin_lock_irqsave(&mapping->lock, flags); + + iova = __fast_smmu_alloc_iova(mapping, attrs, len); + + if (unlikely(iova == DMA_ERROR_CODE)) + goto fail; + + if (unlikely(av8l_fast_map_public(mapping->pgtbl_ops, iova, + phys_to_map, len, prot))) + goto fail_free_iova; + + spin_unlock_irqrestore(&mapping->lock, flags); + + trace_map(to_msm_iommu_domain(mapping->domain), iova, phys_to_map, len, + prot); + return iova + offset_from_phys_to_map; + +fail_free_iova: + __fast_smmu_free_iova(mapping, iova, size); +fail: + spin_unlock_irqrestore(&mapping->lock, flags); + return DMA_ERROR_CODE; +} + +static void fast_smmu_unmap_page(struct device *dev, dma_addr_t iova, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + unsigned long flags; + unsigned long offset = iova & ~FAST_PAGE_MASK; + size_t len = ALIGN(size + offset, FAST_PAGE_SIZE); + bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC); + bool is_coherent = is_dma_coherent(dev, attrs); + + if (!skip_sync && !is_coherent) { + phys_addr_t phys; + + phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova); + WARN_ON(!phys); + + __fast_dma_page_dev_to_cpu(phys_to_page(phys), offset, + size, dir); + } + + spin_lock_irqsave(&mapping->lock, flags); + av8l_fast_unmap_public(mapping->pgtbl_ops, iova, len); + __fast_smmu_free_iova(mapping, iova, len); + spin_unlock_irqrestore(&mapping->lock, flags); + + trace_unmap(to_msm_iommu_domain(mapping->domain), iova - offset, len, + len); +} + +static void fast_smmu_sync_single_for_cpu(struct device *dev, + dma_addr_t iova, size_t size, enum dma_data_direction dir) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + unsigned long offset = iova & ~FAST_PAGE_MASK; + + if (!av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) { + phys_addr_t phys; + + phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova); + WARN_ON(!phys); + + __fast_dma_page_dev_to_cpu(phys_to_page(phys), offset, + size, dir); + } +} + +static void fast_smmu_sync_single_for_device(struct device *dev, + dma_addr_t iova, size_t size, enum dma_data_direction dir) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + unsigned long offset = iova & ~FAST_PAGE_MASK; + + if (!av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) { + phys_addr_t phys; + + phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova); + WARN_ON(!phys); + + __fast_dma_page_cpu_to_dev(phys_to_page(phys), offset, + size, dir); + } +} + +static void fast_smmu_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + dma_addr_t iova = sg_dma_address(sgl); + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + int i; + + if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) + return; + + for_each_sg(sgl, sg, nelems, i) + __dma_unmap_area(sg_virt(sg), sg->length, dir); +} + +static void fast_smmu_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + dma_addr_t iova = sg_dma_address(sgl); + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + int i; + + if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) + return; + + for_each_sg(sgl, sg, nelems, i) + __dma_map_area(sg_virt(sg), sg->length, dir); +} + +static int fast_smmu_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + size_t iova_len; + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = dma_info_to_prot(dir, is_coherent, attrs); + int ret; + dma_addr_t iova; + unsigned long flags; + size_t unused; + + iova_len = iommu_dma_prepare_map_sg(dev, mapping->iovad, sg, nents); + + spin_lock_irqsave(&mapping->lock, flags); + iova = __fast_smmu_alloc_iova(mapping, attrs, iova_len); + spin_unlock_irqrestore(&mapping->lock, flags); + + if (unlikely(iova == DMA_ERROR_CODE)) + goto fail; + + av8l_fast_map_sg_public(mapping->pgtbl_ops, iova, sg, nents, prot, + &unused); + + ret = iommu_dma_finalise_sg(dev, sg, nents, iova); + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + fast_smmu_sync_sg_for_device(dev, sg, nents, dir); + + return ret; +fail: + iommu_dma_invalidate_sg(sg, nents); + return 0; +} + +static void fast_smmu_unmap_sg(struct device *dev, + struct scatterlist *sg, int nelems, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + unsigned long flags; + dma_addr_t start; + size_t len; + struct scatterlist *tmp; + int i; + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + fast_smmu_sync_sg_for_cpu(dev, sg, nelems, dir); + + /* + * The scatterlist segments are mapped into a single + * contiguous IOVA allocation, so this is incredibly easy. + */ + start = sg_dma_address(sg); + for_each_sg(sg_next(sg), tmp, nelems - 1, i) { + if (sg_dma_len(tmp) == 0) + break; + sg = tmp; + } + len = sg_dma_address(sg) + sg_dma_len(sg) - start; + + av8l_fast_unmap_public(mapping->pgtbl_ops, start, len); + + spin_lock_irqsave(&mapping->lock, flags); + __fast_smmu_free_iova(mapping, start, len); + spin_unlock_irqrestore(&mapping->lock, flags); +} + +static void __fast_smmu_free_pages(struct page **pages, int count) +{ + int i; + + for (i = 0; i < count; i++) + __free_page(pages[i]); + kvfree(pages); +} + +static struct page **__fast_smmu_alloc_pages(unsigned int count, gfp_t gfp) +{ + struct page **pages; + unsigned int i = 0, array_size = count * sizeof(*pages); + + if (array_size <= PAGE_SIZE) + pages = kzalloc(array_size, GFP_KERNEL); + else + pages = vzalloc(array_size); + if (!pages) + return NULL; + + /* IOMMU can map any pages, so himem can also be used here */ + gfp |= __GFP_NOWARN | __GFP_HIGHMEM; + + for (i = 0; i < count; ++i) { + struct page *page = alloc_page(gfp); + + if (!page) { + __fast_smmu_free_pages(pages, i); + return NULL; + } + pages[i] = page; + } + return pages; +} + +static void *fast_smmu_alloc(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + struct sg_table sgt; + dma_addr_t dma_addr, iova_iter; + void *addr; + unsigned long flags; + struct sg_mapping_iter miter; + size_t count = ALIGN(size, SZ_4K) >> PAGE_SHIFT; + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = dma_info_to_prot(DMA_BIDIRECTIONAL, is_coherent, attrs); + pgprot_t remap_prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent); + struct page **pages; + + /* + * sg_alloc_table_from_pages accepts unsigned int value for count + * so check count doesn't exceed UINT_MAX. + */ + + if (count > UINT_MAX) { + dev_err(dev, "count: %zx exceeds UNIT_MAX\n", count); + return NULL; + } + + *handle = DMA_ERROR_CODE; + + pages = __fast_smmu_alloc_pages(count, gfp); + if (!pages) { + dev_err(dev, "no pages\n"); + return NULL; + } + + size = ALIGN(size, SZ_4K); + if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, gfp)) { + dev_err(dev, "no sg tablen\n"); + goto out_free_pages; + } + + if (!is_coherent) { + /* + * The CPU-centric flushing implied by SG_MITER_TO_SG isn't + * sufficient here, so skip it by using the "wrong" direction. + */ + sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, + SG_MITER_FROM_SG); + while (sg_miter_next(&miter)) + __dma_flush_area(miter.addr, miter.length); + sg_miter_stop(&miter); + } + + spin_lock_irqsave(&mapping->lock, flags); + dma_addr = __fast_smmu_alloc_iova(mapping, attrs, size); + if (dma_addr == DMA_ERROR_CODE) { + dev_err(dev, "no iova\n"); + spin_unlock_irqrestore(&mapping->lock, flags); + goto out_free_sg; + } + iova_iter = dma_addr; + sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + while (sg_miter_next(&miter)) { + if (unlikely(av8l_fast_map_public( + mapping->pgtbl_ops, iova_iter, + page_to_phys(miter.page), + miter.length, prot))) { + dev_err(dev, "no map public\n"); + /* TODO: unwind previously successful mappings */ + goto out_free_iova; + } + iova_iter += miter.length; + } + sg_miter_stop(&miter); + spin_unlock_irqrestore(&mapping->lock, flags); + + addr = dma_common_pages_remap(pages, size, VM_USERMAP, remap_prot, + __builtin_return_address(0)); + if (!addr) { + dev_err(dev, "no common pages\n"); + goto out_unmap; + } + + *handle = dma_addr; + sg_free_table(&sgt); + return addr; + +out_unmap: + /* need to take the lock again for page tables and iova */ + spin_lock_irqsave(&mapping->lock, flags); + av8l_fast_unmap_public(mapping->pgtbl_ops, dma_addr, size); +out_free_iova: + __fast_smmu_free_iova(mapping, dma_addr, size); + spin_unlock_irqrestore(&mapping->lock, flags); +out_free_sg: + sg_free_table(&sgt); +out_free_pages: + __fast_smmu_free_pages(pages, count); + return NULL; +} + +static void fast_smmu_free(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + struct vm_struct *area; + struct page **pages; + size_t count = ALIGN(size, SZ_4K) >> FAST_PAGE_SHIFT; + unsigned long flags; + + size = ALIGN(size, SZ_4K); + + area = find_vm_area(vaddr); + if (WARN_ON_ONCE(!area)) + return; + + pages = area->pages; + dma_common_free_remap(vaddr, size, VM_USERMAP); + spin_lock_irqsave(&mapping->lock, flags); + av8l_fast_unmap_public(mapping->pgtbl_ops, dma_handle, size); + __fast_smmu_free_iova(mapping, dma_handle, size); + spin_unlock_irqrestore(&mapping->lock, flags); + __fast_smmu_free_pages(pages, count); +} + +static int fast_smmu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, + size_t size, unsigned long attrs) +{ + struct vm_struct *area; + unsigned long uaddr = vma->vm_start; + struct page **pages; + int i, nr_pages, ret = 0; + bool coherent = is_dma_coherent(dev, attrs); + + vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, + coherent); + area = find_vm_area(cpu_addr); + if (!area) + return -EINVAL; + + pages = area->pages; + nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + for (i = vma->vm_pgoff; i < nr_pages && uaddr < vma->vm_end; i++) { + ret = vm_insert_page(vma, uaddr, pages[i]); + if (ret) + break; + uaddr += PAGE_SIZE; + } + + return ret; +} + +static int fast_smmu_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, + size_t size, unsigned long attrs) +{ + unsigned int n_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + struct vm_struct *area; + + area = find_vm_area(cpu_addr); + if (!area || !area->pages) + return -EINVAL; + + return sg_alloc_table_from_pages(sgt, area->pages, n_pages, 0, size, + GFP_KERNEL); +} + +static dma_addr_t fast_smmu_dma_map_resource( + struct device *dev, phys_addr_t phys_addr, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + size_t offset = phys_addr & ~FAST_PAGE_MASK; + size_t len = round_up(size + offset, FAST_PAGE_SIZE); + dma_addr_t dma_addr; + int prot; + unsigned long flags; + + spin_lock_irqsave(&mapping->lock, flags); + dma_addr = __fast_smmu_alloc_iova(mapping, attrs, len); + spin_unlock_irqrestore(&mapping->lock, flags); + + if (dma_addr == DMA_ERROR_CODE) + return dma_addr; + + prot = dma_info_to_prot(dir, false, attrs); + prot |= IOMMU_MMIO; + + if (iommu_map(mapping->domain, dma_addr, phys_addr - offset, + len, prot)) { + spin_lock_irqsave(&mapping->lock, flags); + __fast_smmu_free_iova(mapping, dma_addr, len); + spin_unlock_irqrestore(&mapping->lock, flags); + return DMA_ERROR_CODE; + } + return dma_addr + offset; +} + +static void fast_smmu_dma_unmap_resource( + struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + size_t offset = addr & ~FAST_PAGE_MASK; + size_t len = round_up(size + offset, FAST_PAGE_SIZE); + unsigned long flags; + + iommu_unmap(mapping->domain, addr - offset, len); + spin_lock_irqsave(&mapping->lock, flags); + __fast_smmu_free_iova(mapping, addr, len); + spin_unlock_irqrestore(&mapping->lock, flags); +} + +static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast, + void *data) +{ + av8l_fast_iopte *pmds, *ptep = data; + dma_addr_t iova; + unsigned long bitmap_idx; + struct io_pgtable *tbl; + + tbl = container_of(fast->pgtbl_ops, struct io_pgtable, ops); + pmds = tbl->cfg.av8l_fast_cfg.pmds; + + bitmap_idx = (unsigned long)(ptep - pmds); + iova = bitmap_idx << FAST_PAGE_SHIFT; + dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova); + dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx); + dev_err(fast->dev, "ptep: %pK pmds: %pK diff: %lu\n", ptep, + pmds, bitmap_idx); + print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS, + 32, 8, fast->bitmap, fast->bitmap_size, false); +} + +static int fast_smmu_notify(struct notifier_block *self, + unsigned long action, void *data) +{ + struct dma_fast_smmu_mapping *fast = container_of( + self, struct dma_fast_smmu_mapping, notifier); + + switch (action) { + case MAPPED_OVER_STALE_TLB: + __fast_smmu_mapped_over_stale(fast, data); + return NOTIFY_OK; + default: + WARN(1, "Unhandled notifier action"); + return NOTIFY_DONE; + } +} + +static const struct dma_map_ops fast_smmu_dma_ops = { + .alloc = fast_smmu_alloc, + .free = fast_smmu_free, + .mmap = fast_smmu_mmap_attrs, + .get_sgtable = fast_smmu_get_sgtable, + .map_page = fast_smmu_map_page, + .unmap_page = fast_smmu_unmap_page, + .sync_single_for_cpu = fast_smmu_sync_single_for_cpu, + .sync_single_for_device = fast_smmu_sync_single_for_device, + .map_sg = fast_smmu_map_sg, + .unmap_sg = fast_smmu_unmap_sg, + .sync_sg_for_cpu = fast_smmu_sync_sg_for_cpu, + .sync_sg_for_device = fast_smmu_sync_sg_for_device, + .map_resource = fast_smmu_dma_map_resource, + .unmap_resource = fast_smmu_dma_unmap_resource, +}; + +/** + * __fast_smmu_create_mapping_sized + * @base: bottom of the VA range + * @size: size of the VA range in bytes + * + * Creates a mapping structure which holds information about used/unused IO + * address ranges, which is required to perform mapping with IOMMU aware + * functions. The only VA range supported is [0, 4GB). + * + * The client device need to be attached to the mapping with + * fast_smmu_attach_device function. + */ +static struct dma_fast_smmu_mapping *__fast_smmu_create_mapping_sized( + dma_addr_t base, u64 size) +{ + struct dma_fast_smmu_mapping *fast; + + fast = kzalloc(sizeof(struct dma_fast_smmu_mapping), GFP_KERNEL); + if (!fast) + goto err; + + fast->base = base; + fast->size = size; + fast->num_4k_pages = size >> FAST_PAGE_SHIFT; + fast->bitmap_size = BITS_TO_LONGS(fast->num_4k_pages) * sizeof(long); + + fast->bitmap = kzalloc(fast->bitmap_size, GFP_KERNEL | __GFP_NOWARN | + __GFP_NORETRY); + if (!fast->bitmap) + fast->bitmap = vzalloc(fast->bitmap_size); + + if (!fast->bitmap) + goto err2; + + spin_lock_init(&fast->lock); + + fast->iovad = kzalloc(sizeof(*fast->iovad), GFP_KERNEL); + if (!fast->iovad) + goto err_free_bitmap; + init_iova_domain(fast->iovad, FAST_PAGE_SIZE, + base >> FAST_PAGE_SHIFT); + + return fast; + +err_free_bitmap: + kvfree(fast->bitmap); +err2: + kfree(fast); +err: + return ERR_PTR(-ENOMEM); +} + +/* + * Based off of similar code from dma-iommu.c, but modified to use a different + * iova allocator + */ +static void fast_smmu_reserve_pci_windows(struct device *dev, + struct dma_fast_smmu_mapping *mapping) +{ + struct pci_host_bridge *bridge; + struct resource_entry *window; + phys_addr_t start, end; + struct pci_dev *pci_dev; + unsigned long flags; + + if (!dev_is_pci(dev)) + return; + + pci_dev = to_pci_dev(dev); + bridge = pci_find_host_bridge(pci_dev->bus); + + spin_lock_irqsave(&mapping->lock, flags); + resource_list_for_each_entry(window, &bridge->windows) { + if (resource_type(window->res) != IORESOURCE_MEM && + resource_type(window->res) != IORESOURCE_IO) + continue; + + start = round_down(window->res->start - window->offset, + FAST_PAGE_SIZE); + end = round_up(window->res->end - window->offset, + FAST_PAGE_SIZE); + start = max_t(unsigned long, mapping->base, start); + end = min_t(unsigned long, mapping->base + mapping->size, end); + if (start >= end) + continue; + + dev_dbg(dev, "iova allocator reserved 0x%pa-0x%pa\n", + &start, &end); + + start = (start - mapping->base) >> FAST_PAGE_SHIFT; + end = (end - mapping->base) >> FAST_PAGE_SHIFT; + bitmap_set(mapping->bitmap, start, end - start); + } + spin_unlock_irqrestore(&mapping->lock, flags); +} + +void fast_smmu_put_dma_cookie(struct iommu_domain *domain) +{ + struct dma_fast_smmu_mapping *fast = domain->iova_cookie; + + if (!fast) + return; + + if (fast->iovad) { + put_iova_domain(fast->iovad); + kfree(fast->iovad); + } + + if (fast->bitmap) + kvfree(fast->bitmap); + + kfree(fast); + domain->iova_cookie = NULL; +} + +const struct dma_map_ops *fast_smmu_get_dma_ops(void) +{ + return &fast_smmu_dma_ops; +} + +/** + * fast_smmu_init_mapping + * @dev: valid struct device pointer + * @domain: valid IOMMU domain pointer + * @pgtable_ops: The page table ops associated with this domain + * + * Called the first time a device is attached to this mapping. + * Not for dma client use. + */ +int fast_smmu_init_mapping(struct device *dev, struct iommu_domain *domain, + struct io_pgtable_ops *pgtable_ops) +{ + u64 dma_base = domain->geometry.aperture_start; + u64 dma_end = domain->geometry.aperture_end; + u64 size = dma_end - dma_base + 1; + struct dma_fast_smmu_mapping *fast; + + if (domain->iova_cookie) { + fast = domain->iova_cookie; + goto finish; + } + + if (!pgtable_ops) + return -EINVAL; + + if (dma_base + size > (SZ_1G * 4ULL)) { + dev_err(dev, "Iova end address too large\n"); + return -EINVAL; + } + + fast = __fast_smmu_create_mapping_sized(dma_base, size); + if (IS_ERR(fast)) + return -ENOMEM; + + fast->domain = domain; + fast->dev = dev; + domain->iova_cookie = fast; + + fast->pgtbl_ops = pgtable_ops; + + fast->notifier.notifier_call = fast_smmu_notify; + av8l_register_notify(&fast->notifier); + +finish: + fast_smmu_reserve_pci_windows(dev, fast); + return 0; +} diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 4b6b2f3150a9..74d30523e971 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,7 @@ #include -#define ARM_LPAE_MAX_ADDR_BITS 52 +#define ARM_LPAE_MAX_ADDR_BITS 48 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 @@ -58,6 +59,9 @@ #define ARM_LPAE_PGD_IDX(l,d) \ ((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0) +#define ARM_LPAE_LVL_MASK(l, d) \ + ((l) == ARM_LPAE_START_LVL(d) ? (1 << (d)->pgd_bits) - 1 : \ + (1 << (d)->bits_per_level) - 1) #define ARM_LPAE_LVL_IDX(a,l,d) \ (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) @@ -75,8 +79,7 @@ #define ARM_LPAE_PTE_TYPE_TABLE 3 #define ARM_LPAE_PTE_TYPE_PAGE 3 -#define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12) - +#define ARM_LPAE_PTE_SH_MASK (((arm_lpae_iopte)0x3) << 8) #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) @@ -95,8 +98,11 @@ #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) /* Stage-1 PTE */ +#define ARM_LPAE_PTE_AP_PRIV_RW (((arm_lpae_iopte)0) << 6) #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) -#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_AP_PRIV_RO (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_AP_RO (((arm_lpae_iopte)3) << 6) +#define ARM_LPAE_PTE_ATTRINDX_MASK 0x7 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) @@ -150,42 +156,94 @@ #define ARM_LPAE_TCR_PS_42_BIT 0x3ULL #define ARM_LPAE_TCR_PS_44_BIT 0x4ULL #define ARM_LPAE_TCR_PS_48_BIT 0x5ULL -#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define ARM_LPAE_MAIR1_ATTR_SHIFT(n) ((n-4) << 3) #define ARM_LPAE_MAIR_ATTR_MASK 0xff #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 #define ARM_LPAE_MAIR_ATTR_NC 0x44 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff +#define ARM_LPAE_MAIR_ATTR_UPSTREAM 0xf4 +#define ARM_LPAE_MAIR_ATTR_LLC_NWA 0xe4 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 +#define ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM 3 +#define ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA 0x4ULL #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0) #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2) #define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4) /* IOPTE accessors */ -#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) +#define iopte_deref(pte, d) \ + (__va(iopte_val(pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \ + & ~(ARM_LPAE_GRANULE(d) - 1ULL))) #define iopte_type(pte,l) \ (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) +#define iopte_to_pfn(pte, d) \ + (((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift) + +#define pfn_to_iopte(pfn, d) \ + (((pfn) << (d)->pg_shift) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) + struct arm_lpae_io_pgtable { struct io_pgtable iop; int levels; + unsigned int pgd_bits; size_t pgd_size; unsigned long pg_shift; unsigned long bits_per_level; void *pgd; + void *pgd_ttbr1; }; typedef u64 arm_lpae_iopte; +/* + * We'll use some ignored bits in table entries to keep track of the number + * of page mappings beneath the table. The maximum number of entries + * beneath any table mapping in armv8 is 8192 (which is possible at the + * 2nd- and 3rd-level when using a 64K granule size). The bits at our + * disposal are: + * + * 4k granule: [54..52], [11..2] + * 64k granule: [54..52], [15..2] + * + * [54..52], [11..2] is enough bits for tracking table mappings at any + * level for any granule, so we'll use those. + */ +#define BOTTOM_IGNORED_MASK 0x3ff +#define BOTTOM_IGNORED_SHIFT 2 +#define BOTTOM_IGNORED_NUM_BITS 10 +#define TOP_IGNORED_MASK 0x7ULL +#define TOP_IGNORED_SHIFT 52 +#define IOPTE_RESERVED_MASK ((BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT) | \ + (TOP_IGNORED_MASK << TOP_IGNORED_SHIFT)) + +static arm_lpae_iopte iopte_val(arm_lpae_iopte table_pte) +{ + return table_pte & ~IOPTE_RESERVED_MASK; +} + +static arm_lpae_iopte _iopte_bottom_ignored_val(arm_lpae_iopte table_pte) +{ + return (table_pte & (BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT)) + >> BOTTOM_IGNORED_SHIFT; +} + +static arm_lpae_iopte _iopte_top_ignored_val(arm_lpae_iopte table_pte) +{ + return (table_pte & (TOP_IGNORED_MASK << TOP_IGNORED_SHIFT)) + >> TOP_IGNORED_SHIFT; +} + static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl, enum io_pgtable_fmt fmt) { @@ -195,25 +253,36 @@ static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl, return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK; } -static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, - struct arm_lpae_io_pgtable *data) +static int iopte_tblcnt(arm_lpae_iopte table_pte) { - arm_lpae_iopte pte = paddr; - - /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ - return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; + return (_iopte_bottom_ignored_val(table_pte) | + (_iopte_top_ignored_val(table_pte) << BOTTOM_IGNORED_NUM_BITS)); } -static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, - struct arm_lpae_io_pgtable *data) +static void iopte_tblcnt_set(arm_lpae_iopte *table_pte, int val) { - u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; + arm_lpae_iopte pte = iopte_val(*table_pte); - if (data->pg_shift < 16) - return paddr; + pte |= ((val & BOTTOM_IGNORED_MASK) << BOTTOM_IGNORED_SHIFT) | + (((val & (TOP_IGNORED_MASK << BOTTOM_IGNORED_NUM_BITS)) + >> BOTTOM_IGNORED_NUM_BITS) << TOP_IGNORED_SHIFT); + *table_pte = pte; +} - /* Rotate the packed high-order bits back to the top */ - return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); +static void iopte_tblcnt_sub(arm_lpae_iopte *table_ptep, int cnt) +{ + arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep); + + current_cnt -= cnt; + iopte_tblcnt_set(table_ptep, current_cnt); +} + +static void iopte_tblcnt_add(arm_lpae_iopte *table_ptep, int cnt) +{ + arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep); + + current_cnt += cnt; + iopte_tblcnt_set(table_ptep, current_cnt); } static bool selftest_running = false; @@ -223,22 +292,27 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages) return (dma_addr_t)virt_to_phys(pages); } +static inline void pgtable_dma_sync_single_for_device( + struct io_pgtable_cfg *cfg, + dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) + dma_sync_single_for_device(cfg->iommu_dev, addr, size, + dir); +} + static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, - struct io_pgtable_cfg *cfg) + struct io_pgtable_cfg *cfg, void *cookie) { struct device *dev = cfg->iommu_dev; - int order = get_order(size); - struct page *p; dma_addr_t dma; - void *pages; + void *pages = io_pgtable_alloc_pages_exact(cfg, cookie, size, + gfp | __GFP_ZERO); - VM_BUG_ON((gfp & __GFP_HIGHMEM)); - p = alloc_pages_node(dev ? dev_to_node(dev) : NUMA_NO_NODE, - gfp | __GFP_ZERO, order); - if (!p) + if (!pages) return NULL; - pages = page_address(p); if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) @@ -258,23 +332,23 @@ out_unmap: dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); out_free: - __free_pages(p, order); + io_pgtable_free_pages_exact(cfg, cookie, pages, size); return NULL; } static void __arm_lpae_free_pages(void *pages, size_t size, - struct io_pgtable_cfg *cfg) + struct io_pgtable_cfg *cfg, void *cookie) { if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), size, DMA_TO_DEVICE); - free_pages((unsigned long)pages, get_order(size)); + io_pgtable_free_pages_exact(cfg, cookie, pages, size); } static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg) { - dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), + pgtable_dma_sync_single_for_device(cfg, __arm_lpae_dma_addr(ptep), sizeof(*ptep), DMA_TO_DEVICE); } @@ -293,7 +367,8 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, phys_addr_t paddr, arm_lpae_iopte prot, - int lvl, arm_lpae_iopte *ptep) + int lvl, arm_lpae_iopte *ptep, + bool flush) { arm_lpae_iopte pte = prot; @@ -307,50 +382,48 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, if (data->iop.fmt != ARM_MALI_LPAE) pte |= ARM_LPAE_PTE_AF; - pte |= ARM_LPAE_PTE_SH_IS; - pte |= paddr_to_iopte(paddr, data); + pte |= ARM_LPAE_PTE_SH_OS; + pte |= pfn_to_iopte(paddr >> data->pg_shift, data); - __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); + if (flush) + __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); + else + *ptep = pte; } static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, - arm_lpae_iopte *ptep) + arm_lpae_iopte *ptep, arm_lpae_iopte *prev_ptep, + bool flush) { arm_lpae_iopte pte = *ptep; - if (iopte_leaf(pte, lvl, data->iop.fmt)) { - /* We require an unmap first */ - WARN_ON(!selftest_running); + /* We require an unmap first */ + if (pte & ARM_LPAE_PTE_VALID) { + WARN_RATELIMIT(1, "map without unmap\n"); return -EEXIST; - } else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) { - /* - * We need to unmap and free the old table before - * overwriting it with a block entry. - */ - arm_lpae_iopte *tblp; - size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); - - tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); - if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz)) - return -EINVAL; } - __arm_lpae_init_pte(data, paddr, prot, lvl, ptep); + __arm_lpae_init_pte(data, paddr, prot, lvl, ptep, flush); + + if (prev_ptep) + iopte_tblcnt_add(prev_ptep, 1); return 0; } static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, arm_lpae_iopte *ptep, arm_lpae_iopte curr, - struct io_pgtable_cfg *cfg) + struct io_pgtable_cfg *cfg, + int ref_count) { arm_lpae_iopte old, new; new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) new |= ARM_LPAE_PTE_NSTABLE; + iopte_tblcnt_set(&new, ref_count); /* * Ensure the table itself is visible before its PTE can be. @@ -373,21 +446,69 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, return old; } +struct map_state { + unsigned long iova_end; + unsigned int pgsize; + arm_lpae_iopte *pgtable; + arm_lpae_iopte *prev_pgtable; + arm_lpae_iopte *pte_start; + unsigned int num_pte; +}; +/* map state optimization works at level 3 (the 2nd-to-last level) */ +#define MAP_STATE_LVL 3 + static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, arm_lpae_iopte prot, - int lvl, arm_lpae_iopte *ptep) + int lvl, arm_lpae_iopte *ptep, + arm_lpae_iopte *prev_ptep, struct map_state *ms) { arm_lpae_iopte *cptep, pte; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); size_t tblsz = ARM_LPAE_GRANULE(data); struct io_pgtable_cfg *cfg = &data->iop.cfg; + void *cookie = data->iop.cookie; + arm_lpae_iopte *pgtable = ptep; /* Find our entry at the current level */ ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); /* If we can install a leaf entry at this level, then do so */ - if (size == block_size && (size & cfg->pgsize_bitmap)) - return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); + if (size == block_size && (size & cfg->pgsize_bitmap)) { + if (!ms) + return arm_lpae_init_pte(data, iova, paddr, prot, lvl, + ptep, prev_ptep, true); + + if (lvl == MAP_STATE_LVL) { + if (ms->pgtable) + pgtable_dma_sync_single_for_device(cfg, + __arm_lpae_dma_addr(ms->pte_start), + ms->num_pte * sizeof(*ptep), + DMA_TO_DEVICE); + + ms->iova_end = round_down(iova, SZ_2M) + SZ_2M; + ms->pgtable = pgtable; + ms->prev_pgtable = prev_ptep; + ms->pgsize = size; + ms->pte_start = ptep; + ms->num_pte = 1; + } else { + /* + * We have some map state from previous page + * mappings, but we're about to set up a block + * mapping. Flush out the previous page mappings. + */ + if (ms->pgtable) + pgtable_dma_sync_single_for_device(cfg, + __arm_lpae_dma_addr(ms->pte_start), + ms->num_pte * sizeof(*ptep), + DMA_TO_DEVICE); + memset(ms, 0, sizeof(*ms)); + ms = NULL; + } + + return arm_lpae_init_pte(data, iova, paddr, prot, lvl, + ptep, prev_ptep, ms == NULL); + } /* We can't allocate tables at the final level */ if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) @@ -396,13 +517,13 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = READ_ONCE(*ptep); if (!pte) { - cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); + cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg, cookie); if (!cptep) return -ENOMEM; - pte = arm_lpae_install_table(cptep, ptep, 0, cfg); + pte = arm_lpae_install_table(cptep, ptep, 0, cfg, 0); if (pte) - __arm_lpae_free_pages(cptep, tblsz, cfg); + __arm_lpae_free_pages(cptep, tblsz, cfg, cookie); } else if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) && !(pte & ARM_LPAE_PTE_SW_SYNC)) { __arm_lpae_sync_pte(ptep, cfg); @@ -417,7 +538,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, } /* Rinse, repeat */ - return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); + return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep, + ptep, ms); } static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, @@ -428,10 +550,13 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, if (data->iop.fmt == ARM_64_LPAE_S1 || data->iop.fmt == ARM_32_LPAE_S1) { pte = ARM_LPAE_PTE_nG; - if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) - pte |= ARM_LPAE_PTE_AP_RDONLY; - if (!(prot & IOMMU_PRIV)) - pte |= ARM_LPAE_PTE_AP_UNPRIV; + + if (prot & IOMMU_WRITE) + pte |= (prot & IOMMU_PRIV) ? ARM_LPAE_PTE_AP_PRIV_RW + : ARM_LPAE_PTE_AP_UNPRIV; + else + pte |= (prot & IOMMU_PRIV) ? ARM_LPAE_PTE_AP_PRIV_RO + : ARM_LPAE_PTE_AP_RO; } else { pte = ARM_LPAE_PTE_HAP_FAULT; if (prot & IOMMU_READ) @@ -459,6 +584,12 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, else if (prot & IOMMU_CACHE) pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_USE_UPSTREAM_HINT) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_USE_LLC_NWA) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA + << ARM_LPAE_PTE_ATTRINDX_SHIFT); } if (prot & IOMMU_NOEXEC) @@ -484,7 +615,8 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, return -ERANGE; prot = arm_lpae_prot_to_pte(data, iommu_prot); - ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); + ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, NULL, + NULL); /* * Synchronise all PTE updates for the new mapping before there's * a chance for anything to kick off a table walk for the new iova. @@ -494,11 +626,96 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, return ret; } +static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova, + struct scatterlist *sg, unsigned int nents, + int iommu_prot, size_t *size) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte *ptep = data->pgd; + int lvl = ARM_LPAE_START_LVL(data); + arm_lpae_iopte prot; + struct scatterlist *s; + size_t mapped = 0; + int i, ret; + unsigned int min_pagesz; + struct io_pgtable_cfg *cfg = &data->iop.cfg; + struct map_state ms; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + goto out_err; + + prot = arm_lpae_prot_to_pte(data, iommu_prot); + + min_pagesz = 1 << __ffs(cfg->pgsize_bitmap); + + memset(&ms, 0, sizeof(ms)); + + for_each_sg(sg, s, nents, i) { + phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; + size_t size = s->length; + + /* + * We are mapping on IOMMU page boundaries, so offset within + * the page must be 0. However, the IOMMU may support pages + * smaller than PAGE_SIZE, so s->offset may still represent + * an offset of that boundary within the CPU page. + */ + if (!IS_ALIGNED(s->offset, min_pagesz)) + goto out_err; + + while (size) { + size_t pgsize = iommu_pgsize( + cfg->pgsize_bitmap, iova | phys, size); + + if (ms.pgtable && (iova < ms.iova_end)) { + arm_lpae_iopte *ptep = ms.pgtable + + ARM_LPAE_LVL_IDX(iova, MAP_STATE_LVL, + data); + arm_lpae_init_pte( + data, iova, phys, prot, MAP_STATE_LVL, + ptep, ms.prev_pgtable, false); + ms.num_pte++; + } else { + ret = __arm_lpae_map(data, iova, phys, pgsize, + prot, lvl, ptep, NULL, &ms); + if (ret) + goto out_err; + } + + iova += pgsize; + mapped += pgsize; + phys += pgsize; + size -= pgsize; + } + } + + if (ms.pgtable) + pgtable_dma_sync_single_for_device(cfg, + __arm_lpae_dma_addr(ms.pte_start), + ms.num_pte * sizeof(*ms.pte_start), + DMA_TO_DEVICE); + + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + return mapped; + +out_err: + /* Return the size of the partial mapping so that they can be undone */ + *size = mapped; + return 0; +} + static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, arm_lpae_iopte *ptep) { arm_lpae_iopte *start, *end; unsigned long table_size; + void *cookie = data->iop.cookie; if (lvl == ARM_LPAE_START_LVL(data)) table_size = data->pgd_size; @@ -522,7 +739,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); } - __arm_lpae_free_pages(start, table_size, &data->iop.cfg); + __arm_lpae_free_pages(start, table_size, &data->iop.cfg, cookie); } static void arm_lpae_free_pgtable(struct io_pgtable *iop) @@ -530,6 +747,8 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop) struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd); + __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), + data->pgd_ttbr1); kfree(data); } @@ -544,18 +763,22 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, size_t tablesz = ARM_LPAE_GRANULE(data); size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); int i, unmap_idx = -1; + void *cookie = data->iop.cookie; + int child_cnt = 0; + + size = iommu_pgsize(data->iop.cfg.pgsize_bitmap, iova, size); if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) return 0; - tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); + tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg, cookie); if (!tablep) return 0; /* Bytes unmapped */ if (size == split_sz) unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); - blk_paddr = iopte_to_paddr(blk_pte, data); + blk_paddr = iopte_to_pfn(blk_pte, data) << data->pg_shift; pte = iopte_prot(blk_pte); for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { @@ -563,12 +786,14 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, if (i == unmap_idx) continue; - __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]); + __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i], + true); + child_cnt++; } - pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); + pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg, child_cnt); if (pte != blk_pte) { - __arm_lpae_free_pages(tablep, tablesz, cfg); + __arm_lpae_free_pages(tablep, tablesz, cfg, cookie); /* * We may race against someone unmapping another part of this * block, but anything else is invalid. We can't misinterpret @@ -609,9 +834,6 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, if (!iopte_leaf(pte, lvl, iop->fmt)) { /* Also flush any partial walks */ - io_pgtable_tlb_add_flush(iop, iova, size, - ARM_LPAE_GRANULE(data), false); - io_pgtable_tlb_sync(iop); ptep = iopte_deref(pte, data); __arm_lpae_free_pgtable(data, lvl + 1, ptep); } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) { @@ -621,11 +843,43 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, * has observed it before the TLBIALL can be issued. */ smp_wmb(); - } else { - io_pgtable_tlb_add_flush(iop, iova, size, size, true); } return size; + } else if ((lvl == ARM_LPAE_MAX_LEVELS - 2) && !iopte_leaf(pte, lvl, + iop->fmt)) { + arm_lpae_iopte *table = iopte_deref(pte, data); + arm_lpae_iopte *table_base = table; + int tl_offset = ARM_LPAE_LVL_IDX(iova, lvl + 1, data); + int entry_size = ARM_LPAE_GRANULE(data); + int max_entries = ARM_LPAE_BLOCK_SIZE(lvl, data) >> + data->pg_shift; + int entries = min_t(int, size / entry_size, + max_entries - tl_offset); + int table_len = entries * sizeof(*table); + + /* + * This isn't a block mapping so it must be a table mapping + * and since it's the 2nd-to-last level the next level has + * to be all page mappings. Zero them all out in one fell + * swoop. + */ + + table += tl_offset; + + memset(table, 0, table_len); + pgtable_dma_sync_single_for_device(&iop->cfg, + __arm_lpae_dma_addr(table), + table_len, DMA_TO_DEVICE); + + iopte_tblcnt_sub(ptep, entries); + if (!iopte_tblcnt(*ptep)) { + /* no valid mappings left under this table. free it. */ + __arm_lpae_set_pte(ptep, 0, &iop->cfg); + __arm_lpae_free_pgtable(data, lvl + 1, table_base); + } + + return entries * entry_size; } else if (iopte_leaf(pte, lvl, iop->fmt)) { /* * Insert a table at the next level to map the old region, @@ -641,8 +895,9 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, } static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, - size_t size) + size_t size) { + size_t unmapped = 0; struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); arm_lpae_iopte *ptep = data->pgd; int lvl = ARM_LPAE_START_LVL(data); @@ -650,49 +905,133 @@ static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) return 0; - return __arm_lpae_unmap(data, iova, size, lvl, ptep); + while (unmapped < size) { + size_t ret, size_to_unmap, remaining; + + remaining = (size - unmapped); + size_to_unmap = iommu_pgsize(data->iop.cfg.pgsize_bitmap, iova, + remaining); + size_to_unmap = size_to_unmap >= SZ_2M ? + size_to_unmap : + min_t(unsigned long, remaining, + (ALIGN(iova + 1, SZ_2M) - iova)); + ret = __arm_lpae_unmap(data, iova, size_to_unmap, lvl, ptep); + if (ret == 0) + break; + unmapped += ret; + iova += ret; + } + + if (unmapped) + io_pgtable_tlb_flush_all(&data->iop); + + return unmapped; +} + +static int arm_lpae_iova_to_pte(struct arm_lpae_io_pgtable *data, + unsigned long iova, int *plvl_ret, + arm_lpae_iopte *ptep_ret) +{ + arm_lpae_iopte pte, *ptep = data->pgd; + *plvl_ret = ARM_LPAE_START_LVL(data); + *ptep_ret = 0; + + do { + /* Valid IOPTE pointer? */ + if (!ptep) + return -EINVAL; + + /* Grab the IOPTE we're interested in */ + pte = *(ptep + ARM_LPAE_LVL_IDX(iova, *plvl_ret, data)); + + /* Valid entry? */ + if (!pte) + return -EINVAL; + + /* Leaf entry? */ + if (iopte_leaf(pte, *plvl_ret, data->iop.fmt)) + goto found_translation; + + /* Take it to the next level */ + ptep = iopte_deref(pte, data); + } while (++(*plvl_ret) < ARM_LPAE_MAX_LEVELS); + + /* Ran out of page tables to walk */ + return -EINVAL; + +found_translation: + *ptep_ret = pte; + return 0; +} + +static uint64_t arm_lpae_iova_get_pte(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte pte; + int lvl; + + if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte)) + return pte; + + return 0; } static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); - arm_lpae_iopte pte, *ptep = data->pgd; - int lvl = ARM_LPAE_START_LVL(data); + arm_lpae_iopte pte; + int lvl; + phys_addr_t phys = 0; - do { - /* Valid IOPTE pointer? */ - if (!ptep) - return 0; + if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte)) { + iova &= ((1 << ARM_LPAE_LVL_SHIFT(lvl, data)) - 1); + phys = ((phys_addr_t)iopte_to_pfn(pte, data) + << data->pg_shift) | iova; + } - /* Grab the IOPTE we're interested in */ - ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); - pte = READ_ONCE(*ptep); + return phys; +} - /* Valid entry? */ - if (!pte) - return 0; +static bool __arm_lpae_is_iova_coherent(struct arm_lpae_io_pgtable *data, + arm_lpae_iopte *ptep) +{ + if (data->iop.fmt == ARM_64_LPAE_S1 || + data->iop.fmt == ARM_32_LPAE_S1) { + int attr_idx = (*ptep & (ARM_LPAE_PTE_ATTRINDX_MASK << + ARM_LPAE_PTE_ATTRINDX_SHIFT)) >> + ARM_LPAE_PTE_ATTRINDX_SHIFT; + if ((attr_idx == ARM_LPAE_MAIR_ATTR_IDX_CACHE) && + (((*ptep & ARM_LPAE_PTE_SH_MASK) == ARM_LPAE_PTE_SH_IS) + || + (*ptep & ARM_LPAE_PTE_SH_MASK) == ARM_LPAE_PTE_SH_OS)) + return true; + } else { + if (*ptep & ARM_LPAE_PTE_MEMATTR_OIWB) + return true; + } - /* Leaf entry? */ - if (iopte_leaf(pte, lvl, data->iop.fmt)) - goto found_translation; + return false; +} - /* Take it to the next level */ - ptep = iopte_deref(pte, data); - } while (++lvl < ARM_LPAE_MAX_LEVELS); +static bool arm_lpae_is_iova_coherent(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte pte; + int lvl; + bool ret = false; - /* Ran out of page tables to walk */ - return 0; + if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte)) + ret = __arm_lpae_is_iova_coherent(data, &pte); -found_translation: - iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); - return iopte_to_paddr(pte, data) | iova; + return ret; } static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) { - unsigned long granule, page_sizes; - unsigned int max_addr_bits = 48; + unsigned long granule; /* * We need to restrict the supported page sizes to match the @@ -712,24 +1051,17 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) switch (granule) { case SZ_4K: - page_sizes = (SZ_4K | SZ_2M | SZ_1G); + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); break; case SZ_16K: - page_sizes = (SZ_16K | SZ_32M); + cfg->pgsize_bitmap &= (SZ_16K | SZ_32M); break; case SZ_64K: - max_addr_bits = 52; - page_sizes = (SZ_64K | SZ_512M); - if (cfg->oas > 48) - page_sizes |= 1ULL << 42; /* 4TB */ + cfg->pgsize_bitmap &= (SZ_64K | SZ_512M); break; default: - page_sizes = 0; + cfg->pgsize_bitmap = 0; } - - cfg->pgsize_bitmap &= page_sizes; - cfg->ias = min(cfg->ias, max_addr_bits); - cfg->oas = min(cfg->oas, max_addr_bits); } static struct arm_lpae_io_pgtable * @@ -737,6 +1069,7 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) { unsigned long va_bits, pgd_bits; struct arm_lpae_io_pgtable *data; + struct msm_io_pgtable_info *pgtbl_info = to_msm_io_pgtable_info(cfg); arm_lpae_restrict_pgsizes(cfg); @@ -766,6 +1099,7 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) /* Calculate the actual size of our pgd (without concatenation) */ pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1)); + data->pgd_bits = pgd_bits; data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); data->iop.ops = (struct io_pgtable_ops) { @@ -774,6 +1108,10 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) .iova_to_phys = arm_lpae_iova_to_phys, }; + pgtbl_info->map_sg = arm_lpae_map_sg; + pgtbl_info->is_iova_coherent = arm_lpae_is_iova_coherent; + pgtbl_info->iova_to_pte = arm_lpae_iova_get_pte; + return data; } @@ -783,8 +1121,11 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) u64 reg; struct arm_lpae_io_pgtable *data; - if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA | - IO_PGTABLE_QUIRK_NON_STRICT)) + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS + | IO_PGTABLE_QUIRK_NO_DMA + | IO_PGTABLE_QUIRK_NON_STRICT + | IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT + | IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -792,9 +1133,22 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) return NULL; /* TCR */ - reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + else if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT) + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + else if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA) + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WB << ARM_LPAE_TCR_ORGN0_SHIFT); + else + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT); switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: @@ -827,9 +1181,6 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) case 48: reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; - case 52: - reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_IPS_SHIFT); - break; default: goto out_free_data; } @@ -846,24 +1197,39 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) (ARM_LPAE_MAIR_ATTR_WBRWA << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | (ARM_LPAE_MAIR_ATTR_DEVICE - << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) | + (ARM_LPAE_MAIR_ATTR_UPSTREAM + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM)); cfg->arm_lpae_s1_cfg.mair[0] = reg; - cfg->arm_lpae_s1_cfg.mair[1] = 0; + + reg = ARM_LPAE_MAIR_ATTR_LLC_NWA + << ARM_LPAE_MAIR1_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA); + + cfg->arm_lpae_s1_cfg.mair[1] = reg; /* Looking good; allocate a pgd */ - data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, + cfg, cookie); if (!data->pgd) goto out_free_data; + data->pgd_ttbr1 = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, + cfg, cookie); + if (!data->pgd_ttbr1) + goto out_free_pgd; + /* Ensure the empty pgd is visible before any actual TTBR write */ wmb(); /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); - cfg->arm_lpae_s1_cfg.ttbr[1] = 0; + cfg->arm_lpae_s1_cfg.ttbr[1] = virt_to_phys(data->pgd_ttbr1); return &data->iop; +out_free_pgd: + __arm_lpae_free_pages(data->pgd, data->pgd_size, cfg, cookie); + out_free_data: kfree(data); return NULL; @@ -938,9 +1304,6 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) case 48: reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); break; - case 52: - reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_PS_SHIFT); - break; default: goto out_free_data; } @@ -950,7 +1313,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s2_cfg.vtcr = reg; /* Allocate pgd pages */ - data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, + cfg, cookie); if (!data->pgd) goto out_free_data; @@ -1064,7 +1428,6 @@ static void dummy_tlb_add_flush(unsigned long iova, size_t size, size_t granule, bool leaf, void *cookie) { WARN_ON(cookie != cfg_cookie); - WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); } static void dummy_tlb_sync(void *cookie) @@ -1097,6 +1460,121 @@ static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) -EFAULT; \ }) +/* + * Returns true if there's any mapping in the given iova range in ops. + */ +static bool arm_lpae_range_has_mapping(struct io_pgtable_ops *ops, + unsigned long iova_start, size_t size) +{ + unsigned long iova = iova_start; + + while (iova < (iova_start + size)) { + if (ops->iova_to_phys(ops, iova + 42)) + return true; + iova += SZ_4K; + } + return false; +} + +/* + * Returns true if the iova range is successfully mapped to the contiguous + * phys range in ops. + */ +static bool arm_lpae_range_has_specific_mapping(struct io_pgtable_ops *ops, + const unsigned long iova_start, + const phys_addr_t phys_start, + const size_t size) +{ + unsigned long iova = iova_start; + phys_addr_t phys = phys_start; + + while (iova < (iova_start + size)) { + if (ops->iova_to_phys(ops, iova + 42) != (phys + 42)) + return false; + iova += SZ_4K; + phys += SZ_4K; + } + return true; +} + +static int __init arm_lpae_run_map_sg_tests(struct io_pgtable_cfg *cfg, + struct io_pgtable_ops *ops, + enum io_pgtable_fmt fmt) +{ + size_t mapped; + size_t unused; + struct page *page; + phys_addr_t page_phys; + struct sg_table table; + struct scatterlist *sg; + int nents; + int ret = 0, j, k; + unsigned long total_size; + unsigned long iova; + unsigned long test_sg_sizes[] = { + SZ_4K, + SZ_64K, + SZ_2M, + SZ_1M * 12, + SZ_1M * 20 + }; + int chunk_size = 1UL << find_first_bit(&cfg->pgsize_bitmap, + BITS_PER_LONG); + + for (j = 0; j < ARRAY_SIZE(test_sg_sizes); ++j) { + + total_size = test_sg_sizes[j]; + nents = total_size / chunk_size; + + if (total_size < chunk_size) + continue; + + page = alloc_pages(GFP_KERNEL, get_order(chunk_size)); + if (!page) + return -ENOMEM; + page_phys = page_to_phys(page); + + iova = 0; + ret = sg_alloc_table(&table, nents, GFP_KERNEL); + if (ret) + return ret; + for_each_sg(table.sgl, sg, table.nents, k) + sg_set_page(sg, page, chunk_size, 0); + + mapped = ops->map_sg(ops, iova, table.sgl, table.nents, + IOMMU_READ | IOMMU_WRITE, &unused); + + if (mapped != total_size) + return __FAIL(ops, fmt); + + if (!arm_lpae_range_has_mapping(ops, iova, total_size)) + return __FAIL(ops, fmt); + + if (arm_lpae_range_has_mapping(ops, iova + total_size, + SZ_2G - (iova + total_size))) + return __FAIL(ops, fmt); + + for_each_sg(table.sgl, sg, table.nents, k) { + dma_addr_t newphys = + ops->iova_to_phys(ops, iova + 42); + if (newphys != (page_phys + 42)) + return __FAIL(ops, fmt); + iova += chunk_size; + } + + if (ops->unmap(ops, 0, total_size) != total_size) + return __FAIL(ops, fmt); + + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, fmt); + + sg_free_table(&table); + __free_pages(page, get_order(chunk_size)); + } + + return ret; +} + static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) { static const enum io_pgtable_fmt fmts[] = { @@ -1108,6 +1586,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) unsigned long iova; size_t size; struct io_pgtable_ops *ops; + int map_sg_ret; selftest_running = true; @@ -1120,16 +1599,11 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) } /* - * Initial sanity checks. - * Empty page tables shouldn't provide any translations. + * Initial sanity checks. Empty page tables shouldn't + * provide any translations. TODO: check entire supported + * range for these ops rather than first 2G */ - if (ops->iova_to_phys(ops, 42)) - return __FAIL(ops, i); - - if (ops->iova_to_phys(ops, SZ_1G + 42)) - return __FAIL(ops, i); - - if (ops->iova_to_phys(ops, SZ_2G + 42)) + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) return __FAIL(ops, i); /* @@ -1150,7 +1624,8 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) IOMMU_READ | IOMMU_NOEXEC)) return __FAIL(ops, i); - if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + if (!arm_lpae_range_has_specific_mapping(ops, iova, + iova, size)) return __FAIL(ops, i); iova += SZ_1G; @@ -1161,11 +1636,15 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) if (ops->unmap(ops, SZ_1G + size, size) != size) return __FAIL(ops, i); + if (arm_lpae_range_has_mapping(ops, SZ_1G + size, size)) + return __FAIL(ops, i); + /* Remap of partial unmap */ if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ)) return __FAIL(ops, i); - if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) + if (!arm_lpae_range_has_specific_mapping(ops, SZ_1G + size, + size, size)) return __FAIL(ops, i); /* Full unmap */ @@ -1186,9 +1665,50 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) return __FAIL(ops, i); + if (ops->unmap(ops, iova, size) != size) + return __FAIL(ops, i); + iova += SZ_1G; } + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, i); + + if ((cfg->pgsize_bitmap & SZ_2M) && + (cfg->pgsize_bitmap & SZ_4K)) { + /* mixed block + page mappings */ + iova = 0; + if (ops->map(ops, iova, iova, SZ_2M, IOMMU_READ)) + return __FAIL(ops, i); + + if (ops->map(ops, iova + SZ_2M, iova + SZ_2M, SZ_4K, + IOMMU_READ)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + SZ_2M + 42) != + (iova + SZ_2M + 42)) + return __FAIL(ops, i); + + /* unmap both mappings at once */ + if (ops->unmap(ops, iova, SZ_2M + SZ_4K) != + (SZ_2M + SZ_4K)) + return __FAIL(ops, i); + + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, i); + } + + /* map_sg */ + map_sg_ret = arm_lpae_run_map_sg_tests(cfg, ops, i); + if (map_sg_ret) + return map_sg_ret; + + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, i); + free_io_pgtable_ops(ops); } @@ -1200,8 +1720,6 @@ static int __init arm_lpae_do_selftests(void) { static const unsigned long pgsize[] = { SZ_4K | SZ_2M | SZ_1G, - SZ_16K | SZ_32M, - SZ_64K | SZ_512M, }; static const unsigned int ias[] = { diff --git a/drivers/iommu/io-pgtable-fast.c b/drivers/iommu/io-pgtable-fast.c new file mode 100644 index 000000000000..7585cf11a7c4 --- /dev/null +++ b/drivers/iommu/io-pgtable-fast.c @@ -0,0 +1,778 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + */ + +#define pr_fmt(fmt) "io-pgtable-fast: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define AV8L_FAST_MAX_ADDR_BITS 48 + +/* Struct accessors */ +#define iof_pgtable_to_data(x) \ + container_of((x), struct av8l_fast_io_pgtable, iop) + +#define iof_pgtable_ops_to_pgtable(x) \ + container_of((x), struct io_pgtable, ops) + +#define iof_pgtable_ops_to_data(x) \ + iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x)) + +struct av8l_fast_io_pgtable { + struct io_pgtable iop; + av8l_fast_iopte *pgd; + av8l_fast_iopte *puds[4]; + av8l_fast_iopte *pmds; + struct page **pages; /* page table memory */ +}; + +/* Page table bits */ +#define AV8L_FAST_PTE_TYPE_SHIFT 0 +#define AV8L_FAST_PTE_TYPE_MASK 0x3 + +#define AV8L_FAST_PTE_TYPE_BLOCK 1 +#define AV8L_FAST_PTE_TYPE_TABLE 3 +#define AV8L_FAST_PTE_TYPE_PAGE 3 + +#define AV8L_FAST_PTE_NSTABLE (((av8l_fast_iopte)1) << 63) +#define AV8L_FAST_PTE_XN (((av8l_fast_iopte)3) << 53) +#define AV8L_FAST_PTE_AF (((av8l_fast_iopte)1) << 10) +#define AV8L_FAST_PTE_SH_NS (((av8l_fast_iopte)0) << 8) +#define AV8L_FAST_PTE_SH_OS (((av8l_fast_iopte)2) << 8) +#define AV8L_FAST_PTE_SH_IS (((av8l_fast_iopte)3) << 8) +#define AV8L_FAST_PTE_SH_MASK (((av8l_fast_iopte)3) << 8) +#define AV8L_FAST_PTE_NS (((av8l_fast_iopte)1) << 5) +#define AV8L_FAST_PTE_VALID (((av8l_fast_iopte)1) << 0) + +#define AV8L_FAST_PTE_ATTR_LO_MASK (((av8l_fast_iopte)0x3ff) << 2) +/* Ignore the contiguous bit for block splitting */ +#define AV8L_FAST_PTE_ATTR_HI_MASK (((av8l_fast_iopte)6) << 52) +#define AV8L_FAST_PTE_ATTR_MASK (AV8L_FAST_PTE_ATTR_LO_MASK | \ + AV8L_FAST_PTE_ATTR_HI_MASK) +#define AV8L_FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000) + + +/* Stage-1 PTE */ +#define AV8L_FAST_PTE_AP_PRIV_RW (((av8l_fast_iopte)0) << 6) +#define AV8L_FAST_PTE_AP_RW (((av8l_fast_iopte)1) << 6) +#define AV8L_FAST_PTE_AP_PRIV_RO (((av8l_fast_iopte)2) << 6) +#define AV8L_FAST_PTE_AP_RO (((av8l_fast_iopte)3) << 6) +#define AV8L_FAST_PTE_ATTRINDX_SHIFT 2 +#define AV8L_FAST_PTE_ATTRINDX_MASK 0x7 +#define AV8L_FAST_PTE_nG (((av8l_fast_iopte)1) << 11) + +/* Stage-2 PTE */ +#define AV8L_FAST_PTE_HAP_FAULT (((av8l_fast_iopte)0) << 6) +#define AV8L_FAST_PTE_HAP_READ (((av8l_fast_iopte)1) << 6) +#define AV8L_FAST_PTE_HAP_WRITE (((av8l_fast_iopte)2) << 6) +#define AV8L_FAST_PTE_MEMATTR_OIWB (((av8l_fast_iopte)0xf) << 2) +#define AV8L_FAST_PTE_MEMATTR_NC (((av8l_fast_iopte)0x5) << 2) +#define AV8L_FAST_PTE_MEMATTR_DEV (((av8l_fast_iopte)0x1) << 2) + +/* Register bits */ +#define ARM_32_LPAE_TCR_EAE (1 << 31) +#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) + +#define AV8L_FAST_TCR_TG0_4K (0 << 14) +#define AV8L_FAST_TCR_TG0_64K (1 << 14) +#define AV8L_FAST_TCR_TG0_16K (2 << 14) + +#define AV8L_FAST_TCR_SH0_SHIFT 12 +#define AV8L_FAST_TCR_SH0_MASK 0x3 +#define AV8L_FAST_TCR_SH_NS 0 +#define AV8L_FAST_TCR_SH_OS 2 +#define AV8L_FAST_TCR_SH_IS 3 + +#define AV8L_FAST_TCR_ORGN0_SHIFT 10 +#define AV8L_FAST_TCR_IRGN0_SHIFT 8 +#define AV8L_FAST_TCR_RGN_MASK 0x3 +#define AV8L_FAST_TCR_RGN_NC 0 +#define AV8L_FAST_TCR_RGN_WBWA 1 +#define AV8L_FAST_TCR_RGN_WT 2 +#define AV8L_FAST_TCR_RGN_WB 3 + +#define AV8L_FAST_TCR_SL0_SHIFT 6 +#define AV8L_FAST_TCR_SL0_MASK 0x3 + +#define AV8L_FAST_TCR_T0SZ_SHIFT 0 +#define AV8L_FAST_TCR_SZ_MASK 0xf + +#define AV8L_FAST_TCR_PS_SHIFT 16 +#define AV8L_FAST_TCR_PS_MASK 0x7 + +#define AV8L_FAST_TCR_IPS_SHIFT 32 +#define AV8L_FAST_TCR_IPS_MASK 0x7 + +#define AV8L_FAST_TCR_PS_32_BIT 0x0ULL +#define AV8L_FAST_TCR_PS_36_BIT 0x1ULL +#define AV8L_FAST_TCR_PS_40_BIT 0x2ULL +#define AV8L_FAST_TCR_PS_42_BIT 0x3ULL +#define AV8L_FAST_TCR_PS_44_BIT 0x4ULL +#define AV8L_FAST_TCR_PS_48_BIT 0x5ULL + +#define AV8L_FAST_TCR_EPD1_SHIFT 23 +#define AV8L_FAST_TCR_EPD1_FAULT 1 + +#define AV8L_FAST_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define AV8L_FAST_MAIR_ATTR_MASK 0xff +#define AV8L_FAST_MAIR_ATTR_DEVICE 0x04 +#define AV8L_FAST_MAIR_ATTR_NC 0x44 +#define AV8L_FAST_MAIR_ATTR_WBRWA 0xff +#define AV8L_FAST_MAIR_ATTR_UPSTREAM 0xf4 +#define AV8L_FAST_MAIR_ATTR_IDX_NC 0 +#define AV8L_FAST_MAIR_ATTR_IDX_CACHE 1 +#define AV8L_FAST_MAIR_ATTR_IDX_DEV 2 +#define AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM 3 + +#define AV8L_FAST_PAGE_SHIFT 12 + +#define PTE_MAIR_IDX(pte) \ + ((pte >> AV8L_FAST_PTE_ATTRINDX_SHIFT) & \ + AV8L_FAST_PTE_ATTRINDX_MASK) + +#define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK) + +#define iopte_pmd_offset(pmds, iova) (pmds + (iova >> 12)) + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB + +#include +#include + +static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list); + +void av8l_register_notify(struct notifier_block *nb) +{ + atomic_notifier_chain_register(&av8l_notifier_list, nb); +} +EXPORT_SYMBOL(av8l_register_notify); + +static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep) +{ + if (unlikely(*ptep)) { + atomic_notifier_call_chain( + &av8l_notifier_list, MAPPED_OVER_STALE_TLB, + (void *) ptep); + pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n", + *ptep, ptep); + pr_err("Nearby memory:\n"); + print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS, + 32, 8, ptep - 16, 32 * sizeof(*ptep), false); + } +} + +void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, bool skip_sync) +{ + int i; + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + av8l_fast_iopte *pmdp = data->pmds; + + for (i = 0; i < ((SZ_1G * 4UL) >> AV8L_FAST_PAGE_SHIFT); ++i) { + if (!(*pmdp & AV8L_FAST_PTE_VALID)) { + *pmdp = 0; + if (!skip_sync) + dmac_clean_range(pmdp, pmdp + 1); + } + pmdp++; + } +} +#else +static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep) +{ +} +#endif + +static void av8l_clean_range(struct io_pgtable_ops *ops, + av8l_fast_iopte *start, av8l_fast_iopte *end) +{ + struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops); + + if (!(iop->cfg.quirks & IO_PGTABLE_QUIRK_NO_DMA)) + dmac_clean_range(start, end); +} + +static av8l_fast_iopte +av8l_fast_prot_to_pte(struct av8l_fast_io_pgtable *data, int prot) +{ + av8l_fast_iopte pte = AV8L_FAST_PTE_XN + | AV8L_FAST_PTE_TYPE_PAGE + | AV8L_FAST_PTE_AF + | AV8L_FAST_PTE_nG + | AV8L_FAST_PTE_SH_OS; + + if (prot & IOMMU_MMIO) + pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV + << AV8L_FAST_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_CACHE) + pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE + << AV8L_FAST_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_USE_UPSTREAM_HINT) + pte |= (AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM + << AV8L_FAST_PTE_ATTRINDX_SHIFT); + + if (!(prot & IOMMU_WRITE)) + pte |= AV8L_FAST_PTE_AP_RO; + else + pte |= AV8L_FAST_PTE_AP_RW; + + return pte; +} + +static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova); + unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT; + av8l_fast_iopte pte; + + pte = av8l_fast_prot_to_pte(data, prot); + paddr &= AV8L_FAST_PTE_ADDR_MASK; + for (i = 0; i < nptes; i++, paddr += SZ_4K) { + __av8l_check_for_stale_tlb(ptep + i); + *(ptep + i) = pte | paddr; + } + av8l_clean_range(ops, ptep, ptep + nptes); + + return 0; +} + +int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + return av8l_fast_map(ops, iova, paddr, size, prot); +} + +static size_t +__av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size, bool allow_stale_tlb) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + unsigned long nptes; + av8l_fast_iopte *ptep; + int val = allow_stale_tlb + ? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI + : 0; + + ptep = iopte_pmd_offset(data->pmds, iova); + nptes = size >> AV8L_FAST_PAGE_SHIFT; + + memset(ptep, val, sizeof(*ptep) * nptes); + av8l_clean_range(ops, ptep, ptep + nptes); + if (!allow_stale_tlb) + io_pgtable_tlb_flush_all(&data->iop); + + return size; +} + +/* caller must take care of tlb cache maintenance */ +void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) +{ + __av8l_fast_unmap(ops, iova, size, true); +} + +static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) +{ + return __av8l_fast_unmap(ops, iova, size, false); +} + +static int av8l_fast_map_sg(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, size_t *size) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nents, i) { + av8l_fast_map(ops, iova, sg_phys(sg), sg->length, prot); + iova += sg->length; + } + + return nents; +} + +int av8l_fast_map_sg_public(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, size_t *size) +{ + return av8l_fast_map_sg(ops, iova, sgl, nents, prot, size); +} + +#if defined(CONFIG_ARM64) +#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27) +#elif defined(CONFIG_ARM) +#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27) +#endif + +static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + av8l_fast_iopte pte, *pgdp, *pudp, *pmdp; + unsigned long pgd; + phys_addr_t phys; + const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT; + const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK; + const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE; + const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE; + const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK; + + /* TODO: clean up some of these magic numbers... */ + + pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova); + pgdp = (av8l_fast_iopte *)pgd; + + pte = *pgdp; + if (((pte >> pts) & ptm) != ptt) + return 0; + pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18)); + + pte = *pudp; + if (((pte >> pts) & ptm) != ptt) + return 0; + pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9)); + + pte = *pmdp; + if (((pte >> pts) & ptm) != ptp) + return 0; + phys = pte & am; + + return phys | (iova & 0xfff); +} + +phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return av8l_fast_iova_to_phys(ops, iova); +} + +static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova); + + return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) && + ((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) || + (PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_IS))); +} + +bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return av8l_fast_iova_coherent(ops, iova); +} + +static struct av8l_fast_io_pgtable * +av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg) +{ + struct av8l_fast_io_pgtable *data; + struct msm_io_pgtable_info *pgtbl_info = to_msm_io_pgtable_info(cfg); + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + data->iop.ops = (struct io_pgtable_ops) { + .map = av8l_fast_map, + .unmap = av8l_fast_unmap, + .iova_to_phys = av8l_fast_iova_to_phys, + }; + + pgtbl_info->map_sg = av8l_fast_map_sg; + pgtbl_info->is_iova_coherent = av8l_fast_iova_coherent; + return data; +} + +/* + * We need 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and + * 2048 pages for pmds (each pud page contains 512 table entries, each + * pointing to a pmd). + */ +#define NUM_PGD_PAGES 1 +#define NUM_PUD_PAGES 4 +#define NUM_PMD_PAGES 2048 +#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES) + +static int +av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data, + struct io_pgtable_cfg *cfg, void *cookie) +{ + int i, j, pg = 0; + struct page **pages, *page; + + pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN | + __GFP_NORETRY); + + if (!pages) + pages = vmalloc(sizeof(*pages) * NUM_PGTBL_PAGES); + + if (!pages) + return -ENOMEM; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + goto err_free_pages_arr; + pages[pg++] = page; + data->pgd = page_address(page); + + /* + * We need 2048 entries at level 2 to map 4GB of VA space. A page + * can hold 512 entries, so we need 4 pages. + */ + for (i = 0; i < 4; ++i) { + av8l_fast_iopte pte, *ptep; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + goto err_free_pages; + pages[pg++] = page; + data->puds[i] = page_address(page); + pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE; + ptep = ((av8l_fast_iopte *)data->pgd) + i; + *ptep = pte; + } + dmac_clean_range(data->pgd, data->pgd + 4); + + /* + * We have 4 puds, each of which can point to 512 pmds, so we'll + * have 2048 pmds, each of which can hold 512 ptes, for a grand + * total of 2048*512=1048576 PTEs. + */ + for (i = 0; i < 4; ++i) { + for (j = 0; j < 512; ++j) { + av8l_fast_iopte pte, *pudp; + void *addr; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + goto err_free_pages; + pages[pg++] = page; + + addr = page_address(page); + dmac_clean_range(addr, addr + SZ_4K); + + pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE; + pudp = data->puds[i] + j; + *pudp = pte; + } + dmac_clean_range(data->puds[i], data->puds[i] + 512); + } + + if (WARN_ON(pg != NUM_PGTBL_PAGES)) + goto err_free_pages; + + /* + * We map the pmds into a virtually contiguous space so that we + * don't have to traverse the first two levels of the page tables + * to find the appropriate pud. Instead, it will be a simple + * offset from the virtual base of the pmds. + */ + data->pmds = vmap(&pages[NUM_PGD_PAGES + NUM_PUD_PAGES], NUM_PMD_PAGES, + VM_IOREMAP, PAGE_KERNEL); + if (!data->pmds) + goto err_free_pages; + + data->pages = pages; + return 0; + +err_free_pages: + for (i = 0; i < pg; ++i) + __free_page(pages[i]); +err_free_pages_arr: + kvfree(pages); + return -ENOMEM; +} + +static struct io_pgtable * +av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + u64 reg; + struct av8l_fast_io_pgtable *data = + av8l_fast_alloc_pgtable_data(cfg); + + if (!data) + return NULL; + + /* restrict according to the fast map requirements */ + cfg->ias = 32; + cfg->pgsize_bitmap = SZ_4K; + + /* TCR */ + if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT) + reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) | + (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) | + (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT); + else if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) + reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) | + (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_IRGN0_SHIFT) | + (AV8L_FAST_TCR_RGN_WBWA << AV8L_FAST_TCR_ORGN0_SHIFT); + else + reg = (AV8L_FAST_TCR_SH_OS << AV8L_FAST_TCR_SH0_SHIFT) | + (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) | + (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_ORGN0_SHIFT); + + reg |= AV8L_FAST_TCR_TG0_4K; + + switch (cfg->oas) { + case 32: + reg |= (AV8L_FAST_TCR_PS_32_BIT << AV8L_FAST_TCR_IPS_SHIFT); + break; + case 36: + reg |= (AV8L_FAST_TCR_PS_36_BIT << AV8L_FAST_TCR_IPS_SHIFT); + break; + case 40: + reg |= (AV8L_FAST_TCR_PS_40_BIT << AV8L_FAST_TCR_IPS_SHIFT); + break; + case 42: + reg |= (AV8L_FAST_TCR_PS_42_BIT << AV8L_FAST_TCR_IPS_SHIFT); + break; + case 44: + reg |= (AV8L_FAST_TCR_PS_44_BIT << AV8L_FAST_TCR_IPS_SHIFT); + break; + case 48: + reg |= (AV8L_FAST_TCR_PS_48_BIT << AV8L_FAST_TCR_IPS_SHIFT); + break; + default: + goto out_free_data; + } + + reg |= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT; + reg |= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT; +#if defined(CONFIG_ARM) + reg |= ARM_32_LPAE_TCR_EAE; +#endif + cfg->av8l_fast_cfg.tcr = reg; + + /* MAIRs */ + reg = (AV8L_FAST_MAIR_ATTR_NC + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) | + (AV8L_FAST_MAIR_ATTR_WBRWA + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) | + (AV8L_FAST_MAIR_ATTR_DEVICE + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV)) | + (AV8L_FAST_MAIR_ATTR_UPSTREAM + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM)); + + cfg->av8l_fast_cfg.mair[0] = reg; + cfg->av8l_fast_cfg.mair[1] = 0; + + /* Allocate all page table memory! */ + if (av8l_fast_prepopulate_pgtables(data, cfg, cookie)) + goto out_free_data; + + cfg->av8l_fast_cfg.pmds = data->pmds; + + /* TTBRs */ + cfg->av8l_fast_cfg.ttbr[0] = virt_to_phys(data->pgd); + cfg->av8l_fast_cfg.ttbr[1] = 0; + return &data->iop; + +out_free_data: + kfree(data); + return NULL; +} + +static void av8l_fast_free_pgtable(struct io_pgtable *iop) +{ + int i; + struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop); + + vunmap(data->pmds); + for (i = 0; i < NUM_PGTBL_PAGES; ++i) + __free_page(data->pages[i]); + kvfree(data->pages); + kfree(data); +} + +struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = { + .alloc = av8l_fast_alloc_pgtable, + .free = av8l_fast_free_pgtable, +}; + + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST + +#include + +static struct io_pgtable_cfg *cfg_cookie; + +static void dummy_tlb_flush_all(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void dummy_tlb_add_flush(unsigned long iova, size_t size, size_t granule, + bool leaf, void *cookie) +{ + WARN_ON(cookie != cfg_cookie); + WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); +} + +static void dummy_tlb_sync(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static struct iommu_gather_ops dummy_tlb_ops __initdata = { + .tlb_flush_all = dummy_tlb_flush_all, + .tlb_add_flush = dummy_tlb_add_flush, + .tlb_sync = dummy_tlb_sync, +}; + +/* + * Returns true if the iova range is successfully mapped to the contiguous + * phys range in ops. + */ +static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops, + const unsigned long iova_start, + const phys_addr_t phys_start, + const size_t size) +{ + u64 iova = iova_start; + phys_addr_t phys = phys_start; + + while (iova < (iova_start + size)) { + /* + 42 just to make sure offsetting is working */ + if (ops->iova_to_phys(ops, iova + 42) != (phys + 42)) + return false; + iova += SZ_4K; + phys += SZ_4K; + } + return true; +} + +static int __init av8l_fast_positive_testing(void) +{ + int failed = 0; + u64 iova; + struct io_pgtable_ops *ops; + struct io_pgtable_cfg cfg; + struct av8l_fast_io_pgtable *data; + av8l_fast_iopte *pmds; + u64 max = SZ_1G * 4ULL - 1; + + cfg = (struct io_pgtable_cfg) { + .quirks = 0, + .tlb = &dummy_tlb_ops, + .ias = 32, + .oas = 32, + .pgsize_bitmap = SZ_4K, + }; + + cfg_cookie = &cfg; + ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &cfg, &cfg); + + if (WARN_ON(!ops)) + return 1; + + data = iof_pgtable_ops_to_data(ops); + pmds = data->pmds; + + /* map the entire 4GB VA space with 4K map calls */ + for (iova = 0; iova < max; iova += SZ_4K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) { + failed++; + continue; + } + } + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0, + max))) + failed++; + + /* unmap it all */ + for (iova = 0; iova < max; iova += SZ_4K) { + if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K)) + failed++; + } + + /* sweep up TLB proving PTEs */ + av8l_fast_clear_stale_ptes(ops, false); + + /* map the entire 4GB VA space with 8K map calls */ + for (iova = 0; iova < max; iova += SZ_8K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) { + failed++; + continue; + } + } + + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0, + max))) + failed++; + + /* unmap it all with 8K unmap calls */ + for (iova = 0; iova < max; iova += SZ_8K) { + if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K)) + failed++; + } + + /* sweep up TLB proving PTEs */ + av8l_fast_clear_stale_ptes(ops, false); + + /* map the entire 4GB VA space with 16K map calls */ + for (iova = 0; iova < max; iova += SZ_16K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) { + failed++; + continue; + } + } + + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0, + max))) + failed++; + + /* unmap it all */ + for (iova = 0; iova < max; iova += SZ_16K) { + if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K)) + failed++; + } + + /* sweep up TLB proving PTEs */ + av8l_fast_clear_stale_ptes(ops, false); + + /* map the entire 4GB VA space with 64K map calls */ + for (iova = 0; iova < max; iova += SZ_64K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) { + failed++; + continue; + } + } + + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0, + max))) + failed++; + + /* unmap it all at once */ + if (WARN_ON(ops->unmap(ops, 0, max) != max)) + failed++; + + free_io_pgtable_ops(ops); + return failed; +} + +static int __init av8l_fast_do_selftests(void) +{ + int failed = 0; + + failed += av8l_fast_positive_testing(); + + pr_err("selftest: completed with %d failures\n", failed); + + return 0; +} +subsys_initcall(av8l_fast_do_selftests); +#endif diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index ced53e5b72b5..72d32526ad40 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -11,6 +11,10 @@ #include #include #include +#include +#include +#include +#include static const struct io_pgtable_init_fns * io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { @@ -24,8 +28,13 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { #ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S [ARM_V7S] = &io_pgtable_arm_v7s_init_fns, #endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + [ARM_V8L_FAST] = &io_pgtable_av8l_fast_init_fns, +#endif }; +static struct dentry *io_pgtable_top; + struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, struct io_pgtable_cfg *cfg, void *cookie) @@ -68,3 +77,59 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) io_pgtable_init_table[iop->fmt]->free(iop); } EXPORT_SYMBOL_GPL(free_io_pgtable_ops); + +static atomic_t pages_allocated; + +void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + size_t size, gfp_t gfp_mask) +{ + void *ret; + struct msm_iommu_gather_ops *ops = to_msm_iommu_gather_ops(cfg->tlb); + + if (ops->alloc_pages_exact) + ret = ops->alloc_pages_exact(cookie, size, gfp_mask); + else + ret = alloc_pages_exact(size, gfp_mask); + + if (likely(ret)) + atomic_add(1 << get_order(size), &pages_allocated); + + return ret; +} + +void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + void *virt, size_t size) +{ + struct msm_iommu_gather_ops *ops = to_msm_iommu_gather_ops(cfg->tlb); + + if (ops->free_pages_exact) + ops->free_pages_exact(cookie, virt, size); + else + free_pages_exact(virt, size); + + atomic_sub(1 << get_order(size), &pages_allocated); +} + +static int io_pgtable_init(void) +{ + io_pgtable_top = debugfs_create_dir("io-pgtable", iommu_debugfs_top); + + if (!io_pgtable_top) + return -ENODEV; + + if (!debugfs_create_atomic_t("pages", 0600, + io_pgtable_top, &pages_allocated)) { + debugfs_remove_recursive(io_pgtable_top); + return -ENODEV; + } + + return 0; +} + +static void io_pgtable_exit(void) +{ + debugfs_remove_recursive(io_pgtable_top); +} + +module_init(io_pgtable_init); +module_exit(io_pgtable_exit); diff --git a/drivers/iommu/iommu-debug.c b/drivers/iommu/iommu-debug.c new file mode 100644 index 000000000000..3714634a8d61 --- /dev/null +++ b/drivers/iommu/iommu-debug.c @@ -0,0 +1,2378 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#define pr_fmt(fmt) "iommu-debug: %s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iommu-debug.h" + +#if defined(CONFIG_IOMMU_TESTS) + +static const char *iommu_debug_attr_to_string(enum iommu_attr attr) +{ + unsigned long iommu_attr = (unsigned long)attr; + + switch (iommu_attr) { + case DOMAIN_ATTR_GEOMETRY: + return "DOMAIN_ATTR_GEOMETRY"; + case DOMAIN_ATTR_PAGING: + return "DOMAIN_ATTR_PAGING"; + case DOMAIN_ATTR_WINDOWS: + return "DOMAIN_ATTR_WINDOWS"; + case DOMAIN_ATTR_FSL_PAMU_STASH: + return "DOMAIN_ATTR_FSL_PAMU_STASH"; + case DOMAIN_ATTR_FSL_PAMU_ENABLE: + return "DOMAIN_ATTR_FSL_PAMU_ENABLE"; + case DOMAIN_ATTR_FSL_PAMUV1: + return "DOMAIN_ATTR_FSL_PAMUV1"; + case DOMAIN_ATTR_NESTING: + return "DOMAIN_ATTR_NESTING"; + case DOMAIN_ATTR_PT_BASE_ADDR: + return "DOMAIN_ATTR_PT_BASE_ADDR"; + case DOMAIN_ATTR_SECURE_VMID: + return "DOMAIN_ATTR_SECURE_VMID"; + case DOMAIN_ATTR_ATOMIC: + return "DOMAIN_ATTR_ATOMIC"; + case DOMAIN_ATTR_CONTEXT_BANK: + return "DOMAIN_ATTR_CONTEXT_BANK"; + case DOMAIN_ATTR_TTBR0: + return "DOMAIN_ATTR_TTBR0"; + case DOMAIN_ATTR_CONTEXTIDR: + return "DOMAIN_ATTR_CONTEXTIDR"; + case DOMAIN_ATTR_PROCID: + return "DOMAIN_ATTR_PROCID"; + case DOMAIN_ATTR_DYNAMIC: + return "DOMAIN_ATTR_DYNAMIC"; + case DOMAIN_ATTR_NON_FATAL_FAULTS: + return "DOMAIN_ATTR_NON_FATAL_FAULTS"; + case DOMAIN_ATTR_S1_BYPASS: + return "DOMAIN_ATTR_S1_BYPASS"; + case DOMAIN_ATTR_FAST: + return "DOMAIN_ATTR_FAST"; + case DOMAIN_ATTR_EARLY_MAP: + return "DOMAIN_ATTR_EARLY_MAP"; + case DOMAIN_ATTR_CB_STALL_DISABLE: + return "DOMAIN_ATTR_CB_STALL_DISABLE"; + default: + return "Unknown attr!"; + } +} +#endif + +#ifdef CONFIG_IOMMU_DEBUG_TRACKING + +static DEFINE_MUTEX(iommu_debug_attachments_lock); +static LIST_HEAD(iommu_debug_attachments); + +/* + * Each group may have more than one domain; but each domain may + * only have one group. + * Used by debug tools to display the name of the device(s) associated + * with a particular domain. + */ +struct iommu_debug_attachment { + struct iommu_domain *domain; + struct iommu_group *group; + struct list_head list; +}; + +void iommu_debug_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct iommu_debug_attachment *attach; + struct iommu_group *group; + + group = dev->iommu_group; + if (!group) + return; + + mutex_lock(&iommu_debug_attachments_lock); + list_for_each_entry(attach, &iommu_debug_attachments, list) + if ((attach->domain == domain) && (attach->group == group)) + goto out; + + attach = kzalloc(sizeof(*attach), GFP_KERNEL); + if (!attach) + goto out; + + attach->domain = domain; + attach->group = group; + INIT_LIST_HEAD(&attach->list); + + list_add(&attach->list, &iommu_debug_attachments); +out: + mutex_unlock(&iommu_debug_attachments_lock); +} + +void iommu_debug_domain_remove(struct iommu_domain *domain) +{ + struct iommu_debug_attachment *it, *tmp; + + mutex_lock(&iommu_debug_attachments_lock); + list_for_each_entry_safe(it, tmp, &iommu_debug_attachments, list) { + if (it->domain != domain) + continue; + list_del(&it->list); + kfree(it); + } + + mutex_unlock(&iommu_debug_attachments_lock); +} + +#endif + +#ifdef CONFIG_IOMMU_TESTS + +#ifdef CONFIG_64BIT + +#define kstrtoux kstrtou64 +#define kstrtox_from_user kstrtoull_from_user +#define kstrtosize_t kstrtoul + +#else + +#define kstrtoux kstrtou32 +#define kstrtox_from_user kstrtouint_from_user +#define kstrtosize_t kstrtouint + +#endif + +static LIST_HEAD(iommu_debug_devices); +static struct dentry *debugfs_tests_dir; +static u32 iters_per_op = 1; +static void *test_virt_addr; + +struct iommu_debug_device { + struct device *dev; + struct iommu_domain *domain; + struct dma_iommu_mapping *mapping; + u64 iova; + u64 phys; + size_t len; + struct list_head list; + struct mutex clk_lock; + unsigned int clk_count; + /* Protects domain */ + struct mutex state_lock; +}; + +static int iommu_debug_build_phoney_sg_table(struct device *dev, + struct sg_table *table, + unsigned long total_size, + unsigned long chunk_size) +{ + unsigned long nents = total_size / chunk_size; + struct scatterlist *sg; + int i, j; + struct page *page; + + if (!IS_ALIGNED(total_size, PAGE_SIZE)) + return -EINVAL; + if (!IS_ALIGNED(total_size, chunk_size)) + return -EINVAL; + if (sg_alloc_table(table, nents, GFP_KERNEL)) + return -EINVAL; + + for_each_sg(table->sgl, sg, table->nents, i) { + page = alloc_pages(GFP_KERNEL, get_order(chunk_size)); + if (!page) + goto free_pages; + sg_set_page(sg, page, chunk_size, 0); + } + + return 0; +free_pages: + for_each_sg(table->sgl, sg, i--, j) + __free_pages(sg_page(sg), get_order(chunk_size)); + sg_free_table(table); + return -ENOMEM; +} + +static void iommu_debug_destroy_phoney_sg_table(struct device *dev, + struct sg_table *table, + unsigned long chunk_size) +{ + struct scatterlist *sg; + int i; + + for_each_sg(table->sgl, sg, table->nents, i) + __free_pages(sg_page(sg), get_order(chunk_size)); + sg_free_table(table); +} + +struct iommu_debug_attr { + unsigned long dma_type; + int vmid; +}; + +static struct iommu_debug_attr std_attr = { + .dma_type = 0, + .vmid = 0, +}; + +static struct iommu_debug_attr fastmap_attr = { + .dma_type = DOMAIN_ATTR_FAST, + .vmid = 0, +}; + +static struct iommu_debug_attr secure_attr = { + .dma_type = 0, + .vmid = VMID_CP_PIXEL, +}; + +static int iommu_debug_set_attrs(struct iommu_debug_device *ddev, + struct iommu_domain *domain, + struct iommu_debug_attr *attrs) +{ + int val = 1; + + /* Always set this to avoid clk latency during measurements */ + iommu_domain_set_attr(domain, DOMAIN_ATTR_ATOMIC, &val); + + if (attrs->dma_type == DOMAIN_ATTR_FAST) + iommu_domain_set_attr(domain, DOMAIN_ATTR_FAST, &val); + + if (attrs->vmid != 0) + iommu_domain_set_attr(domain, + DOMAIN_ATTR_SECURE_VMID, &attrs->vmid); + + iommu_domain_set_attr(domain, DOMAIN_ATTR_DEBUG, &val); + + return 0; +} + +static void iommu_debug_print_attrs(struct seq_file *s, + struct iommu_debug_attr *attrs) +{ + seq_puts(s, "Attributes:\n"); + if (attrs->dma_type == DOMAIN_ATTR_FAST) + seq_printf(s, "%s\n", + iommu_debug_attr_to_string(attrs->dma_type)); + + if (attrs->vmid != 0) + seq_printf(s, "SECURE_VMID=%d\n", attrs->vmid); +} + +/* + * Set up a new dma allocator for dev + * Caller should hold state_lock + */ +static int iommu_debug_dma_reconfigure(struct iommu_debug_device *ddev, + struct iommu_debug_attr *attrs, + u64 dma_base, u64 size) +{ + + const struct iommu_ops *iommu; + struct iommu_domain *domain; + struct device *dev = ddev->dev; + int is_fast; + bool coherent; + struct iommu_pgtbl_info info; + + if (ddev->domain) { + dev_err_ratelimited(dev, "Already attached.\n"); + return -EBUSY; + } + + iommu = of_iommu_configure(dev, dev->of_node); + if (!iommu) { + dev_err_ratelimited(dev, "Is not associated with an iommu\n"); + return -EINVAL; + } + + coherent = of_dma_is_coherent(dev->of_node); + + if (!dev->iommu_group) { + dev_err_ratelimited(dev, "Does not have an iommu group\n"); + return -EINVAL; + } + + /* Detach from the default domain */ + domain = iommu_get_domain_for_dev(dev); + if (domain) { + if (domain->type != IOMMU_DOMAIN_DMA) { + dev_err_ratelimited(dev, "Attached, but its not a default domain?\n"); + return -EINVAL; + } + iommu_detach_group(domain, dev->iommu_group); + } + + domain = iommu_domain_alloc(dev->bus); + if (!domain) { + dev_err_ratelimited(dev, "Allocating iommu domain failed\n"); + return -EINVAL; + } + + if (iommu_debug_set_attrs(ddev, domain, attrs)) { + dev_err_ratelimited(dev, "Setting attrs failed\n"); + goto out_free_domain; + } + + if (iommu_attach_group(domain, dev->iommu_group)) { + dev_err_ratelimited(dev, "attach group failed\n"); + goto out_free_domain; + } + + iommu_domain_get_attr(domain, DOMAIN_ATTR_FAST, &is_fast); + + if (is_fast) { + iommu_domain_get_attr(domain, DOMAIN_ATTR_PGTBL_INFO, &info); + /* + * Fix up the domain geometry, as the fastmap implementation + * uses it for determining the IOVA space parameters. Allow + * it to use the entire fastmap IOVA space [0, 4GB). + */ + domain->geometry.aperture_start = 0; + domain->geometry.aperture_end = SZ_4G - 1; + if (fast_smmu_init_mapping(dev, domain, info.ops)) { + dev_err_ratelimited(dev, "fastmap init failed\n"); + goto out_detach_group; + } + } else { + if (iommu_get_dma_cookie(domain)) { + dev_err_ratelimited(dev, "iommu get dma cookie failed\n"); + goto out_detach_group; + } + } + + /* + * Since arch_setup_dma_ops is void, interpret non-null dma-ops + * as success. + */ + set_dma_ops(dev, NULL); + arch_setup_dma_ops(dev, dma_base, size, iommu, coherent); + if (!get_dma_ops(dev)) { + dev_err_ratelimited(dev, "arch_setup_dma_ops failed, dma ops are null.\n"); + goto out_detach_group; + } + + ddev->domain = domain; + return 0; + +out_detach_group: + iommu_detach_group(domain, dev->iommu_group); +out_free_domain: + iommu_domain_free(domain); + return -EINVAL; +} + +/* Caller should hold state_lock */ +static void iommu_debug_dma_deconfigure(struct iommu_debug_device *ddev) +{ + struct iommu_domain *domain; + struct device *dev = ddev->dev; + + if (!dev->iommu_group) { + dev_err_ratelimited(dev, "Does not have an iommu group\n"); + return; + } + + domain = ddev->domain; + if (!domain) { + dev_err_ratelimited(dev, "Is not attached\n"); + return; + } + + + arch_teardown_dma_ops(dev); + iommu_detach_group(domain, dev->iommu_group); + iommu_domain_free(domain); + + ddev->domain = NULL; +} + +static const char * const _size_to_string(unsigned long size) +{ + switch (size) { + case SZ_4K: + return "4K"; + case SZ_8K: + return "8K"; + case SZ_16K: + return "16K"; + case SZ_64K: + return "64K"; + case SZ_1M: + return "1M"; + case SZ_2M: + return "2M"; + case SZ_1M * 12: + return "12M"; + case SZ_1M * 20: + return "20M"; + case SZ_1M * 24: + return "24M"; + case SZ_1M * 32: + return "32M"; + } + + pr_err("unknown size, please add to %s\n", __func__); + return "unknown size"; +} + +static int nr_iters_set(void *data, u64 val) +{ + if (!val) + val = 1; + if (val > 10000) + val = 10000; + *(u32 *)data = val; + return 0; +} + +static int nr_iters_get(void *data, u64 *val) +{ + *val = *(u32 *)data; + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(iommu_debug_nr_iters_ops, nr_iters_get, nr_iters_set, + "%llu\n"); + +static void iommu_debug_device_profiling(struct seq_file *s, + struct iommu_debug_device *ddev, + struct iommu_debug_attr *attrs, + const size_t sizes[]) +{ + const size_t *sz; + struct iommu_domain *domain; + struct device *dev = ddev->dev; + unsigned long iova = 0x10000; + phys_addr_t paddr = 0xa000; + + if (iommu_debug_dma_reconfigure(ddev, attrs, 0, SZ_1G * 4ULL)) + return; + domain = ddev->domain; + + iommu_debug_print_attrs(s, attrs); + + seq_printf(s, "(average over %d iterations)\n", iters_per_op); + seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map", "iommu_unmap"); + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + size_t unmapped; + u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; + u64 map_elapsed_us = 0, unmap_elapsed_us = 0; + u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; + ktime_t tbefore, tafter, diff; + int i; + + for (i = 0; i < iters_per_op; ++i) { + tbefore = ktime_get(); + if (iommu_map(domain, iova, paddr, size, + IOMMU_READ | IOMMU_WRITE)) { + seq_puts(s, "Failed to map\n"); + continue; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + map_elapsed_ns += ktime_to_ns(diff); + + tbefore = ktime_get(); + unmapped = iommu_unmap(domain, iova, size); + if (unmapped != size) { + seq_printf(s, + "Only unmapped %zx instead of %zx\n", + unmapped, size); + continue; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + unmap_elapsed_ns += ktime_to_ns(diff); + } + + map_elapsed_ns = div_u64_rem(map_elapsed_ns, iters_per_op, + &map_elapsed_rem); + unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, iters_per_op, + &unmap_elapsed_rem); + + map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, + &map_elapsed_rem); + unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, + &unmap_elapsed_rem); + + seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", + _size_to_string(size), + map_elapsed_us, map_elapsed_rem, + unmap_elapsed_us, unmap_elapsed_rem); + } + + seq_putc(s, '\n'); + seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map_sg", "iommu_unmap"); + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + size_t unmapped; + u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; + u64 map_elapsed_us = 0, unmap_elapsed_us = 0; + u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; + ktime_t tbefore, tafter, diff; + struct sg_table table; + unsigned long chunk_size = SZ_4K; + int i; + + if (iommu_debug_build_phoney_sg_table(dev, &table, size, + chunk_size)) { + seq_puts(s, + "couldn't build phoney sg table! bailing...\n"); + goto out_detach; + } + + for (i = 0; i < iters_per_op; ++i) { + tbefore = ktime_get(); + if (iommu_map_sg(domain, iova, table.sgl, table.nents, + IOMMU_READ | IOMMU_WRITE) != size) { + seq_puts(s, "Failed to map_sg\n"); + goto next; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + map_elapsed_ns += ktime_to_ns(diff); + + tbefore = ktime_get(); + unmapped = iommu_unmap(domain, iova, size); + if (unmapped != size) { + seq_printf(s, + "Only unmapped %zx instead of %zx\n", + unmapped, size); + goto next; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + unmap_elapsed_ns += ktime_to_ns(diff); + } + + map_elapsed_ns = div_u64_rem(map_elapsed_ns, iters_per_op, + &map_elapsed_rem); + unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, iters_per_op, + &unmap_elapsed_rem); + + map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, + &map_elapsed_rem); + unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, + &unmap_elapsed_rem); + + seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", + _size_to_string(size), + map_elapsed_us, map_elapsed_rem, + unmap_elapsed_us, unmap_elapsed_rem); + +next: + iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size); + } + +out_detach: + iommu_debug_dma_deconfigure(ddev); +} + +static int iommu_debug_profiling_show(struct seq_file *s, void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + const size_t sizes[] = { SZ_4K, SZ_64K, SZ_1M, SZ_2M, SZ_1M * 12, + SZ_1M * 24, SZ_1M * 32, 0 }; + + mutex_lock(&ddev->state_lock); + iommu_debug_device_profiling(s, ddev, &std_attr, sizes); + mutex_unlock(&ddev->state_lock); + + return 0; +} + +static int iommu_debug_profiling_open(struct inode *inode, struct file *file) +{ + return single_open(file, iommu_debug_profiling_show, inode->i_private); +} + +static const struct file_operations iommu_debug_profiling_fops = { + .open = iommu_debug_profiling_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int iommu_debug_secure_profiling_show(struct seq_file *s, void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + const size_t sizes[] = { SZ_4K, SZ_64K, SZ_1M, SZ_2M, SZ_1M * 12, + SZ_1M * 24, SZ_1M * 32, 0 }; + + mutex_lock(&ddev->state_lock); + iommu_debug_device_profiling(s, ddev, &secure_attr, sizes); + mutex_unlock(&ddev->state_lock); + + return 0; +} + +static int iommu_debug_secure_profiling_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_secure_profiling_show, + inode->i_private); +} + +static const struct file_operations iommu_debug_secure_profiling_fops = { + .open = iommu_debug_secure_profiling_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int iommu_debug_profiling_fast_show(struct seq_file *s, void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0}; + + mutex_lock(&ddev->state_lock); + iommu_debug_device_profiling(s, ddev, &fastmap_attr, sizes); + mutex_unlock(&ddev->state_lock); + + return 0; +} + +static int iommu_debug_profiling_fast_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_profiling_fast_show, + inode->i_private); +} + +static const struct file_operations iommu_debug_profiling_fast_fops = { + .open = iommu_debug_profiling_fast_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int iommu_debug_profiling_fast_dma_api_show(struct seq_file *s, + void *ignored) +{ + int i, experiment; + struct iommu_debug_device *ddev = s->private; + struct device *dev = ddev->dev; + u64 map_elapsed_ns[10], unmap_elapsed_ns[10]; + struct iommu_domain *domain; + dma_addr_t dma_addr; + void *virt; + const char * const extra_labels[] = { + "not coherent", + "coherent", + }; + unsigned long extra_attrs[] = { + 0, + DMA_ATTR_SKIP_CPU_SYNC, + }; + + mutex_lock(&ddev->state_lock); + + virt = kmalloc(1518, GFP_KERNEL); + if (!virt) + goto out; + + if (iommu_debug_dma_reconfigure(ddev, &fastmap_attr, 0, SZ_1G * 4ULL)) { + seq_puts(s, "setup failed\n"); + goto out_kfree; + } + domain = ddev->domain; + + if (iommu_enable_config_clocks(domain)) { + seq_puts(s, "Couldn't enable clocks\n"); + goto out_detach; + } + for (experiment = 0; experiment < 2; ++experiment) { + size_t map_avg = 0, unmap_avg = 0; + + for (i = 0; i < 10; ++i) { + ktime_t tbefore, tafter, diff; + u64 ns; + + tbefore = ktime_get(); + dma_addr = dma_map_single_attrs( + dev, virt, SZ_4K, DMA_TO_DEVICE, + extra_attrs[experiment]); + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + ns = ktime_to_ns(diff); + if (dma_mapping_error(dev, dma_addr)) { + seq_puts(s, "dma_map_single failed\n"); + goto out_disable_config_clocks; + } + map_elapsed_ns[i] = ns; + + tbefore = ktime_get(); + dma_unmap_single_attrs( + dev, dma_addr, SZ_4K, DMA_TO_DEVICE, + extra_attrs[experiment]); + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + ns = ktime_to_ns(diff); + unmap_elapsed_ns[i] = ns; + } + + seq_printf(s, "%13s %24s (ns): [", extra_labels[experiment], + "dma_map_single_attrs"); + for (i = 0; i < 10; ++i) { + map_avg += map_elapsed_ns[i]; + seq_printf(s, "%5llu%s", map_elapsed_ns[i], + i < 9 ? ", " : ""); + } + map_avg /= 10; + seq_printf(s, "] (avg: %zu)\n", map_avg); + + seq_printf(s, "%13s %24s (ns): [", extra_labels[experiment], + "dma_unmap_single_attrs"); + for (i = 0; i < 10; ++i) { + unmap_avg += unmap_elapsed_ns[i]; + seq_printf(s, "%5llu%s", unmap_elapsed_ns[i], + i < 9 ? ", " : ""); + } + unmap_avg /= 10; + seq_printf(s, "] (avg: %zu)\n", unmap_avg); + } + +out_disable_config_clocks: + iommu_disable_config_clocks(domain); +out_detach: + iommu_debug_dma_deconfigure(ddev); +out_kfree: + kfree(virt); +out: + mutex_unlock(&ddev->state_lock); + return 0; +} + +static int iommu_debug_profiling_fast_dma_api_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_profiling_fast_dma_api_show, + inode->i_private); +} + +static const struct file_operations iommu_debug_profiling_fast_dma_api_fops = { + .open = iommu_debug_profiling_fast_dma_api_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __tlb_stress_sweep(struct device *dev, struct seq_file *s) +{ + int i, ret = 0; + u64 iova; + const u64 max = SZ_1G * 4ULL - 1; + void *virt; + phys_addr_t phys; + dma_addr_t dma_addr; + + /* + * we'll be doing 4K and 8K mappings. Need to own an entire 8K + * chunk that we can work with. + */ + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(SZ_8K)); + phys = virt_to_phys(virt); + + /* fill the whole 4GB space */ + for (iova = 0, i = 0; iova < max; iova += SZ_8K, ++i) { + dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); + if (dma_addr == DMA_ERROR_CODE) { + dev_err_ratelimited(dev, "Failed map on iter %d\n", i); + ret = -EINVAL; + goto out; + } + } + + if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_ERROR_CODE) { + dev_err_ratelimited(dev, + "dma_map_single unexpectedly (VA should have been exhausted)\n"); + ret = -EINVAL; + goto out; + } + + /* + * free up 4K at the very beginning, then leave one 4K mapping, + * then free up 8K. This will result in the next 8K map to skip + * over the 4K hole and take the 8K one. + */ + dma_unmap_single(dev, 0, SZ_4K, DMA_TO_DEVICE); + dma_unmap_single(dev, SZ_8K, SZ_4K, DMA_TO_DEVICE); + dma_unmap_single(dev, SZ_8K + SZ_4K, SZ_4K, DMA_TO_DEVICE); + + /* remap 8K */ + dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); + if (dma_addr != SZ_8K) { + dma_addr_t expected = SZ_8K; + + dev_err_ratelimited(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", + &dma_addr, &expected); + ret = -EINVAL; + goto out; + } + + /* + * now remap 4K. We should get the first 4K chunk that was skipped + * over during the previous 8K map. If we missed a TLB invalidate + * at that point this should explode. + */ + dma_addr = dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE); + if (dma_addr != 0) { + dma_addr_t expected = 0; + + dev_err_ratelimited(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", + &dma_addr, &expected); + ret = -EINVAL; + goto out; + } + + if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_ERROR_CODE) { + dev_err_ratelimited(dev, + "dma_map_single unexpectedly after remaps (VA should have been exhausted)\n"); + ret = -EINVAL; + goto out; + } + + /* we're all full again. unmap everything. */ + for (iova = 0; iova < max; iova += SZ_8K) + dma_unmap_single(dev, (dma_addr_t)iova, SZ_8K, DMA_TO_DEVICE); + +out: + free_pages((unsigned long)virt, get_order(SZ_8K)); + return ret; +} + +struct fib_state { + unsigned long cur; + unsigned long prev; +}; + +static void fib_init(struct fib_state *f) +{ + f->cur = f->prev = 1; +} + +static unsigned long get_next_fib(struct fib_state *f) +{ + int next = f->cur + f->prev; + + f->prev = f->cur; + f->cur = next; + return next; +} + +/* + * Not actually random. Just testing the fibs (and max - the fibs). + */ +static int __rand_va_sweep(struct device *dev, struct seq_file *s, + const size_t size) +{ + u64 iova; + const u64 max = SZ_1G * 4ULL - 1; + int i, remapped, unmapped, ret = 0; + void *virt; + dma_addr_t dma_addr, dma_addr2; + struct fib_state fib; + + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + if (!virt) { + if (size > SZ_8K) { + dev_err_ratelimited(dev, + "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", + _size_to_string(size)); + return 0; + } + return -ENOMEM; + } + + /* fill the whole 4GB space */ + for (iova = 0, i = 0; iova < max; iova += size, ++i) { + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr == DMA_ERROR_CODE) { + dev_err_ratelimited(dev, "Failed map on iter %d\n", i); + ret = -EINVAL; + goto out; + } + } + + /* now unmap "random" iovas */ + unmapped = 0; + fib_init(&fib); + for (iova = get_next_fib(&fib) * size; + iova < max - size; + iova = (u64)get_next_fib(&fib) * size) { + dma_addr = (dma_addr_t)(iova); + dma_addr2 = (dma_addr_t)((max + 1) - size - iova); + if (dma_addr == dma_addr2) { + WARN(1, + "%s test needs update! The random number sequence is folding in on itself and should be changed.\n", + __func__); + return -EINVAL; + } + dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE); + dma_unmap_single(dev, dma_addr2, size, DMA_TO_DEVICE); + unmapped += 2; + } + + /* and map until everything fills back up */ + for (remapped = 0; ; ++remapped) { + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr == DMA_ERROR_CODE) + break; + } + + if (unmapped != remapped) { + dev_err_ratelimited(dev, + "Unexpected random remap count! Unmapped %d but remapped %d\n", + unmapped, remapped); + ret = -EINVAL; + } + + for (iova = 0; iova < max; iova += size) + dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); + +out: + free_pages((unsigned long)virt, get_order(size)); + return ret; +} + +static int __check_mapping(struct device *dev, struct iommu_domain *domain, + dma_addr_t iova, phys_addr_t expected) +{ + phys_addr_t res = iommu_iova_to_phys_hard(domain, iova); + phys_addr_t res2 = iommu_iova_to_phys(domain, iova); + + WARN(res != res2, "hard/soft iova_to_phys fns don't agree..."); + + if (res != expected) { + dev_err_ratelimited(dev, + "Bad translation for %pa! Expected: %pa Got: %pa\n", + &iova, &expected, &res); + return -EINVAL; + } + + return 0; +} + +static int __full_va_sweep(struct device *dev, struct seq_file *s, + const size_t size, struct iommu_domain *domain) +{ + u64 iova; + dma_addr_t dma_addr; + void *virt; + phys_addr_t phys; + const u64 max = SZ_1G * 4ULL - 1; + int ret = 0, i; + + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + if (!virt) { + if (size > SZ_8K) { + dev_err_ratelimited(dev, + "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", + _size_to_string(size)); + return 0; + } + return -ENOMEM; + } + phys = virt_to_phys(virt); + + for (iova = 0, i = 0; iova < max; iova += size, ++i) { + unsigned long expected = iova; + + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr != expected) { + dev_err_ratelimited(dev, + "Unexpected iova on iter %d (expected: 0x%lx got: 0x%lx)\n", + i, expected, + (unsigned long)dma_addr); + ret = -EINVAL; + goto out; + } + } + + if (domain) { + /* check every mapping from 0..6M */ + for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { + phys_addr_t expected = phys; + + if (__check_mapping(dev, domain, iova, expected)) { + dev_err_ratelimited(dev, "iter: %d\n", i); + ret = -EINVAL; + goto out; + } + } + /* and from 4G..4G-6M */ + for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { + phys_addr_t expected = phys; + unsigned long theiova = ((SZ_1G * 4ULL) - size) - iova; + + if (__check_mapping(dev, domain, theiova, expected)) { + dev_err_ratelimited(dev, "iter: %d\n", i); + ret = -EINVAL; + goto out; + } + } + } + + /* at this point, our VA space should be full */ + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr != DMA_ERROR_CODE) { + dev_err_ratelimited(dev, + "dma_map_single succeeded when it should have failed. Got iova: 0x%lx\n", + (unsigned long)dma_addr); + ret = -EINVAL; + } + +out: + for (iova = 0; iova < max; iova += size) + dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); + + free_pages((unsigned long)virt, get_order(size)); + return ret; +} + +#define ds_printf(d, s, fmt, ...) ({ \ + dev_err(d, fmt, ##__VA_ARGS__); \ + seq_printf(s, fmt, ##__VA_ARGS__); \ + }) + +static int __functional_dma_api_va_test(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, void *priv) +{ + int i, j, ret = 0; + size_t *sz, *sizes = priv; + + for (j = 0; j < 1; ++j) { + for (sz = sizes; *sz; ++sz) { + for (i = 0; i < 2; ++i) { + ds_printf(dev, s, "Full VA sweep @%s %d", + _size_to_string(*sz), i); + if (__full_va_sweep(dev, s, *sz, domain)) { + ds_printf(dev, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ds_printf(dev, s, " -> SUCCEEDED\n"); + } + } + } + } + + ds_printf(dev, s, "bonus map:"); + if (__full_va_sweep(dev, s, SZ_4K, domain)) { + ds_printf(dev, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ds_printf(dev, s, " -> SUCCEEDED\n"); + } + + for (sz = sizes; *sz; ++sz) { + for (i = 0; i < 2; ++i) { + ds_printf(dev, s, "Rand VA sweep @%s %d", + _size_to_string(*sz), i); + if (__rand_va_sweep(dev, s, *sz)) { + ds_printf(dev, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ds_printf(dev, s, " -> SUCCEEDED\n"); + } + } + } + + ds_printf(dev, s, "TLB stress sweep"); + if (__tlb_stress_sweep(dev, s)) { + ds_printf(dev, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ds_printf(dev, s, " -> SUCCEEDED\n"); + } + + ds_printf(dev, s, "second bonus map:"); + if (__full_va_sweep(dev, s, SZ_4K, domain)) { + ds_printf(dev, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ds_printf(dev, s, " -> SUCCEEDED\n"); + } + + return ret; +} + +static int __functional_dma_api_alloc_test(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *ignored) +{ + size_t size = SZ_1K * 742; + int ret = 0; + u8 *data; + dma_addr_t iova; + + /* Make sure we can allocate and use a buffer */ + ds_printf(dev, s, "Allocating coherent buffer"); + data = dma_alloc_coherent(dev, size, &iova, GFP_KERNEL); + if (!data) { + ds_printf(dev, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + int i; + + ds_printf(dev, s, " -> SUCCEEDED\n"); + ds_printf(dev, s, "Using coherent buffer"); + for (i = 0; i < 742; ++i) { + int ind = SZ_1K * i; + u8 *p = data + ind; + u8 val = i % 255; + + memset(data, 0xa5, size); + *p = val; + (*p)++; + if ((*p) != val + 1) { + ds_printf(dev, s, + " -> FAILED on iter %d since %d != %d\n", + i, *p, val + 1); + ret = -EINVAL; + } + } + if (!ret) + ds_printf(dev, s, " -> SUCCEEDED\n"); + dma_free_coherent(dev, size, data, iova); + } + + return ret; +} + +static int __functional_dma_api_basic_test(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *ignored) +{ + size_t size = 1518; + int i, j, ret = 0; + u8 *data; + dma_addr_t iova; + phys_addr_t pa, pa2; + + ds_printf(dev, s, "Basic DMA API test"); + /* Make sure we can allocate and use a buffer */ + for (i = 0; i < 1000; ++i) { + data = kmalloc(size, GFP_KERNEL); + if (!data) { + ds_printf(dev, s, " -> FAILED\n"); + ret = -EINVAL; + goto out; + } + memset(data, 0xa5, size); + iova = dma_map_single(dev, data, size, DMA_TO_DEVICE); + pa = iommu_iova_to_phys(domain, iova); + pa2 = iommu_iova_to_phys_hard(domain, iova); + if (pa != pa2) { + dev_err_ratelimited(dev, + "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n", + &pa, &pa2); + ret = -EINVAL; + goto out; + } + pa2 = virt_to_phys(data); + if (pa != pa2) { + dev_err_ratelimited(dev, + "iova_to_phys doesn't match virt_to_phys: %pa != %pa\n", + &pa, &pa2); + ret = -EINVAL; + goto out; + } + dma_unmap_single(dev, iova, size, DMA_TO_DEVICE); + for (j = 0; j < size; ++j) { + if (data[j] != 0xa5) { + dev_err_ratelimited(dev, + "data[%d] != 0xa5\n", data[j]); + ret = -EINVAL; + goto out; + } + } + kfree(data); + } + +out: + if (ret) + ds_printf(dev, s, " -> FAILED\n"); + else + ds_printf(dev, s, " -> SUCCEEDED\n"); + + return ret; +} + +static int __functional_dma_api_map_sg_test(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + size_t sizes[]) +{ + const size_t *sz; + int i, ret = 0, count = 0; + dma_addr_t iova; + phys_addr_t pa, pa2; + + ds_printf(dev, s, "Map SG DMA API test\n"); + + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + struct sg_table table; + unsigned long chunk_size = SZ_4K; + struct scatterlist *sg; + + /* Build us a table */ + ret = iommu_debug_build_phoney_sg_table(dev, &table, size, + chunk_size); + if (ret) { + seq_puts(s, + "couldn't build phoney sg table! bailing...\n"); + goto out; + } + count = dma_map_sg(dev, table.sgl, table.nents, + DMA_BIDIRECTIONAL); + if (!count) { + ret = -EINVAL; + goto destroy_table; + } + /* Check mappings... */ + for_each_sg(table.sgl, sg, count, i) { + iova = sg_dma_address(sg); + pa = iommu_iova_to_phys(domain, iova); + pa2 = iommu_iova_to_phys_hard(domain, iova); + if (pa != pa2) { + dev_err_ratelimited(dev, + "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n", + &pa, &pa2); + ret = -EINVAL; + goto unmap; + } + /* check mappings at end of buffer */ + iova += sg_dma_len(sg) - 1; + pa = iommu_iova_to_phys(domain, iova); + pa2 = iommu_iova_to_phys_hard(domain, iova); + if (pa != pa2) { + dev_err_ratelimited(dev, + "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n", + &pa, &pa2); + ret = -EINVAL; + goto unmap; + } + } +unmap: + dma_unmap_sg(dev, table.sgl, table.nents, DMA_BIDIRECTIONAL); +destroy_table: + iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size); + } +out: + if (ret) + ds_printf(dev, s, " -> FAILED\n"); + else + ds_printf(dev, s, " -> SUCCEEDED\n"); + + return ret; +} + +/* Creates a fresh fast mapping and applies @fn to it */ +static int __apply_to_new_mapping(struct seq_file *s, + int (*fn)(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *priv), + void *priv) +{ + struct iommu_domain *domain; + struct iommu_debug_device *ddev = s->private; + struct device *dev = ddev->dev; + int ret = -EINVAL; + phys_addr_t pt_phys; + + mutex_lock(&ddev->state_lock); + if (iommu_debug_dma_reconfigure(ddev, &fastmap_attr, 0, SZ_1G * 4ULL)) { + seq_puts(s, "setup failed\n"); + goto out; + } + domain = ddev->domain; + + if (iommu_domain_get_attr(domain, DOMAIN_ATTR_PT_BASE_ADDR, + &pt_phys)) { + ds_printf(dev, s, "Couldn't get page table base address\n"); + goto out_release_mapping; + } + + dev_err_ratelimited(dev, "testing with pgtables at %pa\n", &pt_phys); + if (iommu_enable_config_clocks(domain)) { + ds_printf(dev, s, "Couldn't enable clocks\n"); + goto out_release_mapping; + } + ret = fn(dev, s, domain, priv); + iommu_disable_config_clocks(domain); + +out_release_mapping: + iommu_debug_dma_deconfigure(ddev); +out: + mutex_unlock(&ddev->state_lock); + seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS"); + return 0; +} + +static int iommu_debug_functional_fast_dma_api_show(struct seq_file *s, + void *ignored) +{ + size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0}; + int ret = 0; + + ret |= __apply_to_new_mapping(s, __functional_dma_api_alloc_test, NULL); + ret |= __apply_to_new_mapping(s, __functional_dma_api_basic_test, NULL); + ret |= __apply_to_new_mapping(s, __functional_dma_api_va_test, sizes); + return ret; +} + +static int iommu_debug_functional_fast_dma_api_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_functional_fast_dma_api_show, + inode->i_private); +} + +static const struct file_operations iommu_debug_functional_fast_dma_api_fops = { + .open = iommu_debug_functional_fast_dma_api_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int iommu_debug_functional_arm_dma_api_show(struct seq_file *s, + void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + struct device *dev = ddev->dev; + size_t sizes[] = {SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, 0}; + int ret = -EINVAL; + + mutex_lock(&ddev->state_lock); + if (iommu_debug_dma_reconfigure(ddev, &fastmap_attr, 0, SZ_1G * 4ULL)) + goto out; + + ret = __functional_dma_api_alloc_test(dev, s, ddev->domain, sizes); + ret |= __functional_dma_api_basic_test(dev, s, ddev->domain, sizes); + ret |= __functional_dma_api_map_sg_test(dev, s, ddev->domain, sizes); + + iommu_debug_dma_deconfigure(ddev); +out: + mutex_unlock(&ddev->state_lock); + seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS"); + return 0; +} + +static int iommu_debug_functional_arm_dma_api_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_functional_arm_dma_api_show, + inode->i_private); +} + +static const struct file_operations iommu_debug_functional_arm_dma_api_fops = { + .open = iommu_debug_functional_arm_dma_api_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t __iommu_debug_attach_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset, + struct iommu_debug_attr *attrs) +{ + struct iommu_debug_device *ddev = file->private_data; + ssize_t retval = -EINVAL; + int val, ret; + + if (kstrtoint_from_user(ubuf, count, 0, &val)) { + pr_err_ratelimited("Invalid format. Expected a hex or decimal integer"); + return -EFAULT; + } + + mutex_lock(&ddev->state_lock); + if (val) { + ret = iommu_debug_dma_reconfigure(ddev, attrs, 0, SZ_1G * 4ULL); + if (!ret) + pr_err_ratelimited("Attached\n"); + } else { + iommu_debug_dma_deconfigure(ddev); + pr_err_ratelimited("Detached\n"); + } + mutex_unlock(&ddev->state_lock); + retval = count; + return retval; +} + +static ssize_t iommu_debug_secure_attach_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + return __iommu_debug_attach_write(file, ubuf, count, offset, + &secure_attr); +} + +static ssize_t iommu_debug_attach_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + return __iommu_debug_attach_write(file, ubuf, count, offset, &std_attr); + +} + +static ssize_t iommu_debug_attach_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + char buf[100]; + + snprintf(buf, sizeof(buf), "%d\n", ddev->domain ? 1 : 0); + return simple_read_from_buffer(ubuf, count, offset, buf, strlen(buf)); +} + +static const struct file_operations iommu_debug_dma_attach_fops = { + .open = simple_open, + .write = iommu_debug_attach_write, + .read = iommu_debug_attach_read, +}; + +static ssize_t iommu_debug_test_virt_addr_read(struct file *file, + char __user *ubuf, + size_t count, loff_t *offset) +{ + char buf[100]; + size_t buf_len = sizeof(buf); + + if (*offset) + return 0; + + memset(buf, 0, buf_len); + + if (!test_virt_addr) + strlcpy(buf, "FAIL\n", buf_len); + else + snprintf(buf, buf_len, "0x%pK\n", test_virt_addr); + + return simple_read_from_buffer(ubuf, count, offset, buf, strlen(buf)); +} + +static const struct file_operations iommu_debug_test_virt_addr_fops = { + .open = simple_open, + .read = iommu_debug_test_virt_addr_read, +}; + +static const struct file_operations iommu_debug_attach_fops = { + .open = simple_open, + .write = iommu_debug_attach_write, + .read = iommu_debug_attach_read, +}; + +static const struct file_operations iommu_debug_secure_attach_fops = { + .open = simple_open, + .write = iommu_debug_secure_attach_write, + .read = iommu_debug_attach_read, +}; + +static ssize_t iommu_debug_pte_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + dma_addr_t iova; + + if (kstrtox_from_user(ubuf, count, 0, &iova)) { + pr_err_ratelimited("Invalid format for iova\n"); + ddev->iova = 0; + return -EINVAL; + } + + ddev->iova = iova; + pr_err_ratelimited("Saved iova=%pa for future PTE commands\n", &iova); + return count; +} + + +static ssize_t iommu_debug_pte_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + uint64_t pte; + char buf[100]; + + if (kptr_restrict != 0) { + pr_err_ratelimited("kptr_restrict needs to be disabled.\n"); + return -EPERM; + } + + if (*offset) + return 0; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + memset(buf, 0, sizeof(buf)); + + pte = iommu_iova_to_pte(ddev->domain, ddev->iova); + + if (!pte) + strlcpy(buf, "FAIL\n", sizeof(buf)); + else + snprintf(buf, sizeof(buf), "pte=%016llx\n", pte); + mutex_unlock(&ddev->state_lock); + return simple_read_from_buffer(ubuf, count, offset, buf, strlen(buf)); +} + +static const struct file_operations iommu_debug_pte_fops = { + .open = simple_open, + .write = iommu_debug_pte_write, + .read = iommu_debug_pte_read, +}; + +static ssize_t iommu_debug_atos_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + dma_addr_t iova; + + if (kstrtox_from_user(ubuf, count, 0, &iova)) { + pr_err_ratelimited("Invalid format for iova\n"); + ddev->iova = 0; + return -EINVAL; + } + + ddev->iova = iova; + pr_err_ratelimited("Saved iova=%pa for future ATOS commands\n", &iova); + return count; +} + +static ssize_t iommu_debug_atos_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + phys_addr_t phys; + char buf[100]; + + if (kptr_restrict != 0) { + pr_err_ratelimited("kptr_restrict needs to be disabled.\n"); + return -EPERM; + } + + if (*offset) + return 0; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + memset(buf, 0, 100); + + phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova); + if (!phys) { + strlcpy(buf, "FAIL\n", 100); + phys = iommu_iova_to_phys(ddev->domain, ddev->iova); + dev_err_ratelimited(ddev->dev, "ATOS for %pa failed. Software walk returned: %pa\n", + &ddev->iova, &phys); + } else { + snprintf(buf, 100, "%pa\n", &phys); + } + mutex_unlock(&ddev->state_lock); + return simple_read_from_buffer(ubuf, count, offset, buf, strlen(buf)); +} + +static const struct file_operations iommu_debug_atos_fops = { + .open = simple_open, + .write = iommu_debug_atos_write, + .read = iommu_debug_atos_read, +}; + +static ssize_t iommu_debug_dma_atos_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + phys_addr_t phys; + char buf[100]; + + if (kptr_restrict != 0) { + pr_err_ratelimited("kptr_restrict needs to be disabled.\n"); + return -EPERM; + } + if (*offset) + return 0; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + memset(buf, 0, sizeof(buf)); + + phys = iommu_iova_to_phys_hard(ddev->domain, + ddev->iova); + if (!phys) + strlcpy(buf, "FAIL\n", sizeof(buf)); + else + snprintf(buf, sizeof(buf), "%pa\n", &phys); + mutex_unlock(&ddev->state_lock); + return simple_read_from_buffer(ubuf, count, offset, buf, strlen(buf)); +} + +static const struct file_operations iommu_debug_dma_atos_fops = { + .open = simple_open, + .write = iommu_debug_atos_write, + .read = iommu_debug_dma_atos_read, +}; + +static ssize_t iommu_debug_map_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = -EINVAL; + int ret; + char *comma1, *comma2, *comma3; + char buf[100]; + dma_addr_t iova; + phys_addr_t phys; + size_t size; + int prot; + struct iommu_debug_device *ddev = file->private_data; + + if (count >= 100) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + memset(buf, 0, 100); + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + comma3 = strnchr(comma2 + 1, count, ','); + if (!comma3) + goto invalid_format; + + /* split up the words */ + *comma1 = *comma2 = *comma3 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtoux(comma1 + 1, 0, &phys)) + goto invalid_format; + + if (kstrtosize_t(comma2 + 1, 0, &size)) + goto invalid_format; + + if (kstrtoint(comma3 + 1, 0, &prot)) + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + ret = iommu_map(ddev->domain, iova, phys, size, prot); + if (ret) { + pr_err_ratelimited("iommu_map failed with %d\n", ret); + retval = -EIO; + goto out; + } + + retval = count; + pr_err_ratelimited("Mapped %pa to %pa (len=0x%zx, prot=0x%x)\n", + &iova, &phys, size, prot); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,phys,len,prot where `prot' is the bitwise OR of IOMMU_READ, IOMMU_WRITE, etc.\n"); + return -EINVAL; +} + +static const struct file_operations iommu_debug_map_fops = { + .open = simple_open, + .write = iommu_debug_map_write, +}; + +/* + * Performs DMA mapping of a given virtual address and size to an iova address. + * User input format: (addr,len,dma attr) where dma attr is: + * 0: normal mapping + * 1: force coherent mapping + * 2: force non-cohernet mapping + * 3: use system cache + */ +static ssize_t iommu_debug_dma_map_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *offset) +{ + ssize_t retval = -EINVAL; + int ret; + char *comma1, *comma2; + char buf[100]; + unsigned long addr; + void *v_addr; + dma_addr_t iova; + size_t size; + unsigned int attr; + unsigned long dma_attrs; + struct iommu_debug_device *ddev = file->private_data; + struct device *dev = ddev->dev; + + if (count >= sizeof(buf)) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + memset(buf, 0, sizeof(buf)); + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + return -EFAULT; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + *comma1 = *comma2 = '\0'; + + if (kstrtoul(buf, 0, &addr)) + goto invalid_format; + v_addr = (void *)addr; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + if (kstrtouint(comma2 + 1, 0, &attr)) + goto invalid_format; + + if (v_addr < test_virt_addr || v_addr + size > test_virt_addr + SZ_1M) + goto invalid_addr; + + if (attr == 0) + dma_attrs = 0; + else if (attr == 1) + dma_attrs = DMA_ATTR_FORCE_COHERENT; + else if (attr == 2) + dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; + else if (attr == 3) + dma_attrs = DMA_ATTR_IOMMU_USE_UPSTREAM_HINT; + else + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + iova = dma_map_single_attrs(dev, v_addr, size, + DMA_TO_DEVICE, dma_attrs); + + if (dma_mapping_error(dev, iova)) { + pr_err_ratelimited("Failed to perform dma_map_single\n"); + ret = -EINVAL; + goto out; + } + + retval = count; + pr_err_ratelimited("Mapped 0x%p to %pa (len=0x%zx)\n", + v_addr, &iova, size); + ddev->iova = iova; + pr_err_ratelimited("Saved iova=%pa for future PTE commands\n", + &iova); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: addr,len,dma attr where 'dma attr' is\n0: normal mapping\n1: force coherent\n2: force non-cohernet\n3: use system cache\n"); + return retval; + +invalid_addr: + pr_err_ratelimited("Invalid addr given! Address should be within 1MB size from start addr returned by doing 'cat test_virt_addr'.\n"); + return retval; +} + +static ssize_t iommu_debug_dma_map_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + char buf[100]; + dma_addr_t iova; + + if (*offset) + return 0; + + memset(buf, 0, sizeof(buf)); + + iova = ddev->iova; + snprintf(buf, sizeof(buf), "%pa\n", &iova); + return simple_read_from_buffer(ubuf, count, offset, buf, strlen(buf)); +} + +static const struct file_operations iommu_debug_dma_map_fops = { + .open = simple_open, + .write = iommu_debug_dma_map_write, + .read = iommu_debug_dma_map_read, +}; + +static ssize_t iommu_debug_unmap_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = 0; + char *comma1; + char buf[100]; + dma_addr_t iova; + size_t size; + size_t unmapped; + struct iommu_debug_device *ddev = file->private_data; + + if (count >= 100) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + return -EINVAL; + } + + memset(buf, 0, 100); + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + goto out; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + /* split up the words */ + *comma1 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + unmapped = iommu_unmap(ddev->domain, iova, size); + if (unmapped != size) { + pr_err_ratelimited("iommu_unmap failed. Expected to unmap: 0x%zx, unmapped: 0x%zx", + size, unmapped); + retval = -EIO; + goto out; + } + + retval = count; + pr_err_ratelimited("Unmapped %pa (len=0x%zx)\n", &iova, size); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,len\n"); + return -EINVAL; +} + +static const struct file_operations iommu_debug_unmap_fops = { + .open = simple_open, + .write = iommu_debug_unmap_write, +}; + +static ssize_t iommu_debug_dma_unmap_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = 0; + char *comma1, *comma2; + char buf[100]; + size_t size; + unsigned int attr; + dma_addr_t iova; + unsigned long dma_attrs; + struct iommu_debug_device *ddev = file->private_data; + struct device *dev = ddev->dev; + + if (count >= sizeof(buf)) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + memset(buf, 0, sizeof(buf)); + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + goto out; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + *comma1 = *comma2 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + if (kstrtouint(comma2 + 1, 0, &attr)) + goto invalid_format; + + if (attr == 0) + dma_attrs = 0; + else if (attr == 1) + dma_attrs = DMA_ATTR_FORCE_COHERENT; + else if (attr == 2) + dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; + else if (attr == 3) + dma_attrs = DMA_ATTR_IOMMU_USE_UPSTREAM_HINT; + else + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + dma_unmap_single_attrs(dev, iova, size, DMA_TO_DEVICE, dma_attrs); + + retval = count; + pr_err_ratelimited("Unmapped %pa (len=0x%zx)\n", &iova, size); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,len, dma attr\n"); + return retval; +} + +static const struct file_operations iommu_debug_dma_unmap_fops = { + .open = simple_open, + .write = iommu_debug_dma_unmap_write, +}; + +static ssize_t iommu_debug_config_clocks_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + char buf; + struct iommu_debug_device *ddev = file->private_data; + struct device *dev = ddev->dev; + + /* we're expecting a single character plus (optionally) a newline */ + if (count > 2) { + dev_err_ratelimited(dev, "Invalid value\n"); + return -EINVAL; + } + + if (!ddev->domain) { + dev_err_ratelimited(dev, "No domain. Did you already attach?\n"); + return -EINVAL; + } + + if (copy_from_user(&buf, ubuf, 1)) { + dev_err_ratelimited(dev, "Couldn't copy from user\n"); + return -EFAULT; + } + + mutex_lock(&ddev->clk_lock); + switch (buf) { + case '0': + if (ddev->clk_count == 0) { + dev_err_ratelimited(dev, "Config clocks already disabled\n"); + break; + } + + if (--ddev->clk_count > 0) + break; + + dev_err_ratelimited(dev, "Disabling config clocks\n"); + iommu_disable_config_clocks(ddev->domain); + break; + case '1': + if (ddev->clk_count++ > 0) + break; + + dev_err_ratelimited(dev, "Enabling config clocks\n"); + if (iommu_enable_config_clocks(ddev->domain)) + dev_err_ratelimited(dev, "Failed!\n"); + break; + default: + dev_err_ratelimited(dev, "Invalid value. Should be 0 or 1.\n"); + mutex_unlock(&ddev->clk_lock); + return -EINVAL; + } + mutex_unlock(&ddev->clk_lock); + + return count; +} + +static const struct file_operations iommu_debug_config_clocks_fops = { + .open = simple_open, + .write = iommu_debug_config_clocks_write, +}; + +static ssize_t iommu_debug_trigger_fault_write( + struct file *file, const char __user *ubuf, size_t count, + loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + unsigned long flags; + + if (kstrtoul_from_user(ubuf, count, 0, &flags)) { + pr_err_ratelimited("Invalid flags format\n"); + return -EFAULT; + } + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + iommu_trigger_fault(ddev->domain, flags); + + mutex_unlock(&ddev->state_lock); + return count; +} + +static const struct file_operations iommu_debug_trigger_fault_fops = { + .open = simple_open, + .write = iommu_debug_trigger_fault_write, +}; + +/* + * The following will only work for drivers that implement the generic + * device tree bindings described in + * Documentation/devicetree/bindings/iommu/iommu.txt + */ +static int iommu_debug_device_setup(struct device *dev) +{ + struct iommu_debug_device *ddev; + struct dentry *dir; + + if (!of_find_property(dev->of_node, "iommus", NULL)) + return -EINVAL; + + /* Hold a reference count */ + if (!iommu_group_get(dev)) + return -ENODEV; + + ddev = kzalloc(sizeof(*ddev), GFP_KERNEL); + if (!ddev) + return -ENOMEM; + + mutex_init(&ddev->clk_lock); + mutex_init(&ddev->state_lock); + ddev->dev = dev; + dir = debugfs_create_dir(dev_name(dev), debugfs_tests_dir); + if (!dir) { + pr_err_ratelimited("Couldn't create iommu/devices/%s debugfs dir\n", + dev_name(dev)); + goto err; + } + + if (!debugfs_create_file("nr_iters", 0400, dir, &iters_per_op, + &iommu_debug_nr_iters_ops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/nr_iters debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("test_virt_addr", 0400, dir, ddev, + &iommu_debug_test_virt_addr_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/test_virt_addr debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("profiling", 0400, dir, ddev, + &iommu_debug_profiling_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/profiling debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("secure_profiling", 0400, dir, ddev, + &iommu_debug_secure_profiling_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/secure_profiling debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("profiling_fast", 0400, dir, ddev, + &iommu_debug_profiling_fast_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/profiling_fast debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("profiling_fast_dma_api", 0400, dir, ddev, + &iommu_debug_profiling_fast_dma_api_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/profiling_fast_dma_api debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("functional_fast_dma_api", 0400, dir, ddev, + &iommu_debug_functional_fast_dma_api_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/functional_fast_dma_api debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("functional_arm_dma_api", 0400, dir, ddev, + &iommu_debug_functional_arm_dma_api_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/functional_arm_dma_api debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("dma_attach", 0600, dir, ddev, + &iommu_debug_dma_attach_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/dma_attach debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("attach", 0400, dir, ddev, + &iommu_debug_attach_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/attach debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("secure_attach", 0400, dir, ddev, + &iommu_debug_secure_attach_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/secure_attach debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("atos", 0200, dir, ddev, + &iommu_debug_atos_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/atos debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("dma_atos", 0600, dir, ddev, + &iommu_debug_dma_atos_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/dma_atos debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("map", 0200, dir, ddev, + &iommu_debug_map_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/map debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("dma_map", 0600, dir, ddev, + &iommu_debug_dma_map_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/dma_map debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("unmap", 0200, dir, ddev, + &iommu_debug_unmap_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/unmap debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("dma_unmap", 0200, dir, ddev, + &iommu_debug_dma_unmap_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/dma_unmap debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("pte", 0600, dir, ddev, + &iommu_debug_pte_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/pte debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("config_clocks", 0200, dir, ddev, + &iommu_debug_config_clocks_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/config_clocks debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + if (!debugfs_create_file("trigger-fault", 0200, dir, ddev, + &iommu_debug_trigger_fault_fops)) { + pr_err_ratelimited("Couldn't create iommu/devices/%s/trigger-fault debugfs file\n", + dev_name(dev)); + goto err_rmdir; + } + + list_add(&ddev->list, &iommu_debug_devices); + return 0; + +err_rmdir: + debugfs_remove_recursive(dir); +err: + kfree(ddev); + return 0; +} + +static int iommu_debug_init_tests(void) +{ + debugfs_tests_dir = debugfs_create_dir("tests", + iommu_debugfs_top); + if (!debugfs_tests_dir) { + pr_err_ratelimited("Couldn't create iommu/tests debugfs directory\n"); + return -ENODEV; + } + + test_virt_addr = kzalloc(SZ_1M, GFP_KERNEL); + + if (!test_virt_addr) { + debugfs_remove_recursive(debugfs_tests_dir); + return -ENOMEM; + } + + return 0; +} + +static void iommu_debug_destroy_tests(void) +{ + debugfs_remove_recursive(debugfs_tests_dir); +} +#else +static inline int iommu_debug_init_tests(void) { return 0; } +static inline void iommu_debug_destroy_tests(void) { } +#endif + +/* + * This isn't really a "driver", we just need something in the device tree + * so that our tests can run without any client drivers, and our tests rely + * on parsing the device tree for nodes with the `iommus' property. + */ +static int iommu_debug_probe(struct platform_device *pdev) +{ + return iommu_debug_device_setup(&pdev->dev); +} + +/* + * We'll let the call to iommu_debug_destroy_tests() handle getting rid of + * all the directories that were created. + */ +static int iommu_debug_remove(struct platform_device *pdev) +{ + return 0; +} + +static const struct of_device_id iommu_debug_of_match[] = { + { .compatible = "iommu-debug-test" }, + { }, +}; + +static struct platform_driver iommu_debug_driver = { + .probe = iommu_debug_probe, + .remove = iommu_debug_remove, + .driver = { + .name = "iommu-debug", + .of_match_table = iommu_debug_of_match, + }, +}; + +static int iommu_debug_init(void) +{ + int ret; + + ret = iommu_debug_init_tests(); + if (ret) + return ret; + + ret = platform_driver_register(&iommu_debug_driver); + if (ret) + iommu_debug_destroy_tests(); + return ret; +} + +static void iommu_debug_exit(void) +{ + platform_driver_unregister(&iommu_debug_driver); + iommu_debug_destroy_tests(); +} + +module_init(iommu_debug_init); +module_exit(iommu_debug_exit); diff --git a/drivers/iommu/iommu-debug.h b/drivers/iommu/iommu-debug.h new file mode 100644 index 000000000000..fae4f96b44bc --- /dev/null +++ b/drivers/iommu/iommu-debug.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved. + */ + +#ifndef IOMMU_DEBUG_H +#define IOMMU_DEBUG_H + +#ifdef CONFIG_IOMMU_DEBUG_TRACKING + +void iommu_debug_attach_device(struct iommu_domain *domain, struct device *dev); +void iommu_debug_domain_remove(struct iommu_domain *domain); + +#else /* !CONFIG_IOMMU_DEBUG_TRACKING */ + +static inline void iommu_debug_attach_device(struct iommu_domain *domain, + struct device *dev) +{ +} + +static inline void iommu_debug_domain_remove(struct iommu_domain *domain) +{ +} + +#endif /* CONFIG_IOMMU_DEBUG_TRACKING */ + +#endif /* IOMMU_DEBUG_H */ diff --git a/drivers/iommu/iommu-debugfs.c b/drivers/iommu/iommu-debugfs.c index f03548942096..41ced5ab3bed 100644 --- a/drivers/iommu/iommu-debugfs.c +++ b/drivers/iommu/iommu-debugfs.c @@ -32,6 +32,7 @@ void iommu_debugfs_setup(void) { if (!iommu_debugfs_dir) { iommu_debugfs_dir = debugfs_create_dir("iommu", NULL); + iommu_debugfs_top = iommu_debugfs_dir; pr_warn("\n"); pr_warn("*************************************************************\n"); pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 9f0a2844371c..ad489fa9f455 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -20,10 +20,13 @@ #include #include #include +#include #include #include #include +#include "iommu-debug.h" + static struct kset *iommu_group_kset; static DEFINE_IDA(iommu_group_ida); #ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH @@ -1327,6 +1330,7 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc); void iommu_domain_free(struct iommu_domain *domain) { + iommu_debug_domain_remove(domain); domain->ops->domain_free(domain); } EXPORT_SYMBOL_GPL(iommu_domain_free); @@ -1343,8 +1347,10 @@ static int __iommu_attach_device(struct iommu_domain *domain, return -ENODEV; ret = domain->ops->attach_dev(domain, dev); - if (!ret) + if (!ret) { trace_attach_device_to_domain(dev); + iommu_debug_attach_device(domain, dev); + } return ret; } @@ -1460,9 +1466,6 @@ static int __iommu_attach_group(struct iommu_domain *domain, { int ret; - if (group->default_domain && group->domain != group->default_domain) - return -EBUSY; - ret = __iommu_group_for_each_dev(group, domain, iommu_group_do_attach_device); if (ret == 0) @@ -1492,28 +1495,18 @@ static int iommu_group_do_detach_device(struct device *dev, void *data) return 0; } +/* + * Although upstream implements detaching the default_domain as a noop, + * the "SID switch" secure usecase require complete removal of SIDS/SMRS + * from HLOS iommu registers. + */ static void __iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) { - int ret; - - if (!group->default_domain) { - __iommu_group_for_each_dev(group, domain, + __iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device); - group->domain = NULL; - return; - } - - if (group->domain == group->default_domain) - return; - - /* Detach by re-attaching to the default domain */ - ret = __iommu_group_for_each_dev(group, group->default_domain, - iommu_group_do_attach_device); - if (ret != 0) - WARN_ON(1); - else - group->domain = group->default_domain; + group->domain = NULL; + return; } void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) @@ -1533,8 +1526,40 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) } EXPORT_SYMBOL_GPL(iommu_iova_to_phys); -static size_t iommu_pgsize(struct iommu_domain *domain, - unsigned long addr_merge, size_t size) +phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (unlikely(ops->iova_to_phys_hard == NULL)) + return 0; + + return ops->iova_to_phys_hard(domain, iova); +} + +uint64_t iommu_iova_to_pte(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (unlikely(ops->iova_to_pte == NULL)) + return 0; + + return ops->iova_to_pte(domain, iova); +} + +bool iommu_is_iova_coherent(struct iommu_domain *domain, dma_addr_t iova) +{ + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (unlikely(ops->is_iova_coherent == NULL)) + return false; + + return ops->is_iova_coherent(domain, iova); +} + +size_t iommu_pgsize(unsigned long pgsize_bitmap, + unsigned long addr_merge, size_t size) { unsigned int pgsize_idx; size_t pgsize; @@ -1553,10 +1578,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain, pgsize = (1UL << (pgsize_idx + 1)) - 1; /* throw away page sizes not supported by the hardware */ - pgsize &= domain->pgsize_bitmap; + pgsize &= pgsize_bitmap; /* make sure we're still sane */ - BUG_ON(!pgsize); + if (!pgsize) { + pr_err("invalid pgsize/addr/size! 0x%lx 0x%lx 0x%zx\n", + pgsize_bitmap, addr_merge, size); + BUG(); + } /* pick the biggest page */ pgsize_idx = __fls(pgsize); @@ -1599,7 +1628,8 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); while (size) { - size_t pgsize = iommu_pgsize(domain, iova | paddr, size); + size_t pgsize = iommu_pgsize(domain->pgsize_bitmap, + iova | paddr, size); pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", iova, &paddr, pgsize); @@ -1620,7 +1650,8 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, if (ret) iommu_unmap(domain, orig_iova, orig_size - size); else - trace_map(orig_iova, orig_paddr, orig_size); + trace_map(to_msm_iommu_domain(domain), orig_iova, orig_paddr, + orig_size, prot); return ret; } @@ -1663,14 +1694,14 @@ static size_t __iommu_unmap(struct iommu_domain *domain, * or we hit an area that isn't mapped. */ while (unmapped < size) { - size_t pgsize = iommu_pgsize(domain, iova, size - unmapped); + size_t left = size - unmapped; - unmapped_page = ops->unmap(domain, iova, pgsize); + unmapped_page = ops->unmap(domain, iova, left); if (!unmapped_page) break; if (sync && ops->iotlb_range_add) - ops->iotlb_range_add(domain, iova, pgsize); + ops->iotlb_range_add(domain, iova, left); pr_debug("unmapped: iova 0x%lx size 0x%zx\n", iova, unmapped_page); @@ -1682,7 +1713,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain, if (sync && ops->iotlb_sync) ops->iotlb_sync(domain); - trace_unmap(orig_iova, size, unmapped); + trace_unmap(to_msm_iommu_domain(domain), orig_iova, size, unmapped); return unmapped; } @@ -1702,6 +1733,19 @@ EXPORT_SYMBOL_GPL(iommu_unmap_fast); size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, struct scatterlist *sg, unsigned int nents, int prot) +{ + size_t mapped = 0; + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (ops->map_sg) + mapped = ops->map_sg(domain, iova, sg, nents, prot); + trace_map_sg(to_msm_iommu_domain(domain), iova, mapped, prot); + return mapped; +} +EXPORT_SYMBOL_GPL(iommu_map_sg); + +size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot) { size_t len = 0, mapped = 0; phys_addr_t start; @@ -1740,7 +1784,7 @@ out_err: return 0; } -EXPORT_SYMBOL_GPL(iommu_map_sg); +EXPORT_SYMBOL(default_iommu_map_sg); int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, phys_addr_t paddr, u64 size, int prot) @@ -1804,6 +1848,8 @@ int report_iommu_fault(struct iommu_domain *domain, struct device *dev, } EXPORT_SYMBOL_GPL(report_iommu_fault); +struct dentry *iommu_debugfs_top; + static int __init iommu_init(void) { iommu_group_kset = kset_create_and_add("iommu_groups", @@ -1877,6 +1923,23 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) ops->put_resv_regions(dev, list); } +/** + * iommu_trigger_fault() - trigger an IOMMU fault + * @domain: iommu domain + * + * Triggers a fault on the device to which this domain is attached. + * + * This function should only be used for debugging purposes, for obvious + * reasons. + */ +void iommu_trigger_fault(struct iommu_domain *domain, unsigned long flags) +{ + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (ops->trigger_fault) + ops->trigger_fault(domain, flags); +} + struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, enum iommu_resv_type type) diff --git a/drivers/iommu/msm_dma_iommu_mapping.c b/drivers/iommu/msm_dma_iommu_mapping.c new file mode 100644 index 000000000000..828fc2ab5120 --- /dev/null +++ b/drivers/iommu/msm_dma_iommu_mapping.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/** + * struct msm_iommu_map - represents a mapping of an ion buffer to an iommu + * @lnode - list node to exist in the buffer's list of iommu mappings + * @dev - Device this is mapped to. Used as key + * @sgl - The scatterlist for this mapping + * @nents - Number of entries in sgl + * @dir - The direction for the map. + * @meta - Backpointer to the meta this guy belongs to. + * @ref - for reference counting this mapping + * @attrs - dma mapping attributes + * @buf_start_addr - address of start of buffer + * + * Represents a mapping of one dma_buf buffer to a particular device + * and address range. There may exist other mappings of this buffer in + * different devices. All mappings will have the same cacheability and security. + */ +struct msm_iommu_map { + struct list_head lnode; + struct rb_node node; + struct device *dev; + struct scatterlist *sgl; + unsigned int nents; + enum dma_data_direction dir; + struct msm_iommu_meta *meta; + struct kref ref; + unsigned long attrs; + dma_addr_t buf_start_addr; +}; + +struct msm_iommu_meta { + struct rb_node node; + struct list_head iommu_maps; + struct kref ref; + struct mutex lock; + void *buffer; +}; + +static struct rb_root iommu_root; +static DEFINE_MUTEX(msm_iommu_map_mutex); + +static void msm_iommu_meta_add(struct msm_iommu_meta *meta) +{ + struct rb_root *root = &iommu_root; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct msm_iommu_meta *entry; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct msm_iommu_meta, node); + + if (meta->buffer < entry->buffer) + p = &(*p)->rb_left; + else if (meta->buffer > entry->buffer) + p = &(*p)->rb_right; + else + pr_err("%s: dma_buf %pK already exists\n", __func__, + entry->buffer); + } + + rb_link_node(&meta->node, parent, p); + rb_insert_color(&meta->node, root); +} + +static struct msm_iommu_meta *msm_iommu_meta_lookup(void *buffer) +{ + struct rb_root *root = &iommu_root; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct msm_iommu_meta *entry = NULL; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct msm_iommu_meta, node); + + if (buffer < entry->buffer) + p = &(*p)->rb_left; + else if (buffer > entry->buffer) + p = &(*p)->rb_right; + else + return entry; + } + + return NULL; +} + +static void msm_iommu_add(struct msm_iommu_meta *meta, + struct msm_iommu_map *iommu) +{ + INIT_LIST_HEAD(&iommu->lnode); + list_add(&iommu->lnode, &meta->iommu_maps); +} + + +static struct msm_iommu_map *msm_iommu_lookup(struct msm_iommu_meta *meta, + struct device *dev) +{ + struct msm_iommu_map *entry; + + list_for_each_entry(entry, &meta->iommu_maps, lnode) { + if (entry->dev == dev) + return entry; + } + + return NULL; +} + +static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf) +{ + struct msm_iommu_meta *meta; + + meta = kzalloc(sizeof(*meta), GFP_KERNEL); + + if (!meta) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&meta->iommu_maps); + meta->buffer = dma_buf->priv; + kref_init(&meta->ref); + mutex_init(&meta->lock); + msm_iommu_meta_add(meta); + + return meta; +} + +static void msm_iommu_meta_put(struct msm_iommu_meta *meta); + +static struct scatterlist *clone_sgl(struct scatterlist *sg, int nents) +{ + struct scatterlist *next, *s; + int i; + struct sg_table table; + + if (sg_alloc_table(&table, nents, GFP_KERNEL)) + return NULL; + next = table.sgl; + for_each_sg(sg, s, nents, i) { + *next = *s; + next = sg_next(next); + } + return table.sgl; +} + +static inline int __msm_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, + unsigned long attrs) +{ + struct msm_iommu_map *iommu_map; + struct msm_iommu_meta *iommu_meta = NULL; + int ret = 0; + bool extra_meta_ref_taken = false; + int late_unmap = !(attrs & DMA_ATTR_NO_DELAYED_UNMAP); + + mutex_lock(&msm_iommu_map_mutex); + iommu_meta = msm_iommu_meta_lookup(dma_buf->priv); + + if (!iommu_meta) { + iommu_meta = msm_iommu_meta_create(dma_buf); + + if (IS_ERR(iommu_meta)) { + mutex_unlock(&msm_iommu_map_mutex); + ret = PTR_ERR(iommu_meta); + goto out; + } + if (late_unmap) { + kref_get(&iommu_meta->ref); + extra_meta_ref_taken = true; + } + } else { + kref_get(&iommu_meta->ref); + } + + mutex_unlock(&msm_iommu_map_mutex); + + mutex_lock(&iommu_meta->lock); + iommu_map = msm_iommu_lookup(iommu_meta, dev); + if (!iommu_map) { + iommu_map = kmalloc(sizeof(*iommu_map), GFP_KERNEL); + + if (!iommu_map) { + ret = -ENOMEM; + goto out_unlock; + } + + ret = dma_map_sg_attrs(dev, sg, nents, dir, attrs); + if (!ret) { + kfree(iommu_map); + goto out_unlock; + } + + iommu_map->sgl = clone_sgl(sg, nents); + if (!iommu_map->sgl) { + kfree(iommu_map); + ret = -ENOMEM; + goto out_unlock; + } + iommu_map->nents = nents; + iommu_map->dev = dev; + iommu_map->dir = dir; + iommu_map->attrs = attrs; + iommu_map->buf_start_addr = sg_phys(sg); + + kref_init(&iommu_map->ref); + if (late_unmap) + kref_get(&iommu_map->ref); + iommu_map->meta = iommu_meta; + msm_iommu_add(iommu_meta, iommu_map); + + } else { + if (nents == iommu_map->nents && + dir == iommu_map->dir && + (attrs & ~DMA_ATTR_SKIP_CPU_SYNC) == + (iommu_map->attrs & ~DMA_ATTR_SKIP_CPU_SYNC) && + sg_phys(sg) == iommu_map->buf_start_addr) { + struct scatterlist *sg_tmp = sg; + struct scatterlist *map_sg; + int i; + + for_each_sg(iommu_map->sgl, map_sg, nents, i) { + sg_dma_address(sg_tmp) = sg_dma_address(map_sg); + sg_dma_len(sg_tmp) = sg_dma_len(map_sg); + if (sg_dma_len(map_sg) == 0) + break; + + sg_tmp = sg_next(sg_tmp); + if (sg_tmp == NULL) + break; + } + + kref_get(&iommu_map->ref); + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + dma_sync_sg_for_device(dev, iommu_map->sgl, + iommu_map->nents, iommu_map->dir); + + if (is_device_dma_coherent(dev)) + /* + * Ensure all outstanding changes for coherent + * buffers are applied to the cache before any + * DMA occurs. + */ + dmb(ish); + ret = nents; + } else { + bool start_diff = (sg_phys(sg) != + iommu_map->buf_start_addr); + + dev_err(dev, "lazy map request differs:\n" + "req dir:%d, original dir:%d\n" + "req nents:%d, original nents:%d\n" + "req map attrs:%lu, original map attrs:%lu\n" + "req buffer start address differs:%d\n", + dir, iommu_map->dir, nents, + iommu_map->nents, attrs, iommu_map->attrs, + start_diff); + ret = -EINVAL; + } + } + mutex_unlock(&iommu_meta->lock); + return ret; + +out_unlock: + mutex_unlock(&iommu_meta->lock); +out: + if (!IS_ERR(iommu_meta)) { + if (extra_meta_ref_taken) + msm_iommu_meta_put(iommu_meta); + msm_iommu_meta_put(iommu_meta); + } + return ret; + +} + +/* + * We are not taking a reference to the dma_buf here. It is expected that + * clients hold reference to the dma_buf until they are done with mapping and + * unmapping. + */ +int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_buf *dma_buf, + unsigned long attrs) +{ + int ret; + + if (IS_ERR_OR_NULL(dev)) { + pr_err("%s: dev pointer is invalid\n", __func__); + return -EINVAL; + } + + if (IS_ERR_OR_NULL(sg)) { + pr_err("%s: sg table pointer is invalid\n", __func__); + return -EINVAL; + } + + if (IS_ERR_OR_NULL(dma_buf)) { + pr_err("%s: dma_buf pointer is invalid\n", __func__); + return -EINVAL; + } + + ret = __msm_dma_map_sg(dev, sg, nents, dir, dma_buf, attrs); + + return ret; +} +EXPORT_SYMBOL(msm_dma_map_sg_attrs); + +static void msm_iommu_meta_destroy(struct kref *kref) +{ + struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta, + ref); + + if (!list_empty(&meta->iommu_maps)) { + WARN(1, "%s: DMA Buffer %pK being destroyed with outstanding iommu mappings!\n", + __func__, meta->buffer); + } + rb_erase(&meta->node, &iommu_root); + kfree(meta); +} + +static void msm_iommu_meta_put(struct msm_iommu_meta *meta) +{ + /* + * Need to lock here to prevent race against map/unmap + */ + mutex_lock(&msm_iommu_map_mutex); + kref_put(&meta->ref, msm_iommu_meta_destroy); + mutex_unlock(&msm_iommu_map_mutex); +} + +static void msm_iommu_map_release(struct kref *kref) +{ + struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map, + ref); + struct sg_table table; + + table.nents = table.orig_nents = map->nents; + table.sgl = map->sgl; + list_del(&map->lnode); + + /* Skip an additional cache maintenance on the dma unmap path */ + if (!(map->attrs & DMA_ATTR_SKIP_CPU_SYNC)) + map->attrs |= DMA_ATTR_SKIP_CPU_SYNC; + dma_unmap_sg_attrs(map->dev, map->sgl, map->nents, map->dir, + map->attrs); + sg_free_table(&table); + kfree(map); +} + +void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ + struct msm_iommu_map *iommu_map; + struct msm_iommu_meta *meta; + + mutex_lock(&msm_iommu_map_mutex); + meta = msm_iommu_meta_lookup(dma_buf->priv); + if (!meta) { + WARN(1, "%s: (%pK) was never mapped\n", __func__, dma_buf); + mutex_unlock(&msm_iommu_map_mutex); + goto out; + + } + mutex_unlock(&msm_iommu_map_mutex); + + mutex_lock(&meta->lock); + iommu_map = msm_iommu_lookup(meta, dev); + + if (!iommu_map) { + WARN(1, "%s: (%pK) was never mapped for device %p\n", __func__, + dma_buf, dev); + mutex_unlock(&meta->lock); + goto out; + } + + if (dir != iommu_map->dir) + WARN(1, "%s: (%pK) dir:%d differs from original dir:%d\n", + __func__, dma_buf, dir, iommu_map->dir); + + if (attrs && ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)) + dma_sync_sg_for_cpu(dev, iommu_map->sgl, iommu_map->nents, dir); + + iommu_map->attrs = attrs; + kref_put(&iommu_map->ref, msm_iommu_map_release); + mutex_unlock(&meta->lock); + + msm_iommu_meta_put(meta); + +out: + return; +} +EXPORT_SYMBOL(msm_dma_unmap_sg_attrs); + +int msm_dma_unmap_all_for_dev(struct device *dev) +{ + int ret = 0; + struct msm_iommu_meta *meta; + struct rb_root *root; + struct rb_node *meta_node; + + mutex_lock(&msm_iommu_map_mutex); + root = &iommu_root; + meta_node = rb_first(root); + while (meta_node) { + struct msm_iommu_map *iommu_map; + struct msm_iommu_map *iommu_map_next; + + meta = rb_entry(meta_node, struct msm_iommu_meta, node); + mutex_lock(&meta->lock); + list_for_each_entry_safe(iommu_map, iommu_map_next, + &meta->iommu_maps, lnode) + if (iommu_map->dev == dev) + if (!kref_put(&iommu_map->ref, + msm_iommu_map_release)) + ret = -EINVAL; + + mutex_unlock(&meta->lock); + meta_node = rb_next(meta_node); + } + mutex_unlock(&msm_iommu_map_mutex); + + return ret; +} +EXPORT_SYMBOL(msm_dma_unmap_all_for_dev); + +/* + * Only to be called by ION code when a buffer is freed + */ +void msm_dma_buf_freed(void *buffer) +{ + struct msm_iommu_map *iommu_map; + struct msm_iommu_map *iommu_map_next; + struct msm_iommu_meta *meta; + + mutex_lock(&msm_iommu_map_mutex); + meta = msm_iommu_meta_lookup(buffer); + if (!meta) { + /* Already unmapped (assuming no late unmapping) */ + mutex_unlock(&msm_iommu_map_mutex); + return; + } + mutex_unlock(&msm_iommu_map_mutex); + + mutex_lock(&meta->lock); + + list_for_each_entry_safe(iommu_map, iommu_map_next, &meta->iommu_maps, + lnode) + kref_put(&iommu_map->ref, msm_iommu_map_release); + + if (!list_empty(&meta->iommu_maps)) { + WARN(1, "%s: DMA buffer %pK destroyed with outstanding iommu mappings\n", + __func__, meta->buffer); + } + + INIT_LIST_HEAD(&meta->iommu_maps); + mutex_unlock(&meta->lock); + + msm_iommu_meta_put(meta); +} +EXPORT_SYMBOL(msm_dma_buf_freed); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 614a93aa5305..36732b1a0a44 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -104,8 +104,11 @@ static int of_iommu_xlate(struct device *dev, * IOMMU device we're waiting for, which will be useful if we ever get * a proper probe-ordering dependency mechanism in future. */ - if (!ops) + if (!ops) { + if (IS_ENABLED(CONFIG_MODULES)) + return -EPROBE_DEFER; return driver_deferred_probe_check_state(dev); + } return ops->of_xlate(dev, iommu_spec); } diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index b09544fa2416..a01d25f491bd 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -197,4 +197,13 @@ config QCOM_SECURE_CHAN_MANAGER support for Qualcomm SoC. SCM provides communication channel to communicate with secure world (EL2 and EL3) by using smc call. + +config QCOM_SECURE_BUFFER + tristate "Helper functions for secure buffers through TZ" + depends on QCOM_SECURE_CHAN_MANAGER + help + Enable for targets that need to call into TZ to secure + memory buffers. This ensures that only the correct clients can + use this memory and no unauthorized access is made to the + buffer. endmenu diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index fac3367e7705..514531eea6f9 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -19,6 +19,7 @@ obj-$(CONFIG_QCOM_SMP2P) += smp2p.o obj-$(CONFIG_QCOM_SMSM) += smsm.o CFLAGS_scm.o :=$(call as-instr,.arch_extension sec,-DREQUIRES_SEC=1, -Wno-asm-operand-widths) obj-$(CONFIG_QCOM_SECURE_CHAN_MANAGER) += scm.o +obj-$(CONFIG_QCOM_SECURE_BUFFER) += secure_buffer.o obj-$(CONFIG_QCOM_WCNSS_CTRL) += wcnss_ctrl.o obj-$(CONFIG_QCOM_APR) += apr.o obj-$(CONFIG_QCOM_LLCC) += llcc-slice.o diff --git a/drivers/soc/qcom/secure_buffer.c b/drivers/soc/qcom/secure_buffer.c new file mode 100644 index 000000000000..b0ae8bb3c974 --- /dev/null +++ b/drivers/soc/qcom/secure_buffer.c @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011 Google, Inc + * Copyright (c) 2011-2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MEM_PROT_ASSIGN_ID 0x16 +#define BATCH_MAX_SIZE SZ_2M +#define BATCH_MAX_SECTIONS 32 + +struct dest_vm_and_perm_info { + u32 vm; + u32 perm; + u64 ctx; + u32 ctx_size; +}; + +struct mem_prot_info { + phys_addr_t addr; + u64 size; +}; + +static struct device *qcom_secure_buffer_dev; + +static struct dest_vm_and_perm_info * +populate_dest_info(int *dest_vmids, int nelements, int *dest_perms, + size_t *size_in_bytes) +{ + struct dest_vm_and_perm_info *dest_info; + int i; + size_t size; + + /* Ensure allocated size is less than PAGE_ALLOC_COSTLY_ORDER */ + size = nelements * sizeof(*dest_info); + if (size > PAGE_SIZE) + return NULL; + + dest_info = kzalloc(size, GFP_KERNEL); + if (!dest_info) + return NULL; + + for (i = 0; i < nelements; i++) { + dest_info[i].vm = dest_vmids[i]; + dest_info[i].perm = dest_perms[i]; + dest_info[i].ctx = 0x0; + dest_info[i].ctx_size = 0; + } + + *size_in_bytes = size; + return dest_info; +} + +static unsigned int get_batches_from_sgl(struct mem_prot_info *sg_table_copy, + struct scatterlist *sgl, + struct scatterlist **next_sgl) +{ + u64 batch_size = 0; + unsigned int i = 0; + struct scatterlist *curr_sgl = sgl; + + /* Ensure no zero size batches */ + do { + sg_table_copy[i].addr = page_to_phys(sg_page(curr_sgl)); + sg_table_copy[i].size = curr_sgl->length; + batch_size += sg_table_copy[i].size; + curr_sgl = sg_next(curr_sgl); + i++; + } while (curr_sgl && i < BATCH_MAX_SECTIONS && + curr_sgl->length + batch_size < BATCH_MAX_SIZE); + + *next_sgl = curr_sgl; + return i; +} + +static int batched_hyp_assign(struct sg_table *table, struct scm_desc *desc) +{ + unsigned int entries_size; + unsigned int batch_start = 0; + unsigned int batches_processed; + struct scatterlist *curr_sgl = table->sgl; + struct scatterlist *next_sgl; + int ret = 0; + struct mem_prot_info *sg_table_copy = kcalloc(BATCH_MAX_SECTIONS, + sizeof(*sg_table_copy), + GFP_KERNEL); + dma_addr_t entries_dma_addr; + + if (!sg_table_copy) + return -ENOMEM; + + while (batch_start < table->nents) { + batches_processed = get_batches_from_sgl(sg_table_copy, + curr_sgl, &next_sgl); + curr_sgl = next_sgl; + entries_size = batches_processed * sizeof(*sg_table_copy); + entries_dma_addr = dma_map_single(qcom_secure_buffer_dev, + sg_table_copy, entries_size, + DMA_TO_DEVICE); + if (dma_mapping_error(qcom_secure_buffer_dev, + entries_dma_addr)) { + ret = -EADDRNOTAVAIL; + break; + } + desc->args[0] = entries_dma_addr; + desc->args[1] = entries_size; + + ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP, + MEM_PROT_ASSIGN_ID), desc); + dma_unmap_single(qcom_secure_buffer_dev, entries_dma_addr, + entries_size, DMA_TO_DEVICE); + if (ret) { + pr_info("%s: Failed to assign memory protection, ret = %d\n", + __func__, ret); + /* + * Make it clear to clients that the memory may no + * longer be in a usable state. + */ + ret = -EADDRNOTAVAIL; + break; + } + + batch_start += batches_processed; + } + + kfree(sg_table_copy); + return ret; +} + +/* + * When -EADDRNOTAVAIL is returned the memory may no longer be in + * a usable state and should no longer be accessed by the HLOS. + */ +int hyp_assign_table(struct sg_table *table, + u32 *source_vm_list, int source_nelems, + int *dest_vmids, int *dest_perms, + int dest_nelems) +{ + int ret = 0; + struct scm_desc desc = {0}; + u32 *source_vm_copy; + size_t source_vm_copy_size; + struct dest_vm_and_perm_info *dest_vm_copy; + size_t dest_vm_copy_size; + dma_addr_t source_dma_addr, dest_dma_addr; + + if (!qcom_secure_buffer_dev) + return -EPROBE_DEFER; + + if (!table || !table->sgl || !source_vm_list || !source_nelems || + !dest_vmids || !dest_perms || !dest_nelems) + return -EINVAL; + + /* + * We can only pass cache-aligned sizes to hypervisor, so we need + * to kmalloc and memcpy the source_vm_list here. + */ + source_vm_copy_size = sizeof(*source_vm_copy) * source_nelems; + source_vm_copy = kmemdup(source_vm_list, source_vm_copy_size, + GFP_KERNEL); + if (!source_vm_copy) + return -ENOMEM; + + source_dma_addr = dma_map_single(qcom_secure_buffer_dev, source_vm_copy, + source_vm_copy_size, DMA_TO_DEVICE); + if (dma_mapping_error(qcom_secure_buffer_dev, source_dma_addr)) { + ret = -ENOMEM; + goto out_free_source; + } + + dest_vm_copy = populate_dest_info(dest_vmids, dest_nelems, dest_perms, + &dest_vm_copy_size); + if (!dest_vm_copy) { + ret = -ENOMEM; + goto out_unmap_source; + } + + dest_dma_addr = dma_map_single(qcom_secure_buffer_dev, dest_vm_copy, + dest_vm_copy_size, DMA_TO_DEVICE); + if (dma_mapping_error(qcom_secure_buffer_dev, dest_dma_addr)) { + ret = -ENOMEM; + goto out_free_dest; + } + + + desc.args[2] = source_dma_addr; + desc.args[3] = source_vm_copy_size; + desc.args[4] = dest_dma_addr; + desc.args[5] = dest_vm_copy_size; + desc.args[6] = 0; + + desc.arginfo = SCM_ARGS(7, SCM_RO, SCM_VAL, SCM_RO, SCM_VAL, SCM_RO, + SCM_VAL, SCM_VAL); + + ret = batched_hyp_assign(table, &desc); + + dma_unmap_single(qcom_secure_buffer_dev, dest_dma_addr, + dest_vm_copy_size, DMA_TO_DEVICE); +out_free_dest: + kfree(dest_vm_copy); +out_unmap_source: + dma_unmap_single(qcom_secure_buffer_dev, source_dma_addr, + source_vm_copy_size, DMA_TO_DEVICE); +out_free_source: + kfree(source_vm_copy); + return ret; +} +EXPORT_SYMBOL(hyp_assign_table); + +int hyp_assign_phys(phys_addr_t addr, u64 size, u32 *source_vm_list, + int source_nelems, int *dest_vmids, + int *dest_perms, int dest_nelems) +{ + struct sg_table table; + int ret; + + if (!qcom_secure_buffer_dev) + return -EPROBE_DEFER; + + ret = sg_alloc_table(&table, 1, GFP_KERNEL); + if (ret) + return ret; + + sg_set_page(table.sgl, phys_to_page(addr), size, 0); + + ret = hyp_assign_table(&table, source_vm_list, source_nelems, + dest_vmids, dest_perms, dest_nelems); + + sg_free_table(&table); + return ret; +} +EXPORT_SYMBOL(hyp_assign_phys); + +const char *msm_secure_vmid_to_string(int secure_vmid) +{ + switch (secure_vmid) { + case VMID_HLOS: + return "VMID_HLOS"; + case VMID_CP_TOUCH: + return "VMID_CP_TOUCH"; + case VMID_CP_BITSTREAM: + return "VMID_CP_BITSTREAM"; + case VMID_CP_PIXEL: + return "VMID_CP_PIXEL"; + case VMID_CP_NON_PIXEL: + return "VMID_CP_NON_PIXEL"; + case VMID_CP_CAMERA: + return "VMID_CP_CAMERA"; + case VMID_HLOS_FREE: + return "VMID_HLOS_FREE"; + case VMID_MSS_MSA: + return "VMID_MSS_MSA"; + case VMID_MSS_NONMSA: + return "VMID_MSS_NONMSA"; + case VMID_CP_SEC_DISPLAY: + return "VMID_CP_SEC_DISPLAY"; + case VMID_CP_APP: + return "VMID_CP_APP"; + case VMID_WLAN: + return "VMID_WLAN"; + case VMID_WLAN_CE: + return "VMID_WLAN_CE"; + case VMID_CP_CAMERA_PREVIEW: + return "VMID_CP_CAMERA_PREVIEW"; + case VMID_CP_SPSS_SP: + return "VMID_CP_SPSS_SP"; + case VMID_CP_SPSS_SP_SHARED: + return "VMID_CP_SPSS_SP_SHARED"; + case VMID_CP_SPSS_HLOS_SHARED: + return "VMID_CP_SPSS_HLOS_SHARED"; + case VMID_INVAL: + return "VMID_INVAL"; + default: + return "Unknown VMID"; + } +} +EXPORT_SYMBOL(msm_secure_vmid_to_string); + +u32 msm_secure_get_vmid_perms(u32 vmid) +{ + if (vmid == VMID_CP_SEC_DISPLAY) + return PERM_READ; + else if (vmid == VMID_CP_CDSP) + return PERM_READ | PERM_WRITE | PERM_EXEC; + else + return PERM_READ | PERM_WRITE; +} +EXPORT_SYMBOL(msm_secure_get_vmid_perms); + +static int qcom_secure_buffer_probe(struct platform_device *pdev) +{ + int ret = 0; + + qcom_secure_buffer_dev = &pdev->dev; + +#ifdef CONFIG_ARM64 + ret = dma_set_mask(qcom_secure_buffer_dev, DMA_BIT_MASK(64)); +#else + ret = dma_set_mask(qcom_secure_buffer_dev, DMA_BIT_MASK(32)); +#endif + return ret; +} + +static const struct of_device_id qcom_secure_buffer_of_match[] = { + {.compatible = "qcom,secure-buffer"}, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_secure_buffer_of_match); + +static struct platform_driver qcom_secure_buffer_driver = { + .probe = qcom_secure_buffer_probe, + .driver = { + .name = "qcom_secure_buffer", + .of_match_table = qcom_secure_buffer_of_match, + }, +}; +module_platform_driver(qcom_secure_buffer_driver); + +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index 37258c8b2063..616acc3fdfd9 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -14,6 +14,8 @@ #include #include +struct iova_domain; + int iommu_dma_init(void); /* Domain management interface for IOMMU drivers */ @@ -45,6 +47,11 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, int prot); int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, int prot); +size_t iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad, + struct scatterlist *sg, int nents); +int iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, + int nents, dma_addr_t dma_addr); +void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents); /* * Arch code with no special attribute handling may use these diff --git a/include/linux/dma-mapping-fast.h b/include/linux/dma-mapping-fast.h new file mode 100644 index 000000000000..b2fed2a349ef --- /dev/null +++ b/include/linux/dma-mapping-fast.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + */ + +#ifndef __LINUX_DMA_MAPPING_FAST_H +#define __LINUX_DMA_MAPPING_FAST_H + +#include +#include + +struct dma_iommu_mapping; +struct io_pgtable_ops; +struct iova_domain; + +struct dma_fast_smmu_mapping { + struct device *dev; + struct iommu_domain *domain; + struct iova_domain *iovad; + + dma_addr_t base; + size_t size; + size_t num_4k_pages; + + unsigned int bitmap_size; + unsigned long *bitmap; + unsigned long next_start; + unsigned long upcoming_stale_bit; + bool have_stale_tlbs; + + dma_addr_t pgtbl_dma_handle; + struct io_pgtable_ops *pgtbl_ops; + + spinlock_t lock; + struct notifier_block notifier; +}; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST +int fast_smmu_init_mapping(struct device *dev, struct iommu_domain *domain, + struct io_pgtable_ops *pgtable_ops); +void fast_smmu_put_dma_cookie(struct iommu_domain *domain); +const struct dma_map_ops *fast_smmu_get_dma_ops(void); +#else +static inline int fast_smmu_init_mapping(struct device *dev, + struct iommu_domain *domain, + struct io_pgtable_ops *pgtable_ops) +{ + return -ENODEV; +} + +static inline void fast_smmu_put_dma_cookie(struct iommu_domain *domain) {} +static __maybe_unused const struct dma_map_ops *fast_smmu_get_dma_ops(void) +{ + return NULL; +} +#endif + +#endif /* __LINUX_DMA_MAPPING_FAST_H */ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 6309a721394b..29b4ab683d39 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -70,6 +70,52 @@ */ #define DMA_ATTR_PRIVILEGED (1UL << 9) +/* + * DMA_ATTR_SKIP_ZEROING: Do not zero mapping. + */ +#define DMA_ATTR_SKIP_ZEROING (1UL << 10) +/* + * DMA_ATTR_NO_DELAYED_UNMAP: Used by msm specific lazy mapping to indicate + * that the mapping can be freed on unmap, rather than when the ion_buffer + * is freed. + */ +#define DMA_ATTR_NO_DELAYED_UNMAP (1UL << 11) +/* + * DMA_ATTR_EXEC_MAPPING: The mapping has executable permissions. + */ +#define DMA_ATTR_EXEC_MAPPING (1UL << 12) +/* + * DMA_ATTR_IOMMU_USE_UPSTREAM_HINT: Normally an smmu will override any bus + * attributes (i.e cacheablilty) provided by the client device. Some hardware + * may be designed to use the original attributes instead. + */ +#define DMA_ATTR_IOMMU_USE_UPSTREAM_HINT (1UL << 13) +/* + * When passed to a DMA map call the DMA_ATTR_FORCE_COHERENT DMA + * attribute can be used to force a buffer to be mapped as IO coherent. + */ +#define DMA_ATTR_FORCE_COHERENT (1UL << 14) +/* + * When passed to a DMA map call the DMA_ATTR_FORCE_NON_COHERENT DMA + * attribute can be used to force a buffer to not be mapped as IO + * coherent. + */ +#define DMA_ATTR_FORCE_NON_COHERENT (1UL << 15) +/* + * DMA_ATTR_DELAYED_UNMAP: Used by ION, it will ensure that mappings are not + * removed on unmap but instead are removed when the ion_buffer is freed. + */ +#define DMA_ATTR_DELAYED_UNMAP (1UL << 16) + +/* + * DMA_ATTR_IOMMU_USE_LLC_NWA: Overrides the bus attributes to use the System + * Cache(LLC) with allocation policy as Inner Non-Cacheable, Outer Cacheable: + * Write-Back, Read-Allocate, No Write-Allocate policy. + */ +#define DMA_ATTR_IOMMU_USE_LLC_NWA (1UL << 17) + +#define DMA_ERROR_CODE (~(dma_addr_t)0) + /* * A dma_addr_t can hold any valid DMA or bus address for the platform. * It can be given to a device to use as a DMA source or target. A CPU cannot diff --git a/include/linux/io-pgtable-fast.h b/include/linux/io-pgtable-fast.h new file mode 100644 index 000000000000..a5559c2c42da --- /dev/null +++ b/include/linux/io-pgtable-fast.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + */ + +#ifndef __LINUX_IO_PGTABLE_FAST_H +#define __LINUX_IO_PGTABLE_FAST_H + +#include + +/* + * This ought to be private to io-pgtable-fast, but dma-mapping-fast + * currently requires it for a debug usecase. + */ +typedef u64 av8l_fast_iopte; + +struct io_pgtable_ops; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + +int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot); + +void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova, + size_t size); + +int av8l_fast_map_sg_public(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, size_t *size); + +bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops, + unsigned long iova); + +phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, + unsigned long iova); +#else +static inline int +av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + return -EINVAL; +} +static inline void av8l_fast_unmap_public(struct io_pgtable_ops *ops, + unsigned long iova, size_t size) +{ +} + +static inline int av8l_fast_map_sg_public(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, size_t *size) +{ + return 0; +} + +static inline bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return false; +} +static inline phys_addr_t +av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return 0; +} +#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST */ + + +/* events for notifiers passed to av8l_register_notify */ +#define MAPPED_OVER_STALE_TLB 1 + + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB +/* + * Doesn't matter what we use as long as bit 0 is unset. The reason why we + * need a different value at all is that there are certain hardware + * platforms with erratum that require that a PTE actually be zero'd out + * and not just have its valid bit unset. + */ +#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0xa + +void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, bool skip_sync); +void av8l_register_notify(struct notifier_block *nb); + +#else /* !CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */ + +#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0 + +static inline void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, + bool skip_sync) +{ +} + +static inline void av8l_register_notify(struct notifier_block *nb) +{ +} + +#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */ + +#endif /* __LINUX_IO_PGTABLE_FAST_H */ diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 76969a564831..3c58e9c765b5 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -3,6 +3,13 @@ #define __IO_PGTABLE_H #include +#include + +#define to_msm_iommu_gather_ops(_tlb_ops) \ + container_of(_tlb_ops, struct msm_iommu_gather_ops, tlb_ops) +#define to_msm_io_pgtable_info(_cfg) \ + container_of(_cfg, struct msm_io_pgtable_info, pgtbl_cfg) + /* * Public API for use by IOMMU drivers */ @@ -13,6 +20,9 @@ enum io_pgtable_fmt { ARM_64_LPAE_S2, ARM_V7S, ARM_MALI_LPAE, +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + ARM_V8L_FAST, +#endif IO_PGTABLE_NUM_FMTS, }; @@ -35,6 +45,21 @@ struct iommu_gather_ops { void (*tlb_sync)(void *cookie); }; +/** + * struct msm_iommu_gather_ops - MSM and standard IOMMU callbacks for TLB and + * page table management. + * + * @alloc_pages_exact: Allocate page table memory (optional, defaults to + * alloc_pages_exact) + * @free_pages_exact: Free page table memory (optional, defaults to + * free_pages_exact) + */ +struct msm_iommu_gather_ops { + void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask); + void (*free_pages_exact)(void *cookie, void *virt, size_t size); + const struct iommu_gather_ops tlb_ops; +}; + /** * struct io_pgtable_cfg - Configuration data for a set of page tables. * @@ -68,6 +93,7 @@ struct io_pgtable_cfg { * when the SoC is in "4GB mode" and they can only access the high * remap of DRAM (0x1_00000000 to 0x1_ffffffff). * + * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a * software-emulated IOMMU), such that pagetable updates need not @@ -76,6 +102,15 @@ struct io_pgtable_cfg { * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs * on unmap, for DMA domains using the flush queue mechanism for * delayed invalidation. + + * IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT: Override the attributes + * set in TCR for the page table walker. Use attributes specified + * by the upstream hw instead. + * + * IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA: Override the attributes + * set in TCR for the page table walker with Write-Back, + * no Write-Allocate cacheable encoding. + * */ #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) @@ -83,6 +118,8 @@ struct io_pgtable_cfg { #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) #define IO_PGTABLE_QUIRK_NO_DMA BIT(4) #define IO_PGTABLE_QUIRK_NON_STRICT BIT(5) + #define IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT BIT(6) + #define IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA BIT(7) unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; @@ -114,6 +151,15 @@ struct io_pgtable_cfg { u64 transtab; u64 memattr; } arm_mali_lpae_cfg; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + struct { + u64 ttbr[2]; + u64 tcr; + u64 mair[2]; + void *pmds; + } av8l_fast_cfg; +#endif }; }; @@ -136,6 +182,28 @@ struct io_pgtable_ops { unsigned long iova); }; +/** + * struct msm_io_pgtable_info - MSM specific page table manipulation API for + * IOMMU drivers, and page table configuration. + * + * @map_sg: Map a scatterlist. Returns the number of bytes mapped, + * or 0 on failure. The size parameter contains the size + * of the partial mapping in case of failure. + * @is_iova_coherent: Checks coherency of given IOVA. Returns True if coherent + * and False if non-coherent. + * @iova_to_pte: Translate iova to Page Table Entry (PTE). + * @pgtbl_cfg: The configuration for a set of page tables. + */ +struct msm_io_pgtable_info { + int (*map_sg)(struct io_pgtable_ops *ops, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot, + size_t *size); + bool (*is_iova_coherent)(struct io_pgtable_ops *ops, + unsigned long iova); + uint64_t (*iova_to_pte)(struct io_pgtable_ops *ops, unsigned long iova); + struct io_pgtable_cfg pgtbl_cfg; +}; + /** * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. * @@ -184,17 +252,23 @@ struct io_pgtable { static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) { + if (!iop->cfg.tlb) + return; iop->cfg.tlb->tlb_flush_all(iop->cookie); } static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, unsigned long iova, size_t size, size_t granule, bool leaf) { + if (!iop->cfg.tlb) + return; iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); } static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) { + if (!iop->cfg.tlb) + return; iop->cfg.tlb->tlb_sync(iop->cookie); } @@ -216,5 +290,32 @@ extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns; +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST +extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns; +#endif + +/** + * io_pgtable_alloc_pages_exact: + * allocate an exact number of physically-contiguous pages. + * @size: the number of bytes to allocate + * @gfp_mask: GFP flags for the allocation + * + * Like alloc_pages_exact(), but with some additional accounting for debug + * purposes. + */ +void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + size_t size, gfp_t gfp_mask); + +/** + * io_pgtable_free_pages_exact: + * release memory allocated via io_pgtable_alloc_pages_exact() + * @virt: the value returned by alloc_pages_exact. + * @size: size of allocation, same value as passed to alloc_pages_exact(). + * + * Like free_pages_exact(), but with some additional accounting for debug + * purposes. + */ +void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie, + void *virt, size_t size); #endif /* __IO_PGTABLE_H */ diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e552c3b63f6f..e64ce6d3bca3 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -29,6 +29,11 @@ * if the IOMMU page table format is equivalent. */ #define IOMMU_PRIV (1 << 5) +/* Use upstream device's bus attribute */ +#define IOMMU_USE_UPSTREAM_HINT (1 << 6) + +/* Use upstream device's bus attribute with no write-allocate cache policy */ +#define IOMMU_USE_LLC_NWA (1 << 7) struct iommu_ops; struct iommu_group; @@ -39,8 +44,12 @@ struct notifier_block; struct iommu_sva; /* iommu fault flags */ -#define IOMMU_FAULT_READ 0x0 -#define IOMMU_FAULT_WRITE 0x1 +#define IOMMU_FAULT_READ (1 << 0) +#define IOMMU_FAULT_WRITE (1 << 1) +#define IOMMU_FAULT_TRANSLATION (1 << 2) +#define IOMMU_FAULT_PERMISSION (1 << 3) +#define IOMMU_FAULT_EXTERNAL (1 << 4) +#define IOMMU_FAULT_TRANSACTION_STALLED (1 << 5) typedef int (*iommu_fault_handler_t)(struct iommu_domain *, struct device *, unsigned long, int, void *); @@ -53,6 +62,10 @@ struct iommu_domain_geometry { bool force_aperture; /* DMA only allowed in mappable range? */ }; +struct iommu_pgtbl_info { + void *ops; +}; + /* Domain feature flags */ #define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */ #define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API @@ -77,6 +90,12 @@ struct iommu_domain_geometry { #define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \ __IOMMU_DOMAIN_DMA_API) +#define to_msm_iommu_ops(_iommu_ops) \ + container_of(_iommu_ops, struct msm_iommu_ops, iommu_ops) +#define to_msm_iommu_domain(_iommu_domain) \ + container_of(_iommu_domain, struct msm_iommu_domain, iommu_domain) + +#define IOMMU_DOMAIN_NAME_LEN 32 struct iommu_domain { unsigned type; const struct iommu_ops *ops; @@ -87,6 +106,11 @@ struct iommu_domain { void *iova_cookie; }; +struct msm_iommu_domain { + char name[IOMMU_DOMAIN_NAME_LEN]; + struct iommu_domain iommu_domain; +}; + enum iommu_cap { IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA transactions */ @@ -105,6 +129,11 @@ enum iommu_cap { * DOMAIN_ATTR_FSL_PAMUV1 corresponds to the above mentioned contraints. * The caller can invoke iommu_domain_get_attr to check if the underlying * iommu implementation supports these constraints. + * + * DOMAIN_ATTR_NO_CFRE + * Some bus implementations may enter a bad state if iommu reports an error + * on context fault. As context faults are not always fatal, this must be + * avoided. */ enum iommu_attr { @@ -119,6 +148,28 @@ enum iommu_attr { DOMAIN_ATTR_MAX, }; +#define DOMAIN_ATTR_PT_BASE_ADDR (DOMAIN_ATTR_MAX + 1) +#define DOMAIN_ATTR_CONTEXT_BANK (DOMAIN_ATTR_MAX + 2) +#define DOMAIN_ATTR_DYNAMIC (DOMAIN_ATTR_MAX + 3) +#define DOMAIN_ATTR_TTBR0 (DOMAIN_ATTR_MAX + 4) +#define DOMAIN_ATTR_CONTEXTIDR (DOMAIN_ATTR_MAX + 5) +#define DOMAIN_ATTR_PROCID (DOMAIN_ATTR_MAX + 6) +#define DOMAIN_ATTR_NON_FATAL_FAULTS (DOMAIN_ATTR_MAX + 7) +#define DOMAIN_ATTR_S1_BYPASS (DOMAIN_ATTR_MAX + 8) +#define DOMAIN_ATTR_ATOMIC (DOMAIN_ATTR_MAX + 9) +#define DOMAIN_ATTR_SECURE_VMID (DOMAIN_ATTR_MAX + 10) +#define DOMAIN_ATTR_FAST (DOMAIN_ATTR_MAX + 11) +#define DOMAIN_ATTR_PGTBL_INFO (DOMAIN_ATTR_MAX + 12) +#define DOMAIN_ATTR_USE_UPSTREAM_HINT (DOMAIN_ATTR_MAX + 13) +#define DOMAIN_ATTR_EARLY_MAP (DOMAIN_ATTR_MAX + 14) +#define DOMAIN_ATTR_PAGE_TABLE_IS_COHERENT (DOMAIN_ATTR_MAX + 15) +#define DOMAIN_ATTR_PAGE_TABLE_FORCE_COHERENT (DOMAIN_ATTR_MAX + 16) +#define DOMAIN_ATTR_CB_STALL_DISABLE (DOMAIN_ATTR_MAX + 17) +#define DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR (DOMAIN_ATTR_MAX + 18) +#define DOMAIN_ATTR_USE_LLC_NWA (DOMAIN_ATTR_MAX + 19) +#define DOMAIN_ATTR_NO_CFRE (DOMAIN_ATTR_MAX + 20) +#define DOMAIN_ATTR_DEBUG (DOMAIN_ATTR_MAX + 21) + /* These are the possible reserved region types */ enum iommu_resv_type { /* Memory regions which must be mapped 1:1 at all times */ @@ -147,6 +198,7 @@ struct iommu_resv_region { enum iommu_resv_type type; }; +extern struct dentry *iommu_debugfs_top; /* Per device IOMMU features */ enum iommu_dev_features { IOMMU_DEV_FEAT_AUX, /* Aux-domain feature */ @@ -275,6 +327,34 @@ struct iommu_ops { unsigned long pgsize_bitmap; }; +/** + * struct msm_iommu_ops - standard iommu ops, as well as additional MSM + * specific iommu ops + * @map_sg: map a scatter-gather list of physically contiguous memory chunks + * to an iommu domain + * @iova_to_phys_hard: translate iova to physical address using IOMMU hardware + * @is_iova_coherent: checks coherency of the given iova + * @trigger_fault: trigger a fault on the device attached to an iommu domain + * @tlbi_domain: Invalidate all TLBs covering an iommu domain + * @enable_config_clocks: Enable all config clocks for this domain's IOMMU + * @disable_config_clocks: Disable all config clocks for this domain's IOMMU + * @iova_to_pte: translate iova to Page Table Entry (PTE). + * @iommu_ops: the standard iommu ops + */ +struct msm_iommu_ops { + size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot); + phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain, + dma_addr_t iova); + bool (*is_iova_coherent)(struct iommu_domain *domain, dma_addr_t iova); + void (*trigger_fault)(struct iommu_domain *domain, unsigned long flags); + void (*tlbi_domain)(struct iommu_domain *domain); + int (*enable_config_clocks)(struct iommu_domain *domain); + void (*disable_config_clocks)(struct iommu_domain *domain); + uint64_t (*iova_to_pte)(struct iommu_domain *domain, dma_addr_t iova); + struct iommu_ops iommu_ops; +}; + /** * struct iommu_device - IOMMU core representation of one IOMMU hardware * instance @@ -334,6 +414,8 @@ extern int iommu_attach_device(struct iommu_domain *domain, extern void iommu_detach_device(struct iommu_domain *domain, struct device *dev); extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); +extern size_t iommu_pgsize(unsigned long pgsize_bitmap, + unsigned long addr_merge, size_t size); extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); extern int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot); @@ -343,7 +425,14 @@ extern size_t iommu_unmap_fast(struct iommu_domain *domain, unsigned long iova, size_t size); extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, struct scatterlist *sg,unsigned int nents, int prot); +extern size_t default_iommu_map_sg(struct iommu_domain *domain, + unsigned long iova, struct scatterlist *sg, + unsigned int nents, int prot); extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); +extern phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain, + dma_addr_t iova); +extern bool iommu_is_iova_coherent(struct iommu_domain *domain, + dma_addr_t iova); extern void iommu_set_fault_handler(struct iommu_domain *domain, iommu_fault_handler_t handler, void *token); @@ -393,6 +482,9 @@ extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, int prot); extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr); +extern uint64_t iommu_iova_to_pte(struct iommu_domain *domain, + dma_addr_t iova); + extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev, unsigned long iova, int flags); @@ -415,6 +507,9 @@ static inline void iommu_tlb_sync(struct iommu_domain *domain) domain->ops->iotlb_sync(domain); } +extern void iommu_trigger_fault(struct iommu_domain *domain, + unsigned long flags); + /* PCI device grouping function */ extern struct iommu_group *pci_device_group(struct device *dev); /* Generic device grouping function */ @@ -422,6 +517,31 @@ extern struct iommu_group *generic_device_group(struct device *dev); /* FSL-MC device grouping function */ struct iommu_group *fsl_mc_device_group(struct device *dev); +static inline void iommu_tlbiall(struct iommu_domain *domain) +{ + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (ops->tlbi_domain) + ops->tlbi_domain(domain); +} + +static inline int iommu_enable_config_clocks(struct iommu_domain *domain) +{ + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (ops->enable_config_clocks) + return ops->enable_config_clocks(domain); + return 0; +} + +static inline void iommu_disable_config_clocks(struct iommu_domain *domain) +{ + struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); + + if (ops->disable_config_clocks) + ops->disable_config_clocks(domain); +} + /** * struct iommu_fwspec - per-device IOMMU instance data * @ops: ops for this device's IOMMU @@ -588,6 +708,18 @@ static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_ad return 0; } +static inline phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain, + dma_addr_t iova) +{ + return 0; +} + +static inline bool iommu_is_iova_coherent(struct iommu_domain *domain, + dma_addr_t iova) +{ + return false; +} + static inline void iommu_set_fault_handler(struct iommu_domain *domain, iommu_fault_handler_t handler, void *token) { @@ -747,6 +879,24 @@ static inline void iommu_device_unlink(struct device *dev, struct device *link) { } +static inline void iommu_trigger_fault(struct iommu_domain *domain, + unsigned long flags) +{ +} + +static inline void iommu_tlbiall(struct iommu_domain *domain) +{ +} + +static inline int iommu_enable_config_clocks(struct iommu_domain *domain) +{ + return 0; +} + +static inline void iommu_disable_config_clocks(struct iommu_domain *domain) +{ +} + static inline int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops) diff --git a/include/linux/msm_dma_iommu_mapping.h b/include/linux/msm_dma_iommu_mapping.h new file mode 100644 index 000000000000..6a0c89292f4b --- /dev/null +++ b/include/linux/msm_dma_iommu_mapping.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015-2016, 2018 The Linux Foundation. All rights reserved. + */ + +#ifndef _LINUX_MSM_DMA_IOMMU_MAPPING_H +#define _LINUX_MSM_DMA_IOMMU_MAPPING_H + +#include +#include +#include +#include + +#if IS_ENABLED(CONFIG_QCOM_LAZY_MAPPING) +/* + * This function is not taking a reference to the dma_buf here. It is expected + * that clients hold reference to the dma_buf until they are done with mapping + * and unmapping. + */ +int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_buf *dma_buf, + unsigned long attrs); + +/* + * This function takes an extra reference to the dma_buf. + * What this means is that calling msm_dma_unmap_sg will not result in buffer's + * iommu mapping being removed, which means that subsequent calls to lazy map + * will simply re-use the existing iommu mapping. + * The iommu unmapping of the buffer will occur when the ION buffer is + * destroyed. + * Using lazy mapping can provide a performance benefit because subsequent + * mappings are faster. + * + * The limitation of using this API are that all subsequent iommu mappings + * must be the same as the original mapping, ie they must map the same part of + * the buffer with the same dma data direction. Also there can't be multiple + * mappings of different parts of the buffer. + */ +static inline int msm_dma_map_sg_lazy(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, 0); +} + +static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + unsigned long attrs; + + attrs = DMA_ATTR_NO_DELAYED_UNMAP; + return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, attrs); +} + +void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs); + +int msm_dma_unmap_all_for_dev(struct device *dev); + +/* + * Below is private function only to be called by framework (ION) and not by + * clients. + */ +void msm_dma_buf_freed(void *buffer); + +#else /*CONFIG_QCOM_LAZY_MAPPING*/ + +static inline int msm_dma_map_sg_attrs(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_buf *dma_buf, + unsigned long attrs) +{ + return -EINVAL; +} + +static inline void +msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ +} + +static inline int msm_dma_map_sg_lazy(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + return -EINVAL; +} + +static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + return -EINVAL; +} + +static inline int msm_dma_unmap_all_for_dev(struct device *dev) +{ + return 0; +} + +static inline void msm_dma_buf_freed(void *buffer) {} +#endif /*CONFIG_QCOM_LAZY_MAPPING*/ + +#endif diff --git a/include/soc/qcom/secure_buffer.h b/include/soc/qcom/secure_buffer.h new file mode 100644 index 000000000000..584a0b8814d8 --- /dev/null +++ b/include/soc/qcom/secure_buffer.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved. + */ + +#ifndef __QCOM_SECURE_BUFFER_H__ +#define __QCOM_SECURE_BUFFER_H__ + +#include + +/* + * if you add a secure VMID here make sure you update + * msm_secure_vmid_to_string. + * Make sure to keep the VMID_LAST as the last entry in the enum. + * This is needed in ion to create a list and it's sized using VMID_LAST. + */ +enum vmid { + VMID_HLOS = 0x3, + VMID_CP_TOUCH = 0x8, + VMID_CP_BITSTREAM = 0x9, + VMID_CP_PIXEL = 0xA, + VMID_CP_NON_PIXEL = 0xB, + VMID_CP_CAMERA = 0xD, + VMID_HLOS_FREE = 0xE, + VMID_MSS_MSA = 0xF, + VMID_MSS_NONMSA = 0x10, + VMID_CP_SEC_DISPLAY = 0x11, + VMID_CP_APP = 0x12, + VMID_WLAN = 0x18, + VMID_WLAN_CE = 0x19, + VMID_CP_SPSS_SP = 0x1A, + VMID_CP_CAMERA_PREVIEW = 0x1D, + VMID_CP_SPSS_SP_SHARED = 0x22, + VMID_CP_SPSS_HLOS_SHARED = 0x24, + VMID_CP_CDSP = 0x2A, + VMID_LAST, + VMID_INVAL = -1 +}; + +#define PERM_READ 0x4 +#define PERM_WRITE 0x2 +#define PERM_EXEC 0x1 + +#if IS_ENABLED(CONFIG_QCOM_SECURE_BUFFER) +int hyp_assign_table(struct sg_table *table, + u32 *source_vm_list, int source_nelems, + int *dest_vmids, int *dest_perms, + int dest_nelems); +int hyp_assign_phys(phys_addr_t addr, u64 size, + u32 *source_vmlist, int source_nelems, + int *dest_vmids, int *dest_perms, int dest_nelems); +const char *msm_secure_vmid_to_string(int secure_vmid); +u32 msm_secure_get_vmid_perms(u32 vmid); +#else +static inline int hyp_assign_table(struct sg_table *table, + u32 *source_vm_list, int source_nelems, + int *dest_vmids, int *dest_perms, + int dest_nelems) +{ + return -EINVAL; +} + +static inline int hyp_assign_phys(phys_addr_t addr, u64 size, + u32 *source_vmlist, int source_nelems, + int *dest_vmids, int *dest_perms, int dest_nelems) +{ + return -EINVAL; +} + +static inline const char *msm_secure_vmid_to_string(int secure_vmid) +{ + return "N/A"; +} + +static inline u32 msm_secure_get_vmid_perms(u32 vmid) +{ + return 0; +} +#endif +#endif diff --git a/include/trace/events/iommu.h b/include/trace/events/iommu.h index 72b4582322ff..3c11f72c0fd0 100644 --- a/include/trace/events/iommu.h +++ b/include/trace/events/iommu.h @@ -12,8 +12,10 @@ #define _TRACE_IOMMU_H #include +#include struct device; +struct iommu_domain; DECLARE_EVENT_CLASS(iommu_group_event, @@ -85,47 +87,84 @@ DEFINE_EVENT(iommu_device_event, detach_device_from_domain, TRACE_EVENT(map, - TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size), + TP_PROTO(struct msm_iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot), - TP_ARGS(iova, paddr, size), + TP_ARGS(domain, iova, paddr, size, prot), TP_STRUCT__entry( + __string(name, domain->name) __field(u64, iova) __field(u64, paddr) __field(size_t, size) + __field(int, prot) ), TP_fast_assign( + __assign_str(name, domain->name); __entry->iova = iova; __entry->paddr = paddr; __entry->size = size; + __entry->prot = prot; ), - TP_printk("IOMMU: iova=0x%016llx paddr=0x%016llx size=%zu", - __entry->iova, __entry->paddr, __entry->size + TP_printk("IOMMU:%s iova=0x%016llx paddr=0x%016llx size=0x%zx prot=0x%x", + __get_str(name), __entry->iova, __entry->paddr, + __entry->size, __entry->prot ) ); TRACE_EVENT(unmap, - TP_PROTO(unsigned long iova, size_t size, size_t unmapped_size), + TP_PROTO(struct msm_iommu_domain *domain, unsigned long iova, + size_t size, size_t unmapped_size), - TP_ARGS(iova, size, unmapped_size), + TP_ARGS(domain, iova, size, unmapped_size), TP_STRUCT__entry( + __string(name, domain->name) __field(u64, iova) __field(size_t, size) __field(size_t, unmapped_size) ), TP_fast_assign( + __assign_str(name, domain->name); __entry->iova = iova; __entry->size = size; __entry->unmapped_size = unmapped_size; ), - TP_printk("IOMMU: iova=0x%016llx size=%zu unmapped_size=%zu", - __entry->iova, __entry->size, __entry->unmapped_size + TP_printk("IOMMU:%s iova=0x%016llx size=0x%zx unmapped_size=0x%zx", + __get_str(name), __entry->iova, __entry->size, + __entry->unmapped_size + ) +); + +TRACE_EVENT(map_sg, + + TP_PROTO(struct msm_iommu_domain *domain, unsigned long iova, + size_t size, int prot), + + TP_ARGS(domain, iova, size, prot), + + TP_STRUCT__entry( + __string(name, domain->name) + __field(u64, iova) + __field(size_t, size) + __field(int, prot) + ), + + TP_fast_assign( + __assign_str(name, domain->name); + __entry->iova = iova; + __entry->size = size; + __entry->prot = prot; + ), + + TP_printk("IOMMU:%s iova=0x%016llx size=0x%zx prot=0x%x", + __get_str(name), __entry->iova, __entry->size, + __entry->prot ) ); diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 2c2772e9702a..c3e7e070bd05 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -192,10 +192,22 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, __dma_direct_free_pages(dev, size, virt_to_page(cpu_addr)); } +static bool is_dma_coherent(struct device *dev, unsigned long attrs) +{ + if (attrs & DMA_ATTR_FORCE_COHERENT) + return true; + else if (attrs & DMA_ATTR_FORCE_NON_COHERENT) + return false; + else if (dev_is_dma_coherent(dev)) + return true; + else + return false; +} + void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { - if (!dev_is_dma_coherent(dev)) + if (!is_dma_coherent(dev, attrs)) return arch_dma_alloc(dev, size, dma_handle, gfp, attrs); return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); } @@ -203,7 +215,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - if (!dev_is_dma_coherent(dev)) + if (!is_dma_coherent(dev, attrs)) arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); else dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); @@ -286,7 +298,7 @@ void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, { phys_addr_t phys = dma_to_phys(dev, addr); - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + if (!is_dma_coherent(dev, attrs) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) dma_direct_sync_single_for_cpu(dev, addr, size, dir); if (unlikely(is_swiotlb_buffer(phys))) @@ -327,7 +339,7 @@ dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, return DMA_MAPPING_ERROR; } - if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + if (!is_dma_coherent(dev, attrs) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) arch_sync_dma_for_device(dev, phys, size, dir); return dma_addr; }