From 91723534b5c1fa5ac4b8531755265b1e40879348 Mon Sep 17 00:00:00 2001 From: Patrick Daly Date: Tue, 17 Sep 2019 17:25:42 -0700 Subject: [PATCH 1/2] dma-mapping-fast: Reorganize common behavior fast_smmu_mmap_attrs(), fast_smmu_get_sgtable(), and fast_smmu_free() share the need to classify an input address into one of several cases. Write them in the same general format. Change-Id: I237f76151a47491ba9ecf7c1701eed49de20cd7e Signed-off-by: Patrick Daly [isaacm@codeaurora.org: resolved merge conflicts/exported iommu_dma_mmap] Signed-off-by: Isaac J. Manjarres --- drivers/iommu/dma-iommu.c | 2 +- drivers/iommu/dma-mapping-fast.c | 75 ++++++++++++-------------------- include/linux/dma-iommu.h | 3 ++ 3 files changed, 33 insertions(+), 47 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index da9fac34334a..a3306c78139b 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1071,7 +1071,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, return cpu_addr; } -static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, +int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { diff --git a/drivers/iommu/dma-mapping-fast.c b/drivers/iommu/dma-mapping-fast.c index 77c301e63b6f..5b611162ab86 100644 --- a/drivers/iommu/dma-mapping-fast.c +++ b/drivers/iommu/dma-mapping-fast.c @@ -720,52 +720,42 @@ out_free_pages: } static void fast_smmu_free(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) { struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); struct vm_struct *area; - struct page **pages = NULL; - size_t count = ALIGN(size, SZ_4K) >> FAST_PAGE_SHIFT; unsigned long flags; - size = ALIGN(size, SZ_4K); + size = ALIGN(size, FAST_PAGE_SIZE); - if (fast_dma_in_atomic_pool(vaddr, size) || !is_vmalloc_addr(vaddr)) - goto no_remap; - - area = find_vm_area(vaddr); - if (WARN_ON_ONCE(!area)) - return; - - pages = area->pages; - dma_common_free_remap(vaddr, size); -no_remap: spin_lock_irqsave(&mapping->lock, flags); av8l_fast_unmap_public(mapping->pgtbl_ops, dma_handle, size); __fast_smmu_free_iova(mapping, dma_handle, size); spin_unlock_irqrestore(&mapping->lock, flags); - if (fast_dma_in_atomic_pool(vaddr, size)) - fast_dma_free_from_pool(vaddr, size); - else if (is_vmalloc_addr(vaddr)) - __fast_smmu_free_pages(pages, count); - else - __free_pages(virt_to_page(vaddr), get_order(size)); + + area = find_vm_area(cpu_addr); + if (area && area->pages) { + struct page **pages = area->pages; + + dma_common_free_remap(cpu_addr, size); + __fast_smmu_free_pages(pages, size >> FAST_PAGE_SHIFT); + } else if (!is_vmalloc_addr(cpu_addr)) { + __free_pages(virt_to_page(cpu_addr), get_order(size)); + } else if (fast_dma_in_atomic_pool(cpu_addr, size)) { + // Keep remap + fast_dma_free_from_pool(cpu_addr, size); + } } -static int __vma_remap_range(struct vm_area_struct *vma, void *cpu_addr, +/* __swiotlb_mmap_pfn is not currently exported. */ +static int fast_smmu_mmap_pfn(struct vm_area_struct *vma, unsigned long pfn, size_t size) { int ret = -ENXIO; unsigned long nr_vma_pages = vma_pages(vma); unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned long off = vma->vm_pgoff; - unsigned long pfn; - - if (fast_dma_in_atomic_pool(cpu_addr, size)) - pfn = fast_dma_get_phys(cpu_addr) >> PAGE_SHIFT; - else - pfn = page_to_pfn(virt_to_page(cpu_addr)); if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { ret = remap_pfn_range(vma, vma->vm_start, pfn + off, @@ -781,32 +771,25 @@ static int fast_smmu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, size_t size, unsigned long attrs) { struct vm_struct *area; - unsigned long uaddr = vma->vm_start; - struct page **pages; - int i, nr_pages, ret = 0; bool coherent = is_dma_coherent(dev, attrs); + unsigned long pfn = 0; vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, coherent); - - if (fast_dma_in_atomic_pool(cpu_addr, size) || - !is_vmalloc_addr(cpu_addr)) - return __vma_remap_range(vma, cpu_addr, size); - area = find_vm_area(cpu_addr); - if (!area) - return -EINVAL; + if (area && area->pages) + return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size, + attrs); + else if (!is_vmalloc_addr(cpu_addr)) + pfn = page_to_pfn(virt_to_page(cpu_addr)); + else if (fast_dma_in_atomic_pool(cpu_addr, size)) + pfn = fast_dma_get_phys(cpu_addr) >> PAGE_SHIFT; - pages = area->pages; - nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; - for (i = vma->vm_pgoff; i < nr_pages && uaddr < vma->vm_end; i++) { - ret = vm_insert_page(vma, uaddr, pages[i]); - if (ret) - break; - uaddr += PAGE_SIZE; - } - return ret; + if (pfn) + return fast_smmu_mmap_pfn(vma, pfn, size); + + return -EINVAL; } static int fast_smmu_get_sgtable(struct device *dev, struct sg_table *sgt, diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h index 5cfc5af6233c..b9247e3dbe09 100644 --- a/include/linux/dma-iommu.h +++ b/include/linux/dma-iommu.h @@ -27,6 +27,9 @@ size_t iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad, int iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents, dma_addr_t dma_addr); void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents); +int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); /* Setup call for arch DMA mapping code */ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size); From f3f78a96c709117bc805aba1e31e246649473774 Mon Sep 17 00:00:00 2001 From: Patrick Daly Date: Thu, 19 Sep 2019 15:32:00 -0700 Subject: [PATCH 2/2] dma-mapping-fast: Add support for DMA_ATTR_FORCE_CONTIGUOUS Allow devices with an iommu to allocate from a cma area if the DMA_ATTR_FORCE_CONTIGUOUS flag is given. Change-Id: Ifd8575d1cea57cbc13c602f6edb6ab389844b35d Signed-off-by: Patrick Daly [isaacm@codeaurora.org: resolved merge conflicts] Signed-off-by: Isaac J. Manjarres --- drivers/iommu/dma-mapping-fast.c | 66 ++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/dma-mapping-fast.c b/drivers/iommu/dma-mapping-fast.c index 5b611162ab86..f1a387055c4b 100644 --- a/drivers/iommu/dma-mapping-fast.c +++ b/drivers/iommu/dma-mapping-fast.c @@ -550,7 +550,6 @@ static void *fast_smmu_alloc_atomic(struct dma_fast_smmu_mapping *mapping, dma_addr_t dma_addr; int prot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); - size = ALIGN(size, FAST_PAGE_SIZE); if (coherent) { page = alloc_pages(gfp, get_order(size)); addr = page ? page_address(page) : NULL; @@ -613,6 +612,54 @@ static struct page **__fast_smmu_alloc_pages(unsigned int count, gfp_t gfp) return pages; } +static void *__fast_smmu_alloc_contiguous(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = dma_info_to_prot(DMA_BIDIRECTIONAL, is_coherent, attrs); + pgprot_t remap_prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent); + struct page *page; + dma_addr_t iova; + unsigned long flags; + void *coherent_addr; + + page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, + get_order(size), gfp & __GFP_NOWARN); + if (!page) + return NULL; + + + spin_lock_irqsave(&mapping->lock, flags); + iova = __fast_smmu_alloc_iova(mapping, attrs, size); + spin_unlock_irqrestore(&mapping->lock, flags); + if (iova == DMA_ERROR_CODE) + goto release_page; + + if (av8l_fast_map_public(mapping->pgtbl_ops, iova, page_to_phys(page), + size, prot)) + goto release_iova; + + coherent_addr = dma_common_contiguous_remap(page, size, remap_prot, + __fast_smmu_alloc_contiguous); + if (!coherent_addr) + goto release_mapping; + + if (!is_coherent) + __dma_flush_area(page_to_virt(page), size); + + *handle = iova; + return coherent_addr; + +release_mapping: + av8l_fast_unmap_public(mapping->pgtbl_ops, iova, size); +release_iova: + __fast_smmu_free_iova(mapping, iova, size); +release_page: + dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); + return NULL; +} + static void *fast_smmu_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, unsigned long attrs) @@ -640,11 +687,14 @@ static void *fast_smmu_alloc(struct device *dev, size_t size, } *handle = DMA_ERROR_CODE; + size = ALIGN(size, SZ_4K); - if (!gfpflags_allow_blocking(gfp)) { + if (!gfpflags_allow_blocking(gfp)) return fast_smmu_alloc_atomic(mapping, size, gfp, attrs, handle, is_coherent); - } + else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) + return __fast_smmu_alloc_contiguous(dev, size, handle, gfp, + attrs); pages = __fast_smmu_alloc_pages(count, gfp); if (!pages) { @@ -652,7 +702,6 @@ static void *fast_smmu_alloc(struct device *dev, size_t size, return NULL; } - size = ALIGN(size, SZ_4K); if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, gfp)) { dev_err(dev, "no sg tablen\n"); goto out_free_pages; @@ -740,6 +789,11 @@ static void fast_smmu_free(struct device *dev, size_t size, dma_common_free_remap(cpu_addr, size); __fast_smmu_free_pages(pages, size >> FAST_PAGE_SHIFT); + } else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { + struct page *page = vmalloc_to_page(cpu_addr); + + dma_common_free_remap(cpu_addr, size); + dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); } else if (!is_vmalloc_addr(cpu_addr)) { __free_pages(virt_to_page(cpu_addr), get_order(size)); } else if (fast_dma_in_atomic_pool(cpu_addr, size)) { @@ -780,6 +834,8 @@ static int fast_smmu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, if (area && area->pages) return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); + else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) + pfn = vmalloc_to_pfn(cpu_addr); else if (!is_vmalloc_addr(cpu_addr)) pfn = page_to_pfn(virt_to_page(cpu_addr)); else if (fast_dma_in_atomic_pool(cpu_addr, size)) @@ -805,6 +861,8 @@ static int fast_smmu_get_sgtable(struct device *dev, struct sg_table *sgt, if (area && area->pages) return sg_alloc_table_from_pages(sgt, area->pages, n_pages, 0, size, GFP_KERNEL); + else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) + page = vmalloc_to_page(cpu_addr); else if (!is_vmalloc_addr(cpu_addr)) page = virt_to_page(cpu_addr); else if (fast_dma_in_atomic_pool(cpu_addr, size))