From 253aacc2ad97d99bc39212076b95be5374d78b83 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Mon, 8 Jun 2020 23:29:20 -0700 Subject: [PATCH 1/3] ion: msm: Add support for dynamically preventing dma-buf mappings The current rules for determining if a buffer can or cannot be mapped to depend on how a buffer was allocated. This is restrictive in environments where the security state of a buffer can change dynamically, as the accessibility of the buffer has to change with respect to the security state of the buffer. Thus, add support to track the number of userspace mappings associated with an ION buffer, as well as an interface to allow drivers to lock and unlock a buffer. In this context, locking a buffer means that the buffer will no longer be mappable, and the buffer does not have any outstanding mappings at the time the buffer is locked. Unlocking the buffer means that the buffer can be mapped again. Change-Id: I6aa73b9ac7c301b12106ad3d3bcb4c2aac959e55 Signed-off-by: Isaac J. Manjarres --- .../android/ion/heaps/ion_carveout_heap.c | 30 +++- .../staging/android/ion/heaps/ion_cma_heap.c | 8 + .../android/ion/heaps/ion_secure_util.c | 4 + .../android/ion/heaps/ion_system_heap.c | 19 ++- .../android/ion/heaps/msm_ion_dma_buf.c | 152 ++++++++++++++---- .../staging/android/ion/heaps/msm_ion_priv.h | 5 + include/linux/msm_ion.h | 14 ++ 7 files changed, 201 insertions(+), 31 deletions(-) diff --git a/drivers/staging/android/ion/heaps/ion_carveout_heap.c b/drivers/staging/android/ion/heaps/ion_carveout_heap.c index 53347da25295..5b00e949c1a3 100644 --- a/drivers/staging/android/ion/heaps/ion_carveout_heap.c +++ b/drivers/staging/android/ion/heaps/ion_carveout_heap.c @@ -74,6 +74,7 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, phys_addr_t paddr; int ret; struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); + struct msm_ion_buf_lock_state *lock_state; struct device *dev = carveout_heap->heap.dev; table = kmalloc(sizeof(*table), GFP_KERNEL); @@ -83,10 +84,17 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, if (ret) goto err_free; + lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); + if (!lock_state) { + ret = -ENOMEM; + goto err_free_table; + } + buffer->priv_virt = lock_state; + paddr = ion_carveout_allocate(heap, size); if (paddr == ION_CARVEOUT_ALLOCATE_FAIL) { ret = -ENOMEM; - goto err_free_table; + goto err_free_umap; } sg_set_page(table->sgl, pfn_to_page(PFN_DOWN(paddr)), size, 0); @@ -99,6 +107,8 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, return 0; +err_free_umap: + kfree(lock_state); err_free_table: sg_free_table(table); err_free: @@ -110,18 +120,26 @@ static void ion_carveout_heap_free(struct ion_buffer *buffer) { struct ion_heap *heap = buffer->heap; struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; struct sg_table *table = buffer->sg_table; struct page *page = sg_page(table->sgl); phys_addr_t paddr = page_to_phys(page); struct device *dev = carveout_heap->heap.dev; - ion_buffer_zero(buffer); + mutex_lock(&buffer->lock); + if (hlos_accessible_buffer(buffer)) + ion_buffer_zero(buffer); + + if (lock_state && lock_state->locked) + pr_warn("%s: buffer is locked while being freed\n", __func__); + mutex_unlock(&buffer->lock); if (ion_buffer_cached(buffer)) ion_pages_sync_for_device(dev, page, buffer->size, DMA_BIDIRECTIONAL); ion_carveout_free(heap, paddr, buffer->size); + kfree(buffer->priv_virt); sg_free_table(table); kfree(table); } @@ -370,6 +388,7 @@ static void ion_sc_heap_free(struct ion_buffer *buffer) { struct ion_heap *child; struct sg_table *table = buffer->sg_table; + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; struct page *page = sg_page(table->sgl); phys_addr_t paddr = PFN_PHYS(page_to_pfn(page)); @@ -379,9 +398,16 @@ static void ion_sc_heap_free(struct ion_buffer *buffer) return; } + mutex_lock(&buffer->lock); if (hlos_accessible_buffer(buffer)) ion_buffer_zero(buffer); + + if (lock_state && lock_state->locked) + pr_warn("%s: buffer is locked while being freed\n", __func__); + mutex_unlock(&buffer->lock); + ion_carveout_free(child, paddr, buffer->size); + kfree(buffer->priv_virt); sg_free_table(table); kfree(table); } diff --git a/drivers/staging/android/ion/heaps/ion_cma_heap.c b/drivers/staging/android/ion/heaps/ion_cma_heap.c index 368a572f92a6..643cf9fa0553 100644 --- a/drivers/staging/android/ion/heaps/ion_cma_heap.c +++ b/drivers/staging/android/ion/heaps/ion_cma_heap.c @@ -40,6 +40,7 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, { struct ion_cma_heap *cma_heap = to_cma_heap(heap); struct sg_table *table; + struct msm_ion_buf_lock_state *lock_state; struct page *pages; unsigned long size = PAGE_ALIGN(len); unsigned long nr_pages = size >> PAGE_SHIFT; @@ -96,6 +97,12 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, sg_set_page(table->sgl, pages, size, 0); ; buffer->sg_table = table; + + lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); + if (!lock_state) + goto free_mem; + buffer->priv_virt = lock_state; + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; @@ -117,6 +124,7 @@ static void ion_cma_free(struct ion_buffer *buffer) /* release sg table */ sg_free_table(buffer->sg_table); kfree(buffer->sg_table); + kfree(buffer->priv_virt); } static struct ion_heap_ops ion_cma_ops = { diff --git a/drivers/staging/android/ion/heaps/ion_secure_util.c b/drivers/staging/android/ion/heaps/ion_secure_util.c index 88cc3a6e457f..6f0a6e6070f7 100644 --- a/drivers/staging/android/ion/heaps/ion_secure_util.c +++ b/drivers/staging/android/ion/heaps/ion_secure_util.c @@ -279,6 +279,8 @@ out: bool hlos_accessible_buffer(struct ion_buffer *buffer) { + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + if ((buffer->flags & ION_FLAG_SECURE) && !(buffer->flags & ION_FLAG_CP_HLOS) && !(buffer->flags & ION_FLAG_CP_SPSS_HLOS_SHARED)) @@ -287,6 +289,8 @@ bool hlos_accessible_buffer(struct ion_buffer *buffer) !(buffer->flags & ION_FLAG_CP_HLOS) && !(buffer->flags & ION_FLAG_CP_SPSS_HLOS_SHARED)) return false; + else if (lock_state && lock_state->locked) + return false; return true; } diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c index 333608288de6..afc283e9db66 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -269,6 +269,7 @@ static int ion_system_heap_allocate(struct ion_heap *heap, unsigned long flags) { struct ion_system_heap *sys_heap = to_system_heap(heap); + struct msm_ion_buf_lock_state *lock_state; struct sg_table *table; struct sg_table table_sync = {0}; struct scatterlist *sg; @@ -382,6 +383,14 @@ static int ion_system_heap_allocate(struct ion_heap *heap, buffer->sg_table = table; if (nents_sync) sg_free_table(&table_sync); + + lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); + if (!lock_state) { + ret = -ENOMEM; + goto err_free_sg2; + } + buffer->priv_virt = lock_state; + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; @@ -415,10 +424,11 @@ err: return ret; } -void ion_system_heap_free(struct ion_buffer *buffer) +static void ion_system_heap_free(struct ion_buffer *buffer) { struct ion_heap *heap = buffer->heap; struct ion_system_heap *sys_heap = to_system_heap(heap); + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; struct sg_table *table = buffer->sg_table; struct scatterlist *sg; int i; @@ -426,8 +436,14 @@ void ion_system_heap_free(struct ion_buffer *buffer) if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) && !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { + mutex_lock(&buffer->lock); if (hlos_accessible_buffer(buffer)) ion_buffer_zero(buffer); + + if (lock_state && lock_state->locked) + pr_warn("%s: buffer is locked while being freed\n", + __func__); + mutex_unlock(&buffer->lock); } else if (vmid > 0) { if (ion_hyp_unassign_sg(table, &vmid, 1, true)) return; @@ -438,6 +454,7 @@ void ion_system_heap_free(struct ion_buffer *buffer) get_order(sg->length)); sg_free_table(table); kfree(table); + kfree(buffer->priv_virt); } static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, diff --git a/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c b/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c index eb7402f1fa85..468e9b4b517c 100644 --- a/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c +++ b/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c @@ -160,6 +160,7 @@ static struct sg_table table = a->table; map_attrs = attachment->dma_map_attrs; + mutex_lock(&buffer->lock); if (!(buffer->flags & ION_FLAG_CACHED) || !hlos_accessible_buffer(buffer)) map_attrs |= DMA_ATTR_SKIP_CPU_SYNC; @@ -175,10 +176,10 @@ static struct sg_table !(buffer->flags & ION_FLAG_CACHED)) { pr_warn_ratelimited("dev:%s Cannot DMA map uncached buffer as IO-coherent attrs:0x%lx\n", dev_name(attachment->dev), map_attrs); + mutex_unlock(&buffer->lock); return ERR_PTR(-EINVAL); } - mutex_lock(&buffer->lock); if (map_attrs & DMA_ATTR_SKIP_CPU_SYNC) trace_ion_dma_map_cmo_skip(attachment->dev, ino, @@ -243,6 +244,7 @@ static void msm_ion_unmap_dma_buf(struct dma_buf_attachment *attachment, struct ion_dma_buf_attachment *a = attachment->priv; unsigned long ino = file_inode(attachment->dmabuf->file)->i_ino; + mutex_lock(&buffer->lock); map_attrs = attachment->dma_map_attrs; if (!(buffer->flags & ION_FLAG_CACHED) || !hlos_accessible_buffer(buffer)) @@ -253,7 +255,6 @@ static void msm_ion_unmap_dma_buf(struct dma_buf_attachment *attachment, dev_is_dma_coherent_hint_cached(attachment->dev)) map_attrs |= DMA_ATTR_FORCE_COHERENT; - mutex_lock(&buffer->lock); if (map_attrs & DMA_ATTR_SKIP_CPU_SYNC) trace_ion_dma_unmap_cmo_skip(attachment->dev, ino, @@ -297,31 +298,134 @@ void ion_pages_sync_for_device(struct device *dev, struct page *page, dma_sync_sg_for_device(dev, &sg, 1, dir); } +static void __msm_ion_vm_open(struct ion_buffer *buffer) +{ + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + + lock_state->vma_count++; +} + +static void msm_ion_vm_open(struct vm_area_struct *vma) +{ + struct ion_buffer *buffer = vma->vm_private_data; + + mutex_lock(&buffer->lock); + __msm_ion_vm_open(buffer); + mutex_unlock(&buffer->lock); +} + +static void msm_ion_vm_close(struct vm_area_struct *vma) +{ + struct ion_buffer *buffer = vma->vm_private_data; + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + + mutex_lock(&buffer->lock); + lock_state->vma_count--; + mutex_unlock(&buffer->lock); +} + +static const struct vm_operations_struct msm_ion_vma_ops = { + .open = msm_ion_vm_open, + .close = msm_ion_vm_close, +}; + static int msm_ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) { struct ion_buffer *buffer = dmabuf->priv; + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { pr_err_ratelimited("%s: this buffer cannot be mapped to userspace\n", __func__); + mutex_unlock(&buffer->lock); return -EINVAL; } if (!(buffer->flags & ION_FLAG_CACHED)) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - mutex_lock(&buffer->lock); /* now map it to userspace */ ret = ion_heap_map_user(buffer->heap, buffer, vma); - mutex_unlock(&buffer->lock); - if (ret) + if (ret) { pr_err("%s: failure mapping buffer to userspace\n", __func__); + } else if (lock_state) { + vma->vm_private_data = buffer; + vma->vm_ops = &msm_ion_vma_ops; + __msm_ion_vm_open(buffer); + } + + mutex_unlock(&buffer->lock); + return ret; +} + +static bool is_msm_ion_dma_buf(struct ion_buffer *buffer) +{ + return buffer->heap->buf_ops.attach == msm_ion_dma_buf_attach; +} + +int msm_ion_dma_buf_lock(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer; + struct msm_ion_buf_lock_state *lock_state; + int ret; + + if (!dmabuf) + return -EINVAL; + + buffer = dmabuf->priv; + lock_state = buffer->priv_virt; + + if ((!lock_state) || !is_msm_ion_dma_buf(buffer)) { + pr_err("%s: userspace map locking is not supported for this dma-buf\n", + __func__); + return -EINVAL; + } + + mutex_lock(&buffer->lock); + if (lock_state->locked) { + ret = -EINVAL; + pr_err("%s: buffer is already locked\n", __func__); + } else if (lock_state->vma_count) { + ret = -EBUSY; + } else { + ret = 0; + lock_state->locked = true; + } + mutex_unlock(&buffer->lock); return ret; } +EXPORT_SYMBOL(msm_ion_dma_buf_lock); + +void msm_ion_dma_buf_unlock(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer; + struct msm_ion_buf_lock_state *lock_state; + + if (!dmabuf) + return; + + buffer = dmabuf->priv; + lock_state = buffer->priv_virt; + + if (!lock_state || !is_msm_ion_dma_buf(buffer)) { + pr_err("%s: userspace map unlocking is not supported for this dma-buf\n", + __func__); + return; + } + + mutex_lock(&buffer->lock); + if (!lock_state->locked) + pr_warn("%s: buffer is already unlocked\n", __func__); + else + lock_state->locked = false; + mutex_unlock(&buffer->lock); +} +EXPORT_SYMBOL(msm_ion_dma_buf_unlock); static void msm_ion_dma_buf_release(struct dma_buf *dmabuf) { @@ -336,14 +440,13 @@ static void *msm_ion_dma_buf_vmap(struct dma_buf *dmabuf) struct ion_buffer *buffer = dmabuf->priv; void *vaddr = ERR_PTR(-EINVAL); - if (hlos_accessible_buffer(buffer)) { - mutex_lock(&buffer->lock); + mutex_lock(&buffer->lock); + if (hlos_accessible_buffer(buffer)) vaddr = msm_ion_buffer_kmap_get(buffer); - mutex_unlock(&buffer->lock); - } else { + else pr_warn_ratelimited("heap %s doesn't support map_kernel\n", buffer->heap->name); - } + mutex_unlock(&buffer->lock); return vaddr; } @@ -352,11 +455,10 @@ static void msm_ion_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) { struct ion_buffer *buffer = dmabuf->priv; - if (hlos_accessible_buffer(buffer)) { - mutex_lock(&buffer->lock); + mutex_lock(&buffer->lock); + if (hlos_accessible_buffer(buffer)) msm_ion_buffer_kmap_put(buffer); - mutex_unlock(&buffer->lock); - } + mutex_unlock(&buffer->lock); } static void *msm_ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) @@ -447,6 +549,7 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, unsigned long ino = file_inode(dmabuf->file)->i_ino; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_begin_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -461,7 +564,6 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); @@ -471,8 +573,6 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, trace_ion_begin_cpu_access_cmo_apply(dev, ino, true, true, direction); - - mutex_unlock(&buffer->lock); goto out; } @@ -491,8 +591,8 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, trace_ion_begin_cpu_access_cmo_apply(a->dev, ino, true, true, direction); } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } @@ -504,6 +604,7 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, unsigned long ino = file_inode(dmabuf->file)->i_ino; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_end_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -518,7 +619,6 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); struct sg_table *table = buffer->sg_table; @@ -528,7 +628,6 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, trace_ion_end_cpu_access_cmo_apply(dev, ino, true, true, direction); - mutex_unlock(&buffer->lock); goto out; } @@ -547,9 +646,9 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, trace_ion_end_cpu_access_cmo_apply(a->dev, ino, true, true, direction); } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } @@ -563,6 +662,7 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, unsigned long ino = file_inode(dmabuf->file)->i_ino; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_begin_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -577,7 +677,6 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); struct sg_table *table = buffer->sg_table; @@ -591,7 +690,6 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, else trace_ion_begin_cpu_access_cmo_skip(dev, ino, true, true, dir); - mutex_unlock(&buffer->lock); goto out; } @@ -618,9 +716,9 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, ret = tmp; } } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } @@ -635,6 +733,7 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_end_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -649,7 +748,6 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); struct sg_table *table = buffer->sg_table; @@ -665,8 +763,6 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, trace_ion_end_cpu_access_cmo_skip(dev, ino, true, true, direction); - - mutex_unlock(&buffer->lock); goto out; } @@ -695,9 +791,9 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, ret = tmp; } } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } diff --git a/drivers/staging/android/ion/heaps/msm_ion_priv.h b/drivers/staging/android/ion/heaps/msm_ion_priv.h index a46a78341733..1841ed5c12a2 100644 --- a/drivers/staging/android/ion/heaps/msm_ion_priv.h +++ b/drivers/staging/android/ion/heaps/msm_ion_priv.h @@ -123,6 +123,11 @@ struct msm_ion_heap { struct ion_heap ion_heap; }; +struct msm_ion_buf_lock_state { + bool locked; + int vma_count; +}; + /** * struct ion_platform_data - array of platform heaps passed from board file * @nr: number of structures in the array diff --git a/include/linux/msm_ion.h b/include/linux/msm_ion.h index 743840ace66b..d9b577e573c1 100644 --- a/include/linux/msm_ion.h +++ b/include/linux/msm_ion.h @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -44,6 +45,10 @@ int msm_ion_heap_add_memory(int heap_id, struct sg_table *sgt); int msm_ion_heap_remove_memory(int heap_id, struct sg_table *sgt); +int msm_ion_dma_buf_lock(struct dma_buf *dmabuf); + +void msm_ion_dma_buf_unlock(struct dma_buf *dmabuf); + #else static inline struct device *msm_ion_heap_device_by_id(int heap_id) @@ -96,5 +101,14 @@ static inline int msm_ion_heap_remove_memory(int heap_id, struct sg_table *sgt) return -ENODEV; } +static inline int msm_ion_dma_buf_lock(struct dma_buf *dmabuf) +{ + return -ENODEV; +} + +static inline void msm_ion_dma_buf_unlock(struct dma_buf *dmabuf) +{ +} + #endif /* CONFIG_ION_MSM_HEAPS */ #endif /* _MSM_ION_H */ From 50c067fc65889e2dc2a35259368e6d536e878caf Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Tue, 19 May 2020 16:00:50 -0700 Subject: [PATCH 2/3] soc: qcom: mem-buf: Add support for suppliers to export dma-bufs Currently, mem-buf suppliers can only provide a consumer with a memory buffer if the consumer requests for the memory buffer, in which case the supplier has no control over the contents of the buffer that is given to the consumer. This is not ideal in cases where the supplier virtual machine (VM) may have data that is of interest to the consumer, which can be shared using a memory buffer. Introduce a new IOCTL command which allows a client to specify a dma-buf that they would like to share with a consumer VM, as well as the access control rules that should be applied to the buffer. Change-Id: I1e4ec1205ad4e69039ec0ab26b4209512896a354 Signed-off-by: Isaac J. Manjarres --- drivers/soc/qcom/mem-buf.c | 362 +++++++++++++++++++++++++++++++++-- include/uapi/linux/mem-buf.h | 34 ++++ 2 files changed, 375 insertions(+), 21 deletions(-) diff --git a/drivers/soc/qcom/mem-buf.c b/drivers/soc/qcom/mem-buf.c index 5fc36b604c1c..2a918b070387 100644 --- a/drivers/soc/qcom/mem-buf.c +++ b/drivers/soc/qcom/mem-buf.c @@ -170,6 +170,26 @@ struct mem_buf_xfer_ion_mem { struct dma_buf_attachment *attachment; }; +/** + * struct mem_buf_export: Represents a dmabuf that has been exported to other + * VM(s). + * @dmabuf: The dmabuf that was exported to other VM(s) + * @attachment: The dma-buf attachment for @dmabuf + * @mem_sgt: The SG-Table for the dmabuf that was exported. + * @nr_vmids: The number of VMIDs that have access to the memory that was + * exported. + * @dst_vmids: The VMIDs of the VMs that have access to the buffer. + * @filp: A file structure that corresponds to the buffer that was exported. + */ +struct mem_buf_export { + struct dma_buf *dmabuf; + struct dma_buf_attachment *attachment; + struct sg_table *mem_sgt; + unsigned int nr_vmids; + int *dst_vmids; + struct file *filp; +}; + static int mem_buf_init_txn(struct mem_buf_txn *txn, void *resp_buf) { int ret; @@ -1441,32 +1461,56 @@ err_alloc_acl_list: } EXPORT_SYMBOL(mem_buf_alloc); -int mem_buf_get_fd(void *membuf_desc) +static int _mem_buf_get_fd(struct file *filp) { int fd; - struct mem_buf_desc *membuf = membuf_desc; - if (!membuf_desc) + if (!filp) return -EINVAL; fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) return fd; - fd_install(fd, membuf->filp); + fd_install(fd, filp); return fd; } + +int mem_buf_get_fd(void *membuf_desc) +{ + struct mem_buf_desc *membuf = membuf_desc; + + if (!membuf_desc) + return -EINVAL; + + return _mem_buf_get_fd(membuf->filp); +} EXPORT_SYMBOL(mem_buf_get_fd); +static int mem_buf_get_export_fd(struct mem_buf_export *export_buf) +{ + return _mem_buf_get_fd(export_buf->filp); +} + +static void _mem_buf_put(struct file *filp) +{ + fput(filp); +} + void mem_buf_put(void *membuf_desc) { struct mem_buf_desc *membuf = membuf_desc; if (membuf && membuf->filp) - fput(membuf->filp); + _mem_buf_put(membuf->filp); } EXPORT_SYMBOL(mem_buf_put); +static void mem_buf_export_put(struct mem_buf_export *export_buf) +{ + _mem_buf_put(export_buf->filp); +} + static bool is_mem_buf_file(struct file *filp) { return filp->f_op == &mem_buf_fops; @@ -1572,51 +1616,327 @@ out: return ret; } -static int validate_ioctl_arg(struct mem_buf_alloc_ioctl_arg *allocation) +union mem_buf_ioctl_arg { + struct mem_buf_alloc_ioctl_arg allocation; + struct mem_buf_export_ioctl_arg export; +}; + +static int validate_ioctl_arg(union mem_buf_ioctl_arg *arg, unsigned int cmd) { - if (!allocation->size || !allocation->nr_acl_entries || - !allocation->acl_list || - (allocation->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS) || - !is_valid_mem_type(allocation->src_mem_type) || - !is_valid_mem_type(allocation->dst_mem_type) || - allocation->reserved0 || allocation->reserved1 || - allocation->reserved2) + switch (cmd) { + case MEM_BUF_IOC_ALLOC: + { + struct mem_buf_alloc_ioctl_arg *allocation = &arg->allocation; + + if (!allocation->size || !allocation->nr_acl_entries || + !allocation->acl_list || + (allocation->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS) || + !is_valid_mem_type(allocation->src_mem_type) || + !is_valid_mem_type(allocation->dst_mem_type) || + allocation->reserved0 || allocation->reserved1 || + allocation->reserved2) + return -EINVAL; + break; + } + case MEM_BUF_IOC_EXPORT: + { + struct mem_buf_export_ioctl_arg *export = &arg->export; + + if (!export->nr_acl_entries || !export->acl_list || + export->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS || + export->reserved0 || export->reserved1 || export->reserved2) + return -EINVAL; + break; + } + default: return -EINVAL; + } return 0; } +static bool mem_buf_hlos_accessible(int *vmids, u32 nr_vmids) +{ + int i; + + if (!vmids || !nr_vmids) + return false; + + for (i = 0; i < nr_vmids; i++) + if (vmids[i] == VMID_HLOS) + return true; + + return false; +} + +static int mem_buf_export_release(struct inode *inode, struct file *filp) +{ + int ret; + struct mem_buf_export *export_buf = filp->private_data; + bool dma_buf_freeable = true; + + ret = mem_buf_unassign_mem(export_buf->mem_sgt, export_buf->dst_vmids, + export_buf->nr_vmids); + if (ret < 0) + dma_buf_freeable = false; + + if (!mem_buf_hlos_accessible(export_buf->dst_vmids, + export_buf->nr_vmids) && dma_buf_freeable) + msm_ion_dma_buf_unlock(export_buf->dmabuf); + + kfree(export_buf->dst_vmids); + if (dma_buf_freeable) { + dma_buf_unmap_attachment(export_buf->attachment, + export_buf->mem_sgt, + DMA_BIDIRECTIONAL); + dma_buf_detach(export_buf->dmabuf, export_buf->attachment); + dma_buf_put(export_buf->dmabuf); + } + kfree(export_buf); + return ret; +} + +static const struct file_operations mem_buf_export_fops = { + .release = mem_buf_export_release, +}; + +static int mem_buf_acl_to_vmid_perms_list(unsigned int nr_acl_entries, + const void __user *acl_entries, + int **dst_vmids, int **dst_perms) +{ + int ret, i, *vmids, *perms; + struct acl_entry entry; + + if (!nr_acl_entries || !acl_entries) + return -EINVAL; + + vmids = kmalloc_array(nr_acl_entries, sizeof(*vmids), GFP_KERNEL); + if (!vmids) + return -ENOMEM; + + perms = kmalloc_array(nr_acl_entries, sizeof(*perms), GFP_KERNEL); + if (!perms) { + kfree(vmids); + return -ENOMEM; + } + + for (i = 0; i < nr_acl_entries; i++) { + ret = copy_struct_from_user(&entry, sizeof(entry), + acl_entries + (sizeof(entry) * i), + sizeof(entry)); + if (ret < 0) + goto out; + + vmids[i] = mem_buf_vmid_to_vmid(entry.vmid); + perms[i] = mem_buf_perms_to_perms(entry.perms); + if (vmids[i] < 0 || perms[i] < 0) { + ret = -EINVAL; + goto out; + } + } + + *dst_vmids = vmids; + *dst_perms = perms; + return ret; + +out: + kfree(perms); + kfree(vmids); + return ret; +} + +static struct mem_buf_export *mem_buf_export_dma_buf(int dma_buf_fd, + unsigned int nr_acl_entries, + const void __user *acl_entries, + hh_memparcel_handle_t *memparcel_hdl) +{ + int ret; + struct mem_buf_export *export_buf; + struct dma_buf *dmabuf; + struct dma_buf_attachment *attachment; + struct sg_table *sgt; + int *dst_vmids, *dst_perms; + bool dma_buf_freeable = true; + struct file *filp; + unsigned long flags = 0; + + if (!nr_acl_entries || !acl_entries || !memparcel_hdl) + return ERR_PTR(-EINVAL); + + export_buf = kmalloc(sizeof(*export_buf), GFP_KERNEL); + if (!export_buf) + return ERR_PTR(-ENOMEM); + + dmabuf = dma_buf_get(dma_buf_fd); + if (IS_ERR(dmabuf)) { + pr_err_ratelimited("%s: dma_buf_get failed rc: %d\n", __func__, + PTR_ERR(dmabuf)); + ret = PTR_ERR(dmabuf); + goto err_dma_buf_get; + } + export_buf->dmabuf = dmabuf; + + ret = dma_buf_get_flags(dmabuf, &flags); + if (ret < 0) { + pr_err_ratelimited("%s: dma_buf_get_flags failed rc: %d\n", + __func__, ret); + goto err_dma_buf_attach; + } else if (!(flags & ION_FLAG_CACHED)) { + ret = -EINVAL; + pr_err_ratelimited("%s: only cached buffers can be exported\n", + __func__); + goto err_dma_buf_attach; + } else if (flags & (ION_FLAG_SECURE | ION_FLAGS_CP_MASK)) { + ret = -EINVAL; + pr_err_ratelimited("%s: only non-secure allocations can be exported\n", + __func__); + goto err_dma_buf_attach; + } + + attachment = dma_buf_attach(dmabuf, mem_buf_dev); + if (IS_ERR(attachment)) { + pr_err_ratelimited("%s: dma_buf_attach failed rc: %d\n", + __func__, PTR_ERR(attachment)); + ret = PTR_ERR(attachment); + goto err_dma_buf_attach; + } + export_buf->attachment = attachment; + + sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) { + pr_err_ratelimited("%s dma_buf_map_attachment failed rc: %d\n", + __func__, PTR_ERR(sgt)); + ret = PTR_ERR(sgt); + goto err_map_attachment; + } + export_buf->mem_sgt = sgt; + + ret = mem_buf_acl_to_vmid_perms_list(nr_acl_entries, acl_entries, + &dst_vmids, &dst_perms); + if (ret < 0) { + pr_err_ratelimited("%s failed to copy ACL rc: %d\n", __func__, + ret); + goto err_cpy_acl_entries; + } + export_buf->nr_vmids = nr_acl_entries; + export_buf->dst_vmids = dst_vmids; + + if (!mem_buf_hlos_accessible(dst_vmids, nr_acl_entries)) { + ret = msm_ion_dma_buf_lock(dmabuf); + if (ret < 0) { + pr_err_ratelimited("%s failed to lock buffer rc: %d\n", + __func__, ret); + goto err_lock_mem; + } + } + + ret = mem_buf_assign_mem(sgt, dst_vmids, dst_perms, nr_acl_entries); + if (ret < 0) { + if (ret == -EADDRNOTAVAIL) + dma_buf_freeable = false; + goto err_assign_mem; + } + + ret = mem_buf_retrieve_memparcel_hdl(sgt, dst_vmids, dst_perms, + nr_acl_entries, memparcel_hdl); + if (ret < 0) + goto err_retrieve_hdl; + + filp = anon_inode_getfile("membuf", &mem_buf_export_fops, export_buf, + O_RDWR); + if (IS_ERR(filp)) { + ret = PTR_ERR(filp); + goto err_retrieve_hdl; + } + export_buf->filp = filp; + + kfree(dst_perms); + return export_buf; + +err_retrieve_hdl: + if (mem_buf_unassign_mem(sgt, dst_vmids, nr_acl_entries) < 0) + dma_buf_freeable = false; +err_assign_mem: + if (!mem_buf_hlos_accessible(dst_vmids, nr_acl_entries) && + dma_buf_freeable) + msm_ion_dma_buf_unlock(dmabuf); +err_lock_mem: + kfree(dst_vmids); + kfree(dst_perms); +err_cpy_acl_entries: + if (dma_buf_freeable) + dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL); +err_map_attachment: + if (dma_buf_freeable) + dma_buf_detach(dmabuf, attachment); +err_dma_buf_attach: + if (dma_buf_freeable) + dma_buf_put(dmabuf); +err_dma_buf_get: + kfree(export_buf); + return ERR_PTR(ret); +} + static long mem_buf_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { int fd; unsigned int dir = _IOC_DIR(cmd); - struct mem_buf_alloc_ioctl_arg allocation; + union mem_buf_ioctl_arg ioctl_arg; - if (_IOC_SIZE(cmd) > sizeof(allocation)) + if (_IOC_SIZE(cmd) > sizeof(ioctl_arg)) return -EINVAL; - if (copy_from_user(&allocation, (void __user *)arg, _IOC_SIZE(cmd))) + if (copy_from_user(&ioctl_arg, (void __user *)arg, _IOC_SIZE(cmd))) return -EFAULT; - if (validate_ioctl_arg(&allocation) < 0) + if (validate_ioctl_arg(&ioctl_arg, cmd) < 0) return -EINVAL; if (!(dir & _IOC_WRITE)) - memset(&allocation, 0, sizeof(allocation)); + memset(&ioctl_arg, 0, sizeof(ioctl_arg)); switch (cmd) { case MEM_BUF_IOC_ALLOC: { + struct mem_buf_alloc_ioctl_arg *allocation = + &ioctl_arg.allocation; + if (!(mem_buf_capability & MEM_BUF_CAP_CONSUMER)) return -ENOTSUPP; - fd = mem_buf_alloc_fd(&allocation); + fd = mem_buf_alloc_fd(allocation); if (fd < 0) return fd; - allocation.mem_buf_fd = fd; + allocation->mem_buf_fd = fd; + break; + } + case MEM_BUF_IOC_EXPORT: + { + struct mem_buf_export_ioctl_arg *export = &ioctl_arg.export; + u32 ret_memparcel_hdl; + struct mem_buf_export *export_buf; + + if (!(mem_buf_capability & MEM_BUF_CAP_SUPPLIER)) + return -ENOTSUPP; + + export_buf = mem_buf_export_dma_buf(export->dma_buf_fd, + export->nr_acl_entries, + (const void __user *)export->acl_list, + &ret_memparcel_hdl); + if (IS_ERR(export_buf)) + return PTR_ERR(export_buf); + + fd = mem_buf_get_export_fd(export_buf); + if (fd < 0) { + mem_buf_export_put(export_buf); + return fd; + } + + export->export_fd = fd; + export->memparcel_hdl = ret_memparcel_hdl; break; } default: @@ -1624,7 +1944,7 @@ static long mem_buf_dev_ioctl(struct file *filp, unsigned int cmd, } if (dir & _IOC_READ) { - if (copy_to_user((void __user *)arg, &allocation, + if (copy_to_user((void __user *)arg, &ioctl_arg, _IOC_SIZE(cmd))) return -EFAULT; } diff --git a/include/uapi/linux/mem-buf.h b/include/uapi/linux/mem-buf.h index 025e1acd3bc8..9c9082fd3c1d 100644 --- a/include/uapi/linux/mem-buf.h +++ b/include/uapi/linux/mem-buf.h @@ -97,4 +97,38 @@ struct mem_buf_alloc_ioctl_arg { #define MEM_BUF_IOC_ALLOC _IOWR(MEM_BUF_IOC_MAGIC, 0,\ struct mem_buf_alloc_ioctl_arg) +/** + * struct mem_buf_export_ioctl_arg: An request to allocate memory from another + * VM to other VMs. + * @dma_buf_fd: The fd of the dma-buf that will be exported to another VM. + * @nr_acl_entries: The number of ACL entries in @acl_list. + * @acl_list: An array of structures, where each structure specifies a VMID + * and the access permissions that the VMID will have to the memory to be + * exported. + * @export_fd: An fd that corresponds to the buffer that was exported. This fd + * must be kept open until it is no longer required to export the memory to + * another VM. + * @memparcel_hdl: The handle associated with the memparcel that was created by + * granting access to the dma-buf for the VMIDs specified in @acl_list. + * + * Note: The buffer must not be mmap'ed by any process prior to invoking this + * IOCTL. The buffer must also be a cached buffer from a non-secure ION heap. + * + * All reserved fields must be zeroed out by the caller prior to invoking the + * export IOCTL command with this argument. + */ +struct mem_buf_export_ioctl_arg { + __u32 dma_buf_fd; + __u32 nr_acl_entries; + __u64 acl_list; + __u32 export_fd; + __u32 memparcel_hdl; + __u64 reserved0; + __u64 reserved1; + __u64 reserved2; +}; + +#define MEM_BUF_IOC_EXPORT _IOWR(MEM_BUF_IOC_MAGIC, 1,\ + struct mem_buf_export_ioctl_arg) + #endif /* _UAPI_LINUX_MEM_BUF_H */ From eb749ee5d9c89a75cb3b860263ab97e292ae2d14 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Thu, 21 May 2020 13:53:48 -0700 Subject: [PATCH 3/3] soc: qcom: mem-buf: Add support for consumers to import dma-bufs Supplier virtual machines (VMs) support exporting dma-bufs to a consumer VM. However, consumers do not support importing dma-bufs into their VMs, so add an IOCTL command to import a dma-buf into a consumer VM. The client must provide a memory parcel handle that corresponds to a dma-buf that has been shared with the consumer VM, as well as an access control list that is used for validating the access control rules for the buffer. Upon success, the client is given a dma-buf fd, which they can use to map the buffer and access it from both the CPU, and peripherals. Change-Id: I548e004e73543421b932430fbd9f84ad76658ef8 Signed-off-by: Isaac J. Manjarres --- drivers/soc/qcom/Makefile | 2 +- drivers/soc/qcom/mem-buf-private.h | 43 +++ drivers/soc/qcom/mem-buf.c | 138 +++++++++ drivers/soc/qcom/mem_buf_dma_buf.c | 461 +++++++++++++++++++++++++++++ include/uapi/linux/mem-buf.h | 40 ++- 5 files changed, 677 insertions(+), 7 deletions(-) create mode 100644 drivers/soc/qcom/mem-buf-private.h create mode 100644 drivers/soc/qcom/mem_buf_dma_buf.c diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index 67fa1bdeb884..a178e17b2219 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -42,7 +42,7 @@ obj-$(CONFIG_QCOM_LAHAINA_LLCC) += llcc-lahaina.o obj-$(CONFIG_QCOM_SHIMA_LLCC) += llcc-shima.o obj-$(CONFIG_QCOM_MINIDUMP) += msm_minidump.o minidump_log.o obj-$(CONFIG_QCOM_MEM_OFFLINE) += mem-offline.o -obj-$(CONFIG_QCOM_MEM_BUF) += mem-buf.o +obj-$(CONFIG_QCOM_MEM_BUF) += mem-buf.o mem_buf_dma_buf.o obj-$(CONFIG_QCOM_MEMORY_DUMP_V2) += memory_dump_v2.o obj-$(CONFIG_QCOM_DCC_V2) += dcc_v2.o obj-$(CONFIG_MSM_JTAGV8) += jtagv8.o jtagv8-etm.o diff --git a/drivers/soc/qcom/mem-buf-private.h b/drivers/soc/qcom/mem-buf-private.h new file mode 100644 index 000000000000..763db1d7b318 --- /dev/null +++ b/drivers/soc/qcom/mem-buf-private.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#ifndef MEM_BUF_PRIVATE_H +#define MEM_BUF_PRIVATE_H + +#include +#include +#include +#include + +/** + * strcut mem_buf_import: Represents a memory buffer that was imported from + * another VM. + * @memparcel_hdl: The handle associated with the memparcel that represents the + * memory that was imported from another VM. + * @size: The size of the buffer. + * @sgl_desc: The SG descriptor that represents the memory buffer. + * @dmabuf: The dma-buf that corresponds to the buffer. + * @kmap_cnt: The number of kernel mapping references associated with the buffer + * @vaddr: The virtual address for the buffer after it has been mapped into a + * contiguous range in the kernel virtual address space. + * @lock: protects accesses to attachments. + * @attachments: a list of attachments for the buffer. + */ +struct mem_buf_import { + hh_memparcel_handle_t memparcel_hdl; + size_t size; + struct hh_sgl_desc *sgl_desc; + struct dma_buf *dmabuf; + int kmap_cnt; + void *vaddr; + struct mutex lock; + struct list_head attachments; +}; + +void mem_buf_unimport_dma_buf(struct mem_buf_import *import_buf); + +extern const struct dma_buf_ops mem_buf_dma_buf_ops; +#endif + diff --git a/drivers/soc/qcom/mem-buf.c b/drivers/soc/qcom/mem-buf.c index 2a918b070387..35247e729f1f 100644 --- a/drivers/soc/qcom/mem-buf.c +++ b/drivers/soc/qcom/mem-buf.c @@ -27,6 +27,8 @@ #include #include +#include "mem-buf-private.h" + #define CREATE_TRACE_POINTS #include "trace-mem-buf.h" @@ -1492,6 +1494,11 @@ static int mem_buf_get_export_fd(struct mem_buf_export *export_buf) return _mem_buf_get_fd(export_buf->filp); } +static int mem_buf_get_import_fd(struct mem_buf_import *import_buf) +{ + return dma_buf_fd(import_buf->dmabuf, O_CLOEXEC); +} + static void _mem_buf_put(struct file *filp) { fput(filp); @@ -1511,6 +1518,11 @@ static void mem_buf_export_put(struct mem_buf_export *export_buf) _mem_buf_put(export_buf->filp); } +static void mem_buf_import_put(struct mem_buf_import *import_buf) +{ + dma_buf_put(import_buf->dmabuf); +} + static bool is_mem_buf_file(struct file *filp) { return filp->f_op == &mem_buf_fops; @@ -1619,6 +1631,7 @@ out: union mem_buf_ioctl_arg { struct mem_buf_alloc_ioctl_arg allocation; struct mem_buf_export_ioctl_arg export; + struct mem_buf_import_ioctl_arg import; }; static int validate_ioctl_arg(union mem_buf_ioctl_arg *arg, unsigned int cmd) @@ -1648,6 +1661,16 @@ static int validate_ioctl_arg(union mem_buf_ioctl_arg *arg, unsigned int cmd) return -EINVAL; break; } + case MEM_BUF_IOC_IMPORT: + { + struct mem_buf_import_ioctl_arg *import = &arg->import; + + if (!import->nr_acl_entries || !import->acl_list || + import->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS || + import->reserved0 || import->reserved1 || import->reserved2) + return -EINVAL; + break; + } default: return -EINVAL; } @@ -1877,6 +1900,98 @@ err_dma_buf_get: return ERR_PTR(ret); } +static size_t mem_buf_get_sgl_buf_size(struct hh_sgl_desc *sgl_desc) +{ + size_t size = 0; + unsigned int i; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) + size += sgl_desc->sgl_entries[i].size; + + return size; +} + +static struct mem_buf_import *mem_buf_import_dma_buf( + hh_memparcel_handle_t memparcel_hdl, + unsigned int nr_acl_entries, + const void __user *acl_list) +{ + int ret; + struct mem_buf_import *import; + struct hh_acl_desc *acl_desc; + struct hh_sgl_desc *sgl_desc; + struct acl_entry *k_acl_list; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct dma_buf *dmabuf; + + if (!nr_acl_entries || !acl_list) + return ERR_PTR(-EINVAL); + + import = kzalloc(sizeof(*import), GFP_KERNEL); + if (!import) + return ERR_PTR(-ENOMEM); + import->memparcel_hdl = memparcel_hdl; + mutex_init(&import->lock); + INIT_LIST_HEAD(&import->attachments); + + k_acl_list = memdup_user(acl_list, sizeof(*k_acl_list) * + nr_acl_entries); + if (IS_ERR(k_acl_list)) { + ret = PTR_ERR(k_acl_list); + goto err_out; + } + + acl_desc = mem_buf_acl_to_hh_acl(nr_acl_entries, k_acl_list); + kfree(k_acl_list); + if (IS_ERR(acl_desc)) { + ret = PTR_ERR(acl_desc); + goto err_out; + } + + sgl_desc = mem_buf_map_mem_s2(memparcel_hdl, acl_desc); + kfree(acl_desc); + if (IS_ERR(sgl_desc)) { + ret = PTR_ERR(sgl_desc); + goto err_out; + } + import->sgl_desc = sgl_desc; + import->size = mem_buf_get_sgl_buf_size(sgl_desc); + + ret = mem_buf_map_mem_s1(sgl_desc); + if (ret < 0) + goto err_map_mem_s1; + + exp_info.ops = &mem_buf_dma_buf_ops; + exp_info.size = import->size; + exp_info.flags = O_RDWR; + exp_info.priv = import; + + dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(dmabuf)) + goto err_export_dma_buf; + import->dmabuf = dmabuf; + + return import; + +err_export_dma_buf: + mem_buf_unmap_mem_s1(sgl_desc); +err_map_mem_s1: + kfree(import->sgl_desc); + mem_buf_unmap_mem_s2(memparcel_hdl); +err_out: + kfree(import); + return ERR_PTR(ret); +} + +void mem_buf_unimport_dma_buf(struct mem_buf_import *import_buf) +{ + mem_buf_unmap_mem_s1(import_buf->sgl_desc); + kfree(import_buf->sgl_desc); + mem_buf_unmap_mem_s2(import_buf->memparcel_hdl); + mutex_destroy(&import_buf->lock); + kfree(import_buf); +} + static long mem_buf_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { @@ -1939,6 +2054,29 @@ static long mem_buf_dev_ioctl(struct file *filp, unsigned int cmd, export->memparcel_hdl = ret_memparcel_hdl; break; } + case MEM_BUF_IOC_IMPORT: + { + struct mem_buf_import_ioctl_arg *import = &ioctl_arg.import; + struct mem_buf_import *import_buf; + + if (!(mem_buf_capability & MEM_BUF_CAP_CONSUMER)) + return -ENOTSUPP; + + import_buf = mem_buf_import_dma_buf(import->memparcel_hdl, + import->nr_acl_entries, + (const void __user *)import->acl_list); + if (IS_ERR(import_buf)) + return PTR_ERR(import_buf); + + fd = mem_buf_get_import_fd(import_buf); + if (fd < 0) { + mem_buf_import_put(import_buf); + return fd; + } + + import->dma_buf_import_fd = fd; + break; + } default: return -ENOTTY; } diff --git a/drivers/soc/qcom/mem_buf_dma_buf.c b/drivers/soc/qcom/mem_buf_dma_buf.c new file mode 100644 index 000000000000..1927a7bc70c5 --- /dev/null +++ b/drivers/soc/qcom/mem_buf_dma_buf.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#include "mem-buf-private.h" + +static struct sg_table *dup_hh_sgl_desc_to_sgt(struct hh_sgl_desc *sgl_desc) +{ + struct sg_table *new_table; + int ret, i; + struct scatterlist *sg; + + if (!sgl_desc || !sgl_desc->n_sgl_entries) + return ERR_PTR(-EINVAL); + + new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); + if (!new_table) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table(new_table, sgl_desc->n_sgl_entries, GFP_KERNEL); + if (ret) { + kfree(new_table); + return ERR_PTR(-ENOMEM); + } + + for_each_sg(new_table->sgl, sg, new_table->nents, i) { + sg_set_page(sg, phys_to_page(sgl_desc->sgl_entries[i].ipa_base), + sgl_desc->sgl_entries[i].size, 0); + sg_dma_address(sg) = 0; + sg_dma_len(sg) = 0; + } + + return new_table; +} + +static void free_duped_table(struct sg_table *table) +{ + sg_free_table(table); + kfree(table); +} + +struct mem_buf_dma_buf_attachment { + struct device *dev; + struct sg_table *table; + struct list_head list; + bool dma_mapped; +}; + +static int mem_buf_dma_buf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct mem_buf_dma_buf_attachment *a; + struct sg_table *table; + struct mem_buf_import *import_buf = dmabuf->priv; + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + + table = dup_hh_sgl_desc_to_sgt(import_buf->sgl_desc); + if (IS_ERR(table)) { + kfree(a); + return -ENOMEM; + } + + a->table = table; + a->dev = attachment->dev; + a->dma_mapped = false; + INIT_LIST_HEAD(&a->list); + + attachment->priv = a; + + mutex_lock(&import_buf->lock); + list_add(&a->list, &import_buf->attachments); + mutex_unlock(&import_buf->lock); + + return 0; +} + +static void mem_buf_dma_buf_detatch(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct mem_buf_dma_buf_attachment *a = attachment->priv; + struct mem_buf_import *import_buf = dmabuf->priv; + + mutex_lock(&import_buf->lock); + list_del(&a->list); + mutex_unlock(&import_buf->lock); + free_duped_table(a->table); + kfree(a); +} + +static struct sg_table *mem_buf_dma_map_attachment( + struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct mem_buf_dma_buf_attachment *a = attachment->priv; + struct mem_buf_import *buffer = attachment->dmabuf->priv; + struct sg_table *table; + int count, map_attrs; + + table = a->table; + map_attrs = attachment->dma_map_attrs; + + mutex_lock(&buffer->lock); + count = dma_map_sg_attrs(attachment->dev, table->sgl, table->nents, + direction, map_attrs); + + if (count <= 0) { + mutex_unlock(&buffer->lock); + return ERR_PTR(-ENOMEM); + } + + a->dma_mapped = true; + mutex_unlock(&buffer->lock); + return table; +} + +static void mem_buf_dma_unmap_attachment(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + int map_attrs; + struct mem_buf_import *buffer = attachment->dmabuf->priv; + struct mem_buf_dma_buf_attachment *a = attachment->priv; + + map_attrs = attachment->dma_map_attrs; + + mutex_lock(&buffer->lock); + dma_unmap_sg_attrs(attachment->dev, table->sgl, table->nents, direction, + map_attrs); + a->dma_mapped = false; + mutex_unlock(&buffer->lock); +} + +static int mem_buf_map_user(struct mem_buf_import *import_buf, + struct vm_area_struct *vma) +{ + struct hh_sgl_desc *sgl_desc = import_buf->sgl_desc; + unsigned long addr = vma->vm_start; + unsigned long offset = vma->vm_pgoff * PAGE_SIZE; + int i, ret; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) { + struct page *page = + phys_to_page(sgl_desc->sgl_entries[i].ipa_base); + unsigned long remainder = vma->vm_end - addr; + unsigned long len = sgl_desc->sgl_entries[i].size; + + if (offset >= len) { + offset -= len; + continue; + } else if (offset) { + page += offset / PAGE_SIZE; + len = sgl_desc->sgl_entries[i].size - offset; + offset = 0; + } + + len = min(len, remainder); + ret = remap_pfn_range(vma, addr, page_to_pfn(page), len, + vma->vm_page_prot); + if (ret) + return ret; + addr += len; + if (addr >= vma->vm_end) + return 0; + } + + return 0; +} + +static void *mem_buf_map_kernel(struct mem_buf_import *import_buf) +{ + void *vaddr; + int npages = PAGE_ALIGN(import_buf->size) / PAGE_SIZE; + struct page **pages = + vmalloc(array_size(npages, sizeof(struct page *))); + struct page **tmp = pages; + struct hh_sgl_desc *sgl_desc = import_buf->sgl_desc; + int i, j, n_pages_this_seg; + u64 seg_ipa_base, seg_size; + struct page *page; + + if (!pages) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) { + seg_ipa_base = sgl_desc->sgl_entries[i].ipa_base; + seg_size = sgl_desc->sgl_entries[i].size; + n_pages_this_seg = PAGE_ALIGN(seg_size) / PAGE_SIZE; + page = phys_to_page(seg_ipa_base); + + BUG_ON(i >= npages); + for (j = 0; j < n_pages_this_seg; j++) + *(tmp++) = page++; + } + + vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL); + vfree(pages); + + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} + +static void mem_buf_unmap_kernel(struct mem_buf_import *import_buf) +{ + vunmap(import_buf->vaddr); +} + +static int mem_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + int ret = 0; + + mutex_lock(&import_buf->lock); + ret = mem_buf_map_user(import_buf, vma); + mutex_unlock(&import_buf->lock); + + if (ret) + pr_err_ratelimited("%s: failure mapping buffer to userspace\n", + __func__); + + return ret; +} + +static void mem_buf_dma_buf_release(struct dma_buf *dmabuf) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + + mem_buf_unimport_dma_buf(import_buf); +} + +static void *mem_buf_buffer_kmap_get(struct mem_buf_import *import_buf) +{ + void *vaddr; + + if (import_buf->kmap_cnt) { + import_buf->kmap_cnt++; + return import_buf->vaddr; + } + vaddr = mem_buf_map_kernel(import_buf); + if (IS_ERR(vaddr)) + return vaddr; + import_buf->vaddr = vaddr; + import_buf->kmap_cnt++; + return vaddr; +} + +static void mem_buf_buffer_kmap_put(struct mem_buf_import *import_buf) +{ + if (import_buf->kmap_cnt == 0) { + pr_warn_ratelimited("membuf client likely missing a call to dma_buf_kmap or dma_buf_vmap, pid:%d\n", + current->pid); + return; + } + + import_buf->kmap_cnt--; + if (!import_buf->kmap_cnt) { + mem_buf_unmap_kernel(import_buf); + import_buf->vaddr = NULL; + } +} + +static void *mem_buf_dma_buf_vmap(struct dma_buf *dmabuf) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + void *vaddr; + + mutex_lock(&import_buf->lock); + vaddr = mem_buf_buffer_kmap_get(import_buf); + mutex_unlock(&import_buf->lock); + + return vaddr; +} + +static void mem_buf_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + + mutex_lock(&import_buf->lock); + mem_buf_buffer_kmap_put(import_buf); + mutex_unlock(&import_buf->lock); +} + +static void *mem_buf_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) +{ + /* + * TODO: Once clients remove their hacks where they assume kmap(ed) + * addresses are virtually contiguous implement this properly + */ + void *vaddr = mem_buf_dma_buf_vmap(dmabuf); + + if (IS_ERR(vaddr)) + return vaddr; + + return vaddr + offset * PAGE_SIZE; +} + +static void mem_buf_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset, + void *ptr) +{ + /* + * TODO: Once clients remove their hacks where they assume kmap(ed) + * addresses are virtually contiguous implement this properly + */ + mem_buf_dma_buf_vunmap(dmabuf, ptr); +} + +static int mem_buf_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + if (!a->dma_mapped) + continue; + + dma_sync_sg_for_cpu(a->dev, a->table->sgl, + a->table->nents, direction); + } + mutex_unlock(&import_buf->lock); + return 0; +} + +static int mem_buf_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + if (!a->dma_mapped) + continue; + + dma_sync_sg_for_device(a->dev, a->table->sgl, a->table->nents, + direction); + } + mutex_unlock(&import_buf->lock); + return 0; +} + +static int mem_buf_sgl_sync_range(struct device *dev, struct scatterlist *sgl, + unsigned int nents, unsigned long offset, + unsigned long length, + enum dma_data_direction dir, bool for_cpu) +{ + int i; + struct scatterlist *sg; + unsigned int len = 0; + dma_addr_t sg_dma_addr; + + for_each_sg(sgl, sg, nents, i) { + if (sg_dma_len(sg) == 0) + break; + + if (i > 0) { + pr_warn_ratelimited("Partial cmo only supported with 1 segment\n" + "is dma_set_max_seg_size being set on dev:%s\n", + dev_name(dev)); + return -EINVAL; + } + } + + for_each_sg(sgl, sg, nents, i) { + unsigned int sg_offset, sg_left, size = 0; + + if (i == 0) + sg_dma_addr = sg_dma_address(sg); + + len += sg->length; + if (len <= offset) { + sg_dma_addr += sg->length; + continue; + } + + sg_left = len - offset; + sg_offset = sg->length - sg_left; + + size = (length < sg_left) ? length : sg_left; + if (for_cpu) + dma_sync_single_range_for_cpu(dev, sg_dma_addr, + sg_offset, size, dir); + else + dma_sync_single_range_for_device(dev, sg_dma_addr, + sg_offset, size, dir); + + offset += size; + length -= size; + sg_dma_addr += sg->length; + + if (length == 0) + break; + } + + return 0; +} + +static int mem_buf_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction dir, + unsigned int offset, + unsigned int len) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + int ret = 0; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + if (!a->dma_mapped) + continue; + + ret = mem_buf_sgl_sync_range(a->dev, a->table->sgl, + a->table->nents, offset, len, dir, + true); + + } + mutex_unlock(&import_buf->lock); + return ret; +} + +static int mem_buf_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, + unsigned int len) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + int ret = 0; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + ret = mem_buf_sgl_sync_range(a->dev, a->table->sgl, + a->table->nents, offset, len, + direction, false); + } + mutex_unlock(&import_buf->lock); + return ret; +} + +const struct dma_buf_ops mem_buf_dma_buf_ops = { + .map_dma_buf = mem_buf_dma_map_attachment, + .unmap_dma_buf = mem_buf_dma_unmap_attachment, + .mmap = mem_buf_mmap, + .release = mem_buf_dma_buf_release, + .attach = mem_buf_dma_buf_attach, + .detach = mem_buf_dma_buf_detatch, + .begin_cpu_access = mem_buf_dma_buf_begin_cpu_access, + .end_cpu_access = mem_buf_dma_buf_end_cpu_access, + .begin_cpu_access_partial = mem_buf_dma_buf_begin_cpu_access_partial, + .end_cpu_access_partial = mem_buf_dma_buf_end_cpu_access_partial, + .map = mem_buf_dma_buf_kmap, + .unmap = mem_buf_dma_buf_kunmap, + .vmap = mem_buf_dma_buf_vmap, + .vunmap = mem_buf_dma_buf_vunmap, +}; diff --git a/include/uapi/linux/mem-buf.h b/include/uapi/linux/mem-buf.h index 9c9082fd3c1d..b258d8aa5b11 100644 --- a/include/uapi/linux/mem-buf.h +++ b/include/uapi/linux/mem-buf.h @@ -56,13 +56,13 @@ struct mem_buf_ion_data { }; /** - * struct mem_buf_alloc_ioctl_arg: An request to allocate memory from another + * struct mem_buf_alloc_ioctl_arg: A request to allocate memory from another * VM to other VMs. * @size: The size of the allocation. - * @nr_acl_entries: The number of ACL entries in @acl_list. * @acl_list: An array of structures, where each structure specifies a VMID * and the access permissions that the VMID will have to the memory to be * allocated. + * @nr_acl_entries: The number of ACL entries in @acl_list. * @src_mem_type: The type of memory that the source VM should allocate from. * This should be one of the mem_buf_mem_type enum values. * @src_data: A pointer to data that the source VM should interpret when @@ -70,25 +70,25 @@ struct mem_buf_ion_data { * @dst_mem_type: The type of memory that the destination VM should treat the * incoming allocation from the source VM as. This should be one of the * mem_buf_mem_type enum values. - * @dst_data: A pointer to data that the destination VM should interpret when - * adding the memory to the current VM. * @mem_buf_fd: A file descriptor representing the memory that was allocated * from the source VM and added to the current VM. Calling close() on this file * descriptor will deallocate the memory from the current VM, and return it * to the source VM. + * * @dst_data: A pointer to data that the destination VM should interpret when + * adding the memory to the current VM. * * All reserved fields must be zeroed out by the caller prior to invoking the * allocation IOCTL command with this argument. */ struct mem_buf_alloc_ioctl_arg { __u64 size; - __u32 nr_acl_entries; __u64 acl_list; + __u32 nr_acl_entries; __u32 src_mem_type; __u64 src_data; __u32 dst_mem_type; - __u64 dst_data; __u32 mem_buf_fd; + __u64 dst_data; __u64 reserved0; __u64 reserved1; __u64 reserved2; @@ -131,4 +131,32 @@ struct mem_buf_export_ioctl_arg { #define MEM_BUF_IOC_EXPORT _IOWR(MEM_BUF_IOC_MAGIC, 1,\ struct mem_buf_export_ioctl_arg) +/** + * struct mem_buf_import_ioctl_arg: A request to import memory from another + * VM as a dma-buf + * @memparcel_hdl: The handle that corresponds to the memparcel we are + * importing. + * @nr_acl_entries: The number of ACL entries in @acl_list. + * @acl_list: An array of structures, where each structure specifies a VMID + * and the access permissions that the VMID should have for the memparcel. + * @dma_buf_import_fd: A dma-buf file descriptor that the client can use to + * access the buffer. This fd must be closed to release the memory. + * + * All reserved fields must be zeroed out by the caller prior to invoking the + * import IOCTL command with this argument. + */ +struct mem_buf_import_ioctl_arg { + __u32 memparcel_hdl; + __u32 nr_acl_entries; + __u64 acl_list; + __u32 dma_buf_import_fd; + __u32 reserved0; + __u64 reserved1; + __u64 reserved2; + __u64 reserved3; +}; + +#define MEM_BUF_IOC_IMPORT _IOWR(MEM_BUF_IOC_MAGIC, 2,\ + struct mem_buf_import_ioctl_arg) + #endif /* _UAPI_LINUX_MEM_BUF_H */