diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index 67fa1bdeb884..a178e17b2219 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -42,7 +42,7 @@ obj-$(CONFIG_QCOM_LAHAINA_LLCC) += llcc-lahaina.o obj-$(CONFIG_QCOM_SHIMA_LLCC) += llcc-shima.o obj-$(CONFIG_QCOM_MINIDUMP) += msm_minidump.o minidump_log.o obj-$(CONFIG_QCOM_MEM_OFFLINE) += mem-offline.o -obj-$(CONFIG_QCOM_MEM_BUF) += mem-buf.o +obj-$(CONFIG_QCOM_MEM_BUF) += mem-buf.o mem_buf_dma_buf.o obj-$(CONFIG_QCOM_MEMORY_DUMP_V2) += memory_dump_v2.o obj-$(CONFIG_QCOM_DCC_V2) += dcc_v2.o obj-$(CONFIG_MSM_JTAGV8) += jtagv8.o jtagv8-etm.o diff --git a/drivers/soc/qcom/mem-buf-private.h b/drivers/soc/qcom/mem-buf-private.h new file mode 100644 index 000000000000..763db1d7b318 --- /dev/null +++ b/drivers/soc/qcom/mem-buf-private.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#ifndef MEM_BUF_PRIVATE_H +#define MEM_BUF_PRIVATE_H + +#include +#include +#include +#include + +/** + * strcut mem_buf_import: Represents a memory buffer that was imported from + * another VM. + * @memparcel_hdl: The handle associated with the memparcel that represents the + * memory that was imported from another VM. + * @size: The size of the buffer. + * @sgl_desc: The SG descriptor that represents the memory buffer. + * @dmabuf: The dma-buf that corresponds to the buffer. + * @kmap_cnt: The number of kernel mapping references associated with the buffer + * @vaddr: The virtual address for the buffer after it has been mapped into a + * contiguous range in the kernel virtual address space. + * @lock: protects accesses to attachments. + * @attachments: a list of attachments for the buffer. + */ +struct mem_buf_import { + hh_memparcel_handle_t memparcel_hdl; + size_t size; + struct hh_sgl_desc *sgl_desc; + struct dma_buf *dmabuf; + int kmap_cnt; + void *vaddr; + struct mutex lock; + struct list_head attachments; +}; + +void mem_buf_unimport_dma_buf(struct mem_buf_import *import_buf); + +extern const struct dma_buf_ops mem_buf_dma_buf_ops; +#endif + diff --git a/drivers/soc/qcom/mem-buf.c b/drivers/soc/qcom/mem-buf.c index 5fc36b604c1c..35247e729f1f 100644 --- a/drivers/soc/qcom/mem-buf.c +++ b/drivers/soc/qcom/mem-buf.c @@ -27,6 +27,8 @@ #include #include +#include "mem-buf-private.h" + #define CREATE_TRACE_POINTS #include "trace-mem-buf.h" @@ -170,6 +172,26 @@ struct mem_buf_xfer_ion_mem { struct dma_buf_attachment *attachment; }; +/** + * struct mem_buf_export: Represents a dmabuf that has been exported to other + * VM(s). + * @dmabuf: The dmabuf that was exported to other VM(s) + * @attachment: The dma-buf attachment for @dmabuf + * @mem_sgt: The SG-Table for the dmabuf that was exported. + * @nr_vmids: The number of VMIDs that have access to the memory that was + * exported. + * @dst_vmids: The VMIDs of the VMs that have access to the buffer. + * @filp: A file structure that corresponds to the buffer that was exported. + */ +struct mem_buf_export { + struct dma_buf *dmabuf; + struct dma_buf_attachment *attachment; + struct sg_table *mem_sgt; + unsigned int nr_vmids; + int *dst_vmids; + struct file *filp; +}; + static int mem_buf_init_txn(struct mem_buf_txn *txn, void *resp_buf) { int ret; @@ -1441,32 +1463,66 @@ err_alloc_acl_list: } EXPORT_SYMBOL(mem_buf_alloc); -int mem_buf_get_fd(void *membuf_desc) +static int _mem_buf_get_fd(struct file *filp) { int fd; - struct mem_buf_desc *membuf = membuf_desc; - if (!membuf_desc) + if (!filp) return -EINVAL; fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) return fd; - fd_install(fd, membuf->filp); + fd_install(fd, filp); return fd; } + +int mem_buf_get_fd(void *membuf_desc) +{ + struct mem_buf_desc *membuf = membuf_desc; + + if (!membuf_desc) + return -EINVAL; + + return _mem_buf_get_fd(membuf->filp); +} EXPORT_SYMBOL(mem_buf_get_fd); +static int mem_buf_get_export_fd(struct mem_buf_export *export_buf) +{ + return _mem_buf_get_fd(export_buf->filp); +} + +static int mem_buf_get_import_fd(struct mem_buf_import *import_buf) +{ + return dma_buf_fd(import_buf->dmabuf, O_CLOEXEC); +} + +static void _mem_buf_put(struct file *filp) +{ + fput(filp); +} + void mem_buf_put(void *membuf_desc) { struct mem_buf_desc *membuf = membuf_desc; if (membuf && membuf->filp) - fput(membuf->filp); + _mem_buf_put(membuf->filp); } EXPORT_SYMBOL(mem_buf_put); +static void mem_buf_export_put(struct mem_buf_export *export_buf) +{ + _mem_buf_put(export_buf->filp); +} + +static void mem_buf_import_put(struct mem_buf_import *import_buf) +{ + dma_buf_put(import_buf->dmabuf); +} + static bool is_mem_buf_file(struct file *filp) { return filp->f_op == &mem_buf_fops; @@ -1572,51 +1628,453 @@ out: return ret; } -static int validate_ioctl_arg(struct mem_buf_alloc_ioctl_arg *allocation) +union mem_buf_ioctl_arg { + struct mem_buf_alloc_ioctl_arg allocation; + struct mem_buf_export_ioctl_arg export; + struct mem_buf_import_ioctl_arg import; +}; + +static int validate_ioctl_arg(union mem_buf_ioctl_arg *arg, unsigned int cmd) { - if (!allocation->size || !allocation->nr_acl_entries || - !allocation->acl_list || - (allocation->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS) || - !is_valid_mem_type(allocation->src_mem_type) || - !is_valid_mem_type(allocation->dst_mem_type) || - allocation->reserved0 || allocation->reserved1 || - allocation->reserved2) + switch (cmd) { + case MEM_BUF_IOC_ALLOC: + { + struct mem_buf_alloc_ioctl_arg *allocation = &arg->allocation; + + if (!allocation->size || !allocation->nr_acl_entries || + !allocation->acl_list || + (allocation->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS) || + !is_valid_mem_type(allocation->src_mem_type) || + !is_valid_mem_type(allocation->dst_mem_type) || + allocation->reserved0 || allocation->reserved1 || + allocation->reserved2) + return -EINVAL; + break; + } + case MEM_BUF_IOC_EXPORT: + { + struct mem_buf_export_ioctl_arg *export = &arg->export; + + if (!export->nr_acl_entries || !export->acl_list || + export->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS || + export->reserved0 || export->reserved1 || export->reserved2) + return -EINVAL; + break; + } + case MEM_BUF_IOC_IMPORT: + { + struct mem_buf_import_ioctl_arg *import = &arg->import; + + if (!import->nr_acl_entries || !import->acl_list || + import->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS || + import->reserved0 || import->reserved1 || import->reserved2) + return -EINVAL; + break; + } + default: return -EINVAL; + } return 0; } +static bool mem_buf_hlos_accessible(int *vmids, u32 nr_vmids) +{ + int i; + + if (!vmids || !nr_vmids) + return false; + + for (i = 0; i < nr_vmids; i++) + if (vmids[i] == VMID_HLOS) + return true; + + return false; +} + +static int mem_buf_export_release(struct inode *inode, struct file *filp) +{ + int ret; + struct mem_buf_export *export_buf = filp->private_data; + bool dma_buf_freeable = true; + + ret = mem_buf_unassign_mem(export_buf->mem_sgt, export_buf->dst_vmids, + export_buf->nr_vmids); + if (ret < 0) + dma_buf_freeable = false; + + if (!mem_buf_hlos_accessible(export_buf->dst_vmids, + export_buf->nr_vmids) && dma_buf_freeable) + msm_ion_dma_buf_unlock(export_buf->dmabuf); + + kfree(export_buf->dst_vmids); + if (dma_buf_freeable) { + dma_buf_unmap_attachment(export_buf->attachment, + export_buf->mem_sgt, + DMA_BIDIRECTIONAL); + dma_buf_detach(export_buf->dmabuf, export_buf->attachment); + dma_buf_put(export_buf->dmabuf); + } + kfree(export_buf); + return ret; +} + +static const struct file_operations mem_buf_export_fops = { + .release = mem_buf_export_release, +}; + +static int mem_buf_acl_to_vmid_perms_list(unsigned int nr_acl_entries, + const void __user *acl_entries, + int **dst_vmids, int **dst_perms) +{ + int ret, i, *vmids, *perms; + struct acl_entry entry; + + if (!nr_acl_entries || !acl_entries) + return -EINVAL; + + vmids = kmalloc_array(nr_acl_entries, sizeof(*vmids), GFP_KERNEL); + if (!vmids) + return -ENOMEM; + + perms = kmalloc_array(nr_acl_entries, sizeof(*perms), GFP_KERNEL); + if (!perms) { + kfree(vmids); + return -ENOMEM; + } + + for (i = 0; i < nr_acl_entries; i++) { + ret = copy_struct_from_user(&entry, sizeof(entry), + acl_entries + (sizeof(entry) * i), + sizeof(entry)); + if (ret < 0) + goto out; + + vmids[i] = mem_buf_vmid_to_vmid(entry.vmid); + perms[i] = mem_buf_perms_to_perms(entry.perms); + if (vmids[i] < 0 || perms[i] < 0) { + ret = -EINVAL; + goto out; + } + } + + *dst_vmids = vmids; + *dst_perms = perms; + return ret; + +out: + kfree(perms); + kfree(vmids); + return ret; +} + +static struct mem_buf_export *mem_buf_export_dma_buf(int dma_buf_fd, + unsigned int nr_acl_entries, + const void __user *acl_entries, + hh_memparcel_handle_t *memparcel_hdl) +{ + int ret; + struct mem_buf_export *export_buf; + struct dma_buf *dmabuf; + struct dma_buf_attachment *attachment; + struct sg_table *sgt; + int *dst_vmids, *dst_perms; + bool dma_buf_freeable = true; + struct file *filp; + unsigned long flags = 0; + + if (!nr_acl_entries || !acl_entries || !memparcel_hdl) + return ERR_PTR(-EINVAL); + + export_buf = kmalloc(sizeof(*export_buf), GFP_KERNEL); + if (!export_buf) + return ERR_PTR(-ENOMEM); + + dmabuf = dma_buf_get(dma_buf_fd); + if (IS_ERR(dmabuf)) { + pr_err_ratelimited("%s: dma_buf_get failed rc: %d\n", __func__, + PTR_ERR(dmabuf)); + ret = PTR_ERR(dmabuf); + goto err_dma_buf_get; + } + export_buf->dmabuf = dmabuf; + + ret = dma_buf_get_flags(dmabuf, &flags); + if (ret < 0) { + pr_err_ratelimited("%s: dma_buf_get_flags failed rc: %d\n", + __func__, ret); + goto err_dma_buf_attach; + } else if (!(flags & ION_FLAG_CACHED)) { + ret = -EINVAL; + pr_err_ratelimited("%s: only cached buffers can be exported\n", + __func__); + goto err_dma_buf_attach; + } else if (flags & (ION_FLAG_SECURE | ION_FLAGS_CP_MASK)) { + ret = -EINVAL; + pr_err_ratelimited("%s: only non-secure allocations can be exported\n", + __func__); + goto err_dma_buf_attach; + } + + attachment = dma_buf_attach(dmabuf, mem_buf_dev); + if (IS_ERR(attachment)) { + pr_err_ratelimited("%s: dma_buf_attach failed rc: %d\n", + __func__, PTR_ERR(attachment)); + ret = PTR_ERR(attachment); + goto err_dma_buf_attach; + } + export_buf->attachment = attachment; + + sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL); + if (IS_ERR(sgt)) { + pr_err_ratelimited("%s dma_buf_map_attachment failed rc: %d\n", + __func__, PTR_ERR(sgt)); + ret = PTR_ERR(sgt); + goto err_map_attachment; + } + export_buf->mem_sgt = sgt; + + ret = mem_buf_acl_to_vmid_perms_list(nr_acl_entries, acl_entries, + &dst_vmids, &dst_perms); + if (ret < 0) { + pr_err_ratelimited("%s failed to copy ACL rc: %d\n", __func__, + ret); + goto err_cpy_acl_entries; + } + export_buf->nr_vmids = nr_acl_entries; + export_buf->dst_vmids = dst_vmids; + + if (!mem_buf_hlos_accessible(dst_vmids, nr_acl_entries)) { + ret = msm_ion_dma_buf_lock(dmabuf); + if (ret < 0) { + pr_err_ratelimited("%s failed to lock buffer rc: %d\n", + __func__, ret); + goto err_lock_mem; + } + } + + ret = mem_buf_assign_mem(sgt, dst_vmids, dst_perms, nr_acl_entries); + if (ret < 0) { + if (ret == -EADDRNOTAVAIL) + dma_buf_freeable = false; + goto err_assign_mem; + } + + ret = mem_buf_retrieve_memparcel_hdl(sgt, dst_vmids, dst_perms, + nr_acl_entries, memparcel_hdl); + if (ret < 0) + goto err_retrieve_hdl; + + filp = anon_inode_getfile("membuf", &mem_buf_export_fops, export_buf, + O_RDWR); + if (IS_ERR(filp)) { + ret = PTR_ERR(filp); + goto err_retrieve_hdl; + } + export_buf->filp = filp; + + kfree(dst_perms); + return export_buf; + +err_retrieve_hdl: + if (mem_buf_unassign_mem(sgt, dst_vmids, nr_acl_entries) < 0) + dma_buf_freeable = false; +err_assign_mem: + if (!mem_buf_hlos_accessible(dst_vmids, nr_acl_entries) && + dma_buf_freeable) + msm_ion_dma_buf_unlock(dmabuf); +err_lock_mem: + kfree(dst_vmids); + kfree(dst_perms); +err_cpy_acl_entries: + if (dma_buf_freeable) + dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL); +err_map_attachment: + if (dma_buf_freeable) + dma_buf_detach(dmabuf, attachment); +err_dma_buf_attach: + if (dma_buf_freeable) + dma_buf_put(dmabuf); +err_dma_buf_get: + kfree(export_buf); + return ERR_PTR(ret); +} + +static size_t mem_buf_get_sgl_buf_size(struct hh_sgl_desc *sgl_desc) +{ + size_t size = 0; + unsigned int i; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) + size += sgl_desc->sgl_entries[i].size; + + return size; +} + +static struct mem_buf_import *mem_buf_import_dma_buf( + hh_memparcel_handle_t memparcel_hdl, + unsigned int nr_acl_entries, + const void __user *acl_list) +{ + int ret; + struct mem_buf_import *import; + struct hh_acl_desc *acl_desc; + struct hh_sgl_desc *sgl_desc; + struct acl_entry *k_acl_list; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct dma_buf *dmabuf; + + if (!nr_acl_entries || !acl_list) + return ERR_PTR(-EINVAL); + + import = kzalloc(sizeof(*import), GFP_KERNEL); + if (!import) + return ERR_PTR(-ENOMEM); + import->memparcel_hdl = memparcel_hdl; + mutex_init(&import->lock); + INIT_LIST_HEAD(&import->attachments); + + k_acl_list = memdup_user(acl_list, sizeof(*k_acl_list) * + nr_acl_entries); + if (IS_ERR(k_acl_list)) { + ret = PTR_ERR(k_acl_list); + goto err_out; + } + + acl_desc = mem_buf_acl_to_hh_acl(nr_acl_entries, k_acl_list); + kfree(k_acl_list); + if (IS_ERR(acl_desc)) { + ret = PTR_ERR(acl_desc); + goto err_out; + } + + sgl_desc = mem_buf_map_mem_s2(memparcel_hdl, acl_desc); + kfree(acl_desc); + if (IS_ERR(sgl_desc)) { + ret = PTR_ERR(sgl_desc); + goto err_out; + } + import->sgl_desc = sgl_desc; + import->size = mem_buf_get_sgl_buf_size(sgl_desc); + + ret = mem_buf_map_mem_s1(sgl_desc); + if (ret < 0) + goto err_map_mem_s1; + + exp_info.ops = &mem_buf_dma_buf_ops; + exp_info.size = import->size; + exp_info.flags = O_RDWR; + exp_info.priv = import; + + dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(dmabuf)) + goto err_export_dma_buf; + import->dmabuf = dmabuf; + + return import; + +err_export_dma_buf: + mem_buf_unmap_mem_s1(sgl_desc); +err_map_mem_s1: + kfree(import->sgl_desc); + mem_buf_unmap_mem_s2(memparcel_hdl); +err_out: + kfree(import); + return ERR_PTR(ret); +} + +void mem_buf_unimport_dma_buf(struct mem_buf_import *import_buf) +{ + mem_buf_unmap_mem_s1(import_buf->sgl_desc); + kfree(import_buf->sgl_desc); + mem_buf_unmap_mem_s2(import_buf->memparcel_hdl); + mutex_destroy(&import_buf->lock); + kfree(import_buf); +} + static long mem_buf_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { int fd; unsigned int dir = _IOC_DIR(cmd); - struct mem_buf_alloc_ioctl_arg allocation; + union mem_buf_ioctl_arg ioctl_arg; - if (_IOC_SIZE(cmd) > sizeof(allocation)) + if (_IOC_SIZE(cmd) > sizeof(ioctl_arg)) return -EINVAL; - if (copy_from_user(&allocation, (void __user *)arg, _IOC_SIZE(cmd))) + if (copy_from_user(&ioctl_arg, (void __user *)arg, _IOC_SIZE(cmd))) return -EFAULT; - if (validate_ioctl_arg(&allocation) < 0) + if (validate_ioctl_arg(&ioctl_arg, cmd) < 0) return -EINVAL; if (!(dir & _IOC_WRITE)) - memset(&allocation, 0, sizeof(allocation)); + memset(&ioctl_arg, 0, sizeof(ioctl_arg)); switch (cmd) { case MEM_BUF_IOC_ALLOC: { + struct mem_buf_alloc_ioctl_arg *allocation = + &ioctl_arg.allocation; + if (!(mem_buf_capability & MEM_BUF_CAP_CONSUMER)) return -ENOTSUPP; - fd = mem_buf_alloc_fd(&allocation); + fd = mem_buf_alloc_fd(allocation); if (fd < 0) return fd; - allocation.mem_buf_fd = fd; + allocation->mem_buf_fd = fd; + break; + } + case MEM_BUF_IOC_EXPORT: + { + struct mem_buf_export_ioctl_arg *export = &ioctl_arg.export; + u32 ret_memparcel_hdl; + struct mem_buf_export *export_buf; + + if (!(mem_buf_capability & MEM_BUF_CAP_SUPPLIER)) + return -ENOTSUPP; + + export_buf = mem_buf_export_dma_buf(export->dma_buf_fd, + export->nr_acl_entries, + (const void __user *)export->acl_list, + &ret_memparcel_hdl); + if (IS_ERR(export_buf)) + return PTR_ERR(export_buf); + + fd = mem_buf_get_export_fd(export_buf); + if (fd < 0) { + mem_buf_export_put(export_buf); + return fd; + } + + export->export_fd = fd; + export->memparcel_hdl = ret_memparcel_hdl; + break; + } + case MEM_BUF_IOC_IMPORT: + { + struct mem_buf_import_ioctl_arg *import = &ioctl_arg.import; + struct mem_buf_import *import_buf; + + if (!(mem_buf_capability & MEM_BUF_CAP_CONSUMER)) + return -ENOTSUPP; + + import_buf = mem_buf_import_dma_buf(import->memparcel_hdl, + import->nr_acl_entries, + (const void __user *)import->acl_list); + if (IS_ERR(import_buf)) + return PTR_ERR(import_buf); + + fd = mem_buf_get_import_fd(import_buf); + if (fd < 0) { + mem_buf_import_put(import_buf); + return fd; + } + + import->dma_buf_import_fd = fd; break; } default: @@ -1624,7 +2082,7 @@ static long mem_buf_dev_ioctl(struct file *filp, unsigned int cmd, } if (dir & _IOC_READ) { - if (copy_to_user((void __user *)arg, &allocation, + if (copy_to_user((void __user *)arg, &ioctl_arg, _IOC_SIZE(cmd))) return -EFAULT; } diff --git a/drivers/soc/qcom/mem_buf_dma_buf.c b/drivers/soc/qcom/mem_buf_dma_buf.c new file mode 100644 index 000000000000..1927a7bc70c5 --- /dev/null +++ b/drivers/soc/qcom/mem_buf_dma_buf.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#include "mem-buf-private.h" + +static struct sg_table *dup_hh_sgl_desc_to_sgt(struct hh_sgl_desc *sgl_desc) +{ + struct sg_table *new_table; + int ret, i; + struct scatterlist *sg; + + if (!sgl_desc || !sgl_desc->n_sgl_entries) + return ERR_PTR(-EINVAL); + + new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); + if (!new_table) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table(new_table, sgl_desc->n_sgl_entries, GFP_KERNEL); + if (ret) { + kfree(new_table); + return ERR_PTR(-ENOMEM); + } + + for_each_sg(new_table->sgl, sg, new_table->nents, i) { + sg_set_page(sg, phys_to_page(sgl_desc->sgl_entries[i].ipa_base), + sgl_desc->sgl_entries[i].size, 0); + sg_dma_address(sg) = 0; + sg_dma_len(sg) = 0; + } + + return new_table; +} + +static void free_duped_table(struct sg_table *table) +{ + sg_free_table(table); + kfree(table); +} + +struct mem_buf_dma_buf_attachment { + struct device *dev; + struct sg_table *table; + struct list_head list; + bool dma_mapped; +}; + +static int mem_buf_dma_buf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct mem_buf_dma_buf_attachment *a; + struct sg_table *table; + struct mem_buf_import *import_buf = dmabuf->priv; + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + + table = dup_hh_sgl_desc_to_sgt(import_buf->sgl_desc); + if (IS_ERR(table)) { + kfree(a); + return -ENOMEM; + } + + a->table = table; + a->dev = attachment->dev; + a->dma_mapped = false; + INIT_LIST_HEAD(&a->list); + + attachment->priv = a; + + mutex_lock(&import_buf->lock); + list_add(&a->list, &import_buf->attachments); + mutex_unlock(&import_buf->lock); + + return 0; +} + +static void mem_buf_dma_buf_detatch(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct mem_buf_dma_buf_attachment *a = attachment->priv; + struct mem_buf_import *import_buf = dmabuf->priv; + + mutex_lock(&import_buf->lock); + list_del(&a->list); + mutex_unlock(&import_buf->lock); + free_duped_table(a->table); + kfree(a); +} + +static struct sg_table *mem_buf_dma_map_attachment( + struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct mem_buf_dma_buf_attachment *a = attachment->priv; + struct mem_buf_import *buffer = attachment->dmabuf->priv; + struct sg_table *table; + int count, map_attrs; + + table = a->table; + map_attrs = attachment->dma_map_attrs; + + mutex_lock(&buffer->lock); + count = dma_map_sg_attrs(attachment->dev, table->sgl, table->nents, + direction, map_attrs); + + if (count <= 0) { + mutex_unlock(&buffer->lock); + return ERR_PTR(-ENOMEM); + } + + a->dma_mapped = true; + mutex_unlock(&buffer->lock); + return table; +} + +static void mem_buf_dma_unmap_attachment(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + int map_attrs; + struct mem_buf_import *buffer = attachment->dmabuf->priv; + struct mem_buf_dma_buf_attachment *a = attachment->priv; + + map_attrs = attachment->dma_map_attrs; + + mutex_lock(&buffer->lock); + dma_unmap_sg_attrs(attachment->dev, table->sgl, table->nents, direction, + map_attrs); + a->dma_mapped = false; + mutex_unlock(&buffer->lock); +} + +static int mem_buf_map_user(struct mem_buf_import *import_buf, + struct vm_area_struct *vma) +{ + struct hh_sgl_desc *sgl_desc = import_buf->sgl_desc; + unsigned long addr = vma->vm_start; + unsigned long offset = vma->vm_pgoff * PAGE_SIZE; + int i, ret; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) { + struct page *page = + phys_to_page(sgl_desc->sgl_entries[i].ipa_base); + unsigned long remainder = vma->vm_end - addr; + unsigned long len = sgl_desc->sgl_entries[i].size; + + if (offset >= len) { + offset -= len; + continue; + } else if (offset) { + page += offset / PAGE_SIZE; + len = sgl_desc->sgl_entries[i].size - offset; + offset = 0; + } + + len = min(len, remainder); + ret = remap_pfn_range(vma, addr, page_to_pfn(page), len, + vma->vm_page_prot); + if (ret) + return ret; + addr += len; + if (addr >= vma->vm_end) + return 0; + } + + return 0; +} + +static void *mem_buf_map_kernel(struct mem_buf_import *import_buf) +{ + void *vaddr; + int npages = PAGE_ALIGN(import_buf->size) / PAGE_SIZE; + struct page **pages = + vmalloc(array_size(npages, sizeof(struct page *))); + struct page **tmp = pages; + struct hh_sgl_desc *sgl_desc = import_buf->sgl_desc; + int i, j, n_pages_this_seg; + u64 seg_ipa_base, seg_size; + struct page *page; + + if (!pages) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) { + seg_ipa_base = sgl_desc->sgl_entries[i].ipa_base; + seg_size = sgl_desc->sgl_entries[i].size; + n_pages_this_seg = PAGE_ALIGN(seg_size) / PAGE_SIZE; + page = phys_to_page(seg_ipa_base); + + BUG_ON(i >= npages); + for (j = 0; j < n_pages_this_seg; j++) + *(tmp++) = page++; + } + + vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL); + vfree(pages); + + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} + +static void mem_buf_unmap_kernel(struct mem_buf_import *import_buf) +{ + vunmap(import_buf->vaddr); +} + +static int mem_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + int ret = 0; + + mutex_lock(&import_buf->lock); + ret = mem_buf_map_user(import_buf, vma); + mutex_unlock(&import_buf->lock); + + if (ret) + pr_err_ratelimited("%s: failure mapping buffer to userspace\n", + __func__); + + return ret; +} + +static void mem_buf_dma_buf_release(struct dma_buf *dmabuf) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + + mem_buf_unimport_dma_buf(import_buf); +} + +static void *mem_buf_buffer_kmap_get(struct mem_buf_import *import_buf) +{ + void *vaddr; + + if (import_buf->kmap_cnt) { + import_buf->kmap_cnt++; + return import_buf->vaddr; + } + vaddr = mem_buf_map_kernel(import_buf); + if (IS_ERR(vaddr)) + return vaddr; + import_buf->vaddr = vaddr; + import_buf->kmap_cnt++; + return vaddr; +} + +static void mem_buf_buffer_kmap_put(struct mem_buf_import *import_buf) +{ + if (import_buf->kmap_cnt == 0) { + pr_warn_ratelimited("membuf client likely missing a call to dma_buf_kmap or dma_buf_vmap, pid:%d\n", + current->pid); + return; + } + + import_buf->kmap_cnt--; + if (!import_buf->kmap_cnt) { + mem_buf_unmap_kernel(import_buf); + import_buf->vaddr = NULL; + } +} + +static void *mem_buf_dma_buf_vmap(struct dma_buf *dmabuf) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + void *vaddr; + + mutex_lock(&import_buf->lock); + vaddr = mem_buf_buffer_kmap_get(import_buf); + mutex_unlock(&import_buf->lock); + + return vaddr; +} + +static void mem_buf_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + + mutex_lock(&import_buf->lock); + mem_buf_buffer_kmap_put(import_buf); + mutex_unlock(&import_buf->lock); +} + +static void *mem_buf_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) +{ + /* + * TODO: Once clients remove their hacks where they assume kmap(ed) + * addresses are virtually contiguous implement this properly + */ + void *vaddr = mem_buf_dma_buf_vmap(dmabuf); + + if (IS_ERR(vaddr)) + return vaddr; + + return vaddr + offset * PAGE_SIZE; +} + +static void mem_buf_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset, + void *ptr) +{ + /* + * TODO: Once clients remove their hacks where they assume kmap(ed) + * addresses are virtually contiguous implement this properly + */ + mem_buf_dma_buf_vunmap(dmabuf, ptr); +} + +static int mem_buf_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + if (!a->dma_mapped) + continue; + + dma_sync_sg_for_cpu(a->dev, a->table->sgl, + a->table->nents, direction); + } + mutex_unlock(&import_buf->lock); + return 0; +} + +static int mem_buf_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + if (!a->dma_mapped) + continue; + + dma_sync_sg_for_device(a->dev, a->table->sgl, a->table->nents, + direction); + } + mutex_unlock(&import_buf->lock); + return 0; +} + +static int mem_buf_sgl_sync_range(struct device *dev, struct scatterlist *sgl, + unsigned int nents, unsigned long offset, + unsigned long length, + enum dma_data_direction dir, bool for_cpu) +{ + int i; + struct scatterlist *sg; + unsigned int len = 0; + dma_addr_t sg_dma_addr; + + for_each_sg(sgl, sg, nents, i) { + if (sg_dma_len(sg) == 0) + break; + + if (i > 0) { + pr_warn_ratelimited("Partial cmo only supported with 1 segment\n" + "is dma_set_max_seg_size being set on dev:%s\n", + dev_name(dev)); + return -EINVAL; + } + } + + for_each_sg(sgl, sg, nents, i) { + unsigned int sg_offset, sg_left, size = 0; + + if (i == 0) + sg_dma_addr = sg_dma_address(sg); + + len += sg->length; + if (len <= offset) { + sg_dma_addr += sg->length; + continue; + } + + sg_left = len - offset; + sg_offset = sg->length - sg_left; + + size = (length < sg_left) ? length : sg_left; + if (for_cpu) + dma_sync_single_range_for_cpu(dev, sg_dma_addr, + sg_offset, size, dir); + else + dma_sync_single_range_for_device(dev, sg_dma_addr, + sg_offset, size, dir); + + offset += size; + length -= size; + sg_dma_addr += sg->length; + + if (length == 0) + break; + } + + return 0; +} + +static int mem_buf_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction dir, + unsigned int offset, + unsigned int len) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + int ret = 0; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + if (!a->dma_mapped) + continue; + + ret = mem_buf_sgl_sync_range(a->dev, a->table->sgl, + a->table->nents, offset, len, dir, + true); + + } + mutex_unlock(&import_buf->lock); + return ret; +} + +static int mem_buf_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, + unsigned int len) +{ + struct mem_buf_import *import_buf = dmabuf->priv; + struct mem_buf_dma_buf_attachment *a; + int ret = 0; + + mutex_lock(&import_buf->lock); + list_for_each_entry(a, &import_buf->attachments, list) { + ret = mem_buf_sgl_sync_range(a->dev, a->table->sgl, + a->table->nents, offset, len, + direction, false); + } + mutex_unlock(&import_buf->lock); + return ret; +} + +const struct dma_buf_ops mem_buf_dma_buf_ops = { + .map_dma_buf = mem_buf_dma_map_attachment, + .unmap_dma_buf = mem_buf_dma_unmap_attachment, + .mmap = mem_buf_mmap, + .release = mem_buf_dma_buf_release, + .attach = mem_buf_dma_buf_attach, + .detach = mem_buf_dma_buf_detatch, + .begin_cpu_access = mem_buf_dma_buf_begin_cpu_access, + .end_cpu_access = mem_buf_dma_buf_end_cpu_access, + .begin_cpu_access_partial = mem_buf_dma_buf_begin_cpu_access_partial, + .end_cpu_access_partial = mem_buf_dma_buf_end_cpu_access_partial, + .map = mem_buf_dma_buf_kmap, + .unmap = mem_buf_dma_buf_kunmap, + .vmap = mem_buf_dma_buf_vmap, + .vunmap = mem_buf_dma_buf_vunmap, +}; diff --git a/drivers/staging/android/ion/heaps/ion_carveout_heap.c b/drivers/staging/android/ion/heaps/ion_carveout_heap.c index 53347da25295..5b00e949c1a3 100644 --- a/drivers/staging/android/ion/heaps/ion_carveout_heap.c +++ b/drivers/staging/android/ion/heaps/ion_carveout_heap.c @@ -74,6 +74,7 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, phys_addr_t paddr; int ret; struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); + struct msm_ion_buf_lock_state *lock_state; struct device *dev = carveout_heap->heap.dev; table = kmalloc(sizeof(*table), GFP_KERNEL); @@ -83,10 +84,17 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, if (ret) goto err_free; + lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); + if (!lock_state) { + ret = -ENOMEM; + goto err_free_table; + } + buffer->priv_virt = lock_state; + paddr = ion_carveout_allocate(heap, size); if (paddr == ION_CARVEOUT_ALLOCATE_FAIL) { ret = -ENOMEM; - goto err_free_table; + goto err_free_umap; } sg_set_page(table->sgl, pfn_to_page(PFN_DOWN(paddr)), size, 0); @@ -99,6 +107,8 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, return 0; +err_free_umap: + kfree(lock_state); err_free_table: sg_free_table(table); err_free: @@ -110,18 +120,26 @@ static void ion_carveout_heap_free(struct ion_buffer *buffer) { struct ion_heap *heap = buffer->heap; struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; struct sg_table *table = buffer->sg_table; struct page *page = sg_page(table->sgl); phys_addr_t paddr = page_to_phys(page); struct device *dev = carveout_heap->heap.dev; - ion_buffer_zero(buffer); + mutex_lock(&buffer->lock); + if (hlos_accessible_buffer(buffer)) + ion_buffer_zero(buffer); + + if (lock_state && lock_state->locked) + pr_warn("%s: buffer is locked while being freed\n", __func__); + mutex_unlock(&buffer->lock); if (ion_buffer_cached(buffer)) ion_pages_sync_for_device(dev, page, buffer->size, DMA_BIDIRECTIONAL); ion_carveout_free(heap, paddr, buffer->size); + kfree(buffer->priv_virt); sg_free_table(table); kfree(table); } @@ -370,6 +388,7 @@ static void ion_sc_heap_free(struct ion_buffer *buffer) { struct ion_heap *child; struct sg_table *table = buffer->sg_table; + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; struct page *page = sg_page(table->sgl); phys_addr_t paddr = PFN_PHYS(page_to_pfn(page)); @@ -379,9 +398,16 @@ static void ion_sc_heap_free(struct ion_buffer *buffer) return; } + mutex_lock(&buffer->lock); if (hlos_accessible_buffer(buffer)) ion_buffer_zero(buffer); + + if (lock_state && lock_state->locked) + pr_warn("%s: buffer is locked while being freed\n", __func__); + mutex_unlock(&buffer->lock); + ion_carveout_free(child, paddr, buffer->size); + kfree(buffer->priv_virt); sg_free_table(table); kfree(table); } diff --git a/drivers/staging/android/ion/heaps/ion_cma_heap.c b/drivers/staging/android/ion/heaps/ion_cma_heap.c index 368a572f92a6..643cf9fa0553 100644 --- a/drivers/staging/android/ion/heaps/ion_cma_heap.c +++ b/drivers/staging/android/ion/heaps/ion_cma_heap.c @@ -40,6 +40,7 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, { struct ion_cma_heap *cma_heap = to_cma_heap(heap); struct sg_table *table; + struct msm_ion_buf_lock_state *lock_state; struct page *pages; unsigned long size = PAGE_ALIGN(len); unsigned long nr_pages = size >> PAGE_SHIFT; @@ -96,6 +97,12 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, sg_set_page(table->sgl, pages, size, 0); ; buffer->sg_table = table; + + lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); + if (!lock_state) + goto free_mem; + buffer->priv_virt = lock_state; + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; @@ -117,6 +124,7 @@ static void ion_cma_free(struct ion_buffer *buffer) /* release sg table */ sg_free_table(buffer->sg_table); kfree(buffer->sg_table); + kfree(buffer->priv_virt); } static struct ion_heap_ops ion_cma_ops = { diff --git a/drivers/staging/android/ion/heaps/ion_secure_util.c b/drivers/staging/android/ion/heaps/ion_secure_util.c index 88cc3a6e457f..6f0a6e6070f7 100644 --- a/drivers/staging/android/ion/heaps/ion_secure_util.c +++ b/drivers/staging/android/ion/heaps/ion_secure_util.c @@ -279,6 +279,8 @@ out: bool hlos_accessible_buffer(struct ion_buffer *buffer) { + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + if ((buffer->flags & ION_FLAG_SECURE) && !(buffer->flags & ION_FLAG_CP_HLOS) && !(buffer->flags & ION_FLAG_CP_SPSS_HLOS_SHARED)) @@ -287,6 +289,8 @@ bool hlos_accessible_buffer(struct ion_buffer *buffer) !(buffer->flags & ION_FLAG_CP_HLOS) && !(buffer->flags & ION_FLAG_CP_SPSS_HLOS_SHARED)) return false; + else if (lock_state && lock_state->locked) + return false; return true; } diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c index 1612af0c4fa4..5c4d7833dcbc 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -280,6 +280,7 @@ static int ion_system_heap_allocate(struct ion_heap *heap, unsigned long flags) { struct ion_system_heap *sys_heap = to_system_heap(heap); + struct msm_ion_buf_lock_state *lock_state; struct sg_table *table; struct sg_table table_sync = {0}; struct scatterlist *sg; @@ -399,6 +400,14 @@ static int ion_system_heap_allocate(struct ion_heap *heap, buffer->sg_table = table; if (nents_sync) sg_free_table(&table_sync); + + lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); + if (!lock_state) { + ret = -ENOMEM; + goto err_free_sg2; + } + buffer->priv_virt = lock_state; + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; @@ -432,10 +441,11 @@ err: return ret; } -void ion_system_heap_free(struct ion_buffer *buffer) +static void ion_system_heap_free(struct ion_buffer *buffer) { struct ion_heap *heap = buffer->heap; struct ion_system_heap *sys_heap = to_system_heap(heap); + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; struct sg_table *table = buffer->sg_table; struct scatterlist *sg; int i; @@ -443,8 +453,14 @@ void ion_system_heap_free(struct ion_buffer *buffer) if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) && !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { + mutex_lock(&buffer->lock); if (hlos_accessible_buffer(buffer)) ion_buffer_zero(buffer); + + if (lock_state && lock_state->locked) + pr_warn("%s: buffer is locked while being freed\n", + __func__); + mutex_unlock(&buffer->lock); } else if (vmid > 0) { if (ion_hyp_unassign_sg(table, &vmid, 1, true)) return; @@ -455,6 +471,7 @@ void ion_system_heap_free(struct ion_buffer *buffer) get_order(sg->length)); sg_free_table(table); kfree(table); + kfree(buffer->priv_virt); } static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, diff --git a/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c b/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c index eb7402f1fa85..468e9b4b517c 100644 --- a/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c +++ b/drivers/staging/android/ion/heaps/msm_ion_dma_buf.c @@ -160,6 +160,7 @@ static struct sg_table table = a->table; map_attrs = attachment->dma_map_attrs; + mutex_lock(&buffer->lock); if (!(buffer->flags & ION_FLAG_CACHED) || !hlos_accessible_buffer(buffer)) map_attrs |= DMA_ATTR_SKIP_CPU_SYNC; @@ -175,10 +176,10 @@ static struct sg_table !(buffer->flags & ION_FLAG_CACHED)) { pr_warn_ratelimited("dev:%s Cannot DMA map uncached buffer as IO-coherent attrs:0x%lx\n", dev_name(attachment->dev), map_attrs); + mutex_unlock(&buffer->lock); return ERR_PTR(-EINVAL); } - mutex_lock(&buffer->lock); if (map_attrs & DMA_ATTR_SKIP_CPU_SYNC) trace_ion_dma_map_cmo_skip(attachment->dev, ino, @@ -243,6 +244,7 @@ static void msm_ion_unmap_dma_buf(struct dma_buf_attachment *attachment, struct ion_dma_buf_attachment *a = attachment->priv; unsigned long ino = file_inode(attachment->dmabuf->file)->i_ino; + mutex_lock(&buffer->lock); map_attrs = attachment->dma_map_attrs; if (!(buffer->flags & ION_FLAG_CACHED) || !hlos_accessible_buffer(buffer)) @@ -253,7 +255,6 @@ static void msm_ion_unmap_dma_buf(struct dma_buf_attachment *attachment, dev_is_dma_coherent_hint_cached(attachment->dev)) map_attrs |= DMA_ATTR_FORCE_COHERENT; - mutex_lock(&buffer->lock); if (map_attrs & DMA_ATTR_SKIP_CPU_SYNC) trace_ion_dma_unmap_cmo_skip(attachment->dev, ino, @@ -297,31 +298,134 @@ void ion_pages_sync_for_device(struct device *dev, struct page *page, dma_sync_sg_for_device(dev, &sg, 1, dir); } +static void __msm_ion_vm_open(struct ion_buffer *buffer) +{ + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + + lock_state->vma_count++; +} + +static void msm_ion_vm_open(struct vm_area_struct *vma) +{ + struct ion_buffer *buffer = vma->vm_private_data; + + mutex_lock(&buffer->lock); + __msm_ion_vm_open(buffer); + mutex_unlock(&buffer->lock); +} + +static void msm_ion_vm_close(struct vm_area_struct *vma) +{ + struct ion_buffer *buffer = vma->vm_private_data; + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + + mutex_lock(&buffer->lock); + lock_state->vma_count--; + mutex_unlock(&buffer->lock); +} + +static const struct vm_operations_struct msm_ion_vma_ops = { + .open = msm_ion_vm_open, + .close = msm_ion_vm_close, +}; + static int msm_ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) { struct ion_buffer *buffer = dmabuf->priv; + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { pr_err_ratelimited("%s: this buffer cannot be mapped to userspace\n", __func__); + mutex_unlock(&buffer->lock); return -EINVAL; } if (!(buffer->flags & ION_FLAG_CACHED)) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - mutex_lock(&buffer->lock); /* now map it to userspace */ ret = ion_heap_map_user(buffer->heap, buffer, vma); - mutex_unlock(&buffer->lock); - if (ret) + if (ret) { pr_err("%s: failure mapping buffer to userspace\n", __func__); + } else if (lock_state) { + vma->vm_private_data = buffer; + vma->vm_ops = &msm_ion_vma_ops; + __msm_ion_vm_open(buffer); + } + + mutex_unlock(&buffer->lock); + return ret; +} + +static bool is_msm_ion_dma_buf(struct ion_buffer *buffer) +{ + return buffer->heap->buf_ops.attach == msm_ion_dma_buf_attach; +} + +int msm_ion_dma_buf_lock(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer; + struct msm_ion_buf_lock_state *lock_state; + int ret; + + if (!dmabuf) + return -EINVAL; + + buffer = dmabuf->priv; + lock_state = buffer->priv_virt; + + if ((!lock_state) || !is_msm_ion_dma_buf(buffer)) { + pr_err("%s: userspace map locking is not supported for this dma-buf\n", + __func__); + return -EINVAL; + } + + mutex_lock(&buffer->lock); + if (lock_state->locked) { + ret = -EINVAL; + pr_err("%s: buffer is already locked\n", __func__); + } else if (lock_state->vma_count) { + ret = -EBUSY; + } else { + ret = 0; + lock_state->locked = true; + } + mutex_unlock(&buffer->lock); return ret; } +EXPORT_SYMBOL(msm_ion_dma_buf_lock); + +void msm_ion_dma_buf_unlock(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer; + struct msm_ion_buf_lock_state *lock_state; + + if (!dmabuf) + return; + + buffer = dmabuf->priv; + lock_state = buffer->priv_virt; + + if (!lock_state || !is_msm_ion_dma_buf(buffer)) { + pr_err("%s: userspace map unlocking is not supported for this dma-buf\n", + __func__); + return; + } + + mutex_lock(&buffer->lock); + if (!lock_state->locked) + pr_warn("%s: buffer is already unlocked\n", __func__); + else + lock_state->locked = false; + mutex_unlock(&buffer->lock); +} +EXPORT_SYMBOL(msm_ion_dma_buf_unlock); static void msm_ion_dma_buf_release(struct dma_buf *dmabuf) { @@ -336,14 +440,13 @@ static void *msm_ion_dma_buf_vmap(struct dma_buf *dmabuf) struct ion_buffer *buffer = dmabuf->priv; void *vaddr = ERR_PTR(-EINVAL); - if (hlos_accessible_buffer(buffer)) { - mutex_lock(&buffer->lock); + mutex_lock(&buffer->lock); + if (hlos_accessible_buffer(buffer)) vaddr = msm_ion_buffer_kmap_get(buffer); - mutex_unlock(&buffer->lock); - } else { + else pr_warn_ratelimited("heap %s doesn't support map_kernel\n", buffer->heap->name); - } + mutex_unlock(&buffer->lock); return vaddr; } @@ -352,11 +455,10 @@ static void msm_ion_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) { struct ion_buffer *buffer = dmabuf->priv; - if (hlos_accessible_buffer(buffer)) { - mutex_lock(&buffer->lock); + mutex_lock(&buffer->lock); + if (hlos_accessible_buffer(buffer)) msm_ion_buffer_kmap_put(buffer); - mutex_unlock(&buffer->lock); - } + mutex_unlock(&buffer->lock); } static void *msm_ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) @@ -447,6 +549,7 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, unsigned long ino = file_inode(dmabuf->file)->i_ino; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_begin_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -461,7 +564,6 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); @@ -471,8 +573,6 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, trace_ion_begin_cpu_access_cmo_apply(dev, ino, true, true, direction); - - mutex_unlock(&buffer->lock); goto out; } @@ -491,8 +591,8 @@ static int msm_ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, trace_ion_begin_cpu_access_cmo_apply(a->dev, ino, true, true, direction); } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } @@ -504,6 +604,7 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, unsigned long ino = file_inode(dmabuf->file)->i_ino; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_end_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -518,7 +619,6 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); struct sg_table *table = buffer->sg_table; @@ -528,7 +628,6 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, trace_ion_end_cpu_access_cmo_apply(dev, ino, true, true, direction); - mutex_unlock(&buffer->lock); goto out; } @@ -547,9 +646,9 @@ static int msm_ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, trace_ion_end_cpu_access_cmo_apply(a->dev, ino, true, true, direction); } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } @@ -563,6 +662,7 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, unsigned long ino = file_inode(dmabuf->file)->i_ino; int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_begin_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -577,7 +677,6 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); struct sg_table *table = buffer->sg_table; @@ -591,7 +690,6 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, else trace_ion_begin_cpu_access_cmo_skip(dev, ino, true, true, dir); - mutex_unlock(&buffer->lock); goto out; } @@ -618,9 +716,9 @@ static int msm_ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, ret = tmp; } } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } @@ -635,6 +733,7 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, int ret = 0; + mutex_lock(&buffer->lock); if (!hlos_accessible_buffer(buffer)) { trace_ion_end_cpu_access_cmo_skip(NULL, ino, ion_buffer_cached(buffer), @@ -649,7 +748,6 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, goto out; } - mutex_lock(&buffer->lock); if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { struct device *dev = msm_ion_heap_device(buffer->heap); struct sg_table *table = buffer->sg_table; @@ -665,8 +763,6 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, trace_ion_end_cpu_access_cmo_skip(dev, ino, true, true, direction); - - mutex_unlock(&buffer->lock); goto out; } @@ -695,9 +791,9 @@ static int msm_ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, ret = tmp; } } - mutex_unlock(&buffer->lock); out: + mutex_unlock(&buffer->lock); return ret; } diff --git a/drivers/staging/android/ion/heaps/msm_ion_priv.h b/drivers/staging/android/ion/heaps/msm_ion_priv.h index a46a78341733..1841ed5c12a2 100644 --- a/drivers/staging/android/ion/heaps/msm_ion_priv.h +++ b/drivers/staging/android/ion/heaps/msm_ion_priv.h @@ -123,6 +123,11 @@ struct msm_ion_heap { struct ion_heap ion_heap; }; +struct msm_ion_buf_lock_state { + bool locked; + int vma_count; +}; + /** * struct ion_platform_data - array of platform heaps passed from board file * @nr: number of structures in the array diff --git a/include/linux/msm_ion.h b/include/linux/msm_ion.h index 743840ace66b..d9b577e573c1 100644 --- a/include/linux/msm_ion.h +++ b/include/linux/msm_ion.h @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -44,6 +45,10 @@ int msm_ion_heap_add_memory(int heap_id, struct sg_table *sgt); int msm_ion_heap_remove_memory(int heap_id, struct sg_table *sgt); +int msm_ion_dma_buf_lock(struct dma_buf *dmabuf); + +void msm_ion_dma_buf_unlock(struct dma_buf *dmabuf); + #else static inline struct device *msm_ion_heap_device_by_id(int heap_id) @@ -96,5 +101,14 @@ static inline int msm_ion_heap_remove_memory(int heap_id, struct sg_table *sgt) return -ENODEV; } +static inline int msm_ion_dma_buf_lock(struct dma_buf *dmabuf) +{ + return -ENODEV; +} + +static inline void msm_ion_dma_buf_unlock(struct dma_buf *dmabuf) +{ +} + #endif /* CONFIG_ION_MSM_HEAPS */ #endif /* _MSM_ION_H */ diff --git a/include/uapi/linux/mem-buf.h b/include/uapi/linux/mem-buf.h index 025e1acd3bc8..b258d8aa5b11 100644 --- a/include/uapi/linux/mem-buf.h +++ b/include/uapi/linux/mem-buf.h @@ -56,13 +56,13 @@ struct mem_buf_ion_data { }; /** - * struct mem_buf_alloc_ioctl_arg: An request to allocate memory from another + * struct mem_buf_alloc_ioctl_arg: A request to allocate memory from another * VM to other VMs. * @size: The size of the allocation. - * @nr_acl_entries: The number of ACL entries in @acl_list. * @acl_list: An array of structures, where each structure specifies a VMID * and the access permissions that the VMID will have to the memory to be * allocated. + * @nr_acl_entries: The number of ACL entries in @acl_list. * @src_mem_type: The type of memory that the source VM should allocate from. * This should be one of the mem_buf_mem_type enum values. * @src_data: A pointer to data that the source VM should interpret when @@ -70,25 +70,25 @@ struct mem_buf_ion_data { * @dst_mem_type: The type of memory that the destination VM should treat the * incoming allocation from the source VM as. This should be one of the * mem_buf_mem_type enum values. - * @dst_data: A pointer to data that the destination VM should interpret when - * adding the memory to the current VM. * @mem_buf_fd: A file descriptor representing the memory that was allocated * from the source VM and added to the current VM. Calling close() on this file * descriptor will deallocate the memory from the current VM, and return it * to the source VM. + * * @dst_data: A pointer to data that the destination VM should interpret when + * adding the memory to the current VM. * * All reserved fields must be zeroed out by the caller prior to invoking the * allocation IOCTL command with this argument. */ struct mem_buf_alloc_ioctl_arg { __u64 size; - __u32 nr_acl_entries; __u64 acl_list; + __u32 nr_acl_entries; __u32 src_mem_type; __u64 src_data; __u32 dst_mem_type; - __u64 dst_data; __u32 mem_buf_fd; + __u64 dst_data; __u64 reserved0; __u64 reserved1; __u64 reserved2; @@ -97,4 +97,66 @@ struct mem_buf_alloc_ioctl_arg { #define MEM_BUF_IOC_ALLOC _IOWR(MEM_BUF_IOC_MAGIC, 0,\ struct mem_buf_alloc_ioctl_arg) +/** + * struct mem_buf_export_ioctl_arg: An request to allocate memory from another + * VM to other VMs. + * @dma_buf_fd: The fd of the dma-buf that will be exported to another VM. + * @nr_acl_entries: The number of ACL entries in @acl_list. + * @acl_list: An array of structures, where each structure specifies a VMID + * and the access permissions that the VMID will have to the memory to be + * exported. + * @export_fd: An fd that corresponds to the buffer that was exported. This fd + * must be kept open until it is no longer required to export the memory to + * another VM. + * @memparcel_hdl: The handle associated with the memparcel that was created by + * granting access to the dma-buf for the VMIDs specified in @acl_list. + * + * Note: The buffer must not be mmap'ed by any process prior to invoking this + * IOCTL. The buffer must also be a cached buffer from a non-secure ION heap. + * + * All reserved fields must be zeroed out by the caller prior to invoking the + * export IOCTL command with this argument. + */ +struct mem_buf_export_ioctl_arg { + __u32 dma_buf_fd; + __u32 nr_acl_entries; + __u64 acl_list; + __u32 export_fd; + __u32 memparcel_hdl; + __u64 reserved0; + __u64 reserved1; + __u64 reserved2; +}; + +#define MEM_BUF_IOC_EXPORT _IOWR(MEM_BUF_IOC_MAGIC, 1,\ + struct mem_buf_export_ioctl_arg) + +/** + * struct mem_buf_import_ioctl_arg: A request to import memory from another + * VM as a dma-buf + * @memparcel_hdl: The handle that corresponds to the memparcel we are + * importing. + * @nr_acl_entries: The number of ACL entries in @acl_list. + * @acl_list: An array of structures, where each structure specifies a VMID + * and the access permissions that the VMID should have for the memparcel. + * @dma_buf_import_fd: A dma-buf file descriptor that the client can use to + * access the buffer. This fd must be closed to release the memory. + * + * All reserved fields must be zeroed out by the caller prior to invoking the + * import IOCTL command with this argument. + */ +struct mem_buf_import_ioctl_arg { + __u32 memparcel_hdl; + __u32 nr_acl_entries; + __u64 acl_list; + __u32 dma_buf_import_fd; + __u32 reserved0; + __u64 reserved1; + __u64 reserved2; + __u64 reserved3; +}; + +#define MEM_BUF_IOC_IMPORT _IOWR(MEM_BUF_IOC_MAGIC, 2,\ + struct mem_buf_import_ioctl_arg) + #endif /* _UAPI_LINUX_MEM_BUF_H */