From fe5698e79698cb5b3615bdb90c387ccbedeb13a7 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Fri, 26 Jul 2019 12:45:13 -0700 Subject: [PATCH] ion: Merge all ION related changes from msm-4.19 to msm-lahaina Merge all changes that deal with the ION memory manager from msm-4.19 to msm-lahaina as of commit: 9dd0018410fe: Merge "clk: qcom: npucc-kona: Enable safe config and HW_CTL for all RCGs". Change-Id: I0c362ff9e6938883535467f0809ee360a733638e Signed-off-by: Isaac J. Manjarres --- drivers/dma-buf/Kconfig | 11 + drivers/dma-buf/Makefile | 1 + drivers/dma-buf/dma-buf-ref.c | 114 +++ drivers/dma-buf/dma-buf.c | 26 +- drivers/staging/android/ion/Kconfig | 55 ++ drivers/staging/android/ion/Makefile | 7 +- .../staging/android/ion/ion_carveout_heap.c | 258 ++++++- drivers/staging/android/ion/ion_cma_heap.c | 167 +++- drivers/staging/android/ion/ion_heap.c | 6 +- drivers/staging/android/ion/ion_page_pool.c | 131 +++- drivers/staging/android/ion/ion_page_pool.h | 54 +- drivers/staging/android/ion/ion_secure_util.c | 271 +++++++ drivers/staging/android/ion/ion_secure_util.h | 25 + drivers/staging/android/ion/ion_system_heap.c | 724 ++++++++++++++--- drivers/staging/android/ion/ion_system_heap.h | 59 ++ .../android/ion/ion_system_secure_heap.c | 491 ++++++++++++ .../android/ion/ion_system_secure_heap.h | 24 + drivers/staging/android/ion/msm_ion_dma_buf.c | 726 ++++++++++++++++++ drivers/staging/android/ion/msm_ion_of.c | 504 ++++++++++++ drivers/staging/android/ion/msm_ion_priv.h | 165 ++++ drivers/staging/android/uapi/msm_ion.h | 122 +++ include/linux/dma-buf-ref.h | 30 + include/linux/dma-buf.h | 19 + include/linux/dma-mapping.h | 16 + include/linux/msm_ion_of.h | 11 + include/trace/events/ion.h | 170 ++++ include/uapi/linux/ion.h | 1 + include/uapi/linux/msm_ion.h | 1 + kernel/dma/coherent.c | 7 +- 29 files changed, 3992 insertions(+), 204 deletions(-) create mode 100644 drivers/dma-buf/dma-buf-ref.c create mode 100644 drivers/staging/android/ion/ion_secure_util.c create mode 100644 drivers/staging/android/ion/ion_secure_util.h create mode 100644 drivers/staging/android/ion/ion_system_heap.h create mode 100644 drivers/staging/android/ion/ion_system_secure_heap.c create mode 100644 drivers/staging/android/ion/ion_system_secure_heap.h create mode 100644 drivers/staging/android/ion/msm_ion_dma_buf.c create mode 100644 drivers/staging/android/ion/msm_ion_of.c create mode 100644 drivers/staging/android/ion/msm_ion_priv.h create mode 100644 drivers/staging/android/uapi/msm_ion.h create mode 100644 include/linux/dma-buf-ref.h create mode 100644 include/linux/msm_ion_of.h create mode 100644 include/trace/events/ion.h create mode 120000 include/uapi/linux/ion.h create mode 120000 include/uapi/linux/msm_ion.h diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index b6a9c2f1bc41..531215e4f39e 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -30,6 +30,17 @@ config SW_SYNC WARNING: improper use of this can result in deadlocking kernel drivers from userspace. Intended for test and debug only. +config DEBUG_DMA_BUF_REF + bool "DEBUG Reference Count" + depends on STACKDEPOT + depends on DMA_SHARED_BUFFER + default n + help + Save stack traces for every call to dma_buf_get and dma_buf_put, to + help debug memory leaks. Potential leaks may be found by manually + matching the get/put call stacks. This feature consumes extra memory + in order to save the stack traces using STACKDEPOT. + config UDMABUF bool "userspace dmabuf misc driver" default n diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile index e8c7310cb800..9edbd6253030 100644 --- a/drivers/dma-buf/Makefile +++ b/drivers/dma-buf/Makefile @@ -3,4 +3,5 @@ obj-y := dma-buf.o dma-fence.o dma-fence-array.o dma-fence-chain.o \ reservation.o seqno-fence.o obj-$(CONFIG_SYNC_FILE) += sync_file.o obj-$(CONFIG_SW_SYNC) += sw_sync.o sync_debug.o +obj-$(CONFIG_DEBUG_DMA_BUF_REF) += dma-buf-ref.o obj-$(CONFIG_UDMABUF) += udmabuf.o diff --git a/drivers/dma-buf/dma-buf-ref.c b/drivers/dma-buf/dma-buf-ref.c new file mode 100644 index 000000000000..696699323f3d --- /dev/null +++ b/drivers/dma-buf/dma-buf-ref.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include + +#define DMA_BUF_STACK_DEPTH (16) + +struct dma_buf_ref { + struct list_head list; + depot_stack_handle_t handle; + int count; +}; + +void dma_buf_ref_init(struct msm_dma_buf *msm_dma_buf) +{ + INIT_LIST_HEAD(&msm_dma_buf->refs); +} + +void dma_buf_ref_destroy(struct msm_dma_buf *msm_dma_buf) +{ + struct dma_buf_ref *r, *n; + struct dma_buf *dmabuf = &msm_dma_buf->dma_buf; + + mutex_lock(&dmabuf->lock); + list_for_each_entry_safe(r, n, &msm_dma_buf->refs, list) { + list_del(&r->list); + kfree(r); + } + mutex_unlock(&dmabuf->lock); +} + +static void dma_buf_ref_insert_handle(struct msm_dma_buf *msm_dma_buf, + depot_stack_handle_t handle, + int count) +{ + struct dma_buf_ref *r; + struct dma_buf *dmabuf = &msm_dma_buf->dma_buf; + + mutex_lock(&dmabuf->lock); + list_for_each_entry(r, &msm_dma_buf->refs, list) { + if (r->handle == handle) { + r->count += count; + goto out; + } + } + + r = kzalloc(sizeof(*r), GFP_KERNEL); + if (!r) + goto out; + + INIT_LIST_HEAD(&r->list); + r->handle = handle; + r->count = count; + list_add(&r->list, &msm_dma_buf->refs); + +out: + mutex_unlock(&dmabuf->lock); +} + +void dma_buf_ref_mod(struct msm_dma_buf *msm_dma_buf, int nr) +{ + unsigned long entries[DMA_BUF_STACK_DEPTH]; + struct stack_trace trace = { + .nr_entries = 0, + .entries = entries, + .max_entries = DMA_BUF_STACK_DEPTH, + .skip = 1 + }; + depot_stack_handle_t handle; + + save_stack_trace(&trace); + if (trace.nr_entries != 0 && + trace.entries[trace.nr_entries-1] == ULONG_MAX) + trace.nr_entries--; + + handle = depot_save_stack(&trace, GFP_KERNEL); + if (!handle) + return; + + dma_buf_ref_insert_handle(msm_dma_buf, handle, nr); +} + +/** + * Called with dmabuf->lock held + */ +int dma_buf_ref_show(struct seq_file *s, struct msm_dma_buf *msm_dma_buf) +{ + char *buf; + struct dma_buf_ref *ref; + int count = 0; + struct stack_trace trace; + + buf = (void *)__get_free_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + + list_for_each_entry(ref, &msm_dma_buf->refs, list) { + count += ref->count; + + seq_printf(s, "References: %d\n", ref->count); + depot_fetch_stack(ref->handle, &trace); + snprint_stack_trace(buf, PAGE_SIZE, &trace, 0); + seq_puts(s, buf); + seq_putc(s, '\n'); + } + + seq_printf(s, "Total references: %d\n\n\n", count); + free_page((unsigned long)buf); + + return 0; +} diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index d0f6e7290bcd..c7a20261bf5b 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -79,12 +79,14 @@ static struct file_system_type dma_buf_fs_type = { static int dma_buf_release(struct inode *inode, struct file *file) { + struct msm_dma_buf *msm_dma_buf; struct dma_buf *dmabuf; if (!is_dma_buf_file(file)) return -EINVAL; dmabuf = file->private_data; + msm_dma_buf = to_msm_dma_buf(dmabuf); BUG_ON(dmabuf->vmapping_counter); @@ -98,17 +100,18 @@ static int dma_buf_release(struct inode *inode, struct file *file) */ BUG_ON(dmabuf->cb_shared.active || dmabuf->cb_excl.active); - dmabuf->ops->release(dmabuf); - mutex_lock(&db_list.lock); list_del(&dmabuf->list_node); mutex_unlock(&db_list.lock); + dmabuf->ops->release(dmabuf); + dma_buf_ref_destroy(msm_dma_buf); + if (dmabuf->resv == (struct reservation_object *)&dmabuf[1]) reservation_object_fini(dmabuf->resv); module_put(dmabuf->owner); - kfree(dmabuf); + kfree(msm_dma_buf); return 0; } @@ -505,10 +508,11 @@ err_alloc_file: */ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) { + struct msm_dma_buf *msm_dma_buf; struct dma_buf *dmabuf; struct reservation_object *resv = exp_info->resv; struct file *file; - size_t alloc_size = sizeof(struct dma_buf); + size_t alloc_size = sizeof(struct msm_dma_buf); int ret; if (!exp_info->resv) @@ -528,12 +532,13 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) if (!try_module_get(exp_info->owner)) return ERR_PTR(-ENOENT); - dmabuf = kzalloc(alloc_size, GFP_KERNEL); - if (!dmabuf) { + msm_dma_buf = kzalloc(alloc_size, GFP_KERNEL); + if (!msm_dma_buf) { ret = -ENOMEM; goto err_module; } + dmabuf = &msm_dma_buf->dma_buf; dmabuf->priv = exp_info->priv; dmabuf->ops = exp_info->ops; dmabuf->size = exp_info->size; @@ -561,6 +566,9 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) mutex_init(&dmabuf->lock); INIT_LIST_HEAD(&dmabuf->attachments); + dma_buf_ref_init(msm_dma_buf); + dma_buf_ref_mod(msm_dma_buf, 1); + mutex_lock(&db_list.lock); list_add(&dmabuf->list_node, &db_list.head); mutex_unlock(&db_list.lock); @@ -568,7 +576,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) return dmabuf; err_dmabuf: - kfree(dmabuf); + kfree(msm_dma_buf); err_module: module_put(exp_info->owner); return ERR_PTR(ret); @@ -620,6 +628,7 @@ struct dma_buf *dma_buf_get(int fd) fput(file); return ERR_PTR(-EINVAL); } + dma_buf_ref_mod(to_msm_dma_buf(file->private_data), 1); return file->private_data; } @@ -640,6 +649,7 @@ void dma_buf_put(struct dma_buf *dmabuf) if (WARN_ON(!dmabuf || !dmabuf->file)) return; + dma_buf_ref_mod(to_msm_dma_buf(dmabuf), -1); fput(dmabuf->file); } EXPORT_SYMBOL_GPL(dma_buf_put); @@ -1281,6 +1291,8 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) seq_printf(s, "Total %d devices attached\n\n", attach_count); + dma_buf_ref_show(s, to_msm_dma_buf(buf_obj)); + count++; size += buf_obj->size; mutex_unlock(&buf_obj->lock); diff --git a/drivers/staging/android/ion/Kconfig b/drivers/staging/android/ion/Kconfig index 7a32464d20fa..68dae7e17e20 100644 --- a/drivers/staging/android/ion/Kconfig +++ b/drivers/staging/android/ion/Kconfig @@ -34,3 +34,58 @@ config ION_CMA_HEAP Choose this option to enable CMA heaps with Ion. This heap is backed by the Contiguous Memory Allocator (CMA). If your system has these regions, you should say Y here. + +config ION_MSM_HEAPS + tristate "MSM platform-based Ion heaps support" + depends on ION && DMA_CMA && QCOM_SECURE_BUFFER + help + Enable this option to enable platform-based Ion heaps. The heaps + will register with the Ion core framework, at which point userspace + clients can allocate different types of memory (e.g. secure, cached, + and uncached) from the different types of heaps. The MSM heaps + allow Ion buffers to be shared through the shared DMA buffer framework + and the heaps implement their own cache maintenance operations. + If you're not sure, enable here. + +config ION_FORCE_DMA_SYNC + bool "Force ION to always DMA sync buffer memory" + depends on ION + help + Force ION to DMA sync buffer memory when it is allocated and to + always DMA sync the buffer memory on calls to begin/end cpu + access. This makes ION DMA sync behavior similar to that of the + older version of ION. + We generally don't want to enable this config as it breaks the + cache maintenance model. + If you're not sure say N here. + +config ION_DEFER_FREE_NO_SCHED_IDLE + bool "Increases the priority of ION defer free thread" + depends on ION + help + Certain heaps such as the system heaps make use of a low priority + thread to help free buffer allocations back to the pool which might + result in future allocations requests going to the buddy instead of + the pool when there is a high buffer allocation rate. + Choose this option to remove the SCHED_IDLE flag in case of defer + free thereby increasing the priority of defer free thread. + if you're not sure say Y here. + +config ION_POOL_AUTO_REFILL + bool "Refill the ION heap pools automatically" + depends on ION + help + Choose this option to refill the ION system heap pools (non-secure) + automatically when the pool pages count becomes lower than a set low mark. + This refilling is done by worker thread which is invoked asynchronously + when the pool count reaches below low mark. + if you're not sure say Y here. + +config ION_POOL_FILL_MARK + int "ion pool fillmark size in MB" + depends on ION_POOL_AUTO_REFILL + range 16 256 + default 100 + help + Set the fillmark of the pool in terms of mega bytes and the lowmark is + ION_POOL_LOW_MARK_PERCENT of fillmark value. diff --git a/drivers/staging/android/ion/Makefile b/drivers/staging/android/ion/Makefile index 0b3f169b81da..8a6bd4dd6b70 100644 --- a/drivers/staging/android/ion/Makefile +++ b/drivers/staging/android/ion/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ION) += ion.o ion_heap.o -obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_system_heap.o ion_page_pool.o -obj-$(CONFIG_ION_CARVEOUT_HEAP) += ion_carveout_heap.o -obj-$(CONFIG_ION_CMA_HEAP) += ion_cma_heap.o +obj-$(CONFIG_ION_MSM_HEAPS) += msm_ion_heaps.o +msm_ion_heaps-objs += msm_ion_of.o msm_ion_dma_buf.o ion_page_pool.o \ + ion_system_heap.o ion_carveout_heap.o ion_system_secure_heap.o \ + ion_cma_heap.o ion_secure_util.o diff --git a/drivers/staging/android/ion/ion_carveout_heap.c b/drivers/staging/android/ion/ion_carveout_heap.c index bb9d614767a2..fd7e1d18ee91 100644 --- a/drivers/staging/android/ion/ion_carveout_heap.c +++ b/drivers/staging/android/ion/ion_carveout_heap.c @@ -12,21 +12,29 @@ #include #include #include - +#include +#include +#include +#include #include "ion.h" +#include "msm_ion_priv.h" +#include "ion_secure_util.h" #define ION_CARVEOUT_ALLOCATE_FAIL -1 +#define to_carveout_heap(_heap) \ + container_of(to_msm_ion_heap(_heap), struct ion_carveout_heap, heap) + struct ion_carveout_heap { - struct ion_heap heap; + struct msm_ion_heap heap; struct gen_pool *pool; + phys_addr_t base; }; static phys_addr_t ion_carveout_allocate(struct ion_heap *heap, unsigned long size) { - struct ion_carveout_heap *carveout_heap = - container_of(heap, struct ion_carveout_heap, heap); + struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); unsigned long offset = gen_pool_alloc(carveout_heap->pool, size); if (!offset) @@ -38,8 +46,7 @@ static phys_addr_t ion_carveout_allocate(struct ion_heap *heap, static void ion_carveout_free(struct ion_heap *heap, phys_addr_t addr, unsigned long size) { - struct ion_carveout_heap *carveout_heap = - container_of(heap, struct ion_carveout_heap, heap); + struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); if (addr == ION_CARVEOUT_ALLOCATE_FAIL) return; @@ -55,6 +62,8 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, struct sg_table *table; phys_addr_t paddr; int ret; + struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); + struct device *dev = carveout_heap->heap.dev; table = kmalloc(sizeof(*table), GFP_KERNEL); if (!table) @@ -72,6 +81,11 @@ static int ion_carveout_heap_allocate(struct ion_heap *heap, sg_set_page(table->sgl, pfn_to_page(PFN_DOWN(paddr)), size, 0); buffer->sg_table = table; + if (ion_buffer_cached(buffer)) + ion_pages_sync_for_device(dev, sg_page(table->sgl), + buffer->size, DMA_FROM_DEVICE); + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); + return 0; err_free_table: @@ -84,12 +98,18 @@ err_free: static void ion_carveout_heap_free(struct ion_buffer *buffer) { struct ion_heap *heap = buffer->heap; + struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); struct sg_table *table = buffer->sg_table; struct page *page = sg_page(table->sgl); - phys_addr_t paddr = PFN_PHYS(page_to_pfn(page)); + phys_addr_t paddr = page_to_phys(page); + struct device *dev = carveout_heap->heap.dev; ion_heap_buffer_zero(buffer); + if (ion_buffer_cached(buffer)) + ion_pages_sync_for_device(dev, page, buffer->size, + DMA_BIDIRECTIONAL); + ion_carveout_free(heap, paddr, buffer->size); sg_free_table(table); kfree(table); @@ -103,14 +123,23 @@ static struct ion_heap_ops carveout_heap_ops = { .unmap_kernel = ion_heap_unmap_kernel, }; -struct ion_heap *ion_carveout_heap_create(phys_addr_t base, size_t size) +static struct ion_heap * +__ion_carveout_heap_create(struct ion_platform_heap *heap_data, + bool sync) { struct ion_carveout_heap *carveout_heap; int ret; struct page *page; + size_t size; + struct device *dev = (struct device *)heap_data->priv; + + page = pfn_to_page(PFN_DOWN(heap_data->base)); + size = heap_data->size; + + if (sync) + ion_pages_sync_for_device(dev, page, size, DMA_BIDIRECTIONAL); - page = pfn_to_page(PFN_DOWN(base)); ret = ion_heap_pages_zero(page, size, pgprot_writecombine(PAGE_KERNEL)); if (ret) return ERR_PTR(ret); @@ -124,10 +153,211 @@ struct ion_heap *ion_carveout_heap_create(phys_addr_t base, size_t size) kfree(carveout_heap); return ERR_PTR(-ENOMEM); } - gen_pool_add(carveout_heap->pool, base, size, -1); - carveout_heap->heap.ops = &carveout_heap_ops; - carveout_heap->heap.type = ION_HEAP_TYPE_CARVEOUT; - carveout_heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE; + carveout_heap->base = heap_data->base; + carveout_heap->heap.dev = dev; + gen_pool_add(carveout_heap->pool, carveout_heap->base, heap_data->size, + -1); + carveout_heap->heap.ion_heap.ops = &carveout_heap_ops; + carveout_heap->heap.ion_heap.type = ION_HEAP_TYPE_CARVEOUT; + carveout_heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE; - return &carveout_heap->heap; + return &carveout_heap->heap.ion_heap; +} + +struct ion_heap *ion_carveout_heap_create(struct ion_platform_heap *heap_data) +{ + return __ion_carveout_heap_create(heap_data, true); +} + +static void ion_carveout_heap_destroy(struct ion_heap *heap) +{ + struct ion_carveout_heap *carveout_heap = to_carveout_heap(heap); + + gen_pool_destroy(carveout_heap->pool); + kfree(carveout_heap); + carveout_heap = NULL; +} + +struct ion_sc_entry { + struct list_head list; + struct ion_heap *heap; + u32 token; +}; + +struct ion_sc_heap { + struct msm_ion_heap heap; + struct list_head children; +}; + +static struct ion_heap *ion_sc_find_child(struct ion_heap *heap, u32 flags) +{ + struct ion_sc_heap *manager; + struct ion_sc_entry *entry; + + manager = container_of(to_msm_ion_heap(heap), struct ion_sc_heap, heap); + flags = flags & ION_FLAGS_CP_MASK; + list_for_each_entry(entry, &manager->children, list) { + if (entry->token == flags) + return entry->heap; + } + return NULL; +} + +static int ion_sc_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, unsigned long len, + unsigned long flags) +{ + struct ion_heap *child; + + /* cache maintenance is not possible on secure memory */ + flags &= ~((unsigned long)ION_FLAG_CACHED); + buffer->flags = flags; + + child = ion_sc_find_child(heap, flags); + if (!child) + return -EINVAL; + return ion_carveout_heap_allocate(child, buffer, len, flags); +} + +static void ion_sc_heap_free(struct ion_buffer *buffer) +{ + struct ion_heap *child; + struct sg_table *table = buffer->sg_table; + struct page *page = sg_page(table->sgl); + phys_addr_t paddr = PFN_PHYS(page_to_pfn(page)); + + child = ion_sc_find_child(buffer->heap, buffer->flags); + if (!child) { + WARN(1, "ion_secure_carvout: invalid buffer flags on free. Memory will be leaked\n."); + return; + } + + ion_carveout_free(child, paddr, buffer->size); + sg_free_table(table); + kfree(table); +} + +static struct ion_heap_ops ion_sc_heap_ops = { + .allocate = ion_sc_heap_allocate, + .free = ion_sc_heap_free, +}; + +static int ion_sc_get_dt_token(struct ion_sc_entry *entry, + struct device_node *np, u64 base, u64 size) +{ + u32 token; + int ret = -EINVAL; + + if (of_property_read_u32(np, "token", &token)) + return -EINVAL; + + ret = ion_hyp_assign_from_flags(base, size, token); + if (ret) + pr_err("secure_carveout_heap: Assign token 0x%x failed\n", + token); + else + entry->token = token; + + return ret; +} + +static int ion_sc_add_child(struct ion_sc_heap *manager, + struct device_node *np) +{ + struct device *dev = manager->heap.dev; + struct ion_platform_heap heap_data = {0}; + struct ion_sc_entry *entry; + struct device_node *phandle; + const __be32 *basep; + u64 base, size; + int ret; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + INIT_LIST_HEAD(&entry->list); + + phandle = of_parse_phandle(np, "memory-region", 0); + if (!phandle) + goto out_free; + + basep = of_get_address(phandle, 0, &size, NULL); + if (!basep) + goto out_free; + + base = of_translate_address(phandle, basep); + if (base == OF_BAD_ADDR) + goto out_free; + + heap_data.priv = dev; + heap_data.base = base; + heap_data.size = size; + + /* This will zero memory initially */ + entry->heap = __ion_carveout_heap_create(&heap_data, false); + if (IS_ERR(entry->heap)) + goto out_free; + + ret = ion_sc_get_dt_token(entry, np, base, size); + if (ret) + goto out_free_carveout; + + list_add(&entry->list, &manager->children); + dev_info(dev, "ion_secure_carveout: creating heap@0x%llx, size 0x%llx\n", + base, size); + return 0; + +out_free_carveout: + ion_carveout_heap_destroy(entry->heap); +out_free: + kfree(entry); + return -EINVAL; +} + +static void ion_secure_carveout_heap_destroy(struct ion_heap *heap) +{ + struct ion_sc_heap *manager = + container_of(to_msm_ion_heap(heap), struct ion_sc_heap, heap); + struct ion_sc_entry *entry, *tmp; + + list_for_each_entry_safe(entry, tmp, &manager->children, list) { + ion_carveout_heap_destroy(entry->heap); + kfree(entry); + } + kfree(manager); +} + +struct ion_heap * +ion_secure_carveout_heap_create(struct ion_platform_heap *heap_data) +{ + struct device *dev = heap_data->priv; + int ret; + struct ion_sc_heap *manager; + struct device_node *np; + + manager = kzalloc(sizeof(*manager), GFP_KERNEL); + if (!manager) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&manager->children); + manager->heap.dev = dev; + + for_each_child_of_node(dev->of_node, np) { + ret = ion_sc_add_child(manager, np); + if (ret) { + dev_err(dev, "Creating child pool %s failed\n", + np->name); + goto err; + } + } + + manager->heap.ion_heap.ops = &ion_sc_heap_ops; + manager->heap.ion_heap.type = + (enum ion_heap_type)ION_HEAP_TYPE_SECURE_CARVEOUT; + return &manager->heap.ion_heap; + +err: + ion_secure_carveout_heap_destroy(&manager->heap.ion_heap); + return ERR_PTR(-EINVAL); } diff --git a/drivers/staging/android/ion/ion_cma_heap.c b/drivers/staging/android/ion/ion_cma_heap.c index bf65e67ef9d8..1c1d64c23cdc 100644 --- a/drivers/staging/android/ion/ion_cma_heap.c +++ b/drivers/staging/android/ion/ion_cma_heap.c @@ -4,6 +4,8 @@ * * Copyright (C) Linaro 2012 * Author: for ST-Ericsson. + * + * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. */ #include @@ -12,16 +14,20 @@ #include #include #include +#include #include #include "ion.h" +#include "msm_ion_priv.h" +#include "ion_secure_util.h" struct ion_cma_heap { - struct ion_heap heap; + struct msm_ion_heap heap; struct cma *cma; }; -#define to_cma_heap(x) container_of(x, struct ion_cma_heap, heap) +#define to_cma_heap(x) \ + container_of(to_msm_ion_heap(x), struct ion_cma_heap, heap) /* ION CMA heap operations functions */ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, @@ -35,6 +41,7 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, unsigned long nr_pages = size >> PAGE_SHIFT; unsigned long align = get_order(size); int ret; + struct device *dev = cma_heap->heap.dev; if (align > CONFIG_CMA_ALIGNMENT) align = CONFIG_CMA_ALIGNMENT; @@ -43,22 +50,30 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, if (!pages) return -ENOMEM; - if (PageHighMem(pages)) { - unsigned long nr_clear_pages = nr_pages; - struct page *page = pages; + if (!(flags & ION_FLAG_SECURE)) { + if (PageHighMem(pages)) { + unsigned long nr_clear_pages = nr_pages; + struct page *page = pages; - while (nr_clear_pages > 0) { - void *vaddr = kmap_atomic(page); + while (nr_clear_pages > 0) { + void *vaddr = kmap_atomic(page); - memset(vaddr, 0, PAGE_SIZE); - kunmap_atomic(vaddr); - page++; - nr_clear_pages--; + memset(vaddr, 0, PAGE_SIZE); + kunmap_atomic(vaddr); + page++; + nr_clear_pages--; + } + } else { + memset(page_address(pages), 0, size); } - } else { - memset(page_address(pages), 0, size); } + if (MAKE_ION_ALLOC_DMA_READY || + (flags & ION_FLAG_SECURE) || + (!ion_buffer_cached(buffer))) + ion_pages_sync_for_device(dev, pages, size, + DMA_BIDIRECTIONAL); + table = kmalloc(sizeof(*table), GFP_KERNEL); if (!table) goto err; @@ -71,6 +86,7 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, buffer->priv_virt = pages; buffer->sg_table = table; + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; free_mem: @@ -101,38 +117,125 @@ static struct ion_heap_ops ion_cma_ops = { .unmap_kernel = ion_heap_unmap_kernel, }; -static struct ion_heap *__ion_cma_heap_create(struct cma *cma) +struct ion_heap *ion_cma_heap_create(struct ion_platform_heap *data) { struct ion_cma_heap *cma_heap; + struct device *dev = (struct device *)data->priv; + + if (!dev->cma_area) + return ERR_PTR(-EINVAL); cma_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL); if (!cma_heap) return ERR_PTR(-ENOMEM); - cma_heap->heap.ops = &ion_cma_ops; - cma_heap->cma = cma; - cma_heap->heap.type = ION_HEAP_TYPE_DMA; - return &cma_heap->heap; + cma_heap->heap.ion_heap.ops = &ion_cma_ops; + /* + * get device from private heaps data, later it will be + * used to make the link with reserved CMA memory + */ + cma_heap->heap.dev = dev; + cma_heap->cma = dev->cma_area; + cma_heap->heap.ion_heap.type = ION_HEAP_TYPE_DMA; + return &cma_heap->heap.ion_heap; } -static int __ion_add_cma_heaps(struct cma *cma, void *data) +static void ion_secure_cma_free(struct ion_buffer *buffer) { - struct ion_heap *heap; + if (ion_hyp_unassign_sg_from_flags(buffer->sg_table, buffer->flags, + true)) + return; - heap = __ion_cma_heap_create(cma); - if (IS_ERR(heap)) - return PTR_ERR(heap); - - heap->name = cma_get_name(cma); - - ion_device_add_heap(heap); - return 0; + ion_cma_free(buffer); } -static int ion_add_cma_heaps(void) +static int ion_secure_cma_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, unsigned long len, + unsigned long flags) { - cma_for_each_area(__ion_add_cma_heaps, NULL); - return 0; + int ret; + struct ion_cma_heap *cma_heap = to_cma_heap(heap); + + if (!(flags & ION_FLAGS_CP_MASK)) + return -EINVAL; + + ret = ion_cma_allocate(heap, buffer, len, flags); + if (ret) { + dev_err(cma_heap->heap.dev, "Unable to allocate cma buffer\n"); + goto out; + } + + ret = ion_hyp_assign_sg_from_flags(buffer->sg_table, flags, true); + if (ret) { + if (ret == -EADDRNOTAVAIL) { + goto out_free_buf; + } else { + ion_cma_free(buffer); + goto out; + } + } + + return ret; + +out_free_buf: + ion_secure_cma_free(buffer); +out: + return ret; +} + +static void *ion_secure_cma_map_kernel(struct ion_heap *heap, + struct ion_buffer *buffer) +{ + if (!hlos_accessible_buffer(buffer)) { + pr_info("%s: Mapping non-HLOS accessible buffer disallowed\n", + __func__); + return NULL; + } + return ion_heap_map_kernel(heap, buffer); +} + +static int ion_secure_cma_map_user(struct ion_heap *mapper, + struct ion_buffer *buffer, + struct vm_area_struct *vma) +{ + if (!hlos_accessible_buffer(buffer)) { + pr_info("%s: Mapping non-HLOS accessible buffer disallowed\n", + __func__); + return -EINVAL; + } + return ion_heap_map_user(mapper, buffer, vma); +} + +static struct ion_heap_ops ion_secure_cma_ops = { + .allocate = ion_secure_cma_allocate, + .free = ion_secure_cma_free, + .map_user = ion_secure_cma_map_user, + .map_kernel = ion_secure_cma_map_kernel, + .unmap_kernel = ion_heap_unmap_kernel, +}; + +struct ion_heap *ion_cma_secure_heap_create(struct ion_platform_heap *data) +{ + struct ion_cma_heap *cma_heap; + struct device *dev = (struct device *)data->priv; + + if (!dev->cma_area) + return ERR_PTR(-EINVAL); + + cma_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL); + + if (!cma_heap) + return ERR_PTR(-ENOMEM); + + cma_heap->heap.ion_heap.ops = &ion_secure_cma_ops; + /* + * get device from private heaps data, later it will be + * used to make the link with reserved CMA memory + */ + cma_heap->heap.dev = dev; + cma_heap->cma = dev->cma_area; + cma_heap->heap.ion_heap.type = + (enum ion_heap_type)ION_HEAP_TYPE_HYP_CMA; + return &cma_heap->heap.ion_heap; } -device_initcall(ion_add_cma_heaps); diff --git a/drivers/staging/android/ion/ion_heap.c b/drivers/staging/android/ion/ion_heap.c index 473b465724f1..a837fa010375 100644 --- a/drivers/staging/android/ion/ion_heap.c +++ b/drivers/staging/android/ion/ion_heap.c @@ -244,8 +244,9 @@ static int ion_heap_deferred_free(void *data) int ion_heap_init_deferred_free(struct ion_heap *heap) { +#ifndef CONFIG_ION_DEFER_FREE_NO_SCHED_IDLE struct sched_param param = { .sched_priority = 0 }; - +#endif INIT_LIST_HEAD(&heap->free_list); init_waitqueue_head(&heap->waitqueue); heap->task = kthread_run(ion_heap_deferred_free, heap, @@ -255,8 +256,9 @@ int ion_heap_init_deferred_free(struct ion_heap *heap) __func__); return PTR_ERR_OR_ZERO(heap->task); } +#ifndef CONFIG_ION_DEFER_FREE_NO_SCHED_IDLE sched_setscheduler(heap->task, SCHED_IDLE, ¶m); - +#endif return 0; } diff --git a/drivers/staging/android/ion/ion_page_pool.c b/drivers/staging/android/ion/ion_page_pool.c index f1bc165e644e..a902885abf3c 100644 --- a/drivers/staging/android/ion/ion_page_pool.c +++ b/drivers/staging/android/ion/ion_page_pool.c @@ -10,8 +10,45 @@ #include #include +#include "ion.h" +#include "msm_ion_priv.h" #include "ion_page_pool.h" +/* do a simple check to see if we are in any low memory situation */ +static bool pool_refill_ok(struct ion_page_pool *pool) +{ + struct zonelist *zonelist; + struct zoneref *z; + struct zone *zone; + int mark; + enum zone_type classzone_idx = gfp_zone(pool->gfp_mask); + s64 delta; + + /* check if we are within the refill defer window */ + delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime); + if (delta < ION_POOL_REFILL_DEFER_WINDOW_MS) + return false; + + zonelist = node_zonelist(numa_node_id(), pool->gfp_mask); + /* + * make sure that if we allocate a pool->order page from buddy, + * we don't put the zone watermarks go below the high threshold. + * This makes sure there's no unwanted repetitive refilling and + * reclaiming of buddy pages on the pool. + */ + for_each_zone_zonelist(zone, z, zonelist, classzone_idx) { + mark = high_wmark_pages(zone); + mark += 1 << pool->order; + if (!zone_watermark_ok_safe(zone, pool->order, mark, + classzone_idx)) { + pool->last_low_watermark_ktime = ktime_get(); + return false; + } + } + + return true; +} + static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool) { if (fatal_signal_pending(current)) @@ -36,11 +73,34 @@ static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) pool->low_count++; } + atomic_inc(&pool->count); mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, - 1 << pool->order); + (1 << (PAGE_SHIFT + pool->order))); mutex_unlock(&pool->mutex); } +void ion_page_pool_refill(struct ion_page_pool *pool) +{ + struct page *page; + gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY; + struct device *dev = pool->heap_dev; + + /* skip refilling order 0 pools */ + if (!pool->order) + return; + + while (!pool_fillmark_reached(pool) && pool_refill_ok(pool)) { + page = alloc_pages(gfp_refill, pool->order); + if (!page) + break; + if (!pool->cached) + ion_pages_sync_for_device(dev, page, + PAGE_SIZE << pool->order, + DMA_BIDIRECTIONAL); + ion_page_pool_add(pool, page); + } +} + static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) { struct page *page; @@ -55,39 +115,73 @@ static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) pool->low_count--; } + atomic_dec(&pool->count); list_del(&page->lru); mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, - -(1 << pool->order)); + -(1 << (PAGE_SHIFT + pool->order))); return page; } -struct page *ion_page_pool_alloc(struct ion_page_pool *pool) +struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool) { struct page *page = NULL; BUG_ON(!pool); - mutex_lock(&pool->mutex); - if (pool->high_count) - page = ion_page_pool_remove(pool, true); - else if (pool->low_count) - page = ion_page_pool_remove(pool, false); - mutex_unlock(&pool->mutex); + if (fatal_signal_pending(current)) + return ERR_PTR(-EINTR); + + if (*from_pool && mutex_trylock(&pool->mutex)) { + if (pool->high_count) + page = ion_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + } + if (!page) { + page = ion_page_pool_alloc_pages(pool); + *from_pool = false; + } if (!page) - page = ion_page_pool_alloc_pages(pool); + return ERR_PTR(-ENOMEM); + return page; +} +/* + * Tries to allocate from only the specified Pool and returns NULL otherwise + */ +struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *pool) +{ + struct page *page = NULL; + + if (!pool) + return ERR_PTR(-EINVAL); + + if (mutex_trylock(&pool->mutex)) { + if (pool->high_count) + page = ion_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + } + + if (!page) + return ERR_PTR(-ENOMEM); return page; } void ion_page_pool_free(struct ion_page_pool *pool, struct page *page) { - BUG_ON(pool->order != compound_order(page)); - ion_page_pool_add(pool, page); } -static int ion_page_pool_total(struct ion_page_pool *pool, bool high) +void ion_page_pool_free_immediate(struct ion_page_pool *pool, struct page *page) +{ + ion_page_pool_free_pages(pool, page); +} + +int ion_page_pool_total(struct ion_page_pool *pool, bool high) { int count = pool->low_count; @@ -131,20 +225,21 @@ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, return freed; } -struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order, + bool cached) { - struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); + struct ion_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); if (!pool) return NULL; - pool->high_count = 0; - pool->low_count = 0; INIT_LIST_HEAD(&pool->low_items); INIT_LIST_HEAD(&pool->high_items); - pool->gfp_mask = gfp_mask | __GFP_COMP; + pool->gfp_mask = gfp_mask; pool->order = order; mutex_init(&pool->mutex); plist_node_init(&pool->list, order); + if (cached) + pool->cached = true; return pool; } diff --git a/drivers/staging/android/ion/ion_page_pool.h b/drivers/staging/android/ion/ion_page_pool.h index e205d0086833..646a5224532a 100644 --- a/drivers/staging/android/ion/ion_page_pool.h +++ b/drivers/staging/android/ion/ion_page_pool.h @@ -13,6 +13,19 @@ #include #include +/* ION page pool marks in bytes */ +#ifdef CONFIG_ION_POOL_AUTO_REFILL +#define ION_POOL_FILL_MARK (CONFIG_ION_POOL_FILL_MARK * SZ_1M) +#define POOL_LOW_MARK_PERCENT 40UL +#define ION_POOL_LOW_MARK ((ION_POOL_FILL_MARK * POOL_LOW_MARK_PERCENT) / 100) +#else +#define ION_POOL_FILL_MARK 0UL +#define ION_POOL_LOW_MARK 0UL +#endif + +/* if low watermark of zones have reached, defer the refill in this window */ +#define ION_POOL_REFILL_DEFER_WINDOW_MS 10 + /** * functions for creating and destroying a heap pool -- allows you * to keep a pool of pre allocated memory to use from your heap. Keeping @@ -25,13 +38,18 @@ * struct ion_page_pool - pagepool struct * @high_count: number of highmem items in the pool * @low_count: number of lowmem items in the pool + * @count: total number of pages/items in the pool * @high_items: list of highmem items * @low_items: list of lowmem items + * @last_low_watermark_ktime: most recent time at which the zone watermarks were + * low * @mutex: lock protecting this struct and especially the count * item list * @gfp_mask: gfp_mask to use from alloc * @order: order of pages in the pool * @list: plist node for list of pools + * @cached: it's cached pool or not + * @heap_dev: device for the ion heap associated with this pool * * Allows you to keep a pool of pre allocated pages to use from your heap. * Keeping a pool of pages that is ready for dma, ie any cached mapping have @@ -41,18 +59,28 @@ struct ion_page_pool { int high_count; int low_count; + atomic_t count; struct list_head high_items; struct list_head low_items; + ktime_t last_low_watermark_ktime; struct mutex mutex; gfp_t gfp_mask; unsigned int order; struct plist_node list; + bool cached; + struct device *heap_dev; }; -struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order); +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order, + bool cached); void ion_page_pool_destroy(struct ion_page_pool *pool); -struct page *ion_page_pool_alloc(struct ion_page_pool *pool); +struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool); void ion_page_pool_free(struct ion_page_pool *pool, struct page *page); +struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *a); +void ion_page_pool_free_immediate(struct ion_page_pool *pool, + struct page *page); +int ion_page_pool_total(struct ion_page_pool *pool, bool high); +size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid); /** ion_page_pool_shrink - shrinks the size of the memory cached in the pool * @pool: the pool @@ -63,4 +91,26 @@ void ion_page_pool_free(struct ion_page_pool *pool, struct page *page); */ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, int nr_to_scan); + +void ion_page_pool_refill(struct ion_page_pool *pool); + +static __always_inline int get_pool_fillmark(struct ion_page_pool *pool) +{ + return ION_POOL_FILL_MARK / (PAGE_SIZE << pool->order); +} + +static __always_inline int get_pool_lowmark(struct ion_page_pool *pool) +{ + return ION_POOL_LOW_MARK / (PAGE_SIZE << pool->order); +} + +static __always_inline bool pool_count_below_lowmark(struct ion_page_pool *pool) +{ + return atomic_read(&pool->count) < get_pool_lowmark(pool); +} + +static __always_inline bool pool_fillmark_reached(struct ion_page_pool *pool) +{ + return atomic_read(&pool->count) >= get_pool_fillmark(pool); +} #endif /* _ION_PAGE_POOL_H */ diff --git a/drivers/staging/android/ion/ion_secure_util.c b/drivers/staging/android/ion/ion_secure_util.c new file mode 100644 index 000000000000..39fa09132c59 --- /dev/null +++ b/drivers/staging/android/ion/ion_secure_util.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include "ion_secure_util.h" +#include "ion.h" +#include "msm_ion_priv.h" + +bool is_secure_vmid_valid(int vmid) +{ + return (vmid == VMID_CP_TOUCH || + vmid == VMID_CP_BITSTREAM || + vmid == VMID_CP_PIXEL || + vmid == VMID_CP_NON_PIXEL || + vmid == VMID_CP_CAMERA || + vmid == VMID_CP_SEC_DISPLAY || + vmid == VMID_CP_APP || + vmid == VMID_CP_CAMERA_PREVIEW || + vmid == VMID_CP_SPSS_SP || + vmid == VMID_CP_SPSS_SP_SHARED || + vmid == VMID_CP_SPSS_HLOS_SHARED || + vmid == VMID_CP_CDSP); +} + +int get_secure_vmid(unsigned long flags) +{ + if (flags & ION_FLAG_CP_TOUCH) + return VMID_CP_TOUCH; + if (flags & ION_FLAG_CP_BITSTREAM) + return VMID_CP_BITSTREAM; + if (flags & ION_FLAG_CP_PIXEL) + return VMID_CP_PIXEL; + if (flags & ION_FLAG_CP_NON_PIXEL) + return VMID_CP_NON_PIXEL; + if (flags & ION_FLAG_CP_CAMERA) + return VMID_CP_CAMERA; + if (flags & ION_FLAG_CP_SEC_DISPLAY) + return VMID_CP_SEC_DISPLAY; + if (flags & ION_FLAG_CP_APP) + return VMID_CP_APP; + if (flags & ION_FLAG_CP_CAMERA_PREVIEW) + return VMID_CP_CAMERA_PREVIEW; + if (flags & ION_FLAG_CP_SPSS_SP) + return VMID_CP_SPSS_SP; + if (flags & ION_FLAG_CP_SPSS_SP_SHARED) + return VMID_CP_SPSS_SP_SHARED; + if (flags & ION_FLAG_CP_SPSS_HLOS_SHARED) + return VMID_CP_SPSS_HLOS_SHARED; + if (flags & ION_FLAG_CP_CDSP) + return VMID_CP_CDSP; + return -EINVAL; +} + +static unsigned int count_set_bits(unsigned long val) +{ + return ((unsigned int)bitmap_weight(&val, BITS_PER_LONG)); +} + +static int get_vmid(unsigned long flags) +{ + int vmid; + + vmid = get_secure_vmid(flags); + if (vmid < 0) { + if (flags & ION_FLAG_CP_HLOS) + vmid = VMID_HLOS; + } + return vmid; +} + +int ion_populate_vm_list(unsigned long flags, unsigned int *vm_list, + int nelems) +{ + unsigned int itr = 0; + int vmid; + + flags = flags & ION_FLAGS_CP_MASK; + if (!flags) + return -EINVAL; + + for_each_set_bit(itr, &flags, BITS_PER_LONG) { + vmid = get_vmid(0x1UL << itr); + if (vmid < 0 || !nelems) + return -EINVAL; + + vm_list[nelems - 1] = vmid; + nelems--; + } + return 0; +} + +int ion_hyp_unassign_sg(struct sg_table *sgt, int *source_vm_list, + int source_nelems, bool clear_page_private) +{ + u32 dest_vmid = VMID_HLOS; + u32 dest_perms = PERM_READ | PERM_WRITE | PERM_EXEC; + struct scatterlist *sg; + int ret, i; + + if (source_nelems <= 0) { + pr_err("%s: source_nelems invalid\n", + __func__); + ret = -EINVAL; + goto out; + } + + ret = hyp_assign_table(sgt, source_vm_list, source_nelems, &dest_vmid, + &dest_perms, 1); + if (ret) + goto out; + + if (clear_page_private) + for_each_sg(sgt->sgl, sg, sgt->nents, i) + ClearPagePrivate(sg_page(sg)); +out: + return ret; +} + +int ion_hyp_assign_sg(struct sg_table *sgt, int *dest_vm_list, + int dest_nelems, bool set_page_private) +{ + u32 source_vmid = VMID_HLOS; + struct scatterlist *sg; + int *dest_perms; + int i; + int ret = 0; + + if (dest_nelems <= 0) { + pr_err("%s: dest_nelems invalid\n", + __func__); + ret = -EINVAL; + goto out; + } + + dest_perms = kcalloc(dest_nelems, sizeof(*dest_perms), GFP_KERNEL); + if (!dest_perms) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < dest_nelems; i++) + dest_perms[i] = msm_secure_get_vmid_perms(dest_vm_list[i]); + + ret = hyp_assign_table(sgt, &source_vmid, 1, + dest_vm_list, dest_perms, dest_nelems); + + if (ret) { + pr_err("%s: Assign call failed\n", + __func__); + goto out_free_dest; + } + if (set_page_private) + for_each_sg(sgt->sgl, sg, sgt->nents, i) + SetPagePrivate(sg_page(sg)); + +out_free_dest: + kfree(dest_perms); +out: + return ret; +} + +int ion_hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private) +{ + int ret = 0; + int *source_vm_list; + int source_nelems; + + source_nelems = count_set_bits(flags & ION_FLAGS_CP_MASK); + source_vm_list = kcalloc(source_nelems, sizeof(*source_vm_list), + GFP_KERNEL); + if (!source_vm_list) + return -ENOMEM; + ret = ion_populate_vm_list(flags, source_vm_list, source_nelems); + if (ret) { + pr_err("%s: Failed to get secure vmids\n", __func__); + goto out_free_source; + } + + ret = ion_hyp_unassign_sg(sgt, source_vm_list, source_nelems, + set_page_private); + +out_free_source: + kfree(source_vm_list); + return ret; +} + +int ion_hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private) +{ + int ret = 0; + int *dest_vm_list = NULL; + int dest_nelems; + + dest_nelems = count_set_bits(flags & ION_FLAGS_CP_MASK); + dest_vm_list = kcalloc(dest_nelems, sizeof(*dest_vm_list), GFP_KERNEL); + if (!dest_vm_list) { + ret = -ENOMEM; + goto out; + } + + ret = ion_populate_vm_list(flags, dest_vm_list, dest_nelems); + if (ret) { + pr_err("%s: Failed to get secure vmid(s)\n", __func__); + goto out_free_dest_vm; + } + + ret = ion_hyp_assign_sg(sgt, dest_vm_list, dest_nelems, + set_page_private); + +out_free_dest_vm: + kfree(dest_vm_list); +out: + return ret; +} + +bool hlos_accessible_buffer(struct ion_buffer *buffer) +{ + if ((buffer->flags & ION_FLAG_SECURE) && + !(buffer->flags & ION_FLAG_CP_HLOS) && + !(buffer->flags & ION_FLAG_CP_SPSS_HLOS_SHARED)) + return false; + else if ((get_secure_vmid(buffer->flags) > 0) && + !(buffer->flags & ION_FLAG_CP_HLOS) && + !(buffer->flags & ION_FLAG_CP_SPSS_HLOS_SHARED)) + return false; + + return true; +} + +int ion_hyp_assign_from_flags(u64 base, u64 size, unsigned long flags) +{ + u32 *vmids, *modes; + u32 nr, i; + int ret = -EINVAL; + u32 src_vm = VMID_HLOS; + + nr = count_set_bits(flags); + vmids = kcalloc(nr, sizeof(*vmids), GFP_KERNEL); + if (!vmids) + return -ENOMEM; + + modes = kcalloc(nr, sizeof(*modes), GFP_KERNEL); + if (!modes) { + kfree(vmids); + return -ENOMEM; + } + + if ((flags & ~ION_FLAGS_CP_MASK) || + ion_populate_vm_list(flags, vmids, nr)) { + pr_err("%s: Failed to parse secure flags 0x%lx\n", __func__, + flags); + goto out; + } + + for (i = 0; i < nr; i++) + modes[i] = msm_secure_get_vmid_perms(vmids[i]); + + ret = hyp_assign_phys(base, size, &src_vm, 1, vmids, modes, nr); + if (ret) + pr_err("%s: Assign call failed, flags 0x%lx\n", __func__, + flags); + +out: + kfree(modes); + kfree(vmids); + return ret; +} diff --git a/drivers/staging/android/ion/ion_secure_util.h b/drivers/staging/android/ion/ion_secure_util.h new file mode 100644 index 000000000000..7bb56e798f0d --- /dev/null +++ b/drivers/staging/android/ion/ion_secure_util.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + */ + +#include "ion.h" + +#ifndef _ION_SECURE_UTIL_H +#define _ION_SECURE_UTIL_H + +int get_secure_vmid(unsigned long flags); +bool is_secure_vmid_valid(int vmid); +int ion_hyp_assign_sg(struct sg_table *sgt, int *dest_vm_list, + int dest_nelems, bool set_page_private); +int ion_hyp_unassign_sg(struct sg_table *sgt, int *source_vm_list, + int source_nelems, bool clear_page_private); +int ion_hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private); +int ion_hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private); +int ion_hyp_assign_from_flags(u64 base, u64 size, unsigned long flags); + +bool hlos_accessible_buffer(struct ion_buffer *buffer); + +#endif /* _ION_SECURE_UTIL_H */ diff --git a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c index fbf5f39c22db..b567f6f3b296 100644 --- a/drivers/staging/android/ion/ion_system_heap.c +++ b/drivers/staging/android/ion/ion_system_heap.c @@ -3,6 +3,8 @@ * ION Memory Allocator system heap exporter * * Copyright (C) 2011 Google, Inc. + * Copyright (c) 2011-2019, The Linux Foundation. All rights reserved. + * */ #include @@ -13,18 +15,26 @@ #include #include #include - +#include +#include +#include +#include +#include "ion_system_heap.h" #include "ion.h" #include "ion_page_pool.h" - -#define NUM_ORDERS ARRAY_SIZE(orders) +#include "msm_ion_priv.h" +#include "ion_system_heap.h" +#include "ion_system_secure_heap.h" +#include "ion_secure_util.h" static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_RECLAIM; static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO; -static const unsigned int orders[] = {8, 4, 0}; -static int order_to_index(unsigned int order) +bool pool_auto_refill_en __read_mostly = + IS_ENABLED(CONFIG_ION_POOL_AUTO_REFILL); + +int order_to_index(unsigned int order) { int i; @@ -40,44 +50,132 @@ static inline unsigned int order_to_size(int order) return PAGE_SIZE << order; } -struct ion_system_heap { - struct ion_heap heap; - struct ion_page_pool *pools[NUM_ORDERS]; +struct pages_mem { + struct page **pages; + u32 size; }; -static struct page *alloc_buffer_page(struct ion_system_heap *heap, - struct ion_buffer *buffer, - unsigned long order) +int ion_heap_is_system_heap_type(enum ion_heap_type type) { - struct ion_page_pool *pool = heap->pools[order_to_index(order)]; - - return ion_page_pool_alloc(pool); + return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM); } -static void free_buffer_page(struct ion_system_heap *heap, - struct ion_buffer *buffer, struct page *page) +static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, + struct ion_buffer *buffer, + unsigned long order, + bool *from_pool) { + int cached = (int)ion_buffer_cached(buffer); + struct page *page; struct ion_page_pool *pool; - unsigned int order = compound_order(page); + int vmid = get_secure_vmid(buffer->flags); + struct device *dev = sys_heap->heap.dev; - /* go to system */ - if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) { - __free_pages(page, order); - return; + if (vmid > 0) + pool = sys_heap->secure_pools[vmid][order_to_index(order)]; + else if (!cached) + pool = sys_heap->uncached_pools[order_to_index(order)]; + else + pool = sys_heap->cached_pools[order_to_index(order)]; + + page = ion_page_pool_alloc(pool, from_pool); + + if (pool_auto_refill_en && + pool_count_below_lowmark(pool)) { + wake_up_process(sys_heap->kworker[cached]); } - pool = heap->pools[order_to_index(order)]; + if (IS_ERR(page)) + return page; - ion_page_pool_free(pool, page); + if ((MAKE_ION_ALLOC_DMA_READY && vmid <= 0) || !(*from_pool)) + ion_pages_sync_for_device(dev, page, PAGE_SIZE << order, + DMA_BIDIRECTIONAL); + + return page; } -static struct page *alloc_largest_available(struct ion_system_heap *heap, - struct ion_buffer *buffer, - unsigned long size, - unsigned int max_order) +/* + * For secure pages that need to be freed and not added back to the pool; the + * hyp_unassign should be called before calling this function + */ +void free_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, struct page *page, + unsigned int order) +{ + bool cached = ion_buffer_cached(buffer); + int vmid = get_secure_vmid(buffer->flags); + + if (!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { + struct ion_page_pool *pool; + + if (vmid > 0) + pool = heap->secure_pools[vmid][order_to_index(order)]; + else if (cached) + pool = heap->cached_pools[order_to_index(order)]; + else + pool = heap->uncached_pools[order_to_index(order)]; + + if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) + ion_page_pool_free_immediate(pool, page); + else + ion_page_pool_free(pool, page); + } else { + __free_pages(page, order); + } +} + +static struct page_info *alloc_largest_available(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) { struct page *page; + struct page_info *info; int i; + bool from_pool; + + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < order_to_size(orders[i])) + continue; + if (max_order < orders[i]) + continue; + from_pool = !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC); + page = alloc_buffer_page(heap, buffer, orders[i], &from_pool); + if (IS_ERR(page)) + continue; + + info->page = page; + info->order = orders[i]; + info->from_pool = from_pool; + INIT_LIST_HEAD(&info->list); + return info; + } + kfree(info); + + return ERR_PTR(-ENOMEM); +} + +static struct page_info * +alloc_from_pool_preferred(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) +{ + struct page *page; + struct page_info *info; + int i; + + if (buffer->flags & ION_FLAG_POOL_FORCE_ALLOC) + goto force_alloc; + + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return ERR_PTR(-ENOMEM); for (i = 0; i < NUM_ORDERS; i++) { if (size < order_to_size(orders[i])) @@ -85,14 +183,89 @@ static struct page *alloc_largest_available(struct ion_system_heap *heap, if (max_order < orders[i]) continue; - page = alloc_buffer_page(heap, buffer, orders[i]); - if (!page) + page = alloc_from_secure_pool_order(heap, buffer, orders[i]); + if (IS_ERR(page)) continue; - return page; + info->page = page; + info->order = orders[i]; + info->from_pool = true; + INIT_LIST_HEAD(&info->list); + return info; } - return NULL; + page = split_page_from_secure_pool(heap, buffer); + if (!IS_ERR(page)) { + info->page = page; + info->order = 0; + info->from_pool = true; + INIT_LIST_HEAD(&info->list); + return info; + } + + kfree(info); +force_alloc: + return alloc_largest_available(heap, buffer, size, max_order); +} + +static unsigned int process_info(struct page_info *info, + struct scatterlist *sg, + struct scatterlist *sg_sync, + struct pages_mem *data, unsigned int i) +{ + struct page *page = info->page; + unsigned int j; + + if (sg_sync) { + sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0); + sg_dma_address(sg_sync) = page_to_phys(page); + } + sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0); + /* + * This is not correct - sg_dma_address needs a dma_addr_t + * that is valid for the the targeted device, but this works + * on the currently targeted hardware. + */ + sg_dma_address(sg) = page_to_phys(page); + if (data) { + for (j = 0; j < (1 << info->order); ++j) + data->pages[i++] = nth_page(page, j); + } + list_del(&info->list); + kfree(info); + return i; +} + +static int ion_heap_alloc_pages_mem(struct pages_mem *pages_mem) +{ + struct page **pages; + unsigned int page_tbl_size; + + page_tbl_size = sizeof(struct page *) * (pages_mem->size >> PAGE_SHIFT); + if (page_tbl_size > SZ_8K) { + /* + * Do fallback to ensure we have a balance between + * performance and availability. + */ + pages = kmalloc(page_tbl_size, + __GFP_COMP | __GFP_NORETRY | + __GFP_NOWARN); + if (!pages) + pages = vmalloc(page_tbl_size); + } else { + pages = kmalloc(page_tbl_size, GFP_KERNEL); + } + + if (!pages) + return -ENOMEM; + + pages_mem->pages = pages; + return 0; +} + +static void ion_heap_free_pages_mem(struct pages_mem *pages_mem) +{ + kvfree(pages_mem->pages); } static int ion_system_heap_allocate(struct ion_heap *heap, @@ -100,107 +273,229 @@ static int ion_system_heap_allocate(struct ion_heap *heap, unsigned long size, unsigned long flags) { - struct ion_system_heap *sys_heap = container_of(heap, - struct ion_system_heap, - heap); + struct ion_system_heap *sys_heap = to_system_heap(heap); struct sg_table *table; + struct sg_table table_sync = {0}; struct scatterlist *sg; + struct scatterlist *sg_sync; + int ret = -ENOMEM; struct list_head pages; - struct page *page, *tmp_page; + struct list_head pages_from_pool; + struct page_info *info, *tmp_info; int i = 0; + unsigned int nents_sync = 0; unsigned long size_remaining = PAGE_ALIGN(size); unsigned int max_order = orders[0]; + struct pages_mem data; + unsigned int sz; + int vmid = get_secure_vmid(buffer->flags); if (size / PAGE_SIZE > totalram_pages() / 2) return -ENOMEM; + if (ion_heap_is_system_heap_type(buffer->heap->type) && + is_secure_vmid_valid(vmid)) { + pr_info("%s: System heap doesn't support secure allocations\n", + __func__); + return -EINVAL; + } + + data.size = 0; INIT_LIST_HEAD(&pages); + INIT_LIST_HEAD(&pages_from_pool); + while (size_remaining > 0) { - page = alloc_largest_available(sys_heap, buffer, size_remaining, - max_order); - if (!page) - goto free_pages; - list_add_tail(&page->lru, &pages); - size_remaining -= PAGE_SIZE << compound_order(page); - max_order = compound_order(page); + if (is_secure_vmid_valid(vmid)) + info = alloc_from_pool_preferred(sys_heap, buffer, + size_remaining, + max_order); + else + info = alloc_largest_available(sys_heap, buffer, + size_remaining, + max_order); + + if (IS_ERR(info)) { + ret = PTR_ERR(info); + goto err; + } + + sz = (1 << info->order) * PAGE_SIZE; + + if (info->from_pool) { + list_add_tail(&info->list, &pages_from_pool); + } else { + list_add_tail(&info->list, &pages); + data.size += sz; + ++nents_sync; + } + size_remaining -= sz; + max_order = info->order; i++; } - table = kmalloc(sizeof(*table), GFP_KERNEL); - if (!table) - goto free_pages; - if (sg_alloc_table(table, i, GFP_KERNEL)) - goto free_table; + ret = ion_heap_alloc_pages_mem(&data); + if (ret) + goto err; + + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) { + ret = -ENOMEM; + goto err_free_data_pages; + } + + ret = sg_alloc_table(table, i, GFP_KERNEL); + if (ret) + goto err1; + + if (nents_sync) { + ret = sg_alloc_table(&table_sync, nents_sync, GFP_KERNEL); + if (ret) + goto err_free_sg; + } + + i = 0; sg = table->sgl; - list_for_each_entry_safe(page, tmp_page, &pages, lru) { - sg_set_page(sg, page, PAGE_SIZE << compound_order(page), 0); + sg_sync = table_sync.sgl; + + /* + * We now have two separate lists. One list contains pages from the + * pool and the other pages from buddy. We want to merge these + * together while preserving the ordering of the pages (higher order + * first). + */ + do { + info = list_first_entry_or_null(&pages, struct page_info, list); + tmp_info = list_first_entry_or_null(&pages_from_pool, + struct page_info, list); + if (info && tmp_info) { + if (info->order >= tmp_info->order) { + i = process_info(info, sg, sg_sync, &data, i); + sg_sync = sg_next(sg_sync); + } else { + i = process_info(tmp_info, sg, 0, 0, i); + } + } else if (info) { + i = process_info(info, sg, sg_sync, &data, i); + sg_sync = sg_next(sg_sync); + } else if (tmp_info) { + i = process_info(tmp_info, sg, 0, 0, i); + } sg = sg_next(sg); - list_del(&page->lru); + + } while (sg); + + if (nents_sync) { + if (vmid > 0) { + ret = ion_hyp_assign_sg(&table_sync, &vmid, 1, true); + if (ret) + goto err_free_sg2; + } } buffer->sg_table = table; + if (nents_sync) + sg_free_table(&table_sync); + ion_heap_free_pages_mem(&data); + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; -free_table: +err_free_sg2: + /* We failed to zero buffers. Bypass pool */ + buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE; + + if (vmid > 0) + ion_hyp_unassign_sg(table, &vmid, 1, true); + + for_each_sg(table->sgl, sg, table->nents, i) + free_buffer_page(sys_heap, buffer, sg_page(sg), + get_order(sg->length)); + if (nents_sync) + sg_free_table(&table_sync); +err_free_sg: + sg_free_table(table); +err1: kfree(table); -free_pages: - list_for_each_entry_safe(page, tmp_page, &pages, lru) - free_buffer_page(sys_heap, buffer, page); - return -ENOMEM; +err_free_data_pages: + ion_heap_free_pages_mem(&data); +err: + list_for_each_entry_safe(info, tmp_info, &pages, list) { + free_buffer_page(sys_heap, buffer, info->page, info->order); + kfree(info); + } + list_for_each_entry_safe(info, tmp_info, &pages_from_pool, list) { + free_buffer_page(sys_heap, buffer, info->page, info->order); + kfree(info); + } + return ret; } -static void ion_system_heap_free(struct ion_buffer *buffer) +void ion_system_heap_free(struct ion_buffer *buffer) { - struct ion_system_heap *sys_heap = container_of(buffer->heap, - struct ion_system_heap, - heap); + struct ion_heap *heap = buffer->heap; + struct ion_system_heap *sys_heap = to_system_heap(heap); struct sg_table *table = buffer->sg_table; struct scatterlist *sg; int i; + int vmid = get_secure_vmid(buffer->flags); - /* zero the buffer before goto page pool */ - if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) - ion_heap_buffer_zero(buffer); + if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) && + !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { + if (vmid < 0) + ion_heap_buffer_zero(buffer); + } else if (vmid > 0) { + if (ion_hyp_unassign_sg(table, &vmid, 1, true)) + return; + } for_each_sg(table->sgl, sg, table->nents, i) - free_buffer_page(sys_heap, buffer, sg_page(sg)); + free_buffer_page(sys_heap, buffer, sg_page(sg), + get_order(sg->length)); sg_free_table(table); kfree(table); } static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, - int nr_to_scan) + int nr_to_scan) { - struct ion_page_pool *pool; struct ion_system_heap *sys_heap; int nr_total = 0; - int i, nr_freed; + int i, j, nr_freed = 0; int only_scan = 0; + struct ion_page_pool *pool; - sys_heap = container_of(heap, struct ion_system_heap, heap); + sys_heap = to_system_heap(heap); if (!nr_to_scan) only_scan = 1; - for (i = 0; i < NUM_ORDERS; i++) { - pool = sys_heap->pools[i]; + /* shrink the pools starting from lower order ones */ + for (i = NUM_ORDERS - 1; i >= 0; i--) { + nr_freed = 0; - if (only_scan) { - nr_total += ion_page_pool_shrink(pool, - gfp_mask, - nr_to_scan); + for (j = 0; j < VMID_LAST; j++) { + if (is_secure_vmid_valid(j)) + nr_freed += + ion_secure_page_pool_shrink(sys_heap, + j, i, + nr_to_scan); + } - } else { - nr_freed = ion_page_pool_shrink(pool, - gfp_mask, - nr_to_scan); + pool = sys_heap->uncached_pools[i]; + nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan); + + pool = sys_heap->cached_pools[i]; + nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan); + nr_total += nr_freed; + + if (!only_scan) { nr_to_scan -= nr_freed; - nr_total += nr_freed; + /* shrink completed */ if (nr_to_scan <= 0) break; } } + return nr_total; } @@ -213,16 +508,127 @@ static struct ion_heap_ops system_heap_ops = { .shrink = ion_system_heap_shrink, }; +static __maybe_unused int ion_system_heap_debug_show(struct ion_heap *heap, + struct seq_file *s, + void *unused) +{ + struct ion_system_heap *sys_heap; + bool use_seq = s; + unsigned long uncached_total = 0; + unsigned long cached_total = 0; + unsigned long secure_total = 0; + struct ion_page_pool *pool; + int i, j; + + sys_heap = to_system_heap(heap); + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->uncached_pools[i]; + if (use_seq) { + seq_printf(s, + "%d order %u highmem pages in uncached pool = %lu total\n", + pool->high_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->high_count); + seq_printf(s, + "%d order %u lowmem pages in uncached pool = %lu total\n", + pool->low_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->low_count); + } + + uncached_total += (1 << pool->order) * PAGE_SIZE * + pool->high_count; + uncached_total += (1 << pool->order) * PAGE_SIZE * + pool->low_count; + } + + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->cached_pools[i]; + if (use_seq) { + seq_printf(s, + "%d order %u highmem pages in cached pool = %lu total\n", + pool->high_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->high_count); + seq_printf(s, + "%d order %u lowmem pages in cached pool = %lu total\n", + pool->low_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->low_count); + } + + cached_total += (1 << pool->order) * PAGE_SIZE * + pool->high_count; + cached_total += (1 << pool->order) * PAGE_SIZE * + pool->low_count; + } + + for (i = 0; i < NUM_ORDERS; i++) { + for (j = 0; j < VMID_LAST; j++) { + if (!is_secure_vmid_valid(j)) + continue; + pool = sys_heap->secure_pools[j][i]; + + if (use_seq) { + seq_printf(s, + "VMID %d: %d order %u highmem pages in secure pool = %lu total\n", + j, pool->high_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->high_count); + seq_printf(s, + "VMID %d: %d order %u lowmem pages in secure pool = %lu total\n", + j, pool->low_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->low_count); + } + + secure_total += (1 << pool->order) * PAGE_SIZE * + pool->high_count; + secure_total += (1 << pool->order) * PAGE_SIZE * + pool->low_count; + } + } + + if (use_seq) { + seq_puts(s, "--------------------------------------------\n"); + seq_printf(s, "uncached pool = %lu cached pool = %lu secure pool = %lu\n", + uncached_total, cached_total, secure_total); + seq_printf(s, "pool total (uncached + cached + secure) = %lu\n", + uncached_total + cached_total + secure_total); + seq_puts(s, "--------------------------------------------\n"); + } else { + pr_info("-------------------------------------------------\n"); + pr_info("uncached pool = %lu cached pool = %lu secure pool = %lu\n", + uncached_total, cached_total, secure_total); + pr_info("pool total (uncached + cached + secure) = %lu\n", + uncached_total + cached_total + secure_total); + pr_info("-------------------------------------------------\n"); + } + + return 0; +} + static void ion_system_heap_destroy_pools(struct ion_page_pool **pools) { int i; for (i = 0; i < NUM_ORDERS; i++) - if (pools[i]) + if (pools[i]) { ion_page_pool_destroy(pools[i]); + pools[i] = NULL; + } } -static int ion_system_heap_create_pools(struct ion_page_pool **pools) +/** + * ion_system_heap_create_pools - Creates pools for all orders + * + * If this fails you don't need to destroy any pools. It's all or + * nothing. If it succeeds you'll eventually need to use + * ion_system_heap_destroy_pools to destroy the pools. + */ +static int ion_system_heap_create_pools(struct ion_system_heap *sys_heap, + struct ion_page_pool **pools, + bool cached) { int i; @@ -230,58 +636,140 @@ static int ion_system_heap_create_pools(struct ion_page_pool **pools) struct ion_page_pool *pool; gfp_t gfp_flags = low_order_gfp_flags; - if (orders[i] > 4) + if (orders[i]) gfp_flags = high_order_gfp_flags; - - pool = ion_page_pool_create(gfp_flags, orders[i]); + pool = ion_page_pool_create(gfp_flags, orders[i], cached); if (!pool) goto err_create_pool; + pool->heap_dev = sys_heap->heap.dev; pools[i] = pool; } return 0; - err_create_pool: ion_system_heap_destroy_pools(pools); return -ENOMEM; } -static struct ion_heap *__ion_system_heap_create(void) +static int ion_sys_heap_worker(void *data) +{ + struct ion_page_pool **pools = (struct ion_page_pool **)data; + int i; + + for (;;) { + for (i = 0; i < NUM_ORDERS; i++) { + if (pool_count_below_lowmark(pools[i])) + ion_page_pool_refill(pools[i]); + } + set_current_state(TASK_INTERRUPTIBLE); + if (unlikely(kthread_should_stop())) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); + + set_current_state(TASK_RUNNING); + } + + return 0; +} + +static struct task_struct *ion_create_kworker(struct ion_page_pool **pools, + bool cached) +{ + struct sched_attr attr = { 0 }; + struct task_struct *thread; + int ret; + char *buf; + cpumask_t *cpumask; + DECLARE_BITMAP(bmap, nr_cpumask_bits); + + attr.sched_nice = ION_KTHREAD_NICE_VAL; + buf = cached ? "cached" : "uncached"; + /* + * Affine the kthreads to min capacity CPUs + * TODO: remove this hack once is_min_capability_cpu is available + */ + bitmap_fill(bmap, 0x4); + cpumask = to_cpumask(bmap); + + thread = kthread_create(ion_sys_heap_worker, pools, + "ion-pool-%s-worker", buf); + if (IS_ERR(thread)) { + pr_err("%s: failed to create %s worker thread: %ld\n", + __func__, buf, PTR_ERR(thread)); + return thread; + } + ret = sched_setattr(thread, &attr); + if (ret) { + kthread_stop(thread); + pr_warn("%s: failed to set task priority for %s worker thread: ret = %d\n", + __func__, buf, ret); + return ERR_PTR(ret); + } + kthread_bind_mask(thread, cpumask); + return thread; +} + +struct ion_heap *ion_system_heap_create(struct ion_platform_heap *data) { struct ion_system_heap *heap; + int ret = -ENOMEM; + int i; heap = kzalloc(sizeof(*heap), GFP_KERNEL); if (!heap) return ERR_PTR(-ENOMEM); - heap->heap.ops = &system_heap_ops; - heap->heap.type = ION_HEAP_TYPE_SYSTEM; - heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE; + heap->heap.dev = data->priv; + heap->heap.ion_heap.ops = &system_heap_ops; + heap->heap.ion_heap.type = ION_HEAP_TYPE_SYSTEM; + heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE; - if (ion_system_heap_create_pools(heap->pools)) - goto free_heap; + for (i = 0; i < VMID_LAST; i++) + if (is_secure_vmid_valid(i)) + if (ion_system_heap_create_pools(heap, + heap->secure_pools[i], + false)) + goto destroy_secure_pools; - return &heap->heap; + if (ion_system_heap_create_pools(heap, heap->uncached_pools, false)) + goto destroy_secure_pools; -free_heap: + if (ion_system_heap_create_pools(heap, heap->cached_pools, true)) + goto destroy_uncached_pools; + + if (pool_auto_refill_en) { + heap->kworker[ION_KTHREAD_UNCACHED] = + ion_create_kworker(heap->uncached_pools, false); + if (IS_ERR(heap->kworker[ION_KTHREAD_UNCACHED])) { + ret = PTR_ERR(heap->kworker[ION_KTHREAD_UNCACHED]); + goto destroy_pools; + } + heap->kworker[ION_KTHREAD_CACHED] = + ion_create_kworker(heap->cached_pools, true); + if (IS_ERR(heap->kworker[ION_KTHREAD_CACHED])) { + kthread_stop(heap->kworker[ION_KTHREAD_UNCACHED]); + ret = PTR_ERR(heap->kworker[ION_KTHREAD_CACHED]); + goto destroy_pools; + } + } + + mutex_init(&heap->split_page_mutex); + + return &heap->heap.ion_heap; +destroy_pools: + ion_system_heap_destroy_pools(heap->cached_pools); +destroy_uncached_pools: + ion_system_heap_destroy_pools(heap->uncached_pools); +destroy_secure_pools: + for (i = 0; i < VMID_LAST; i++) { + if (heap->secure_pools[i]) + ion_system_heap_destroy_pools(heap->secure_pools[i]); + } kfree(heap); - return ERR_PTR(-ENOMEM); + return ERR_PTR(ret); } -static int ion_system_heap_create(void) -{ - struct ion_heap *heap; - - heap = __ion_system_heap_create(); - if (IS_ERR(heap)) - return PTR_ERR(heap); - heap->name = "ion_system_heap"; - - ion_device_add_heap(heap); - - return 0; -} -device_initcall(ion_system_heap_create); - static int ion_system_contig_heap_allocate(struct ion_heap *heap, struct ion_buffer *buffer, unsigned long len, @@ -317,6 +805,8 @@ static int ion_system_contig_heap_allocate(struct ion_heap *heap, buffer->sg_table = table; + ion_pages_sync_for_device(NULL, page, len, DMA_BIDIRECTIONAL); + return 0; free_table: @@ -349,7 +839,7 @@ static struct ion_heap_ops kmalloc_ops = { .map_user = ion_heap_map_user, }; -static struct ion_heap *__ion_system_contig_heap_create(void) +struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused) { struct ion_heap *heap; @@ -358,21 +848,5 @@ static struct ion_heap *__ion_system_contig_heap_create(void) return ERR_PTR(-ENOMEM); heap->ops = &kmalloc_ops; heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG; - heap->name = "ion_system_contig_heap"; - return heap; } - -static int ion_system_contig_heap_create(void) -{ - struct ion_heap *heap; - - heap = __ion_system_contig_heap_create(); - if (IS_ERR(heap)) - return PTR_ERR(heap); - - ion_device_add_heap(heap); - - return 0; -} -device_initcall(ion_system_contig_heap_create); diff --git a/drivers/staging/android/ion/ion_system_heap.h b/drivers/staging/android/ion/ion_system_heap.h new file mode 100644 index 000000000000..069afbcb6f06 --- /dev/null +++ b/drivers/staging/android/ion/ion_system_heap.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + */ +#include +#include "ion.h" +#include "msm_ion_priv.h" + +#ifndef _ION_SYSTEM_HEAP_H +#define _ION_SYSTEM_HEAP_H + +#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS +#if defined(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) +static const unsigned int orders[] = {8, 4, 0}; +#else +static const unsigned int orders[] = {9, 4, 0}; +#endif +#else +static const unsigned int orders[] = {0}; +#endif + +#define NUM_ORDERS ARRAY_SIZE(orders) + +#define ION_KTHREAD_NICE_VAL 10 + +#define to_system_heap(_heap) \ + container_of(to_msm_ion_heap(_heap), struct ion_system_heap, heap) + +enum ion_kthread_type { + ION_KTHREAD_UNCACHED, + ION_KTHREAD_CACHED, + ION_MAX_NUM_KTHREADS +}; + +struct ion_system_heap { + struct msm_ion_heap heap; + struct ion_page_pool *uncached_pools[MAX_ORDER]; + struct ion_page_pool *cached_pools[MAX_ORDER]; + /* worker threads to refill the pool */ + struct task_struct *kworker[ION_MAX_NUM_KTHREADS]; + struct ion_page_pool *secure_pools[VMID_LAST][MAX_ORDER]; + /* Prevents unnecessary page splitting */ + struct mutex split_page_mutex; +}; + +struct page_info { + struct page *page; + bool from_pool; + unsigned int order; + struct list_head list; +}; + +int order_to_index(unsigned int order); + +void free_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, struct page *page, + unsigned int order); + +#endif /* _ION_SYSTEM_HEAP_H */ diff --git a/drivers/staging/android/ion/ion_system_secure_heap.c b/drivers/staging/android/ion/ion_system_secure_heap.c new file mode 100644 index 000000000000..a7cb38d81173 --- /dev/null +++ b/drivers/staging/android/ion/ion_system_secure_heap.c @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014-2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "ion_system_secure_heap.h" +#include "ion_system_heap.h" +#include "ion.h" +#include "ion_page_pool.h" +#include "msm_ion_priv.h" +#include "ion_secure_util.h" + +#define to_system_secure_heap(_heap) \ + container_of(to_msm_ion_heap(_heap), struct ion_system_secure_heap, \ + heap) + +struct ion_system_secure_heap { + struct ion_heap *sys_heap; + struct msm_ion_heap heap; + + /* Protects prefetch_list */ + spinlock_t work_lock; + bool destroy_heap; + struct list_head prefetch_list; + struct delayed_work prefetch_work; +}; + +struct prefetch_info { + struct list_head list; + int vmid; + u64 size; + bool shrink; +}; + +/* + * The video client may not hold the last reference count on the + * ion_buffer(s). Delay for a short time after the video client sends + * the IOC_DRAIN event to increase the chance that the reference + * count drops to zero. Time in milliseconds. + */ +#define SHRINK_DELAY 1000 + +int ion_heap_is_system_secure_heap_type(enum ion_heap_type type) +{ + return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM_SECURE); +} + +static bool is_cp_flag_present(unsigned long flags) +{ + return flags & (ION_FLAG_CP_TOUCH | + ION_FLAG_CP_BITSTREAM | + ION_FLAG_CP_PIXEL | + ION_FLAG_CP_NON_PIXEL | + ION_FLAG_CP_CAMERA); +} + +static void ion_system_secure_heap_free(struct ion_buffer *buffer) +{ + struct ion_heap *heap = buffer->heap; + struct ion_system_secure_heap *secure_heap = + to_system_secure_heap(heap); + + secure_heap->sys_heap->ops->free(buffer); +} + +static int ion_system_secure_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned long flags) +{ + int ret = 0; + struct ion_system_secure_heap *secure_heap = + to_system_secure_heap(heap); + enum ion_heap_type type = secure_heap->heap.ion_heap.type; + + if (!ion_heap_is_system_secure_heap_type(type) || + !is_cp_flag_present(flags)) { + pr_info("%s: Incorrect heap type or incorrect flags\n", + __func__); + return -EINVAL; + } + + ret = secure_heap->sys_heap->ops->allocate(secure_heap->sys_heap, + buffer, size, flags); + if (ret) { + pr_info("%s: Failed to get allocation for %s, ret = %d\n", + __func__, heap->name, ret); + return ret; + } + return ret; +} + +static void process_one_prefetch(struct ion_heap *sys_heap, + struct prefetch_info *info) +{ + struct ion_buffer buffer; + int ret; + int vmid; + + memset(&buffer, 0, sizeof(struct ion_buffer)); + buffer.heap = sys_heap; + + ret = sys_heap->ops->allocate(sys_heap, &buffer, info->size, + buffer.flags); + if (ret) { + pr_debug("%s: Failed to prefetch 0x%llx, ret = %d\n", + __func__, info->size, ret); + return; + } + + vmid = get_secure_vmid(info->vmid); + if (vmid < 0) + goto out; + + ret = ion_hyp_assign_sg(buffer.sg_table, &vmid, 1, true); + if (ret) + goto out; + + /* Now free it to the secure heap */ + buffer.heap = sys_heap; + buffer.flags = info->vmid; + +out: + sys_heap->ops->free(&buffer); +} + +/* + * Since no lock is held, results are approximate. + */ +size_t ion_system_secure_heap_page_pool_total(struct ion_heap *heap, + int vmid_flags) +{ + struct ion_system_heap *sys_heap; + struct ion_page_pool *pool; + size_t total = 0; + int vmid, i; + + sys_heap = to_system_heap(heap); + vmid = get_secure_vmid(vmid_flags); + if (vmid < 0) + return 0; + + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->secure_pools[vmid][i]; + total += ion_page_pool_total(pool, true); + } + + return total << PAGE_SHIFT; +} + +static void process_one_shrink(struct ion_system_secure_heap *secure_heap, + struct ion_heap *sys_heap, + struct prefetch_info *info) +{ + struct ion_buffer buffer; + size_t pool_size, size; + int ret; + + memset(&buffer, 0, sizeof(struct ion_buffer)); + buffer.heap = &secure_heap->heap.ion_heap; + buffer.flags = info->vmid; + + pool_size = ion_system_secure_heap_page_pool_total(sys_heap, + info->vmid); + size = min_t(size_t, pool_size, info->size); + ret = sys_heap->ops->allocate(sys_heap, &buffer, size, buffer.flags); + if (ret) { + pr_debug("%s: Failed to shrink 0x%llx, ret = %d\n", + __func__, info->size, ret); + return; + } + + buffer.private_flags = ION_PRIV_FLAG_SHRINKER_FREE; + buffer.heap = sys_heap; + sys_heap->ops->free(&buffer); +} + +static void ion_system_secure_heap_prefetch_work(struct work_struct *work) +{ + struct ion_system_secure_heap *secure_heap = container_of(work, + struct ion_system_secure_heap, + prefetch_work.work); + struct ion_heap *sys_heap = secure_heap->sys_heap; + struct prefetch_info *info, *tmp; + unsigned long flags; + + spin_lock_irqsave(&secure_heap->work_lock, flags); + list_for_each_entry_safe(info, tmp, + &secure_heap->prefetch_list, list) { + list_del(&info->list); + spin_unlock_irqrestore(&secure_heap->work_lock, flags); + + if (info->shrink) + process_one_shrink(secure_heap, sys_heap, info); + else + process_one_prefetch(sys_heap, info); + + kfree(info); + spin_lock_irqsave(&secure_heap->work_lock, flags); + } + spin_unlock_irqrestore(&secure_heap->work_lock, flags); +} + +static int alloc_prefetch_info(struct ion_prefetch_regions __user * + user_regions, bool shrink, + struct list_head *items) +{ + struct prefetch_info *info; + u64 user_sizes; + int err; + unsigned int nr_sizes, vmid, i; + + err = get_user(nr_sizes, &user_regions->nr_sizes); + err |= get_user(user_sizes, &user_regions->sizes); + err |= get_user(vmid, &user_regions->vmid); + if (err) + return -EFAULT; + + if (!is_secure_vmid_valid(get_secure_vmid(vmid))) + return -EINVAL; + + if (nr_sizes > 0x10) + return -EINVAL; + + for (i = 0; i < nr_sizes; i++) { + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + err = get_user(info->size, ((u64 __user *)user_sizes + i)); + if (err) + goto out_free; + + info->vmid = vmid; + info->shrink = shrink; + INIT_LIST_HEAD(&info->list); + list_add_tail(&info->list, items); + } + return err; +out_free: + kfree(info); + return err; +} + +static int __ion_system_secure_heap_resize(struct ion_heap *heap, void *ptr, + bool shrink) +{ + struct ion_system_secure_heap *secure_heap = + to_system_secure_heap(heap); + struct ion_prefetch_data *data = ptr; + int i, ret = 0; + struct prefetch_info *info, *tmp; + unsigned long flags; + LIST_HEAD(items); + + if ((int)heap->type != ION_HEAP_TYPE_SYSTEM_SECURE) + return -EINVAL; + + if (data->nr_regions > 0x10) + return -EINVAL; + + for (i = 0; i < data->nr_regions; i++) { + struct ion_prefetch_regions *r; + + r = (struct ion_prefetch_regions *)data->regions + i; + ret = alloc_prefetch_info(r, shrink, &items); + if (ret) + goto out_free; + } + + spin_lock_irqsave(&secure_heap->work_lock, flags); + if (secure_heap->destroy_heap) { + spin_unlock_irqrestore(&secure_heap->work_lock, flags); + goto out_free; + } + list_splice_tail_init(&items, &secure_heap->prefetch_list); + queue_delayed_work(system_unbound_wq, &secure_heap->prefetch_work, + shrink ? msecs_to_jiffies(SHRINK_DELAY) : 0); + spin_unlock_irqrestore(&secure_heap->work_lock, flags); + + return 0; + +out_free: + list_for_each_entry_safe(info, tmp, &items, list) { + list_del(&info->list); + kfree(info); + } + return ret; +} + +int ion_system_secure_heap_prefetch(struct ion_heap *heap, void *ptr) +{ + return __ion_system_secure_heap_resize(heap, ptr, false); +} + +int ion_system_secure_heap_drain(struct ion_heap *heap, void *ptr) +{ + return __ion_system_secure_heap_resize(heap, ptr, true); +} + +static void *ion_system_secure_heap_map_kernel(struct ion_heap *heap, + struct ion_buffer *buffer) +{ + pr_info("%s: Kernel mapping from secure heap %s disallowed\n", + __func__, heap->name); + return ERR_PTR(-EINVAL); +} + +static void ion_system_secure_heap_unmap_kernel(struct ion_heap *heap, + struct ion_buffer *buffer) +{ +} + +static int ion_system_secure_heap_map_user(struct ion_heap *mapper, + struct ion_buffer *buffer, + struct vm_area_struct *vma) +{ + pr_info("%s: Mapping from secure heap %s disallowed\n", + __func__, mapper->name); + return -EINVAL; +} + +static int ion_system_secure_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, + int nr_to_scan) +{ + struct ion_system_secure_heap *secure_heap = + to_system_secure_heap(heap); + + return secure_heap->sys_heap->ops->shrink(secure_heap->sys_heap, + gfp_mask, nr_to_scan); +} + +static struct ion_heap_ops system_secure_heap_ops = { + .allocate = ion_system_secure_heap_allocate, + .free = ion_system_secure_heap_free, + .map_kernel = ion_system_secure_heap_map_kernel, + .unmap_kernel = ion_system_secure_heap_unmap_kernel, + .map_user = ion_system_secure_heap_map_user, + .shrink = ion_system_secure_heap_shrink, +}; + +struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *unused) +{ + struct ion_system_secure_heap *heap; + + heap = kzalloc(sizeof(*heap), GFP_KERNEL); + if (!heap) + return ERR_PTR(-ENOMEM); + heap->heap.ion_heap.ops = &system_secure_heap_ops; + heap->heap.ion_heap.type = + (enum ion_heap_type)ION_HEAP_TYPE_SYSTEM_SECURE; + heap->sys_heap = get_ion_heap(ION_SYSTEM_HEAP_ID); + + heap->destroy_heap = false; + heap->work_lock = __SPIN_LOCK_UNLOCKED(heap->work_lock); + INIT_LIST_HEAD(&heap->prefetch_list); + INIT_DELAYED_WORK(&heap->prefetch_work, + ion_system_secure_heap_prefetch_work); + return &heap->heap.ion_heap; +} + +struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long order) +{ + int vmid = get_secure_vmid(buffer->flags); + struct ion_page_pool *pool; + + if (!is_secure_vmid_valid(vmid)) + return ERR_PTR(-EINVAL); + + pool = heap->secure_pools[vmid][order_to_index(order)]; + return ion_page_pool_alloc_pool_only(pool); +} + +struct page *split_page_from_secure_pool(struct ion_system_heap *heap, + struct ion_buffer *buffer) +{ + int i, j; + struct page *page; + unsigned int order; + + mutex_lock(&heap->split_page_mutex); + + /* + * Someone may have just split a page and returned the unused portion + * back to the pool, so try allocating from the pool one more time + * before splitting. We want to maintain large pages sizes when + * possible. + */ + page = alloc_from_secure_pool_order(heap, buffer, 0); + if (!IS_ERR(page)) + goto got_page; + + for (i = NUM_ORDERS - 2; i >= 0; i--) { + order = orders[i]; + page = alloc_from_secure_pool_order(heap, buffer, order); + if (IS_ERR(page)) + continue; + + split_page(page, order); + break; + } + /* + * Return the remaining order-0 pages to the pool. + * SetPagePrivate flag to mark memory as secure. + */ + if (!IS_ERR(page)) { + for (j = 1; j < (1 << order); j++) { + SetPagePrivate(page + j); + free_buffer_page(heap, buffer, page + j, 0); + } + } +got_page: + mutex_unlock(&heap->split_page_mutex); + + return page; +} + +int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, + int vmid, int order_idx, int nr_to_scan) +{ + int ret, freed = 0; + int order = orders[order_idx]; + struct page *page, *tmp; + struct sg_table sgt; + struct scatterlist *sg; + struct ion_page_pool *pool = sys_heap->secure_pools[vmid][order_idx]; + LIST_HEAD(pages); + + if (nr_to_scan == 0) + return ion_page_pool_total(pool, true); + + while (freed < nr_to_scan) { + page = ion_page_pool_alloc_pool_only(pool); + if (IS_ERR(page)) + break; + list_add(&page->lru, &pages); + freed += (1 << order); + } + + if (!freed) + return freed; + + ret = sg_alloc_table(&sgt, (freed >> order), GFP_KERNEL); + if (ret) + goto out1; + sg = sgt.sgl; + list_for_each_entry(page, &pages, lru) { + sg_set_page(sg, page, (1 << order) * PAGE_SIZE, 0); + sg_dma_address(sg) = page_to_phys(page); + sg = sg_next(sg); + } + + ret = ion_hyp_unassign_sg(&sgt, &vmid, 1, true); + if (ret == -EADDRNOTAVAIL) + goto out3; + else if (ret < 0) + goto out2; + + list_for_each_entry_safe(page, tmp, &pages, lru) { + list_del(&page->lru); + ion_page_pool_free_immediate(pool, page); + } + + sg_free_table(&sgt); + return freed; + +out2: + sg_free_table(&sgt); +out1: + /* Restore pages to secure pool */ + list_for_each_entry_safe(page, tmp, &pages, lru) { + list_del(&page->lru); + ion_page_pool_free(pool, page); + } + return 0; +out3: + /* + * The security state of the pages is unknown after a failure; + * They can neither be added back to the secure pool nor buddy system. + */ + sg_free_table(&sgt); + return 0; +} diff --git a/drivers/staging/android/ion/ion_system_secure_heap.h b/drivers/staging/android/ion/ion_system_secure_heap.h new file mode 100644 index 000000000000..23b36bd7203c --- /dev/null +++ b/drivers/staging/android/ion/ion_system_secure_heap.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + */ +#include "ion.h" +#include "ion_system_heap.h" + +#ifndef _ION_SYSTEM_SECURE_HEAP_H +#define _ION_SYSTEM_SECURE_HEAP_H + +int ion_system_secure_heap_prefetch(struct ion_heap *heap, void *data); +int ion_system_secure_heap_drain(struct ion_heap *heap, void *data); + +struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long order); + +struct page *split_page_from_secure_pool(struct ion_system_heap *heap, + struct ion_buffer *buffer); + +int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, + int vmid, int order_idx, int nr_to_scan); + +#endif /* _ION_SYSTEM_SECURE_HEAP_H */ diff --git a/drivers/staging/android/ion/msm_ion_dma_buf.c b/drivers/staging/android/ion/msm_ion_dma_buf.c new file mode 100644 index 000000000000..749842bdbc8d --- /dev/null +++ b/drivers/staging/android/ion/msm_ion_dma_buf.c @@ -0,0 +1,726 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011 Google, Inc. + * Copyright (c) 2011-2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CREATE_TRACE_POINTS +#include + +#include "ion.h" +#include "msm_ion_priv.h" +#include "ion_secure_util.h" + +static void _ion_buffer_destroy(struct ion_buffer *buffer) +{ + struct ion_heap *heap = buffer->heap; + struct ion_device *dev = buffer->dev; + + msm_dma_buf_freed(buffer); + + mutex_lock(&dev->buffer_lock); + rb_erase(&buffer->node, &dev->buffers); + mutex_unlock(&dev->buffer_lock); + + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) + ion_heap_freelist_add(heap, buffer); + else + ion_buffer_destroy(buffer); +} + +static void *ion_buffer_kmap_get(struct ion_buffer *buffer) +{ + void *vaddr; + + if (buffer->kmap_cnt) { + buffer->kmap_cnt++; + return buffer->vaddr; + } + vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer); + if (WARN_ONCE(!vaddr, + "heap->ops->map_kernel should return ERR_PTR on error")) + return ERR_PTR(-EINVAL); + if (IS_ERR(vaddr)) + return vaddr; + buffer->vaddr = vaddr; + buffer->kmap_cnt++; + return vaddr; +} + +static void ion_buffer_kmap_put(struct ion_buffer *buffer) +{ + if (buffer->kmap_cnt == 0) { + pr_warn_ratelimited("ION client likely missing a call to dma_buf_kmap or dma_buf_vmap, pid:%d\n", + current->pid); + return; + } + + buffer->kmap_cnt--; + if (!buffer->kmap_cnt) { + buffer->heap->ops->unmap_kernel(buffer->heap, buffer); + buffer->vaddr = NULL; + } +} + +static struct sg_table *dup_sg_table(struct sg_table *table) +{ + struct sg_table *new_table; + int ret, i; + struct scatterlist *sg, *new_sg; + + new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); + if (!new_table) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table(new_table, table->nents, GFP_KERNEL); + if (ret) { + kfree(new_table); + return ERR_PTR(-ENOMEM); + } + + new_sg = new_table->sgl; + for_each_sg(table->sgl, sg, table->nents, i) { + memcpy(new_sg, sg, sizeof(*sg)); + sg_dma_address(new_sg) = 0; + sg_dma_len(new_sg) = 0; + new_sg = sg_next(new_sg); + } + + return new_table; +} + +static void free_duped_table(struct sg_table *table) +{ + sg_free_table(table); + kfree(table); +} + +struct ion_dma_buf_attachment { + struct device *dev; + struct sg_table *table; + struct list_head list; + bool dma_mapped; +}; + +static int ion_dma_buf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct ion_dma_buf_attachment *a; + struct sg_table *table; + struct ion_buffer *buffer = dmabuf->priv; + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + + table = dup_sg_table(buffer->sg_table); + if (IS_ERR(table)) { + kfree(a); + return -ENOMEM; + } + + a->table = table; + a->dev = attachment->dev; + a->dma_mapped = false; + INIT_LIST_HEAD(&a->list); + + attachment->priv = a; + + mutex_lock(&buffer->lock); + list_add(&a->list, &buffer->attachments); + mutex_unlock(&buffer->lock); + + return 0; +} + +static void ion_dma_buf_detatch(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct ion_dma_buf_attachment *a = attachment->priv; + struct ion_buffer *buffer = dmabuf->priv; + + mutex_lock(&buffer->lock); + list_del(&a->list); + mutex_unlock(&buffer->lock); + free_duped_table(a->table); + + kfree(a); +} + +bool ion_buffer_cached(struct ion_buffer *buffer) +{ + return !!(buffer->flags & ION_FLAG_CACHED); +} + +static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct ion_dma_buf_attachment *a = attachment->priv; + struct sg_table *table; + int count, map_attrs; + struct ion_buffer *buffer = attachment->dmabuf->priv; + unsigned long ino = file_inode(attachment->dmabuf->file)->i_ino; + + table = a->table; + + map_attrs = attachment->dma_map_attrs; + if (!(buffer->flags & ION_FLAG_CACHED) || + !hlos_accessible_buffer(buffer)) + map_attrs |= DMA_ATTR_SKIP_CPU_SYNC; + + mutex_lock(&buffer->lock); + if (map_attrs & DMA_ATTR_SKIP_CPU_SYNC) + trace_ion_dma_map_cmo_skip(attachment->dev, + ino, + ion_buffer_cached(buffer), + hlos_accessible_buffer(buffer), + attachment->dma_map_attrs, + direction); + else + trace_ion_dma_map_cmo_apply(attachment->dev, + ino, + ion_buffer_cached(buffer), + hlos_accessible_buffer(buffer), + attachment->dma_map_attrs, + direction); + + if (map_attrs & DMA_ATTR_DELAYED_UNMAP) { + count = msm_dma_map_sg_attrs(attachment->dev, table->sgl, + table->nents, direction, + attachment->dmabuf, map_attrs); + } else { + count = dma_map_sg_attrs(attachment->dev, table->sgl, + table->nents, direction, + map_attrs); + } + + if (count <= 0) { + mutex_unlock(&buffer->lock); + return ERR_PTR(-ENOMEM); + } + + a->dma_mapped = true; + mutex_unlock(&buffer->lock); + return table; +} + +void ion_prepare_sgl_for_force_dma_sync(struct sg_table *table) +{ + struct scatterlist *sg; + int i; + + if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { + /* + * this will set up dma addresses for the sglist -- it is not + * technically correct as per the dma api -- a specific + * device isn't really taking ownership here. However, in + * practice on our systems the only dma_address space is + * physical addresses. + */ + for_each_sg(table->sgl, sg, table->nents, i) { + sg_dma_address(sg) = sg_phys(sg); + sg_dma_len(sg) = sg->length; + } + } +} + +static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + int map_attrs; + struct ion_buffer *buffer = attachment->dmabuf->priv; + struct ion_dma_buf_attachment *a = attachment->priv; + unsigned long ino = file_inode(attachment->dmabuf->file)->i_ino; + + map_attrs = attachment->dma_map_attrs; + if (!(buffer->flags & ION_FLAG_CACHED) || + !hlos_accessible_buffer(buffer)) + map_attrs |= DMA_ATTR_SKIP_CPU_SYNC; + + mutex_lock(&buffer->lock); + if (map_attrs & DMA_ATTR_SKIP_CPU_SYNC) + trace_ion_dma_unmap_cmo_skip(attachment->dev, + ino, + ion_buffer_cached(buffer), + hlos_accessible_buffer(buffer), + attachment->dma_map_attrs, + direction); + else + trace_ion_dma_unmap_cmo_apply(attachment->dev, + ino, + ion_buffer_cached(buffer), + hlos_accessible_buffer(buffer), + attachment->dma_map_attrs, + direction); + + if (map_attrs & DMA_ATTR_DELAYED_UNMAP) + msm_dma_unmap_sg_attrs(attachment->dev, table->sgl, + table->nents, direction, + attachment->dmabuf, + map_attrs); + else + dma_unmap_sg_attrs(attachment->dev, table->sgl, table->nents, + direction, map_attrs); + a->dma_mapped = false; + mutex_unlock(&buffer->lock); +} + +void ion_pages_sync_for_device(struct device *dev, struct page *page, + size_t size, enum dma_data_direction dir) +{ + struct scatterlist sg; + + sg_init_table(&sg, 1); + sg_set_page(&sg, page, size, 0); + /* + * This is not correct - sg_dma_address needs a dma_addr_t that is valid + * for the targeted device, but this works on the currently targeted + * hardware. + */ + sg_dma_address(&sg) = page_to_phys(page); + dma_sync_sg_for_device(dev, &sg, 1, dir); +} + +static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct ion_buffer *buffer = dmabuf->priv; + int ret = 0; + + if (!buffer->heap->ops->map_user) { + pr_err("%s: this heap does not define a method for mapping to userspace\n", + __func__); + return -EINVAL; + } + + if (!(buffer->flags & ION_FLAG_CACHED)) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + + mutex_lock(&buffer->lock); + /* now map it to userspace */ + ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma); + mutex_unlock(&buffer->lock); + + if (ret) + pr_err("%s: failure mapping buffer to userspace\n", + __func__); + + return ret; +} + +static void ion_dma_buf_release(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer = dmabuf->priv; + + _ion_buffer_destroy(buffer); + kfree(dmabuf->exp_name); +} + +static void *ion_dma_buf_vmap(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer = dmabuf->priv; + void *vaddr = ERR_PTR(-EINVAL); + + if (buffer->heap->ops->map_kernel) { + mutex_lock(&buffer->lock); + vaddr = ion_buffer_kmap_get(buffer); + mutex_unlock(&buffer->lock); + } else { + pr_warn_ratelimited("heap %s doesn't support map_kernel\n", + buffer->heap->name); + } + + return vaddr; +} + +static void ion_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) +{ + struct ion_buffer *buffer = dmabuf->priv; + + if (buffer->heap->ops->map_kernel) { + mutex_lock(&buffer->lock); + ion_buffer_kmap_put(buffer); + mutex_unlock(&buffer->lock); + } +} + +static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset) +{ + /* + * TODO: Once clients remove their hacks where they assume kmap(ed) + * addresses are virtually contiguous implement this properly + */ + void *vaddr = ion_dma_buf_vmap(dmabuf); + + if (IS_ERR(vaddr)) + return vaddr; + + return vaddr + offset * PAGE_SIZE; +} + +static void ion_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset, + void *ptr) +{ + /* + * TODO: Once clients remove their hacks where they assume kmap(ed) + * addresses are virtually contiguous implement this properly + */ + ion_dma_buf_vunmap(dmabuf, ptr); +} + +static int ion_sgl_sync_range(struct device *dev, struct scatterlist *sgl, + unsigned int nents, unsigned long offset, + unsigned long length, + enum dma_data_direction dir, bool for_cpu) +{ + int i; + struct scatterlist *sg; + unsigned int len = 0; + dma_addr_t sg_dma_addr; + + for_each_sg(sgl, sg, nents, i) { + if (sg_dma_len(sg) == 0) + break; + + if (i > 0) { + pr_warn("Partial cmo only supported with 1 segment\n" + "is dma_set_max_seg_size being set on dev:%s\n", + dev_name(dev)); + return -EINVAL; + } + } + + for_each_sg(sgl, sg, nents, i) { + unsigned int sg_offset, sg_left, size = 0; + + if (i == 0) + sg_dma_addr = sg_dma_address(sg); + + len += sg->length; + if (len <= offset) { + sg_dma_addr += sg->length; + continue; + } + + sg_left = len - offset; + sg_offset = sg->length - sg_left; + + size = (length < sg_left) ? length : sg_left; + if (for_cpu) + dma_sync_single_range_for_cpu(dev, sg_dma_addr, + sg_offset, size, dir); + else + dma_sync_single_range_for_device(dev, sg_dma_addr, + sg_offset, size, dir); + + offset += size; + length -= size; + sg_dma_addr += sg->length; + + if (length == 0) + break; + } + + return 0; +} + +static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_dma_buf_attachment *a; + unsigned long ino = file_inode(dmabuf->file)->i_ino; + int ret = 0; + + if (!hlos_accessible_buffer(buffer)) { + trace_ion_begin_cpu_access_cmo_skip(NULL, ino, + ion_buffer_cached(buffer), + false, direction); + ret = -EPERM; + goto out; + } + + if (!(buffer->flags & ION_FLAG_CACHED)) { + trace_ion_begin_cpu_access_cmo_skip(NULL, ino, false, true, + direction); + goto out; + } + + mutex_lock(&buffer->lock); + + if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { + struct device *dev = msm_ion_heap_device(buffer->heap); + struct sg_table *table = buffer->sg_table; + + dma_sync_sg_for_cpu(dev, table->sgl, table->nents, direction); + + trace_ion_begin_cpu_access_cmo_apply(dev, ino, true, true, + direction); + + mutex_unlock(&buffer->lock); + goto out; + } + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->dma_mapped) { + trace_ion_begin_cpu_access_notmapped(a->dev, + ino, + true, true, + direction); + continue; + } + + dma_sync_sg_for_cpu(a->dev, a->table->sgl, + a->table->nents, direction); + + trace_ion_begin_cpu_access_cmo_apply(a->dev, ino, true, + true, direction); + } + mutex_unlock(&buffer->lock); +out: + return ret; +} + +static int ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_dma_buf_attachment *a; + unsigned long ino = file_inode(dmabuf->file)->i_ino; + int ret = 0; + + if (!hlos_accessible_buffer(buffer)) { + trace_ion_end_cpu_access_cmo_skip(NULL, ino, + ion_buffer_cached(buffer), + false, direction); + ret = -EPERM; + goto out; + } + + if (!(buffer->flags & ION_FLAG_CACHED)) { + trace_ion_end_cpu_access_cmo_skip(NULL, ino, false, true, + direction); + goto out; + } + + mutex_lock(&buffer->lock); + if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { + struct device *dev = msm_ion_heap_device(buffer->heap); + struct sg_table *table = buffer->sg_table; + + dma_sync_sg_for_device(dev, table->sgl, table->nents, + direction); + + trace_ion_end_cpu_access_cmo_apply(dev, ino, true, + true, direction); + mutex_unlock(&buffer->lock); + goto out; + } + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->dma_mapped) { + trace_ion_end_cpu_access_notmapped(a->dev, + ino, + true, true, + direction); + continue; + } + + dma_sync_sg_for_device(a->dev, a->table->sgl, a->table->nents, + direction); + + trace_ion_end_cpu_access_cmo_apply(a->dev, ino, true, + true, direction); + } + mutex_unlock(&buffer->lock); + +out: + return ret; +} + +static int ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction dir, + unsigned int offset, + unsigned int len) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_dma_buf_attachment *a; + unsigned long ino = file_inode(dmabuf->file)->i_ino; + int ret = 0; + + if (!hlos_accessible_buffer(buffer)) { + trace_ion_begin_cpu_access_cmo_skip(NULL, ino, + ion_buffer_cached(buffer), + false, dir); + ret = -EPERM; + goto out; + } + + if (!(buffer->flags & ION_FLAG_CACHED)) { + trace_ion_begin_cpu_access_cmo_skip(NULL, ino, false, true, + dir); + goto out; + } + + mutex_lock(&buffer->lock); + if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { + struct device *dev = msm_ion_heap_device(buffer->heap); + struct sg_table *table = buffer->sg_table; + + ret = ion_sgl_sync_range(dev, table->sgl, table->nents, + offset, len, dir, true); + + if (!ret) + trace_ion_begin_cpu_access_cmo_apply(dev, ino, + true, true, dir); + else + trace_ion_begin_cpu_access_cmo_skip(dev, ino, + true, true, dir); + mutex_unlock(&buffer->lock); + goto out; + } + + list_for_each_entry(a, &buffer->attachments, list) { + int tmp = 0; + + if (!a->dma_mapped) { + trace_ion_begin_cpu_access_notmapped(a->dev, + ino, + true, true, + dir); + continue; + } + + tmp = ion_sgl_sync_range(a->dev, a->table->sgl, a->table->nents, + offset, len, dir, true); + + if (!tmp) { + trace_ion_begin_cpu_access_cmo_apply(a->dev, ino, + true, true, dir); + } else { + trace_ion_begin_cpu_access_cmo_skip(a->dev, ino, + true, true, dir); + ret = tmp; + } + } + mutex_unlock(&buffer->lock); + +out: + return ret; +} + +static int ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, + unsigned int len) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_dma_buf_attachment *a; + unsigned long ino = file_inode(dmabuf->file)->i_ino; + + int ret = 0; + + if (!hlos_accessible_buffer(buffer)) { + trace_ion_end_cpu_access_cmo_skip(NULL, ino, + ion_buffer_cached(buffer), + false, direction); + ret = -EPERM; + goto out; + } + + if (!(buffer->flags & ION_FLAG_CACHED)) { + trace_ion_end_cpu_access_cmo_skip(NULL, ino, false, + true, direction); + goto out; + } + + mutex_lock(&buffer->lock); + if (IS_ENABLED(CONFIG_ION_FORCE_DMA_SYNC)) { + struct device *dev = msm_ion_heap_device(buffer->heap); + struct sg_table *table = buffer->sg_table; + + ret = ion_sgl_sync_range(dev, table->sgl, table->nents, + offset, len, direction, false); + + if (!ret) + trace_ion_end_cpu_access_cmo_apply(dev, ino, + true, true, + direction); + else + trace_ion_end_cpu_access_cmo_skip(dev, ino, + true, true, + direction); + + mutex_unlock(&buffer->lock); + goto out; + } + + list_for_each_entry(a, &buffer->attachments, list) { + int tmp = 0; + + if (!a->dma_mapped) { + trace_ion_end_cpu_access_notmapped(a->dev, + ino, + true, true, + direction); + continue; + } + + tmp = ion_sgl_sync_range(a->dev, a->table->sgl, a->table->nents, + offset, len, direction, false); + + if (!tmp) { + trace_ion_end_cpu_access_cmo_apply(a->dev, ino, + true, true, + direction); + + } else { + trace_ion_end_cpu_access_cmo_skip(a->dev, ino, true, + true, direction); + ret = tmp; + } + } + mutex_unlock(&buffer->lock); + +out: + return ret; +} + +static int ion_dma_buf_get_flags(struct dma_buf *dmabuf, + unsigned long *flags) +{ + struct ion_buffer *buffer = dmabuf->priv; + *flags = buffer->flags; + + return 0; +} + +const struct dma_buf_ops msm_ion_dma_buf_ops = { + .map_dma_buf = ion_map_dma_buf, + .unmap_dma_buf = ion_unmap_dma_buf, + .mmap = ion_mmap, + .release = ion_dma_buf_release, + .attach = ion_dma_buf_attach, + .detach = ion_dma_buf_detatch, + .begin_cpu_access = ion_dma_buf_begin_cpu_access, + .end_cpu_access = ion_dma_buf_end_cpu_access, + .begin_cpu_access_partial = ion_dma_buf_begin_cpu_access_partial, + .end_cpu_access_partial = ion_dma_buf_end_cpu_access_partial, + .map = ion_dma_buf_kmap, + .unmap = ion_dma_buf_kunmap, + .vmap = ion_dma_buf_vmap, + .vunmap = ion_dma_buf_vunmap, + .get_flags = ion_dma_buf_get_flags, +}; diff --git a/drivers/staging/android/ion/msm_ion_of.c b/drivers/staging/android/ion/msm_ion_of.c new file mode 100644 index 000000000000..f9a3f207847a --- /dev/null +++ b/drivers/staging/android/ion/msm_ion_of.c @@ -0,0 +1,504 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ion.h" +#include "msm_ion_priv.h" + +#define ION_COMPAT_STR "qcom,msm-ion" +#define ION_NOT_READY 0 +#define ION_INIT_FAILURE 1 +#define ION_READY 2 + +static int num_heaps; +static int status = ION_NOT_READY; +static struct ion_heap **heaps; + +struct ion_heap_desc { + unsigned int id; + enum ion_heap_type type; + const char *name; +}; + +#ifdef CONFIG_OF +static struct ion_heap_desc ion_heap_meta[] = { + { + .id = ION_SYSTEM_HEAP_ID, + .name = ION_SYSTEM_HEAP_NAME, + }, + { + .id = ION_SECURE_HEAP_ID, + .name = ION_SECURE_HEAP_NAME, + }, + { + .id = ION_CP_MM_HEAP_ID, + .name = ION_MM_HEAP_NAME, + }, + { + .id = ION_USER_CONTIG_HEAP_ID, + .name = ION_USER_CONTIG_HEAP_NAME, + }, + { + .id = ION_QSECOM_HEAP_ID, + .name = ION_QSECOM_HEAP_NAME, + }, + { + .id = ION_QSECOM_TA_HEAP_ID, + .name = ION_QSECOM_TA_HEAP_NAME, + }, + { + .id = ION_SPSS_HEAP_ID, + .name = ION_SPSS_HEAP_NAME, + }, + { + .id = ION_ADSP_HEAP_ID, + .name = ION_ADSP_HEAP_NAME, + }, + { + .id = ION_SECURE_DISPLAY_HEAP_ID, + .name = ION_SECURE_DISPLAY_HEAP_NAME, + }, + { + .id = ION_AUDIO_HEAP_ID, + .name = ION_AUDIO_HEAP_NAME, + }, + { + .id = ION_SECURE_CARVEOUT_HEAP_ID, + .name = ION_SECURE_CARVEOUT_HEAP_NAME, + } +}; +#endif + +#ifdef CONFIG_OF +#define MAKE_HEAP_TYPE_MAPPING(h) { .name = #h, \ + .heap_type = ION_HEAP_TYPE_##h, } + +static struct heap_types_info { + const char *name; + int heap_type; +} heap_types_info[] = { + MAKE_HEAP_TYPE_MAPPING(SYSTEM), + MAKE_HEAP_TYPE_MAPPING(SYSTEM_CONTIG), + MAKE_HEAP_TYPE_MAPPING(CARVEOUT), + MAKE_HEAP_TYPE_MAPPING(SECURE_CARVEOUT), + MAKE_HEAP_TYPE_MAPPING(DMA), + MAKE_HEAP_TYPE_MAPPING(SYSTEM_SECURE), + MAKE_HEAP_TYPE_MAPPING(HYP_CMA), +}; + +static struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) +{ + struct ion_heap *heap = NULL; + int heap_type = heap_data->type; + + switch (heap_type) { + case ION_HEAP_TYPE_SYSTEM_CONTIG: + pr_err("%s: Heap type is disabled: %d\n", __func__, + heap_data->type); + break; + case ION_HEAP_TYPE_SYSTEM: + heap = ion_system_heap_create(heap_data); + break; + case ION_HEAP_TYPE_CARVEOUT: + heap = ion_carveout_heap_create(heap_data); + break; +#ifdef CONFIG_CMA + case ION_HEAP_TYPE_DMA: + heap = ion_cma_heap_create(heap_data); + break; + case (enum ion_heap_type)ION_HEAP_TYPE_HYP_CMA: + heap = ion_cma_secure_heap_create(heap_data); + break; +#endif + case (enum ion_heap_type)ION_HEAP_TYPE_SYSTEM_SECURE: + heap = ion_system_secure_heap_create(heap_data); + break; + case (enum ion_heap_type)ION_HEAP_TYPE_SECURE_CARVEOUT: + heap = ion_secure_carveout_heap_create(heap_data); + break; + default: + pr_err("%s: Invalid heap type %d\n", __func__, + heap_data->type); + return ERR_PTR(-EINVAL); + } + + if (IS_ERR_OR_NULL(heap)) { + pr_err("%s: error creating heap %s type %d base %pa size %zu\n", + __func__, heap_data->name, heap_data->type, + &heap_data->base, heap_data->size); + return ERR_PTR(-EINVAL); + } + + heap->name = heap_data->name; + heap->id = heap_data->id; + return heap; +} + +struct device *msm_ion_heap_device(struct ion_heap *heap) +{ + if (!heap) + return ERR_PTR(-EINVAL); + + if (status == ION_NOT_READY) + return ERR_PTR(-EPROBE_DEFER); + else if (status == ION_INIT_FAILURE) + return ERR_PTR(-ENODEV); + + return to_msm_ion_heap(heap)->dev; +} + +struct device *msm_ion_heap_device_by_id(int heap_id) +{ + struct ion_heap *heap; + + if (status == ION_NOT_READY) + return ERR_PTR(-EPROBE_DEFER); + else if (status == ION_INIT_FAILURE) + return ERR_PTR(-ENODEV); + + heap = get_ion_heap(heap_id); + if (heap) + return msm_ion_heap_device(heap); + + return ERR_PTR(-EINVAL); +} +EXPORT_SYMBOL(msm_ion_heap_device_by_id); + +static int msm_ion_get_heap_type_from_dt_node(struct device_node *node, + int *heap_type) +{ + const char *name; + int i, ret = -EINVAL; + + ret = of_property_read_string(node, "qcom,ion-heap-type", &name); + if (ret) + goto out; + for (i = 0; i < ARRAY_SIZE(heap_types_info); ++i) { + if (!strcmp(heap_types_info[i].name, name)) { + *heap_type = heap_types_info[i].heap_type; + ret = 0; + goto out; + } + } + WARN(1, "Unknown heap type: %s. You might need to update heap_types_info in %s", + name, __FILE__); +out: + return ret; +} + +static int msm_ion_populate_heap(struct device_node *node, + struct ion_platform_heap *heap) +{ + unsigned int i; + int ret = -EINVAL, heap_type = -1; + unsigned int len = ARRAY_SIZE(ion_heap_meta); + + for (i = 0; i < len; ++i) { + if (ion_heap_meta[i].id == heap->id) { + heap->name = ion_heap_meta[i].name; + ret = msm_ion_get_heap_type_from_dt_node(node, + &heap_type); + if (ret) + break; + heap->type = heap_type; + break; + } + } + if (ret) + pr_err("%s: Unable to populate heap, error: %d\n", __func__, + ret); + return ret; +} + +static void free_pdata(const struct ion_platform_data *pdata) +{ + kfree(pdata->heaps); + kfree(pdata); +} + +static int init_reserved_memory(struct device *dev, struct device_node *pnode) +{ + int ret = 0; + struct reserved_mem *rmem = of_reserved_mem_lookup(pnode); + + if (!rmem) { + dev_err(dev, "Failed to find reserved memory region\n"); + return -EINVAL; + } + + /* + * We only need to call this when the memory-region is managed by + * a reserved memory region driver (e.g. CMA, coherent, etc). In that + * case, they will have ops for device specific initialization for + * the memory region. Otherwise, we have a pure carveout, which needs + * not be initialized. + */ + if (rmem->ops) { + ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0); + if (ret) + dev_err(dev, "Failed to initialize memory region\n"); + } + + return ret; +} + +static void release_reserved_memory(struct device *dev, + struct device_node *pnode) +{ + struct reserved_mem *rmem = of_reserved_mem_lookup(pnode); + + if (rmem && rmem->ops) + of_reserved_mem_device_release(dev); +} + +static void release_reserved_memory_regions(struct ion_platform_heap *heaps, + int idx) +{ + struct device *dev; + struct device_node *node, *mem_node; + + for (idx = idx - 1; idx >= 0; idx--) { + dev = heaps[idx].priv; + node = dev->of_node; + mem_node = of_parse_phandle(node, "memory-region", 0); + + if (mem_node) + release_reserved_memory(dev, mem_node); + } +} + +static int msm_ion_get_heap_dt_data(struct device_node *node, + struct ion_platform_heap *heap) +{ + struct device_node *pnode; + struct device *dev = heap->priv; + int ret = -EINVAL; + + pnode = of_parse_phandle(node, "memory-region", 0); + if (pnode) { + const __be32 *basep; + u64 size = 0; + u64 base = 0; + + ret = init_reserved_memory(dev, pnode); + if (ret) + goto out; + + basep = of_get_address(pnode, 0, &size, NULL); + if (!basep) { + if (dev->cma_area) { + base = cma_get_base(dev->cma_area); + size = cma_get_size(dev->cma_area); + ret = 0; + } else if (dev->dma_mem) { + base = dma_get_device_base(dev, dev->dma_mem); + size = dma_get_size(dev->dma_mem); + ret = 0; + } + } else { + base = of_translate_address(pnode, basep); + if (base != OF_BAD_ADDR) + ret = 0; + } + + if (!ret) { + heap->base = base; + heap->size = size; + } + of_node_put(pnode); + } else { + ret = 0; + } + +out: + WARN(ret, "Failed to parse DT node for heap %s\n", heap->name); + return ret; +} + +static struct ion_platform_data *msm_ion_parse_dt(struct platform_device *pdev) +{ + struct ion_platform_data *pdata = 0; + struct ion_platform_heap *heaps = NULL; + struct device_node *node; + struct platform_device *new_dev = NULL; + const struct device_node *dt_node = pdev->dev.of_node; + const __be32 *val; + int ret = -EINVAL; + u32 num_heaps = 0; + int idx = 0; + + for_each_available_child_of_node(dt_node, node) + num_heaps++; + + if (!num_heaps) + return ERR_PTR(-EINVAL); + + pdata = kzalloc(sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + heaps = kcalloc(num_heaps, sizeof(struct ion_platform_heap), + GFP_KERNEL); + if (!heaps) { + kfree(pdata); + return ERR_PTR(-ENOMEM); + } + + pdata->heaps = heaps; + pdata->nr = num_heaps; + + for_each_available_child_of_node(dt_node, node) { + new_dev = of_platform_device_create(node, NULL, &pdev->dev); + if (!new_dev) { + pr_err("Failed to create device %s\n", node->name); + goto free_heaps; + } + of_dma_configure(&new_dev->dev, node, true); + + pdata->heaps[idx].priv = &new_dev->dev; + val = of_get_address(node, 0, NULL, NULL); + if (!val) { + pr_err("%s: Unable to find reg key\n", __func__); + goto free_heaps; + } + pdata->heaps[idx].id = (u32)of_read_number(val, 1); + + ret = msm_ion_populate_heap(node, &pdata->heaps[idx]); + if (ret) + goto free_heaps; + + ret = msm_ion_get_heap_dt_data(node, &pdata->heaps[idx]); + if (ret) + goto free_heaps; + + ++idx; + } + return pdata; + +free_heaps: + release_reserved_memory_regions(pdata->heaps, idx); + free_pdata(pdata); + return ERR_PTR(ret); +} +#else +static struct ion_platform_data *msm_ion_parse_dt(struct platform_device *pdev) +{ + return NULL; +} + +static void free_pdata(const struct ion_platform_data *pdata) +{ +} +#endif + +struct ion_heap *get_ion_heap(int heap_id) +{ + int i; + struct ion_heap *heap; + + for (i = 0; i < num_heaps; i++) { + heap = heaps[i]; + if (heap->id == heap_id) + return heap; + } + + pr_err("%s: heap_id %d not found\n", __func__, heap_id); + return NULL; +} + +static int msm_ion_probe(struct platform_device *pdev) +{ + struct ion_platform_data *pdata; + unsigned int pdata_needs_to_be_freed; + int err = -1; + int i; + + if (pdev->dev.of_node) { + pdata = msm_ion_parse_dt(pdev); + if (IS_ERR(pdata)) { + status = ION_INIT_FAILURE; + return PTR_ERR(pdata); + } + pdata_needs_to_be_freed = 1; + } else { + pdata = pdev->dev.platform_data; + pdata_needs_to_be_freed = 0; + } + + num_heaps = pdata->nr; + + heaps = kcalloc(pdata->nr, sizeof(struct ion_heap *), GFP_KERNEL); + + if (!heaps) { + err = -ENOMEM; + goto out; + } + + /* create the heaps as specified in the board file */ + for (i = 0; i < num_heaps; i++) { + struct ion_platform_heap *heap_data = &pdata->heaps[i]; + + heaps[i] = ion_heap_create(heap_data); + if (IS_ERR_OR_NULL(heaps[i])) { + heaps[i] = 0; + continue; + } else { + if (heap_data->size) + pr_info("ION heap %s created at %pa with size %zx\n", + heap_data->name, + &heap_data->base, + heap_data->size); + else + pr_info("ION heap %s created\n", + heap_data->name); + } + + ion_device_add_heap(heaps[i]); + } + if (pdata_needs_to_be_freed) + free_pdata(pdata); + + /* + * Publish the status at the end, so our interfaces know that they + * can stop returning -EPROBE_DEFER. + */ + status = ION_READY; + + return 0; + +out: + kfree(heaps); + if (pdata_needs_to_be_freed) + free_pdata(pdata); + status = ION_INIT_FAILURE; + return err; +} + +static const struct of_device_id msm_ion_match_table[] = { + {.compatible = ION_COMPAT_STR}, + {}, +}; + +static struct platform_driver msm_ion_driver = { + .probe = msm_ion_probe, + .driver = { + .name = "ion-msm", + .of_match_table = msm_ion_match_table, + }, +}; +module_platform_driver(msm_ion_driver); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/staging/android/ion/msm_ion_priv.h b/drivers/staging/android/ion/msm_ion_priv.h new file mode 100644 index 000000000000..36c04b28abd8 --- /dev/null +++ b/drivers/staging/android/ion/msm_ion_priv.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2011 Google, Inc. + * Copyright (c) 2011-2019, The Linux Foundation. All rights reserved. + */ + +#ifndef _MSM_ION_PRIV_H +#define _MSM_ION_PRIV_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../uapi/ion.h" +#include "../uapi/msm_ion.h" + +#define ION_ADSP_HEAP_NAME "adsp" +#define ION_SYSTEM_HEAP_NAME "system" +#define ION_MM_HEAP_NAME "mm" +#define ION_SPSS_HEAP_NAME "spss" +#define ION_SECURE_CARVEOUT_HEAP_NAME "secure_carveout" +#define ION_USER_CONTIG_HEAP_NAME "user_contig" +#define ION_QSECOM_HEAP_NAME "qsecom" +#define ION_QSECOM_TA_HEAP_NAME "qsecom_ta" +#define ION_SECURE_HEAP_NAME "secure_heap" +#define ION_SECURE_DISPLAY_HEAP_NAME "secure_display" +#define ION_AUDIO_HEAP_NAME "audio" + +/** + * Debug feature. Make ION allocations DMA + * ready to help identify clients who are wrongly + * dependending on ION allocations being DMA + * ready. + * + * As default set to 'false' since ION allocations + * are no longer required to be DMA ready + */ +#ifdef CONFIG_ION_FORCE_DMA_SYNC +#define MAKE_ION_ALLOC_DMA_READY 1 +#else +#define MAKE_ION_ALLOC_DMA_READY 0 +#endif + +#define to_msm_ion_heap(x) container_of(x, struct msm_ion_heap, ion_heap) + +/** + * struct ion_platform_heap - defines a heap in the given platform + * @type: type of the heap from ion_heap_type enum + * @id: unique identifier for heap. When allocating higher numb ers + * will be allocated from first. At allocation these are passed + * as a bit mask and therefore can not exceed ION_NUM_HEAP_IDS. + * @name: used for debug purposes + * @base: base address of heap in physical memory if applicable + * @size: size of the heap in bytes if applicable + * @priv: private info passed from the board file + * + * Provided by the board file. + */ +struct ion_platform_heap { + enum ion_heap_type type; + unsigned int id; + const char *name; + phys_addr_t base; + size_t size; + phys_addr_t align; + void *priv; +}; + +/** + * struct msm_ion_heap - defines an ion heap, as well as additional information + * relevant to the heap. + * @dev: the device structure associated with the heap + * @ion_heap: ion heap + * + */ +struct msm_ion_heap { + struct device *dev; + struct ion_heap ion_heap; +}; + +/** + * struct ion_platform_data - array of platform heaps passed from board file + * @nr: number of structures in the array + * @heaps: array of platform_heap structions + * + * Provided by the board file in the form of platform data to a platform device. + */ +struct ion_platform_data { + int nr; + struct ion_platform_heap *heaps; +}; + +/** + * ion_buffer_cached - this ion buffer is cached + * @buffer: buffer + * + * indicates whether this ion buffer is cached + */ +bool ion_buffer_cached(struct ion_buffer *buffer); + +/** + * functions for creating and destroying the built in ion heaps. + * architectures can add their own custom architecture specific + * heaps as appropriate. + */ + +struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused); + +struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *heap); + +struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *heap); + +struct ion_heap *ion_carveout_heap_create(struct ion_platform_heap *heap_data); + +struct ion_heap +*ion_secure_carveout_heap_create(struct ion_platform_heap *heap); + +struct ion_heap *ion_chunk_heap_create(struct ion_platform_heap *heap_data); + +#ifdef CONFIG_CMA +struct ion_heap *ion_cma_secure_heap_create(struct ion_platform_heap *heap); + +struct ion_heap *ion_cma_heap_create(struct ion_platform_heap *data); +#else +static inline struct ion_heap + *ion_cma_secure_heap_create(struct ion_platform_heap *h) +{ + return NULL; +} + +static inline struct ion_heap *ion_cma_heap_create(struct ion_platform_heap *h) +{ + return NULL; +} +#endif + +struct device *msm_ion_heap_device(struct ion_heap *heap); + +struct ion_heap *get_ion_heap(int heap_id); + +void ion_prepare_sgl_for_force_dma_sync(struct sg_table *table); + +/** + * ion_pages_sync_for_device - cache flush pages for use with the specified + * device + * @dev: the device the pages will be used with + * @page: the first page to be flushed + * @size: size in bytes of region to be flushed + * @dir: direction of dma transfer + */ +void ion_pages_sync_for_device(struct device *dev, struct page *page, + size_t size, enum dma_data_direction dir); + +extern const struct dma_buf_ops msm_ion_dma_buf_ops; +#endif /* _MSM_ION_PRIV_H */ diff --git a/drivers/staging/android/uapi/msm_ion.h b/drivers/staging/android/uapi/msm_ion.h new file mode 100644 index 000000000000..87d8d8cb36eb --- /dev/null +++ b/drivers/staging/android/uapi/msm_ion.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + */ +#ifndef _UAPI_LINUX_MSM_ION_H +#define _UAPI_LINUX_MSM_ION_H + +#include + +#define ION_BIT(nr) (1U << (nr)) + +/** + * TARGET_ION_ABI_VERSION can be used by user space clients to ensure that at + * compile time only their code which uses the appropriate ION APIs for + * this kernel is included. + */ +#define TARGET_ION_ABI_VERSION 2 + +enum msm_ion_heap_types { + ION_HEAP_TYPE_MSM_START = 6, + ION_HEAP_TYPE_SECURE_DMA = ION_HEAP_TYPE_MSM_START, + ION_HEAP_TYPE_SYSTEM_SECURE, + ION_HEAP_TYPE_HYP_CMA, + ION_HEAP_TYPE_SECURE_CARVEOUT, +}; + +/** + * These are the only ids that should be used for Ion heap ids. + * The ids listed are the order in which allocation will be attempted + * if specified. Don't swap the order of heap ids unless you know what + * you are doing! + * Id's are spaced by purpose to allow new Id's to be inserted in-between (for + * possible fallbacks) + */ + +enum ion_heap_ids { + INVALID_HEAP_ID = -1, + ION_CP_MM_HEAP_ID = 8, + ION_SECURE_HEAP_ID = 9, + ION_SECURE_DISPLAY_HEAP_ID = 10, + ION_SPSS_HEAP_ID = 13, /* Secure Processor ION heap */ + ION_ADSP_HEAP_ID = 22, + ION_SYSTEM_HEAP_ID = 25, + ION_QSECOM_HEAP_ID = 27, + ION_HEAP_ID_RESERVED = 31 /** Bit reserved for ION_FLAG_SECURE flag */ +}; + +/** + * Newly added heap ids have to be #define(d) since all API changes must + * include a new #define. + */ +#define ION_SECURE_CARVEOUT_HEAP_ID 14 +#define ION_QSECOM_TA_HEAP_ID 19 +#define ION_AUDIO_HEAP_ID 28 +#define ION_CAMERA_HEAP_ID 20 +#define ION_USER_CONTIG_HEAP_ID 26 +/** + * Flags to be used when allocating from the secure heap for + * content protection + */ +#define ION_FLAG_CP_TOUCH ION_BIT(17) +#define ION_FLAG_CP_BITSTREAM ION_BIT(18) +#define ION_FLAG_CP_PIXEL ION_BIT(19) +#define ION_FLAG_CP_NON_PIXEL ION_BIT(20) +#define ION_FLAG_CP_CAMERA ION_BIT(21) +#define ION_FLAG_CP_HLOS ION_BIT(22) +#define ION_FLAG_CP_SPSS_SP ION_BIT(23) +#define ION_FLAG_CP_SPSS_SP_SHARED ION_BIT(24) +#define ION_FLAG_CP_SEC_DISPLAY ION_BIT(25) +#define ION_FLAG_CP_APP ION_BIT(26) +#define ION_FLAG_CP_CAMERA_PREVIEW ION_BIT(27) +/* ION_FLAG_ALLOW_NON_CONTIG uses ION_BIT(28) */ +#define ION_FLAG_CP_CDSP ION_BIT(29) +#define ION_FLAG_CP_SPSS_HLOS_SHARED ION_BIT(30) + +#define ION_FLAGS_CP_MASK 0x6FFE0000 + +/** + * Flag to allow non continguous allocation of memory from secure + * heap + */ +#define ION_FLAG_ALLOW_NON_CONTIG ION_BIT(28) + +/** + * Flag to use when allocating to indicate that a heap is secure. + * Do NOT use BIT macro since it is defined in #ifdef __KERNEL__ + */ +#define ION_FLAG_SECURE ION_BIT(ION_HEAP_ID_RESERVED) + +/* + * Used in conjunction with heap which pool memory to force an allocation + * to come from the page allocator directly instead of from the pool allocation + */ +#define ION_FLAG_POOL_FORCE_ALLOC ION_BIT(16) + +/** + * Macro should be used with ion_heap_ids defined above. + */ +#define ION_HEAP(bit) ION_BIT(bit) + +#define ION_IOC_MSM_MAGIC 'M' + +struct ion_prefetch_regions { + __u64 sizes; + __u32 vmid; + __u32 nr_sizes; +}; + +struct ion_prefetch_data { + __u64 len; + __u64 regions; + __u32 heap_id; + __u32 nr_regions; +}; + +#define ION_IOC_PREFETCH _IOWR(ION_IOC_MSM_MAGIC, 3, \ + struct ion_prefetch_data) + +#define ION_IOC_DRAIN _IOWR(ION_IOC_MSM_MAGIC, 4, \ + struct ion_prefetch_data) + +#endif /* _UAPI_LINUX_MSM_ION_H */ diff --git a/include/linux/dma-buf-ref.h b/include/linux/dma-buf-ref.h new file mode 100644 index 000000000000..5aa363966d9d --- /dev/null +++ b/include/linux/dma-buf-ref.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + */ + +#ifndef _DMA_BUF_REF_H +#define _DMA_BUF_REF_H + +struct msm_dma_buf; +struct seq_file; + +#ifdef CONFIG_DEBUG_DMA_BUF_REF +void dma_buf_ref_init(struct msm_dma_buf *b); +void dma_buf_ref_destroy(struct msm_dma_buf *b); +void dma_buf_ref_mod(struct msm_dma_buf *b, int nr); +int dma_buf_ref_show(struct seq_file *s, struct msm_dma_buf *msm_dma_buf); + +#else +static inline void dma_buf_ref_init(struct msm_dma_buf *b) {} +static inline void dma_buf_ref_destroy(struct msm_dma_buf *b) {} +static inline void dma_buf_ref_mod(struct msm_dma_buf *b, int nr) {} +static inline int dma_buf_ref_show(struct seq_file *s, + struct msm_dma_buf *msm_dma_buf) +{ + return -ENOMEM; +} +#endif + + +#endif /* _DMA_BUF_REF_H */ diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index 1e148a48d706..3315d343d0b6 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -20,6 +20,7 @@ #include #include #include +#include #include struct device; @@ -390,6 +391,17 @@ struct dma_buf { } cb_excl, cb_shared; }; +/** + * struct msm_dma_buf - Holds the meta data associated with a shared buffer + * object, as well as the buffer object. + * @refs: list entry for dma-buf reference tracking + * @dma_buf: the shared buffer object + */ +struct msm_dma_buf { + struct list_head refs; + struct dma_buf dma_buf; +}; + /** * struct dma_buf_attachment - holds device-buffer attachment data * @dmabuf: buffer for this attachment. @@ -454,6 +466,12 @@ struct dma_buf_export_info { struct dma_buf_export_info name = { .exp_name = KBUILD_MODNAME, \ .owner = THIS_MODULE } +/** + * to_msm_dma_buf - helper macro for deriving an msm_dma_buf from a dma_buf. + */ +#define to_msm_dma_buf(_dma_buf) \ + container_of(_dma_buf, struct msm_dma_buf, dma_buf) + /** * get_dma_buf - convenience wrapper for get_file. * @dmabuf: [in] pointer to dma_buf @@ -466,6 +484,7 @@ struct dma_buf_export_info { static inline void get_dma_buf(struct dma_buf *dmabuf) { get_file(dmabuf->file); + dma_buf_ref_mod(to_msm_dma_buf(dmabuf), 1); } struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf, diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 8816b945bc4a..3f90bf8fcad5 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -798,10 +798,15 @@ static inline int dma_get_cache_alignment(void) } #ifdef CONFIG_DMA_DECLARE_COHERENT +struct dma_coherent_mem; int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, dma_addr_t device_addr, size_t size); void dma_release_declared_memory(struct device *dev); +dma_addr_t dma_get_device_base(struct device *dev, + struct dma_coherent_mem *mem); +unsigned long dma_get_size(struct dma_coherent_mem *mem); #else +struct dma_coherent_mem {}; static inline int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, dma_addr_t device_addr, size_t size) @@ -813,6 +818,17 @@ static inline void dma_release_declared_memory(struct device *dev) { } + +static inline dma_addr_t dma_get_device_base(struct device *dev, + struct dma_coherent_mem *mem) +{ + return 0; +} + +static inline unsigned long dma_get_size(struct dma_coherent_mem *mem) +{ + return 0; +} #endif /* CONFIG_DMA_DECLARE_COHERENT */ static inline void *dmam_alloc_coherent(struct device *dev, size_t size, diff --git a/include/linux/msm_ion_of.h b/include/linux/msm_ion_of.h new file mode 100644 index 000000000000..1492eca8451f --- /dev/null +++ b/include/linux/msm_ion_of.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#ifndef _MSM_ION_KERNEL_H +#define _MSM_ION_KERNEL_H + +struct device *msm_ion_heap_device_by_id(int heap_id); + +#endif /* _MSM_ION_H */ diff --git a/include/trace/events/ion.h b/include/trace/events/ion.h new file mode 100644 index 000000000000..377d847d66e2 --- /dev/null +++ b/include/trace/events/ion.h @@ -0,0 +1,170 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018 The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ion + +#if !defined(_TRACE_ION_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ION_H + +#include +#include + +#define DEV_NAME_NONE "None" + +DECLARE_EVENT_CLASS(ion_dma_map_cmo_class, + + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, unsigned long map_attrs, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, map_attrs, dir), + + TP_STRUCT__entry( + __string(dev_name, dev ? dev_name(dev) : DEV_NAME_NONE) + __field(unsigned long, ino) + __field(bool, cached) + __field(bool, hlos_accessible) + __field(unsigned long, map_attrs) + __field(enum dma_data_direction, dir) + ), + + TP_fast_assign( + __assign_str(dev_name, dev ? dev_name(dev) : DEV_NAME_NONE); + __entry->ino = ino; + __entry->cached = cached; + __entry->hlos_accessible = hlos_accessible; + __entry->map_attrs = map_attrs; + __entry->dir = dir; + ), + + TP_printk("dev=%s ino=%lu cached=%d access=%d map_attrs=0x%lx dir=%d", + __get_str(dev_name), + __entry->ino, + __entry->cached, + __entry->hlos_accessible, + __entry->map_attrs, + __entry->dir) +); + +DEFINE_EVENT(ion_dma_map_cmo_class, ion_dma_map_cmo_apply, + + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, unsigned long map_attrs, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, map_attrs, dir) +); + +DEFINE_EVENT(ion_dma_map_cmo_class, ion_dma_map_cmo_skip, + + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, unsigned long map_attrs, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, map_attrs, dir) +); + +DEFINE_EVENT(ion_dma_map_cmo_class, ion_dma_unmap_cmo_apply, + + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, unsigned long map_attrs, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, map_attrs, dir) +); + +DEFINE_EVENT(ion_dma_map_cmo_class, ion_dma_unmap_cmo_skip, + + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, unsigned long map_attrs, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, map_attrs, dir) +); + +DECLARE_EVENT_CLASS(ion_access_cmo_class, + + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, dir), + + TP_STRUCT__entry( + __string(dev_name, dev ? dev_name(dev) : DEV_NAME_NONE) + __field(unsigned long, ino) + __field(bool, cached) + __field(bool, hlos_accessible) + __field(enum dma_data_direction, dir) + ), + + TP_fast_assign( + __assign_str(dev_name, dev ? dev_name(dev) : DEV_NAME_NONE); + __entry->ino = ino; + __entry->cached = cached; + __entry->hlos_accessible = hlos_accessible; + __entry->dir = dir; + ), + + TP_printk("dev=%s ino=%ld cached=%d access=%d dir=%d", + __get_str(dev_name), + __entry->ino, + __entry->cached, + __entry->hlos_accessible, + __entry->dir) +); + +DEFINE_EVENT(ion_access_cmo_class, ion_begin_cpu_access_cmo_apply, + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, dir) +); + +DEFINE_EVENT(ion_access_cmo_class, ion_begin_cpu_access_cmo_skip, + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, dir) +); + +DEFINE_EVENT(ion_access_cmo_class, ion_begin_cpu_access_notmapped, + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, dir) +); + +DEFINE_EVENT(ion_access_cmo_class, ion_end_cpu_access_cmo_apply, + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, dir) +); + +DEFINE_EVENT(ion_access_cmo_class, ion_end_cpu_access_cmo_skip, + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, dir) +); + +DEFINE_EVENT(ion_access_cmo_class, ion_end_cpu_access_notmapped, + TP_PROTO(const struct device *dev, unsigned long ino, + bool cached, bool hlos_accessible, + enum dma_data_direction dir), + + TP_ARGS(dev, ino, cached, hlos_accessible, dir) +); +#endif /* _TRACE_ION_H */ + +#include + diff --git a/include/uapi/linux/ion.h b/include/uapi/linux/ion.h new file mode 120000 index 000000000000..17e8dbb24265 --- /dev/null +++ b/include/uapi/linux/ion.h @@ -0,0 +1 @@ +../../../drivers/staging/android/uapi/ion.h \ No newline at end of file diff --git a/include/uapi/linux/msm_ion.h b/include/uapi/linux/msm_ion.h new file mode 120000 index 000000000000..94349d27efa6 --- /dev/null +++ b/include/uapi/linux/msm_ion.h @@ -0,0 +1 @@ +../../../drivers/staging/android/uapi/msm_ion.h \ No newline at end of file diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c index 29fd6590dc1e..57552b545a29 100644 --- a/kernel/dma/coherent.c +++ b/kernel/dma/coherent.c @@ -28,7 +28,7 @@ static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *de return NULL; } -static inline dma_addr_t dma_get_device_base(struct device *dev, +dma_addr_t dma_get_device_base(struct device *dev, struct dma_coherent_mem * mem) { if (mem->use_dev_dma_pfn_offset) @@ -37,6 +37,11 @@ static inline dma_addr_t dma_get_device_base(struct device *dev, return mem->device_base; } +unsigned long dma_get_size(struct dma_coherent_mem *mem) +{ + return mem->size << PAGE_SHIFT; +} + static int dma_init_coherent_memory(phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, struct dma_coherent_mem **mem)