From 0191b64327162c73e634980bd950c61daee15bc3 Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Tue, 18 Feb 2020 23:58:33 -0800 Subject: [PATCH 1/2] ion: Reduce secure system heap allocation time by stealing pages When performing a secure system heap allocation, and allocating from a secure pool fails, attempt to steal pages from the uncached system heap pools, before falling back to the buddy allocator. Additionally, remove redundant code that was once used for zeroing out pages allocated from buddy. Change-Id: Id0b08f49b1fb037b26cd09e6336a0b0dc5768d82 Signed-off-by: Chris Goldsworthy --- .../android/ion/heaps/ion_system_heap.c | 123 ++++++++---------- 1 file changed, 53 insertions(+), 70 deletions(-) diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c index 71c193f48222..6611daa756e6 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -51,11 +51,6 @@ static inline unsigned int order_to_size(int order) return PAGE_SIZE << order; } -struct pages_mem { - struct page **pages; - u32 size; -}; - int ion_heap_is_system_heap_type(enum ion_heap_type type) { return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM); @@ -71,14 +66,52 @@ static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, struct ion_page_pool *pool; int vmid = get_secure_vmid(buffer->flags); struct device *dev = sys_heap->heap.dev; + int order_ind = order_to_index(order); + struct task_struct *worker; - if (vmid > 0) - pool = sys_heap->secure_pools[vmid][order_to_index(order)]; - else if (!cached) - pool = sys_heap->uncached_pools[order_to_index(order)]; - else - pool = sys_heap->cached_pools[order_to_index(order)]; + if (vmid > 0) { + pool = sys_heap->secure_pools[vmid][order_ind]; + /* + * We should skip stealing pages if (1) we're focing our + * allocations to come from buddy; or (2) pool refilling is + * disabled, in which case stealing pages could deplete the + * uncached pools. + */ + if (!(*from_pool && pool_auto_refill_en)) + goto normal_alloc; + + page = ion_page_pool_alloc_pool_only(pool); + if (!IS_ERR(page)) + return page; + + pool = sys_heap->uncached_pools[order_ind]; + page = ion_page_pool_alloc_pool_only(pool); + if (IS_ERR(page)) { + pool = sys_heap->secure_pools[vmid][order_ind]; + goto normal_alloc; + } + + /* + * Here, setting `from_pool = false` indicates that the + * page didn't come from the secure pool, and causes + * the page to be hyp-assigned. + */ + *from_pool = false; + + if (pool_auto_refill_en && + pool_count_below_lowmark(pool)) { + worker = sys_heap->kworker[ION_KTHREAD_UNCACHED]; + wake_up_process(worker); + } + return page; + } else if (!cached) { + pool = sys_heap->uncached_pools[order_ind]; + } else { + pool = sys_heap->cached_pools[order_ind]; + } + +normal_alloc: page = ion_page_pool_alloc(pool, from_pool); if (pool_auto_refill_en && @@ -209,13 +242,11 @@ force_alloc: return alloc_largest_available(heap, buffer, size, max_order); } -static unsigned int process_info(struct page_info *info, - struct scatterlist *sg, - struct scatterlist *sg_sync, - struct pages_mem *data, unsigned int i) +static void process_info(struct page_info *info, + struct scatterlist *sg, + struct scatterlist *sg_sync) { struct page *page = info->page; - unsigned int j; if (sg_sync) { sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0); @@ -228,45 +259,9 @@ static unsigned int process_info(struct page_info *info, * on the currently targeted hardware. */ sg_dma_address(sg) = page_to_phys(page); - if (data) { - for (j = 0; j < (1 << info->order); ++j) - data->pages[i++] = nth_page(page, j); - } + list_del(&info->list); kfree(info); - return i; -} - -static int ion_heap_alloc_pages_mem(struct pages_mem *pages_mem) -{ - struct page **pages; - unsigned int page_tbl_size; - - page_tbl_size = sizeof(struct page *) * (pages_mem->size >> PAGE_SHIFT); - if (page_tbl_size > SZ_8K) { - /* - * Do fallback to ensure we have a balance between - * performance and availability. - */ - pages = kmalloc(page_tbl_size, - __GFP_COMP | __GFP_NORETRY | - __GFP_NOWARN); - if (!pages) - pages = vmalloc(page_tbl_size); - } else { - pages = kmalloc(page_tbl_size, GFP_KERNEL); - } - - if (!pages) - return -ENOMEM; - - pages_mem->pages = pages; - return 0; -} - -static void ion_heap_free_pages_mem(struct pages_mem *pages_mem) -{ - kvfree(pages_mem->pages); } static int ion_system_heap_allocate(struct ion_heap *heap, @@ -287,7 +282,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, unsigned int nents_sync = 0; unsigned long size_remaining = PAGE_ALIGN(size); unsigned int max_order = orders[0]; - struct pages_mem data; unsigned int sz; int vmid = get_secure_vmid(buffer->flags); @@ -301,7 +295,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, return -EINVAL; } - data.size = 0; INIT_LIST_HEAD(&pages); INIT_LIST_HEAD(&pages_from_pool); @@ -326,7 +319,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, list_add_tail(&info->list, &pages_from_pool); } else { list_add_tail(&info->list, &pages); - data.size += sz; ++nents_sync; } size_remaining -= sz; @@ -334,15 +326,10 @@ static int ion_system_heap_allocate(struct ion_heap *heap, i++; } - ret = ion_heap_alloc_pages_mem(&data); - - if (ret) - goto err; - table = kzalloc(sizeof(*table), GFP_KERNEL); if (!table) { ret = -ENOMEM; - goto err_free_data_pages; + goto err; } ret = sg_alloc_table(table, i, GFP_KERNEL); @@ -355,7 +342,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, goto err_free_sg; } - i = 0; sg = table->sgl; sg_sync = table_sync.sgl; @@ -371,16 +357,16 @@ static int ion_system_heap_allocate(struct ion_heap *heap, struct page_info, list); if (info && tmp_info) { if (info->order >= tmp_info->order) { - i = process_info(info, sg, sg_sync, &data, i); + process_info(info, sg, sg_sync); sg_sync = sg_next(sg_sync); } else { - i = process_info(tmp_info, sg, 0, 0, i); + process_info(tmp_info, sg, NULL); } } else if (info) { - i = process_info(info, sg, sg_sync, &data, i); + process_info(info, sg, sg_sync); sg_sync = sg_next(sg_sync); } else if (tmp_info) { - i = process_info(tmp_info, sg, 0, 0, i); + process_info(tmp_info, sg, NULL); } sg = sg_next(sg); @@ -397,7 +383,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, buffer->sg_table = table; if (nents_sync) sg_free_table(&table_sync); - ion_heap_free_pages_mem(&data); ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; @@ -417,8 +402,6 @@ err_free_sg: sg_free_table(table); err1: kfree(table); -err_free_data_pages: - ion_heap_free_pages_mem(&data); err: list_for_each_entry_safe(info, tmp_info, &pages, list) { free_buffer_page(sys_heap, buffer, info->page, info->order); From 6dccca705bbaf1c6223c3a0ef47070915df02af2 Mon Sep 17 00:00:00 2001 From: Charan Teja Reddy Date: Fri, 24 Jan 2020 13:58:11 +0530 Subject: [PATCH 2/2] ion: don't wakeup refill threads for order-0 requests We check the pool->count against the ION_POOL_LOW_MARK before deciding to wakeup the refill threads. But as we never refill the order-0 pool, we will keep waking up these refill threads only they goto sleep again without doing any work. The overhead of waking up of these threads is quite high. Fix it. Change-Id: I0d10b96a46de24b45822ebeda1f20b75e43c2a5d Signed-off-by: Charan Teja Reddy [cgoldswo@codeaurora.org: Fix merge conflicts, apply change elsewhere] Signed-off-by: Chris Goldsworthy --- drivers/staging/android/ion/heaps/ion_system_heap.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c index 6611daa756e6..a4034bb31fb9 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -99,7 +99,7 @@ static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, */ *from_pool = false; - if (pool_auto_refill_en && + if (pool_auto_refill_en && pool->order && pool_count_below_lowmark(pool)) { worker = sys_heap->kworker[ION_KTHREAD_UNCACHED]; wake_up_process(worker); @@ -114,10 +114,9 @@ static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, normal_alloc: page = ion_page_pool_alloc(pool, from_pool); - if (pool_auto_refill_en && - pool_count_below_lowmark(pool) && vmid <= 0) { + if (pool_auto_refill_en && pool->order && + pool_count_below_lowmark(pool) && vmid <= 0) wake_up_process(sys_heap->kworker[cached]); - } if (IS_ERR(page)) return page;