diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c index 71c193f48222..6611daa756e6 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -51,11 +51,6 @@ static inline unsigned int order_to_size(int order) return PAGE_SIZE << order; } -struct pages_mem { - struct page **pages; - u32 size; -}; - int ion_heap_is_system_heap_type(enum ion_heap_type type) { return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM); @@ -71,14 +66,52 @@ static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, struct ion_page_pool *pool; int vmid = get_secure_vmid(buffer->flags); struct device *dev = sys_heap->heap.dev; + int order_ind = order_to_index(order); + struct task_struct *worker; - if (vmid > 0) - pool = sys_heap->secure_pools[vmid][order_to_index(order)]; - else if (!cached) - pool = sys_heap->uncached_pools[order_to_index(order)]; - else - pool = sys_heap->cached_pools[order_to_index(order)]; + if (vmid > 0) { + pool = sys_heap->secure_pools[vmid][order_ind]; + /* + * We should skip stealing pages if (1) we're focing our + * allocations to come from buddy; or (2) pool refilling is + * disabled, in which case stealing pages could deplete the + * uncached pools. + */ + if (!(*from_pool && pool_auto_refill_en)) + goto normal_alloc; + + page = ion_page_pool_alloc_pool_only(pool); + if (!IS_ERR(page)) + return page; + + pool = sys_heap->uncached_pools[order_ind]; + page = ion_page_pool_alloc_pool_only(pool); + if (IS_ERR(page)) { + pool = sys_heap->secure_pools[vmid][order_ind]; + goto normal_alloc; + } + + /* + * Here, setting `from_pool = false` indicates that the + * page didn't come from the secure pool, and causes + * the page to be hyp-assigned. + */ + *from_pool = false; + + if (pool_auto_refill_en && + pool_count_below_lowmark(pool)) { + worker = sys_heap->kworker[ION_KTHREAD_UNCACHED]; + wake_up_process(worker); + } + return page; + } else if (!cached) { + pool = sys_heap->uncached_pools[order_ind]; + } else { + pool = sys_heap->cached_pools[order_ind]; + } + +normal_alloc: page = ion_page_pool_alloc(pool, from_pool); if (pool_auto_refill_en && @@ -209,13 +242,11 @@ force_alloc: return alloc_largest_available(heap, buffer, size, max_order); } -static unsigned int process_info(struct page_info *info, - struct scatterlist *sg, - struct scatterlist *sg_sync, - struct pages_mem *data, unsigned int i) +static void process_info(struct page_info *info, + struct scatterlist *sg, + struct scatterlist *sg_sync) { struct page *page = info->page; - unsigned int j; if (sg_sync) { sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0); @@ -228,45 +259,9 @@ static unsigned int process_info(struct page_info *info, * on the currently targeted hardware. */ sg_dma_address(sg) = page_to_phys(page); - if (data) { - for (j = 0; j < (1 << info->order); ++j) - data->pages[i++] = nth_page(page, j); - } + list_del(&info->list); kfree(info); - return i; -} - -static int ion_heap_alloc_pages_mem(struct pages_mem *pages_mem) -{ - struct page **pages; - unsigned int page_tbl_size; - - page_tbl_size = sizeof(struct page *) * (pages_mem->size >> PAGE_SHIFT); - if (page_tbl_size > SZ_8K) { - /* - * Do fallback to ensure we have a balance between - * performance and availability. - */ - pages = kmalloc(page_tbl_size, - __GFP_COMP | __GFP_NORETRY | - __GFP_NOWARN); - if (!pages) - pages = vmalloc(page_tbl_size); - } else { - pages = kmalloc(page_tbl_size, GFP_KERNEL); - } - - if (!pages) - return -ENOMEM; - - pages_mem->pages = pages; - return 0; -} - -static void ion_heap_free_pages_mem(struct pages_mem *pages_mem) -{ - kvfree(pages_mem->pages); } static int ion_system_heap_allocate(struct ion_heap *heap, @@ -287,7 +282,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, unsigned int nents_sync = 0; unsigned long size_remaining = PAGE_ALIGN(size); unsigned int max_order = orders[0]; - struct pages_mem data; unsigned int sz; int vmid = get_secure_vmid(buffer->flags); @@ -301,7 +295,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, return -EINVAL; } - data.size = 0; INIT_LIST_HEAD(&pages); INIT_LIST_HEAD(&pages_from_pool); @@ -326,7 +319,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, list_add_tail(&info->list, &pages_from_pool); } else { list_add_tail(&info->list, &pages); - data.size += sz; ++nents_sync; } size_remaining -= sz; @@ -334,15 +326,10 @@ static int ion_system_heap_allocate(struct ion_heap *heap, i++; } - ret = ion_heap_alloc_pages_mem(&data); - - if (ret) - goto err; - table = kzalloc(sizeof(*table), GFP_KERNEL); if (!table) { ret = -ENOMEM; - goto err_free_data_pages; + goto err; } ret = sg_alloc_table(table, i, GFP_KERNEL); @@ -355,7 +342,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, goto err_free_sg; } - i = 0; sg = table->sgl; sg_sync = table_sync.sgl; @@ -371,16 +357,16 @@ static int ion_system_heap_allocate(struct ion_heap *heap, struct page_info, list); if (info && tmp_info) { if (info->order >= tmp_info->order) { - i = process_info(info, sg, sg_sync, &data, i); + process_info(info, sg, sg_sync); sg_sync = sg_next(sg_sync); } else { - i = process_info(tmp_info, sg, 0, 0, i); + process_info(tmp_info, sg, NULL); } } else if (info) { - i = process_info(info, sg, sg_sync, &data, i); + process_info(info, sg, sg_sync); sg_sync = sg_next(sg_sync); } else if (tmp_info) { - i = process_info(tmp_info, sg, 0, 0, i); + process_info(tmp_info, sg, NULL); } sg = sg_next(sg); @@ -397,7 +383,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, buffer->sg_table = table; if (nents_sync) sg_free_table(&table_sync); - ion_heap_free_pages_mem(&data); ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; @@ -417,8 +402,6 @@ err_free_sg: sg_free_table(table); err1: kfree(table); -err_free_data_pages: - ion_heap_free_pages_mem(&data); err: list_for_each_entry_safe(info, tmp_info, &pages, list) { free_buffer_page(sys_heap, buffer, info->page, info->order);