From e2bc9cefcb494e124c38de0a552e479a6ce7bc43 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Tue, 14 Jul 2020 10:46:58 -0700 Subject: [PATCH] ion: Restore GKI system heap implementation Restore the generic system heap implementation such that it does not have any MSM specific additions as of android11-5.4 commit a9a13eeea952 ("Merge 5.4.45 into android-5.4-stable"). Change-Id: Iedc33377d5fa64513d43459d305ff98ce2091a61 Signed-off-by: Isaac J. Manjarres --- drivers/staging/android/ion/heaps/Makefile | 2 + .../staging/android/ion/heaps/ion_page_pool.c | 155 ++++++++++ .../staging/android/ion/heaps/ion_page_pool.h | 66 +++++ .../android/ion/heaps/ion_system_heap.c | 276 ++++++++++++++++++ 4 files changed, 499 insertions(+) create mode 100644 drivers/staging/android/ion/heaps/ion_page_pool.c create mode 100644 drivers/staging/android/ion/heaps/ion_page_pool.h create mode 100644 drivers/staging/android/ion/heaps/ion_system_heap.c diff --git a/drivers/staging/android/ion/heaps/Makefile b/drivers/staging/android/ion/heaps/Makefile index 931204394bc6..5cbcdaf1fb98 100644 --- a/drivers/staging/android/ion/heaps/Makefile +++ b/drivers/staging/android/ion/heaps/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_sys_heap.o +ion_sys_heap-y := ion_system_heap.o ion_page_pool.o obj-$(CONFIG_ION_MSM_HEAPS) += msm_ion_heaps.o msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_msm_page_pool.o \ ion_msm_system_heap.o ion_carveout_heap.o \ diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.c b/drivers/staging/android/ion/heaps/ion_page_pool.c new file mode 100644 index 000000000000..f1bc165e644e --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_page_pool.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator page pool helpers + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include + +#include "ion_page_pool.h" + +static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool) +{ + if (fatal_signal_pending(current)) + return NULL; + return alloc_pages(pool->gfp_mask, pool->order); +} + +static void ion_page_pool_free_pages(struct ion_page_pool *pool, + struct page *page) +{ + __free_pages(page, pool->order); +} + +static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) +{ + mutex_lock(&pool->mutex); + if (PageHighMem(page)) { + list_add_tail(&page->lru, &pool->high_items); + pool->high_count++; + } else { + list_add_tail(&page->lru, &pool->low_items); + pool->low_count++; + } + + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + 1 << pool->order); + mutex_unlock(&pool->mutex); +} + +static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) +{ + struct page *page; + + if (high) { + BUG_ON(!pool->high_count); + page = list_first_entry(&pool->high_items, struct page, lru); + pool->high_count--; + } else { + BUG_ON(!pool->low_count); + page = list_first_entry(&pool->low_items, struct page, lru); + pool->low_count--; + } + + list_del(&page->lru); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + -(1 << pool->order)); + return page; +} + +struct page *ion_page_pool_alloc(struct ion_page_pool *pool) +{ + struct page *page = NULL; + + BUG_ON(!pool); + + mutex_lock(&pool->mutex); + if (pool->high_count) + page = ion_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + + if (!page) + page = ion_page_pool_alloc_pages(pool); + + return page; +} + +void ion_page_pool_free(struct ion_page_pool *pool, struct page *page) +{ + BUG_ON(pool->order != compound_order(page)); + + ion_page_pool_add(pool, page); +} + +static int ion_page_pool_total(struct ion_page_pool *pool, bool high) +{ + int count = pool->low_count; + + if (high) + count += pool->high_count; + + return count << pool->order; +} + +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) +{ + int freed = 0; + bool high; + + if (current_is_kswapd()) + high = true; + else + high = !!(gfp_mask & __GFP_HIGHMEM); + + if (nr_to_scan == 0) + return ion_page_pool_total(pool, high); + + while (freed < nr_to_scan) { + struct page *page; + + mutex_lock(&pool->mutex); + if (pool->low_count) { + page = ion_page_pool_remove(pool, false); + } else if (high && pool->high_count) { + page = ion_page_pool_remove(pool, true); + } else { + mutex_unlock(&pool->mutex); + break; + } + mutex_unlock(&pool->mutex); + ion_page_pool_free_pages(pool, page); + freed += (1 << pool->order); + } + + return freed; +} + +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) +{ + struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return NULL; + pool->high_count = 0; + pool->low_count = 0; + INIT_LIST_HEAD(&pool->low_items); + INIT_LIST_HEAD(&pool->high_items); + pool->gfp_mask = gfp_mask | __GFP_COMP; + pool->order = order; + mutex_init(&pool->mutex); + plist_node_init(&pool->list, order); + + return pool; +} + +void ion_page_pool_destroy(struct ion_page_pool *pool) +{ + kfree(pool); +} diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.h b/drivers/staging/android/ion/heaps/ion_page_pool.h new file mode 100644 index 000000000000..e205d0086833 --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_page_pool.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ION Page Pool kernel interface header + * + * Copyright (C) 2011 Google, Inc. + */ + +#ifndef _ION_PAGE_POOL_H +#define _ION_PAGE_POOL_H + +#include +#include +#include +#include + +/** + * functions for creating and destroying a heap pool -- allows you + * to keep a pool of pre allocated memory to use from your heap. Keeping + * a pool of memory that is ready for dma, ie any cached mapping have been + * invalidated from the cache, provides a significant performance benefit on + * many systems + */ + +/** + * struct ion_page_pool - pagepool struct + * @high_count: number of highmem items in the pool + * @low_count: number of lowmem items in the pool + * @high_items: list of highmem items + * @low_items: list of lowmem items + * @mutex: lock protecting this struct and especially the count + * item list + * @gfp_mask: gfp_mask to use from alloc + * @order: order of pages in the pool + * @list: plist node for list of pools + * + * Allows you to keep a pool of pre allocated pages to use from your heap. + * Keeping a pool of pages that is ready for dma, ie any cached mapping have + * been invalidated from the cache, provides a significant performance benefit + * on many systems + */ +struct ion_page_pool { + int high_count; + int low_count; + struct list_head high_items; + struct list_head low_items; + struct mutex mutex; + gfp_t gfp_mask; + unsigned int order; + struct plist_node list; +}; + +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order); +void ion_page_pool_destroy(struct ion_page_pool *pool); +struct page *ion_page_pool_alloc(struct ion_page_pool *pool); +void ion_page_pool_free(struct ion_page_pool *pool, struct page *page); + +/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool + * @pool: the pool + * @gfp_mask: the memory type to reclaim + * @nr_to_scan: number of items to shrink in pages + * + * returns the number of items freed in pages + */ +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan); +#endif /* _ION_PAGE_POOL_H */ diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c new file mode 100644 index 000000000000..6052b843cdeb --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator system heap exporter + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ion_page_pool.h" + +#define NUM_ORDERS ARRAY_SIZE(orders) + +static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | + __GFP_NORETRY) & ~__GFP_RECLAIM; +static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO; +static const unsigned int orders[] = {8, 4, 0}; + +static int order_to_index(unsigned int order) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (order == orders[i]) + return i; + BUG(); + return -1; +} + +static inline unsigned int order_to_size(int order) +{ + return PAGE_SIZE << order; +} + +struct ion_system_heap { + struct ion_heap heap; + struct ion_page_pool *pools[NUM_ORDERS]; +}; + +static struct page *alloc_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long order) +{ + struct ion_page_pool *pool = heap->pools[order_to_index(order)]; + + return ion_page_pool_alloc(pool); +} + +static void free_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, struct page *page) +{ + struct ion_page_pool *pool; + unsigned int order = compound_order(page); + + /* go to system */ + if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) { + __free_pages(page, order); + return; + } + + pool = heap->pools[order_to_index(order)]; + + ion_page_pool_free(pool, page); +} + +static struct page *alloc_largest_available(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) +{ + struct page *page; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < order_to_size(orders[i])) + continue; + if (max_order < orders[i]) + continue; + + page = alloc_buffer_page(heap, buffer, orders[i]); + if (!page) + continue; + + return page; + } + + return NULL; +} + +static int ion_system_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned long flags) +{ + struct ion_system_heap *sys_heap = container_of(heap, + struct ion_system_heap, + heap); + struct sg_table *table; + struct scatterlist *sg; + struct list_head pages; + struct page *page, *tmp_page; + int i = 0; + unsigned long size_remaining = PAGE_ALIGN(size); + unsigned int max_order = orders[0]; + + if (size / PAGE_SIZE > totalram_pages() / 2) + return -ENOMEM; + + INIT_LIST_HEAD(&pages); + while (size_remaining > 0) { + page = alloc_largest_available(sys_heap, buffer, size_remaining, + max_order); + if (!page) + goto free_pages; + list_add_tail(&page->lru, &pages); + size_remaining -= page_size(page); + max_order = compound_order(page); + i++; + } + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + goto free_pages; + + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_table; + + sg = table->sgl; + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + list_del(&page->lru); + } + + buffer->sg_table = table; + + ion_buffer_prep_noncached(buffer); + + return 0; + +free_table: + kfree(table); +free_pages: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + free_buffer_page(sys_heap, buffer, page); + return -ENOMEM; +} + +static void ion_system_heap_free(struct ion_buffer *buffer) +{ + struct ion_system_heap *sys_heap = container_of(buffer->heap, + struct ion_system_heap, + heap); + struct sg_table *table = buffer->sg_table; + struct scatterlist *sg; + int i; + + /* zero the buffer before goto page pool */ + if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) + ion_buffer_zero(buffer); + + for_each_sg(table->sgl, sg, table->nents, i) + free_buffer_page(sys_heap, buffer, sg_page(sg)); + sg_free_table(table); + kfree(table); +} + +static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, + int nr_to_scan) +{ + struct ion_page_pool *pool; + struct ion_system_heap *sys_heap; + int nr_total = 0; + int i, nr_freed; + int only_scan = 0; + + sys_heap = container_of(heap, struct ion_system_heap, heap); + + if (!nr_to_scan) + only_scan = 1; + + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->pools[i]; + + if (only_scan) { + nr_total += ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); + + } else { + nr_freed = ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); + nr_to_scan -= nr_freed; + nr_total += nr_freed; + if (nr_to_scan <= 0) + break; + } + } + return nr_total; +} + +static void ion_system_heap_destroy_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (pools[i]) + ion_page_pool_destroy(pools[i]); +} + +static int ion_system_heap_create_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + struct ion_page_pool *pool; + gfp_t gfp_flags = low_order_gfp_flags; + + if (orders[i] > 4) + gfp_flags = high_order_gfp_flags; + + pool = ion_page_pool_create(gfp_flags, orders[i]); + if (!pool) + goto err_create_pool; + pools[i] = pool; + } + + return 0; + +err_create_pool: + ion_system_heap_destroy_pools(pools); + return -ENOMEM; +} + +static struct ion_heap_ops system_heap_ops = { + .allocate = ion_system_heap_allocate, + .free = ion_system_heap_free, + .shrink = ion_system_heap_shrink, +}; + +static struct ion_system_heap system_heap = { + .heap = { + .ops = &system_heap_ops, + .type = ION_HEAP_TYPE_SYSTEM, + .flags = ION_HEAP_FLAG_DEFER_FREE, + .name = "ion_system_heap", + } +}; + +static int __init ion_system_heap_init(void) +{ + int ret = ion_system_heap_create_pools(system_heap.pools); + if (ret) + return ret; + + return ion_device_add_heap(&system_heap.heap); +} + +static void __exit ion_system_heap_exit(void) +{ + ion_device_remove_heap(&system_heap.heap); + ion_system_heap_destroy_pools(system_heap.pools); +} + +module_init(ion_system_heap_init); +module_exit(ion_system_heap_exit); +MODULE_LICENSE("GPL v2");