From 9cae2378c0ec8855c567eb068ef6a9d6798779a4 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Thu, 25 Jun 2020 22:41:08 -0700 Subject: [PATCH 1/3] ion: Move the msm system heap source into its own files The MSM implementation of the system heap currently lives in the same source files as where the GKI system heap source lives. This is problematic, as the MSM system heap needs to co-exist with the GKI system heap. While they share a lot of code, it is not possible for them to use the same source files, as the same symbols would have to be available for GKI for the system heap, as well as for the MSM system heap. Additionally, the MSM system heap changes some of the signatures of the shared functions, which makes it even more complicated for the two heaps to share source files. Thus, in preparation to bring back the GKI system heap, move the MSM system heap to its own files. This change does not introduce any functional impact. Also, namespace the page pool interface, as it has common global symbols with the GKI ION page pool interface, which will lead to linking issues with duplicate symbols. Change-Id: I7cadb752add96d2f3a392240946c643c673711a1 Signed-off-by: Isaac J. Manjarres --- drivers/staging/android/ion/heaps/Makefile | 6 +- .../{ion_page_pool.c => ion_msm_page_pool.c} | 68 ++++---- .../{ion_page_pool.h => ion_msm_page_pool.h} | 60 +++---- ...on_system_heap.c => ion_msm_system_heap.c} | 149 +++++++++--------- ...on_system_heap.h => ion_msm_system_heap.h} | 22 +-- .../ion/heaps/ion_system_secure_heap.c | 33 ++-- .../ion/heaps/ion_system_secure_heap.h | 10 +- drivers/staging/android/ion/heaps/msm_ion.c | 2 +- .../staging/android/ion/heaps/msm_ion_priv.h | 2 +- 9 files changed, 184 insertions(+), 168 deletions(-) rename drivers/staging/android/ion/heaps/{ion_page_pool.c => ion_msm_page_pool.c} (69%) rename drivers/staging/android/ion/heaps/{ion_page_pool.h => ion_msm_page_pool.h} (61%) rename drivers/staging/android/ion/heaps/{ion_system_heap.c => ion_msm_system_heap.c} (83%) rename drivers/staging/android/ion/heaps/{ion_system_heap.h => ion_msm_system_heap.h} (64%) diff --git a/drivers/staging/android/ion/heaps/Makefile b/drivers/staging/android/ion/heaps/Makefile index d6752eab5cd6..931204394bc6 100644 --- a/drivers/staging/android/ion/heaps/Makefile +++ b/drivers/staging/android/ion/heaps/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ION_MSM_HEAPS) += msm_ion_heaps.o -msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_page_pool.o \ - ion_system_heap.o ion_carveout_heap.o ion_system_secure_heap.o \ - ion_cma_heap.o ion_secure_util.o +msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_msm_page_pool.o \ + ion_msm_system_heap.o ion_carveout_heap.o \ + ion_system_secure_heap.o ion_cma_heap.o ion_secure_util.o diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.c b/drivers/staging/android/ion/heaps/ion_msm_page_pool.c similarity index 69% rename from drivers/staging/android/ion/heaps/ion_page_pool.c rename to drivers/staging/android/ion/heaps/ion_msm_page_pool.c index cf1c0620402e..0253b5fa5fd5 100644 --- a/drivers/staging/android/ion/heaps/ion_page_pool.c +++ b/drivers/staging/android/ion/heaps/ion_msm_page_pool.c @@ -11,22 +11,24 @@ #include #include "msm_ion_priv.h" -#include "ion_page_pool.h" +#include "ion_msm_page_pool.h" -static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool) +static inline struct page +*ion_msm_page_pool_alloc_pages(struct ion_msm_page_pool *pool) { if (fatal_signal_pending(current)) return NULL; return alloc_pages(pool->gfp_mask, pool->order); } -static void ion_page_pool_free_pages(struct ion_page_pool *pool, - struct page *page) +static void ion_msm_page_pool_free_pages(struct ion_msm_page_pool *pool, + struct page *page) { __free_pages(page, pool->order); } -static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) +static void ion_msm_page_pool_add(struct ion_msm_page_pool *pool, + struct page *page) { mutex_lock(&pool->mutex); if (PageHighMem(page)) { @@ -45,7 +47,7 @@ static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) #ifdef CONFIG_ION_POOL_AUTO_REFILL /* do a simple check to see if we are in any low memory situation */ -static bool pool_refill_ok(struct ion_page_pool *pool) +static bool pool_refill_ok(struct ion_msm_page_pool *pool) { struct zonelist *zonelist; struct zoneref *z; @@ -79,7 +81,7 @@ static bool pool_refill_ok(struct ion_page_pool *pool) return true; } -void ion_page_pool_refill(struct ion_page_pool *pool) +void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool) { struct page *page; gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY; @@ -97,12 +99,13 @@ void ion_page_pool_refill(struct ion_page_pool *pool) ion_pages_sync_for_device(dev, page, PAGE_SIZE << pool->order, DMA_BIDIRECTIONAL); - ion_page_pool_add(pool, page); + ion_msm_page_pool_add(pool, page); } } #endif /* CONFIG_ION_PAGE_POOL_REFILL */ -static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) +static struct page *ion_msm_page_pool_remove(struct ion_msm_page_pool *pool, + bool high) { struct page *page; @@ -123,7 +126,8 @@ static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) return page; } -struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool) +struct page *ion_msm_page_pool_alloc(struct ion_msm_page_pool *pool, + bool *from_pool) { struct page *page = NULL; @@ -134,13 +138,13 @@ struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool) if (*from_pool && mutex_trylock(&pool->mutex)) { if (pool->high_count) - page = ion_page_pool_remove(pool, true); + page = ion_msm_page_pool_remove(pool, true); else if (pool->low_count) - page = ion_page_pool_remove(pool, false); + page = ion_msm_page_pool_remove(pool, false); mutex_unlock(&pool->mutex); } if (!page) { - page = ion_page_pool_alloc_pages(pool); + page = ion_msm_page_pool_alloc_pages(pool); *from_pool = false; } @@ -152,7 +156,7 @@ struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool) /* * Tries to allocate from only the specified Pool and returns NULL otherwise */ -struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *pool) +struct page *ion_msm_page_pool_alloc_pool_only(struct ion_msm_page_pool *pool) { struct page *page = NULL; @@ -161,9 +165,9 @@ struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *pool) if (mutex_trylock(&pool->mutex)) { if (pool->high_count) - page = ion_page_pool_remove(pool, true); + page = ion_msm_page_pool_remove(pool, true); else if (pool->low_count) - page = ion_page_pool_remove(pool, false); + page = ion_msm_page_pool_remove(pool, false); mutex_unlock(&pool->mutex); } @@ -172,17 +176,18 @@ struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *pool) return page; } -void ion_page_pool_free(struct ion_page_pool *pool, struct page *page) +void ion_msm_page_pool_free(struct ion_msm_page_pool *pool, struct page *page) { - ion_page_pool_add(pool, page); + ion_msm_page_pool_add(pool, page); } -void ion_page_pool_free_immediate(struct ion_page_pool *pool, struct page *page) +void ion_msm_page_pool_free_immediate(struct ion_msm_page_pool *pool, + struct page *page) { - ion_page_pool_free_pages(pool, page); + ion_msm_page_pool_free_pages(pool, page); } -int ion_page_pool_total(struct ion_page_pool *pool, bool high) +int ion_msm_page_pool_total(struct ion_msm_page_pool *pool, bool high) { int count = pool->low_count; @@ -192,8 +197,8 @@ int ion_page_pool_total(struct ion_page_pool *pool, bool high) return count << pool->order; } -int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, - int nr_to_scan) +int ion_msm_page_pool_shrink(struct ion_msm_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) { int freed = 0; bool high; @@ -204,32 +209,33 @@ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, high = !!(gfp_mask & __GFP_HIGHMEM); if (nr_to_scan == 0) - return ion_page_pool_total(pool, high); + return ion_msm_page_pool_total(pool, high); while (freed < nr_to_scan) { struct page *page; mutex_lock(&pool->mutex); if (pool->low_count) { - page = ion_page_pool_remove(pool, false); + page = ion_msm_page_pool_remove(pool, false); } else if (high && pool->high_count) { - page = ion_page_pool_remove(pool, true); + page = ion_msm_page_pool_remove(pool, true); } else { mutex_unlock(&pool->mutex); break; } mutex_unlock(&pool->mutex); - ion_page_pool_free_pages(pool, page); + ion_msm_page_pool_free_pages(pool, page); freed += (1 << pool->order); } return freed; } -struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order, - bool cached) +struct ion_msm_page_pool *ion_msm_page_pool_create(gfp_t gfp_mask, + unsigned int order, + bool cached) { - struct ion_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); + struct ion_msm_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); if (!pool) return NULL; @@ -245,7 +251,7 @@ struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order, return pool; } -void ion_page_pool_destroy(struct ion_page_pool *pool) +void ion_msm_page_pool_destroy(struct ion_msm_page_pool *pool) { kfree(pool); } diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.h b/drivers/staging/android/ion/heaps/ion_msm_page_pool.h similarity index 61% rename from drivers/staging/android/ion/heaps/ion_page_pool.h rename to drivers/staging/android/ion/heaps/ion_msm_page_pool.h index dfe3f8b59426..88b0d95a099a 100644 --- a/drivers/staging/android/ion/heaps/ion_page_pool.h +++ b/drivers/staging/android/ion/heaps/ion_msm_page_pool.h @@ -5,8 +5,8 @@ * Copyright (C) 2011 Google, Inc. */ -#ifndef _ION_PAGE_POOL_H -#define _ION_PAGE_POOL_H +#ifndef _ION_MSM_PAGE_POOL_H +#define _ION_MSM_PAGE_POOL_H #include #include @@ -35,7 +35,7 @@ */ /** - * struct ion_page_pool - pagepool struct + * struct ion_msm_page_pool - pagepool struct * @high_count: number of highmem items in the pool * @low_count: number of lowmem items in the pool * @count: total number of pages/items in the pool @@ -56,7 +56,7 @@ * been invalidated from the cache, provides a significant performance benefit * on many systems */ -struct ion_page_pool { +struct ion_msm_page_pool { int high_count; int low_count; atomic_t count; @@ -71,72 +71,78 @@ struct ion_page_pool { struct device *heap_dev; }; -struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order, - bool cached); -void ion_page_pool_destroy(struct ion_page_pool *pool); -struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool); -void ion_page_pool_free(struct ion_page_pool *pool, struct page *page); -struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *a); -void ion_page_pool_free_immediate(struct ion_page_pool *pool, - struct page *page); -int ion_page_pool_total(struct ion_page_pool *pool, bool high); +struct ion_msm_page_pool *ion_msm_page_pool_create(gfp_t gfp_mask, + unsigned int order, + bool cached); +void ion_msm_page_pool_destroy(struct ion_msm_page_pool *pool); +struct page *ion_msm_page_pool_alloc(struct ion_msm_page_pool *pool, + bool *from_pool); +void ion_msm_page_pool_free(struct ion_msm_page_pool *pool, struct page *page); +struct page *ion_msm_page_pool_alloc_pool_only(struct ion_msm_page_pool *a); +void ion_msm_page_pool_free_immediate(struct ion_msm_page_pool *pool, + struct page *page); +int ion_msm_page_pool_total(struct ion_msm_page_pool *pool, bool high); size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid); -/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool +/** ion_msm_page_pool_shrink - shrinks the size of the memory cached in the pool * @pool: the pool * @gfp_mask: the memory type to reclaim * @nr_to_scan: number of items to shrink in pages * * returns the number of items freed in pages */ -int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, - int nr_to_scan); +int ion_msm_page_pool_shrink(struct ion_msm_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan); #ifdef CONFIG_ION_POOL_AUTO_REFILL -void ion_page_pool_refill(struct ion_page_pool *pool); +void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool); -static __always_inline int get_pool_fillmark(struct ion_page_pool *pool) +static __always_inline int get_pool_fillmark(struct ion_msm_page_pool *pool) { return ION_POOL_FILL_MARK / (PAGE_SIZE << pool->order); } -static __always_inline int get_pool_lowmark(struct ion_page_pool *pool) +static __always_inline int get_pool_lowmark(struct ion_msm_page_pool *pool) { return ION_POOL_LOW_MARK / (PAGE_SIZE << pool->order); } -static __always_inline bool pool_count_below_lowmark(struct ion_page_pool *pool) +static __always_inline bool +pool_count_below_lowmark(struct ion_msm_page_pool *pool) { return atomic_read(&pool->count) < get_pool_lowmark(pool); } -static __always_inline bool pool_fillmark_reached(struct ion_page_pool *pool) +static __always_inline bool +pool_fillmark_reached(struct ion_msm_page_pool *pool) { return atomic_read(&pool->count) >= get_pool_fillmark(pool); } #else -static inline void ion_page_pool_refill(struct ion_page_pool *pool) +static inline void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool) { } -static __always_inline int get_pool_fillmark(struct ion_page_pool *pool) +static __always_inline int get_pool_fillmark(struct ion_msm_page_pool *pool) { return 0; } -static __always_inline int get_pool_lowmark(struct ion_page_pool *pool) +static __always_inline int get_pool_lowmark(struct ion_msm_page_pool *pool) { return 0; } -static __always_inline bool pool_count_below_lowmark(struct ion_page_pool *pool) +static __always_inline bool +pool_count_below_lowmark(struct ion_msm_page_pool *pool) { return false; } -static __always_inline bool pool_fillmark_reached(struct ion_page_pool *pool) +static __always_inline bool +pool_fillmark_reached(struct ion_msm_page_pool *pool) { return false; } #endif /* CONFIG_ION_POOL_AUTO_REFILL */ -#endif /* _ION_PAGE_POOL_H */ +#endif /* _ION_MSM_PAGE_POOL_H */ diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_msm_system_heap.c similarity index 83% rename from drivers/staging/android/ion/heaps/ion_system_heap.c rename to drivers/staging/android/ion/heaps/ion_msm_system_heap.c index 5c4d7833dcbc..1d78ce811d95 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_msm_system_heap.c @@ -21,10 +21,9 @@ #include #include #include -#include "ion_system_heap.h" -#include "ion_page_pool.h" +#include "ion_msm_system_heap.h" +#include "ion_msm_page_pool.h" #include "msm_ion_priv.h" -#include "ion_system_heap.h" #include "ion_system_secure_heap.h" #include "ion_secure_util.h" @@ -32,8 +31,8 @@ static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_RECLAIM; static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO; -bool pool_auto_refill_en __read_mostly = - IS_ENABLED(CONFIG_ION_POOL_AUTO_REFILL); +static bool pool_auto_refill_en __read_mostly = +IS_ENABLED(CONFIG_ION_POOL_AUTO_REFILL); int order_to_index(unsigned int order) { @@ -51,19 +50,19 @@ static inline unsigned int order_to_size(int order) return PAGE_SIZE << order; } -int ion_heap_is_system_heap_type(enum ion_heap_type type) +static int ion_heap_is_system_heap_type(enum ion_heap_type type) { return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM); } -static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, +static struct page *alloc_buffer_page(struct ion_msm_system_heap *sys_heap, struct ion_buffer *buffer, unsigned long order, bool *from_pool) { int cached = (int)ion_buffer_cached(buffer); struct page *page; - struct ion_page_pool *pool; + struct ion_msm_page_pool *pool; int vmid = get_secure_vmid(buffer->flags); struct device *dev = sys_heap->heap.dev; int order_ind = order_to_index(order); @@ -81,12 +80,12 @@ static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, if (!(*from_pool && pool_auto_refill_en)) goto normal_alloc; - page = ion_page_pool_alloc_pool_only(pool); + page = ion_msm_page_pool_alloc_pool_only(pool); if (!IS_ERR(page)) return page; pool = sys_heap->uncached_pools[order_ind]; - page = ion_page_pool_alloc_pool_only(pool); + page = ion_msm_page_pool_alloc_pool_only(pool); if (IS_ERR(page)) { pool = sys_heap->secure_pools[vmid][order_ind]; goto normal_alloc; @@ -112,7 +111,7 @@ static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, } normal_alloc: - page = ion_page_pool_alloc(pool, from_pool); + page = ion_msm_page_pool_alloc(pool, from_pool); if (pool_auto_refill_en && pool->order && pool_count_below_lowmark(pool) && vmid <= 0) @@ -132,7 +131,7 @@ normal_alloc: * For secure pages that need to be freed and not added back to the pool; the * hyp_unassign should be called before calling this function */ -void free_buffer_page(struct ion_system_heap *heap, +void free_buffer_page(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, struct page *page, unsigned int order) { @@ -140,7 +139,7 @@ void free_buffer_page(struct ion_system_heap *heap, int vmid = get_secure_vmid(buffer->flags); if (!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { - struct ion_page_pool *pool; + struct ion_msm_page_pool *pool; if (vmid > 0) pool = heap->secure_pools[vmid][order_to_index(order)]; @@ -150,9 +149,9 @@ void free_buffer_page(struct ion_system_heap *heap, pool = heap->uncached_pools[order_to_index(order)]; if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) - ion_page_pool_free_immediate(pool, page); + ion_msm_page_pool_free_immediate(pool, page); else - ion_page_pool_free(pool, page); + ion_msm_page_pool_free(pool, page); #ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES, @@ -169,10 +168,11 @@ void free_buffer_page(struct ion_system_heap *heap, } } -static struct page_info *alloc_largest_available(struct ion_system_heap *heap, - struct ion_buffer *buffer, - unsigned long size, - unsigned int max_order) +static struct +page_info *alloc_largest_available(struct ion_msm_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) { struct page *page; struct page_info *info; @@ -205,7 +205,7 @@ static struct page_info *alloc_largest_available(struct ion_system_heap *heap, } static struct page_info * -alloc_from_pool_preferred(struct ion_system_heap *heap, +alloc_from_pool_preferred(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, unsigned long size, unsigned int max_order) @@ -274,12 +274,12 @@ static void process_info(struct page_info *info, kfree(info); } -static int ion_system_heap_allocate(struct ion_heap *heap, - struct ion_buffer *buffer, - unsigned long size, - unsigned long flags) +static int ion_msm_system_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned long flags) { - struct ion_system_heap *sys_heap = to_system_heap(heap); + struct ion_msm_system_heap *sys_heap = to_msm_system_heap(heap); struct msm_ion_buf_lock_state *lock_state; struct sg_table *table; struct sg_table table_sync = {0}; @@ -441,10 +441,10 @@ err: return ret; } -static void ion_system_heap_free(struct ion_buffer *buffer) +static void ion_msm_system_heap_free(struct ion_buffer *buffer) { struct ion_heap *heap = buffer->heap; - struct ion_system_heap *sys_heap = to_system_heap(heap); + struct ion_msm_system_heap *sys_heap = to_msm_system_heap(heap); struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; struct sg_table *table = buffer->sg_table; struct scatterlist *sg; @@ -474,16 +474,16 @@ static void ion_system_heap_free(struct ion_buffer *buffer) kfree(buffer->priv_virt); } -static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, - int nr_to_scan) +static int ion_msm_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, + int nr_to_scan) { - struct ion_system_heap *sys_heap; + struct ion_msm_system_heap *sys_heap; int nr_total = 0; int i, j, nr_freed = 0; int only_scan = 0; - struct ion_page_pool *pool; + struct ion_msm_page_pool *pool; - sys_heap = to_system_heap(heap); + sys_heap = to_msm_system_heap(heap); if (!nr_to_scan) only_scan = 1; @@ -501,10 +501,12 @@ static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, } pool = sys_heap->uncached_pools[i]; - nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan); + nr_freed += + ion_msm_page_pool_shrink(pool, gfp_mask, nr_to_scan); pool = sys_heap->cached_pools[i]; - nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan); + nr_freed += + ion_msm_page_pool_shrink(pool, gfp_mask, nr_to_scan); nr_total += nr_freed; if (!only_scan) { @@ -519,23 +521,23 @@ static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, } static struct ion_heap_ops system_heap_ops = { - .allocate = ion_system_heap_allocate, - .free = ion_system_heap_free, - .shrink = ion_system_heap_shrink, + .allocate = ion_msm_system_heap_allocate, + .free = ion_msm_system_heap_free, + .shrink = ion_msm_system_heap_shrink, }; -static int ion_system_heap_debug_show(struct ion_heap *heap, - struct seq_file *s, void *unused) +static int ion_msm_system_heap_debug_show(struct ion_heap *heap, + struct seq_file *s, void *unused) { - struct ion_system_heap *sys_heap; + struct ion_msm_system_heap *sys_heap; bool use_seq = s; unsigned long uncached_total = 0; unsigned long cached_total = 0; unsigned long secure_total = 0; - struct ion_page_pool *pool; + struct ion_msm_page_pool *pool; int i, j; - sys_heap = to_system_heap(heap); + sys_heap = to_msm_system_heap(heap); for (i = 0; i < NUM_ORDERS; i++) { pool = sys_heap->uncached_pools[i]; if (use_seq) { @@ -624,40 +626,43 @@ static int ion_system_heap_debug_show(struct ion_heap *heap, } static struct msm_ion_heap_ops msm_system_heap_ops = { - .debug_show = ion_system_heap_debug_show, + .debug_show = ion_msm_system_heap_debug_show, }; -static void ion_system_heap_destroy_pools(struct ion_page_pool **pools) +static void ion_msm_system_heap_destroy_pools(struct ion_msm_page_pool **pools) { int i; + if (!pools) + return; + for (i = 0; i < NUM_ORDERS; i++) if (pools[i]) { - ion_page_pool_destroy(pools[i]); + ion_msm_page_pool_destroy(pools[i]); pools[i] = NULL; } } /** - * ion_system_heap_create_pools - Creates pools for all orders + * ion_msm_system_heap_create_pools - Creates pools for all orders * * If this fails you don't need to destroy any pools. It's all or * nothing. If it succeeds you'll eventually need to use - * ion_system_heap_destroy_pools to destroy the pools. + * ion_msm_system_heap_destroy_pools to destroy the pools. */ -static int ion_system_heap_create_pools(struct ion_system_heap *sys_heap, - struct ion_page_pool **pools, - bool cached) +static int +ion_msm_system_heap_create_pools(struct ion_msm_system_heap *sys_heap, + struct ion_msm_page_pool **pools, bool cached) { int i; for (i = 0; i < NUM_ORDERS; i++) { - struct ion_page_pool *pool; + struct ion_msm_page_pool *pool; gfp_t gfp_flags = low_order_gfp_flags; if (orders[i]) gfp_flags = high_order_gfp_flags; - pool = ion_page_pool_create(gfp_flags, orders[i], cached); + pool = ion_msm_page_pool_create(gfp_flags, orders[i], cached); if (!pool) goto err_create_pool; pool->heap_dev = sys_heap->heap.dev; @@ -666,19 +671,19 @@ static int ion_system_heap_create_pools(struct ion_system_heap *sys_heap, return 0; err_create_pool: - ion_system_heap_destroy_pools(pools); + ion_msm_system_heap_destroy_pools(pools); return -ENOMEM; } -static int ion_sys_heap_worker(void *data) +static int ion_msm_sys_heap_worker(void *data) { - struct ion_page_pool **pools = (struct ion_page_pool **)data; + struct ion_msm_page_pool **pools = (struct ion_msm_page_pool **)data; int i; for (;;) { for (i = 0; i < NUM_ORDERS; i++) { if (pool_count_below_lowmark(pools[i])) - ion_page_pool_refill(pools[i]); + ion_msm_page_pool_refill(pools[i]); } set_current_state(TASK_INTERRUPTIBLE); if (unlikely(kthread_should_stop())) { @@ -693,7 +698,7 @@ static int ion_sys_heap_worker(void *data) return 0; } -static struct task_struct *ion_create_kworker(struct ion_page_pool **pools, +static struct task_struct *ion_create_kworker(struct ion_msm_page_pool **pools, bool cached) { struct sched_attr attr = { 0 }; @@ -704,7 +709,7 @@ static struct task_struct *ion_create_kworker(struct ion_page_pool **pools, attr.sched_nice = ION_KTHREAD_NICE_VAL; buf = cached ? "cached" : "uncached"; - thread = kthread_run(ion_sys_heap_worker, pools, + thread = kthread_run(ion_msm_sys_heap_worker, pools, "ion-pool-%s-worker", buf); if (IS_ERR(thread)) { pr_err("%s: failed to create %s worker thread: %ld\n", @@ -722,9 +727,9 @@ static struct task_struct *ion_create_kworker(struct ion_page_pool **pools, return thread; } -struct ion_heap *ion_system_heap_create(struct ion_platform_heap *data) +struct ion_heap *ion_msm_system_heap_create(struct ion_platform_heap *data) { - struct ion_system_heap *heap; + struct ion_msm_system_heap *heap; int ret = -ENOMEM; int i; @@ -739,16 +744,16 @@ struct ion_heap *ion_system_heap_create(struct ion_platform_heap *data) heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE; for (i = 0; i < VMID_LAST; i++) - if (is_secure_vmid_valid(i)) - if (ion_system_heap_create_pools(heap, - heap->secure_pools[i], - false)) - goto destroy_secure_pools; + if (is_secure_vmid_valid(i) && + ion_msm_system_heap_create_pools(heap, + heap->secure_pools[i], + false)) + goto destroy_secure_pools; - if (ion_system_heap_create_pools(heap, heap->uncached_pools, false)) + if (ion_msm_system_heap_create_pools(heap, heap->uncached_pools, false)) goto destroy_secure_pools; - if (ion_system_heap_create_pools(heap, heap->cached_pools, true)) + if (ion_msm_system_heap_create_pools(heap, heap->cached_pools, true)) goto destroy_uncached_pools; if (pool_auto_refill_en) { @@ -771,14 +776,12 @@ struct ion_heap *ion_system_heap_create(struct ion_platform_heap *data) return &heap->heap.ion_heap; destroy_pools: - ion_system_heap_destroy_pools(heap->cached_pools); + ion_msm_system_heap_destroy_pools(heap->cached_pools); destroy_uncached_pools: - ion_system_heap_destroy_pools(heap->uncached_pools); + ion_msm_system_heap_destroy_pools(heap->uncached_pools); destroy_secure_pools: - for (i = 0; i < VMID_LAST; i++) { - if (heap->secure_pools[i]) - ion_system_heap_destroy_pools(heap->secure_pools[i]); - } + for (i = 0; i < VMID_LAST; i++) + ion_msm_system_heap_destroy_pools(heap->secure_pools[i]); kfree(heap); return ERR_PTR(ret); } diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.h b/drivers/staging/android/ion/heaps/ion_msm_system_heap.h similarity index 64% rename from drivers/staging/android/ion/heaps/ion_system_heap.h rename to drivers/staging/android/ion/heaps/ion_msm_system_heap.h index ee95a6a108c0..4e1986ae8916 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.h +++ b/drivers/staging/android/ion/heaps/ion_msm_system_heap.h @@ -1,12 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. */ #include #include "msm_ion_priv.h" -#ifndef _ION_SYSTEM_HEAP_H -#define _ION_SYSTEM_HEAP_H +#ifndef _ION_MSM_SYSTEM_HEAP_H +#define _ION_MSM_SYSTEM_HEAP_H #ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS #if defined(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) @@ -22,8 +22,8 @@ static const unsigned int orders[] = {0}; #define ION_KTHREAD_NICE_VAL 10 -#define to_system_heap(_heap) \ - container_of(to_msm_ion_heap(_heap), struct ion_system_heap, heap) +#define to_msm_system_heap(_heap) \ + container_of(to_msm_ion_heap(_heap), struct ion_msm_system_heap, heap) enum ion_kthread_type { ION_KTHREAD_UNCACHED, @@ -31,13 +31,13 @@ enum ion_kthread_type { ION_MAX_NUM_KTHREADS }; -struct ion_system_heap { +struct ion_msm_system_heap { struct msm_ion_heap heap; - struct ion_page_pool *uncached_pools[MAX_ORDER]; - struct ion_page_pool *cached_pools[MAX_ORDER]; + struct ion_msm_page_pool *uncached_pools[MAX_ORDER]; + struct ion_msm_page_pool *cached_pools[MAX_ORDER]; /* worker threads to refill the pool */ struct task_struct *kworker[ION_MAX_NUM_KTHREADS]; - struct ion_page_pool *secure_pools[VMID_LAST][MAX_ORDER]; + struct ion_msm_page_pool *secure_pools[VMID_LAST][MAX_ORDER]; /* Prevents unnecessary page splitting */ struct mutex split_page_mutex; }; @@ -51,8 +51,8 @@ struct page_info { int order_to_index(unsigned int order); -void free_buffer_page(struct ion_system_heap *heap, +void free_buffer_page(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, struct page *page, unsigned int order); -#endif /* _ION_SYSTEM_HEAP_H */ +#endif /* _ION_MSM_SYSTEM_HEAP_H */ diff --git a/drivers/staging/android/ion/heaps/ion_system_secure_heap.c b/drivers/staging/android/ion/heaps/ion_system_secure_heap.c index 7b252d7ad169..470bc53a2bb3 100644 --- a/drivers/staging/android/ion/heaps/ion_system_secure_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_secure_heap.c @@ -10,8 +10,8 @@ #include #include "ion_system_secure_heap.h" -#include "ion_system_heap.h" -#include "ion_page_pool.h" +#include "ion_msm_system_heap.h" +#include "ion_msm_page_pool.h" #include "msm_ion_priv.h" #include "ion_secure_util.h" @@ -146,19 +146,19 @@ out1: size_t ion_system_secure_heap_page_pool_total(struct ion_heap *heap, int vmid_flags) { - struct ion_system_heap *sys_heap; - struct ion_page_pool *pool; + struct ion_msm_system_heap *sys_heap; + struct ion_msm_page_pool *pool; size_t total = 0; int vmid, i; - sys_heap = to_system_heap(heap); + sys_heap = to_msm_system_heap(heap); vmid = get_secure_vmid(vmid_flags); if (vmid < 0) return 0; for (i = 0; i < NUM_ORDERS; i++) { pool = sys_heap->secure_pools[vmid][i]; - total += ion_page_pool_total(pool, true); + total += ion_msm_page_pool_total(pool, true); } return total << PAGE_SHIFT; @@ -342,21 +342,21 @@ struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *unused) return &heap->heap.ion_heap; } -struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap, +struct page *alloc_from_secure_pool_order(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, unsigned long order) { int vmid = get_secure_vmid(buffer->flags); - struct ion_page_pool *pool; + struct ion_msm_page_pool *pool; if (!is_secure_vmid_valid(vmid)) return ERR_PTR(-EINVAL); pool = heap->secure_pools[vmid][order_to_index(order)]; - return ion_page_pool_alloc_pool_only(pool); + return ion_msm_page_pool_alloc_pool_only(pool); } -struct page *split_page_from_secure_pool(struct ion_system_heap *heap, +struct page *split_page_from_secure_pool(struct ion_msm_system_heap *heap, struct ion_buffer *buffer) { int i, j; @@ -400,7 +400,7 @@ got_page: return page; } -int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, +int ion_secure_page_pool_shrink(struct ion_msm_system_heap *sys_heap, int vmid, int order_idx, int nr_to_scan) { int ret, freed = 0; @@ -408,14 +408,15 @@ int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, struct page *page, *tmp; struct sg_table sgt; struct scatterlist *sg; - struct ion_page_pool *pool = sys_heap->secure_pools[vmid][order_idx]; + struct ion_msm_page_pool *pool = + sys_heap->secure_pools[vmid][order_idx]; LIST_HEAD(pages); if (nr_to_scan == 0) - return ion_page_pool_total(pool, true); + return ion_msm_page_pool_total(pool, true); while (freed < nr_to_scan) { - page = ion_page_pool_alloc_pool_only(pool); + page = ion_msm_page_pool_alloc_pool_only(pool); if (IS_ERR(page)) break; list_add(&page->lru, &pages); @@ -443,7 +444,7 @@ int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, list_for_each_entry_safe(page, tmp, &pages, lru) { list_del(&page->lru); - ion_page_pool_free_immediate(pool, page); + ion_msm_page_pool_free_immediate(pool, page); } sg_free_table(&sgt); @@ -455,7 +456,7 @@ out1: /* Restore pages to secure pool */ list_for_each_entry_safe(page, tmp, &pages, lru) { list_del(&page->lru); - ion_page_pool_free(pool, page); + ion_msm_page_pool_free(pool, page); } return 0; out3: diff --git a/drivers/staging/android/ion/heaps/ion_system_secure_heap.h b/drivers/staging/android/ion/heaps/ion_system_secure_heap.h index fdeee71ac5fc..9e808b397fde 100644 --- a/drivers/staging/android/ion/heaps/ion_system_secure_heap.h +++ b/drivers/staging/android/ion/heaps/ion_system_secure_heap.h @@ -1,21 +1,21 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. */ #include "msm_ion_priv.h" -#include "ion_system_heap.h" +#include "ion_msm_system_heap.h" #ifndef _ION_SYSTEM_SECURE_HEAP_H #define _ION_SYSTEM_SECURE_HEAP_H -struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap, +struct page *alloc_from_secure_pool_order(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, unsigned long order); -struct page *split_page_from_secure_pool(struct ion_system_heap *heap, +struct page *split_page_from_secure_pool(struct ion_msm_system_heap *heap, struct ion_buffer *buffer); -int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, +int ion_secure_page_pool_shrink(struct ion_msm_system_heap *sys_heap, int vmid, int order_idx, int nr_to_scan); #endif /* _ION_SYSTEM_SECURE_HEAP_H */ diff --git a/drivers/staging/android/ion/heaps/msm_ion.c b/drivers/staging/android/ion/heaps/msm_ion.c index 78f08907f378..6a4cf23e1760 100644 --- a/drivers/staging/android/ion/heaps/msm_ion.c +++ b/drivers/staging/android/ion/heaps/msm_ion.c @@ -159,7 +159,7 @@ static struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) switch (heap_type) { case ION_HEAP_TYPE_SYSTEM: - heap = ion_system_heap_create(heap_data); + heap = ion_msm_system_heap_create(heap_data); break; case ION_HEAP_TYPE_MSM_CARVEOUT: heap = ion_carveout_heap_create(heap_data); diff --git a/drivers/staging/android/ion/heaps/msm_ion_priv.h b/drivers/staging/android/ion/heaps/msm_ion_priv.h index cf1231eb3478..66ea173e853c 100644 --- a/drivers/staging/android/ion/heaps/msm_ion_priv.h +++ b/drivers/staging/android/ion/heaps/msm_ion_priv.h @@ -156,7 +156,7 @@ bool ion_buffer_cached(struct ion_buffer *buffer); * heaps as appropriate. */ -struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused); +struct ion_heap *ion_msm_system_heap_create(struct ion_platform_heap *unused); struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *heap); From 149000cb212d0d6f6c1caccf81415e08b65a2c90 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Thu, 25 Jun 2020 22:55:52 -0700 Subject: [PATCH 2/3] ion: Update the system heap type to MSM_SYSTEM and update sys heap ID In preparation for restoring the GKI system heap, move the existing system heap into the vendor heap space, leaving the GKI system heap ID slot empty. For now, alias allocations targeted at ION_HEAP_SYSTEM to ION_SYSTEM_HEAP_ID until all clients have migrated. Once that happens, we can remove the aliasing. Change-Id: Ib1bd74c2a54ebfdcb14da2b04f237ae1b88db189 Signed-off-by: Isaac J. Manjarres --- .../android/ion/heaps/ion_msm_system_heap.c | 8 ++++---- drivers/staging/android/ion/heaps/msm_ion.c | 4 ++-- drivers/staging/android/ion/ion_buffer.c | 15 +++++++++++++++ include/uapi/linux/msm_ion.h | 1 + include/uapi/linux/msm_ion_ids.h | 3 ++- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/drivers/staging/android/ion/heaps/ion_msm_system_heap.c b/drivers/staging/android/ion/heaps/ion_msm_system_heap.c index 1d78ce811d95..f57918149716 100644 --- a/drivers/staging/android/ion/heaps/ion_msm_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_msm_system_heap.c @@ -50,9 +50,9 @@ static inline unsigned int order_to_size(int order) return PAGE_SIZE << order; } -static int ion_heap_is_system_heap_type(enum ion_heap_type type) +static int ion_heap_is_msm_system_heap_type(enum ion_heap_type type) { - return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM); + return type == ((enum ion_heap_type)ION_HEAP_TYPE_MSM_SYSTEM); } static struct page *alloc_buffer_page(struct ion_msm_system_heap *sys_heap, @@ -299,7 +299,7 @@ static int ion_msm_system_heap_allocate(struct ion_heap *heap, if (size / PAGE_SIZE > totalram_pages() / 2) return -ENOMEM; - if (ion_heap_is_system_heap_type(buffer->heap->type) && + if (ion_heap_is_msm_system_heap_type(buffer->heap->type) && is_secure_allocation(buffer->flags)) { pr_info("%s: System heap doesn't support secure allocations\n", __func__); @@ -740,7 +740,7 @@ struct ion_heap *ion_msm_system_heap_create(struct ion_platform_heap *data) heap->heap.msm_heap_ops = &msm_system_heap_ops; heap->heap.ion_heap.ops = &system_heap_ops; heap->heap.ion_heap.buf_ops = msm_ion_dma_buf_ops; - heap->heap.ion_heap.type = ION_HEAP_TYPE_SYSTEM; + heap->heap.ion_heap.type = (enum ion_heap_type)ION_HEAP_TYPE_MSM_SYSTEM; heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE; for (i = 0; i < VMID_LAST; i++) diff --git a/drivers/staging/android/ion/heaps/msm_ion.c b/drivers/staging/android/ion/heaps/msm_ion.c index 6a4cf23e1760..2bed708ab61c 100644 --- a/drivers/staging/android/ion/heaps/msm_ion.c +++ b/drivers/staging/android/ion/heaps/msm_ion.c @@ -100,7 +100,7 @@ static struct heap_types_info { const char *name; int heap_type; } heap_types_info[] = { - MAKE_HEAP_TYPE_MAPPING(SYSTEM), + MAKE_HEAP_TYPE_MAPPING(MSM_SYSTEM), MAKE_HEAP_TYPE_MAPPING(MSM_CARVEOUT), MAKE_HEAP_TYPE_MAPPING(SECURE_CARVEOUT), MAKE_HEAP_TYPE_MAPPING(DMA), @@ -158,7 +158,7 @@ static struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) int heap_type = heap_data->type; switch (heap_type) { - case ION_HEAP_TYPE_SYSTEM: + case ION_HEAP_TYPE_MSM_SYSTEM: heap = ion_msm_system_heap_create(heap_data); break; case ION_HEAP_TYPE_MSM_CARVEOUT: diff --git a/drivers/staging/android/ion/ion_buffer.c b/drivers/staging/android/ion/ion_buffer.c index 12f180a84ea8..862da1a32551 100644 --- a/drivers/staging/android/ion/ion_buffer.c +++ b/drivers/staging/android/ion/ion_buffer.c @@ -12,6 +12,7 @@ #include #include "ion_private.h" +#define ION_SYSTEM_HEAP_ID BIT(25) /* this function should only be called while dev->lock is held */ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, @@ -115,11 +116,25 @@ struct ion_buffer *ion_buffer_alloc(struct ion_device *dev, size_t len, { struct ion_buffer *buffer = NULL; struct ion_heap *heap; + char task_comm[TASK_COMM_LEN]; if (!dev || !len) { return ERR_PTR(-EINVAL); } + /* + * Temporarily reroute generic system heap allocations to the MSM system + * heap. Once clients have stopped using the generic system heap ID, we + * can remove this. + */ + if (heap_id_mask & ION_HEAP_SYSTEM) { + get_task_comm(task_comm, current->group_leader); + pr_warn_ratelimited("%s: Rerouting allocation from generic sys heap to msm sys heap for %s-%d\n", + __func__, task_comm, current->tgid); + heap_id_mask &= ~ION_HEAP_SYSTEM; + heap_id_mask |= ION_SYSTEM_HEAP_ID; + } + /* * traverse the list of heaps available in this system in priority * order. If the heap type is supported by the client, and matches the diff --git a/include/uapi/linux/msm_ion.h b/include/uapi/linux/msm_ion.h index 46eef92343f0..e68fc5540567 100644 --- a/include/uapi/linux/msm_ion.h +++ b/include/uapi/linux/msm_ion.h @@ -22,6 +22,7 @@ enum msm_ion_heap_types { ION_HEAP_TYPE_HYP_CMA, ION_HEAP_TYPE_MSM_CARVEOUT, ION_HEAP_TYPE_SECURE_CARVEOUT, + ION_HEAP_TYPE_MSM_SYSTEM, }; /** diff --git a/include/uapi/linux/msm_ion_ids.h b/include/uapi/linux/msm_ion_ids.h index 9a36cae13063..857661963724 100644 --- a/include/uapi/linux/msm_ion_ids.h +++ b/include/uapi/linux/msm_ion_ids.h @@ -16,7 +16,7 @@ * possible fallbacks) */ -#define ION_SYSTEM_HEAP_ID ION_BIT(0) +/* ION_BIT(0) is reserved for the generic system heap. */ #define ION_QSECOM_TA_HEAP_ID ION_BIT(1) #define ION_CAMERA_HEAP_ID ION_BIT(2) #define ION_DISPLAY_HEAP_ID ION_BIT(3) @@ -31,6 +31,7 @@ #define ION_SPSS_HEAP_ID ION_BIT(14) #define ION_SECURE_CARVEOUT_HEAP_ID ION_BIT(15) #define ION_TUI_CARVEOUT_HEAP_ID ION_BIT(16) +#define ION_SYSTEM_HEAP_ID ION_BIT(25) #define ION_HEAP_ID_RESERVED ION_BIT(31) #endif /* _MSM_ION_IDS_H */ From e2bc9cefcb494e124c38de0a552e479a6ce7bc43 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Tue, 14 Jul 2020 10:46:58 -0700 Subject: [PATCH 3/3] ion: Restore GKI system heap implementation Restore the generic system heap implementation such that it does not have any MSM specific additions as of android11-5.4 commit a9a13eeea952 ("Merge 5.4.45 into android-5.4-stable"). Change-Id: Iedc33377d5fa64513d43459d305ff98ce2091a61 Signed-off-by: Isaac J. Manjarres --- drivers/staging/android/ion/heaps/Makefile | 2 + .../staging/android/ion/heaps/ion_page_pool.c | 155 ++++++++++ .../staging/android/ion/heaps/ion_page_pool.h | 66 +++++ .../android/ion/heaps/ion_system_heap.c | 276 ++++++++++++++++++ 4 files changed, 499 insertions(+) create mode 100644 drivers/staging/android/ion/heaps/ion_page_pool.c create mode 100644 drivers/staging/android/ion/heaps/ion_page_pool.h create mode 100644 drivers/staging/android/ion/heaps/ion_system_heap.c diff --git a/drivers/staging/android/ion/heaps/Makefile b/drivers/staging/android/ion/heaps/Makefile index 931204394bc6..5cbcdaf1fb98 100644 --- a/drivers/staging/android/ion/heaps/Makefile +++ b/drivers/staging/android/ion/heaps/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_sys_heap.o +ion_sys_heap-y := ion_system_heap.o ion_page_pool.o obj-$(CONFIG_ION_MSM_HEAPS) += msm_ion_heaps.o msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_msm_page_pool.o \ ion_msm_system_heap.o ion_carveout_heap.o \ diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.c b/drivers/staging/android/ion/heaps/ion_page_pool.c new file mode 100644 index 000000000000..f1bc165e644e --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_page_pool.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator page pool helpers + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include + +#include "ion_page_pool.h" + +static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool) +{ + if (fatal_signal_pending(current)) + return NULL; + return alloc_pages(pool->gfp_mask, pool->order); +} + +static void ion_page_pool_free_pages(struct ion_page_pool *pool, + struct page *page) +{ + __free_pages(page, pool->order); +} + +static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) +{ + mutex_lock(&pool->mutex); + if (PageHighMem(page)) { + list_add_tail(&page->lru, &pool->high_items); + pool->high_count++; + } else { + list_add_tail(&page->lru, &pool->low_items); + pool->low_count++; + } + + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + 1 << pool->order); + mutex_unlock(&pool->mutex); +} + +static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) +{ + struct page *page; + + if (high) { + BUG_ON(!pool->high_count); + page = list_first_entry(&pool->high_items, struct page, lru); + pool->high_count--; + } else { + BUG_ON(!pool->low_count); + page = list_first_entry(&pool->low_items, struct page, lru); + pool->low_count--; + } + + list_del(&page->lru); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + -(1 << pool->order)); + return page; +} + +struct page *ion_page_pool_alloc(struct ion_page_pool *pool) +{ + struct page *page = NULL; + + BUG_ON(!pool); + + mutex_lock(&pool->mutex); + if (pool->high_count) + page = ion_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + + if (!page) + page = ion_page_pool_alloc_pages(pool); + + return page; +} + +void ion_page_pool_free(struct ion_page_pool *pool, struct page *page) +{ + BUG_ON(pool->order != compound_order(page)); + + ion_page_pool_add(pool, page); +} + +static int ion_page_pool_total(struct ion_page_pool *pool, bool high) +{ + int count = pool->low_count; + + if (high) + count += pool->high_count; + + return count << pool->order; +} + +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) +{ + int freed = 0; + bool high; + + if (current_is_kswapd()) + high = true; + else + high = !!(gfp_mask & __GFP_HIGHMEM); + + if (nr_to_scan == 0) + return ion_page_pool_total(pool, high); + + while (freed < nr_to_scan) { + struct page *page; + + mutex_lock(&pool->mutex); + if (pool->low_count) { + page = ion_page_pool_remove(pool, false); + } else if (high && pool->high_count) { + page = ion_page_pool_remove(pool, true); + } else { + mutex_unlock(&pool->mutex); + break; + } + mutex_unlock(&pool->mutex); + ion_page_pool_free_pages(pool, page); + freed += (1 << pool->order); + } + + return freed; +} + +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) +{ + struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return NULL; + pool->high_count = 0; + pool->low_count = 0; + INIT_LIST_HEAD(&pool->low_items); + INIT_LIST_HEAD(&pool->high_items); + pool->gfp_mask = gfp_mask | __GFP_COMP; + pool->order = order; + mutex_init(&pool->mutex); + plist_node_init(&pool->list, order); + + return pool; +} + +void ion_page_pool_destroy(struct ion_page_pool *pool) +{ + kfree(pool); +} diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.h b/drivers/staging/android/ion/heaps/ion_page_pool.h new file mode 100644 index 000000000000..e205d0086833 --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_page_pool.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ION Page Pool kernel interface header + * + * Copyright (C) 2011 Google, Inc. + */ + +#ifndef _ION_PAGE_POOL_H +#define _ION_PAGE_POOL_H + +#include +#include +#include +#include + +/** + * functions for creating and destroying a heap pool -- allows you + * to keep a pool of pre allocated memory to use from your heap. Keeping + * a pool of memory that is ready for dma, ie any cached mapping have been + * invalidated from the cache, provides a significant performance benefit on + * many systems + */ + +/** + * struct ion_page_pool - pagepool struct + * @high_count: number of highmem items in the pool + * @low_count: number of lowmem items in the pool + * @high_items: list of highmem items + * @low_items: list of lowmem items + * @mutex: lock protecting this struct and especially the count + * item list + * @gfp_mask: gfp_mask to use from alloc + * @order: order of pages in the pool + * @list: plist node for list of pools + * + * Allows you to keep a pool of pre allocated pages to use from your heap. + * Keeping a pool of pages that is ready for dma, ie any cached mapping have + * been invalidated from the cache, provides a significant performance benefit + * on many systems + */ +struct ion_page_pool { + int high_count; + int low_count; + struct list_head high_items; + struct list_head low_items; + struct mutex mutex; + gfp_t gfp_mask; + unsigned int order; + struct plist_node list; +}; + +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order); +void ion_page_pool_destroy(struct ion_page_pool *pool); +struct page *ion_page_pool_alloc(struct ion_page_pool *pool); +void ion_page_pool_free(struct ion_page_pool *pool, struct page *page); + +/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool + * @pool: the pool + * @gfp_mask: the memory type to reclaim + * @nr_to_scan: number of items to shrink in pages + * + * returns the number of items freed in pages + */ +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan); +#endif /* _ION_PAGE_POOL_H */ diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c new file mode 100644 index 000000000000..6052b843cdeb --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator system heap exporter + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ion_page_pool.h" + +#define NUM_ORDERS ARRAY_SIZE(orders) + +static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | + __GFP_NORETRY) & ~__GFP_RECLAIM; +static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO; +static const unsigned int orders[] = {8, 4, 0}; + +static int order_to_index(unsigned int order) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (order == orders[i]) + return i; + BUG(); + return -1; +} + +static inline unsigned int order_to_size(int order) +{ + return PAGE_SIZE << order; +} + +struct ion_system_heap { + struct ion_heap heap; + struct ion_page_pool *pools[NUM_ORDERS]; +}; + +static struct page *alloc_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long order) +{ + struct ion_page_pool *pool = heap->pools[order_to_index(order)]; + + return ion_page_pool_alloc(pool); +} + +static void free_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, struct page *page) +{ + struct ion_page_pool *pool; + unsigned int order = compound_order(page); + + /* go to system */ + if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) { + __free_pages(page, order); + return; + } + + pool = heap->pools[order_to_index(order)]; + + ion_page_pool_free(pool, page); +} + +static struct page *alloc_largest_available(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) +{ + struct page *page; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < order_to_size(orders[i])) + continue; + if (max_order < orders[i]) + continue; + + page = alloc_buffer_page(heap, buffer, orders[i]); + if (!page) + continue; + + return page; + } + + return NULL; +} + +static int ion_system_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned long flags) +{ + struct ion_system_heap *sys_heap = container_of(heap, + struct ion_system_heap, + heap); + struct sg_table *table; + struct scatterlist *sg; + struct list_head pages; + struct page *page, *tmp_page; + int i = 0; + unsigned long size_remaining = PAGE_ALIGN(size); + unsigned int max_order = orders[0]; + + if (size / PAGE_SIZE > totalram_pages() / 2) + return -ENOMEM; + + INIT_LIST_HEAD(&pages); + while (size_remaining > 0) { + page = alloc_largest_available(sys_heap, buffer, size_remaining, + max_order); + if (!page) + goto free_pages; + list_add_tail(&page->lru, &pages); + size_remaining -= page_size(page); + max_order = compound_order(page); + i++; + } + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + goto free_pages; + + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_table; + + sg = table->sgl; + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + list_del(&page->lru); + } + + buffer->sg_table = table; + + ion_buffer_prep_noncached(buffer); + + return 0; + +free_table: + kfree(table); +free_pages: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + free_buffer_page(sys_heap, buffer, page); + return -ENOMEM; +} + +static void ion_system_heap_free(struct ion_buffer *buffer) +{ + struct ion_system_heap *sys_heap = container_of(buffer->heap, + struct ion_system_heap, + heap); + struct sg_table *table = buffer->sg_table; + struct scatterlist *sg; + int i; + + /* zero the buffer before goto page pool */ + if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) + ion_buffer_zero(buffer); + + for_each_sg(table->sgl, sg, table->nents, i) + free_buffer_page(sys_heap, buffer, sg_page(sg)); + sg_free_table(table); + kfree(table); +} + +static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, + int nr_to_scan) +{ + struct ion_page_pool *pool; + struct ion_system_heap *sys_heap; + int nr_total = 0; + int i, nr_freed; + int only_scan = 0; + + sys_heap = container_of(heap, struct ion_system_heap, heap); + + if (!nr_to_scan) + only_scan = 1; + + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->pools[i]; + + if (only_scan) { + nr_total += ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); + + } else { + nr_freed = ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); + nr_to_scan -= nr_freed; + nr_total += nr_freed; + if (nr_to_scan <= 0) + break; + } + } + return nr_total; +} + +static void ion_system_heap_destroy_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (pools[i]) + ion_page_pool_destroy(pools[i]); +} + +static int ion_system_heap_create_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + struct ion_page_pool *pool; + gfp_t gfp_flags = low_order_gfp_flags; + + if (orders[i] > 4) + gfp_flags = high_order_gfp_flags; + + pool = ion_page_pool_create(gfp_flags, orders[i]); + if (!pool) + goto err_create_pool; + pools[i] = pool; + } + + return 0; + +err_create_pool: + ion_system_heap_destroy_pools(pools); + return -ENOMEM; +} + +static struct ion_heap_ops system_heap_ops = { + .allocate = ion_system_heap_allocate, + .free = ion_system_heap_free, + .shrink = ion_system_heap_shrink, +}; + +static struct ion_system_heap system_heap = { + .heap = { + .ops = &system_heap_ops, + .type = ION_HEAP_TYPE_SYSTEM, + .flags = ION_HEAP_FLAG_DEFER_FREE, + .name = "ion_system_heap", + } +}; + +static int __init ion_system_heap_init(void) +{ + int ret = ion_system_heap_create_pools(system_heap.pools); + if (ret) + return ret; + + return ion_device_add_heap(&system_heap.heap); +} + +static void __exit ion_system_heap_exit(void) +{ + ion_device_remove_heap(&system_heap.heap); + ion_system_heap_destroy_pools(system_heap.pools); +} + +module_init(ion_system_heap_init); +module_exit(ion_system_heap_exit); +MODULE_LICENSE("GPL v2");