diff --git a/drivers/staging/android/ion/heaps/Makefile b/drivers/staging/android/ion/heaps/Makefile index d6752eab5cd6..5cbcdaf1fb98 100644 --- a/drivers/staging/android/ion/heaps/Makefile +++ b/drivers/staging/android/ion/heaps/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_sys_heap.o +ion_sys_heap-y := ion_system_heap.o ion_page_pool.o obj-$(CONFIG_ION_MSM_HEAPS) += msm_ion_heaps.o -msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_page_pool.o \ - ion_system_heap.o ion_carveout_heap.o ion_system_secure_heap.o \ - ion_cma_heap.o ion_secure_util.o +msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_msm_page_pool.o \ + ion_msm_system_heap.o ion_carveout_heap.o \ + ion_system_secure_heap.o ion_cma_heap.o ion_secure_util.o diff --git a/drivers/staging/android/ion/heaps/ion_msm_page_pool.c b/drivers/staging/android/ion/heaps/ion_msm_page_pool.c new file mode 100644 index 000000000000..0253b5fa5fd5 --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_msm_page_pool.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator page pool helpers + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include + +#include "msm_ion_priv.h" +#include "ion_msm_page_pool.h" + +static inline struct page +*ion_msm_page_pool_alloc_pages(struct ion_msm_page_pool *pool) +{ + if (fatal_signal_pending(current)) + return NULL; + return alloc_pages(pool->gfp_mask, pool->order); +} + +static void ion_msm_page_pool_free_pages(struct ion_msm_page_pool *pool, + struct page *page) +{ + __free_pages(page, pool->order); +} + +static void ion_msm_page_pool_add(struct ion_msm_page_pool *pool, + struct page *page) +{ + mutex_lock(&pool->mutex); + if (PageHighMem(page)) { + list_add_tail(&page->lru, &pool->high_items); + pool->high_count++; + } else { + list_add_tail(&page->lru, &pool->low_items); + pool->low_count++; + } + + atomic_inc(&pool->count); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + (1 << pool->order)); + mutex_unlock(&pool->mutex); +} + +#ifdef CONFIG_ION_POOL_AUTO_REFILL +/* do a simple check to see if we are in any low memory situation */ +static bool pool_refill_ok(struct ion_msm_page_pool *pool) +{ + struct zonelist *zonelist; + struct zoneref *z; + struct zone *zone; + int mark; + enum zone_type classzone_idx = gfp_zone(pool->gfp_mask); + s64 delta; + + /* check if we are within the refill defer window */ + delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime); + if (delta < ION_POOL_REFILL_DEFER_WINDOW_MS) + return false; + + zonelist = node_zonelist(numa_node_id(), pool->gfp_mask); + /* + * make sure that if we allocate a pool->order page from buddy, + * we don't put the zone watermarks go below the high threshold. + * This makes sure there's no unwanted repetitive refilling and + * reclaiming of buddy pages on the pool. + */ + for_each_zone_zonelist(zone, z, zonelist, classzone_idx) { + mark = high_wmark_pages(zone); + mark += 1 << pool->order; + if (!zone_watermark_ok_safe(zone, pool->order, mark, + classzone_idx)) { + pool->last_low_watermark_ktime = ktime_get(); + return false; + } + } + + return true; +} + +void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool) +{ + struct page *page; + gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY; + struct device *dev = pool->heap_dev; + + /* skip refilling order 0 pools */ + if (!pool->order) + return; + + while (!pool_fillmark_reached(pool) && pool_refill_ok(pool)) { + page = alloc_pages(gfp_refill, pool->order); + if (!page) + break; + if (!pool->cached) + ion_pages_sync_for_device(dev, page, + PAGE_SIZE << pool->order, + DMA_BIDIRECTIONAL); + ion_msm_page_pool_add(pool, page); + } +} +#endif /* CONFIG_ION_PAGE_POOL_REFILL */ + +static struct page *ion_msm_page_pool_remove(struct ion_msm_page_pool *pool, + bool high) +{ + struct page *page; + + if (high) { + BUG_ON(!pool->high_count); + page = list_first_entry(&pool->high_items, struct page, lru); + pool->high_count--; + } else { + BUG_ON(!pool->low_count); + page = list_first_entry(&pool->low_items, struct page, lru); + pool->low_count--; + } + + atomic_dec(&pool->count); + list_del(&page->lru); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + -(1 << pool->order)); + return page; +} + +struct page *ion_msm_page_pool_alloc(struct ion_msm_page_pool *pool, + bool *from_pool) +{ + struct page *page = NULL; + + BUG_ON(!pool); + + if (fatal_signal_pending(current)) + return ERR_PTR(-EINTR); + + if (*from_pool && mutex_trylock(&pool->mutex)) { + if (pool->high_count) + page = ion_msm_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_msm_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + } + if (!page) { + page = ion_msm_page_pool_alloc_pages(pool); + *from_pool = false; + } + + if (!page) + return ERR_PTR(-ENOMEM); + return page; +} + +/* + * Tries to allocate from only the specified Pool and returns NULL otherwise + */ +struct page *ion_msm_page_pool_alloc_pool_only(struct ion_msm_page_pool *pool) +{ + struct page *page = NULL; + + if (!pool) + return ERR_PTR(-EINVAL); + + if (mutex_trylock(&pool->mutex)) { + if (pool->high_count) + page = ion_msm_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_msm_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + } + + if (!page) + return ERR_PTR(-ENOMEM); + return page; +} + +void ion_msm_page_pool_free(struct ion_msm_page_pool *pool, struct page *page) +{ + ion_msm_page_pool_add(pool, page); +} + +void ion_msm_page_pool_free_immediate(struct ion_msm_page_pool *pool, + struct page *page) +{ + ion_msm_page_pool_free_pages(pool, page); +} + +int ion_msm_page_pool_total(struct ion_msm_page_pool *pool, bool high) +{ + int count = pool->low_count; + + if (high) + count += pool->high_count; + + return count << pool->order; +} + +int ion_msm_page_pool_shrink(struct ion_msm_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) +{ + int freed = 0; + bool high; + + if (current_is_kswapd()) + high = true; + else + high = !!(gfp_mask & __GFP_HIGHMEM); + + if (nr_to_scan == 0) + return ion_msm_page_pool_total(pool, high); + + while (freed < nr_to_scan) { + struct page *page; + + mutex_lock(&pool->mutex); + if (pool->low_count) { + page = ion_msm_page_pool_remove(pool, false); + } else if (high && pool->high_count) { + page = ion_msm_page_pool_remove(pool, true); + } else { + mutex_unlock(&pool->mutex); + break; + } + mutex_unlock(&pool->mutex); + ion_msm_page_pool_free_pages(pool, page); + freed += (1 << pool->order); + } + + return freed; +} + +struct ion_msm_page_pool *ion_msm_page_pool_create(gfp_t gfp_mask, + unsigned int order, + bool cached) +{ + struct ion_msm_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return NULL; + INIT_LIST_HEAD(&pool->low_items); + INIT_LIST_HEAD(&pool->high_items); + pool->gfp_mask = gfp_mask; + pool->order = order; + mutex_init(&pool->mutex); + plist_node_init(&pool->list, order); + if (cached) + pool->cached = true; + + return pool; +} + +void ion_msm_page_pool_destroy(struct ion_msm_page_pool *pool) +{ + kfree(pool); +} diff --git a/drivers/staging/android/ion/heaps/ion_msm_page_pool.h b/drivers/staging/android/ion/heaps/ion_msm_page_pool.h new file mode 100644 index 000000000000..88b0d95a099a --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_msm_page_pool.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ION Page Pool kernel interface header + * + * Copyright (C) 2011 Google, Inc. + */ + +#ifndef _ION_MSM_PAGE_POOL_H +#define _ION_MSM_PAGE_POOL_H + +#include +#include +#include +#include + +/* ION page pool marks in bytes */ +#ifdef CONFIG_ION_POOL_AUTO_REFILL +#define ION_POOL_FILL_MARK (CONFIG_ION_POOL_FILL_MARK * SZ_1M) +#define POOL_LOW_MARK_PERCENT 40UL +#define ION_POOL_LOW_MARK ((ION_POOL_FILL_MARK * POOL_LOW_MARK_PERCENT) / 100) +#else +#define ION_POOL_FILL_MARK 0UL +#define ION_POOL_LOW_MARK 0UL +#endif + +/* if low watermark of zones have reached, defer the refill in this window */ +#define ION_POOL_REFILL_DEFER_WINDOW_MS 10 + +/** + * functions for creating and destroying a heap pool -- allows you + * to keep a pool of pre allocated memory to use from your heap. Keeping + * a pool of memory that is ready for dma, ie any cached mapping have been + * invalidated from the cache, provides a significant performance benefit on + * many systems + */ + +/** + * struct ion_msm_page_pool - pagepool struct + * @high_count: number of highmem items in the pool + * @low_count: number of lowmem items in the pool + * @count: total number of pages/items in the pool + * @high_items: list of highmem items + * @low_items: list of lowmem items + * @last_low_watermark_ktime: most recent time at which the zone watermarks were + * low + * @mutex: lock protecting this struct and especially the count + * item list + * @gfp_mask: gfp_mask to use from alloc + * @order: order of pages in the pool + * @list: plist node for list of pools + * @cached: it's cached pool or not + * @heap_dev: device for the ion heap associated with this pool + * + * Allows you to keep a pool of pre allocated pages to use from your heap. + * Keeping a pool of pages that is ready for dma, ie any cached mapping have + * been invalidated from the cache, provides a significant performance benefit + * on many systems + */ +struct ion_msm_page_pool { + int high_count; + int low_count; + atomic_t count; + struct list_head high_items; + struct list_head low_items; + ktime_t last_low_watermark_ktime; + struct mutex mutex; + gfp_t gfp_mask; + unsigned int order; + struct plist_node list; + bool cached; + struct device *heap_dev; +}; + +struct ion_msm_page_pool *ion_msm_page_pool_create(gfp_t gfp_mask, + unsigned int order, + bool cached); +void ion_msm_page_pool_destroy(struct ion_msm_page_pool *pool); +struct page *ion_msm_page_pool_alloc(struct ion_msm_page_pool *pool, + bool *from_pool); +void ion_msm_page_pool_free(struct ion_msm_page_pool *pool, struct page *page); +struct page *ion_msm_page_pool_alloc_pool_only(struct ion_msm_page_pool *a); +void ion_msm_page_pool_free_immediate(struct ion_msm_page_pool *pool, + struct page *page); +int ion_msm_page_pool_total(struct ion_msm_page_pool *pool, bool high); +size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid); + +/** ion_msm_page_pool_shrink - shrinks the size of the memory cached in the pool + * @pool: the pool + * @gfp_mask: the memory type to reclaim + * @nr_to_scan: number of items to shrink in pages + * + * returns the number of items freed in pages + */ +int ion_msm_page_pool_shrink(struct ion_msm_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan); + +#ifdef CONFIG_ION_POOL_AUTO_REFILL +void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool); + +static __always_inline int get_pool_fillmark(struct ion_msm_page_pool *pool) +{ + return ION_POOL_FILL_MARK / (PAGE_SIZE << pool->order); +} + +static __always_inline int get_pool_lowmark(struct ion_msm_page_pool *pool) +{ + return ION_POOL_LOW_MARK / (PAGE_SIZE << pool->order); +} + +static __always_inline bool +pool_count_below_lowmark(struct ion_msm_page_pool *pool) +{ + return atomic_read(&pool->count) < get_pool_lowmark(pool); +} + +static __always_inline bool +pool_fillmark_reached(struct ion_msm_page_pool *pool) +{ + return atomic_read(&pool->count) >= get_pool_fillmark(pool); +} +#else +static inline void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool) +{ +} + +static __always_inline int get_pool_fillmark(struct ion_msm_page_pool *pool) +{ + return 0; +} + +static __always_inline int get_pool_lowmark(struct ion_msm_page_pool *pool) +{ + return 0; +} + +static __always_inline bool +pool_count_below_lowmark(struct ion_msm_page_pool *pool) +{ + return false; +} + +static __always_inline bool +pool_fillmark_reached(struct ion_msm_page_pool *pool) +{ + return false; +} +#endif /* CONFIG_ION_POOL_AUTO_REFILL */ +#endif /* _ION_MSM_PAGE_POOL_H */ diff --git a/drivers/staging/android/ion/heaps/ion_msm_system_heap.c b/drivers/staging/android/ion/heaps/ion_msm_system_heap.c new file mode 100644 index 000000000000..f57918149716 --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_msm_system_heap.c @@ -0,0 +1,789 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator system heap exporter + * + * Copyright (C) 2011 Google, Inc. + * Copyright (c) 2011-2020, The Linux Foundation. All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ion_msm_system_heap.h" +#include "ion_msm_page_pool.h" +#include "msm_ion_priv.h" +#include "ion_system_secure_heap.h" +#include "ion_secure_util.h" + +static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | + __GFP_NORETRY) & ~__GFP_RECLAIM; +static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO; + +static bool pool_auto_refill_en __read_mostly = +IS_ENABLED(CONFIG_ION_POOL_AUTO_REFILL); + +int order_to_index(unsigned int order) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (order == orders[i]) + return i; + BUG(); + return -1; +} + +static inline unsigned int order_to_size(int order) +{ + return PAGE_SIZE << order; +} + +static int ion_heap_is_msm_system_heap_type(enum ion_heap_type type) +{ + return type == ((enum ion_heap_type)ION_HEAP_TYPE_MSM_SYSTEM); +} + +static struct page *alloc_buffer_page(struct ion_msm_system_heap *sys_heap, + struct ion_buffer *buffer, + unsigned long order, + bool *from_pool) +{ + int cached = (int)ion_buffer_cached(buffer); + struct page *page; + struct ion_msm_page_pool *pool; + int vmid = get_secure_vmid(buffer->flags); + struct device *dev = sys_heap->heap.dev; + int order_ind = order_to_index(order); + struct task_struct *worker; + + if (vmid > 0) { + pool = sys_heap->secure_pools[vmid][order_ind]; + + /* + * We should skip stealing pages if (1) we're focing our + * allocations to come from buddy; or (2) pool refilling is + * disabled, in which case stealing pages could deplete the + * uncached pools. + */ + if (!(*from_pool && pool_auto_refill_en)) + goto normal_alloc; + + page = ion_msm_page_pool_alloc_pool_only(pool); + if (!IS_ERR(page)) + return page; + + pool = sys_heap->uncached_pools[order_ind]; + page = ion_msm_page_pool_alloc_pool_only(pool); + if (IS_ERR(page)) { + pool = sys_heap->secure_pools[vmid][order_ind]; + goto normal_alloc; + } + + /* + * Here, setting `from_pool = false` indicates that the + * page didn't come from the secure pool, and causes + * the page to be hyp-assigned. + */ + *from_pool = false; + + if (pool_auto_refill_en && pool->order && + pool_count_below_lowmark(pool)) { + worker = sys_heap->kworker[ION_KTHREAD_UNCACHED]; + wake_up_process(worker); + } + return page; + } else if (!cached) { + pool = sys_heap->uncached_pools[order_ind]; + } else { + pool = sys_heap->cached_pools[order_ind]; + } + +normal_alloc: + page = ion_msm_page_pool_alloc(pool, from_pool); + + if (pool_auto_refill_en && pool->order && + pool_count_below_lowmark(pool) && vmid <= 0) + wake_up_process(sys_heap->kworker[cached]); + + if (IS_ERR(page)) + return page; + + if ((MAKE_ION_ALLOC_DMA_READY && vmid <= 0) || !(*from_pool)) + ion_pages_sync_for_device(dev, page, PAGE_SIZE << order, + DMA_BIDIRECTIONAL); + + return page; +} + +/* + * For secure pages that need to be freed and not added back to the pool; the + * hyp_unassign should be called before calling this function + */ +void free_buffer_page(struct ion_msm_system_heap *heap, + struct ion_buffer *buffer, struct page *page, + unsigned int order) +{ + bool cached = ion_buffer_cached(buffer); + int vmid = get_secure_vmid(buffer->flags); + + if (!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { + struct ion_msm_page_pool *pool; + + if (vmid > 0) + pool = heap->secure_pools[vmid][order_to_index(order)]; + else if (cached) + pool = heap->cached_pools[order_to_index(order)]; + else + pool = heap->uncached_pools[order_to_index(order)]; + + if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) + ion_msm_page_pool_free_immediate(pool, page); + else + ion_msm_page_pool_free(pool, page); + +#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES + mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES, + -(1 << pool->order)); +#endif + } else { + __free_pages(page, order); + +#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES + mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES, + -(1 << order)); +#endif + + } +} + +static struct +page_info *alloc_largest_available(struct ion_msm_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) +{ + struct page *page; + struct page_info *info; + int i; + bool from_pool; + + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < order_to_size(orders[i])) + continue; + if (max_order < orders[i]) + continue; + from_pool = !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC); + page = alloc_buffer_page(heap, buffer, orders[i], &from_pool); + if (IS_ERR(page)) + continue; + + info->page = page; + info->order = orders[i]; + info->from_pool = from_pool; + INIT_LIST_HEAD(&info->list); + return info; + } + kfree(info); + + return ERR_PTR(-ENOMEM); +} + +static struct page_info * +alloc_from_pool_preferred(struct ion_msm_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) +{ + struct page *page; + struct page_info *info; + int i; + + if (buffer->flags & ION_FLAG_POOL_FORCE_ALLOC) + goto force_alloc; + + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < order_to_size(orders[i])) + continue; + if (max_order < orders[i]) + continue; + + page = alloc_from_secure_pool_order(heap, buffer, orders[i]); + if (IS_ERR(page)) + continue; + + info->page = page; + info->order = orders[i]; + info->from_pool = true; + INIT_LIST_HEAD(&info->list); + return info; + } + + page = split_page_from_secure_pool(heap, buffer); + if (!IS_ERR(page)) { + info->page = page; + info->order = 0; + info->from_pool = true; + INIT_LIST_HEAD(&info->list); + return info; + } + + kfree(info); +force_alloc: + return alloc_largest_available(heap, buffer, size, max_order); +} + +static void process_info(struct page_info *info, + struct scatterlist *sg, + struct scatterlist *sg_sync) +{ + struct page *page = info->page; + + if (sg_sync) { + sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0); + sg_dma_address(sg_sync) = page_to_phys(page); + } + sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0); + /* + * This is not correct - sg_dma_address needs a dma_addr_t + * that is valid for the the targeted device, but this works + * on the currently targeted hardware. + */ + sg_dma_address(sg) = page_to_phys(page); + + list_del(&info->list); + kfree(info); +} + +static int ion_msm_system_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned long flags) +{ + struct ion_msm_system_heap *sys_heap = to_msm_system_heap(heap); + struct msm_ion_buf_lock_state *lock_state; + struct sg_table *table; + struct sg_table table_sync = {0}; + struct scatterlist *sg; + struct scatterlist *sg_sync; + int ret = -ENOMEM; + struct list_head pages; + struct list_head pages_from_pool; + struct page_info *info, *tmp_info; + int i = 0; + unsigned int nents_sync = 0; + unsigned long size_remaining = PAGE_ALIGN(size); + unsigned int max_order = orders[0]; + unsigned int sz; + int vmid = get_secure_vmid(buffer->flags); + + if (size / PAGE_SIZE > totalram_pages() / 2) + return -ENOMEM; + + if (ion_heap_is_msm_system_heap_type(buffer->heap->type) && + is_secure_allocation(buffer->flags)) { + pr_info("%s: System heap doesn't support secure allocations\n", + __func__); + return -EINVAL; + } + + INIT_LIST_HEAD(&pages); + INIT_LIST_HEAD(&pages_from_pool); + + while (size_remaining > 0) { + if (is_secure_vmid_valid(vmid)) + info = alloc_from_pool_preferred(sys_heap, buffer, + size_remaining, + max_order); + else + info = alloc_largest_available(sys_heap, buffer, + size_remaining, + max_order); + + if (IS_ERR(info)) { + ret = PTR_ERR(info); + goto err; + } + + sz = (1 << info->order) * PAGE_SIZE; + +#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES + mod_node_page_state(page_pgdat(info->page), + NR_UNRECLAIMABLE_PAGES, + (1 << (info->order))); +#endif + + if (info->from_pool) { + list_add_tail(&info->list, &pages_from_pool); + } else { + list_add_tail(&info->list, &pages); + ++nents_sync; + } + size_remaining -= sz; + max_order = info->order; + i++; + } + + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) { + ret = -ENOMEM; + goto err; + } + + ret = sg_alloc_table(table, i, GFP_KERNEL); + if (ret) + goto err1; + + if (nents_sync) { + ret = sg_alloc_table(&table_sync, nents_sync, GFP_KERNEL); + if (ret) + goto err_free_sg; + } + + sg = table->sgl; + sg_sync = table_sync.sgl; + + /* + * We now have two separate lists. One list contains pages from the + * pool and the other pages from buddy. We want to merge these + * together while preserving the ordering of the pages (higher order + * first). + */ + do { + info = list_first_entry_or_null(&pages, struct page_info, list); + tmp_info = list_first_entry_or_null(&pages_from_pool, + struct page_info, list); + if (info && tmp_info) { + if (info->order >= tmp_info->order) { + process_info(info, sg, sg_sync); + sg_sync = sg_next(sg_sync); + } else { + process_info(tmp_info, sg, NULL); + } + } else if (info) { + process_info(info, sg, sg_sync); + sg_sync = sg_next(sg_sync); + } else if (tmp_info) { + process_info(tmp_info, sg, NULL); + } + sg = sg_next(sg); + + } while (sg); + + if (nents_sync) { + if (vmid > 0) { + ret = ion_hyp_assign_sg(&table_sync, &vmid, 1, true); + if (ret) + goto err_free_sg2; + } + } + + buffer->sg_table = table; + if (nents_sync) + sg_free_table(&table_sync); + + lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); + if (!lock_state) { + ret = -ENOMEM; + goto err_free_sg2; + } + buffer->priv_virt = lock_state; + + ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); + return 0; + +err_free_sg2: + /* We failed to zero buffers. Bypass pool */ + buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE; + + if (vmid > 0) + if (ion_hyp_unassign_sg(table, &vmid, 1, true)) + goto err_free_table_sync; + + for_each_sg(table->sgl, sg, table->nents, i) + free_buffer_page(sys_heap, buffer, sg_page(sg), + get_order(sg->length)); +err_free_table_sync: + if (nents_sync) + sg_free_table(&table_sync); +err_free_sg: + sg_free_table(table); +err1: + kfree(table); +err: + list_for_each_entry_safe(info, tmp_info, &pages, list) { + free_buffer_page(sys_heap, buffer, info->page, info->order); + kfree(info); + } + list_for_each_entry_safe(info, tmp_info, &pages_from_pool, list) { + free_buffer_page(sys_heap, buffer, info->page, info->order); + kfree(info); + } + return ret; +} + +static void ion_msm_system_heap_free(struct ion_buffer *buffer) +{ + struct ion_heap *heap = buffer->heap; + struct ion_msm_system_heap *sys_heap = to_msm_system_heap(heap); + struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + struct sg_table *table = buffer->sg_table; + struct scatterlist *sg; + int i; + int vmid = get_secure_vmid(buffer->flags); + + if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) && + !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { + mutex_lock(&buffer->lock); + if (hlos_accessible_buffer(buffer)) + ion_buffer_zero(buffer); + + if (lock_state && lock_state->locked) + pr_warn("%s: buffer is locked while being freed\n", + __func__); + mutex_unlock(&buffer->lock); + } else if (vmid > 0) { + if (ion_hyp_unassign_sg(table, &vmid, 1, true)) + return; + } + + for_each_sg(table->sgl, sg, table->nents, i) + free_buffer_page(sys_heap, buffer, sg_page(sg), + get_order(sg->length)); + sg_free_table(table); + kfree(table); + kfree(buffer->priv_virt); +} + +static int ion_msm_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, + int nr_to_scan) +{ + struct ion_msm_system_heap *sys_heap; + int nr_total = 0; + int i, j, nr_freed = 0; + int only_scan = 0; + struct ion_msm_page_pool *pool; + + sys_heap = to_msm_system_heap(heap); + + if (!nr_to_scan) + only_scan = 1; + + /* shrink the pools starting from lower order ones */ + for (i = NUM_ORDERS - 1; i >= 0; i--) { + nr_freed = 0; + + for (j = 0; j < VMID_LAST; j++) { + if (is_secure_vmid_valid(j)) + nr_freed += + ion_secure_page_pool_shrink(sys_heap, + j, i, + nr_to_scan); + } + + pool = sys_heap->uncached_pools[i]; + nr_freed += + ion_msm_page_pool_shrink(pool, gfp_mask, nr_to_scan); + + pool = sys_heap->cached_pools[i]; + nr_freed += + ion_msm_page_pool_shrink(pool, gfp_mask, nr_to_scan); + nr_total += nr_freed; + + if (!only_scan) { + nr_to_scan -= nr_freed; + /* shrink completed */ + if (nr_to_scan <= 0) + break; + } + } + + return nr_total; +} + +static struct ion_heap_ops system_heap_ops = { + .allocate = ion_msm_system_heap_allocate, + .free = ion_msm_system_heap_free, + .shrink = ion_msm_system_heap_shrink, +}; + +static int ion_msm_system_heap_debug_show(struct ion_heap *heap, + struct seq_file *s, void *unused) +{ + struct ion_msm_system_heap *sys_heap; + bool use_seq = s; + unsigned long uncached_total = 0; + unsigned long cached_total = 0; + unsigned long secure_total = 0; + struct ion_msm_page_pool *pool; + int i, j; + + sys_heap = to_msm_system_heap(heap); + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->uncached_pools[i]; + if (use_seq) { + seq_printf(s, + "%d order %u highmem pages in uncached pool = %lu total\n", + pool->high_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->high_count); + seq_printf(s, + "%d order %u lowmem pages in uncached pool = %lu total\n", + pool->low_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->low_count); + } + + uncached_total += (1 << pool->order) * PAGE_SIZE * + pool->high_count; + uncached_total += (1 << pool->order) * PAGE_SIZE * + pool->low_count; + } + + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->cached_pools[i]; + if (use_seq) { + seq_printf(s, + "%d order %u highmem pages in cached pool = %lu total\n", + pool->high_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->high_count); + seq_printf(s, + "%d order %u lowmem pages in cached pool = %lu total\n", + pool->low_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->low_count); + } + + cached_total += (1 << pool->order) * PAGE_SIZE * + pool->high_count; + cached_total += (1 << pool->order) * PAGE_SIZE * + pool->low_count; + } + + for (i = 0; i < NUM_ORDERS; i++) { + for (j = 0; j < VMID_LAST; j++) { + if (!is_secure_vmid_valid(j)) + continue; + pool = sys_heap->secure_pools[j][i]; + + if (use_seq) { + seq_printf(s, + "VMID %d: %d order %u highmem pages in secure pool = %lu total\n", + j, pool->high_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->high_count); + seq_printf(s, + "VMID %d: %d order %u lowmem pages in secure pool = %lu total\n", + j, pool->low_count, pool->order, + (1 << pool->order) * PAGE_SIZE * + pool->low_count); + } + + secure_total += (1 << pool->order) * PAGE_SIZE * + pool->high_count; + secure_total += (1 << pool->order) * PAGE_SIZE * + pool->low_count; + } + } + + if (use_seq) { + seq_puts(s, "--------------------------------------------\n"); + seq_printf(s, "uncached pool = %lu cached pool = %lu secure pool = %lu\n", + uncached_total, cached_total, secure_total); + seq_printf(s, "pool total (uncached + cached + secure) = %lu\n", + uncached_total + cached_total + secure_total); + seq_puts(s, "--------------------------------------------\n"); + } else { + pr_info("-------------------------------------------------\n"); + pr_info("uncached pool = %lu cached pool = %lu secure pool = %lu\n", + uncached_total, cached_total, secure_total); + pr_info("pool total (uncached + cached + secure) = %lu\n", + uncached_total + cached_total + secure_total); + pr_info("-------------------------------------------------\n"); + } + + return 0; +} + +static struct msm_ion_heap_ops msm_system_heap_ops = { + .debug_show = ion_msm_system_heap_debug_show, +}; + +static void ion_msm_system_heap_destroy_pools(struct ion_msm_page_pool **pools) +{ + int i; + + if (!pools) + return; + + for (i = 0; i < NUM_ORDERS; i++) + if (pools[i]) { + ion_msm_page_pool_destroy(pools[i]); + pools[i] = NULL; + } +} + +/** + * ion_msm_system_heap_create_pools - Creates pools for all orders + * + * If this fails you don't need to destroy any pools. It's all or + * nothing. If it succeeds you'll eventually need to use + * ion_msm_system_heap_destroy_pools to destroy the pools. + */ +static int +ion_msm_system_heap_create_pools(struct ion_msm_system_heap *sys_heap, + struct ion_msm_page_pool **pools, bool cached) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + struct ion_msm_page_pool *pool; + gfp_t gfp_flags = low_order_gfp_flags; + + if (orders[i]) + gfp_flags = high_order_gfp_flags; + pool = ion_msm_page_pool_create(gfp_flags, orders[i], cached); + if (!pool) + goto err_create_pool; + pool->heap_dev = sys_heap->heap.dev; + pools[i] = pool; + } + + return 0; +err_create_pool: + ion_msm_system_heap_destroy_pools(pools); + return -ENOMEM; +} + +static int ion_msm_sys_heap_worker(void *data) +{ + struct ion_msm_page_pool **pools = (struct ion_msm_page_pool **)data; + int i; + + for (;;) { + for (i = 0; i < NUM_ORDERS; i++) { + if (pool_count_below_lowmark(pools[i])) + ion_msm_page_pool_refill(pools[i]); + } + set_current_state(TASK_INTERRUPTIBLE); + if (unlikely(kthread_should_stop())) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); + + set_current_state(TASK_RUNNING); + } + + return 0; +} + +static struct task_struct *ion_create_kworker(struct ion_msm_page_pool **pools, + bool cached) +{ + struct sched_attr attr = { 0 }; + struct task_struct *thread; + int ret; + char *buf; + + attr.sched_nice = ION_KTHREAD_NICE_VAL; + buf = cached ? "cached" : "uncached"; + + thread = kthread_run(ion_msm_sys_heap_worker, pools, + "ion-pool-%s-worker", buf); + if (IS_ERR(thread)) { + pr_err("%s: failed to create %s worker thread: %ld\n", + __func__, buf, PTR_ERR(thread)); + return thread; + } + ret = sched_setattr(thread, &attr); + if (ret) { + kthread_stop(thread); + pr_warn("%s: failed to set task priority for %s worker thread: ret = %d\n", + __func__, buf, ret); + return ERR_PTR(ret); + } + + return thread; +} + +struct ion_heap *ion_msm_system_heap_create(struct ion_platform_heap *data) +{ + struct ion_msm_system_heap *heap; + int ret = -ENOMEM; + int i; + + heap = kzalloc(sizeof(*heap), GFP_KERNEL); + if (!heap) + return ERR_PTR(-ENOMEM); + heap->heap.dev = data->priv; + heap->heap.msm_heap_ops = &msm_system_heap_ops; + heap->heap.ion_heap.ops = &system_heap_ops; + heap->heap.ion_heap.buf_ops = msm_ion_dma_buf_ops; + heap->heap.ion_heap.type = (enum ion_heap_type)ION_HEAP_TYPE_MSM_SYSTEM; + heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE; + + for (i = 0; i < VMID_LAST; i++) + if (is_secure_vmid_valid(i) && + ion_msm_system_heap_create_pools(heap, + heap->secure_pools[i], + false)) + goto destroy_secure_pools; + + if (ion_msm_system_heap_create_pools(heap, heap->uncached_pools, false)) + goto destroy_secure_pools; + + if (ion_msm_system_heap_create_pools(heap, heap->cached_pools, true)) + goto destroy_uncached_pools; + + if (pool_auto_refill_en) { + heap->kworker[ION_KTHREAD_UNCACHED] = + ion_create_kworker(heap->uncached_pools, false); + if (IS_ERR(heap->kworker[ION_KTHREAD_UNCACHED])) { + ret = PTR_ERR(heap->kworker[ION_KTHREAD_UNCACHED]); + goto destroy_pools; + } + heap->kworker[ION_KTHREAD_CACHED] = + ion_create_kworker(heap->cached_pools, true); + if (IS_ERR(heap->kworker[ION_KTHREAD_CACHED])) { + kthread_stop(heap->kworker[ION_KTHREAD_UNCACHED]); + ret = PTR_ERR(heap->kworker[ION_KTHREAD_CACHED]); + goto destroy_pools; + } + } + + mutex_init(&heap->split_page_mutex); + + return &heap->heap.ion_heap; +destroy_pools: + ion_msm_system_heap_destroy_pools(heap->cached_pools); +destroy_uncached_pools: + ion_msm_system_heap_destroy_pools(heap->uncached_pools); +destroy_secure_pools: + for (i = 0; i < VMID_LAST; i++) + ion_msm_system_heap_destroy_pools(heap->secure_pools[i]); + kfree(heap); + return ERR_PTR(ret); +} + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.h b/drivers/staging/android/ion/heaps/ion_msm_system_heap.h similarity index 64% rename from drivers/staging/android/ion/heaps/ion_system_heap.h rename to drivers/staging/android/ion/heaps/ion_msm_system_heap.h index ee95a6a108c0..4e1986ae8916 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.h +++ b/drivers/staging/android/ion/heaps/ion_msm_system_heap.h @@ -1,12 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. */ #include #include "msm_ion_priv.h" -#ifndef _ION_SYSTEM_HEAP_H -#define _ION_SYSTEM_HEAP_H +#ifndef _ION_MSM_SYSTEM_HEAP_H +#define _ION_MSM_SYSTEM_HEAP_H #ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS #if defined(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) @@ -22,8 +22,8 @@ static const unsigned int orders[] = {0}; #define ION_KTHREAD_NICE_VAL 10 -#define to_system_heap(_heap) \ - container_of(to_msm_ion_heap(_heap), struct ion_system_heap, heap) +#define to_msm_system_heap(_heap) \ + container_of(to_msm_ion_heap(_heap), struct ion_msm_system_heap, heap) enum ion_kthread_type { ION_KTHREAD_UNCACHED, @@ -31,13 +31,13 @@ enum ion_kthread_type { ION_MAX_NUM_KTHREADS }; -struct ion_system_heap { +struct ion_msm_system_heap { struct msm_ion_heap heap; - struct ion_page_pool *uncached_pools[MAX_ORDER]; - struct ion_page_pool *cached_pools[MAX_ORDER]; + struct ion_msm_page_pool *uncached_pools[MAX_ORDER]; + struct ion_msm_page_pool *cached_pools[MAX_ORDER]; /* worker threads to refill the pool */ struct task_struct *kworker[ION_MAX_NUM_KTHREADS]; - struct ion_page_pool *secure_pools[VMID_LAST][MAX_ORDER]; + struct ion_msm_page_pool *secure_pools[VMID_LAST][MAX_ORDER]; /* Prevents unnecessary page splitting */ struct mutex split_page_mutex; }; @@ -51,8 +51,8 @@ struct page_info { int order_to_index(unsigned int order); -void free_buffer_page(struct ion_system_heap *heap, +void free_buffer_page(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, struct page *page, unsigned int order); -#endif /* _ION_SYSTEM_HEAP_H */ +#endif /* _ION_MSM_SYSTEM_HEAP_H */ diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.c b/drivers/staging/android/ion/heaps/ion_page_pool.c index cf1c0620402e..f1bc165e644e 100644 --- a/drivers/staging/android/ion/heaps/ion_page_pool.c +++ b/drivers/staging/android/ion/heaps/ion_page_pool.c @@ -10,7 +10,6 @@ #include #include -#include "msm_ion_priv.h" #include "ion_page_pool.h" static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool) @@ -37,71 +36,11 @@ static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) pool->low_count++; } - atomic_inc(&pool->count); mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, - (1 << pool->order)); + 1 << pool->order); mutex_unlock(&pool->mutex); } -#ifdef CONFIG_ION_POOL_AUTO_REFILL -/* do a simple check to see if we are in any low memory situation */ -static bool pool_refill_ok(struct ion_page_pool *pool) -{ - struct zonelist *zonelist; - struct zoneref *z; - struct zone *zone; - int mark; - enum zone_type classzone_idx = gfp_zone(pool->gfp_mask); - s64 delta; - - /* check if we are within the refill defer window */ - delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime); - if (delta < ION_POOL_REFILL_DEFER_WINDOW_MS) - return false; - - zonelist = node_zonelist(numa_node_id(), pool->gfp_mask); - /* - * make sure that if we allocate a pool->order page from buddy, - * we don't put the zone watermarks go below the high threshold. - * This makes sure there's no unwanted repetitive refilling and - * reclaiming of buddy pages on the pool. - */ - for_each_zone_zonelist(zone, z, zonelist, classzone_idx) { - mark = high_wmark_pages(zone); - mark += 1 << pool->order; - if (!zone_watermark_ok_safe(zone, pool->order, mark, - classzone_idx)) { - pool->last_low_watermark_ktime = ktime_get(); - return false; - } - } - - return true; -} - -void ion_page_pool_refill(struct ion_page_pool *pool) -{ - struct page *page; - gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY; - struct device *dev = pool->heap_dev; - - /* skip refilling order 0 pools */ - if (!pool->order) - return; - - while (!pool_fillmark_reached(pool) && pool_refill_ok(pool)) { - page = alloc_pages(gfp_refill, pool->order); - if (!page) - break; - if (!pool->cached) - ion_pages_sync_for_device(dev, page, - PAGE_SIZE << pool->order, - DMA_BIDIRECTIONAL); - ion_page_pool_add(pool, page); - } -} -#endif /* CONFIG_ION_PAGE_POOL_REFILL */ - static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) { struct page *page; @@ -116,73 +55,39 @@ static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) pool->low_count--; } - atomic_dec(&pool->count); list_del(&page->lru); mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, - -(1 << pool->order)); + -(1 << pool->order)); return page; } -struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool) +struct page *ion_page_pool_alloc(struct ion_page_pool *pool) { struct page *page = NULL; BUG_ON(!pool); - if (fatal_signal_pending(current)) - return ERR_PTR(-EINTR); + mutex_lock(&pool->mutex); + if (pool->high_count) + page = ion_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); - if (*from_pool && mutex_trylock(&pool->mutex)) { - if (pool->high_count) - page = ion_page_pool_remove(pool, true); - else if (pool->low_count) - page = ion_page_pool_remove(pool, false); - mutex_unlock(&pool->mutex); - } - if (!page) { + if (!page) page = ion_page_pool_alloc_pages(pool); - *from_pool = false; - } - if (!page) - return ERR_PTR(-ENOMEM); - return page; -} - -/* - * Tries to allocate from only the specified Pool and returns NULL otherwise - */ -struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *pool) -{ - struct page *page = NULL; - - if (!pool) - return ERR_PTR(-EINVAL); - - if (mutex_trylock(&pool->mutex)) { - if (pool->high_count) - page = ion_page_pool_remove(pool, true); - else if (pool->low_count) - page = ion_page_pool_remove(pool, false); - mutex_unlock(&pool->mutex); - } - - if (!page) - return ERR_PTR(-ENOMEM); return page; } void ion_page_pool_free(struct ion_page_pool *pool, struct page *page) { + BUG_ON(pool->order != compound_order(page)); + ion_page_pool_add(pool, page); } -void ion_page_pool_free_immediate(struct ion_page_pool *pool, struct page *page) -{ - ion_page_pool_free_pages(pool, page); -} - -int ion_page_pool_total(struct ion_page_pool *pool, bool high) +static int ion_page_pool_total(struct ion_page_pool *pool, bool high) { int count = pool->low_count; @@ -226,21 +131,20 @@ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, return freed; } -struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order, - bool cached) +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) { - struct ion_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); + struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); if (!pool) return NULL; + pool->high_count = 0; + pool->low_count = 0; INIT_LIST_HEAD(&pool->low_items); INIT_LIST_HEAD(&pool->high_items); - pool->gfp_mask = gfp_mask; + pool->gfp_mask = gfp_mask | __GFP_COMP; pool->order = order; mutex_init(&pool->mutex); plist_node_init(&pool->list, order); - if (cached) - pool->cached = true; return pool; } diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.h b/drivers/staging/android/ion/heaps/ion_page_pool.h index dfe3f8b59426..e205d0086833 100644 --- a/drivers/staging/android/ion/heaps/ion_page_pool.h +++ b/drivers/staging/android/ion/heaps/ion_page_pool.h @@ -13,19 +13,6 @@ #include #include -/* ION page pool marks in bytes */ -#ifdef CONFIG_ION_POOL_AUTO_REFILL -#define ION_POOL_FILL_MARK (CONFIG_ION_POOL_FILL_MARK * SZ_1M) -#define POOL_LOW_MARK_PERCENT 40UL -#define ION_POOL_LOW_MARK ((ION_POOL_FILL_MARK * POOL_LOW_MARK_PERCENT) / 100) -#else -#define ION_POOL_FILL_MARK 0UL -#define ION_POOL_LOW_MARK 0UL -#endif - -/* if low watermark of zones have reached, defer the refill in this window */ -#define ION_POOL_REFILL_DEFER_WINDOW_MS 10 - /** * functions for creating and destroying a heap pool -- allows you * to keep a pool of pre allocated memory to use from your heap. Keeping @@ -38,18 +25,13 @@ * struct ion_page_pool - pagepool struct * @high_count: number of highmem items in the pool * @low_count: number of lowmem items in the pool - * @count: total number of pages/items in the pool * @high_items: list of highmem items * @low_items: list of lowmem items - * @last_low_watermark_ktime: most recent time at which the zone watermarks were - * low * @mutex: lock protecting this struct and especially the count * item list * @gfp_mask: gfp_mask to use from alloc * @order: order of pages in the pool * @list: plist node for list of pools - * @cached: it's cached pool or not - * @heap_dev: device for the ion heap associated with this pool * * Allows you to keep a pool of pre allocated pages to use from your heap. * Keeping a pool of pages that is ready for dma, ie any cached mapping have @@ -59,28 +41,18 @@ struct ion_page_pool { int high_count; int low_count; - atomic_t count; struct list_head high_items; struct list_head low_items; - ktime_t last_low_watermark_ktime; struct mutex mutex; gfp_t gfp_mask; unsigned int order; struct plist_node list; - bool cached; - struct device *heap_dev; }; -struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order, - bool cached); +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order); void ion_page_pool_destroy(struct ion_page_pool *pool); -struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool); +struct page *ion_page_pool_alloc(struct ion_page_pool *pool); void ion_page_pool_free(struct ion_page_pool *pool, struct page *page); -struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *a); -void ion_page_pool_free_immediate(struct ion_page_pool *pool, - struct page *page); -int ion_page_pool_total(struct ion_page_pool *pool, bool high); -size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid); /** ion_page_pool_shrink - shrinks the size of the memory cached in the pool * @pool: the pool @@ -91,52 +63,4 @@ size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid); */ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, int nr_to_scan); - -#ifdef CONFIG_ION_POOL_AUTO_REFILL -void ion_page_pool_refill(struct ion_page_pool *pool); - -static __always_inline int get_pool_fillmark(struct ion_page_pool *pool) -{ - return ION_POOL_FILL_MARK / (PAGE_SIZE << pool->order); -} - -static __always_inline int get_pool_lowmark(struct ion_page_pool *pool) -{ - return ION_POOL_LOW_MARK / (PAGE_SIZE << pool->order); -} - -static __always_inline bool pool_count_below_lowmark(struct ion_page_pool *pool) -{ - return atomic_read(&pool->count) < get_pool_lowmark(pool); -} - -static __always_inline bool pool_fillmark_reached(struct ion_page_pool *pool) -{ - return atomic_read(&pool->count) >= get_pool_fillmark(pool); -} -#else -static inline void ion_page_pool_refill(struct ion_page_pool *pool) -{ -} - -static __always_inline int get_pool_fillmark(struct ion_page_pool *pool) -{ - return 0; -} - -static __always_inline int get_pool_lowmark(struct ion_page_pool *pool) -{ - return 0; -} - -static __always_inline bool pool_count_below_lowmark(struct ion_page_pool *pool) -{ - return false; -} - -static __always_inline bool pool_fillmark_reached(struct ion_page_pool *pool) -{ - return false; -} -#endif /* CONFIG_ION_POOL_AUTO_REFILL */ #endif /* _ION_PAGE_POOL_H */ diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c index 5c4d7833dcbc..6052b843cdeb 100644 --- a/drivers/staging/android/ion/heaps/ion_system_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -3,8 +3,6 @@ * ION Memory Allocator system heap exporter * * Copyright (C) 2011 Google, Inc. - * Copyright (c) 2011-2020, The Linux Foundation. All rights reserved. - * */ #include @@ -17,25 +15,17 @@ #include #include #include -#include -#include -#include -#include -#include "ion_system_heap.h" + #include "ion_page_pool.h" -#include "msm_ion_priv.h" -#include "ion_system_heap.h" -#include "ion_system_secure_heap.h" -#include "ion_secure_util.h" + +#define NUM_ORDERS ARRAY_SIZE(orders) static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_RECLAIM; static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO; +static const unsigned int orders[] = {8, 4, 0}; -bool pool_auto_refill_en __read_mostly = - IS_ENABLED(CONFIG_ION_POOL_AUTO_REFILL); - -int order_to_index(unsigned int order) +static int order_to_index(unsigned int order) { int i; @@ -51,175 +41,44 @@ static inline unsigned int order_to_size(int order) return PAGE_SIZE << order; } -int ion_heap_is_system_heap_type(enum ion_heap_type type) -{ - return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM); -} +struct ion_system_heap { + struct ion_heap heap; + struct ion_page_pool *pools[NUM_ORDERS]; +}; -static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, +static struct page *alloc_buffer_page(struct ion_system_heap *heap, struct ion_buffer *buffer, - unsigned long order, - bool *from_pool) + unsigned long order) +{ + struct ion_page_pool *pool = heap->pools[order_to_index(order)]; + + return ion_page_pool_alloc(pool); +} + +static void free_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, struct page *page) { - int cached = (int)ion_buffer_cached(buffer); - struct page *page; struct ion_page_pool *pool; - int vmid = get_secure_vmid(buffer->flags); - struct device *dev = sys_heap->heap.dev; - int order_ind = order_to_index(order); - struct task_struct *worker; + unsigned int order = compound_order(page); - if (vmid > 0) { - pool = sys_heap->secure_pools[vmid][order_ind]; - - /* - * We should skip stealing pages if (1) we're focing our - * allocations to come from buddy; or (2) pool refilling is - * disabled, in which case stealing pages could deplete the - * uncached pools. - */ - if (!(*from_pool && pool_auto_refill_en)) - goto normal_alloc; - - page = ion_page_pool_alloc_pool_only(pool); - if (!IS_ERR(page)) - return page; - - pool = sys_heap->uncached_pools[order_ind]; - page = ion_page_pool_alloc_pool_only(pool); - if (IS_ERR(page)) { - pool = sys_heap->secure_pools[vmid][order_ind]; - goto normal_alloc; - } - - /* - * Here, setting `from_pool = false` indicates that the - * page didn't come from the secure pool, and causes - * the page to be hyp-assigned. - */ - *from_pool = false; - - if (pool_auto_refill_en && pool->order && - pool_count_below_lowmark(pool)) { - worker = sys_heap->kworker[ION_KTHREAD_UNCACHED]; - wake_up_process(worker); - } - return page; - } else if (!cached) { - pool = sys_heap->uncached_pools[order_ind]; - } else { - pool = sys_heap->cached_pools[order_ind]; - } - -normal_alloc: - page = ion_page_pool_alloc(pool, from_pool); - - if (pool_auto_refill_en && pool->order && - pool_count_below_lowmark(pool) && vmid <= 0) - wake_up_process(sys_heap->kworker[cached]); - - if (IS_ERR(page)) - return page; - - if ((MAKE_ION_ALLOC_DMA_READY && vmid <= 0) || !(*from_pool)) - ion_pages_sync_for_device(dev, page, PAGE_SIZE << order, - DMA_BIDIRECTIONAL); - - return page; -} - -/* - * For secure pages that need to be freed and not added back to the pool; the - * hyp_unassign should be called before calling this function - */ -void free_buffer_page(struct ion_system_heap *heap, - struct ion_buffer *buffer, struct page *page, - unsigned int order) -{ - bool cached = ion_buffer_cached(buffer); - int vmid = get_secure_vmid(buffer->flags); - - if (!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { - struct ion_page_pool *pool; - - if (vmid > 0) - pool = heap->secure_pools[vmid][order_to_index(order)]; - else if (cached) - pool = heap->cached_pools[order_to_index(order)]; - else - pool = heap->uncached_pools[order_to_index(order)]; - - if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) - ion_page_pool_free_immediate(pool, page); - else - ion_page_pool_free(pool, page); - -#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES - mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES, - -(1 << pool->order)); -#endif - } else { + /* go to system */ + if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) { __free_pages(page, order); - -#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES - mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES, - -(1 << order)); -#endif - + return; } + + pool = heap->pools[order_to_index(order)]; + + ion_page_pool_free(pool, page); } -static struct page_info *alloc_largest_available(struct ion_system_heap *heap, - struct ion_buffer *buffer, - unsigned long size, - unsigned int max_order) +static struct page *alloc_largest_available(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) { struct page *page; - struct page_info *info; int i; - bool from_pool; - - info = kmalloc(sizeof(*info), GFP_KERNEL); - if (!info) - return ERR_PTR(-ENOMEM); - - for (i = 0; i < NUM_ORDERS; i++) { - if (size < order_to_size(orders[i])) - continue; - if (max_order < orders[i]) - continue; - from_pool = !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC); - page = alloc_buffer_page(heap, buffer, orders[i], &from_pool); - if (IS_ERR(page)) - continue; - - info->page = page; - info->order = orders[i]; - info->from_pool = from_pool; - INIT_LIST_HEAD(&info->list); - return info; - } - kfree(info); - - return ERR_PTR(-ENOMEM); -} - -static struct page_info * -alloc_from_pool_preferred(struct ion_system_heap *heap, - struct ion_buffer *buffer, - unsigned long size, - unsigned int max_order) -{ - struct page *page; - struct page_info *info; - int i; - - if (buffer->flags & ION_FLAG_POOL_FORCE_ALLOC) - goto force_alloc; - - info = kmalloc(sizeof(*info), GFP_KERNEL); - if (!info) - return ERR_PTR(-ENOMEM); for (i = 0; i < NUM_ORDERS; i++) { if (size < order_to_size(orders[i])) @@ -227,51 +86,14 @@ alloc_from_pool_preferred(struct ion_system_heap *heap, if (max_order < orders[i]) continue; - page = alloc_from_secure_pool_order(heap, buffer, orders[i]); - if (IS_ERR(page)) + page = alloc_buffer_page(heap, buffer, orders[i]); + if (!page) continue; - info->page = page; - info->order = orders[i]; - info->from_pool = true; - INIT_LIST_HEAD(&info->list); - return info; + return page; } - page = split_page_from_secure_pool(heap, buffer); - if (!IS_ERR(page)) { - info->page = page; - info->order = 0; - info->from_pool = true; - INIT_LIST_HEAD(&info->list); - return info; - } - - kfree(info); -force_alloc: - return alloc_largest_available(heap, buffer, size, max_order); -} - -static void process_info(struct page_info *info, - struct scatterlist *sg, - struct scatterlist *sg_sync) -{ - struct page *page = info->page; - - if (sg_sync) { - sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0); - sg_dma_address(sg_sync) = page_to_phys(page); - } - sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0); - /* - * This is not correct - sg_dma_address needs a dma_addr_t - * that is valid for the the targeted device, but this works - * on the currently targeted hardware. - */ - sg_dma_address(sg) = page_to_phys(page); - - list_del(&info->list); - kfree(info); + return NULL; } static int ion_system_heap_allocate(struct ion_heap *heap, @@ -279,508 +101,176 @@ static int ion_system_heap_allocate(struct ion_heap *heap, unsigned long size, unsigned long flags) { - struct ion_system_heap *sys_heap = to_system_heap(heap); - struct msm_ion_buf_lock_state *lock_state; + struct ion_system_heap *sys_heap = container_of(heap, + struct ion_system_heap, + heap); struct sg_table *table; - struct sg_table table_sync = {0}; struct scatterlist *sg; - struct scatterlist *sg_sync; - int ret = -ENOMEM; struct list_head pages; - struct list_head pages_from_pool; - struct page_info *info, *tmp_info; + struct page *page, *tmp_page; int i = 0; - unsigned int nents_sync = 0; unsigned long size_remaining = PAGE_ALIGN(size); unsigned int max_order = orders[0]; - unsigned int sz; - int vmid = get_secure_vmid(buffer->flags); if (size / PAGE_SIZE > totalram_pages() / 2) return -ENOMEM; - if (ion_heap_is_system_heap_type(buffer->heap->type) && - is_secure_allocation(buffer->flags)) { - pr_info("%s: System heap doesn't support secure allocations\n", - __func__); - return -EINVAL; - } - INIT_LIST_HEAD(&pages); - INIT_LIST_HEAD(&pages_from_pool); - while (size_remaining > 0) { - if (is_secure_vmid_valid(vmid)) - info = alloc_from_pool_preferred(sys_heap, buffer, - size_remaining, - max_order); - else - info = alloc_largest_available(sys_heap, buffer, - size_remaining, - max_order); - - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto err; - } - - sz = (1 << info->order) * PAGE_SIZE; - -#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES - mod_node_page_state(page_pgdat(info->page), - NR_UNRECLAIMABLE_PAGES, - (1 << (info->order))); -#endif - - if (info->from_pool) { - list_add_tail(&info->list, &pages_from_pool); - } else { - list_add_tail(&info->list, &pages); - ++nents_sync; - } - size_remaining -= sz; - max_order = info->order; + page = alloc_largest_available(sys_heap, buffer, size_remaining, + max_order); + if (!page) + goto free_pages; + list_add_tail(&page->lru, &pages); + size_remaining -= page_size(page); + max_order = compound_order(page); i++; } + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + goto free_pages; - table = kzalloc(sizeof(*table), GFP_KERNEL); - if (!table) { - ret = -ENOMEM; - goto err; - } - - ret = sg_alloc_table(table, i, GFP_KERNEL); - if (ret) - goto err1; - - if (nents_sync) { - ret = sg_alloc_table(&table_sync, nents_sync, GFP_KERNEL); - if (ret) - goto err_free_sg; - } + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_table; sg = table->sgl; - sg_sync = table_sync.sgl; - - /* - * We now have two separate lists. One list contains pages from the - * pool and the other pages from buddy. We want to merge these - * together while preserving the ordering of the pages (higher order - * first). - */ - do { - info = list_first_entry_or_null(&pages, struct page_info, list); - tmp_info = list_first_entry_or_null(&pages_from_pool, - struct page_info, list); - if (info && tmp_info) { - if (info->order >= tmp_info->order) { - process_info(info, sg, sg_sync); - sg_sync = sg_next(sg_sync); - } else { - process_info(tmp_info, sg, NULL); - } - } else if (info) { - process_info(info, sg, sg_sync); - sg_sync = sg_next(sg_sync); - } else if (tmp_info) { - process_info(tmp_info, sg, NULL); - } + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); sg = sg_next(sg); - - } while (sg); - - if (nents_sync) { - if (vmid > 0) { - ret = ion_hyp_assign_sg(&table_sync, &vmid, 1, true); - if (ret) - goto err_free_sg2; - } + list_del(&page->lru); } buffer->sg_table = table; - if (nents_sync) - sg_free_table(&table_sync); - lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL); - if (!lock_state) { - ret = -ENOMEM; - goto err_free_sg2; - } - buffer->priv_virt = lock_state; + ion_buffer_prep_noncached(buffer); - ion_prepare_sgl_for_force_dma_sync(buffer->sg_table); return 0; -err_free_sg2: - /* We failed to zero buffers. Bypass pool */ - buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE; - - if (vmid > 0) - if (ion_hyp_unassign_sg(table, &vmid, 1, true)) - goto err_free_table_sync; - - for_each_sg(table->sgl, sg, table->nents, i) - free_buffer_page(sys_heap, buffer, sg_page(sg), - get_order(sg->length)); -err_free_table_sync: - if (nents_sync) - sg_free_table(&table_sync); -err_free_sg: - sg_free_table(table); -err1: +free_table: kfree(table); -err: - list_for_each_entry_safe(info, tmp_info, &pages, list) { - free_buffer_page(sys_heap, buffer, info->page, info->order); - kfree(info); - } - list_for_each_entry_safe(info, tmp_info, &pages_from_pool, list) { - free_buffer_page(sys_heap, buffer, info->page, info->order); - kfree(info); - } - return ret; +free_pages: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + free_buffer_page(sys_heap, buffer, page); + return -ENOMEM; } static void ion_system_heap_free(struct ion_buffer *buffer) { - struct ion_heap *heap = buffer->heap; - struct ion_system_heap *sys_heap = to_system_heap(heap); - struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt; + struct ion_system_heap *sys_heap = container_of(buffer->heap, + struct ion_system_heap, + heap); struct sg_table *table = buffer->sg_table; struct scatterlist *sg; int i; - int vmid = get_secure_vmid(buffer->flags); - if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) && - !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) { - mutex_lock(&buffer->lock); - if (hlos_accessible_buffer(buffer)) - ion_buffer_zero(buffer); - - if (lock_state && lock_state->locked) - pr_warn("%s: buffer is locked while being freed\n", - __func__); - mutex_unlock(&buffer->lock); - } else if (vmid > 0) { - if (ion_hyp_unassign_sg(table, &vmid, 1, true)) - return; - } + /* zero the buffer before goto page pool */ + if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) + ion_buffer_zero(buffer); for_each_sg(table->sgl, sg, table->nents, i) - free_buffer_page(sys_heap, buffer, sg_page(sg), - get_order(sg->length)); + free_buffer_page(sys_heap, buffer, sg_page(sg)); sg_free_table(table); kfree(table); - kfree(buffer->priv_virt); } static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, - int nr_to_scan) + int nr_to_scan) { + struct ion_page_pool *pool; struct ion_system_heap *sys_heap; int nr_total = 0; - int i, j, nr_freed = 0; + int i, nr_freed; int only_scan = 0; - struct ion_page_pool *pool; - sys_heap = to_system_heap(heap); + sys_heap = container_of(heap, struct ion_system_heap, heap); if (!nr_to_scan) only_scan = 1; - /* shrink the pools starting from lower order ones */ - for (i = NUM_ORDERS - 1; i >= 0; i--) { - nr_freed = 0; + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->pools[i]; - for (j = 0; j < VMID_LAST; j++) { - if (is_secure_vmid_valid(j)) - nr_freed += - ion_secure_page_pool_shrink(sys_heap, - j, i, - nr_to_scan); - } + if (only_scan) { + nr_total += ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); - pool = sys_heap->uncached_pools[i]; - nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan); - - pool = sys_heap->cached_pools[i]; - nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan); - nr_total += nr_freed; - - if (!only_scan) { + } else { + nr_freed = ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); nr_to_scan -= nr_freed; - /* shrink completed */ + nr_total += nr_freed; if (nr_to_scan <= 0) break; } } - return nr_total; } +static void ion_system_heap_destroy_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (pools[i]) + ion_page_pool_destroy(pools[i]); +} + +static int ion_system_heap_create_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + struct ion_page_pool *pool; + gfp_t gfp_flags = low_order_gfp_flags; + + if (orders[i] > 4) + gfp_flags = high_order_gfp_flags; + + pool = ion_page_pool_create(gfp_flags, orders[i]); + if (!pool) + goto err_create_pool; + pools[i] = pool; + } + + return 0; + +err_create_pool: + ion_system_heap_destroy_pools(pools); + return -ENOMEM; +} + static struct ion_heap_ops system_heap_ops = { .allocate = ion_system_heap_allocate, .free = ion_system_heap_free, .shrink = ion_system_heap_shrink, }; -static int ion_system_heap_debug_show(struct ion_heap *heap, - struct seq_file *s, void *unused) -{ - struct ion_system_heap *sys_heap; - bool use_seq = s; - unsigned long uncached_total = 0; - unsigned long cached_total = 0; - unsigned long secure_total = 0; - struct ion_page_pool *pool; - int i, j; - - sys_heap = to_system_heap(heap); - for (i = 0; i < NUM_ORDERS; i++) { - pool = sys_heap->uncached_pools[i]; - if (use_seq) { - seq_printf(s, - "%d order %u highmem pages in uncached pool = %lu total\n", - pool->high_count, pool->order, - (1 << pool->order) * PAGE_SIZE * - pool->high_count); - seq_printf(s, - "%d order %u lowmem pages in uncached pool = %lu total\n", - pool->low_count, pool->order, - (1 << pool->order) * PAGE_SIZE * - pool->low_count); - } - - uncached_total += (1 << pool->order) * PAGE_SIZE * - pool->high_count; - uncached_total += (1 << pool->order) * PAGE_SIZE * - pool->low_count; +static struct ion_system_heap system_heap = { + .heap = { + .ops = &system_heap_ops, + .type = ION_HEAP_TYPE_SYSTEM, + .flags = ION_HEAP_FLAG_DEFER_FREE, + .name = "ion_system_heap", } - - for (i = 0; i < NUM_ORDERS; i++) { - pool = sys_heap->cached_pools[i]; - if (use_seq) { - seq_printf(s, - "%d order %u highmem pages in cached pool = %lu total\n", - pool->high_count, pool->order, - (1 << pool->order) * PAGE_SIZE * - pool->high_count); - seq_printf(s, - "%d order %u lowmem pages in cached pool = %lu total\n", - pool->low_count, pool->order, - (1 << pool->order) * PAGE_SIZE * - pool->low_count); - } - - cached_total += (1 << pool->order) * PAGE_SIZE * - pool->high_count; - cached_total += (1 << pool->order) * PAGE_SIZE * - pool->low_count; - } - - for (i = 0; i < NUM_ORDERS; i++) { - for (j = 0; j < VMID_LAST; j++) { - if (!is_secure_vmid_valid(j)) - continue; - pool = sys_heap->secure_pools[j][i]; - - if (use_seq) { - seq_printf(s, - "VMID %d: %d order %u highmem pages in secure pool = %lu total\n", - j, pool->high_count, pool->order, - (1 << pool->order) * PAGE_SIZE * - pool->high_count); - seq_printf(s, - "VMID %d: %d order %u lowmem pages in secure pool = %lu total\n", - j, pool->low_count, pool->order, - (1 << pool->order) * PAGE_SIZE * - pool->low_count); - } - - secure_total += (1 << pool->order) * PAGE_SIZE * - pool->high_count; - secure_total += (1 << pool->order) * PAGE_SIZE * - pool->low_count; - } - } - - if (use_seq) { - seq_puts(s, "--------------------------------------------\n"); - seq_printf(s, "uncached pool = %lu cached pool = %lu secure pool = %lu\n", - uncached_total, cached_total, secure_total); - seq_printf(s, "pool total (uncached + cached + secure) = %lu\n", - uncached_total + cached_total + secure_total); - seq_puts(s, "--------------------------------------------\n"); - } else { - pr_info("-------------------------------------------------\n"); - pr_info("uncached pool = %lu cached pool = %lu secure pool = %lu\n", - uncached_total, cached_total, secure_total); - pr_info("pool total (uncached + cached + secure) = %lu\n", - uncached_total + cached_total + secure_total); - pr_info("-------------------------------------------------\n"); - } - - return 0; -} - -static struct msm_ion_heap_ops msm_system_heap_ops = { - .debug_show = ion_system_heap_debug_show, }; -static void ion_system_heap_destroy_pools(struct ion_page_pool **pools) +static int __init ion_system_heap_init(void) { - int i; + int ret = ion_system_heap_create_pools(system_heap.pools); + if (ret) + return ret; - for (i = 0; i < NUM_ORDERS; i++) - if (pools[i]) { - ion_page_pool_destroy(pools[i]); - pools[i] = NULL; - } + return ion_device_add_heap(&system_heap.heap); } -/** - * ion_system_heap_create_pools - Creates pools for all orders - * - * If this fails you don't need to destroy any pools. It's all or - * nothing. If it succeeds you'll eventually need to use - * ion_system_heap_destroy_pools to destroy the pools. - */ -static int ion_system_heap_create_pools(struct ion_system_heap *sys_heap, - struct ion_page_pool **pools, - bool cached) +static void __exit ion_system_heap_exit(void) { - int i; - - for (i = 0; i < NUM_ORDERS; i++) { - struct ion_page_pool *pool; - gfp_t gfp_flags = low_order_gfp_flags; - - if (orders[i]) - gfp_flags = high_order_gfp_flags; - pool = ion_page_pool_create(gfp_flags, orders[i], cached); - if (!pool) - goto err_create_pool; - pool->heap_dev = sys_heap->heap.dev; - pools[i] = pool; - } - - return 0; -err_create_pool: - ion_system_heap_destroy_pools(pools); - return -ENOMEM; -} - -static int ion_sys_heap_worker(void *data) -{ - struct ion_page_pool **pools = (struct ion_page_pool **)data; - int i; - - for (;;) { - for (i = 0; i < NUM_ORDERS; i++) { - if (pool_count_below_lowmark(pools[i])) - ion_page_pool_refill(pools[i]); - } - set_current_state(TASK_INTERRUPTIBLE); - if (unlikely(kthread_should_stop())) { - set_current_state(TASK_RUNNING); - break; - } - schedule(); - - set_current_state(TASK_RUNNING); - } - - return 0; -} - -static struct task_struct *ion_create_kworker(struct ion_page_pool **pools, - bool cached) -{ - struct sched_attr attr = { 0 }; - struct task_struct *thread; - int ret; - char *buf; - - attr.sched_nice = ION_KTHREAD_NICE_VAL; - buf = cached ? "cached" : "uncached"; - - thread = kthread_run(ion_sys_heap_worker, pools, - "ion-pool-%s-worker", buf); - if (IS_ERR(thread)) { - pr_err("%s: failed to create %s worker thread: %ld\n", - __func__, buf, PTR_ERR(thread)); - return thread; - } - ret = sched_setattr(thread, &attr); - if (ret) { - kthread_stop(thread); - pr_warn("%s: failed to set task priority for %s worker thread: ret = %d\n", - __func__, buf, ret); - return ERR_PTR(ret); - } - - return thread; -} - -struct ion_heap *ion_system_heap_create(struct ion_platform_heap *data) -{ - struct ion_system_heap *heap; - int ret = -ENOMEM; - int i; - - heap = kzalloc(sizeof(*heap), GFP_KERNEL); - if (!heap) - return ERR_PTR(-ENOMEM); - heap->heap.dev = data->priv; - heap->heap.msm_heap_ops = &msm_system_heap_ops; - heap->heap.ion_heap.ops = &system_heap_ops; - heap->heap.ion_heap.buf_ops = msm_ion_dma_buf_ops; - heap->heap.ion_heap.type = ION_HEAP_TYPE_SYSTEM; - heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE; - - for (i = 0; i < VMID_LAST; i++) - if (is_secure_vmid_valid(i)) - if (ion_system_heap_create_pools(heap, - heap->secure_pools[i], - false)) - goto destroy_secure_pools; - - if (ion_system_heap_create_pools(heap, heap->uncached_pools, false)) - goto destroy_secure_pools; - - if (ion_system_heap_create_pools(heap, heap->cached_pools, true)) - goto destroy_uncached_pools; - - if (pool_auto_refill_en) { - heap->kworker[ION_KTHREAD_UNCACHED] = - ion_create_kworker(heap->uncached_pools, false); - if (IS_ERR(heap->kworker[ION_KTHREAD_UNCACHED])) { - ret = PTR_ERR(heap->kworker[ION_KTHREAD_UNCACHED]); - goto destroy_pools; - } - heap->kworker[ION_KTHREAD_CACHED] = - ion_create_kworker(heap->cached_pools, true); - if (IS_ERR(heap->kworker[ION_KTHREAD_CACHED])) { - kthread_stop(heap->kworker[ION_KTHREAD_UNCACHED]); - ret = PTR_ERR(heap->kworker[ION_KTHREAD_CACHED]); - goto destroy_pools; - } - } - - mutex_init(&heap->split_page_mutex); - - return &heap->heap.ion_heap; -destroy_pools: - ion_system_heap_destroy_pools(heap->cached_pools); -destroy_uncached_pools: - ion_system_heap_destroy_pools(heap->uncached_pools); -destroy_secure_pools: - for (i = 0; i < VMID_LAST; i++) { - if (heap->secure_pools[i]) - ion_system_heap_destroy_pools(heap->secure_pools[i]); - } - kfree(heap); - return ERR_PTR(ret); + ion_device_remove_heap(&system_heap.heap); + ion_system_heap_destroy_pools(system_heap.pools); } +module_init(ion_system_heap_init); +module_exit(ion_system_heap_exit); MODULE_LICENSE("GPL v2"); diff --git a/drivers/staging/android/ion/heaps/ion_system_secure_heap.c b/drivers/staging/android/ion/heaps/ion_system_secure_heap.c index 7b252d7ad169..470bc53a2bb3 100644 --- a/drivers/staging/android/ion/heaps/ion_system_secure_heap.c +++ b/drivers/staging/android/ion/heaps/ion_system_secure_heap.c @@ -10,8 +10,8 @@ #include #include "ion_system_secure_heap.h" -#include "ion_system_heap.h" -#include "ion_page_pool.h" +#include "ion_msm_system_heap.h" +#include "ion_msm_page_pool.h" #include "msm_ion_priv.h" #include "ion_secure_util.h" @@ -146,19 +146,19 @@ out1: size_t ion_system_secure_heap_page_pool_total(struct ion_heap *heap, int vmid_flags) { - struct ion_system_heap *sys_heap; - struct ion_page_pool *pool; + struct ion_msm_system_heap *sys_heap; + struct ion_msm_page_pool *pool; size_t total = 0; int vmid, i; - sys_heap = to_system_heap(heap); + sys_heap = to_msm_system_heap(heap); vmid = get_secure_vmid(vmid_flags); if (vmid < 0) return 0; for (i = 0; i < NUM_ORDERS; i++) { pool = sys_heap->secure_pools[vmid][i]; - total += ion_page_pool_total(pool, true); + total += ion_msm_page_pool_total(pool, true); } return total << PAGE_SHIFT; @@ -342,21 +342,21 @@ struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *unused) return &heap->heap.ion_heap; } -struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap, +struct page *alloc_from_secure_pool_order(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, unsigned long order) { int vmid = get_secure_vmid(buffer->flags); - struct ion_page_pool *pool; + struct ion_msm_page_pool *pool; if (!is_secure_vmid_valid(vmid)) return ERR_PTR(-EINVAL); pool = heap->secure_pools[vmid][order_to_index(order)]; - return ion_page_pool_alloc_pool_only(pool); + return ion_msm_page_pool_alloc_pool_only(pool); } -struct page *split_page_from_secure_pool(struct ion_system_heap *heap, +struct page *split_page_from_secure_pool(struct ion_msm_system_heap *heap, struct ion_buffer *buffer) { int i, j; @@ -400,7 +400,7 @@ got_page: return page; } -int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, +int ion_secure_page_pool_shrink(struct ion_msm_system_heap *sys_heap, int vmid, int order_idx, int nr_to_scan) { int ret, freed = 0; @@ -408,14 +408,15 @@ int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, struct page *page, *tmp; struct sg_table sgt; struct scatterlist *sg; - struct ion_page_pool *pool = sys_heap->secure_pools[vmid][order_idx]; + struct ion_msm_page_pool *pool = + sys_heap->secure_pools[vmid][order_idx]; LIST_HEAD(pages); if (nr_to_scan == 0) - return ion_page_pool_total(pool, true); + return ion_msm_page_pool_total(pool, true); while (freed < nr_to_scan) { - page = ion_page_pool_alloc_pool_only(pool); + page = ion_msm_page_pool_alloc_pool_only(pool); if (IS_ERR(page)) break; list_add(&page->lru, &pages); @@ -443,7 +444,7 @@ int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, list_for_each_entry_safe(page, tmp, &pages, lru) { list_del(&page->lru); - ion_page_pool_free_immediate(pool, page); + ion_msm_page_pool_free_immediate(pool, page); } sg_free_table(&sgt); @@ -455,7 +456,7 @@ out1: /* Restore pages to secure pool */ list_for_each_entry_safe(page, tmp, &pages, lru) { list_del(&page->lru); - ion_page_pool_free(pool, page); + ion_msm_page_pool_free(pool, page); } return 0; out3: diff --git a/drivers/staging/android/ion/heaps/ion_system_secure_heap.h b/drivers/staging/android/ion/heaps/ion_system_secure_heap.h index fdeee71ac5fc..9e808b397fde 100644 --- a/drivers/staging/android/ion/heaps/ion_system_secure_heap.h +++ b/drivers/staging/android/ion/heaps/ion_system_secure_heap.h @@ -1,21 +1,21 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. */ #include "msm_ion_priv.h" -#include "ion_system_heap.h" +#include "ion_msm_system_heap.h" #ifndef _ION_SYSTEM_SECURE_HEAP_H #define _ION_SYSTEM_SECURE_HEAP_H -struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap, +struct page *alloc_from_secure_pool_order(struct ion_msm_system_heap *heap, struct ion_buffer *buffer, unsigned long order); -struct page *split_page_from_secure_pool(struct ion_system_heap *heap, +struct page *split_page_from_secure_pool(struct ion_msm_system_heap *heap, struct ion_buffer *buffer); -int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap, +int ion_secure_page_pool_shrink(struct ion_msm_system_heap *sys_heap, int vmid, int order_idx, int nr_to_scan); #endif /* _ION_SYSTEM_SECURE_HEAP_H */ diff --git a/drivers/staging/android/ion/heaps/msm_ion.c b/drivers/staging/android/ion/heaps/msm_ion.c index 78f08907f378..2bed708ab61c 100644 --- a/drivers/staging/android/ion/heaps/msm_ion.c +++ b/drivers/staging/android/ion/heaps/msm_ion.c @@ -100,7 +100,7 @@ static struct heap_types_info { const char *name; int heap_type; } heap_types_info[] = { - MAKE_HEAP_TYPE_MAPPING(SYSTEM), + MAKE_HEAP_TYPE_MAPPING(MSM_SYSTEM), MAKE_HEAP_TYPE_MAPPING(MSM_CARVEOUT), MAKE_HEAP_TYPE_MAPPING(SECURE_CARVEOUT), MAKE_HEAP_TYPE_MAPPING(DMA), @@ -158,8 +158,8 @@ static struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data) int heap_type = heap_data->type; switch (heap_type) { - case ION_HEAP_TYPE_SYSTEM: - heap = ion_system_heap_create(heap_data); + case ION_HEAP_TYPE_MSM_SYSTEM: + heap = ion_msm_system_heap_create(heap_data); break; case ION_HEAP_TYPE_MSM_CARVEOUT: heap = ion_carveout_heap_create(heap_data); diff --git a/drivers/staging/android/ion/heaps/msm_ion_priv.h b/drivers/staging/android/ion/heaps/msm_ion_priv.h index cf1231eb3478..66ea173e853c 100644 --- a/drivers/staging/android/ion/heaps/msm_ion_priv.h +++ b/drivers/staging/android/ion/heaps/msm_ion_priv.h @@ -156,7 +156,7 @@ bool ion_buffer_cached(struct ion_buffer *buffer); * heaps as appropriate. */ -struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused); +struct ion_heap *ion_msm_system_heap_create(struct ion_platform_heap *unused); struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *heap); diff --git a/drivers/staging/android/ion/ion_buffer.c b/drivers/staging/android/ion/ion_buffer.c index 12f180a84ea8..862da1a32551 100644 --- a/drivers/staging/android/ion/ion_buffer.c +++ b/drivers/staging/android/ion/ion_buffer.c @@ -12,6 +12,7 @@ #include #include "ion_private.h" +#define ION_SYSTEM_HEAP_ID BIT(25) /* this function should only be called while dev->lock is held */ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, @@ -115,11 +116,25 @@ struct ion_buffer *ion_buffer_alloc(struct ion_device *dev, size_t len, { struct ion_buffer *buffer = NULL; struct ion_heap *heap; + char task_comm[TASK_COMM_LEN]; if (!dev || !len) { return ERR_PTR(-EINVAL); } + /* + * Temporarily reroute generic system heap allocations to the MSM system + * heap. Once clients have stopped using the generic system heap ID, we + * can remove this. + */ + if (heap_id_mask & ION_HEAP_SYSTEM) { + get_task_comm(task_comm, current->group_leader); + pr_warn_ratelimited("%s: Rerouting allocation from generic sys heap to msm sys heap for %s-%d\n", + __func__, task_comm, current->tgid); + heap_id_mask &= ~ION_HEAP_SYSTEM; + heap_id_mask |= ION_SYSTEM_HEAP_ID; + } + /* * traverse the list of heaps available in this system in priority * order. If the heap type is supported by the client, and matches the diff --git a/include/uapi/linux/msm_ion.h b/include/uapi/linux/msm_ion.h index 46eef92343f0..e68fc5540567 100644 --- a/include/uapi/linux/msm_ion.h +++ b/include/uapi/linux/msm_ion.h @@ -22,6 +22,7 @@ enum msm_ion_heap_types { ION_HEAP_TYPE_HYP_CMA, ION_HEAP_TYPE_MSM_CARVEOUT, ION_HEAP_TYPE_SECURE_CARVEOUT, + ION_HEAP_TYPE_MSM_SYSTEM, }; /** diff --git a/include/uapi/linux/msm_ion_ids.h b/include/uapi/linux/msm_ion_ids.h index 9a36cae13063..857661963724 100644 --- a/include/uapi/linux/msm_ion_ids.h +++ b/include/uapi/linux/msm_ion_ids.h @@ -16,7 +16,7 @@ * possible fallbacks) */ -#define ION_SYSTEM_HEAP_ID ION_BIT(0) +/* ION_BIT(0) is reserved for the generic system heap. */ #define ION_QSECOM_TA_HEAP_ID ION_BIT(1) #define ION_CAMERA_HEAP_ID ION_BIT(2) #define ION_DISPLAY_HEAP_ID ION_BIT(3) @@ -31,6 +31,7 @@ #define ION_SPSS_HEAP_ID ION_BIT(14) #define ION_SECURE_CARVEOUT_HEAP_ID ION_BIT(15) #define ION_TUI_CARVEOUT_HEAP_ID ION_BIT(16) +#define ION_SYSTEM_HEAP_ID ION_BIT(25) #define ION_HEAP_ID_RESERVED ION_BIT(31) #endif /* _MSM_ION_IDS_H */