A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-10 14:19:09 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge "ion: Restore GKI system heap implementation"

Browse source
This commit is contained in:
qctecmdr 2020-07-15 18:03:32 -07:00 • committed by Gerrit - the friendly Code Review server
commit 8b21a243f7
15 changed files with 1380 additions and 848 deletions
Download patch file Download diff file Expand all files Collapse all files

8
drivers/staging/android/ion/heaps/Makefile
View file

@ -1,5 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_sys_heap.o
ion_sys_heap-y := ion_system_heap.o ion_page_pool.o
obj-$(CONFIG_ION_MSM_HEAPS) += msm_ion_heaps.o
msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_page_pool.o \
ion_system_heap.o ion_carveout_heap.o ion_system_secure_heap.o \
ion_cma_heap.o ion_secure_util.o
msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_msm_page_pool.o \
ion_msm_system_heap.o ion_carveout_heap.o \
ion_system_secure_heap.o ion_cma_heap.o ion_secure_util.o

257
drivers/staging/android/ion/heaps/ion_msm_page_pool.c Normal file
View file

@ -0,0 +1,257 @@
// SPDX-License-Identifier: GPL-2.0
/*
* ION Memory Allocator page pool helpers
*
* Copyright (C) 2011 Google, Inc.
*/
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/sched/signal.h>
#include "msm_ion_priv.h"
#include "ion_msm_page_pool.h"
static inline struct page
*ion_msm_page_pool_alloc_pages(struct ion_msm_page_pool *pool)
{
if (fatal_signal_pending(current))
return NULL;
return alloc_pages(pool->gfp_mask, pool->order);
}
static void ion_msm_page_pool_free_pages(struct ion_msm_page_pool *pool,
struct page *page)
{
__free_pages(page, pool->order);
}
static void ion_msm_page_pool_add(struct ion_msm_page_pool *pool,
struct page *page)
{
mutex_lock(&pool->mutex);
if (PageHighMem(page)) {
list_add_tail(&page->lru, &pool->high_items);
pool->high_count++;
} else {
list_add_tail(&page->lru, &pool->low_items);
pool->low_count++;
}
atomic_inc(&pool->count);
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
(1 << pool->order));
mutex_unlock(&pool->mutex);
}
#ifdef CONFIG_ION_POOL_AUTO_REFILL
/* do a simple check to see if we are in any low memory situation */
static bool pool_refill_ok(struct ion_msm_page_pool *pool)
{
struct zonelist *zonelist;
struct zoneref *z;
struct zone *zone;
int mark;
enum zone_type classzone_idx = gfp_zone(pool->gfp_mask);
s64 delta;
/* check if we are within the refill defer window */
delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime);
if (delta < ION_POOL_REFILL_DEFER_WINDOW_MS)
return false;
zonelist = node_zonelist(numa_node_id(), pool->gfp_mask);
/*
* make sure that if we allocate a pool->order page from buddy,
* we don't put the zone watermarks go below the high threshold.
* This makes sure there's no unwanted repetitive refilling and
* reclaiming of buddy pages on the pool.
*/
for_each_zone_zonelist(zone, z, zonelist, classzone_idx) {
mark = high_wmark_pages(zone);
mark += 1 << pool->order;
if (!zone_watermark_ok_safe(zone, pool->order, mark,
classzone_idx)) {
pool->last_low_watermark_ktime = ktime_get();
return false;
}
}
return true;
}
void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool)
{
struct page *page;
gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY;
struct device *dev = pool->heap_dev;
/* skip refilling order 0 pools */
if (!pool->order)
return;
while (!pool_fillmark_reached(pool) && pool_refill_ok(pool)) {
page = alloc_pages(gfp_refill, pool->order);
if (!page)
break;
if (!pool->cached)
ion_pages_sync_for_device(dev, page,
PAGE_SIZE << pool->order,
DMA_BIDIRECTIONAL);
ion_msm_page_pool_add(pool, page);
}
}
#endif /* CONFIG_ION_PAGE_POOL_REFILL */
static struct page *ion_msm_page_pool_remove(struct ion_msm_page_pool *pool,
bool high)
{
struct page *page;
if (high) {
BUG_ON(!pool->high_count);
page = list_first_entry(&pool->high_items, struct page, lru);
pool->high_count--;
} else {
BUG_ON(!pool->low_count);
page = list_first_entry(&pool->low_items, struct page, lru);
pool->low_count--;
}
atomic_dec(&pool->count);
list_del(&page->lru);
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
-(1 << pool->order));
return page;
}
struct page *ion_msm_page_pool_alloc(struct ion_msm_page_pool *pool,
bool *from_pool)
{
struct page *page = NULL;
BUG_ON(!pool);
if (fatal_signal_pending(current))
return ERR_PTR(-EINTR);
if (*from_pool && mutex_trylock(&pool->mutex)) {
if (pool->high_count)
page = ion_msm_page_pool_remove(pool, true);
else if (pool->low_count)
page = ion_msm_page_pool_remove(pool, false);
mutex_unlock(&pool->mutex);
}
if (!page) {
page = ion_msm_page_pool_alloc_pages(pool);
*from_pool = false;
}
if (!page)
return ERR_PTR(-ENOMEM);
return page;
}
/*
* Tries to allocate from only the specified Pool and returns NULL otherwise
*/
struct page *ion_msm_page_pool_alloc_pool_only(struct ion_msm_page_pool *pool)
{
struct page *page = NULL;
if (!pool)
return ERR_PTR(-EINVAL);
if (mutex_trylock(&pool->mutex)) {
if (pool->high_count)
page = ion_msm_page_pool_remove(pool, true);
else if (pool->low_count)
page = ion_msm_page_pool_remove(pool, false);
mutex_unlock(&pool->mutex);
}
if (!page)
return ERR_PTR(-ENOMEM);
return page;
}
void ion_msm_page_pool_free(struct ion_msm_page_pool *pool, struct page *page)
{
ion_msm_page_pool_add(pool, page);
}
void ion_msm_page_pool_free_immediate(struct ion_msm_page_pool *pool,
struct page *page)
{
ion_msm_page_pool_free_pages(pool, page);
}
int ion_msm_page_pool_total(struct ion_msm_page_pool *pool, bool high)
{
int count = pool->low_count;
if (high)
count += pool->high_count;
return count << pool->order;
}
int ion_msm_page_pool_shrink(struct ion_msm_page_pool *pool, gfp_t gfp_mask,
int nr_to_scan)
{
int freed = 0;
bool high;
if (current_is_kswapd())
high = true;
else
high = !!(gfp_mask & __GFP_HIGHMEM);
if (nr_to_scan == 0)
return ion_msm_page_pool_total(pool, high);
while (freed < nr_to_scan) {
struct page *page;
mutex_lock(&pool->mutex);
if (pool->low_count) {
page = ion_msm_page_pool_remove(pool, false);
} else if (high && pool->high_count) {
page = ion_msm_page_pool_remove(pool, true);
} else {
mutex_unlock(&pool->mutex);
break;
}
mutex_unlock(&pool->mutex);
ion_msm_page_pool_free_pages(pool, page);
freed += (1 << pool->order);
}
return freed;
}
struct ion_msm_page_pool *ion_msm_page_pool_create(gfp_t gfp_mask,
unsigned int order,
bool cached)
{
struct ion_msm_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
if (!pool)
return NULL;
INIT_LIST_HEAD(&pool->low_items);
INIT_LIST_HEAD(&pool->high_items);
pool->gfp_mask = gfp_mask;
pool->order = order;
mutex_init(&pool->mutex);
plist_node_init(&pool->list, order);
if (cached)
pool->cached = true;
return pool;
}
void ion_msm_page_pool_destroy(struct ion_msm_page_pool *pool)
{
kfree(pool);
}

148
drivers/staging/android/ion/heaps/ion_msm_page_pool.h Normal file
View file

@ -0,0 +1,148 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* ION Page Pool kernel interface header
*
* Copyright (C) 2011 Google, Inc.
*/
#ifndef _ION_MSM_PAGE_POOL_H
#define _ION_MSM_PAGE_POOL_H
#include <linux/mm_types.h>
#include <linux/mutex.h>
#include <linux/shrinker.h>
#include <linux/types.h>
/* ION page pool marks in bytes */
#ifdef CONFIG_ION_POOL_AUTO_REFILL
#define ION_POOL_FILL_MARK (CONFIG_ION_POOL_FILL_MARK * SZ_1M)
#define POOL_LOW_MARK_PERCENT 40UL
#define ION_POOL_LOW_MARK ((ION_POOL_FILL_MARK * POOL_LOW_MARK_PERCENT) / 100)
#else
#define ION_POOL_FILL_MARK 0UL
#define ION_POOL_LOW_MARK 0UL
#endif
/* if low watermark of zones have reached, defer the refill in this window */
#define ION_POOL_REFILL_DEFER_WINDOW_MS 10
/**
* functions for creating and destroying a heap pool -- allows you
* to keep a pool of pre allocated memory to use from your heap. Keeping
* a pool of memory that is ready for dma, ie any cached mapping have been
* invalidated from the cache, provides a significant performance benefit on
* many systems
*/
/**
* struct ion_msm_page_pool - pagepool struct
* @high_count: number of highmem items in the pool
* @low_count: number of lowmem items in the pool
* @count: total number of pages/items in the pool
* @high_items: list of highmem items
* @low_items: list of lowmem items
* @last_low_watermark_ktime: most recent time at which the zone watermarks were
* low
* @mutex: lock protecting this struct and especially the count
* item list
* @gfp_mask: gfp_mask to use from alloc
* @order: order of pages in the pool
* @list: plist node for list of pools
* @cached: it's cached pool or not
* @heap_dev: device for the ion heap associated with this pool
*
* Allows you to keep a pool of pre allocated pages to use from your heap.
* Keeping a pool of pages that is ready for dma, ie any cached mapping have
* been invalidated from the cache, provides a significant performance benefit
* on many systems
*/
struct ion_msm_page_pool {
int high_count;
int low_count;
atomic_t count;
struct list_head high_items;
struct list_head low_items;
ktime_t last_low_watermark_ktime;
struct mutex mutex;
gfp_t gfp_mask;
unsigned int order;
struct plist_node list;
bool cached;
struct device *heap_dev;
};
struct ion_msm_page_pool *ion_msm_page_pool_create(gfp_t gfp_mask,
unsigned int order,
bool cached);
void ion_msm_page_pool_destroy(struct ion_msm_page_pool *pool);
struct page *ion_msm_page_pool_alloc(struct ion_msm_page_pool *pool,
bool *from_pool);
void ion_msm_page_pool_free(struct ion_msm_page_pool *pool, struct page *page);
struct page *ion_msm_page_pool_alloc_pool_only(struct ion_msm_page_pool *a);
void ion_msm_page_pool_free_immediate(struct ion_msm_page_pool *pool,
struct page *page);
int ion_msm_page_pool_total(struct ion_msm_page_pool *pool, bool high);
size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid);
/** ion_msm_page_pool_shrink - shrinks the size of the memory cached in the pool
* @pool: the pool
* @gfp_mask: the memory type to reclaim
* @nr_to_scan: number of items to shrink in pages
*
* returns the number of items freed in pages
*/
int ion_msm_page_pool_shrink(struct ion_msm_page_pool *pool, gfp_t gfp_mask,
int nr_to_scan);
#ifdef CONFIG_ION_POOL_AUTO_REFILL
void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool);
static __always_inline int get_pool_fillmark(struct ion_msm_page_pool *pool)
{
return ION_POOL_FILL_MARK / (PAGE_SIZE << pool->order);
}
static __always_inline int get_pool_lowmark(struct ion_msm_page_pool *pool)
{
return ION_POOL_LOW_MARK / (PAGE_SIZE << pool->order);
}
static __always_inline bool
pool_count_below_lowmark(struct ion_msm_page_pool *pool)
{
return atomic_read(&pool->count) < get_pool_lowmark(pool);
}
static __always_inline bool
pool_fillmark_reached(struct ion_msm_page_pool *pool)
{
return atomic_read(&pool->count) >= get_pool_fillmark(pool);
}
#else
static inline void ion_msm_page_pool_refill(struct ion_msm_page_pool *pool)
{
}
static __always_inline int get_pool_fillmark(struct ion_msm_page_pool *pool)
{
return 0;
}
static __always_inline int get_pool_lowmark(struct ion_msm_page_pool *pool)
{
return 0;
}
static __always_inline bool
pool_count_below_lowmark(struct ion_msm_page_pool *pool)
{
return false;
}
static __always_inline bool
pool_fillmark_reached(struct ion_msm_page_pool *pool)
{
return false;
}
#endif /* CONFIG_ION_POOL_AUTO_REFILL */
#endif /* _ION_MSM_PAGE_POOL_H */

789
drivers/staging/android/ion/heaps/ion_msm_system_heap.c Normal file
View file

@ -0,0 +1,789 @@
// SPDX-License-Identifier: GPL-2.0
/*
* ION Memory Allocator system heap exporter
*
* Copyright (C) 2011 Google, Inc.
* Copyright (c) 2011-2020, The Linux Foundation. All rights reserved.
*
*/
#include <asm/page.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/highmem.h>
#include <linux/ion.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/sched.h>
#include <uapi/linux/sched/types.h>
#include <linux/seq_file.h>
#include <soc/qcom/secure_buffer.h>
#include "ion_msm_system_heap.h"
#include "ion_msm_page_pool.h"
#include "msm_ion_priv.h"
#include "ion_system_secure_heap.h"
#include "ion_secure_util.h"
static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
__GFP_NORETRY) & ~__GFP_RECLAIM;
static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO;
static bool pool_auto_refill_en __read_mostly =
IS_ENABLED(CONFIG_ION_POOL_AUTO_REFILL);
int order_to_index(unsigned int order)
{
int i;
for (i = 0; i < NUM_ORDERS; i++)
if (order == orders[i])
return i;
BUG();
return -1;
}
static inline unsigned int order_to_size(int order)
{
return PAGE_SIZE << order;
}
static int ion_heap_is_msm_system_heap_type(enum ion_heap_type type)
{
return type == ((enum ion_heap_type)ION_HEAP_TYPE_MSM_SYSTEM);
}
static struct page *alloc_buffer_page(struct ion_msm_system_heap *sys_heap,
struct ion_buffer *buffer,
unsigned long order,
bool *from_pool)
{
int cached = (int)ion_buffer_cached(buffer);
struct page *page;
struct ion_msm_page_pool *pool;
int vmid = get_secure_vmid(buffer->flags);
struct device *dev = sys_heap->heap.dev;
int order_ind = order_to_index(order);
struct task_struct *worker;
if (vmid > 0) {
pool = sys_heap->secure_pools[vmid][order_ind];
/*
* We should skip stealing pages if (1) we're focing our
* allocations to come from buddy; or (2) pool refilling is
* disabled, in which case stealing pages could deplete the
* uncached pools.
*/
if (!(*from_pool && pool_auto_refill_en))
goto normal_alloc;
page = ion_msm_page_pool_alloc_pool_only(pool);
if (!IS_ERR(page))
return page;
pool = sys_heap->uncached_pools[order_ind];
page = ion_msm_page_pool_alloc_pool_only(pool);
if (IS_ERR(page)) {
pool = sys_heap->secure_pools[vmid][order_ind];
goto normal_alloc;
}
/*
* Here, setting `from_pool = false` indicates that the
* page didn't come from the secure pool, and causes
* the page to be hyp-assigned.
*/
*from_pool = false;
if (pool_auto_refill_en && pool->order &&
pool_count_below_lowmark(pool)) {
worker = sys_heap->kworker[ION_KTHREAD_UNCACHED];
wake_up_process(worker);
}
return page;
} else if (!cached) {
pool = sys_heap->uncached_pools[order_ind];
} else {
pool = sys_heap->cached_pools[order_ind];
}
normal_alloc:
page = ion_msm_page_pool_alloc(pool, from_pool);
if (pool_auto_refill_en && pool->order &&
pool_count_below_lowmark(pool) && vmid <= 0)
wake_up_process(sys_heap->kworker[cached]);
if (IS_ERR(page))
return page;
if ((MAKE_ION_ALLOC_DMA_READY && vmid <= 0) || !(*from_pool))
ion_pages_sync_for_device(dev, page, PAGE_SIZE << order,
DMA_BIDIRECTIONAL);
return page;
}
/*
* For secure pages that need to be freed and not added back to the pool; the
* hyp_unassign should be called before calling this function
*/
void free_buffer_page(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer, struct page *page,
unsigned int order)
{
bool cached = ion_buffer_cached(buffer);
int vmid = get_secure_vmid(buffer->flags);
if (!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) {
struct ion_msm_page_pool *pool;
if (vmid > 0)
pool = heap->secure_pools[vmid][order_to_index(order)];
else if (cached)
pool = heap->cached_pools[order_to_index(order)];
else
pool = heap->uncached_pools[order_to_index(order)];
if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)
ion_msm_page_pool_free_immediate(pool, page);
else
ion_msm_page_pool_free(pool, page);
#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES
mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES,
-(1 << pool->order));
#endif
} else {
__free_pages(page, order);
#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES
mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES,
-(1 << order));
#endif
}
}
static struct
page_info *alloc_largest_available(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer,
unsigned long size,
unsigned int max_order)
{
struct page *page;
struct page_info *info;
int i;
bool from_pool;
info = kmalloc(sizeof(*info), GFP_KERNEL);
if (!info)
return ERR_PTR(-ENOMEM);
for (i = 0; i < NUM_ORDERS; i++) {
if (size < order_to_size(orders[i]))
continue;
if (max_order < orders[i])
continue;
from_pool = !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC);
page = alloc_buffer_page(heap, buffer, orders[i], &from_pool);
if (IS_ERR(page))
continue;
info->page = page;
info->order = orders[i];
info->from_pool = from_pool;
INIT_LIST_HEAD(&info->list);
return info;
}
kfree(info);
return ERR_PTR(-ENOMEM);
}
static struct page_info *
alloc_from_pool_preferred(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer,
unsigned long size,
unsigned int max_order)
{
struct page *page;
struct page_info *info;
int i;
if (buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)
goto force_alloc;
info = kmalloc(sizeof(*info), GFP_KERNEL);
if (!info)
return ERR_PTR(-ENOMEM);
for (i = 0; i < NUM_ORDERS; i++) {
if (size < order_to_size(orders[i]))
continue;
if (max_order < orders[i])
continue;
page = alloc_from_secure_pool_order(heap, buffer, orders[i]);
if (IS_ERR(page))
continue;
info->page = page;
info->order = orders[i];
info->from_pool = true;
INIT_LIST_HEAD(&info->list);
return info;
}
page = split_page_from_secure_pool(heap, buffer);
if (!IS_ERR(page)) {
info->page = page;
info->order = 0;
info->from_pool = true;
INIT_LIST_HEAD(&info->list);
return info;
}
kfree(info);
force_alloc:
return alloc_largest_available(heap, buffer, size, max_order);
}
static void process_info(struct page_info *info,
struct scatterlist *sg,
struct scatterlist *sg_sync)
{
struct page *page = info->page;
if (sg_sync) {
sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0);
sg_dma_address(sg_sync) = page_to_phys(page);
}
sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0);
/*
* This is not correct - sg_dma_address needs a dma_addr_t
* that is valid for the the targeted device, but this works
* on the currently targeted hardware.
*/
sg_dma_address(sg) = page_to_phys(page);
list_del(&info->list);
kfree(info);
}
static int ion_msm_system_heap_allocate(struct ion_heap *heap,
struct ion_buffer *buffer,
unsigned long size,
unsigned long flags)
{
struct ion_msm_system_heap *sys_heap = to_msm_system_heap(heap);
struct msm_ion_buf_lock_state *lock_state;
struct sg_table *table;
struct sg_table table_sync = {0};
struct scatterlist *sg;
struct scatterlist *sg_sync;
int ret = -ENOMEM;
struct list_head pages;
struct list_head pages_from_pool;
struct page_info *info, *tmp_info;
int i = 0;
unsigned int nents_sync = 0;
unsigned long size_remaining = PAGE_ALIGN(size);
unsigned int max_order = orders[0];
unsigned int sz;
int vmid = get_secure_vmid(buffer->flags);
if (size / PAGE_SIZE > totalram_pages() / 2)
return -ENOMEM;
if (ion_heap_is_msm_system_heap_type(buffer->heap->type) &&
is_secure_allocation(buffer->flags)) {
pr_info("%s: System heap doesn't support secure allocations\n",
__func__);
return -EINVAL;
}
INIT_LIST_HEAD(&pages);
INIT_LIST_HEAD(&pages_from_pool);
while (size_remaining > 0) {
if (is_secure_vmid_valid(vmid))
info = alloc_from_pool_preferred(sys_heap, buffer,
size_remaining,
max_order);
else
info = alloc_largest_available(sys_heap, buffer,
size_remaining,
max_order);
if (IS_ERR(info)) {
ret = PTR_ERR(info);
goto err;
}
sz = (1 << info->order) * PAGE_SIZE;
#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES
mod_node_page_state(page_pgdat(info->page),
NR_UNRECLAIMABLE_PAGES,
(1 << (info->order)));
#endif
if (info->from_pool) {
list_add_tail(&info->list, &pages_from_pool);
} else {
list_add_tail(&info->list, &pages);
++nents_sync;
}
size_remaining -= sz;
max_order = info->order;
i++;
}
table = kzalloc(sizeof(*table), GFP_KERNEL);
if (!table) {
ret = -ENOMEM;
goto err;
}
ret = sg_alloc_table(table, i, GFP_KERNEL);
if (ret)
goto err1;
if (nents_sync) {
ret = sg_alloc_table(&table_sync, nents_sync, GFP_KERNEL);
if (ret)
goto err_free_sg;
}
sg = table->sgl;
sg_sync = table_sync.sgl;
/*
* We now have two separate lists. One list contains pages from the
* pool and the other pages from buddy. We want to merge these
* together while preserving the ordering of the pages (higher order
* first).
*/
do {
info = list_first_entry_or_null(&pages, struct page_info, list);
tmp_info = list_first_entry_or_null(&pages_from_pool,
struct page_info, list);
if (info && tmp_info) {
if (info->order >= tmp_info->order) {
process_info(info, sg, sg_sync);
sg_sync = sg_next(sg_sync);
} else {
process_info(tmp_info, sg, NULL);
}
} else if (info) {
process_info(info, sg, sg_sync);
sg_sync = sg_next(sg_sync);
} else if (tmp_info) {
process_info(tmp_info, sg, NULL);
}
sg = sg_next(sg);
} while (sg);
if (nents_sync) {
if (vmid > 0) {
ret = ion_hyp_assign_sg(&table_sync, &vmid, 1, true);
if (ret)
goto err_free_sg2;
}
}
buffer->sg_table = table;
if (nents_sync)
sg_free_table(&table_sync);
lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL);
if (!lock_state) {
ret = -ENOMEM;
goto err_free_sg2;
}
buffer->priv_virt = lock_state;
ion_prepare_sgl_for_force_dma_sync(buffer->sg_table);
return 0;
err_free_sg2:
/* We failed to zero buffers. Bypass pool */
buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE;
if (vmid > 0)
if (ion_hyp_unassign_sg(table, &vmid, 1, true))
goto err_free_table_sync;
for_each_sg(table->sgl, sg, table->nents, i)
free_buffer_page(sys_heap, buffer, sg_page(sg),
get_order(sg->length));
err_free_table_sync:
if (nents_sync)
sg_free_table(&table_sync);
err_free_sg:
sg_free_table(table);
err1:
kfree(table);
err:
list_for_each_entry_safe(info, tmp_info, &pages, list) {
free_buffer_page(sys_heap, buffer, info->page, info->order);
kfree(info);
}
list_for_each_entry_safe(info, tmp_info, &pages_from_pool, list) {
free_buffer_page(sys_heap, buffer, info->page, info->order);
kfree(info);
}
return ret;
}
static void ion_msm_system_heap_free(struct ion_buffer *buffer)
{
struct ion_heap *heap = buffer->heap;
struct ion_msm_system_heap *sys_heap = to_msm_system_heap(heap);
struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt;
struct sg_table *table = buffer->sg_table;
struct scatterlist *sg;
int i;
int vmid = get_secure_vmid(buffer->flags);
if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) &&
!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) {
mutex_lock(&buffer->lock);
if (hlos_accessible_buffer(buffer))
ion_buffer_zero(buffer);
if (lock_state && lock_state->locked)
pr_warn("%s: buffer is locked while being freed\n",
__func__);
mutex_unlock(&buffer->lock);
} else if (vmid > 0) {
if (ion_hyp_unassign_sg(table, &vmid, 1, true))
return;
}
for_each_sg(table->sgl, sg, table->nents, i)
free_buffer_page(sys_heap, buffer, sg_page(sg),
get_order(sg->length));
sg_free_table(table);
kfree(table);
kfree(buffer->priv_virt);
}
static int ion_msm_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,
int nr_to_scan)
{
struct ion_msm_system_heap *sys_heap;
int nr_total = 0;
int i, j, nr_freed = 0;
int only_scan = 0;
struct ion_msm_page_pool *pool;
sys_heap = to_msm_system_heap(heap);
if (!nr_to_scan)
only_scan = 1;
/* shrink the pools starting from lower order ones */
for (i = NUM_ORDERS - 1; i >= 0; i--) {
nr_freed = 0;
for (j = 0; j < VMID_LAST; j++) {
if (is_secure_vmid_valid(j))
nr_freed +=
ion_secure_page_pool_shrink(sys_heap,
j, i,
nr_to_scan);
}
pool = sys_heap->uncached_pools[i];
nr_freed +=
ion_msm_page_pool_shrink(pool, gfp_mask, nr_to_scan);
pool = sys_heap->cached_pools[i];
nr_freed +=
ion_msm_page_pool_shrink(pool, gfp_mask, nr_to_scan);
nr_total += nr_freed;
if (!only_scan) {
nr_to_scan -= nr_freed;
/* shrink completed */
if (nr_to_scan <= 0)
break;
}
}
return nr_total;
}
static struct ion_heap_ops system_heap_ops = {
.allocate = ion_msm_system_heap_allocate,
.free = ion_msm_system_heap_free,
.shrink = ion_msm_system_heap_shrink,
};
static int ion_msm_system_heap_debug_show(struct ion_heap *heap,
struct seq_file *s, void *unused)
{
struct ion_msm_system_heap *sys_heap;
bool use_seq = s;
unsigned long uncached_total = 0;
unsigned long cached_total = 0;
unsigned long secure_total = 0;
struct ion_msm_page_pool *pool;
int i, j;
sys_heap = to_msm_system_heap(heap);
for (i = 0; i < NUM_ORDERS; i++) {
pool = sys_heap->uncached_pools[i];
if (use_seq) {
seq_printf(s,
"%d order %u highmem pages in uncached pool = %lu total\n",
pool->high_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->high_count);
seq_printf(s,
"%d order %u lowmem pages in uncached pool = %lu total\n",
pool->low_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->low_count);
}
uncached_total += (1 << pool->order) * PAGE_SIZE *
pool->high_count;
uncached_total += (1 << pool->order) * PAGE_SIZE *
pool->low_count;
}
for (i = 0; i < NUM_ORDERS; i++) {
pool = sys_heap->cached_pools[i];
if (use_seq) {
seq_printf(s,
"%d order %u highmem pages in cached pool = %lu total\n",
pool->high_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->high_count);
seq_printf(s,
"%d order %u lowmem pages in cached pool = %lu total\n",
pool->low_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->low_count);
}
cached_total += (1 << pool->order) * PAGE_SIZE *
pool->high_count;
cached_total += (1 << pool->order) * PAGE_SIZE *
pool->low_count;
}
for (i = 0; i < NUM_ORDERS; i++) {
for (j = 0; j < VMID_LAST; j++) {
if (!is_secure_vmid_valid(j))
continue;
pool = sys_heap->secure_pools[j][i];
if (use_seq) {
seq_printf(s,
"VMID %d: %d order %u highmem pages in secure pool = %lu total\n",
j, pool->high_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->high_count);
seq_printf(s,
"VMID %d: %d order %u lowmem pages in secure pool = %lu total\n",
j, pool->low_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->low_count);
}
secure_total += (1 << pool->order) * PAGE_SIZE *
pool->high_count;
secure_total += (1 << pool->order) * PAGE_SIZE *
pool->low_count;
}
}
if (use_seq) {
seq_puts(s, "--------------------------------------------\n");
seq_printf(s, "uncached pool = %lu cached pool = %lu secure pool = %lu\n",
uncached_total, cached_total, secure_total);
seq_printf(s, "pool total (uncached + cached + secure) = %lu\n",
uncached_total + cached_total + secure_total);
seq_puts(s, "--------------------------------------------\n");
} else {
pr_info("-------------------------------------------------\n");
pr_info("uncached pool = %lu cached pool = %lu secure pool = %lu\n",
uncached_total, cached_total, secure_total);
pr_info("pool total (uncached + cached + secure) = %lu\n",
uncached_total + cached_total + secure_total);
pr_info("-------------------------------------------------\n");
}
return 0;
}
static struct msm_ion_heap_ops msm_system_heap_ops = {
.debug_show = ion_msm_system_heap_debug_show,
};
static void ion_msm_system_heap_destroy_pools(struct ion_msm_page_pool **pools)
{
int i;
if (!pools)
return;
for (i = 0; i < NUM_ORDERS; i++)
if (pools[i]) {
ion_msm_page_pool_destroy(pools[i]);
pools[i] = NULL;
}
}
/**
* ion_msm_system_heap_create_pools - Creates pools for all orders
*
* If this fails you don't need to destroy any pools. It's all or
* nothing. If it succeeds you'll eventually need to use
* ion_msm_system_heap_destroy_pools to destroy the pools.
*/
static int
ion_msm_system_heap_create_pools(struct ion_msm_system_heap *sys_heap,
struct ion_msm_page_pool **pools, bool cached)
{
int i;
for (i = 0; i < NUM_ORDERS; i++) {
struct ion_msm_page_pool *pool;
gfp_t gfp_flags = low_order_gfp_flags;
if (orders[i])
gfp_flags = high_order_gfp_flags;
pool = ion_msm_page_pool_create(gfp_flags, orders[i], cached);
if (!pool)
goto err_create_pool;
pool->heap_dev = sys_heap->heap.dev;
pools[i] = pool;
}
return 0;
err_create_pool:
ion_msm_system_heap_destroy_pools(pools);
return -ENOMEM;
}
static int ion_msm_sys_heap_worker(void *data)
{
struct ion_msm_page_pool **pools = (struct ion_msm_page_pool **)data;
int i;
for (;;) {
for (i = 0; i < NUM_ORDERS; i++) {
if (pool_count_below_lowmark(pools[i]))
ion_msm_page_pool_refill(pools[i]);
}
set_current_state(TASK_INTERRUPTIBLE);
if (unlikely(kthread_should_stop())) {
set_current_state(TASK_RUNNING);
break;
}
schedule();
set_current_state(TASK_RUNNING);
}
return 0;
}
static struct task_struct *ion_create_kworker(struct ion_msm_page_pool **pools,
bool cached)
{
struct sched_attr attr = { 0 };
struct task_struct *thread;
int ret;
char *buf;
attr.sched_nice = ION_KTHREAD_NICE_VAL;
buf = cached ? "cached" : "uncached";
thread = kthread_run(ion_msm_sys_heap_worker, pools,
"ion-pool-%s-worker", buf);
if (IS_ERR(thread)) {
pr_err("%s: failed to create %s worker thread: %ld\n",
__func__, buf, PTR_ERR(thread));
return thread;
}
ret = sched_setattr(thread, &attr);
if (ret) {
kthread_stop(thread);
pr_warn("%s: failed to set task priority for %s worker thread: ret = %d\n",
__func__, buf, ret);
return ERR_PTR(ret);
}
return thread;
}
struct ion_heap *ion_msm_system_heap_create(struct ion_platform_heap *data)
{
struct ion_msm_system_heap *heap;
int ret = -ENOMEM;
int i;
heap = kzalloc(sizeof(*heap), GFP_KERNEL);
if (!heap)
return ERR_PTR(-ENOMEM);
heap->heap.dev = data->priv;
heap->heap.msm_heap_ops = &msm_system_heap_ops;
heap->heap.ion_heap.ops = &system_heap_ops;
heap->heap.ion_heap.buf_ops = msm_ion_dma_buf_ops;
heap->heap.ion_heap.type = (enum ion_heap_type)ION_HEAP_TYPE_MSM_SYSTEM;
heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE;
for (i = 0; i < VMID_LAST; i++)
if (is_secure_vmid_valid(i) &&
ion_msm_system_heap_create_pools(heap,
heap->secure_pools[i],
false))
goto destroy_secure_pools;
if (ion_msm_system_heap_create_pools(heap, heap->uncached_pools, false))
goto destroy_secure_pools;
if (ion_msm_system_heap_create_pools(heap, heap->cached_pools, true))
goto destroy_uncached_pools;
if (pool_auto_refill_en) {
heap->kworker[ION_KTHREAD_UNCACHED] =
ion_create_kworker(heap->uncached_pools, false);
if (IS_ERR(heap->kworker[ION_KTHREAD_UNCACHED])) {
ret = PTR_ERR(heap->kworker[ION_KTHREAD_UNCACHED]);
goto destroy_pools;
}
heap->kworker[ION_KTHREAD_CACHED] =
ion_create_kworker(heap->cached_pools, true);
if (IS_ERR(heap->kworker[ION_KTHREAD_CACHED])) {
kthread_stop(heap->kworker[ION_KTHREAD_UNCACHED]);
ret = PTR_ERR(heap->kworker[ION_KTHREAD_CACHED]);
goto destroy_pools;
}
}
mutex_init(&heap->split_page_mutex);
return &heap->heap.ion_heap;
destroy_pools:
ion_msm_system_heap_destroy_pools(heap->cached_pools);
destroy_uncached_pools:
ion_msm_system_heap_destroy_pools(heap->uncached_pools);
destroy_secure_pools:
for (i = 0; i < VMID_LAST; i++)
ion_msm_system_heap_destroy_pools(heap->secure_pools[i]);
kfree(heap);
return ERR_PTR(ret);
}
MODULE_LICENSE("GPL v2");

22
drivers/staging/android/ion/heaps/ion_system_heap.h → drivers/staging/android/ion/heaps/ion_msm_system_heap.h
View file

@ -1,12 +1,12 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
* Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
*/
#include <soc/qcom/secure_buffer.h>
#include "msm_ion_priv.h"
#ifndef _ION_SYSTEM_HEAP_H
#define _ION_SYSTEM_HEAP_H
#ifndef _ION_MSM_SYSTEM_HEAP_H
#define _ION_MSM_SYSTEM_HEAP_H
#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS
#if defined(CONFIG_IOMMU_IO_PGTABLE_ARMV7S)
@ -22,8 +22,8 @@ static const unsigned int orders[] = {0};
#define ION_KTHREAD_NICE_VAL 10
#define to_system_heap(_heap) \
container_of(to_msm_ion_heap(_heap), struct ion_system_heap, heap)
#define to_msm_system_heap(_heap) \
container_of(to_msm_ion_heap(_heap), struct ion_msm_system_heap, heap)
enum ion_kthread_type {
ION_KTHREAD_UNCACHED,
@ -31,13 +31,13 @@ enum ion_kthread_type {
ION_MAX_NUM_KTHREADS
};
struct ion_system_heap {
struct ion_msm_system_heap {
struct msm_ion_heap heap;
struct ion_page_pool *uncached_pools[MAX_ORDER];
struct ion_page_pool *cached_pools[MAX_ORDER];
struct ion_msm_page_pool *uncached_pools[MAX_ORDER];
struct ion_msm_page_pool *cached_pools[MAX_ORDER];
/* worker threads to refill the pool */
struct task_struct *kworker[ION_MAX_NUM_KTHREADS];
struct ion_page_pool *secure_pools[VMID_LAST][MAX_ORDER];
struct ion_msm_page_pool *secure_pools[VMID_LAST][MAX_ORDER];
/* Prevents unnecessary page splitting */
struct mutex split_page_mutex;
};
@ -51,8 +51,8 @@ struct page_info {
int order_to_index(unsigned int order);
void free_buffer_page(struct ion_system_heap *heap,
void free_buffer_page(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer, struct page *page,
unsigned int order);
#endif /* _ION_SYSTEM_HEAP_H */
#endif /* _ION_MSM_SYSTEM_HEAP_H */

132
drivers/staging/android/ion/heaps/ion_page_pool.c
View file

@ -10,7 +10,6 @@
#include <linux/swap.h>
#include <linux/sched/signal.h>
#include "msm_ion_priv.h"
#include "ion_page_pool.h"
static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool)
@ -37,71 +36,11 @@ static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page)
pool->low_count++;
}
atomic_inc(&pool->count);
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
(1 << pool->order));
1 << pool->order);
mutex_unlock(&pool->mutex);
}
#ifdef CONFIG_ION_POOL_AUTO_REFILL
/* do a simple check to see if we are in any low memory situation */
static bool pool_refill_ok(struct ion_page_pool *pool)
{
struct zonelist *zonelist;
struct zoneref *z;
struct zone *zone;
int mark;
enum zone_type classzone_idx = gfp_zone(pool->gfp_mask);
s64 delta;
/* check if we are within the refill defer window */
delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime);
if (delta < ION_POOL_REFILL_DEFER_WINDOW_MS)
return false;
zonelist = node_zonelist(numa_node_id(), pool->gfp_mask);
/*
* make sure that if we allocate a pool->order page from buddy,
* we don't put the zone watermarks go below the high threshold.
* This makes sure there's no unwanted repetitive refilling and
* reclaiming of buddy pages on the pool.
*/
for_each_zone_zonelist(zone, z, zonelist, classzone_idx) {
mark = high_wmark_pages(zone);
mark += 1 << pool->order;
if (!zone_watermark_ok_safe(zone, pool->order, mark,
classzone_idx)) {
pool->last_low_watermark_ktime = ktime_get();
return false;
}
}
return true;
}
void ion_page_pool_refill(struct ion_page_pool *pool)
{
struct page *page;
gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY;
struct device *dev = pool->heap_dev;
/* skip refilling order 0 pools */
if (!pool->order)
return;
while (!pool_fillmark_reached(pool) && pool_refill_ok(pool)) {
page = alloc_pages(gfp_refill, pool->order);
if (!page)
break;
if (!pool->cached)
ion_pages_sync_for_device(dev, page,
PAGE_SIZE << pool->order,
DMA_BIDIRECTIONAL);
ion_page_pool_add(pool, page);
}
}
#endif /* CONFIG_ION_PAGE_POOL_REFILL */
static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high)
{
struct page *page;
@ -116,73 +55,39 @@ static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high)
pool->low_count--;
}
atomic_dec(&pool->count);
list_del(&page->lru);
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
-(1 << pool->order));
-(1 << pool->order));
return page;
}
struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool)
struct page *ion_page_pool_alloc(struct ion_page_pool *pool)
{
struct page *page = NULL;
BUG_ON(!pool);
if (fatal_signal_pending(current))
return ERR_PTR(-EINTR);
mutex_lock(&pool->mutex);
if (pool->high_count)
page = ion_page_pool_remove(pool, true);
else if (pool->low_count)
page = ion_page_pool_remove(pool, false);
mutex_unlock(&pool->mutex);
if (*from_pool && mutex_trylock(&pool->mutex)) {
if (pool->high_count)
page = ion_page_pool_remove(pool, true);
else if (pool->low_count)
page = ion_page_pool_remove(pool, false);
mutex_unlock(&pool->mutex);
}
if (!page) {
if (!page)
page = ion_page_pool_alloc_pages(pool);
*from_pool = false;
}
if (!page)
return ERR_PTR(-ENOMEM);
return page;
}
/*
* Tries to allocate from only the specified Pool and returns NULL otherwise
*/
struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *pool)
{
struct page *page = NULL;
if (!pool)
return ERR_PTR(-EINVAL);
if (mutex_trylock(&pool->mutex)) {
if (pool->high_count)
page = ion_page_pool_remove(pool, true);
else if (pool->low_count)
page = ion_page_pool_remove(pool, false);
mutex_unlock(&pool->mutex);
}
if (!page)
return ERR_PTR(-ENOMEM);
return page;
}
void ion_page_pool_free(struct ion_page_pool *pool, struct page *page)
{
BUG_ON(pool->order != compound_order(page));
ion_page_pool_add(pool, page);
}
void ion_page_pool_free_immediate(struct ion_page_pool *pool, struct page *page)
{
ion_page_pool_free_pages(pool, page);
}
int ion_page_pool_total(struct ion_page_pool *pool, bool high)
static int ion_page_pool_total(struct ion_page_pool *pool, bool high)
{
int count = pool->low_count;
@ -226,21 +131,20 @@ int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
return freed;
}
struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order,
bool cached)
struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)
{
struct ion_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
if (!pool)
return NULL;
pool->high_count = 0;
pool->low_count = 0;
INIT_LIST_HEAD(&pool->low_items);
INIT_LIST_HEAD(&pool->high_items);
pool->gfp_mask = gfp_mask;
pool->gfp_mask = gfp_mask | __GFP_COMP;
pool->order = order;
mutex_init(&pool->mutex);
plist_node_init(&pool->list, order);
if (cached)
pool->cached = true;
return pool;
}

80
drivers/staging/android/ion/heaps/ion_page_pool.h
View file

@ -13,19 +13,6 @@
#include <linux/shrinker.h>
#include <linux/types.h>
/* ION page pool marks in bytes */
#ifdef CONFIG_ION_POOL_AUTO_REFILL
#define ION_POOL_FILL_MARK (CONFIG_ION_POOL_FILL_MARK * SZ_1M)
#define POOL_LOW_MARK_PERCENT 40UL
#define ION_POOL_LOW_MARK ((ION_POOL_FILL_MARK * POOL_LOW_MARK_PERCENT) / 100)
#else
#define ION_POOL_FILL_MARK 0UL
#define ION_POOL_LOW_MARK 0UL
#endif
/* if low watermark of zones have reached, defer the refill in this window */
#define ION_POOL_REFILL_DEFER_WINDOW_MS 10
/**
* functions for creating and destroying a heap pool -- allows you
* to keep a pool of pre allocated memory to use from your heap. Keeping
@ -38,18 +25,13 @@
* struct ion_page_pool - pagepool struct
* @high_count: number of highmem items in the pool
* @low_count: number of lowmem items in the pool
* @count: total number of pages/items in the pool
* @high_items: list of highmem items
* @low_items: list of lowmem items
* @last_low_watermark_ktime: most recent time at which the zone watermarks were
* low
* @mutex: lock protecting this struct and especially the count
* item list
* @gfp_mask: gfp_mask to use from alloc
* @order: order of pages in the pool
* @list: plist node for list of pools
* @cached: it's cached pool or not
* @heap_dev: device for the ion heap associated with this pool
*
* Allows you to keep a pool of pre allocated pages to use from your heap.
* Keeping a pool of pages that is ready for dma, ie any cached mapping have
@ -59,28 +41,18 @@
struct ion_page_pool {
int high_count;
int low_count;
atomic_t count;
struct list_head high_items;
struct list_head low_items;
ktime_t last_low_watermark_ktime;
struct mutex mutex;
gfp_t gfp_mask;
unsigned int order;
struct plist_node list;
bool cached;
struct device *heap_dev;
};
struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order,
bool cached);
struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order);
void ion_page_pool_destroy(struct ion_page_pool *pool);
struct page *ion_page_pool_alloc(struct ion_page_pool *pool, bool *from_pool);
struct page *ion_page_pool_alloc(struct ion_page_pool *pool);
void ion_page_pool_free(struct ion_page_pool *pool, struct page *page);
struct page *ion_page_pool_alloc_pool_only(struct ion_page_pool *a);
void ion_page_pool_free_immediate(struct ion_page_pool *pool,
struct page *page);
int ion_page_pool_total(struct ion_page_pool *pool, bool high);
size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid);
/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool
* @pool: the pool
@ -91,52 +63,4 @@ size_t ion_system_heap_secure_page_pool_total(struct ion_heap *heap, int vmid);
*/
int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
int nr_to_scan);
#ifdef CONFIG_ION_POOL_AUTO_REFILL
void ion_page_pool_refill(struct ion_page_pool *pool);
static __always_inline int get_pool_fillmark(struct ion_page_pool *pool)
{
return ION_POOL_FILL_MARK / (PAGE_SIZE << pool->order);
}
static __always_inline int get_pool_lowmark(struct ion_page_pool *pool)
{
return ION_POOL_LOW_MARK / (PAGE_SIZE << pool->order);
}
static __always_inline bool pool_count_below_lowmark(struct ion_page_pool *pool)
{
return atomic_read(&pool->count) < get_pool_lowmark(pool);
}
static __always_inline bool pool_fillmark_reached(struct ion_page_pool *pool)
{
return atomic_read(&pool->count) >= get_pool_fillmark(pool);
}
#else
static inline void ion_page_pool_refill(struct ion_page_pool *pool)
{
}
static __always_inline int get_pool_fillmark(struct ion_page_pool *pool)
{
return 0;
}
static __always_inline int get_pool_lowmark(struct ion_page_pool *pool)
{
return 0;
}
static __always_inline bool pool_count_below_lowmark(struct ion_page_pool *pool)
{
return false;
}
static __always_inline bool pool_fillmark_reached(struct ion_page_pool *pool)
{
return false;
}
#endif /* CONFIG_ION_POOL_AUTO_REFILL */
#endif /* _ION_PAGE_POOL_H */

772
drivers/staging/android/ion/heaps/ion_system_heap.c
View file

@ -3,8 +3,6 @@
* ION Memory Allocator system heap exporter
*
* Copyright (C) 2011 Google, Inc.
* Copyright (c) 2011-2020, The Linux Foundation. All rights reserved.
*
*/
#include <asm/page.h>
@ -17,25 +15,17 @@
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/sched.h>
#include <uapi/linux/sched/types.h>
#include <linux/seq_file.h>
#include <soc/qcom/secure_buffer.h>
#include "ion_system_heap.h"
#include "ion_page_pool.h"
#include "msm_ion_priv.h"
#include "ion_system_heap.h"
#include "ion_system_secure_heap.h"
#include "ion_secure_util.h"
#define NUM_ORDERS ARRAY_SIZE(orders)
static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
__GFP_NORETRY) & ~__GFP_RECLAIM;
static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO;
static const unsigned int orders[] = {8, 4, 0};
bool pool_auto_refill_en __read_mostly =
IS_ENABLED(CONFIG_ION_POOL_AUTO_REFILL);
int order_to_index(unsigned int order)
static int order_to_index(unsigned int order)
{
int i;
@ -51,175 +41,44 @@ static inline unsigned int order_to_size(int order)
return PAGE_SIZE << order;
}
int ion_heap_is_system_heap_type(enum ion_heap_type type)
{
return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM);
}
struct ion_system_heap {
struct ion_heap heap;
struct ion_page_pool *pools[NUM_ORDERS];
};
static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap,
static struct page *alloc_buffer_page(struct ion_system_heap *heap,
struct ion_buffer *buffer,
unsigned long order,
bool *from_pool)
unsigned long order)
{
struct ion_page_pool *pool = heap->pools[order_to_index(order)];
return ion_page_pool_alloc(pool);
}
static void free_buffer_page(struct ion_system_heap *heap,
struct ion_buffer *buffer, struct page *page)
{
int cached = (int)ion_buffer_cached(buffer);
struct page *page;
struct ion_page_pool *pool;
int vmid = get_secure_vmid(buffer->flags);
struct device *dev = sys_heap->heap.dev;
int order_ind = order_to_index(order);
struct task_struct *worker;
unsigned int order = compound_order(page);
if (vmid > 0) {
pool = sys_heap->secure_pools[vmid][order_ind];
/*
* We should skip stealing pages if (1) we're focing our
* allocations to come from buddy; or (2) pool refilling is
* disabled, in which case stealing pages could deplete the
* uncached pools.
*/
if (!(*from_pool && pool_auto_refill_en))
goto normal_alloc;
page = ion_page_pool_alloc_pool_only(pool);
if (!IS_ERR(page))
return page;
pool = sys_heap->uncached_pools[order_ind];
page = ion_page_pool_alloc_pool_only(pool);
if (IS_ERR(page)) {
pool = sys_heap->secure_pools[vmid][order_ind];
goto normal_alloc;
}
/*
* Here, setting `from_pool = false` indicates that the
* page didn't come from the secure pool, and causes
* the page to be hyp-assigned.
*/
*from_pool = false;
if (pool_auto_refill_en && pool->order &&
pool_count_below_lowmark(pool)) {
worker = sys_heap->kworker[ION_KTHREAD_UNCACHED];
wake_up_process(worker);
}
return page;
} else if (!cached) {
pool = sys_heap->uncached_pools[order_ind];
} else {
pool = sys_heap->cached_pools[order_ind];
}
normal_alloc:
page = ion_page_pool_alloc(pool, from_pool);
if (pool_auto_refill_en && pool->order &&
pool_count_below_lowmark(pool) && vmid <= 0)
wake_up_process(sys_heap->kworker[cached]);
if (IS_ERR(page))
return page;
if ((MAKE_ION_ALLOC_DMA_READY && vmid <= 0) || !(*from_pool))
ion_pages_sync_for_device(dev, page, PAGE_SIZE << order,
DMA_BIDIRECTIONAL);
return page;
}
/*
* For secure pages that need to be freed and not added back to the pool; the
* hyp_unassign should be called before calling this function
*/
void free_buffer_page(struct ion_system_heap *heap,
struct ion_buffer *buffer, struct page *page,
unsigned int order)
{
bool cached = ion_buffer_cached(buffer);
int vmid = get_secure_vmid(buffer->flags);
if (!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) {
struct ion_page_pool *pool;
if (vmid > 0)
pool = heap->secure_pools[vmid][order_to_index(order)];
else if (cached)
pool = heap->cached_pools[order_to_index(order)];
else
pool = heap->uncached_pools[order_to_index(order)];
if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)
ion_page_pool_free_immediate(pool, page);
else
ion_page_pool_free(pool, page);
#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES
mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES,
-(1 << pool->order));
#endif
} else {
/* go to system */
if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) {
__free_pages(page, order);
#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES
mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES,
-(1 << order));
#endif
return;
}
pool = heap->pools[order_to_index(order)];
ion_page_pool_free(pool, page);
}
static struct page_info *alloc_largest_available(struct ion_system_heap *heap,
struct ion_buffer *buffer,
unsigned long size,
unsigned int max_order)
static struct page *alloc_largest_available(struct ion_system_heap *heap,
struct ion_buffer *buffer,
unsigned long size,
unsigned int max_order)
{
struct page *page;
struct page_info *info;
int i;
bool from_pool;
info = kmalloc(sizeof(*info), GFP_KERNEL);
if (!info)
return ERR_PTR(-ENOMEM);
for (i = 0; i < NUM_ORDERS; i++) {
if (size < order_to_size(orders[i]))
continue;
if (max_order < orders[i])
continue;
from_pool = !(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC);
page = alloc_buffer_page(heap, buffer, orders[i], &from_pool);
if (IS_ERR(page))
continue;
info->page = page;
info->order = orders[i];
info->from_pool = from_pool;
INIT_LIST_HEAD(&info->list);
return info;
}
kfree(info);
return ERR_PTR(-ENOMEM);
}
static struct page_info *
alloc_from_pool_preferred(struct ion_system_heap *heap,
struct ion_buffer *buffer,
unsigned long size,
unsigned int max_order)
{
struct page *page;
struct page_info *info;
int i;
if (buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)
goto force_alloc;
info = kmalloc(sizeof(*info), GFP_KERNEL);
if (!info)
return ERR_PTR(-ENOMEM);
for (i = 0; i < NUM_ORDERS; i++) {
if (size < order_to_size(orders[i]))
@ -227,51 +86,14 @@ alloc_from_pool_preferred(struct ion_system_heap *heap,
if (max_order < orders[i])
continue;
page = alloc_from_secure_pool_order(heap, buffer, orders[i]);
if (IS_ERR(page))
page = alloc_buffer_page(heap, buffer, orders[i]);
if (!page)
continue;
info->page = page;
info->order = orders[i];
info->from_pool = true;
INIT_LIST_HEAD(&info->list);
return info;
return page;
}
page = split_page_from_secure_pool(heap, buffer);
if (!IS_ERR(page)) {
info->page = page;
info->order = 0;
info->from_pool = true;
INIT_LIST_HEAD(&info->list);
return info;
}
kfree(info);
force_alloc:
return alloc_largest_available(heap, buffer, size, max_order);
}
static void process_info(struct page_info *info,
struct scatterlist *sg,
struct scatterlist *sg_sync)
{
struct page *page = info->page;
if (sg_sync) {
sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0);
sg_dma_address(sg_sync) = page_to_phys(page);
}
sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0);
/*
* This is not correct - sg_dma_address needs a dma_addr_t
* that is valid for the the targeted device, but this works
* on the currently targeted hardware.
*/
sg_dma_address(sg) = page_to_phys(page);
list_del(&info->list);
kfree(info);
return NULL;
}
static int ion_system_heap_allocate(struct ion_heap *heap,
@ -279,508 +101,176 @@ static int ion_system_heap_allocate(struct ion_heap *heap,
unsigned long size,
unsigned long flags)
{
struct ion_system_heap *sys_heap = to_system_heap(heap);
struct msm_ion_buf_lock_state *lock_state;
struct ion_system_heap *sys_heap = container_of(heap,
struct ion_system_heap,
heap);
struct sg_table *table;
struct sg_table table_sync = {0};
struct scatterlist *sg;
struct scatterlist *sg_sync;
int ret = -ENOMEM;
struct list_head pages;
struct list_head pages_from_pool;
struct page_info *info, *tmp_info;
struct page *page, *tmp_page;
int i = 0;
unsigned int nents_sync = 0;
unsigned long size_remaining = PAGE_ALIGN(size);
unsigned int max_order = orders[0];
unsigned int sz;
int vmid = get_secure_vmid(buffer->flags);
if (size / PAGE_SIZE > totalram_pages() / 2)
return -ENOMEM;
if (ion_heap_is_system_heap_type(buffer->heap->type) &&
is_secure_allocation(buffer->flags)) {
pr_info("%s: System heap doesn't support secure allocations\n",
__func__);
return -EINVAL;
}
INIT_LIST_HEAD(&pages);
INIT_LIST_HEAD(&pages_from_pool);
while (size_remaining > 0) {
if (is_secure_vmid_valid(vmid))
info = alloc_from_pool_preferred(sys_heap, buffer,
size_remaining,
max_order);
else
info = alloc_largest_available(sys_heap, buffer,
size_remaining,
max_order);
if (IS_ERR(info)) {
ret = PTR_ERR(info);
goto err;
}
sz = (1 << info->order) * PAGE_SIZE;
#ifdef CONFIG_MM_STAT_UNRECLAIMABLE_PAGES
mod_node_page_state(page_pgdat(info->page),
NR_UNRECLAIMABLE_PAGES,
(1 << (info->order)));
#endif
if (info->from_pool) {
list_add_tail(&info->list, &pages_from_pool);
} else {
list_add_tail(&info->list, &pages);
++nents_sync;
}
size_remaining -= sz;
max_order = info->order;
page = alloc_largest_available(sys_heap, buffer, size_remaining,
max_order);
if (!page)
goto free_pages;
list_add_tail(&page->lru, &pages);
size_remaining -= page_size(page);
max_order = compound_order(page);
i++;
}
table = kmalloc(sizeof(*table), GFP_KERNEL);
if (!table)
goto free_pages;
table = kzalloc(sizeof(*table), GFP_KERNEL);
if (!table) {
ret = -ENOMEM;
goto err;
}
ret = sg_alloc_table(table, i, GFP_KERNEL);
if (ret)
goto err1;
if (nents_sync) {
ret = sg_alloc_table(&table_sync, nents_sync, GFP_KERNEL);
if (ret)
goto err_free_sg;
}
if (sg_alloc_table(table, i, GFP_KERNEL))
goto free_table;
sg = table->sgl;
sg_sync = table_sync.sgl;
/*
* We now have two separate lists. One list contains pages from the
* pool and the other pages from buddy. We want to merge these
* together while preserving the ordering of the pages (higher order
* first).
*/
do {
info = list_first_entry_or_null(&pages, struct page_info, list);
tmp_info = list_first_entry_or_null(&pages_from_pool,
struct page_info, list);
if (info && tmp_info) {
if (info->order >= tmp_info->order) {
process_info(info, sg, sg_sync);
sg_sync = sg_next(sg_sync);
} else {
process_info(tmp_info, sg, NULL);
}
} else if (info) {
process_info(info, sg, sg_sync);
sg_sync = sg_next(sg_sync);
} else if (tmp_info) {
process_info(tmp_info, sg, NULL);
}
list_for_each_entry_safe(page, tmp_page, &pages, lru) {
sg_set_page(sg, page, page_size(page), 0);
sg = sg_next(sg);
} while (sg);
if (nents_sync) {
if (vmid > 0) {
ret = ion_hyp_assign_sg(&table_sync, &vmid, 1, true);
if (ret)
goto err_free_sg2;
}
list_del(&page->lru);
}
buffer->sg_table = table;
if (nents_sync)
sg_free_table(&table_sync);
lock_state = kzalloc(sizeof(*lock_state), GFP_KERNEL);
if (!lock_state) {
ret = -ENOMEM;
goto err_free_sg2;
}
buffer->priv_virt = lock_state;
ion_buffer_prep_noncached(buffer);
ion_prepare_sgl_for_force_dma_sync(buffer->sg_table);
return 0;
err_free_sg2:
/* We failed to zero buffers. Bypass pool */
buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE;
if (vmid > 0)
if (ion_hyp_unassign_sg(table, &vmid, 1, true))
goto err_free_table_sync;
for_each_sg(table->sgl, sg, table->nents, i)
free_buffer_page(sys_heap, buffer, sg_page(sg),
get_order(sg->length));
err_free_table_sync:
if (nents_sync)
sg_free_table(&table_sync);
err_free_sg:
sg_free_table(table);
err1:
free_table:
kfree(table);
err:
list_for_each_entry_safe(info, tmp_info, &pages, list) {
free_buffer_page(sys_heap, buffer, info->page, info->order);
kfree(info);
}
list_for_each_entry_safe(info, tmp_info, &pages_from_pool, list) {
free_buffer_page(sys_heap, buffer, info->page, info->order);
kfree(info);
}
return ret;
free_pages:
list_for_each_entry_safe(page, tmp_page, &pages, lru)
free_buffer_page(sys_heap, buffer, page);
return -ENOMEM;
}
static void ion_system_heap_free(struct ion_buffer *buffer)
{
struct ion_heap *heap = buffer->heap;
struct ion_system_heap *sys_heap = to_system_heap(heap);
struct msm_ion_buf_lock_state *lock_state = buffer->priv_virt;
struct ion_system_heap *sys_heap = container_of(buffer->heap,
struct ion_system_heap,
heap);
struct sg_table *table = buffer->sg_table;
struct scatterlist *sg;
int i;
int vmid = get_secure_vmid(buffer->flags);
if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) &&
!(buffer->flags & ION_FLAG_POOL_FORCE_ALLOC)) {
mutex_lock(&buffer->lock);
if (hlos_accessible_buffer(buffer))
ion_buffer_zero(buffer);
if (lock_state && lock_state->locked)
pr_warn("%s: buffer is locked while being freed\n",
__func__);
mutex_unlock(&buffer->lock);
} else if (vmid > 0) {
if (ion_hyp_unassign_sg(table, &vmid, 1, true))
return;
}
/* zero the buffer before goto page pool */
if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE))
ion_buffer_zero(buffer);
for_each_sg(table->sgl, sg, table->nents, i)
free_buffer_page(sys_heap, buffer, sg_page(sg),
get_order(sg->length));
free_buffer_page(sys_heap, buffer, sg_page(sg));
sg_free_table(table);
kfree(table);
kfree(buffer->priv_virt);
}
static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,
int nr_to_scan)
int nr_to_scan)
{
struct ion_page_pool *pool;
struct ion_system_heap *sys_heap;
int nr_total = 0;
int i, j, nr_freed = 0;
int i, nr_freed;
int only_scan = 0;
struct ion_page_pool *pool;
sys_heap = to_system_heap(heap);
sys_heap = container_of(heap, struct ion_system_heap, heap);
if (!nr_to_scan)
only_scan = 1;
/* shrink the pools starting from lower order ones */
for (i = NUM_ORDERS - 1; i >= 0; i--) {
nr_freed = 0;
for (i = 0; i < NUM_ORDERS; i++) {
pool = sys_heap->pools[i];
for (j = 0; j < VMID_LAST; j++) {
if (is_secure_vmid_valid(j))
nr_freed +=
ion_secure_page_pool_shrink(sys_heap,
j, i,
nr_to_scan);
}
if (only_scan) {
nr_total += ion_page_pool_shrink(pool,
gfp_mask,
nr_to_scan);
pool = sys_heap->uncached_pools[i];
nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan);
pool = sys_heap->cached_pools[i];
nr_freed += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan);
nr_total += nr_freed;
if (!only_scan) {
} else {
nr_freed = ion_page_pool_shrink(pool,
gfp_mask,
nr_to_scan);
nr_to_scan -= nr_freed;
/* shrink completed */
nr_total += nr_freed;
if (nr_to_scan <= 0)
break;
}
}
return nr_total;
}
static void ion_system_heap_destroy_pools(struct ion_page_pool **pools)
{
int i;
for (i = 0; i < NUM_ORDERS; i++)
if (pools[i])
ion_page_pool_destroy(pools[i]);
}
static int ion_system_heap_create_pools(struct ion_page_pool **pools)
{
int i;
for (i = 0; i < NUM_ORDERS; i++) {
struct ion_page_pool *pool;
gfp_t gfp_flags = low_order_gfp_flags;
if (orders[i] > 4)
gfp_flags = high_order_gfp_flags;
pool = ion_page_pool_create(gfp_flags, orders[i]);
if (!pool)
goto err_create_pool;
pools[i] = pool;
}
return 0;
err_create_pool:
ion_system_heap_destroy_pools(pools);
return -ENOMEM;
}
static struct ion_heap_ops system_heap_ops = {
.allocate = ion_system_heap_allocate,
.free = ion_system_heap_free,
.shrink = ion_system_heap_shrink,
};
static int ion_system_heap_debug_show(struct ion_heap *heap,
struct seq_file *s, void *unused)
{
struct ion_system_heap *sys_heap;
bool use_seq = s;
unsigned long uncached_total = 0;
unsigned long cached_total = 0;
unsigned long secure_total = 0;
struct ion_page_pool *pool;
int i, j;
sys_heap = to_system_heap(heap);
for (i = 0; i < NUM_ORDERS; i++) {
pool = sys_heap->uncached_pools[i];
if (use_seq) {
seq_printf(s,
"%d order %u highmem pages in uncached pool = %lu total\n",
pool->high_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->high_count);
seq_printf(s,
"%d order %u lowmem pages in uncached pool = %lu total\n",
pool->low_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->low_count);
}
uncached_total += (1 << pool->order) * PAGE_SIZE *
pool->high_count;
uncached_total += (1 << pool->order) * PAGE_SIZE *
pool->low_count;
static struct ion_system_heap system_heap = {
.heap = {
.ops = &system_heap_ops,
.type = ION_HEAP_TYPE_SYSTEM,
.flags = ION_HEAP_FLAG_DEFER_FREE,
.name = "ion_system_heap",
}
for (i = 0; i < NUM_ORDERS; i++) {
pool = sys_heap->cached_pools[i];
if (use_seq) {
seq_printf(s,
"%d order %u highmem pages in cached pool = %lu total\n",
pool->high_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->high_count);
seq_printf(s,
"%d order %u lowmem pages in cached pool = %lu total\n",
pool->low_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->low_count);
}
cached_total += (1 << pool->order) * PAGE_SIZE *
pool->high_count;
cached_total += (1 << pool->order) * PAGE_SIZE *
pool->low_count;
}
for (i = 0; i < NUM_ORDERS; i++) {
for (j = 0; j < VMID_LAST; j++) {
if (!is_secure_vmid_valid(j))
continue;
pool = sys_heap->secure_pools[j][i];
if (use_seq) {
seq_printf(s,
"VMID %d: %d order %u highmem pages in secure pool = %lu total\n",
j, pool->high_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->high_count);
seq_printf(s,
"VMID %d: %d order %u lowmem pages in secure pool = %lu total\n",
j, pool->low_count, pool->order,
(1 << pool->order) * PAGE_SIZE *
pool->low_count);
}
secure_total += (1 << pool->order) * PAGE_SIZE *
pool->high_count;
secure_total += (1 << pool->order) * PAGE_SIZE *
pool->low_count;
}
}
if (use_seq) {
seq_puts(s, "--------------------------------------------\n");
seq_printf(s, "uncached pool = %lu cached pool = %lu secure pool = %lu\n",
uncached_total, cached_total, secure_total);
seq_printf(s, "pool total (uncached + cached + secure) = %lu\n",
uncached_total + cached_total + secure_total);
seq_puts(s, "--------------------------------------------\n");
} else {
pr_info("-------------------------------------------------\n");
pr_info("uncached pool = %lu cached pool = %lu secure pool = %lu\n",
uncached_total, cached_total, secure_total);
pr_info("pool total (uncached + cached + secure) = %lu\n",
uncached_total + cached_total + secure_total);
pr_info("-------------------------------------------------\n");
}
return 0;
}
static struct msm_ion_heap_ops msm_system_heap_ops = {
.debug_show = ion_system_heap_debug_show,
};
static void ion_system_heap_destroy_pools(struct ion_page_pool **pools)
static int __init ion_system_heap_init(void)
{
int i;
int ret = ion_system_heap_create_pools(system_heap.pools);
if (ret)
return ret;
for (i = 0; i < NUM_ORDERS; i++)
if (pools[i]) {
ion_page_pool_destroy(pools[i]);
pools[i] = NULL;
}
return ion_device_add_heap(&system_heap.heap);
}
/**
* ion_system_heap_create_pools - Creates pools for all orders
*
* If this fails you don't need to destroy any pools. It's all or
* nothing. If it succeeds you'll eventually need to use
* ion_system_heap_destroy_pools to destroy the pools.
*/
static int ion_system_heap_create_pools(struct ion_system_heap *sys_heap,
struct ion_page_pool **pools,
bool cached)
static void __exit ion_system_heap_exit(void)
{
int i;
for (i = 0; i < NUM_ORDERS; i++) {
struct ion_page_pool *pool;
gfp_t gfp_flags = low_order_gfp_flags;
if (orders[i])
gfp_flags = high_order_gfp_flags;
pool = ion_page_pool_create(gfp_flags, orders[i], cached);
if (!pool)
goto err_create_pool;
pool->heap_dev = sys_heap->heap.dev;
pools[i] = pool;
}
return 0;
err_create_pool:
ion_system_heap_destroy_pools(pools);
return -ENOMEM;
}
static int ion_sys_heap_worker(void *data)
{
struct ion_page_pool **pools = (struct ion_page_pool **)data;
int i;
for (;;) {
for (i = 0; i < NUM_ORDERS; i++) {
if (pool_count_below_lowmark(pools[i]))
ion_page_pool_refill(pools[i]);
}
set_current_state(TASK_INTERRUPTIBLE);
if (unlikely(kthread_should_stop())) {
set_current_state(TASK_RUNNING);
break;
}
schedule();
set_current_state(TASK_RUNNING);
}
return 0;
}
static struct task_struct *ion_create_kworker(struct ion_page_pool **pools,
bool cached)
{
struct sched_attr attr = { 0 };
struct task_struct *thread;
int ret;
char *buf;
attr.sched_nice = ION_KTHREAD_NICE_VAL;
buf = cached ? "cached" : "uncached";
thread = kthread_run(ion_sys_heap_worker, pools,
"ion-pool-%s-worker", buf);
if (IS_ERR(thread)) {
pr_err("%s: failed to create %s worker thread: %ld\n",
__func__, buf, PTR_ERR(thread));
return thread;
}
ret = sched_setattr(thread, &attr);
if (ret) {
kthread_stop(thread);
pr_warn("%s: failed to set task priority for %s worker thread: ret = %d\n",
__func__, buf, ret);
return ERR_PTR(ret);
}
return thread;
}
struct ion_heap *ion_system_heap_create(struct ion_platform_heap *data)
{
struct ion_system_heap *heap;
int ret = -ENOMEM;
int i;
heap = kzalloc(sizeof(*heap), GFP_KERNEL);
if (!heap)
return ERR_PTR(-ENOMEM);
heap->heap.dev = data->priv;
heap->heap.msm_heap_ops = &msm_system_heap_ops;
heap->heap.ion_heap.ops = &system_heap_ops;
heap->heap.ion_heap.buf_ops = msm_ion_dma_buf_ops;
heap->heap.ion_heap.type = ION_HEAP_TYPE_SYSTEM;
heap->heap.ion_heap.flags = ION_HEAP_FLAG_DEFER_FREE;
for (i = 0; i < VMID_LAST; i++)
if (is_secure_vmid_valid(i))
if (ion_system_heap_create_pools(heap,
heap->secure_pools[i],
false))
goto destroy_secure_pools;
if (ion_system_heap_create_pools(heap, heap->uncached_pools, false))
goto destroy_secure_pools;
if (ion_system_heap_create_pools(heap, heap->cached_pools, true))
goto destroy_uncached_pools;
if (pool_auto_refill_en) {
heap->kworker[ION_KTHREAD_UNCACHED] =
ion_create_kworker(heap->uncached_pools, false);
if (IS_ERR(heap->kworker[ION_KTHREAD_UNCACHED])) {
ret = PTR_ERR(heap->kworker[ION_KTHREAD_UNCACHED]);
goto destroy_pools;
}
heap->kworker[ION_KTHREAD_CACHED] =
ion_create_kworker(heap->cached_pools, true);
if (IS_ERR(heap->kworker[ION_KTHREAD_CACHED])) {
kthread_stop(heap->kworker[ION_KTHREAD_UNCACHED]);
ret = PTR_ERR(heap->kworker[ION_KTHREAD_CACHED]);
goto destroy_pools;
}
}
mutex_init(&heap->split_page_mutex);
return &heap->heap.ion_heap;
destroy_pools:
ion_system_heap_destroy_pools(heap->cached_pools);
destroy_uncached_pools:
ion_system_heap_destroy_pools(heap->uncached_pools);
destroy_secure_pools:
for (i = 0; i < VMID_LAST; i++) {
if (heap->secure_pools[i])
ion_system_heap_destroy_pools(heap->secure_pools[i]);
}
kfree(heap);
return ERR_PTR(ret);
ion_device_remove_heap(&system_heap.heap);
ion_system_heap_destroy_pools(system_heap.pools);
}
module_init(ion_system_heap_init);
module_exit(ion_system_heap_exit);
MODULE_LICENSE("GPL v2");

33
drivers/staging/android/ion/heaps/ion_system_secure_heap.c
View file

@ -10,8 +10,8 @@
#include <linux/kernel.h>
#include "ion_system_secure_heap.h"
#include "ion_system_heap.h"
#include "ion_page_pool.h"
#include "ion_msm_system_heap.h"
#include "ion_msm_page_pool.h"
#include "msm_ion_priv.h"
#include "ion_secure_util.h"
@ -146,19 +146,19 @@ out1:
size_t ion_system_secure_heap_page_pool_total(struct ion_heap *heap,
int vmid_flags)
{
struct ion_system_heap *sys_heap;
struct ion_page_pool *pool;
struct ion_msm_system_heap *sys_heap;
struct ion_msm_page_pool *pool;
size_t total = 0;
int vmid, i;
sys_heap = to_system_heap(heap);
sys_heap = to_msm_system_heap(heap);
vmid = get_secure_vmid(vmid_flags);
if (vmid < 0)
return 0;
for (i = 0; i < NUM_ORDERS; i++) {
pool = sys_heap->secure_pools[vmid][i];
total += ion_page_pool_total(pool, true);
total += ion_msm_page_pool_total(pool, true);
}
return total << PAGE_SHIFT;
@ -342,21 +342,21 @@ struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *unused)
return &heap->heap.ion_heap;
}
struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap,
struct page *alloc_from_secure_pool_order(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer,
unsigned long order)
{
int vmid = get_secure_vmid(buffer->flags);
struct ion_page_pool *pool;
struct ion_msm_page_pool *pool;
if (!is_secure_vmid_valid(vmid))
return ERR_PTR(-EINVAL);
pool = heap->secure_pools[vmid][order_to_index(order)];
return ion_page_pool_alloc_pool_only(pool);
return ion_msm_page_pool_alloc_pool_only(pool);
}
struct page *split_page_from_secure_pool(struct ion_system_heap *heap,
struct page *split_page_from_secure_pool(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer)
{
int i, j;
@ -400,7 +400,7 @@ got_page:
return page;
}
int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap,
int ion_secure_page_pool_shrink(struct ion_msm_system_heap *sys_heap,
int vmid, int order_idx, int nr_to_scan)
{
int ret, freed = 0;
@ -408,14 +408,15 @@ int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap,
struct page *page, *tmp;
struct sg_table sgt;
struct scatterlist *sg;
struct ion_page_pool *pool = sys_heap->secure_pools[vmid][order_idx];
struct ion_msm_page_pool *pool =
sys_heap->secure_pools[vmid][order_idx];
LIST_HEAD(pages);
if (nr_to_scan == 0)
return ion_page_pool_total(pool, true);
return ion_msm_page_pool_total(pool, true);
while (freed < nr_to_scan) {
page = ion_page_pool_alloc_pool_only(pool);
page = ion_msm_page_pool_alloc_pool_only(pool);
if (IS_ERR(page))
break;
list_add(&page->lru, &pages);
@ -443,7 +444,7 @@ int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap,
list_for_each_entry_safe(page, tmp, &pages, lru) {
list_del(&page->lru);
ion_page_pool_free_immediate(pool, page);
ion_msm_page_pool_free_immediate(pool, page);
}
sg_free_table(&sgt);
@ -455,7 +456,7 @@ out1:
/* Restore pages to secure pool */
list_for_each_entry_safe(page, tmp, &pages, lru) {
list_del(&page->lru);
ion_page_pool_free(pool, page);
ion_msm_page_pool_free(pool, page);
}
return 0;
out3:

10
drivers/staging/android/ion/heaps/ion_system_secure_heap.h
View file

@ -1,21 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
* Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
*/
#include "msm_ion_priv.h"
#include "ion_system_heap.h"
#include "ion_msm_system_heap.h"
#ifndef _ION_SYSTEM_SECURE_HEAP_H
#define _ION_SYSTEM_SECURE_HEAP_H
struct page *alloc_from_secure_pool_order(struct ion_system_heap *heap,
struct page *alloc_from_secure_pool_order(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer,
unsigned long order);
struct page *split_page_from_secure_pool(struct ion_system_heap *heap,
struct page *split_page_from_secure_pool(struct ion_msm_system_heap *heap,
struct ion_buffer *buffer);
int ion_secure_page_pool_shrink(struct ion_system_heap *sys_heap,
int ion_secure_page_pool_shrink(struct ion_msm_system_heap *sys_heap,
int vmid, int order_idx, int nr_to_scan);
#endif /* _ION_SYSTEM_SECURE_HEAP_H */

6
drivers/staging/android/ion/heaps/msm_ion.c
View file

@ -100,7 +100,7 @@ static struct heap_types_info {
const char *name;
int heap_type;
} heap_types_info[] = {
MAKE_HEAP_TYPE_MAPPING(SYSTEM),
MAKE_HEAP_TYPE_MAPPING(MSM_SYSTEM),
MAKE_HEAP_TYPE_MAPPING(MSM_CARVEOUT),
MAKE_HEAP_TYPE_MAPPING(SECURE_CARVEOUT),
MAKE_HEAP_TYPE_MAPPING(DMA),
@ -158,8 +158,8 @@ static struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data)
int heap_type = heap_data->type;
switch (heap_type) {
case ION_HEAP_TYPE_SYSTEM:
heap = ion_system_heap_create(heap_data);
case ION_HEAP_TYPE_MSM_SYSTEM:
heap = ion_msm_system_heap_create(heap_data);
break;
case ION_HEAP_TYPE_MSM_CARVEOUT:
heap = ion_carveout_heap_create(heap_data);

2
drivers/staging/android/ion/heaps/msm_ion_priv.h
View file

@ -156,7 +156,7 @@ bool ion_buffer_cached(struct ion_buffer *buffer);
* heaps as appropriate.
*/
struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused);
struct ion_heap *ion_msm_system_heap_create(struct ion_platform_heap *unused);
struct ion_heap *ion_system_secure_heap_create(struct ion_platform_heap *heap);

15
drivers/staging/android/ion/ion_buffer.c
View file

@ -12,6 +12,7 @@
#include <linux/dma-noncoherent.h>
#include "ion_private.h"
#define ION_SYSTEM_HEAP_ID BIT(25)
/* this function should only be called while dev->lock is held */
static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
@ -115,11 +116,25 @@ struct ion_buffer *ion_buffer_alloc(struct ion_device *dev, size_t len,
{
struct ion_buffer *buffer = NULL;
struct ion_heap *heap;
char task_comm[TASK_COMM_LEN];
if (!dev || !len) {
return ERR_PTR(-EINVAL);
}
/*
* Temporarily reroute generic system heap allocations to the MSM system
* heap. Once clients have stopped using the generic system heap ID, we
* can remove this.
*/
if (heap_id_mask & ION_HEAP_SYSTEM) {
get_task_comm(task_comm, current->group_leader);
pr_warn_ratelimited("%s: Rerouting allocation from generic sys heap to msm sys heap for %s-%d\n",
__func__, task_comm, current->tgid);
heap_id_mask &= ~ION_HEAP_SYSTEM;
heap_id_mask |= ION_SYSTEM_HEAP_ID;
}
/*
* traverse the list of heaps available in this system in priority
* order. If the heap type is supported by the client, and matches the

1
include/uapi/linux/msm_ion.h
View file

@ -22,6 +22,7 @@ enum msm_ion_heap_types {
ION_HEAP_TYPE_HYP_CMA,
ION_HEAP_TYPE_MSM_CARVEOUT,
ION_HEAP_TYPE_SECURE_CARVEOUT,
ION_HEAP_TYPE_MSM_SYSTEM,
};
/**

3
include/uapi/linux/msm_ion_ids.h
View file

@ -16,7 +16,7 @@
* possible fallbacks)
*/
#define ION_SYSTEM_HEAP_ID ION_BIT(0)
/* ION_BIT(0) is reserved for the generic system heap. */
#define ION_QSECOM_TA_HEAP_ID ION_BIT(1)
#define ION_CAMERA_HEAP_ID ION_BIT(2)
#define ION_DISPLAY_HEAP_ID ION_BIT(3)
@ -31,6 +31,7 @@
#define ION_SPSS_HEAP_ID ION_BIT(14)
#define ION_SECURE_CARVEOUT_HEAP_ID ION_BIT(15)
#define ION_TUI_CARVEOUT_HEAP_ID ION_BIT(16)
#define ION_SYSTEM_HEAP_ID ION_BIT(25)
#define ION_HEAP_ID_RESERVED ION_BIT(31)
#endif /* _MSM_ION_IDS_H */