diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 61f2f6ff9467..0a5e1e7983ed 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -39,8 +39,17 @@ struct vm_area_struct; #define ___GFP_HARDWALL 0x100000u #define ___GFP_THISNODE 0x200000u #define ___GFP_ACCOUNT 0x400000u +#ifdef CONFIG_CMA_DIRECT_UTILIZATION +#define ___GFP_CMA 0x800000u +#else +#define ___GFP_CMA 0 +#endif #ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x800000u +#ifdef CONFIG_CMA_DIRECT_UTILIZATION +#define ___GFP_NOLOCKDEP 0x1000000u +#else +#define __GFP_NOLOCKDEP 0x800000u +#endif #else #define ___GFP_NOLOCKDEP 0 #endif @@ -57,6 +66,7 @@ struct vm_area_struct; #define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) #define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ +#define __GFP_CMA ((__force gfp_t)___GFP_CMA) #define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) /** @@ -217,7 +227,11 @@ struct vm_area_struct; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ +#ifdef CONFIG_CMA_DIRECT_UTILIZATION +#define __GFP_BITS_SHIFT (24 + IS_ENABLED(CONFIG_LOCKDEP)) +#else #define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP)) +#endif #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** diff --git a/include/linux/highmem.h b/include/linux/highmem.h index ea5cdbd8c2c3..fd093eb91a94 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -205,7 +205,12 @@ static inline struct page * alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { +#ifndef CONFIG_CMA_DIRECT_UTILIZATION return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr); +#else + return __alloc_zeroed_user_highpage(__GFP_MOVABLE|__GFP_CMA, vma, + vaddr); +#endif } static inline void clear_highpage(struct page *page) diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index bda20282746b..f85c1cef213e 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -441,6 +441,10 @@ struct zone { struct pglist_data *zone_pgdat; struct per_cpu_pageset __percpu *pageset; +#ifdef CONFIG_CMA_DIRECT_UTILIZATION + bool cma_alloc; +#endif + #ifndef CONFIG_SPARSEMEM /* * Flags for a pageblock_nr_pages block. See pageblock-flags.h. diff --git a/mm/Kconfig b/mm/Kconfig index 880a5221a5ea..ec445cdbc5cd 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -539,6 +539,23 @@ config CMA_AREAS If unsure, leave the default value "7". +config CMA_DIRECT_UTILIZATION + bool "Allow clients to directly use CMA for page allocations" + depends on CMA && QGKI + help + CMA pages are currently used as a fallback for moveable page + allocations only when all regular moveable pages have been + allocated. This can cause stalls for non-movable allocations, + as non-movable allocations will not consider CMA memory as free + memory, and thus, will cause the system to reclaim movable pages + to satisfy non-movable allocations, ignoring the CMA pages. Since + the system now has movable memory, CMA pages will also be ignored, + since they are used only when there is not enough movable memory + to satisfy a movable request. Enable this option to specify to the + page allocator that movable allocations should be satisfied with + CMA pages, so that CMA pages can be used directly for allocations, + instead of as a fallback, thus, improving CMA utilization. + config MEM_SOFT_DIRTY bool "Track memory changes" depends on CHECKPOINT_RESTORE && HAVE_ARCH_SOFT_DIRTY && PROC_FS diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3771c0fb4d15..93e2aa504852 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2737,9 +2737,12 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, retry: page = __rmqueue_smallest(zone, order, migratetype); + if (unlikely(!page)) { +#ifndef CONFIG_CMA_DIRECT_UTILIZATION if (migratetype == MIGRATE_MOVABLE) page = __rmqueue_cma_fallback(zone, order); +#endif if (!page && __rmqueue_fallback(zone, order, migratetype, alloc_flags)) @@ -2750,6 +2753,28 @@ retry: return page; } +static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + struct page *page = 0; + +retry: +#ifdef CONFIG_CMA_DIRECT_UTILIZATION + if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc) + page = __rmqueue_cma_fallback(zone, order); + else +#endif + page = __rmqueue_smallest(zone, order, migratetype); + + if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype, + alloc_flags)) + goto retry; + + trace_mm_page_alloc_zone_locked(page, order, migratetype); + return page; +} + /* * Obtain a specified number of elements from the buddy allocator, all under * a single hold of the lock, for efficiency. Add them to the supplied list. @@ -2757,14 +2782,20 @@ retry: */ static int rmqueue_bulk(struct zone *zone, unsigned int order, unsigned long count, struct list_head *list, - int migratetype, unsigned int alloc_flags) + int migratetype, unsigned int alloc_flags, int cma) { int i, alloced = 0; spin_lock(&zone->lock); for (i = 0; i < count; ++i) { - struct page *page = __rmqueue(zone, order, migratetype, - alloc_flags); + struct page *page; + + if (cma) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, alloc_flags); + if (unlikely(page == NULL)) break; @@ -3228,7 +3259,7 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, unsigned int alloc_flags, struct per_cpu_pages *pcp, - struct list_head *list) + struct list_head *list, gfp_t gfp_flags) { struct page *page; @@ -3236,7 +3267,8 @@ static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, if (list_empty(list)) { pcp->count += rmqueue_bulk(zone, 0, pcp->batch, list, - migratetype, alloc_flags); + migratetype, alloc_flags, + gfp_flags & __GFP_CMA); if (unlikely(list_empty(list))) return NULL; } @@ -3262,7 +3294,8 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, local_irq_save(flags); pcp = &this_cpu_ptr(zone->pageset)->pcp; list = &pcp->lists[migratetype]; - page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list); + page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list, + gfp_flags); if (page) { __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); zone_statistics(preferred_zone, zone); @@ -3303,8 +3336,14 @@ struct page *rmqueue(struct zone *preferred_zone, if (page) trace_mm_page_alloc_zone_locked(page, order, migratetype); } - if (!page) - page = __rmqueue(zone, order, migratetype, alloc_flags); + if (!page) { + if (gfp_flags & __GFP_CMA) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, + alloc_flags); + } } while (page && check_new_pages(page, order)); spin_unlock(&zone->lock); if (!page) @@ -8438,6 +8477,9 @@ int alloc_contig_range(unsigned long start, unsigned long end, if (ret < 0) return ret; +#ifdef CONFIG_CMA_DIRECT_UTILIZATION + cc.zone->cma_alloc = 1; +#endif /* * In case of -EBUSY, we'd like to know which page causes problem. * So, just fall through. test_pages_isolated() has a tracepoint @@ -8519,6 +8561,9 @@ int alloc_contig_range(unsigned long start, unsigned long end, done: undo_isolate_page_range(pfn_max_align_down(start), pfn_max_align_up(end), migratetype); +#ifdef CONFIG_CMA_DIRECT_UTILIZATION + cc.zone->cma_alloc = 0; +#endif return ret; } #endif /* CONFIG_CONTIG_ALLOC */