From a8f20aa6dc46e37eb46e90e429e34c597c9517a0 Mon Sep 17 00:00:00 2001 From: Susheel Khiani Date: Tue, 8 Sep 2015 15:05:43 +0530 Subject: [PATCH 1/6] msm: Allow lowmem to be non contiguous and mixed Currently on 32 bit systems, virtual space above PAGE_OFFSET is reserved for direct mapped lowmem and part of virtual address space is reserved for vmalloc. We want to optimize such as to have as much direct mapped memory as possible since there is penalty for mapping/unmapping highmem. Now, we may have an image that is expected to have a lifetime of the entire system and is reserved in physical region that would be part of direct mapped lowmem. The physical memory which is thus reserved is never used by Linux. This means that even though the system is not actually accessing the virtual memory corresponding to the reserved physical memory, we are still losing that portion of direct mapped lowmem space. So by allowing lowmem to be non contiguous we can give this unused virtual address space of reserved region back for use in vmalloc. Change-Id: I980b3dfafac71884dcdcb8cd2e4a6363cde5746a Signed-off-by: Susheel Khiani Signed-off-by: Qingqing Zhou --- arch/arm/mm/ioremap.c | 3 ++- arch/arm/mm/mmu.c | 40 ++++++++++++++++++++++++++++++++++++++-- include/linux/vmalloc.h | 2 ++ mm/vmalloc.c | 26 ++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index d42b93316183..60da4db46f77 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -93,7 +93,8 @@ void __init add_static_vm_early(struct static_vm *svm) void *vaddr; vm = &svm->vm; - vm_area_add_early(vm); + if (!vm_area_check_early(vm)) + vm_area_add_early(vm); vaddr = vm->addr; list_for_each_entry(curr_svm, &static_vmlist, list) { diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 5c3ab53fb8e2..10ab37a2cfba 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1452,12 +1452,21 @@ static void __init map_lowmem(void) struct memblock_region *reg; phys_addr_t kernel_x_start = round_down(__pa(KERNEL_START), SECTION_SIZE); phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); + struct static_vm *svm; + phys_addr_t start; + phys_addr_t end; + unsigned long vaddr; + unsigned long pfn; + unsigned long length; + unsigned int type; + int nr = 0; /* Map all the lowmem memory banks. */ for_each_memblock(memory, reg) { - phys_addr_t start = reg->base; - phys_addr_t end = start + reg->size; struct map_desc map; + start = reg->base; + end = start + reg->size; + nr++; if (memblock_is_nomap(reg)) continue; @@ -1509,6 +1518,33 @@ static void __init map_lowmem(void) } } } + svm = memblock_alloc(sizeof(*svm) * nr, __alignof__(*svm)); + + for_each_memblock(memory, reg) { + struct vm_struct *vm; + + start = reg->base; + end = start + reg->size; + + if (end > arm_lowmem_limit) + end = arm_lowmem_limit; + if (start >= end) + break; + + vm = &svm->vm; + pfn = __phys_to_pfn(start); + vaddr = __phys_to_virt(start); + length = end - start; + type = MT_MEMORY_RW; + + vm->addr = (void *)(vaddr & PAGE_MASK); + vm->size = PAGE_ALIGN(length + (vaddr & ~PAGE_MASK)); + vm->phys_addr = __pfn_to_phys(pfn); + vm->flags = VM_LOWMEM; + vm->flags |= VM_ARM_MTYPE(type); + vm->caller = map_lowmem; + add_static_vm_early(svm++); + } } #ifdef CONFIG_ARM_PV_FIXUP diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 01a1334c5fc5..b81467562d2b 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -27,6 +27,7 @@ struct notifier_block; /* in notifier.h */ * vfree_atomic(). */ #define VM_FLUSH_RESET_PERMS 0x00000100 /* Reset direct map and flush TLB on unmap */ +#define VM_LOWMEM 0x00000200 /* Tracking of direct mapped lowmem */ /* bits [20..32] reserved for arch specific ioremap internals */ @@ -203,6 +204,7 @@ extern long vwrite(char *buf, char *addr, unsigned long count); extern struct list_head vmap_area_list; extern __init void vm_area_add_early(struct vm_struct *vm); extern __init void vm_area_register_early(struct vm_struct *vm, size_t align); +extern __init int vm_area_check_early(struct vm_struct *vm); #ifdef CONFIG_SMP # ifdef CONFIG_MMU diff --git a/mm/vmalloc.c b/mm/vmalloc.c index ad4d00bd7914..51dbf88fcdb3 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1806,6 +1806,32 @@ EXPORT_SYMBOL(vm_map_ram); static struct vm_struct *vmlist __initdata; +/** + * vm_area_check_early - check if vmap area is already mapped + * @vm: vm_struct to be checked + * + * This function is used to check if the vmap area has been + * mapped already. @vm->addr, @vm->size and @vm->flags should + * contain proper values. + * + */ +int __init vm_area_check_early(struct vm_struct *vm) +{ + struct vm_struct *tmp, **p; + + BUG_ON(vmap_initialized); + for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { + if (tmp->addr >= vm->addr) { + if (tmp->addr < vm->addr + vm->size) + return 1; + } else { + if (tmp->addr + tmp->size > vm->addr) + return 1; + } + } + return 0; +} + /** * vm_area_add_early - add vmap area early during boot * @vm: vm_struct to add From 674893e4529a0af02fbe883133f6e8bd62ec98cc Mon Sep 17 00:00:00 2001 From: Susheel Khiani Date: Thu, 22 Aug 2013 13:46:07 -0700 Subject: [PATCH 2/6] mm: Update is_vmalloc_addr to account for vmalloc savings is_vmalloc_addr currently assumes that all vmalloc addresses exist between VMALLOC_START and VMALLOC_END. This may not be the case when interleaving vmalloc and lowmem. Update the is_vmalloc_addr to properly check for this. Correspondingly we need to ensure that VMALLOC_TOTAL accounts for all the vmalloc regions when CONFIG_ENABLE_VMALLOC_SAVING is enabled. Change-Id: I5def3d6ae1a4de59ea36f095b8c73649a37b1f36 Signed-off-by: Susheel Khiani Signed-off-by: Zhenhua Huang Signed-off-by: Qingqing Zhou Signed-off-by: Sudarshan Rajagopalan --- arch/arm/mm/mmu.c | 1 + include/linux/mm.h | 5 +++++ include/linux/vmalloc.h | 11 ++++++++++ mm/vmalloc.c | 48 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 10ab37a2cfba..4e4522f30898 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1544,6 +1544,7 @@ static void __init map_lowmem(void) vm->flags |= VM_ARM_MTYPE(type); vm->caller = map_lowmem; add_static_vm_early(svm++); + mark_vmalloc_reserved_area(vm->addr, vm->size); } } diff --git a/include/linux/mm.h b/include/linux/mm.h index 3a97481a5383..2d2df6838ded 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -675,6 +675,10 @@ unsigned long vmalloc_to_pfn(const void *addr); * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there * is no special casing required. */ + +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +extern bool is_vmalloc_addr(const void *x); +#else static inline bool is_vmalloc_addr(const void *x) { #ifdef CONFIG_MMU @@ -685,6 +689,7 @@ static inline bool is_vmalloc_addr(const void *x) return false; #endif } +#endif //CONFIG_ENABLE_VMALLOC_SAVING #ifndef is_ioremap_addr #define is_ioremap_addr(x) is_vmalloc_addr(x) diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index b81467562d2b..7e1c887d10a4 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -205,6 +205,12 @@ extern struct list_head vmap_area_list; extern __init void vm_area_add_early(struct vm_struct *vm); extern __init void vm_area_register_early(struct vm_struct *vm, size_t align); extern __init int vm_area_check_early(struct vm_struct *vm); +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +extern void mark_vmalloc_reserved_area(void *addr, unsigned long size); +#else +static inline void mark_vmalloc_reserved_area(void *addr, unsigned long size) +{ }; +#endif #ifdef CONFIG_SMP # ifdef CONFIG_MMU @@ -230,7 +236,12 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) #endif #ifdef CONFIG_MMU +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +extern unsigned long total_vmalloc_size; +#define VMALLOC_TOTAL total_vmalloc_size +#else #define VMALLOC_TOTAL (VMALLOC_END - VMALLOC_START) +#endif #else #define VMALLOC_TOTAL 0UL #endif diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 51dbf88fcdb3..5bc93e42c3a8 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -249,6 +249,50 @@ static int vmap_page_range(unsigned long start, unsigned long end, return ret; } +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +#define POSSIBLE_VMALLOC_START PAGE_OFFSET + +#define VMALLOC_BITMAP_SIZE ((VMALLOC_END - PAGE_OFFSET) >> \ + PAGE_SHIFT) +#define VMALLOC_TO_BIT(addr) ((addr - PAGE_OFFSET) >> PAGE_SHIFT) +#define BIT_TO_VMALLOC(i) (PAGE_OFFSET + i * PAGE_SIZE) + +unsigned long total_vmalloc_size; +unsigned long vmalloc_reserved; + +DECLARE_BITMAP(possible_areas, VMALLOC_BITMAP_SIZE); + +void mark_vmalloc_reserved_area(void *x, unsigned long size) +{ + unsigned long addr = (unsigned long)x; + + bitmap_set(possible_areas, VMALLOC_TO_BIT(addr), size >> PAGE_SHIFT); + vmalloc_reserved += size; +} + +bool is_vmalloc_addr(const void *x) +{ + unsigned long addr = (unsigned long)x; + + if (addr < POSSIBLE_VMALLOC_START || addr >= VMALLOC_END) + return false; + + if (test_bit(VMALLOC_TO_BIT(addr), possible_areas)) + return false; + + return true; +} +EXPORT_SYMBOL(is_vmalloc_addr); + +static void calc_total_vmalloc_size(void) +{ + total_vmalloc_size = VMALLOC_END - POSSIBLE_VMALLOC_START - + vmalloc_reserved; +} +#else +static void calc_total_vmalloc_size(void) { } +#endif + int is_vmalloc_or_module_addr(const void *x) { /* @@ -1963,6 +2007,7 @@ void __init vmalloc_init(void) * Now we can initialize a free vmap space. */ vmap_init_free_space(); + calc_total_vmalloc_size(); vmap_initialized = true; } @@ -3564,6 +3609,9 @@ static int s_show(struct seq_file *m, void *p) if (is_vmalloc_addr(v->pages)) seq_puts(m, " vpages"); + if (v->flags & VM_LOWMEM) + seq_puts(m, " lowmem"); + show_numa_info(m, v); seq_putc(m, '\n'); From 5bf84788a15a51c71b418d08a0cf7f866d3cf2bb Mon Sep 17 00:00:00 2001 From: Zhenhua Huang Date: Tue, 18 Sep 2018 18:14:58 +0800 Subject: [PATCH 3/6] ARM: enable vmalloc saving For some targets that have less vmalloc space this can be increased by enabling config ENABLE_VMALLOC_SAVING. With this config we can reclaim virtual mappings which remains unused because of non hlos carveout reservations in lowmem. Select the default method of reclaiming virtual memory as vmalloc saving. Change-Id: I249992871babe8c64c34d52ef43bbc7c81636d47 Signed-off-by: Zhenhua Huang Signed-off-by: Qingqing Zhou --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index fae644eb5a46..bb58c9c1979a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1633,7 +1633,7 @@ config ARM_MODULE_PLTS choice prompt "Virtual Memory Reclaim" - default NO_VM_RECLAIM + default ENABLE_VMALLOC_SAVING help Select the method of reclaiming virtual memory. Two values are allowed to choose, one is NO_VM_RECLAIM, the other is From 644acac772930037eaf7f0406f71cde55b2e5f71 Mon Sep 17 00:00:00 2001 From: Susheel Khiani Date: Thu, 3 Sep 2015 18:21:23 +0530 Subject: [PATCH 4/6] msm: Increase the kernel virtual area to include lowmem Even though lowmem is accounted for in vmalloc space, allocation comes only from the region bounded by VMALLOC_START and VMALLOC_END. The kernel virtual area can now allocate from any unmapped region starting from PAGE_OFFSET. Change-Id: I291b9eb443d3f7445fd979bd7b09e9241ff22ba3 Signed-off-by: Neeti Desai Signed-off-by: Susheel Khiani --- mm/vmalloc.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 5bc93e42c3a8..7b14761cde8b 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2172,16 +2172,27 @@ struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, */ struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) { +#ifdef CONFIG_ENABLE_VMALLOC_SAVING + return __get_vm_area_node(size, 1, flags, PAGE_OFFSET, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, + __builtin_return_address(0)); +#else return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0)); +#endif } struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, const void *caller) { +#ifdef CONFIG_ENABLE_VMALLOC_SAVING + return __get_vm_area_node(size, 1, flags, PAGE_OFFSET, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); +#else return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, caller); +#endif } /** From b588978a22ff7daae00afd7b30be97c69197e4cc Mon Sep 17 00:00:00 2001 From: Qingqing Zhou Date: Fri, 3 Jan 2020 20:08:43 +0800 Subject: [PATCH 5/6] mm/vmalloc.c: increase vm area range in __vmalloc_node When the CONFIG_ENABLE_VMALLOC_SAVING is enabled, even though the unused lowmem is used as vmalloc space, but __vmalloc_node() still uses VMALLOC_START as the start point to allocate vm area, but the range [VMALLOC_START, VMALLOC_END] is limited, this may cause vmalloc allocation failure. Change the start point to allocate the vm area from VMALLOC_START to PAGE_OFFSET to make that allocation can be chosen from [PAGE_OFFSET, VMALLOC_START) also. Change-Id: I89c099484846c4bcd9b607c1925f253c165d0dd9 Signed-off-by: Qingqing Zhou --- mm/vmalloc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 7b14761cde8b..66e394d0b8a0 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2631,8 +2631,13 @@ static void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask, pgprot_t prot, int node, const void *caller) { +#ifdef CONFIG_ENABLE_VMALLOC_SAVING + return __vmalloc_node_range(size, align, PAGE_OFFSET, VMALLOC_END, + gfp_mask, prot, 0, node, caller); +#else return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, gfp_mask, prot, 0, node, caller); +#endif } void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) From b85b3f46843797863ec8b329dd30b42fdf03c207 Mon Sep 17 00:00:00 2001 From: Zhenhua Huang Date: Thu, 21 Jun 2018 12:54:40 +0800 Subject: [PATCH 6/6] mm: Kconfig: Add support for config size of purging vmap_area This size is the maximum amount of virtual address space we gather up before attempting to purge with a TLB flush. It is 128M in most cases. With repeated and high size vmalloc operations, it may easily generate more fragments. This is wasting limited vmalloc area, for 32bits. So make it configable and the default multiplier as 8, 32bits only. Change-Id: I68a75acb16d3cff05f8b13c05ae78922269e219f Signed-off-by: Zhenhua Huang --- mm/Kconfig | 10 ++++++++++ mm/vmalloc.c | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mm/Kconfig b/mm/Kconfig index 9b9651d302d5..d02e896f381e 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -628,6 +628,16 @@ config ZSMALLOC_STAT information to userspace via debugfs. If unsure, say N. +config VMAP_LAZY_PURGING_FACTOR + int "multiplier to the size of purged vmap areas" + default "8" if ARM + default "32" + help + It is used as a multiplier to the max VA pages purged in a + single attempt. For 32-bit in order to reduce fragmentation + of vmalloc space, we decrease the default value to "8". + + config GENERIC_EARLY_IOREMAP bool diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 66e394d0b8a0..8e3188fe134e 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -1260,7 +1260,8 @@ static unsigned long lazy_max_pages(void) log = fls(num_online_cpus()); - return log * (32UL * 1024 * 1024 / PAGE_SIZE); + return log * (1UL * CONFIG_VMAP_LAZY_PURGING_FACTOR * + 1024 * 1024 / PAGE_SIZE); } static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0);