diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index fae644eb5a46..bb58c9c1979a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1633,7 +1633,7 @@ config ARM_MODULE_PLTS choice prompt "Virtual Memory Reclaim" - default NO_VM_RECLAIM + default ENABLE_VMALLOC_SAVING help Select the method of reclaiming virtual memory. Two values are allowed to choose, one is NO_VM_RECLAIM, the other is diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index d42b93316183..60da4db46f77 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -93,7 +93,8 @@ void __init add_static_vm_early(struct static_vm *svm) void *vaddr; vm = &svm->vm; - vm_area_add_early(vm); + if (!vm_area_check_early(vm)) + vm_area_add_early(vm); vaddr = vm->addr; list_for_each_entry(curr_svm, &static_vmlist, list) { diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 5c3ab53fb8e2..4e4522f30898 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1452,12 +1452,21 @@ static void __init map_lowmem(void) struct memblock_region *reg; phys_addr_t kernel_x_start = round_down(__pa(KERNEL_START), SECTION_SIZE); phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); + struct static_vm *svm; + phys_addr_t start; + phys_addr_t end; + unsigned long vaddr; + unsigned long pfn; + unsigned long length; + unsigned int type; + int nr = 0; /* Map all the lowmem memory banks. */ for_each_memblock(memory, reg) { - phys_addr_t start = reg->base; - phys_addr_t end = start + reg->size; struct map_desc map; + start = reg->base; + end = start + reg->size; + nr++; if (memblock_is_nomap(reg)) continue; @@ -1509,6 +1518,34 @@ static void __init map_lowmem(void) } } } + svm = memblock_alloc(sizeof(*svm) * nr, __alignof__(*svm)); + + for_each_memblock(memory, reg) { + struct vm_struct *vm; + + start = reg->base; + end = start + reg->size; + + if (end > arm_lowmem_limit) + end = arm_lowmem_limit; + if (start >= end) + break; + + vm = &svm->vm; + pfn = __phys_to_pfn(start); + vaddr = __phys_to_virt(start); + length = end - start; + type = MT_MEMORY_RW; + + vm->addr = (void *)(vaddr & PAGE_MASK); + vm->size = PAGE_ALIGN(length + (vaddr & ~PAGE_MASK)); + vm->phys_addr = __pfn_to_phys(pfn); + vm->flags = VM_LOWMEM; + vm->flags |= VM_ARM_MTYPE(type); + vm->caller = map_lowmem; + add_static_vm_early(svm++); + mark_vmalloc_reserved_area(vm->addr, vm->size); + } } #ifdef CONFIG_ARM_PV_FIXUP diff --git a/include/linux/mm.h b/include/linux/mm.h index 3a97481a5383..2d2df6838ded 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -675,6 +675,10 @@ unsigned long vmalloc_to_pfn(const void *addr); * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there * is no special casing required. */ + +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +extern bool is_vmalloc_addr(const void *x); +#else static inline bool is_vmalloc_addr(const void *x) { #ifdef CONFIG_MMU @@ -685,6 +689,7 @@ static inline bool is_vmalloc_addr(const void *x) return false; #endif } +#endif //CONFIG_ENABLE_VMALLOC_SAVING #ifndef is_ioremap_addr #define is_ioremap_addr(x) is_vmalloc_addr(x) diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 01a1334c5fc5..7e1c887d10a4 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -27,6 +27,7 @@ struct notifier_block; /* in notifier.h */ * vfree_atomic(). */ #define VM_FLUSH_RESET_PERMS 0x00000100 /* Reset direct map and flush TLB on unmap */ +#define VM_LOWMEM 0x00000200 /* Tracking of direct mapped lowmem */ /* bits [20..32] reserved for arch specific ioremap internals */ @@ -203,6 +204,13 @@ extern long vwrite(char *buf, char *addr, unsigned long count); extern struct list_head vmap_area_list; extern __init void vm_area_add_early(struct vm_struct *vm); extern __init void vm_area_register_early(struct vm_struct *vm, size_t align); +extern __init int vm_area_check_early(struct vm_struct *vm); +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +extern void mark_vmalloc_reserved_area(void *addr, unsigned long size); +#else +static inline void mark_vmalloc_reserved_area(void *addr, unsigned long size) +{ }; +#endif #ifdef CONFIG_SMP # ifdef CONFIG_MMU @@ -228,7 +236,12 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) #endif #ifdef CONFIG_MMU +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +extern unsigned long total_vmalloc_size; +#define VMALLOC_TOTAL total_vmalloc_size +#else #define VMALLOC_TOTAL (VMALLOC_END - VMALLOC_START) +#endif #else #define VMALLOC_TOTAL 0UL #endif diff --git a/mm/Kconfig b/mm/Kconfig index 9b9651d302d5..d02e896f381e 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -628,6 +628,16 @@ config ZSMALLOC_STAT information to userspace via debugfs. If unsure, say N. +config VMAP_LAZY_PURGING_FACTOR + int "multiplier to the size of purged vmap areas" + default "8" if ARM + default "32" + help + It is used as a multiplier to the max VA pages purged in a + single attempt. For 32-bit in order to reduce fragmentation + of vmalloc space, we decrease the default value to "8". + + config GENERIC_EARLY_IOREMAP bool diff --git a/mm/vmalloc.c b/mm/vmalloc.c index ad4d00bd7914..8e3188fe134e 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -249,6 +249,50 @@ static int vmap_page_range(unsigned long start, unsigned long end, return ret; } +#ifdef CONFIG_ENABLE_VMALLOC_SAVING +#define POSSIBLE_VMALLOC_START PAGE_OFFSET + +#define VMALLOC_BITMAP_SIZE ((VMALLOC_END - PAGE_OFFSET) >> \ + PAGE_SHIFT) +#define VMALLOC_TO_BIT(addr) ((addr - PAGE_OFFSET) >> PAGE_SHIFT) +#define BIT_TO_VMALLOC(i) (PAGE_OFFSET + i * PAGE_SIZE) + +unsigned long total_vmalloc_size; +unsigned long vmalloc_reserved; + +DECLARE_BITMAP(possible_areas, VMALLOC_BITMAP_SIZE); + +void mark_vmalloc_reserved_area(void *x, unsigned long size) +{ + unsigned long addr = (unsigned long)x; + + bitmap_set(possible_areas, VMALLOC_TO_BIT(addr), size >> PAGE_SHIFT); + vmalloc_reserved += size; +} + +bool is_vmalloc_addr(const void *x) +{ + unsigned long addr = (unsigned long)x; + + if (addr < POSSIBLE_VMALLOC_START || addr >= VMALLOC_END) + return false; + + if (test_bit(VMALLOC_TO_BIT(addr), possible_areas)) + return false; + + return true; +} +EXPORT_SYMBOL(is_vmalloc_addr); + +static void calc_total_vmalloc_size(void) +{ + total_vmalloc_size = VMALLOC_END - POSSIBLE_VMALLOC_START - + vmalloc_reserved; +} +#else +static void calc_total_vmalloc_size(void) { } +#endif + int is_vmalloc_or_module_addr(const void *x) { /* @@ -1216,7 +1260,8 @@ static unsigned long lazy_max_pages(void) log = fls(num_online_cpus()); - return log * (32UL * 1024 * 1024 / PAGE_SIZE); + return log * (1UL * CONFIG_VMAP_LAZY_PURGING_FACTOR * + 1024 * 1024 / PAGE_SIZE); } static atomic_long_t vmap_lazy_nr = ATOMIC_LONG_INIT(0); @@ -1806,6 +1851,32 @@ EXPORT_SYMBOL(vm_map_ram); static struct vm_struct *vmlist __initdata; +/** + * vm_area_check_early - check if vmap area is already mapped + * @vm: vm_struct to be checked + * + * This function is used to check if the vmap area has been + * mapped already. @vm->addr, @vm->size and @vm->flags should + * contain proper values. + * + */ +int __init vm_area_check_early(struct vm_struct *vm) +{ + struct vm_struct *tmp, **p; + + BUG_ON(vmap_initialized); + for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { + if (tmp->addr >= vm->addr) { + if (tmp->addr < vm->addr + vm->size) + return 1; + } else { + if (tmp->addr + tmp->size > vm->addr) + return 1; + } + } + return 0; +} + /** * vm_area_add_early - add vmap area early during boot * @vm: vm_struct to add @@ -1937,6 +2008,7 @@ void __init vmalloc_init(void) * Now we can initialize a free vmap space. */ vmap_init_free_space(); + calc_total_vmalloc_size(); vmap_initialized = true; } @@ -2101,16 +2173,27 @@ struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, */ struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) { +#ifdef CONFIG_ENABLE_VMALLOC_SAVING + return __get_vm_area_node(size, 1, flags, PAGE_OFFSET, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, + __builtin_return_address(0)); +#else return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0)); +#endif } struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, const void *caller) { +#ifdef CONFIG_ENABLE_VMALLOC_SAVING + return __get_vm_area_node(size, 1, flags, PAGE_OFFSET, VMALLOC_END, + NUMA_NO_NODE, GFP_KERNEL, caller); +#else return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, caller); +#endif } /** @@ -2549,8 +2632,13 @@ static void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask, pgprot_t prot, int node, const void *caller) { +#ifdef CONFIG_ENABLE_VMALLOC_SAVING + return __vmalloc_node_range(size, align, PAGE_OFFSET, VMALLOC_END, + gfp_mask, prot, 0, node, caller); +#else return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, gfp_mask, prot, 0, node, caller); +#endif } void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) @@ -3538,6 +3626,9 @@ static int s_show(struct seq_file *m, void *p) if (is_vmalloc_addr(v->pages)) seq_puts(m, " vpages"); + if (v->flags & VM_LOWMEM) + seq_puts(m, " lowmem"); + show_numa_info(m, v); seq_putc(m, '\n');