From ecba20dd597e9a37e40d59cc3574e32a7a4d0471 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Mon, 22 Apr 2024 14:24:59 -0700 Subject: [PATCH] ANDROID: 16K: Handle pad VMA splits and merges In some cases a VMA with padding representation may be split, and therefore the padding flags must be updated accordingly. There are 3 cases to handle: Given: | DDDDPPPP | where: - D represents 1 page of data; - P represents 1 page of padding; - | represents the boundaries (start/end) of the VMA 1) Split exactly at the padding boundary | DDDDPPPP | --> | DDDD | PPPP | - Remove padding flags from the first VMA. - The second VMA is all padding 2) Split within the padding area | DDDDPPPP | --> | DDDDPP | PP | - Subtract the length of the second VMA from the first VMA's padding. - The second VMA is all padding, adjust its padding length (flags) 3) Split within the data area | DDDDPPPP | --> | DD | DDPPPP | - Remove padding flags from the first VMA. - The second VMA is has the same padding as from before the split. To simplify the semantics merging of padding VMAs is not allowed. If a split produces a VMA that is entirely padding, show_[s]maps() only outputs the padding VMA entry (as the data entry is of length 0). Bug: 330117029 Bug: 327600007 Bug: 330767927 Bug: 328266487 Bug: 329803029 Change-Id: Ie2628ced5512e2c7f8af25fabae1f38730c8bb1a Signed-off-by: Kalesh Singh --- fs/proc/task_mmu.c | 7 ++-- include/linux/pgsize_migration.h | 34 ++++++++++++++++ mm/mlock.c | 3 +- mm/mmap.c | 7 +++- mm/mprotect.c | 3 +- mm/pgsize_migration.c | 69 +++++++++++++++++++++++++++++++- 6 files changed, 115 insertions(+), 8 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 05ffaa1f55c9..4843a0473487 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -423,7 +423,8 @@ static int show_map(struct seq_file *m, void *v) struct vm_area_struct *pad_vma = get_pad_vma(v); struct vm_area_struct *vma = get_data_vma(v); - show_map_vma(m, vma); + if (vma_pages(vma)) + show_map_vma(m, vma); show_map_pad_vma(vma, pad_vma, m, show_map_vma); @@ -913,7 +914,6 @@ static void show_smap_vma(struct seq_file *m, void *v) if (arch_pkeys_enabled()) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); show_smap_vma_flags(m, vma); - } static int show_smap(struct seq_file *m, void *v) @@ -921,7 +921,8 @@ static int show_smap(struct seq_file *m, void *v) struct vm_area_struct *pad_vma = get_pad_vma(v); struct vm_area_struct *vma = get_data_vma(v); - show_smap_vma(m, vma); + if (vma_pages(vma)) + show_smap_vma(m, vma); show_map_pad_vma(vma, pad_vma, m, (show_pad_vma_fn)show_smap_vma); diff --git a/include/linux/pgsize_migration.h b/include/linux/pgsize_migration.h index 7ab0f288bcf9..5c47ec28ea7d 100644 --- a/include/linux/pgsize_migration.h +++ b/include/linux/pgsize_migration.h @@ -61,6 +61,9 @@ extern struct vm_area_struct *get_data_vma(struct vm_area_struct *vma); extern void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, struct seq_file *m, show_pad_vma_fn func); + +extern void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, + unsigned long addr, int new_below); #else /* PAGE_SIZE != SZ_4K || !defined(CONFIG_64BIT) */ static inline void vma_set_pad_pages(struct vm_area_struct *vma, unsigned long nr_pages) @@ -92,10 +95,41 @@ static inline void show_map_pad_vma(struct vm_area_struct *vma, struct seq_file *m, show_pad_vma_fn func) { } + +static inline void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, + unsigned long addr, int new_below) +{ +} #endif /* PAGE_SIZE == SZ_4K && defined(CONFIG_64BIT) */ static inline unsigned long vma_data_pages(struct vm_area_struct *vma) { return vma_pages(vma) - vma_pad_pages(vma); } + +/* + * Sets the correct padding bits / flags for a VMA split. + */ +static inline unsigned long vma_pad_fixup_flags(struct vm_area_struct *vma, + unsigned long newflags) +{ + if (newflags & VM_PAD_MASK) + return (newflags & ~VM_PAD_MASK) | (vma->vm_flags & VM_PAD_MASK); + else + return newflags; +} + +/* + * Merging of padding VMAs is uncommon, as padding is only allowed + * from the linker context. + * + * To simplify the semantics, adjacent VMAs with padding are not + * allowed to merge. + */ +static inline bool is_mergable_pad_vma(struct vm_area_struct *vma, + unsigned long vm_flags) +{ + /* Padding VMAs cannot be merged with other padding or real VMAs */ + return !((vma->vm_flags | vm_flags) & VM_PAD_MASK); +} #endif /* _LINUX_PAGE_SIZE_MIGRATION_H */ diff --git a/mm/mlock.c b/mm/mlock.c index 646acba3045b..c76fb104507a 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -571,7 +572,7 @@ success: */ if (lock) - vma->vm_flags = newflags; + vma->vm_flags = vma_pad_fixup_flags(vma, newflags); else munlock_vma_pages_range(vma, start, end); diff --git a/mm/mmap.c b/mm/mmap.c index 8f1716292c63..bcc12a4f0662 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1030,6 +1031,8 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma, return 0; if (vma_get_anon_name(vma) != anon_name) return 0; + if (!is_mergable_pad_vma(vma, vm_flags)) + return 0; return 1; } @@ -2731,8 +2734,10 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new); /* Success. */ - if (!err) + if (!err) { + split_pad_vma(vma, new, addr, new_below); return 0; + } /* Clean everything up if vma_adjust failed. */ if (new->vm_ops && new->vm_ops->close) diff --git a/mm/mprotect.c b/mm/mprotect.c index 87a59d46b4de..99cf8a97b2e4 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -454,7 +455,7 @@ success: * vm_flags and vm_page_prot are protected by the mmap_sem * held in write mode. */ - vma->vm_flags = newflags; + vma->vm_flags = vma_pad_fixup_flags(vma, newflags); dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); vma_set_page_prot(vma); diff --git a/mm/pgsize_migration.c b/mm/pgsize_migration.c index e427950a6d6d..305036b3d45a 100644 --- a/mm/pgsize_migration.c +++ b/mm/pgsize_migration.c @@ -114,6 +114,7 @@ void vma_set_pad_pages(struct vm_area_struct *vma, if (!is_pgsize_migration_enabled()) return; + vma->vm_flags &= ~VM_PAD_MASK; vma->vm_flags |= (nr_pages << VM_PAD_SHIFT); } @@ -269,10 +270,10 @@ struct vm_area_struct *get_pad_vma(struct vm_area_struct *vma) pad->vm_start = VMA_PAD_START(pad); /* Make the pad vma PROT_NONE */ - pad->vm_flags = pad->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC); + pad->vm_flags &= ~(VM_READ|VM_WRITE|VM_EXEC); /* Remove padding bits */ - pad->vm_flags = pad->vm_flags & ~VM_PAD_MASK; + pad->vm_flags &= ~VM_PAD_MASK; return pad; } @@ -325,5 +326,69 @@ void show_map_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *pad, kfree(pad); kfree(vma); } + +/* + * When splitting a padding VMA there are a couple of cases to handle. + * + * Given: + * + * | DDDDPPPP | + * + * where: + * - D represents 1 page of data; + * - P represents 1 page of padding; + * - | represents the boundaries (start/end) of the VMA + * + * + * 1) Split exactly at the padding boundary + * + * | DDDDPPPP | --> | DDDD | PPPP | + * + * - Remove padding flags from the first VMA. + * - The second VMA is all padding + * + * 2) Split within the padding area + * + * | DDDDPPPP | --> | DDDDPP | PP | + * + * - Subtract the length of the second VMA from the first VMA's padding. + * - The second VMA is all padding, adjust its padding length (flags) + * + * 3) Split within the data area + * + * | DDDDPPPP | --> | DD | DDPPPP | + * + * - Remove padding flags from the first VMA. + * - The second VMA is has the same padding as from before the split. + */ +void split_pad_vma(struct vm_area_struct *vma, struct vm_area_struct *new, + unsigned long addr, int new_below) +{ + unsigned long nr_pad_pages = vma_pad_pages(vma); + unsigned long nr_vma2_pages; + struct vm_area_struct *first; + struct vm_area_struct *second; + + if (!nr_pad_pages) + return; + + if (new_below) { + first = new; + second = vma; + } else { + first = vma; + second = new; + } + + nr_vma2_pages = vma_pages(second); + + if (nr_vma2_pages >= nr_pad_pages) { /* Case 1 & 3 */ + first->vm_flags &= ~VM_PAD_MASK; + vma_set_pad_pages(second, nr_pad_pages); + } else { /* Case 2 */ + vma_set_pad_pages(first, nr_pad_pages - nr_vma2_pages); + vma_set_pad_pages(second, nr_vma2_pages); + } +} #endif /* PAGE_SIZE == SZ_4K */ #endif /* CONFIG_64BIT */