From bff8c1b3e76640c2a9db783d4e1fb804a916a2cd Mon Sep 17 00:00:00 2001 From: xiewen3 Date: Tue, 16 May 2023 22:43:43 +0800 Subject: [PATCH] LUR: Revert "BACKPORT: mm: multi-gen LRU: kill switch" The LRU function is not compatible with GKI, causing GKI to fail to start, so revert. This reverts commit 16dc2574dd742a5a27f2588601d5b646175f45ba. Change-Id: Idfece6dedd466ebc4f58ebfccd9ba9f59494466e Reviewed-on: https://gerrit.mot.com/2609758 SME-Granted: SME Approvals Granted SLTApproved: Slta Waiver Tested-by: Jira Key Reviewed-by: Xiangpo Zhao Submit-Approved: Jira Key --- include/linux/cgroup.h | 15 +-- include/linux/mm_inline.h | 15 +-- include/linux/mmzone.h | 9 -- kernel/cgroup/cgroup-internal.h | 1 + mm/Kconfig | 6 - mm/vmscan.c | 229 +------------------------------- 6 files changed, 9 insertions(+), 266 deletions(-) diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 3933ddd082e9..bf290358507c 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -428,18 +428,6 @@ static inline void cgroup_put(struct cgroup *cgrp) css_put(&cgrp->self); } -extern struct mutex cgroup_mutex; - -static inline void cgroup_lock(void) -{ - mutex_lock(&cgroup_mutex); -} - -static inline void cgroup_unlock(void) -{ - mutex_unlock(&cgroup_mutex); -} - /** * task_css_set_check - obtain a task's css_set with extra access conditions * @task: the task to obtain css_set for @@ -454,6 +442,7 @@ static inline void cgroup_unlock(void) * as locks used during the cgroup_subsys::attach() methods. */ #ifdef CONFIG_PROVE_RCU +extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; #define task_css_set_check(task, __c) \ rcu_dereference_check((task)->cgroups, \ @@ -718,8 +707,6 @@ struct cgroup; static inline void css_get(struct cgroup_subsys_state *css) {} static inline void css_put(struct cgroup_subsys_state *css) {} -static inline void cgroup_lock(void) {} -static inline void cgroup_unlock(void) {} static inline int cgroup_attach_task_all(struct task_struct *from, struct task_struct *t) { return 0; } static inline int cgroupstats_build(struct cgroupstats *stats, diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 663dc926dbac..64b33e79165b 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -105,21 +105,10 @@ static __always_inline enum lru_list page_lru(struct page *page) #ifdef CONFIG_LRU_GEN -#ifdef CONFIG_LRU_GEN_ENABLED static inline bool lru_gen_enabled(void) { - DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); - - return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); + return true; } -#else -static inline bool lru_gen_enabled(void) -{ - DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); - - return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); -} -#endif static inline bool lru_gen_in_fault(void) { @@ -232,7 +221,7 @@ static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bo VM_WARN_ON_ONCE_PAGE(gen != -1, page); - if (PageUnevictable(page) || !lrugen->enabled) + if (PageUnevictable(page)) return false; /* * There are three common cases for this page: diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 5a9c5192155b..f4fb963c963e 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -381,13 +381,6 @@ enum { LRU_GEN_FILE, }; -enum { - LRU_GEN_CORE, - LRU_GEN_MM_WALK, - LRU_GEN_NONLEAF_YOUNG, - NR_LRU_GEN_CAPS -}; - #define MIN_LRU_BATCH BITS_PER_LONG #define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) @@ -429,8 +422,6 @@ struct lru_gen_struct { /* can be modified without holding the LRU lock */ atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; - /* whether the multi-gen LRU is enabled */ - bool enabled; }; enum { diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index ec985923f7c8..803989eae99e 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -165,6 +165,7 @@ struct cgroup_mgctx { #define DEFINE_CGROUP_MGCTX(name) \ struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) +extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; extern struct cgroup_subsys *cgroup_subsys[]; extern struct list_head cgroup_roots; diff --git a/mm/Kconfig b/mm/Kconfig index b96975666710..5f7638bc90e4 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -865,12 +865,6 @@ config LRU_GEN help A high performance LRU implementation to overcommit memory. -config LRU_GEN_ENABLED - bool "Enable by default" - depends on LRU_GEN - help - This option enables the multi-gen LRU by default. - config LRU_GEN_STATS bool "Full stats for debugging" depends on LRU_GEN diff --git a/mm/vmscan.c b/mm/vmscan.c index 3c68054373d9..cdf2aded13e9 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -53,7 +53,6 @@ #include #include #include -#include #include #include @@ -2641,14 +2640,6 @@ out: #ifdef CONFIG_LRU_GEN -#ifdef CONFIG_LRU_GEN_ENABLED -DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); -#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) -#else -DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); -#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) -#endif - /****************************************************************************** * shorthand helpers ******************************************************************************/ @@ -3527,8 +3518,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area goto next; if (!pmd_trans_huge(pmd[i])) { - if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && - get_cap(LRU_GEN_NONLEAF_YOUNG)) + if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)) pmdp_test_and_clear_young(vma, addr, pmd + i); goto next; } @@ -3627,12 +3617,10 @@ restart: walk->mm_stats[MM_NONLEAF_TOTAL]++; #ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG - if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { - if (!pmd_young(val)) - continue; + if (!pmd_young(val)) + continue; - walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); - } + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); #endif if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) continue; @@ -3897,7 +3885,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, * handful of PTEs. Spreading the work out over a period of time usually * is less efficient, but it avoids bursty page faults. */ - if (!(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { + if (!arch_has_hw_pte_young()) { success = iterate_mm_list_nowalk(lruvec, max_seq); goto done; } @@ -4567,209 +4555,6 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc blk_finish_plug(&plug); } -/****************************************************************************** - * state change - ******************************************************************************/ - -static bool __maybe_unused state_is_valid(struct lruvec *lruvec) -{ - struct lru_gen_struct *lrugen = &lruvec->lrugen; - - if (lrugen->enabled) { - enum lru_list lru; - - for_each_evictable_lru(lru) { - if (!list_empty(&lruvec->lists[lru])) - return false; - } - } else { - int gen, type, zone; - - for_each_gen_type_zone(gen, type, zone) { - if (!list_empty(&lrugen->lists[gen][type][zone])) - return false; - } - } - - return true; -} - -static bool fill_evictable(struct lruvec *lruvec) -{ - enum lru_list lru; - int remaining = MAX_LRU_BATCH; - - for_each_evictable_lru(lru) { - int type = is_file_lru(lru); - bool active = is_active_lru(lru); - struct list_head *head = &lruvec->lists[lru]; - - while (!list_empty(head)) { - bool success; - struct page *page = lru_to_page(head); - - VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page); - VM_WARN_ON_ONCE_PAGE(PageActive(page) != active, page); - VM_WARN_ON_ONCE_PAGE(page_is_file_cache(page) != type, page); - VM_WARN_ON_ONCE_PAGE(page_lru_gen(page) != -1, page); - - del_page_from_lru_list(page, lruvec); - success = lru_gen_add_page(lruvec, page, false); - VM_WARN_ON_ONCE(!success); - - if (!--remaining) - return false; - } - } - - return true; -} - -static bool drain_evictable(struct lruvec *lruvec) -{ - int gen, type, zone; - int remaining = MAX_LRU_BATCH; - - for_each_gen_type_zone(gen, type, zone) { - struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; - - while (!list_empty(head)) { - bool success; - struct page *page = lru_to_page(head); - - VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page); - VM_WARN_ON_ONCE_PAGE(PageActive(page), page); - VM_WARN_ON_ONCE_PAGE(page_is_file_cache(page) != type, page); - VM_WARN_ON_ONCE_PAGE(page_zonenum(page) != zone, page); - - success = lru_gen_del_page(lruvec, page, false); - VM_WARN_ON_ONCE(!success); - add_page_to_lru_list(page, lruvec); - - if (!--remaining) - return false; - } - } - - return true; -} - -static void lru_gen_change_state(bool enabled) -{ - static DEFINE_MUTEX(state_mutex); - - struct mem_cgroup *memcg; - - cgroup_lock(); - cpus_read_lock(); - get_online_mems(); - mutex_lock(&state_mutex); - - if (enabled == lru_gen_enabled()) - goto unlock; - - if (enabled) - static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); - else - static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); - - memcg = mem_cgroup_iter(NULL, NULL, NULL); - do { - int nid; - - for_each_node(nid) { - struct pglist_data *pgdat = NODE_DATA(nid); - struct lruvec *lruvec = get_lruvec(memcg, nid); - - if (!lruvec) - continue; - - spin_lock_irq(&pgdat->lru_lock); - - VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); - VM_WARN_ON_ONCE(!state_is_valid(lruvec)); - - lruvec->lrugen.enabled = enabled; - - while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { - spin_unlock_irq(&pgdat->lru_lock); - cond_resched(); - spin_lock_irq(&pgdat->lru_lock); - } - - spin_unlock_irq(&pgdat->lru_lock); - } - - cond_resched(); - } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); -unlock: - mutex_unlock(&state_mutex); - put_online_mems(); - cpus_read_unlock(); - cgroup_unlock(); -} - -/****************************************************************************** - * sysfs interface - ******************************************************************************/ - -static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) -{ - unsigned int caps = 0; - - if (get_cap(LRU_GEN_CORE)) - caps |= BIT(LRU_GEN_CORE); - - if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) - caps |= BIT(LRU_GEN_MM_WALK); - - if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) - caps |= BIT(LRU_GEN_NONLEAF_YOUNG); - - return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); -} - -static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, - const char *buf, size_t len) -{ - int i; - unsigned int caps; - - if (tolower(*buf) == 'n') - caps = 0; - else if (tolower(*buf) == 'y') - caps = -1; - else if (kstrtouint(buf, 0, &caps)) - return -EINVAL; - - for (i = 0; i < NR_LRU_GEN_CAPS; i++) { - bool enabled = caps & BIT(i); - - if (i == LRU_GEN_CORE) - lru_gen_change_state(enabled); - else if (enabled) - static_branch_enable(&lru_gen_caps[i]); - else - static_branch_disable(&lru_gen_caps[i]); - } - - return len; -} - -static struct kobj_attribute lru_gen_enabled_attr = __ATTR( - enabled, 0644, show_enabled, store_enabled -); - -static struct attribute *lru_gen_attrs[] = { - &lru_gen_enabled_attr.attr, - NULL -}; - -static struct attribute_group lru_gen_attr_group = { - .name = "lru_gen", - .attrs = lru_gen_attrs, -}; - /****************************************************************************** * initialization ******************************************************************************/ @@ -4780,7 +4565,6 @@ void lru_gen_init_lruvec(struct lruvec *lruvec) struct lru_gen_struct *lrugen = &lruvec->lrugen; lrugen->max_seq = MIN_NR_GENS + 1; - lrugen->enabled = lru_gen_enabled(); for_each_gen_type_zone(gen, type, zone) INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); @@ -4820,9 +4604,6 @@ static int __init init_lru_gen(void) BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); - if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) - pr_err("lru_gen: failed to create sysfs group\n"); - return 0; }; late_initcall(init_lru_gen);