A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-05 19:31:57 -04:00
Code Issues Projects Releases Packages Wiki Activity

LUR: Revert "BACKPORT: mm: multi-gen LRU: minimal implementation"

Browse source 
The LRU function is not compatible with GKI,
causing GKI to fail to start, so revert.

This reverts commit bdab47a49135e8ded0e5158717bc3a29dbf79e8c.

Change-Id: I48190e01afcb59dae18d87a1ca785fcec73b2730
Reviewed-on: https://gerrit.mot.com/2609762
SME-Granted: SME Approvals Granted
SLTApproved: Slta Waiver
Tested-by: Jira Key
Reviewed-by: Xiangpo Zhao <zhaoxp3@motorola.com>
Submit-Approved: Jira Key
This commit is contained in:
xiewen3 2023-05-16 22:48:58 +08:00 • committed by Xiaojun Ji
commit a246d9f08a
10 changed files with 14 additions and 1047 deletions
Download patch file Download diff file Expand all files Collapse all files

36
include/linux/mm_inline.h
View file

@ -120,33 +120,6 @@ static inline int lru_gen_from_seq(unsigned long seq)
return seq % MAX_NR_GENS;
}
static inline int lru_hist_from_seq(unsigned long seq)
{
return seq % NR_HIST_GENS;
}
static inline int lru_tier_from_refs(int refs)
{
VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH));
/* see the comment in page_lru_refs() */
return order_base_2(refs + 1);
}
static inline int page_lru_refs(struct page *page)
{
unsigned long flags = READ_ONCE(page->flags);
bool workingset = flags & BIT(PG_workingset);
/*
* Return the number of accesses beyond PG_referenced, i.e., N-1 if the
* total number of accesses is N>1, since N=0,1 both map to the first
* tier. lru_tier_from_refs() will account for this off-by-one. Also see
* the comment on MAX_NR_TIERS.
*/
return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset;
}
static inline int page_lru_gen(struct page *page)
{
unsigned long flags = READ_ONCE(page->flags);
@ -199,15 +172,6 @@ static inline void lru_gen_update_size(struct lruvec *lruvec, struct page *page,
__update_lru_size(lruvec, lru, zone, -delta);
return;
}
/* promotion */
if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
__update_lru_size(lruvec, lru, zone, -delta);
__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
}
/* demotion requires isolation, e.g., lru_deactivate_fn() */
VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
}
static inline bool lru_gen_add_page(struct lruvec *lruvec, struct page *page, bool reclaiming)

44
include/linux/mmzone.h
View file

@ -341,28 +341,6 @@ struct zone_reclaim_stat {
#define MIN_NR_GENS 2U
#define MAX_NR_GENS 4U
/*
* Each generation is divided into multiple tiers. A page accessed N times
* through file descriptors is in tier order_base_2(N). A page in the first tier
* (N=0,1) is marked by PG_referenced unless it was faulted in through page
* tables or read ahead. A page in any other tier (N>1) is marked by
* PG_referenced and PG_workingset. This implies a minimum of two tiers is
* supported without using additional bits in page->flags.
*
* In contrast to moving across generations which requires the LRU lock, moving
* across tiers only involves atomic operations on page->flags and therefore
* has a negligible cost in the buffered access path. In the eviction path,
* comparisons of refaulted/(evicted+protected) from the first tier and the
* rest infer whether pages accessed multiple times through file descriptors
* are statistically hot and thus worth protecting.
*
* MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the
* number of categories of the active/inactive LRU when keeping track of
* accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in
* page->flags.
*/
#define MAX_NR_TIERS 4U
#ifndef __GENERATING_BOUNDS_H
struct lruvec;
@ -370,9 +348,6 @@ struct lruvec;
#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
void *lru_gen_eviction(struct page *page);
void lru_gen_refault(struct page *page, void *shadow);
#ifdef CONFIG_LRU_GEN
enum {
@ -380,16 +355,6 @@ enum {
LRU_GEN_FILE,
};
#define MIN_LRU_BATCH BITS_PER_LONG
#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64)
/* whether to keep historical stats from evicted generations */
#ifdef CONFIG_LRU_GEN_STATS
#define NR_HIST_GENS MAX_NR_GENS
#else
#define NR_HIST_GENS 1U
#endif
/*
* The youngest generation number is stored in max_seq for both anon and file
* types as they are aged on an equal footing. The oldest generation numbers are
@ -412,15 +377,6 @@ struct lru_gen_struct {
struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the multi-gen LRU sizes, eventually consistent */
long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES];
/* the exponential moving average of refaulted */
unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS];
/* the exponential moving average of evicted+protected */
unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS];
/* the first tier doesn't need protection, hence the minus one */
unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1];
/* can be modified without holding the LRU lock */
atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS];
};
void lru_gen_init_lruvec(struct lruvec *lruvec);

5
include/linux/page-flags-layout.h
View file

@ -96,10 +96,7 @@
#error "Not enough bits in page flags"
#endif
/* see the comment on MAX_NR_TIERS */
#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \
ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \
NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH)
#define LRU_REFS_WIDTH 0
/*
* We are going to use the flags for the page to node mapping if its in

2
kernel/bounds.c
View file

@ -24,10 +24,8 @@ int main(void)
DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
#ifdef CONFIG_LRU_GEN
DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1));
DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2);
#else
DEFINE(LRU_GEN_WIDTH, 0);
DEFINE(__LRU_REFS_WIDTH, 0);
#endif
/* End of constants */

11
mm/Kconfig
View file

@ -856,7 +856,6 @@ config OOM_TASK_PRIORITY_ADJ_LIMIT
With this parameter you can limit the OOM killer to first kill
tasks with an oom_score_adj value above a certain threshold
before considering tasks with a lower oom_score_adj value.
# multi-gen LRU {
config LRU_GEN
bool "Multi-Gen LRU"
depends on MMU
@ -865,16 +864,6 @@ config LRU_GEN
help
A high performance LRU implementation to overcommit memory.
config LRU_GEN_STATS
bool "Full stats for debugging"
depends on LRU_GEN
help
Do not enable this option unless you plan to look at historical stats
from evicted generations for debugging purpose.
This option has a per-memcg and per-node memory overhead.
# }
endmenu
config PROCESS_RECLAIM

19
mm/memory.c
View file

@ -71,8 +71,6 @@
#include <linux/dax.h>
#include <linux/oom.h>
#include <linux/numa.h>
#include <linux/mm_inline.h>
#include <linux/mmzone.h>
#include <trace/events/kmem.h>
@ -3019,19 +3017,6 @@ void unmap_mapping_range(struct address_space *mapping,
}
EXPORT_SYMBOL(unmap_mapping_range);
static void lru_gen_swap_refault(struct page *page, swp_entry_t entry)
{
if (lru_gen_enabled()) {
void *item;
struct address_space *mapping = swap_address_space(entry);
pgoff_t index = swp_offset(entry);
item = xa_load(&mapping->i_pages, index);
if (xa_is_value(item))
lru_gen_refault(page, item);
}
}
/*
* We enter with non-exclusive mmap_sem (to exclude vma changes,
* but allow concurrent faults), and pte mapped but not yet locked.
@ -3112,7 +3097,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
__SetPageLocked(page);
__SetPageSwapBacked(page);
set_page_private(page, entry.val);
lru_gen_swap_refault(page, entry);
lru_cache_add_anon(page);
swap_readpage(page, true);
}
@ -3246,8 +3230,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
} else {
do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
mem_cgroup_commit_charge(page, memcg, true, false);
if (!lru_gen_enabled())
activate_page(page);
activate_page(page);
}
swap_free(entry);

41
mm/swap.c
View file

@ -357,41 +357,6 @@ static void __lru_cache_activate_page(struct page *page)
put_cpu_var(lru_add_pvec);
}
#ifdef CONFIG_LRU_GEN
static void page_inc_refs(struct page *page)
{
unsigned long new_flags, old_flags;
if (PageUnevictable(page))
return;
if (!PageReferenced(page)) {
SetPageReferenced(page);
return;
}
if (!PageWorkingset(page)) {
SetPageWorkingset(page);
return;
}
/* see the comment on MAX_NR_TIERS */
do {
old_flags = READ_ONCE(page->flags);
new_flags = old_flags & LRU_REFS_MASK;
if (new_flags == LRU_REFS_MASK)
break;
new_flags += BIT(LRU_REFS_PGOFF);
new_flags |= old_flags & ~LRU_REFS_MASK;
} while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags);
}
#else
static void page_inc_refs(struct page *page)
{
}
#endif /* CONFIG_LRU_GEN */
/*
* Mark a page as having seen activity.
*
@ -405,12 +370,6 @@ static void page_inc_refs(struct page *page)
void mark_page_accessed(struct page *page)
{
page = compound_head(page);
if (lru_gen_enabled()) {
page_inc_refs(page);
return;
}
if (!PageActive(page) && !PageUnevictable(page) &&
PageReferenced(page)) {

9
mm/swap_state.c
View file

@ -21,7 +21,6 @@
#include <linux/vmalloc.h>
#include <linux/swap_slots.h>
#include <linux/huge_mm.h>
#include <linux/mm_inline.h>
#include <asm/pgtable.h>
#include "internal.h"
@ -410,7 +409,6 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
struct page *found_page = NULL, *new_page = NULL;
struct swap_info_struct *si;
int err;
void *shadow;
*new_page_allocated = false;
do {
@ -467,13 +465,10 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
__SetPageLocked(new_page);
__SetPageSwapBacked(new_page);
err = add_to_swap_cache(new_page, entry,
gfp_mask & GFP_RECLAIM_MASK, &shadow);
gfp_mask & GFP_RECLAIM_MASK, NULL);
if (likely(!err)) {
/* Initiate read into locked page */
if (!lru_gen_enabled())
SetPageWorkingset(new_page);
else if (shadow)
lru_gen_refault(new_page, shadow);
SetPageWorkingset(new_page);
lru_cache_add_anon(new_page);
*new_page_allocated = true;
return new_page;

780
mm/vmscan.c
View file

@ -923,7 +923,6 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
{
unsigned long flags;
int refcount;
void *shadow = NULL;
BUG_ON(!PageLocked(page));
BUG_ON(mapping != page_mapping(page));
@ -965,11 +964,8 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
if (PageSwapCache(page)) {
swp_entry_t swap = { .val = page_private(page) };
if (lru_gen_enabled())
shadow = lru_gen_eviction(page);
mem_cgroup_swapout(page, swap);
__delete_from_swap_cache(page, swap, shadow);
__delete_from_swap_cache(page, swap, NULL);
xa_unlock_irqrestore(&mapping->i_pages, flags);
put_swap_page(page, swap);
} else {
@ -2637,17 +2633,6 @@ out:
* shorthand helpers
******************************************************************************/
#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset))
#define DEFINE_MAX_SEQ(lruvec) \
unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
#define DEFINE_MIN_SEQ(lruvec) \
unsigned long min_seq[ANON_AND_FILE] = { \
READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
}
#define for_each_gen_type_zone(gen, type, zone) \
for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \
for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \
@ -2673,734 +2658,6 @@ static struct lruvec __maybe_unused *get_lruvec(struct mem_cgroup *memcg, int ni
return pgdat ? &pgdat->lruvec : NULL;
}
static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
{
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
if (mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH)
return 0;
return mem_cgroup_swappiness(memcg);
}
static int get_nr_gens(struct lruvec *lruvec, int type)
{
return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
}
static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
{
/* see the comment on lru_gen_struct */
return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS &&
get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) &&
get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS;
}
/******************************************************************************
* refault feedback loop
******************************************************************************/
/*
* A feedback loop based on Proportional-Integral-Derivative (PID) controller.
*
* The P term is refaulted/(evicted+protected) from a tier in the generation
* currently being evicted; the I term is the exponential moving average of the
* P term over the generations previously evicted, using the smoothing factor
* 1/2; the D term isn't supported.
*
* The setpoint (SP) is always the first tier of one type; the process variable
* (PV) is either any tier of the other type or any other tier of the same
* type.
*
* The error is the difference between the SP and the PV; the correction is to
* turn off protection when SP>PV or turn on protection when SP<PV.
*
* For future optimizations:
* 1. The D term may discount the other two terms over time so that long-lived
* generations can resist stale information.
*/
struct ctrl_pos {
unsigned long refaulted;
unsigned long total;
int gain;
};
static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
struct ctrl_pos *pos)
{
struct lru_gen_struct *lrugen = &lruvec->lrugen;
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
pos->refaulted = lrugen->avg_refaulted[type][tier] +
atomic_long_read(&lrugen->refaulted[hist][type][tier]);
pos->total = lrugen->avg_total[type][tier] +
atomic_long_read(&lrugen->evicted[hist][type][tier]);
if (tier)
pos->total += lrugen->protected[hist][type][tier - 1];
pos->gain = gain;
}
static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
{
int hist, tier;
struct lru_gen_struct *lrugen = &lruvec->lrugen;
bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1;
unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1;
lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock);
if (!carryover && !clear)
return;
hist = lru_hist_from_seq(seq);
for (tier = 0; tier < MAX_NR_TIERS; tier++) {
if (carryover) {
unsigned long sum;
sum = lrugen->avg_refaulted[type][tier] +
atomic_long_read(&lrugen->refaulted[hist][type][tier]);
WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2);
sum = lrugen->avg_total[type][tier] +
atomic_long_read(&lrugen->evicted[hist][type][tier]);
if (tier)
sum += lrugen->protected[hist][type][tier - 1];
WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2);
}
if (clear) {
atomic_long_set(&lrugen->refaulted[hist][type][tier], 0);
atomic_long_set(&lrugen->evicted[hist][type][tier], 0);
if (tier)
WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0);
}
}
}
static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
{
/*
* Return true if the PV has a limited number of refaults or a lower
* refaulted/total than the SP.
*/
return pv->refaulted < MIN_LRU_BATCH ||
pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <=
(sp->refaulted + 1) * pv->total * pv->gain;
}
/******************************************************************************
* the aging
******************************************************************************/
/* protect pages accessed multiple times through file descriptors */
static int page_inc_gen(struct lruvec *lruvec, struct page *page, bool reclaiming)
{
int type = page_is_file_cache(page);
struct lru_gen_struct *lrugen = &lruvec->lrugen;
int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
unsigned long new_flags, old_flags;
do {
old_flags = READ_ONCE(page->flags);
VM_WARN_ON_ONCE_PAGE(!(old_flags & LRU_GEN_MASK), page);
new_gen = (old_gen + 1) % MAX_NR_GENS;
new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS);
new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
/* for end_page_writeback() */
if (reclaiming)
new_flags |= BIT(PG_reclaim);
} while (cmpxchg(&page->flags, old_flags, new_flags) != old_flags);
lru_gen_update_size(lruvec, page, old_gen, new_gen);
return new_gen;
}
static void inc_min_seq(struct lruvec *lruvec, int type)
{
struct lru_gen_struct *lrugen = &lruvec->lrugen;
reset_ctrl_pos(lruvec, type, true);
WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1);
}
static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap)
{
int gen, type, zone;
bool success = false;
struct lru_gen_struct *lrugen = &lruvec->lrugen;
DEFINE_MIN_SEQ(lruvec);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
/* find the oldest populated generation */
for (type = !can_swap; type < ANON_AND_FILE; type++) {
while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) {
gen = lru_gen_from_seq(min_seq[type]);
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
if (!list_empty(&lrugen->lists[gen][type][zone]))
goto next;
}
min_seq[type]++;
}
next:
;
}
/* see the comment on lru_gen_struct */
if (can_swap) {
min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]);
min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]);
}
for (type = !can_swap; type < ANON_AND_FILE; type++) {
if (min_seq[type] == lrugen->min_seq[type])
continue;
reset_ctrl_pos(lruvec, type, true);
WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
success = true;
}
return success;
}
static void inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, bool can_swap)
{
int prev, next;
int type, zone;
struct lru_gen_struct *lrugen = &lruvec->lrugen;
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
spin_lock_irq(&pgdat->lru_lock);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
if (max_seq != lrugen->max_seq)
goto unlock;
for (type = ANON_AND_FILE - 1; type >= 0; type--) {
if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
continue;
VM_WARN_ON_ONCE(type == LRU_GEN_FILE || can_swap);
inc_min_seq(lruvec, type);
}
/*
* Update the active/inactive LRU sizes for compatibility. Both sides of
* the current max_seq need to be covered, since max_seq+1 can overlap
* with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do
* overlap, cold/hot inversion happens.
*/
prev = lru_gen_from_seq(lrugen->max_seq - 1);
next = lru_gen_from_seq(lrugen->max_seq + 1);
for (type = 0; type < ANON_AND_FILE; type++) {
for (zone = 0; zone < MAX_NR_ZONES; zone++) {
enum lru_list lru = type * LRU_INACTIVE_FILE;
long delta = lrugen->nr_pages[prev][type][zone] -
lrugen->nr_pages[next][type][zone];
if (!delta)
continue;
__update_lru_size(lruvec, lru, zone, delta);
__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta);
}
}
for (type = 0; type < ANON_AND_FILE; type++)
reset_ctrl_pos(lruvec, type, false);
/* make sure preceding modifications appear */
smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1);
unlock:
spin_unlock_irq(&pgdat->lru_lock);
}
static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq,
struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan)
{
int gen, type, zone;
unsigned long old = 0;
unsigned long young = 0;
unsigned long total = 0;
struct lru_gen_struct *lrugen = &lruvec->lrugen;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
for (type = !can_swap; type < ANON_AND_FILE; type++) {
unsigned long seq;
for (seq = min_seq[type]; seq <= max_seq; seq++) {
unsigned long size = 0;
gen = lru_gen_from_seq(seq);
for (zone = 0; zone < MAX_NR_ZONES; zone++)
size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
total += size;
if (seq == max_seq)
young += size;
else if (seq + MIN_NR_GENS == max_seq)
old += size;
}
}
/* try to scrape all its memory if this memcg was deleted */
*nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
/*
* The aging tries to be lazy to reduce the overhead, while the eviction
* stalls when the number of generations reaches MIN_NR_GENS. Hence, the
* ideal number of generations is MIN_NR_GENS+1.
*/
if (min_seq[!can_swap] + MIN_NR_GENS > max_seq)
return true;
if (min_seq[!can_swap] + MIN_NR_GENS < max_seq)
return false;
/*
* It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1)
* of the total number of pages for each generation. A reasonable range
* for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The
* aging cares about the upper bound of hot pages, while the eviction
* cares about the lower bound of cold pages.
*/
if (young * MIN_NR_GENS > total)
return true;
if (old * (MIN_NR_GENS + 2) < total)
return true;
return false;
}
static void age_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
bool need_aging;
unsigned long nr_to_scan;
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
enum mem_cgroup_protection prot = mem_cgroup_protected(NULL, memcg);
DEFINE_MAX_SEQ(lruvec);
DEFINE_MIN_SEQ(lruvec);
VM_WARN_ON_ONCE(sc->memcg_low_reclaim);
if (prot == MEMCG_PROT_MIN)
return;
need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan);
if (need_aging)
inc_max_seq(lruvec, max_seq, swappiness);
}
static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
{
struct mem_cgroup *memcg;
VM_WARN_ON_ONCE(!current_is_kswapd());
memcg = mem_cgroup_iter(NULL, NULL, NULL);
do {
struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
age_lruvec(lruvec, sc);
cond_resched();
} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)));
}
/******************************************************************************
* the eviction
******************************************************************************/
static bool sort_page(struct lruvec *lruvec, struct page *page, int tier_idx)
{
bool success;
int gen = page_lru_gen(page);
int type = page_is_file_cache(page);
int zone = page_zonenum(page);
int delta = hpage_nr_pages(page);
int refs = page_lru_refs(page);
int tier = lru_tier_from_refs(refs);
struct lru_gen_struct *lrugen = &lruvec->lrugen;
VM_WARN_ON_ONCE_PAGE(gen >= MAX_NR_GENS, page);
/* unevictable */
if (!page_evictable(page)) {
success = lru_gen_del_page(lruvec, page, true);
VM_WARN_ON_ONCE_PAGE(!success, page);
SetPageUnevictable(page);
add_page_to_lru_list(page, lruvec);
__count_vm_events(UNEVICTABLE_PGCULLED, delta);
return true;
}
/* dirty lazyfree */
if (type == LRU_GEN_FILE && PageAnon(page) && PageDirty(page)) {
success = lru_gen_del_page(lruvec, page, true);
VM_WARN_ON_ONCE_PAGE(!success, page);
SetPageSwapBacked(page);
add_page_to_lru_list_tail(page, lruvec);
return true;
}
/* protected */
if (tier > tier_idx) {
int hist = lru_hist_from_seq(lrugen->min_seq[type]);
gen = page_inc_gen(lruvec, page, false);
list_move_tail(&page->lru, &lrugen->lists[gen][type][zone]);
WRITE_ONCE(lrugen->protected[hist][type][tier - 1],
lrugen->protected[hist][type][tier - 1] + delta);
__mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE, delta);
return true;
}
/* waiting for writeback */
if (PageLocked(page) || PageWriteback(page) ||
(type == LRU_GEN_FILE && PageDirty(page))) {
gen = page_inc_gen(lruvec, page, true);
list_move(&page->lru, &lrugen->lists[gen][type][zone]);
return true;
}
return false;
}
static bool isolate_page(struct lruvec *lruvec, struct page *page, struct scan_control *sc)
{
bool success;
/* unmapping inhibited */
if (!sc->may_unmap && page_mapped(page))
return false;
/* swapping inhibited */
if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) &&
(PageDirty(page) ||
(PageAnon(page) && !PageSwapCache(page))))
return false;
/* raced with release_pages() */
if (!get_page_unless_zero(page))
return false;
ClearPageLRU(page);
/* see the comment on MAX_NR_TIERS */
if (!PageReferenced(page))
set_mask_bits(&page->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);
/* for shrink_page_list() */
ClearPageReclaim(page);
ClearPageReferenced(page);
success = lru_gen_del_page(lruvec, page, true);
VM_WARN_ON_ONCE_PAGE(!success, page);
return true;
}
static int scan_pages(struct lruvec *lruvec, struct scan_control *sc,
int type, int tier, struct list_head *list)
{
int gen, zone;
enum vm_event_item item;
int sorted = 0;
int scanned = 0;
int isolated = 0;
int remaining = MAX_LRU_BATCH;
struct lru_gen_struct *lrugen = &lruvec->lrugen;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
VM_WARN_ON_ONCE(!list_empty(list));
if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
return 0;
gen = lru_gen_from_seq(lrugen->min_seq[type]);
for (zone = sc->reclaim_idx; zone >= 0; zone--) {
LIST_HEAD(moved);
int skipped = 0;
struct list_head *head = &lrugen->lists[gen][type][zone];
while (!list_empty(head)) {
struct page *page = lru_to_page(head);
int delta = hpage_nr_pages(page);
VM_WARN_ON_ONCE_PAGE(PageUnevictable(page), page);
VM_WARN_ON_ONCE_PAGE(PageActive(page), page);
VM_WARN_ON_ONCE_PAGE(page_is_file_cache(page) != type, page);
VM_WARN_ON_ONCE_PAGE(page_zonenum(page) != zone, page);
scanned += delta;
if (sort_page(lruvec, page, tier))
sorted += delta;
else if (isolate_page(lruvec, page, sc)) {
list_add(&page->lru, list);
isolated += delta;
} else {
list_move(&page->lru, &moved);
skipped += delta;
}
if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH)
break;
}
if (skipped) {
list_splice(&moved, head);
__count_zid_vm_events(PGSCAN_SKIP, zone, skipped);
}
if (!remaining || isolated >= MIN_LRU_BATCH)
break;
}
item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
if (global_reclaim(sc)) {
__count_vm_events(item, isolated);
__count_vm_events(PGREFILL, sorted);
}
__count_memcg_events(memcg, item, isolated);
__count_memcg_events(memcg, PGREFILL, sorted);
/*
* There might not be eligible pages due to reclaim_idx, may_unmap and
* may_writepage. Check the remaining to prevent livelock if it's not
* making progress.
*/
return isolated || !remaining ? scanned : 0;
}
static int get_tier_idx(struct lruvec *lruvec, int type)
{
int tier;
struct ctrl_pos sp, pv;
/*
* To leave a margin for fluctuations, use a larger gain factor (1:2).
* This value is chosen because any other tier would have at least twice
* as many refaults as the first tier.
*/
read_ctrl_pos(lruvec, type, 0, 1, &sp);
for (tier = 1; tier < MAX_NR_TIERS; tier++) {
read_ctrl_pos(lruvec, type, tier, 2, &pv);
if (!positive_ctrl_err(&sp, &pv))
break;
}
return tier - 1;
}
static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx)
{
int type, tier;
struct ctrl_pos sp, pv;
int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness };
/*
* Compare the first tier of anon with that of file to determine which
* type to scan. Also need to compare other tiers of the selected type
* with the first tier of the other type to determine the last tier (of
* the selected type) to evict.
*/
read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp);
read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv);
type = positive_ctrl_err(&sp, &pv);
read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp);
for (tier = 1; tier < MAX_NR_TIERS; tier++) {
read_ctrl_pos(lruvec, type, tier, gain[type], &pv);
if (!positive_ctrl_err(&sp, &pv))
break;
}
*tier_idx = tier - 1;
return type;
}
static int isolate_pages(struct lruvec *lruvec, struct scan_control *sc, int swappiness,
int *type_scanned, struct list_head *list)
{
int i;
int type;
int scanned;
int tier = -1;
DEFINE_MIN_SEQ(lruvec);
/*
* Try to make the obvious choice first. When anon and file are both
* available from the same generation, interpret swappiness 1 as file
* first and 200 as anon first.
*/
if (!swappiness)
type = LRU_GEN_FILE;
else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE])
type = LRU_GEN_ANON;
else if (swappiness == 1)
type = LRU_GEN_FILE;
else if (swappiness == 200)
type = LRU_GEN_ANON;
else
type = get_type_to_scan(lruvec, swappiness, &tier);
for (i = !swappiness; i < ANON_AND_FILE; i++) {
if (tier < 0)
tier = get_tier_idx(lruvec, type);
scanned = scan_pages(lruvec, sc, type, tier, list);
if (scanned)
break;
type = !type;
tier = -1;
}
*type_scanned = type;
return scanned;
}
static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swappiness)
{
int type;
int scanned;
int reclaimed;
LIST_HEAD(list);
struct page *page;
enum vm_event_item item;
struct reclaim_stat stat;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
spin_lock_irq(&pgdat->lru_lock);
scanned = isolate_pages(lruvec, sc, swappiness, &type, &list);
scanned += try_to_inc_min_seq(lruvec, swappiness);
if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS)
scanned = 0;
spin_unlock_irq(&pgdat->lru_lock);
if (list_empty(&list))
return scanned;
reclaimed = shrink_page_list(&list, pgdat, sc, 0, &stat, false);
list_for_each_entry(page, &list, lru) {
/* restore LRU_REFS_FLAGS cleared by isolate_page() */
if (PageWorkingset(page))
SetPageReferenced(page);
/* don't add rejected pages to the oldest generation */
if (PageReclaim(page) &&
(PageDirty(page) || PageWriteback(page)))
ClearPageActive(page);
else
SetPageActive(page);
}
spin_lock_irq(&pgdat->lru_lock);
move_pages_to_lru(lruvec, &list);
item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
if (global_reclaim(sc))
__count_vm_events(item, reclaimed);
__count_memcg_events(memcg, item, reclaimed);
spin_unlock_irq(&pgdat->lru_lock);
mem_cgroup_uncharge_list(&list);
free_unref_page_list(&list);
sc->nr_reclaimed += reclaimed;
return scanned;
}
static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
bool can_swap)
{
bool need_aging;
unsigned long nr_to_scan;
DEFINE_MAX_SEQ(lruvec);
DEFINE_MIN_SEQ(lruvec);
need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan);
if (!need_aging)
return nr_to_scan;
/* skip the aging path at the default priority */
if (sc->priority == DEF_PRIORITY)
goto done;
/* leave the work to lru_gen_age_node() */
if (current_is_kswapd())
return 0;
inc_max_seq(lruvec, max_seq, can_swap);
done:
return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0;
}
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
struct blk_plug plug;
unsigned long scanned = 0;
lru_add_drain();
blk_start_plug(&plug);
while (true) {
int delta;
int swappiness;
unsigned long nr_to_scan;
if (sc->may_swap)
swappiness = get_swappiness(lruvec, sc);
else if (global_reclaim(sc) && get_swappiness(lruvec, sc))
swappiness = 1;
else
swappiness = 0;
nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
if (!nr_to_scan)
break;
delta = evict_pages(lruvec, sc, swappiness);
if (!delta)
break;
scanned += delta;
if (scanned >= nr_to_scan)
break;
cond_resched();
}
blk_finish_plug(&plug);
}
/******************************************************************************
* initialization
******************************************************************************/
@ -3443,16 +2700,6 @@ static int __init init_lru_gen(void)
};
late_initcall(init_lru_gen);
#else /* !CONFIG_LRU_GEN */
static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
{
}
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
{
}
#endif /* CONFIG_LRU_GEN */
/*
@ -3471,11 +2718,6 @@ static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memc
struct blk_plug plug;
bool scan_adjusted;
if (lru_gen_enabled()) {
lru_gen_shrink_lruvec(lruvec, sc);
return;
}
get_scan_count(lruvec, memcg, sc, nr, lru_pages);
/* Record the original scan target for proportional adjustments later */
@ -3950,9 +3192,6 @@ static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat)
{
struct mem_cgroup *memcg;
if (lru_gen_enabled())
return;
memcg = mem_cgroup_iter(root_memcg, NULL, NULL);
do {
unsigned long refaults;
@ -4310,15 +3549,11 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
}
#endif
static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc)
static void age_active_anon(struct pglist_data *pgdat,
struct scan_control *sc)
{
struct mem_cgroup *memcg;
if (lru_gen_enabled()) {
lru_gen_age_node(pgdat, sc);
return;
}
if (!total_swap_pages)
return;
@ -4598,11 +3833,12 @@ restart:
sc.may_swap = !nr_boost_reclaim;
/*
* Do some background aging, to give pages a chance to be
* referenced before reclaiming. All pages are rotated
* regardless of classzone as this is about consistent aging.
* Do some background aging of the anon list, to give
* pages a chance to be referenced before reclaiming. All
* pages are rotated regardless of classzone as this is
* about consistent aging.
*/
kswapd_age_node(pgdat, &sc);
age_active_anon(pgdat, &sc);
/*
* If we're getting trouble reclaiming, start doing writepage

114
mm/workingset.c
View file

@ -7,7 +7,6 @@
#include <linux/memcontrol.h>
#include <linux/writeback.h>
#include <linux/mm_inline.h>
#include <linux/shmem_fs.h>
#include <linux/pagemap.h>
#include <linux/atomic.h>
@ -185,6 +184,7 @@ static unsigned int bucket_order __read_mostly;
static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction,
bool workingset)
{
eviction >>= bucket_order;
eviction &= EVICTION_MASK;
eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
@ -209,110 +209,10 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
*memcgidp = memcgid;
*pgdat = NODE_DATA(nid);
*evictionp = entry;
*evictionp = entry << bucket_order;
*workingsetp = workingset;
}
#ifdef CONFIG_LRU_GEN
void *lru_gen_eviction(struct page *page)
{
int hist;
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
struct lru_gen_struct *lrugen;
int type = page_is_file_cache(page);
int delta = hpage_nr_pages(page);
int refs = page_lru_refs(page);
int tier = lru_tier_from_refs(refs);
struct mem_cgroup *memcg = page_memcg(page);
struct pglist_data *pgdat = page_pgdat(page);
if (!mem_cgroup_disabled() && !memcg)
return NULL;
BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT);
lruvec = mem_cgroup_lruvec(pgdat, memcg);
lrugen = &lruvec->lrugen;
min_seq = READ_ONCE(lrugen->min_seq[type]);
token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
hist = lru_hist_from_seq(min_seq);
atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs);
}
void lru_gen_refault(struct page *page, void *shadow)
{
int hist, tier, refs;
int memcg_id;
bool workingset;
unsigned long token;
unsigned long min_seq;
struct lruvec *lruvec;
struct lru_gen_struct *lrugen;
struct mem_cgroup *memcg;
struct pglist_data *pgdat;
int type = page_is_file_cache(page);
int delta = hpage_nr_pages(page);
unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset);
if (pgdat != page_pgdat(page))
return;
rcu_read_lock();
memcg = page_memcg_rcu(page);
if ((!mem_cgroup_disabled() && !memcg) || memcg_id != mem_cgroup_id(memcg))
goto unlock;
lruvec = mem_cgroup_lruvec(pgdat, memcg);
lrugen = &lruvec->lrugen;
min_seq = READ_ONCE(lrugen->min_seq[type]);
if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH)))
goto unlock;
hist = lru_hist_from_seq(min_seq);
/* see the comment in page_lru_refs() */
refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset;
tier = lru_tier_from_refs(refs);
atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
mod_lruvec_state(lruvec, WORKINGSET_REFAULT, delta);
/*
* Count the following two cases as stalls:
* 1. For pages accessed through page tables, hotter pages pushed out
* hot pages which refaulted immediately.
* 2. For pages accessed multiple times through file descriptors,
* numbers of accesses might have been out of the range.
*/
if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) {
SetPageWorkingset(page);
mod_lruvec_state(lruvec, WORKINGSET_RESTORE, delta);
}
unlock:
rcu_read_unlock();
}
#else /* !CONFIG_LRU_GEN */
void *lru_gen_eviction(struct page *page)
{
return NULL;
}
void lru_gen_refault(struct page *page, void *shadow)
{
}
#endif /* CONFIG_LRU_GEN */
/**
* workingset_eviction - note the eviction of a page from memory
* @page: the page being evicted
@ -328,9 +228,6 @@ void *workingset_eviction(struct page *page)
unsigned long eviction;
struct lruvec *lruvec;
if (lru_gen_enabled())
return lru_gen_eviction(page);
/* Page is fully exclusive and pins page->mem_cgroup */
VM_BUG_ON_PAGE(PageLRU(page), page);
VM_BUG_ON_PAGE(page_count(page), page);
@ -338,7 +235,6 @@ void *workingset_eviction(struct page *page)
lruvec = mem_cgroup_lruvec(pgdat, memcg);
eviction = atomic_long_inc_return(&lruvec->inactive_age);
eviction >>= bucket_order;
return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
}
@ -362,13 +258,7 @@ void workingset_refault(struct page *page, void *shadow)
bool workingset;
int memcgid;
if (lru_gen_enabled()) {
lru_gen_refault(page, shadow);
return;
}
unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset);
eviction <<= bucket_order;
rcu_read_lock();
/*