diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 04872cd8fb1c..d01a06dae3b7 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -123,11 +123,10 @@ static __always_inline void add_page_to_lru_list_tail(struct page *page, } static __always_inline void del_page_from_lru_list(struct page *page, - struct lruvec *lruvec) + struct lruvec *lruvec, enum lru_list lru) { list_del(&page->lru); - update_lru_size(lruvec, page_lru(page), page_zonenum(page), - -hpage_nr_pages(page)); + update_lru_size(lruvec, lru, page_zonenum(page), -hpage_nr_pages(page)); } #endif diff --git a/mm/compaction.c b/mm/compaction.c index 195dce9221c2..2293de8dffe5 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -984,7 +984,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, VM_BUG_ON_PAGE(PageCompound(page), page); /* Successfully isolated */ - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, page_lru(page)); inc_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page)); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 9171eccfd971..d464f39389a2 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2741,7 +2741,7 @@ static void lock_page_lru(struct page *page, int *isolated) lruvec = mem_cgroup_page_lruvec(page, pgdat); ClearPageLRU(page); - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, page_lru(page)); *isolated = 1; } else *isolated = 0; diff --git a/mm/mlock.c b/mm/mlock.c index 94a4828072a7..395bf08f86d8 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -115,7 +115,7 @@ static bool __munlock_isolate_lru_page(struct page *page, bool getpage) if (getpage) get_page(page); ClearPageLRU(page); - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, page_lru(page)); return true; } diff --git a/mm/swap.c b/mm/swap.c index 26e2dd8cf33e..1be4213f4807 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -68,8 +68,7 @@ static void __page_cache_release(struct page *page) lruvec = mem_cgroup_page_lruvec(page, pgdat); VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); - del_page_from_lru_list(page, lruvec); - page_off_lru(page); + del_page_from_lru_list(page, lruvec, page_off_lru(page)); spin_unlock_irqrestore(&pgdat->lru_lock, flags); } __ClearPageWaiters(page); @@ -223,7 +222,7 @@ static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, int *pgmoved = arg; if (PageLRU(page) && !PageUnevictable(page)) { - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, page_lru(page)); ClearPageActive(page); add_page_to_lru_list_tail(page, lruvec); (*pgmoved)++; @@ -278,8 +277,9 @@ static void __activate_page(struct page *page, struct lruvec *lruvec, { if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { int file = page_is_file_cache(page); + int lru = page_lru_base_type(page); - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, lru); SetPageActive(page); add_page_to_lru_list(page, lruvec); trace_mm_lru_activate(page); @@ -495,7 +495,7 @@ void __lru_cache_add_active_or_unevictable(struct page *page, static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, void *arg) { - int file; + int lru, file; bool active; if (!PageLRU(page)) @@ -510,8 +510,9 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, active = PageActive(page); file = page_is_file_cache(page); + lru = page_lru_base_type(page); - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, lru + active); ClearPageActive(page); ClearPageReferenced(page); @@ -542,8 +543,9 @@ static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, { if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { int file = page_is_file_cache(page); + int lru = page_lru_base_type(page); - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); ClearPageActive(page); ClearPageReferenced(page); add_page_to_lru_list(page, lruvec); @@ -558,8 +560,10 @@ static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, { if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page) && !PageUnevictable(page)) { + bool active = PageActive(page); - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, + LRU_INACTIVE_ANON + active); ClearPageActive(page); ClearPageReferenced(page); /* @@ -821,8 +825,7 @@ void release_pages(struct page **pages, int nr) lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); - del_page_from_lru_list(page, lruvec); - page_off_lru(page); + del_page_from_lru_list(page, lruvec, page_off_lru(page)); } __ClearPageWaiters(page); diff --git a/mm/vmscan.c b/mm/vmscan.c index 677b408b154d..214e43dd38c3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1874,9 +1874,10 @@ int isolate_lru_page(struct page *page) spin_lock_irq(&pgdat->lru_lock); lruvec = mem_cgroup_page_lruvec(page, pgdat); if (PageLRU(page)) { + int lru = page_lru(page); get_page(page); ClearPageLRU(page); - del_page_from_lru_list(page, lruvec); + del_page_from_lru_list(page, lruvec, lru); ret = 0; } spin_unlock_irq(&pgdat->lru_lock); @@ -1948,6 +1949,7 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, int nr_pages, nr_moved = 0; LIST_HEAD(pages_to_free); struct page *page; + enum lru_list lru; while (!list_empty(list)) { page = lru_to_page(list); @@ -1962,12 +1964,13 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, lruvec = mem_cgroup_page_lruvec(page, pgdat); SetPageLRU(page); + lru = page_lru(page); add_page_to_lru_list(page, lruvec); if (put_page_testzero(page)) { __ClearPageLRU(page); - del_page_from_lru_list(page, lruvec); __ClearPageActive(page); + del_page_from_lru_list(page, lruvec, lru); if (unlikely(PageCompound(page))) { spin_unlock_irq(&pgdat->lru_lock); @@ -4564,8 +4567,8 @@ void check_move_unevictable_pages(struct pagevec *pvec) if (page_evictable(page)) { VM_BUG_ON_PAGE(PageActive(page), page); - del_page_from_lru_list(page, lruvec); ClearPageUnevictable(page); + del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE); add_page_to_lru_list(page, lruvec); pgrescued++; }