mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-28 10:41:38 -04:00
mm + fs: prepare for non-page entries in page cache radix trees
shmem mappings already contain exceptional entries where swap slot information is remembered. To be able to store eviction information for regular page cache, prepare every site dealing with the radix trees directly to handle entries other than pages. The common lookup functions will filter out non-page entries and return NULL for page cache holes, just as before. But provide a raw version of the API which returns non-page entries as well, and switch shmem over to use it. Signed-off-by: Johannes Weiner <[email protected]> Reviewed-by: Rik van Riel <[email protected]> Reviewed-by: Minchan Kim <[email protected]> Cc: Andrea Arcangeli <[email protected]> Cc: Bob Liu <[email protected]> Cc: Christoph Hellwig <[email protected]> Cc: Dave Chinner <[email protected]> Cc: Greg Thelen <[email protected]> Cc: Hugh Dickins <[email protected]> Cc: Jan Kara <[email protected]> Cc: KOSAKI Motohiro <[email protected]> Cc: Luigi Semenzato <[email protected]> Cc: Mel Gorman <[email protected]> Cc: Metin Doslu <[email protected]> Cc: Michel Lespinasse <[email protected]> Cc: Ozgun Erdogan <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Roman Gushchin <[email protected]> Cc: Ryan Mallon <[email protected]> Cc: Tejun Heo <[email protected]> Cc: Vlastimil Babka <[email protected]> Signed-off-by: Andrew Morton <[email protected]> Signed-off-by: Linus Torvalds <[email protected]>
This commit is contained in:
Johannes Weiner
Linus Torvalds
parent
e7b563bb2a
commit
0cd6144aad
11 changed files with 349 additions and 130 deletions
202
mm/filemap.c
202
mm/filemap.c
|
|
@ -446,6 +446,29 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(replace_page_cache_page);
|
||||
|
||||
static int page_cache_tree_insert(struct address_space *mapping,
|
||||
struct page *page)
|
||||
{
|
||||
void **slot;
|
||||
int error;
|
||||
|
||||
slot = radix_tree_lookup_slot(&mapping->page_tree, page->index);
|
||||
if (slot) {
|
||||
void *p;
|
||||
|
||||
p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
|
||||
if (!radix_tree_exceptional_entry(p))
|
||||
return -EEXIST;
|
||||
radix_tree_replace_slot(slot, page);
|
||||
mapping->nrpages++;
|
||||
return 0;
|
||||
}
|
||||
error = radix_tree_insert(&mapping->page_tree, page->index, page);
|
||||
if (!error)
|
||||
mapping->nrpages++;
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* add_to_page_cache_locked - add a locked page to the pagecache
|
||||
* @page: page to add
|
||||
|
|
@ -480,11 +503,10 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
|
|||
page->index = offset;
|
||||
|
||||
spin_lock_irq(&mapping->tree_lock);
|
||||
error = radix_tree_insert(&mapping->page_tree, offset, page);
|
||||
error = page_cache_tree_insert(mapping, page);
|
||||
radix_tree_preload_end();
|
||||
if (unlikely(error))
|
||||
goto err_insert;
|
||||
mapping->nrpages++;
|
||||
__inc_zone_page_state(page, NR_FILE_PAGES);
|
||||
spin_unlock_irq(&mapping->tree_lock);
|
||||
trace_mm_filemap_add_to_page_cache(page);
|
||||
|
|
@ -712,7 +734,10 @@ pgoff_t page_cache_next_hole(struct address_space *mapping,
|
|||
unsigned long i;
|
||||
|
||||
for (i = 0; i < max_scan; i++) {
|
||||
if (!radix_tree_lookup(&mapping->page_tree, index))
|
||||
struct page *page;
|
||||
|
||||
page = radix_tree_lookup(&mapping->page_tree, index);
|
||||
if (!page || radix_tree_exceptional_entry(page))
|
||||
break;
|
||||
index++;
|
||||
if (index == 0)
|
||||
|
|
@ -750,7 +775,10 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping,
|
|||
unsigned long i;
|
||||
|
||||
for (i = 0; i < max_scan; i++) {
|
||||
if (!radix_tree_lookup(&mapping->page_tree, index))
|
||||
struct page *page;
|
||||
|
||||
page = radix_tree_lookup(&mapping->page_tree, index);
|
||||
if (!page || radix_tree_exceptional_entry(page))
|
||||
break;
|
||||
index--;
|
||||
if (index == ULONG_MAX)
|
||||
|
|
@ -762,14 +790,19 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping,
|
|||
EXPORT_SYMBOL(page_cache_prev_hole);
|
||||
|
||||
/**
|
||||
* find_get_page - find and get a page reference
|
||||
* find_get_entry - find and get a page cache entry
|
||||
* @mapping: the address_space to search
|
||||
* @offset: the page index
|
||||
* @offset: the page cache index
|
||||
*
|
||||
* Is there a pagecache struct page at the given (mapping, offset) tuple?
|
||||
* If yes, increment its refcount and return it; if no, return NULL.
|
||||
* Looks up the page cache slot at @mapping & @offset. If there is a
|
||||
* page cache page, it is returned with an increased refcount.
|
||||
*
|
||||
* If the slot holds a shadow entry of a previously evicted page, it
|
||||
* is returned.
|
||||
*
|
||||
* Otherwise, %NULL is returned.
|
||||
*/
|
||||
struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
|
||||
struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
|
||||
{
|
||||
void **pagep;
|
||||
struct page *page;
|
||||
|
|
@ -810,24 +843,50 @@ out:
|
|||
|
||||
return page;
|
||||
}
|
||||
EXPORT_SYMBOL(find_get_page);
|
||||
EXPORT_SYMBOL(find_get_entry);
|
||||
|
||||
/**
|
||||
* find_lock_page - locate, pin and lock a pagecache page
|
||||
* find_get_page - find and get a page reference
|
||||
* @mapping: the address_space to search
|
||||
* @offset: the page index
|
||||
*
|
||||
* Locates the desired pagecache page, locks it, increments its reference
|
||||
* count and returns its address.
|
||||
* Looks up the page cache slot at @mapping & @offset. If there is a
|
||||
* page cache page, it is returned with an increased refcount.
|
||||
*
|
||||
* Returns zero if the page was not present. find_lock_page() may sleep.
|
||||
* Otherwise, %NULL is returned.
|
||||
*/
|
||||
struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
|
||||
struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
|
||||
{
|
||||
struct page *page = find_get_entry(mapping, offset);
|
||||
|
||||
if (radix_tree_exceptional_entry(page))
|
||||
page = NULL;
|
||||
return page;
|
||||
}
|
||||
EXPORT_SYMBOL(find_get_page);
|
||||
|
||||
/**
|
||||
* find_lock_entry - locate, pin and lock a page cache entry
|
||||
* @mapping: the address_space to search
|
||||
* @offset: the page cache index
|
||||
*
|
||||
* Looks up the page cache slot at @mapping & @offset. If there is a
|
||||
* page cache page, it is returned locked and with an increased
|
||||
* refcount.
|
||||
*
|
||||
* If the slot holds a shadow entry of a previously evicted page, it
|
||||
* is returned.
|
||||
*
|
||||
* Otherwise, %NULL is returned.
|
||||
*
|
||||
* find_lock_entry() may sleep.
|
||||
*/
|
||||
struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
|
||||
{
|
||||
struct page *page;
|
||||
|
||||
repeat:
|
||||
page = find_get_page(mapping, offset);
|
||||
page = find_get_entry(mapping, offset);
|
||||
if (page && !radix_tree_exception(page)) {
|
||||
lock_page(page);
|
||||
/* Has the page been truncated? */
|
||||
|
|
@ -840,6 +899,29 @@ repeat:
|
|||
}
|
||||
return page;
|
||||
}
|
||||
EXPORT_SYMBOL(find_lock_entry);
|
||||
|
||||
/**
|
||||
* find_lock_page - locate, pin and lock a pagecache page
|
||||
* @mapping: the address_space to search
|
||||
* @offset: the page index
|
||||
*
|
||||
* Looks up the page cache slot at @mapping & @offset. If there is a
|
||||
* page cache page, it is returned locked and with an increased
|
||||
* refcount.
|
||||
*
|
||||
* Otherwise, %NULL is returned.
|
||||
*
|
||||
* find_lock_page() may sleep.
|
||||
*/
|
||||
struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
|
||||
{
|
||||
struct page *page = find_lock_entry(mapping, offset);
|
||||
|
||||
if (radix_tree_exceptional_entry(page))
|
||||
page = NULL;
|
||||
return page;
|
||||
}
|
||||
EXPORT_SYMBOL(find_lock_page);
|
||||
|
||||
/**
|
||||
|
|
@ -848,16 +930,18 @@ EXPORT_SYMBOL(find_lock_page);
|
|||
* @index: the page's index into the mapping
|
||||
* @gfp_mask: page allocation mode
|
||||
*
|
||||
* Locates a page in the pagecache. If the page is not present, a new page
|
||||
* is allocated using @gfp_mask and is added to the pagecache and to the VM's
|
||||
* LRU list. The returned page is locked and has its reference count
|
||||
* incremented.
|
||||
* Looks up the page cache slot at @mapping & @offset. If there is a
|
||||
* page cache page, it is returned locked and with an increased
|
||||
* refcount.
|
||||
*
|
||||
* find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
|
||||
* allocation!
|
||||
* If the page is not present, a new page is allocated using @gfp_mask
|
||||
* and added to the page cache and the VM's LRU list. The page is
|
||||
* returned locked and with an increased refcount.
|
||||
*
|
||||
* find_or_create_page() returns the desired page's address, or zero on
|
||||
* memory exhaustion.
|
||||
* On memory exhaustion, %NULL is returned.
|
||||
*
|
||||
* find_or_create_page() may sleep, even if @gfp_flags specifies an
|
||||
* atomic allocation!
|
||||
*/
|
||||
struct page *find_or_create_page(struct address_space *mapping,
|
||||
pgoff_t index, gfp_t gfp_mask)
|
||||
|
|
@ -889,6 +973,76 @@ repeat:
|
|||
}
|
||||
EXPORT_SYMBOL(find_or_create_page);
|
||||
|
||||
/**
|
||||
* find_get_entries - gang pagecache lookup
|
||||
* @mapping: The address_space to search
|
||||
* @start: The starting page cache index
|
||||
* @nr_entries: The maximum number of entries
|
||||
* @entries: Where the resulting entries are placed
|
||||
* @indices: The cache indices corresponding to the entries in @entries
|
||||
*
|
||||
* find_get_entries() will search for and return a group of up to
|
||||
* @nr_entries entries in the mapping. The entries are placed at
|
||||
* @entries. find_get_entries() takes a reference against any actual
|
||||
* pages it returns.
|
||||
*
|
||||
* The search returns a group of mapping-contiguous page cache entries
|
||||
* with ascending indexes. There may be holes in the indices due to
|
||||
* not-present pages.
|
||||
*
|
||||
* Any shadow entries of evicted pages are included in the returned
|
||||
* array.
|
||||
*
|
||||
* find_get_entries() returns the number of pages and shadow entries
|
||||
* which were found.
|
||||
*/
|
||||
unsigned find_get_entries(struct address_space *mapping,
|
||||
pgoff_t start, unsigned int nr_entries,
|
||||
struct page **entries, pgoff_t *indices)
|
||||
{
|
||||
void **slot;
|
||||
unsigned int ret = 0;
|
||||
struct radix_tree_iter iter;
|
||||
|
||||
if (!nr_entries)
|
||||
return 0;
|
||||
|
||||
rcu_read_lock();
|
||||
restart:
|
||||
radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
|
||||
struct page *page;
|
||||
repeat:
|
||||
page = radix_tree_deref_slot(slot);
|
||||
if (unlikely(!page))
|
||||
continue;
|
||||
if (radix_tree_exception(page)) {
|
||||
if (radix_tree_deref_retry(page))
|
||||
goto restart;
|
||||
/*
|
||||
* Otherwise, we must be storing a swap entry
|
||||
* here as an exceptional entry: so return it
|
||||
* without attempting to raise page count.
|
||||
*/
|
||||
goto export;
|
||||
}
|
||||
if (!page_cache_get_speculative(page))
|
||||
goto repeat;
|
||||
|
||||
/* Has the page moved? */
|
||||
if (unlikely(page != *slot)) {
|
||||
page_cache_release(page);
|
||||
goto repeat;
|
||||
}
|
||||
export:
|
||||
indices[ret] = iter.index;
|
||||
entries[ret] = page;
|
||||
if (++ret == nr_entries)
|
||||
break;
|
||||
}
|
||||
rcu_read_unlock();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* find_get_pages - gang pagecache lookup
|
||||
* @mapping: The address_space to search
|
||||
|
|
|
|||
Loading…
Reference in a new issue