mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-28 02:32:09 -04:00
mm: turn migrate_vma upside down
There isn't any good reason to pass callbacks to migrate_vma. Instead we can just export the three steps done by this function to drivers and let them sequence the operation without callbacks. This removes a lot of boilerplate code as-is, and will allow the drivers to drastically improve code flow and error handling further on. Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Christoph Hellwig <[email protected]> Reviewed-by: Ralph Campbell <[email protected]> Tested-by: Ralph Campbell <[email protected]> Signed-off-by: Jason Gunthorpe <[email protected]>
This commit is contained in:
Christoph Hellwig
Jason Gunthorpe
parent
f4fb3b9c19
commit
a7d1f22bb7
4 changed files with 194 additions and 344 deletions
244
mm/migrate.c
244
mm/migrate.c
|
|
@ -2118,16 +2118,6 @@ out_unlock:
|
|||
#endif /* CONFIG_NUMA */
|
||||
|
||||
#if defined(CONFIG_MIGRATE_VMA_HELPER)
|
||||
struct migrate_vma {
|
||||
struct vm_area_struct *vma;
|
||||
unsigned long *dst;
|
||||
unsigned long *src;
|
||||
unsigned long cpages;
|
||||
unsigned long npages;
|
||||
unsigned long start;
|
||||
unsigned long end;
|
||||
};
|
||||
|
||||
static int migrate_vma_collect_hole(unsigned long start,
|
||||
unsigned long end,
|
||||
struct mm_walk *walk)
|
||||
|
|
@ -2578,6 +2568,110 @@ restore:
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* migrate_vma_setup() - prepare to migrate a range of memory
|
||||
* @args: contains the vma, start, and and pfns arrays for the migration
|
||||
*
|
||||
* Returns: negative errno on failures, 0 when 0 or more pages were migrated
|
||||
* without an error.
|
||||
*
|
||||
* Prepare to migrate a range of memory virtual address range by collecting all
|
||||
* the pages backing each virtual address in the range, saving them inside the
|
||||
* src array. Then lock those pages and unmap them. Once the pages are locked
|
||||
* and unmapped, check whether each page is pinned or not. Pages that aren't
|
||||
* pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
|
||||
* corresponding src array entry. Then restores any pages that are pinned, by
|
||||
* remapping and unlocking those pages.
|
||||
*
|
||||
* The caller should then allocate destination memory and copy source memory to
|
||||
* it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
|
||||
* flag set). Once these are allocated and copied, the caller must update each
|
||||
* corresponding entry in the dst array with the pfn value of the destination
|
||||
* page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_LOCKED flags set
|
||||
* (destination pages must have their struct pages locked, via lock_page()).
|
||||
*
|
||||
* Note that the caller does not have to migrate all the pages that are marked
|
||||
* with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
|
||||
* device memory to system memory. If the caller cannot migrate a device page
|
||||
* back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
|
||||
* consequences for the userspace process, so it must be avoided if at all
|
||||
* possible.
|
||||
*
|
||||
* For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
|
||||
* do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
|
||||
* allowing the caller to allocate device memory for those unback virtual
|
||||
* address. For this the caller simply has to allocate device memory and
|
||||
* properly set the destination entry like for regular migration. Note that
|
||||
* this can still fails and thus inside the device driver must check if the
|
||||
* migration was successful for those entries after calling migrate_vma_pages()
|
||||
* just like for regular migration.
|
||||
*
|
||||
* After that, the callers must call migrate_vma_pages() to go over each entry
|
||||
* in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
|
||||
* set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
|
||||
* then migrate_vma_pages() to migrate struct page information from the source
|
||||
* struct page to the destination struct page. If it fails to migrate the
|
||||
* struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
|
||||
* src array.
|
||||
*
|
||||
* At this point all successfully migrated pages have an entry in the src
|
||||
* array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
|
||||
* array entry with MIGRATE_PFN_VALID flag set.
|
||||
*
|
||||
* Once migrate_vma_pages() returns the caller may inspect which pages were
|
||||
* successfully migrated, and which were not. Successfully migrated pages will
|
||||
* have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
|
||||
*
|
||||
* It is safe to update device page table after migrate_vma_pages() because
|
||||
* both destination and source page are still locked, and the mmap_sem is held
|
||||
* in read mode (hence no one can unmap the range being migrated).
|
||||
*
|
||||
* Once the caller is done cleaning up things and updating its page table (if it
|
||||
* chose to do so, this is not an obligation) it finally calls
|
||||
* migrate_vma_finalize() to update the CPU page table to point to new pages
|
||||
* for successfully migrated pages or otherwise restore the CPU page table to
|
||||
* point to the original source pages.
|
||||
*/
|
||||
int migrate_vma_setup(struct migrate_vma *args)
|
||||
{
|
||||
long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
|
||||
|
||||
args->start &= PAGE_MASK;
|
||||
args->end &= PAGE_MASK;
|
||||
if (!args->vma || is_vm_hugetlb_page(args->vma) ||
|
||||
(args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
|
||||
return -EINVAL;
|
||||
if (nr_pages <= 0)
|
||||
return -EINVAL;
|
||||
if (args->start < args->vma->vm_start ||
|
||||
args->start >= args->vma->vm_end)
|
||||
return -EINVAL;
|
||||
if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
|
||||
return -EINVAL;
|
||||
if (!args->src || !args->dst)
|
||||
return -EINVAL;
|
||||
|
||||
memset(args->src, 0, sizeof(*args->src) * nr_pages);
|
||||
args->cpages = 0;
|
||||
args->npages = 0;
|
||||
|
||||
migrate_vma_collect(args);
|
||||
|
||||
if (args->cpages)
|
||||
migrate_vma_prepare(args);
|
||||
if (args->cpages)
|
||||
migrate_vma_unmap(args);
|
||||
|
||||
/*
|
||||
* At this point pages are locked and unmapped, and thus they have
|
||||
* stable content and can safely be copied to destination memory that
|
||||
* is allocated by the drivers.
|
||||
*/
|
||||
return 0;
|
||||
|
||||
}
|
||||
EXPORT_SYMBOL(migrate_vma_setup);
|
||||
|
||||
static void migrate_vma_insert_page(struct migrate_vma *migrate,
|
||||
unsigned long addr,
|
||||
struct page *page,
|
||||
|
|
@ -2709,7 +2803,7 @@ abort:
|
|||
*src &= ~MIGRATE_PFN_MIGRATE;
|
||||
}
|
||||
|
||||
/*
|
||||
/**
|
||||
* migrate_vma_pages() - migrate meta-data from src page to dst page
|
||||
* @migrate: migrate struct containing all migration information
|
||||
*
|
||||
|
|
@ -2717,7 +2811,7 @@ abort:
|
|||
* struct page. This effectively finishes the migration from source page to the
|
||||
* destination page.
|
||||
*/
|
||||
static void migrate_vma_pages(struct migrate_vma *migrate)
|
||||
void migrate_vma_pages(struct migrate_vma *migrate)
|
||||
{
|
||||
const unsigned long npages = migrate->npages;
|
||||
const unsigned long start = migrate->start;
|
||||
|
|
@ -2791,8 +2885,9 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
|
|||
if (notified)
|
||||
mmu_notifier_invalidate_range_only_end(&range);
|
||||
}
|
||||
EXPORT_SYMBOL(migrate_vma_pages);
|
||||
|
||||
/*
|
||||
/**
|
||||
* migrate_vma_finalize() - restore CPU page table entry
|
||||
* @migrate: migrate struct containing all migration information
|
||||
*
|
||||
|
|
@ -2803,7 +2898,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
|
|||
* This also unlocks the pages and puts them back on the lru, or drops the extra
|
||||
* refcount, for device pages.
|
||||
*/
|
||||
static void migrate_vma_finalize(struct migrate_vma *migrate)
|
||||
void migrate_vma_finalize(struct migrate_vma *migrate)
|
||||
{
|
||||
const unsigned long npages = migrate->npages;
|
||||
unsigned long i;
|
||||
|
|
@ -2846,124 +2941,5 @@ static void migrate_vma_finalize(struct migrate_vma *migrate)
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* migrate_vma() - migrate a range of memory inside vma
|
||||
*
|
||||
* @ops: migration callback for allocating destination memory and copying
|
||||
* @vma: virtual memory area containing the range to be migrated
|
||||
* @start: start address of the range to migrate (inclusive)
|
||||
* @end: end address of the range to migrate (exclusive)
|
||||
* @src: array of hmm_pfn_t containing source pfns
|
||||
* @dst: array of hmm_pfn_t containing destination pfns
|
||||
* @private: pointer passed back to each of the callback
|
||||
* Returns: 0 on success, error code otherwise
|
||||
*
|
||||
* This function tries to migrate a range of memory virtual address range, using
|
||||
* callbacks to allocate and copy memory from source to destination. First it
|
||||
* collects all the pages backing each virtual address in the range, saving this
|
||||
* inside the src array. Then it locks those pages and unmaps them. Once the pages
|
||||
* are locked and unmapped, it checks whether each page is pinned or not. Pages
|
||||
* that aren't pinned have the MIGRATE_PFN_MIGRATE flag set (by this function)
|
||||
* in the corresponding src array entry. It then restores any pages that are
|
||||
* pinned, by remapping and unlocking those pages.
|
||||
*
|
||||
* At this point it calls the alloc_and_copy() callback. For documentation on
|
||||
* what is expected from that callback, see struct migrate_vma_ops comments in
|
||||
* include/linux/migrate.h
|
||||
*
|
||||
* After the alloc_and_copy() callback, this function goes over each entry in
|
||||
* the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
|
||||
* set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
|
||||
* then the function tries to migrate struct page information from the source
|
||||
* struct page to the destination struct page. If it fails to migrate the struct
|
||||
* page information, then it clears the MIGRATE_PFN_MIGRATE flag in the src
|
||||
* array.
|
||||
*
|
||||
* At this point all successfully migrated pages have an entry in the src
|
||||
* array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
|
||||
* array entry with MIGRATE_PFN_VALID flag set.
|
||||
*
|
||||
* It then calls the finalize_and_map() callback. See comments for "struct
|
||||
* migrate_vma_ops", in include/linux/migrate.h for details about
|
||||
* finalize_and_map() behavior.
|
||||
*
|
||||
* After the finalize_and_map() callback, for successfully migrated pages, this
|
||||
* function updates the CPU page table to point to new pages, otherwise it
|
||||
* restores the CPU page table to point to the original source pages.
|
||||
*
|
||||
* Function returns 0 after the above steps, even if no pages were migrated
|
||||
* (The function only returns an error if any of the arguments are invalid.)
|
||||
*
|
||||
* Both src and dst array must be big enough for (end - start) >> PAGE_SHIFT
|
||||
* unsigned long entries.
|
||||
*/
|
||||
int migrate_vma(const struct migrate_vma_ops *ops,
|
||||
struct vm_area_struct *vma,
|
||||
unsigned long start,
|
||||
unsigned long end,
|
||||
unsigned long *src,
|
||||
unsigned long *dst,
|
||||
void *private)
|
||||
{
|
||||
struct migrate_vma migrate;
|
||||
|
||||
/* Sanity check the arguments */
|
||||
start &= PAGE_MASK;
|
||||
end &= PAGE_MASK;
|
||||
if (!vma || is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL) ||
|
||||
vma_is_dax(vma))
|
||||
return -EINVAL;
|
||||
if (start < vma->vm_start || start >= vma->vm_end)
|
||||
return -EINVAL;
|
||||
if (end <= vma->vm_start || end > vma->vm_end)
|
||||
return -EINVAL;
|
||||
if (!ops || !src || !dst || start >= end)
|
||||
return -EINVAL;
|
||||
|
||||
memset(src, 0, sizeof(*src) * ((end - start) >> PAGE_SHIFT));
|
||||
migrate.src = src;
|
||||
migrate.dst = dst;
|
||||
migrate.start = start;
|
||||
migrate.npages = 0;
|
||||
migrate.cpages = 0;
|
||||
migrate.end = end;
|
||||
migrate.vma = vma;
|
||||
|
||||
/* Collect, and try to unmap source pages */
|
||||
migrate_vma_collect(&migrate);
|
||||
if (!migrate.cpages)
|
||||
return 0;
|
||||
|
||||
/* Lock and isolate page */
|
||||
migrate_vma_prepare(&migrate);
|
||||
if (!migrate.cpages)
|
||||
return 0;
|
||||
|
||||
/* Unmap pages */
|
||||
migrate_vma_unmap(&migrate);
|
||||
if (!migrate.cpages)
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* At this point pages are locked and unmapped, and thus they have
|
||||
* stable content and can safely be copied to destination memory that
|
||||
* is allocated by the callback.
|
||||
*
|
||||
* Note that migration can fail in migrate_vma_struct_page() for each
|
||||
* individual page.
|
||||
*/
|
||||
ops->alloc_and_copy(vma, src, dst, start, end, private);
|
||||
|
||||
/* This does the real migration of struct page */
|
||||
migrate_vma_pages(&migrate);
|
||||
|
||||
ops->finalize_and_map(vma, src, dst, start, end, private);
|
||||
|
||||
/* Unlock and remap pages */
|
||||
migrate_vma_finalize(&migrate);
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(migrate_vma);
|
||||
EXPORT_SYMBOL(migrate_vma_finalize);
|
||||
#endif /* defined(MIGRATE_VMA_HELPER) */
|
||||
|
|
|
|||
Loading…
Reference in a new issue