diff --git a/mm/memory.c b/mm/memory.c index ae9a9821a8b4..c56b341f608c 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4458,11 +4458,15 @@ int __handle_speculative_fault(struct mm_struct *mm, unsigned long address, goto out_walk; p4d = p4d_offset(pgd, address); + if (pgd_val(READ_ONCE(*pgd)) != pgd_val(pgdval)) + goto out_walk; p4dval = READ_ONCE(*p4d); if (p4d_none(p4dval) || unlikely(p4d_bad(p4dval))) goto out_walk; vmf.pud = pud_offset(p4d, address); + if (p4d_val(READ_ONCE(*p4d)) != p4d_val(p4dval)) + goto out_walk; pudval = READ_ONCE(*vmf.pud); if (pud_none(pudval) || unlikely(pud_bad(pudval))) goto out_walk; @@ -4472,6 +4476,8 @@ int __handle_speculative_fault(struct mm_struct *mm, unsigned long address, goto out_walk; vmf.pmd = pmd_offset(vmf.pud, address); + if (pud_val(READ_ONCE(*vmf.pud)) != pud_val(pudval)) + goto out_walk; vmf.orig_pmd = READ_ONCE(*vmf.pmd); /* * pmd_none could mean that a hugepage collapse is in progress @@ -4499,6 +4505,11 @@ int __handle_speculative_fault(struct mm_struct *mm, unsigned long address, */ vmf.pte = pte_offset_map(vmf.pmd, address); + if (pmd_val(READ_ONCE(*vmf.pmd)) != pmd_val(vmf.orig_pmd)) { + pte_unmap(vmf.pte); + vmf.pte = NULL; + goto out_walk; + } vmf.orig_pte = READ_ONCE(*vmf.pte); barrier(); /* See comment in handle_pte_fault() */ if (pte_none(vmf.orig_pte)) {