A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-05 19:31:57 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'sm8350/lineage-20' into lineage-22.2

Browse source 
* sm8350/lineage-20:
  power: supply: core: Use blocking_notifier_call_chain to avoid RCU complaint
  qcacld-3.0: Fix potential OOB memory access
  qcacmn: Avoid incrementing usable channel count for 0 freq
  Revert "ANDROID: GKI: mm: add struct vm_fault fields for SPECULATIVE_PAGE_FAULTS"
  Revert "mm: introduce CONFIG_SPECULATIVE_PAGE_FAULT"
  Revert "mm: prepare for FAULT_FLAG_SPECULATIVE"
  Revert "mm: introduce pte_spinlock for FAULT_FLAG_SPECULATIVE"
  Revert "mm: make pte_unmap_same compatible with SPF"
  Revert "mm: introduce INIT_VMA()"
  Revert "mm: VMA sequence count"
  Revert "mm: protect VMA modifications using VMA sequence count"
  Revert "mm: protect mremap() against SPF hanlder"
  Revert "mm: protect SPF handler against anon_vma changes"
  Revert "mm: cache some VMA fields in the vm_fault structure"
  Revert "mm/migrate: Pass vm_fault pointer to migrate_misplaced_page()"
  Revert "mm: introduce __lru_cache_add_active_or_unevictable"
  Revert "mm: introduce __vm_normal_page()"
  Revert "mm: introduce __page_add_new_anon_rmap()"
  Revert "mm: protect mm_rb tree with a rwlock"
  Revert "mm: provide speculative fault infrastructure"
  ...

Change-Id: I8ecf0cbbcd4eda942793b759c108c33b5341a76e
This commit is contained in:
Michael Bestas 2025-06-02 13:35:48 +03:00
commit 894163621a
No known key found for this signature in database
38 changed files with 242 additions and 1290 deletions
Download patch file Download diff file Expand all files Collapse all files

1
arch/arm64/Kconfig
View file

@ -191,7 +191,6 @@ config ARM64
select SYSCTL_EXCEPTION_TRACE
select THREAD_INFO_IN_TASK
select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD
select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
help
ARM 64-bit (AArch64) Linux support.

26
arch/arm64/mm/fault.c
View file

@ -410,9 +410,10 @@ static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *re
#define VM_FAULT_BADMAP ((__force vm_fault_t)0x010000)
#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x020000)
static int __do_page_fault(struct vm_area_struct *vma, unsigned long addr,
static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr,
unsigned int mm_flags, unsigned long vm_flags)
{
struct vm_area_struct *vma = find_vma(mm, addr);
if (unlikely(!vma))
return VM_FAULT_BADMAP;
@ -459,7 +460,6 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
vm_fault_t fault, major = 0;
unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
unsigned int mm_flags = FAULT_FLAG_DEFAULT;
struct vm_area_struct *vma = NULL;
if (kprobe_page_fault(regs, esr))
return 0;
@ -499,14 +499,6 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
/*
* let's try a speculative page fault without grabbing the
* mmap_sem.
*/
fault = handle_speculative_fault(mm, addr, mm_flags, &vma);
if (fault != VM_FAULT_RETRY)
goto done;
/*
* As per x86, we may deadlock here. However, since the kernel only
* validly references user space from well defined areas of the code,
@ -531,10 +523,7 @@ retry:
#endif
}
if (!vma || !can_reuse_spf_vma(vma, addr))
vma = find_vma(mm, addr);
fault = __do_page_fault(vma, addr, mm_flags, vm_flags);
fault = __do_page_fault(mm, addr, mm_flags, vm_flags);
major |= fault & VM_FAULT_MAJOR;
/* Quick path to respond to signals */
@ -547,20 +536,11 @@ retry:
if (fault & VM_FAULT_RETRY) {
if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
mm_flags |= FAULT_FLAG_TRIED;
/*
* Do not try to reuse this vma and fetch it
* again since we will release the mmap_sem.
*/
vma = NULL;
goto retry;
}
}
up_read(&mm->mmap_sem);
done:
/*
* Handle the "normal" (no error) case first.
*/

8
drivers/power/supply/power_supply_core.c
View file

@ -27,7 +27,7 @@
struct class *power_supply_class;
EXPORT_SYMBOL_GPL(power_supply_class);
ATOMIC_NOTIFIER_HEAD(power_supply_notifier);
BLOCKING_NOTIFIER_HEAD(power_supply_notifier);
EXPORT_SYMBOL_GPL(power_supply_notifier);
static struct device_type power_supply_dev_type;
@ -95,7 +95,7 @@ static void power_supply_changed_work(struct work_struct *work)
class_for_each_device(power_supply_class, NULL, psy,
__power_supply_changed_work);
power_supply_update_leds(psy);
atomic_notifier_call_chain(&power_supply_notifier,
blocking_notifier_call_chain(&power_supply_notifier,
PSY_EVENT_PROP_CHANGED, psy);
kobject_uevent(&psy->dev.kobj, KOBJ_CHANGE);
spin_lock_irqsave(&psy->changed_lock, flags);
@ -913,13 +913,13 @@ static void power_supply_dev_release(struct device *dev)
int power_supply_reg_notifier(struct notifier_block *nb)
{
return atomic_notifier_chain_register(&power_supply_notifier, nb);
return blocking_notifier_chain_register(&power_supply_notifier, nb);
}
EXPORT_SYMBOL_GPL(power_supply_reg_notifier);
void power_supply_unreg_notifier(struct notifier_block *nb)
{
atomic_notifier_chain_unregister(&power_supply_notifier, nb);
blocking_notifier_chain_unregister(&power_supply_notifier, nb);
}
EXPORT_SYMBOL_GPL(power_supply_unreg_notifier);

6
drivers/power/supply/qti_battery_charger.c
View file

@ -1801,13 +1801,13 @@ static ssize_t charging_enabled_store(struct class *c,
if (val) {
/*
* Enable charging, i.e. set the restricted current back to
* its default value and unset the restriction boolean flag.
* the thermal limit and unset the restriction boolean flag.
*/
rc = __battery_psy_set_charge_current(bcdev,
DEFAULT_RESTRICT_FCC_UA);
bcdev->thermal_fcc_ua);
if (rc < 0)
return rc;
bcdev->restrict_fcc_ua = DEFAULT_RESTRICT_FCC_UA;
bcdev->restrict_fcc_ua = bcdev->thermal_fcc_ua;
bcdev->restrict_chg_en = 0;
} else {
/*

13
drivers/staging/qca-wifi-host-cmn/umac/regulatory/core/src/reg_services_common.c
View file

@ -3338,7 +3338,7 @@ reg_update_usable_chan_resp(struct wlan_objmgr_pdev *pdev,
struct ch_params ch_params = {0};
int index = *count;
for (i = 0; i < len; i++) {
for (i = 0; i < len && index < NUM_CHANNELS; i++) {
/* In case usable channels are required for multiple filter
* mask, Some frequencies may present in res_msg . To avoid
* frequency duplication, only mode mask is updated for
@ -3690,6 +3690,8 @@ reg_get_usable_channel_coex_filter(struct wlan_objmgr_pdev *pdev,
chan_list[chan_enum].center_freq &&
freq_range.end_freq >=
chan_list[chan_enum].center_freq) {
reg_debug("avoid freq %d",
chan_list[chan_enum].center_freq);
reg_remove_freq(res_msg, chan_enum);
}
}
@ -3808,16 +3810,15 @@ wlan_reg_get_usable_channel(struct wlan_objmgr_pdev *pdev,
}
}
if (req_msg.filter_mask & 1 << FILTER_CELLULAR_COEX)
status =
reg_get_usable_channel_coex_filter(pdev, req_msg, res_msg,
chan_list, usable_channels);
if (req_msg.filter_mask & 1 << FILTER_WLAN_CONCURRENCY)
status =
reg_get_usable_channel_con_filter(pdev, req_msg, res_msg,
usable_channels);
if (req_msg.filter_mask & 1 << FILTER_CELLULAR_COEX)
status =
reg_get_usable_channel_coex_filter(pdev, req_msg, res_msg,
chan_list, usable_channels);
if (!(req_msg.filter_mask & 1 << FILTER_CELLULAR_COEX) &&
!(req_msg.filter_mask & 1 << FILTER_WLAN_CONCURRENCY))
status =

8
drivers/staging/qcacld-3.0/core/wma/src/wma_utils.c
View file

@ -1,6 +1,6 @@
/*
* Copyright (c) 2013-2021 The Linux Foundation. All rights reserved.
* Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.
* Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
@ -708,7 +708,6 @@ int wma_stats_ext_event_handler(void *handle, uint8_t *event_buf,
}
stats_ext_info = param_buf->fixed_param;
buf_ptr = (uint8_t *)stats_ext_info;
alloc_len = sizeof(tSirStatsExtEvent);
alloc_len += stats_ext_info->data_len;
@ -725,7 +724,7 @@ int wma_stats_ext_event_handler(void *handle, uint8_t *event_buf,
if (!stats_ext_event)
return -ENOMEM;
buf_ptr += sizeof(wmi_stats_ext_event_fixed_param) + WMI_TLV_HDR_SIZE;
buf_ptr = (uint8_t *)param_buf->data;
stats_ext_event->vdev_id = stats_ext_info->vdev_id;
stats_ext_event->event_data_len = stats_ext_info->data_len;
@ -775,7 +774,6 @@ int wma_stats_ext_event_handler(void *handle, uint8_t *event_buf,
}
stats_ext_info = param_buf->fixed_param;
buf_ptr = (uint8_t *)stats_ext_info;
alloc_len = sizeof(tSirStatsExtEvent);
alloc_len += stats_ext_info->data_len;
@ -791,7 +789,7 @@ int wma_stats_ext_event_handler(void *handle, uint8_t *event_buf,
if (!stats_ext_event)
return -ENOMEM;
buf_ptr += sizeof(wmi_stats_ext_event_fixed_param) + WMI_TLV_HDR_SIZE;
buf_ptr = (uint8_t *)param_buf->data;
stats_ext_event->vdev_id = stats_ext_info->vdev_id;
stats_ext_event->event_data_len = stats_ext_info->data_len;

5
fs/proc/task_mmu.c
View file

@ -1296,11 +1296,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
goto out_mm;
}
for (vma = mm->mmap; vma; vma = vma->vm_next) {
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags,
vma->vm_flags & ~VM_SOFTDIRTY);
vma->vm_flags &= ~VM_SOFTDIRTY;
vma_set_page_prot(vma);
vm_write_end(vma);
}
downgrade_write(&mm->mmap_sem);
break;

17
fs/userfaultfd.c
View file

@ -678,11 +678,8 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
octx = vma->vm_userfaultfd_ctx.ctx;
if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
vm_write_begin(vma);
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
WRITE_ONCE(vma->vm_flags,
vma->vm_flags & ~__VM_UFFD_FLAGS);
vm_write_end(vma);
vma->vm_flags &= ~__VM_UFFD_FLAGS;
return 0;
}
@ -924,10 +921,8 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
else
prev = vma;
}
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags, new_flags);
vma->vm_flags = new_flags;
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vm_write_end(vma);
}
up_write(&mm->mmap_sem);
mmput(mm);
@ -1499,10 +1494,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
* the next vma was merged into the current one and
* the current one has not been updated yet.
*/
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags, vma_pad_fixup_flags(vma, new_flags));
vma->vm_flags = vma_pad_fixup_flags(vma, new_flags);
vma->vm_userfaultfd_ctx.ctx = ctx;
vm_write_end(vma);
if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
hugetlb_unshare_all_pmds(vma);
@ -1674,10 +1667,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
* the next vma was merged into the current one and
* the current one has not been updated yet.
*/
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags, vma_pad_fixup_flags(vma, new_flags));
vma->vm_flags = vma_pad_fixup_flags(vma, new_flags);
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vm_write_end(vma);
skip:
prev = vma;

2
include/linux/hugetlb_inline.h
View file

@ -8,7 +8,7 @@
static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma)
{
return !!(READ_ONCE(vma->vm_flags) & VM_HUGETLB);
return !!(vma->vm_flags & VM_HUGETLB);
}
#else

4
include/linux/migrate.h
View file

@ -131,14 +131,14 @@ static inline void __ClearPageMovable(struct page *page)
#ifdef CONFIG_NUMA_BALANCING
extern bool pmd_trans_migrating(pmd_t pmd);
extern int migrate_misplaced_page(struct page *page,
struct vm_fault *vmf, int node);
struct vm_area_struct *vma, int node);
#else
static inline bool pmd_trans_migrating(pmd_t pmd)
{
return false;
}
static inline int migrate_misplaced_page(struct page *page,
struct vm_fault *vmf, int node)
struct vm_area_struct *vma, int node)
{
return -EAGAIN; /* can't migrate now */
}

121
include/linux/mm.h
View file

@ -417,7 +417,6 @@ extern pgprot_t protection_map[16];
* @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch.
* @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals.
* @FAULT_FLAG_PREFAULT_OLD: Make faultaround ptes old.
* @FAULT_FLAG_SPECULATIVE: Speculative fault, not holding mmap_sem.
*
* About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify
* whether we would allow page faults to retry by specifying these two
@ -449,7 +448,6 @@ extern pgprot_t protection_map[16];
#define FAULT_FLAG_INSTRUCTION 0x100
#define FAULT_FLAG_INTERRUPTIBLE 0x200
#define FAULT_FLAG_PREFAULT_OLD 0x400
#define FAULT_FLAG_SPECULATIVE 0x800
/*
* The default fault flags that should be used by most of the
@ -505,10 +503,6 @@ struct vm_fault {
gfp_t gfp_mask; /* gfp mask to be used for allocations */
pgoff_t pgoff; /* Logical page offset based on vma */
unsigned long address; /* Faulting virtual address */
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
unsigned int sequence;
pmd_t orig_pmd; /* value of PMD at the time of fault */
#endif
pmd_t *pmd; /* Pointer to pmd entry matching
* the 'address' */
pud_t *pud; /* Pointer to pud entry matching
@ -539,8 +533,6 @@ struct vm_fault {
* page table to avoid allocation from
* atomic context.
*/
unsigned long vma_flags; /* Speculative Page Fault field */
pgprot_t vma_page_prot; /* Speculative Page Fault field */
ANDROID_VENDOR_DATA(1);
ANDROID_VENDOR_DATA(2);
};
@ -624,15 +616,6 @@ struct vm_operations_struct {
ANDROID_KABI_RESERVE(4);
};
static inline void INIT_VMA(struct vm_area_struct *vma)
{
INIT_LIST_HEAD(&vma->anon_vma_chain);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
seqcount_init(&vma->vm_sequence);
atomic_set(&vma->vm_ref_count, 1);
#endif
}
static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
{
static const struct vm_operations_struct dummy_vm_ops = {};
@ -640,7 +623,7 @@ static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
memset(vma, 0, sizeof(*vma));
vma->vm_mm = mm;
vma->vm_ops = &dummy_vm_ops;
INIT_VMA(vma);
INIT_LIST_HEAD(&vma->anon_vma_chain);
}
static inline void vma_set_anonymous(struct vm_area_struct *vma)
@ -951,9 +934,9 @@ void free_compound_page(struct page *page);
* pte_mkwrite. But get_user_pages can cause write faults for mappings
* that do not have writing enabled, when used by access_process_vm.
*/
static inline pte_t maybe_mkwrite(pte_t pte, unsigned long vma_flags)
static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
{
if (likely(vma_flags & VM_WRITE))
if (likely(vma->vm_flags & VM_WRITE))
pte = pte_mkwrite(pte);
return pte;
}
@ -1574,14 +1557,8 @@ struct zap_details {
struct page *single_page; /* Locked page to be unmapped */
};
struct page *_vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
pte_t pte, unsigned long vma_flags);
static inline struct page *vm_normal_page(struct vm_area_struct *vma,
unsigned long addr, pte_t pte)
{
return _vm_normal_page(vma, addr, pte, vma->vm_flags);
}
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
pte_t pte);
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
pmd_t pmd);
@ -1610,34 +1587,6 @@ int follow_phys(struct vm_area_struct *vma, unsigned long address,
int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
static inline void vm_write_begin(struct vm_area_struct *vma)
{
/*
* Isolated vma might be freed without exclusive mmap_lock but
* speculative page fault handler still needs to know it was changed.
*/
if (!RB_EMPTY_NODE(&vma->vm_rb))
WARN_ON_ONCE(!rwsem_is_locked(&(vma->vm_mm)->mmap_sem));
/*
* The reads never spins and preemption
* disablement is not required.
*/
raw_write_seqcount_begin(&vma->vm_sequence);
}
static inline void vm_write_end(struct vm_area_struct *vma)
{
raw_write_seqcount_end(&vma->vm_sequence);
}
#else
static inline void vm_write_begin(struct vm_area_struct *vma)
{
}
static inline void vm_write_end(struct vm_area_struct *vma)
{
}
#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
extern void truncate_pagecache(struct inode *inode, loff_t new);
extern void truncate_setsize(struct inode *inode, loff_t newsize);
void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
@ -1649,43 +1598,6 @@ int invalidate_inode_page(struct page *page);
#ifdef CONFIG_MMU
extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
unsigned long address, unsigned int flags);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
extern int __handle_speculative_fault(struct mm_struct *mm,
unsigned long address,
unsigned int flags,
struct vm_area_struct **vma);
static inline int handle_speculative_fault(struct mm_struct *mm,
unsigned long address,
unsigned int flags,
struct vm_area_struct **vma)
{
/*
* Try speculative page fault for multithreaded user space task only.
*/
if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1) {
*vma = NULL;
return VM_FAULT_RETRY;
}
return __handle_speculative_fault(mm, address, flags, vma);
}
extern bool can_reuse_spf_vma(struct vm_area_struct *vma,
unsigned long address);
#else
static inline int handle_speculative_fault(struct mm_struct *mm,
unsigned long address,
unsigned int flags,
struct vm_area_struct **vma)
{
return VM_FAULT_RETRY;
}
static inline bool can_reuse_spf_vma(struct vm_area_struct *vma,
unsigned long address)
{
return false;
}
#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
unsigned long address, unsigned int fault_flags,
bool *unlocked);
@ -2488,29 +2400,16 @@ void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
extern int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
struct vm_area_struct *expand, bool keep_locked);
struct vm_area_struct *expand);
static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
{
return __vma_adjust(vma, start, end, pgoff, insert, NULL, false);
return __vma_adjust(vma, start, end, pgoff, insert, NULL);
}
extern struct vm_area_struct *__vma_merge(struct mm_struct *mm,
extern struct vm_area_struct *vma_merge(struct mm_struct *,
struct vm_area_struct *prev, unsigned long addr, unsigned long end,
unsigned long vm_flags, struct anon_vma *anon, struct file *file,
pgoff_t pgoff, struct mempolicy *mpol, struct vm_userfaultfd_ctx uff,
const char __user *user, bool keep_locked);
static inline struct vm_area_struct *vma_merge(struct mm_struct *mm,
struct vm_area_struct *prev, unsigned long addr, unsigned long end,
unsigned long vm_flags, struct anon_vma *anon, struct file *file,
pgoff_t off, struct mempolicy *pol, struct vm_userfaultfd_ctx uff,
const char __user *user)
{
return __vma_merge(mm, prev, addr, end, vm_flags, anon, file, off,
pol, uff, user, false);
}
unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
struct mempolicy *, struct vm_userfaultfd_ctx, const char __user *);
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
extern int __split_vma(struct mm_struct *, struct vm_area_struct *,
unsigned long addr, int new_below);

9
include/linux/mm_types.h
View file

@ -362,10 +362,7 @@ struct vm_area_struct {
struct mempolicy *vm_policy; /* NUMA policy for the VMA */
#endif
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
seqcount_t vm_sequence;
atomic_t vm_ref_count; /* see vma_get(), vma_put() */
#endif
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
ANDROID_KABI_RESERVE(3);
@ -390,9 +387,6 @@ struct mm_struct {
struct vm_area_struct *mmap; /* list of VMAs */
struct rb_root mm_rb;
u64 vmacache_seqnum; /* per-thread vmacache */
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
rwlock_t mm_rb_lock;
#endif
#ifdef CONFIG_MMU
unsigned long (*get_unmapped_area) (struct file *filp,
unsigned long addr, unsigned long len,
@ -711,7 +705,6 @@ enum vm_fault_reason {
VM_FAULT_FALLBACK = (__force vm_fault_t)0x000800,
VM_FAULT_DONE_COW = (__force vm_fault_t)0x001000,
VM_FAULT_NEEDDSYNC = (__force vm_fault_t)0x002000,
VM_FAULT_PTNOTSAME = (__force vm_fault_t)0x004000,
VM_FAULT_HINDEX_MASK = (__force vm_fault_t)0x0f0000,
};

4
include/linux/pagemap.h
View file

@ -494,8 +494,8 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma,
pgoff_t pgoff;
if (unlikely(is_vm_hugetlb_page(vma)))
return linear_hugepage_index(vma, address);
pgoff = (address - READ_ONCE(vma->vm_start)) >> PAGE_SHIFT;
pgoff += READ_ONCE(vma->vm_pgoff);
pgoff = (address - vma->vm_start) >> PAGE_SHIFT;
pgoff += vma->vm_pgoff;
return pgoff;
}

2
include/linux/power_supply.h
View file

@ -402,7 +402,7 @@ struct power_supply_battery_info {
int resist_table_size;
};
extern struct atomic_notifier_head power_supply_notifier;
extern struct blocking_notifier_head power_supply_notifier;
extern int power_supply_reg_notifier(struct notifier_block *nb);
extern void power_supply_unreg_notifier(struct notifier_block *nb);
extern struct power_supply *power_supply_get_by_name(const char *name);

12
include/linux/rmap.h
View file

@ -198,16 +198,8 @@ void page_add_anon_rmap(struct page *, struct vm_area_struct *,
unsigned long, bool);
void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
unsigned long, int);
void __page_add_new_anon_rmap(struct page *page, struct vm_area_struct *vma,
unsigned long address, bool compound);
static inline void page_add_new_anon_rmap(struct page *page,
struct vm_area_struct *vma,
unsigned long address, bool compound)
{
VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
__page_add_new_anon_rmap(page, vma, address, compound);
}
void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
unsigned long, bool);
void page_add_file_rmap(struct page *, bool);
void page_remove_rmap(struct page *, bool);

10
include/linux/swap.h
View file

@ -344,14 +344,8 @@ extern void deactivate_page(struct page *page);
extern void mark_page_lazyfree(struct page *page);
extern void swap_setup(void);
extern void __lru_cache_add_active_or_unevictable(struct page *page,
unsigned long vma_flags);
static inline void lru_cache_add_active_or_unevictable(struct page *page,
struct vm_area_struct *vma)
{
return __lru_cache_add_active_or_unevictable(page, vma->vm_flags);
}
extern void lru_cache_add_active_or_unevictable(struct page *page,
struct vm_area_struct *vma);
/* linux/mm/vmscan.c */
extern unsigned long zone_reclaimable_pages(struct zone *zone);

4
include/linux/vm_event_item.h
View file

@ -113,10 +113,6 @@ enum vm_event_item { PGPGIN, PGPGOUT,
#ifdef CONFIG_SWAP
SWAP_RA,
SWAP_RA_HIT,
#endif
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
SPECULATIVE_PGFAULT_ANON,
SPECULATIVE_PGFAULT_FILE,
#endif
NR_VM_EVENT_ITEMS
};

88
include/trace/events/pagefault.h
View file

@ -1,88 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM pagefault
#if !defined(_TRACE_PAGEFAULT_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_PAGEFAULT_H
#include <linux/tracepoint.h>
#include <linux/mm.h>
DECLARE_EVENT_CLASS(spf,
TP_PROTO(unsigned long caller,
struct vm_area_struct *vma, unsigned long address),
TP_ARGS(caller, vma, address),
TP_STRUCT__entry(
__field(unsigned long, caller)
__field(unsigned long, vm_start)
__field(unsigned long, vm_end)
__field(unsigned long, address)
),
TP_fast_assign(
__entry->caller = caller;
__entry->vm_start = vma->vm_start;
__entry->vm_end = vma->vm_end;
__entry->address = address;
),
TP_printk("ip:%lx vma:%lx-%lx address:%lx",
__entry->caller, __entry->vm_start, __entry->vm_end,
__entry->address)
);
DEFINE_EVENT(spf, spf_pte_lock,
TP_PROTO(unsigned long caller,
struct vm_area_struct *vma, unsigned long address),
TP_ARGS(caller, vma, address)
);
DEFINE_EVENT(spf, spf_vma_changed,
TP_PROTO(unsigned long caller,
struct vm_area_struct *vma, unsigned long address),
TP_ARGS(caller, vma, address)
);
DEFINE_EVENT(spf, spf_vma_noanon,
TP_PROTO(unsigned long caller,
struct vm_area_struct *vma, unsigned long address),
TP_ARGS(caller, vma, address)
);
DEFINE_EVENT(spf, spf_vma_notsup,
TP_PROTO(unsigned long caller,
struct vm_area_struct *vma, unsigned long address),
TP_ARGS(caller, vma, address)
);
DEFINE_EVENT(spf, spf_vma_access,
TP_PROTO(unsigned long caller,
struct vm_area_struct *vma, unsigned long address),
TP_ARGS(caller, vma, address)
);
DEFINE_EVENT(spf, spf_pmd_changed,
TP_PROTO(unsigned long caller,
struct vm_area_struct *vma, unsigned long address),
TP_ARGS(caller, vma, address)
);
#endif /* _TRACE_PAGEFAULT_H */
/* This part must be outside protection */
#include <trace/define_trace.h>

35
kernel/fork.c
View file

@ -362,7 +362,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
if (new) {
*new = *orig;
INIT_VMA(new);
INIT_LIST_HEAD(&new->anon_vma_chain);
}
return new;
}
@ -486,7 +486,7 @@ EXPORT_SYMBOL(free_task);
static __latent_entropy int dup_mmap(struct mm_struct *mm,
struct mm_struct *oldmm)
{
struct vm_area_struct *mpnt, *tmp, *prev, **pprev, *last = NULL;
struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
struct rb_node **rb_link, *rb_parent;
int retval;
unsigned long charge;
@ -605,18 +605,8 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
rb_parent = &tmp->vm_rb;
mm->map_count++;
if (!(tmp->vm_flags & VM_WIPEONFORK)) {
if (IS_ENABLED(CONFIG_SPECULATIVE_PAGE_FAULT)) {
/*
* Mark this VMA as changing to prevent the
* speculative page fault hanlder to process
* it until the TLB are flushed below.
*/
last = mpnt;
vm_write_begin(mpnt);
}
if (!(tmp->vm_flags & VM_WIPEONFORK))
retval = copy_page_range(mm, oldmm, mpnt);
}
if (tmp->vm_ops && tmp->vm_ops->open)
tmp->vm_ops->open(tmp);
@ -629,22 +619,6 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
out:
up_write(&mm->mmap_sem);
flush_tlb_mm(oldmm);
if (IS_ENABLED(CONFIG_SPECULATIVE_PAGE_FAULT)) {
/*
* Since the TLB has been flush, we can safely unmark the
* copied VMAs and allows the speculative page fault handler to
* process them again.
* Walk back the VMA list from the last marked VMA.
*/
for (; last; last = last->vm_prev) {
if (last->vm_flags & VM_DONTCOPY)
continue;
if (!(last->vm_flags & VM_WIPEONFORK))
vm_write_end(last);
}
}
up_write(&oldmm->mmap_sem);
dup_userfaultfd_complete(&uf);
fail_uprobe_end:
@ -1062,9 +1036,6 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
mm->mmap = NULL;
mm->mm_rb = RB_ROOT;
mm->vmacache_seqnum = 0;
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
rwlock_init(&mm->mm_rb_lock);
#endif
atomic_set(&mm->mm_users, 1);
atomic_set(&mm->mm_count, 1);
init_rwsem(&mm->mmap_sem);

23
mm/Kconfig
View file

@ -780,29 +780,6 @@ config HAVE_USERSPACE_LOW_MEMORY_KILLER
when the OOM killer and userspace memory killer both have the
potential to run).
config ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
def_bool n
config SPECULATIVE_PAGE_FAULT
bool "Speculative page faults"
default y
depends on ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
depends on MMU && SMP
depends on QGKI
help
Try to handle user space page faults without holding the mmap_sem.
This should allow better concurrency for massively threaded process
since the page fault handler will not wait for other threads memory
layout change to be done, assuming that this change is done in another
part of the process's memory space. This type of page fault is named
speculative page fault.
If the speculative page fault fails because of a concurrency is
detected or because underlying PMD or PTE tables are not yet
allocating, it is failing its processing and a classic page fault
is then tried.
config GUP_BENCHMARK
bool "Enable infrastructure for get_user_pages_fast() benchmarking"
help

56
mm/filemap.c
View file

@ -2567,12 +2567,12 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
#endif
/* If we don't want any read-ahead, don't bother */
if (vmf->vma_flags & VM_RAND_READ)
if (vmf->vma->vm_flags & VM_RAND_READ)
return fpin;
if (!ra->ra_pages)
return fpin;
if (vmf->vma_flags & VM_SEQ_READ) {
if (vmf->vma->vm_flags & VM_SEQ_READ) {
fpin = maybe_unlock_mmap_for_io(vmf, fpin);
page_cache_sync_readahead(mapping, ra, file, offset,
ra->ra_pages);
@ -2624,7 +2624,7 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
pgoff_t offset = vmf->pgoff;
/* If we don't want any read-ahead, don't bother */
if (vmf->vma_flags & VM_RAND_READ || !ra->ra_pages)
if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages)
return fpin;
if (ra->mmap_miss > 0)
ra->mmap_miss--;
@ -2647,9 +2647,7 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
* it in the page cache, and handles the special cases reasonably without
* having a lot of duplicated code.
*
* If FAULT_FLAG_SPECULATIVE is set, this function runs with elevated vma
* refcount and with mmap lock not held.
* Otherwise, vma->vm_mm->mmap_sem must be held on entry.
* vma->vm_mm->mmap_sem must be held on entry.
*
* If our return value has VM_FAULT_RETRY set, it's because the mmap_sem
* may be dropped before doing I/O or by lock_page_maybe_drop_mmap().
@ -2674,52 +2672,6 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
struct page *page;
vm_fault_t ret = 0;
if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
page = find_get_page(mapping, offset);
if (unlikely(!page))
return VM_FAULT_RETRY;
if (unlikely(PageReadahead(page)))
goto page_put;
if (!trylock_page(page))
goto page_put;
if (unlikely(compound_head(page)->mapping != mapping))
goto page_unlock;
VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
if (unlikely(!PageUptodate(page)))
goto page_unlock;
max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
if (unlikely(offset >= max_off))
goto page_unlock;
/*
* Update readahead mmap_miss statistic.
*
* Note that we are not sure if finish_fault() will
* manage to complete the transaction. If it fails,
* we'll come back to filemap_fault() non-speculative
* case which will update mmap_miss a second time.
* This is not ideal, we would prefer to guarantee the
* update will happen exactly once.
*/
if (!(vmf->vma->vm_flags & VM_RAND_READ) && ra->ra_pages) {
unsigned int mmap_miss = READ_ONCE(ra->mmap_miss);
if (mmap_miss)
WRITE_ONCE(ra->mmap_miss, --mmap_miss);
}
vmf->page = page;
return VM_FAULT_LOCKED;
page_unlock:
unlock_page(page);
page_put:
put_page(page);
return VM_FAULT_RETRY;
}
max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
if (unlikely(offset >= max_off))
return VM_FAULT_SIGBUS;

6
mm/huge_memory.c
View file

@ -1270,8 +1270,8 @@ static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
pte_t entry;
entry = mk_pte(pages[i], vmf->vma_page_prot);
entry = maybe_mkwrite(pte_mkdirty(entry), vmf->vma_flags);
entry = mk_pte(pages[i], vma->vm_page_prot);
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
memcg = (void *)page_private(pages[i]);
set_page_private(pages[i], 0);
page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false);
@ -2263,7 +2263,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
entry = pte_swp_mksoft_dirty(entry);
} else {
entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
entry = maybe_mkwrite(entry, vma->vm_flags);
entry = maybe_mkwrite(entry, vma);
if (!write)
entry = pte_wrprotect(entry);
if (!young)

3
mm/init-mm.c
View file

@ -28,9 +28,6 @@
*/
struct mm_struct init_mm = {
.mm_rb = RB_ROOT,
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
.mm_rb_lock = __RW_LOCK_UNLOCKED(init_mm.mm_rb_lock),
#endif
.pgd = swapper_pg_dir,
.mm_users = ATOMIC_INIT(2),
.mm_count = ATOMIC_INIT(1),

20
mm/internal.h
View file

@ -36,26 +36,6 @@ void page_writeback_init(void);
vm_fault_t do_swap_page(struct vm_fault *vmf);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
extern struct vm_area_struct *get_vma(struct mm_struct *mm,
unsigned long addr);
extern void put_vma(struct vm_area_struct *vma);
static inline bool vma_has_changed(struct vm_fault *vmf)
{
int ret = RB_EMPTY_NODE(&vmf->vma->vm_rb);
unsigned int seq = READ_ONCE(vmf->vma->vm_sequence.sequence);
/*
* Matches both the wmb in write_seqlock_{begin,end}() and
* the wmb in vma_rb_erase().
*/
smp_rmb();
return ret || seq != vmf->sequence;
}
#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
unsigned long floor, unsigned long ceiling);

11
mm/khugepaged.c
View file

@ -918,8 +918,6 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm,
.flags = FAULT_FLAG_ALLOW_RETRY,
.pmd = pmd,
.pgoff = linear_page_index(vma, address),
.vma_flags = vma->vm_flags,
.vma_page_prot = vma->vm_page_prot,
};
/* we only decide to swapin, if there is enough young ptes */
@ -1043,7 +1041,6 @@ static void collapse_huge_page(struct mm_struct *mm,
if (mm_find_pmd(mm, address) != pmd)
goto out;
vm_write_begin(vma);
anon_vma_lock_write(vma->anon_vma);
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm,
@ -1081,7 +1078,6 @@ static void collapse_huge_page(struct mm_struct *mm,
pmd_populate(mm, pmd, pmd_pgtable(_pmd));
spin_unlock(pmd_ptl);
anon_vma_unlock_write(vma->anon_vma);
vm_write_end(vma);
result = SCAN_FAIL;
goto out;
}
@ -1117,7 +1113,6 @@ static void collapse_huge_page(struct mm_struct *mm,
set_pmd_at(mm, address, pmd, _pmd);
update_mmu_cache_pmd(vma, address, pmd);
spin_unlock(pmd_ptl);
vm_write_end(vma);
*hpage = NULL;
@ -1345,8 +1340,6 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr)
if (!pmd)
goto drop_hpage;
vm_write_begin(vma);
/*
* We need to lock the mapping so that from here on, only GUP-fast and
* hardware page walks can access the parts of the page tables that
@ -1414,7 +1407,6 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr)
haddr + HPAGE_PMD_SIZE);
mmu_notifier_invalidate_range_start(&range);
_pmd = pmdp_collapse_flush(vma, haddr, pmd);
vm_write_end(vma);
mm_dec_nr_ptes(mm);
tlb_remove_table_sync_one();
mmu_notifier_invalidate_range_end(&range);
@ -1431,7 +1423,6 @@ drop_hpage:
abort:
pte_unmap_unlock(start_pte, ptl);
vm_write_end(vma);
i_mmap_unlock_write(vma->vm_file->f_mapping);
goto drop_hpage;
}
@ -1512,10 +1503,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
NULL, mm, addr,
addr + HPAGE_PMD_SIZE);
mmu_notifier_invalidate_range_start(&range);
vm_write_begin(vma);
/* assume page table is clear */
_pmd = pmdp_collapse_flush(vma, addr, pmd);
vm_write_end(vma);
mm_dec_nr_ptes(mm);
tlb_remove_table_sync_one();
pte_free(mm, pmd_pgtable(_pmd));

4
mm/madvise.c
View file

@ -172,9 +172,7 @@ success:
/*
* vm_flags is protected by the mmap_sem held in write mode.
*/
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags, vma_pad_fixup_flags(vma, new_flags));
vm_write_end(vma);
vma->vm_flags = vma_pad_fixup_flags(vma, new_flags);
out_convert_errno:
/*

650
mm/memory.c
View file

File diff suppressed because it is too large Load diff

49
mm/mempolicy.c
View file

@ -380,11 +380,8 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
struct vm_area_struct *vma;
down_write(&mm->mmap_sem);
for (vma = mm->mmap; vma; vma = vma->vm_next) {
vm_write_begin(vma);
for (vma = mm->mmap; vma; vma = vma->vm_next)
mpol_rebind_policy(vma->vm_policy, new);
vm_write_end(vma);
}
up_write(&mm->mmap_sem);
}
@ -715,7 +712,6 @@ static int vma_replace_policy(struct vm_area_struct *vma,
if (IS_ERR(new))
return PTR_ERR(new);
vm_write_begin(vma);
if (vma->vm_ops && vma->vm_ops->set_policy) {
err = vma->vm_ops->set_policy(vma, new);
if (err)
@ -723,17 +719,11 @@ static int vma_replace_policy(struct vm_area_struct *vma,
}
old = vma->vm_policy;
/*
* The speculative page fault handler accesses this field without
* hodling the mmap_sem.
*/
WRITE_ONCE(vma->vm_policy, new);
vm_write_end(vma);
vma->vm_policy = new; /* protected by mmap_sem */
mpol_put(old);
return 0;
err_out:
vm_write_end(vma);
mpol_put(new);
return err;
}
@ -1708,28 +1698,23 @@ COMPAT_SYSCALL_DEFINE4(migrate_pages, compat_pid_t, pid,
struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
unsigned long addr)
{
struct mempolicy *pol;
struct mempolicy *pol = NULL;
if (!vma)
return NULL;
if (vma) {
if (vma->vm_ops && vma->vm_ops->get_policy) {
pol = vma->vm_ops->get_policy(vma, addr);
} else if (vma->vm_policy) {
pol = vma->vm_policy;
if (vma->vm_ops && vma->vm_ops->get_policy)
return vma->vm_ops->get_policy(vma, addr);
/*
* This could be called without holding the mmap_sem in the
* speculative page fault handler's path.
*/
pol = READ_ONCE(vma->vm_policy);
if (pol) {
/*
* shmem_alloc_page() passes MPOL_F_SHARED policy with
* a pseudo vma whose vma->vm_ops=NULL. Take a reference
* count on these policies which will be dropped by
* mpol_cond_put() later
*/
if (mpol_needs_cond_ref(pol))
mpol_get(pol);
/*
* shmem_alloc_page() passes MPOL_F_SHARED policy with
* a pseudo vma whose vma->vm_ops=NULL. Take a reference
* count on these policies which will be dropped by
* mpol_cond_put() later
*/
if (mpol_needs_cond_ref(pol))
mpol_get(pol);
}
}
return pol;

6
mm/migrate.c
View file

@ -241,7 +241,7 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
*/
entry = pte_to_swp_entry(*pvmw.pte);
if (is_write_migration_entry(entry))
pte = maybe_mkwrite(pte, vma->vm_flags);
pte = maybe_mkwrite(pte, vma);
if (unlikely(is_zone_device_page(new))) {
if (is_device_private_page(new)) {
@ -1976,7 +1976,7 @@ bool pmd_trans_migrating(pmd_t pmd)
* node. Caller is expected to have an elevated reference count on
* the page that will be dropped by this function before returning.
*/
int migrate_misplaced_page(struct page *page, struct vm_fault *vmf,
int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
int node)
{
pg_data_t *pgdat = NODE_DATA(node);
@ -1989,7 +1989,7 @@ int migrate_misplaced_page(struct page *page, struct vm_fault *vmf,
* with execute permissions as they are probably shared libraries.
*/
if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
(vmf->vma_flags & VM_EXEC))
(vma->vm_flags & VM_EXEC))
goto out;
/*

14
mm/mlock.c
View file

@ -446,9 +446,7 @@ static unsigned long __munlock_pagevec_fill(struct pagevec *pvec,
void munlock_vma_pages_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags, vma->vm_flags & VM_LOCKED_CLEAR_MASK);
vm_write_end(vma);
vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
while (start < end) {
struct page *page;
@ -572,11 +570,11 @@ success:
* It's okay if try_to_unmap_one unmaps a page just after we
* set VM_LOCKED, populate_vma_page_range will bring it back.
*/
if (lock) {
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags, vma_pad_fixup_flags(vma, newflags));
vm_write_end(vma);
} else
if (lock)
vma->vm_flags = vma_pad_fixup_flags(vma, newflags);
else
munlock_vma_pages_range(vma, start, end);
out:

226
mm/mmap.c
View file

@ -166,27 +166,6 @@ void unlink_file_vma(struct vm_area_struct *vma)
}
}
static void __free_vma(struct vm_area_struct *vma)
{
if (vma->vm_file)
fput(vma->vm_file);
mpol_put(vma_policy(vma));
vm_area_free(vma);
}
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
void put_vma(struct vm_area_struct *vma)
{
if (atomic_dec_and_test(&vma->vm_ref_count))
__free_vma(vma);
}
#else
static inline void put_vma(struct vm_area_struct *vma)
{
__free_vma(vma);
}
#endif
/*
* Close a vm structure and free it, returning the next.
*/
@ -197,7 +176,10 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
might_sleep();
if (vma->vm_ops && vma->vm_ops->close)
vma->vm_ops->close(vma);
put_vma(vma);
if (vma->vm_file)
fput(vma->vm_file);
mpol_put(vma_policy(vma));
vm_area_free(vma);
return next;
}
@ -450,13 +432,6 @@ static void validate_mm(struct mm_struct *mm)
RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks,
struct vm_area_struct, vm_rb,
unsigned long, rb_subtree_gap, vma_compute_gap)
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
#define mm_rb_write_lock(mm) write_lock(&(mm)->mm_rb_lock)
#define mm_rb_write_unlock(mm) write_unlock(&(mm)->mm_rb_lock)
#else
#define mm_rb_write_lock(mm) do { } while (0)
#define mm_rb_write_unlock(mm) do { } while (0)
#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
/*
* Update augmented rbtree rb_subtree_gap values after vma->vm_start or
@ -473,37 +448,26 @@ static void vma_gap_update(struct vm_area_struct *vma)
}
static inline void vma_rb_insert(struct vm_area_struct *vma,
struct mm_struct *mm)
struct rb_root *root)
{
struct rb_root *root = &mm->mm_rb;
/* All rb_subtree_gap values must be consistent prior to insertion */
validate_mm_rb(root, NULL);
rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
}
static void __vma_rb_erase(struct vm_area_struct *vma, struct mm_struct *mm)
static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
{
struct rb_root *root = &mm->mm_rb;
/*
* Note rb_erase_augmented is a fairly large inline function,
* so make sure we instantiate it only once with our desired
* augmented rbtree callbacks.
*/
mm_rb_write_lock(mm);
rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
mm_rb_write_unlock(mm); /* wmb */
/*
* Ensure the removal is complete before clearing the node.
* Matched by vma_has_changed()/handle_speculative_fault().
*/
RB_CLEAR_NODE(&vma->vm_rb);
}
static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma,
struct mm_struct *mm,
struct rb_root *root,
struct vm_area_struct *ignore)
{
/*
@ -511,21 +475,21 @@ static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma,
* with the possible exception of the "next" vma being erased if
* next->vm_start was reduced.
*/
validate_mm_rb(&mm->mm_rb, ignore);
validate_mm_rb(root, ignore);
__vma_rb_erase(vma, mm);
__vma_rb_erase(vma, root);
}
static __always_inline void vma_rb_erase(struct vm_area_struct *vma,
struct mm_struct *mm)
struct rb_root *root)
{
/*
* All rb_subtree_gap values must be consistent prior to erase,
* with the possible exception of the vma being erased.
*/
validate_mm_rb(&mm->mm_rb, vma);
validate_mm_rb(root, vma);
__vma_rb_erase(vma, mm);
__vma_rb_erase(vma, root);
}
/*
@ -640,12 +604,10 @@ void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
* immediately update the gap to the correct value. Finally we
* rebalance the rbtree after all augmented values have been set.
*/
mm_rb_write_lock(mm);
rb_link_node(&vma->vm_rb, rb_parent, rb_link);
vma->rb_subtree_gap = 0;
vma_gap_update(vma);
vma_rb_insert(vma, mm);
mm_rb_write_unlock(mm);
vma_rb_insert(vma, &mm->mm_rb);
}
static void __vma_link_file(struct vm_area_struct *vma)
@ -721,7 +683,7 @@ static __always_inline void __vma_unlink_common(struct mm_struct *mm,
{
struct vm_area_struct *next;
vma_rb_erase_ignore(vma, mm, ignore);
vma_rb_erase_ignore(vma, &mm->mm_rb, ignore);
next = vma->vm_next;
if (has_prev)
prev->vm_next = next;
@ -755,7 +717,7 @@ static inline void __vma_unlink_prev(struct mm_struct *mm,
*/
int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
struct vm_area_struct *expand, bool keep_locked)
struct vm_area_struct *expand)
{
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *next = vma->vm_next, *orig_vma = vma;
@ -767,10 +729,6 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
long adjust_next = 0;
int remove_next = 0;
vm_write_begin(vma);
if (next)
vm_write_begin(next);
if (next && !insert) {
struct vm_area_struct *exporter = NULL, *importer = NULL;
@ -851,12 +809,8 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
importer->anon_vma = exporter->anon_vma;
error = anon_vma_clone(importer, exporter);
if (error) {
if (next && next != vma)
vm_write_end(next);
vm_write_end(vma);
if (error)
return error;
}
}
}
again:
@ -902,18 +856,17 @@ again:
}
if (start != vma->vm_start) {
WRITE_ONCE(vma->vm_start, start);
vma->vm_start = start;
start_changed = true;
}
if (end != vma->vm_end) {
WRITE_ONCE(vma->vm_end, end);
vma->vm_end = end;
end_changed = true;
}
WRITE_ONCE(vma->vm_pgoff, pgoff);
vma->vm_pgoff = pgoff;
if (adjust_next) {
WRITE_ONCE(next->vm_start,
next->vm_start + (adjust_next << PAGE_SHIFT));
WRITE_ONCE(next->vm_pgoff, next->vm_pgoff + adjust_next);
next->vm_start += adjust_next << PAGE_SHIFT;
next->vm_pgoff += adjust_next;
}
if (root) {
@ -978,13 +931,15 @@ again:
}
if (remove_next) {
if (file)
if (file) {
uprobe_munmap(next, next->vm_start, next->vm_end);
fput(file);
}
if (next->anon_vma)
anon_vma_merge(vma, next);
mm->map_count--;
vm_write_end(next);
put_vma(next);
mpol_put(vma_policy(next));
vm_area_free(next);
/*
* In mprotect's case 6 (see comments on vma_merge),
* we must remove another next too. It would clutter
@ -998,8 +953,6 @@ again:
* "vma->vm_next" gap must be updated.
*/
next = vma->vm_next;
if (next)
vm_write_begin(next);
} else {
/*
* For the scope of the comment "next" and
@ -1046,11 +999,6 @@ again:
if (insert && file)
uprobe_mmap(insert);
if (next && next != vma)
vm_write_end(next);
if (!keep_locked)
vm_write_end(vma);
validate_mm(mm);
return 0;
@ -1192,13 +1140,13 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
* parameter) may establish ptes with the wrong permissions of NNNN
* instead of the right permissions of XXXX.
*/
struct vm_area_struct *__vma_merge(struct mm_struct *mm,
struct vm_area_struct *vma_merge(struct mm_struct *mm,
struct vm_area_struct *prev, unsigned long addr,
unsigned long end, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file,
pgoff_t pgoff, struct mempolicy *policy,
struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
const char __user *anon_name, bool keep_locked)
const char __user *anon_name)
{
pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
struct vm_area_struct *area, *next;
@ -1248,11 +1196,10 @@ struct vm_area_struct *__vma_merge(struct mm_struct *mm,
/* cases 1, 6 */
err = __vma_adjust(prev, prev->vm_start,
next->vm_end, prev->vm_pgoff, NULL,
prev, keep_locked);
prev);
} else /* cases 2, 5, 7 */
err = __vma_adjust(prev, prev->vm_start,
end, prev->vm_pgoff, NULL, prev,
keep_locked);
end, prev->vm_pgoff, NULL, prev);
if (err)
return NULL;
khugepaged_enter_vma_merge(prev, vm_flags);
@ -1270,12 +1217,10 @@ struct vm_area_struct *__vma_merge(struct mm_struct *mm,
anon_name)) {
if (prev && addr < prev->vm_end) /* case 4 */
err = __vma_adjust(prev, prev->vm_start,
addr, prev->vm_pgoff, NULL, next,
keep_locked);
addr, prev->vm_pgoff, NULL, next);
else { /* cases 3, 8 */
err = __vma_adjust(area, addr, next->vm_end,
next->vm_pgoff - pglen, NULL, next,
keep_locked);
next->vm_pgoff - pglen, NULL, next);
/*
* In case 3 area is already equal to next and
* this is a noop, but in case 8 "area" has
@ -1899,14 +1844,12 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
out:
perf_event_mmap(vma);
vm_write_begin(vma);
vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
if (vm_flags & VM_LOCKED) {
if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
is_vm_hugetlb_page(vma) ||
vma == get_gate_vma(current->mm))
WRITE_ONCE(vma->vm_flags,
vma->vm_flags & VM_LOCKED_CLEAR_MASK);
vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
else
mm->locked_vm += (len >> PAGE_SHIFT);
}
@ -1921,10 +1864,9 @@ out:
* then new mapped in-place (which must be aimed as
* a completely new data area).
*/
WRITE_ONCE(vma->vm_flags, vma->vm_flags | VM_SOFTDIRTY);
vma->vm_flags |= VM_SOFTDIRTY;
vma_set_page_prot(vma);
vm_write_end(vma);
return addr;
@ -2299,11 +2241,15 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
EXPORT_SYMBOL(get_unmapped_area);
/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
static struct vm_area_struct *__find_vma(struct mm_struct *mm,
unsigned long addr)
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
struct rb_node *rb_node;
struct vm_area_struct *vma = NULL;
struct vm_area_struct *vma;
/* Check the cache first. */
vma = vmacache_find(mm, addr);
if (likely(vma))
return vma;
rb_node = mm->mm_rb.rb_node;
@ -2321,54 +2267,13 @@ static struct vm_area_struct *__find_vma(struct mm_struct *mm,
rb_node = rb_node->rb_right;
}
return vma;
}
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma;
/* Check the cache first. */
vma = vmacache_find(mm, addr);
if (likely(vma))
return vma;
vma = __find_vma(mm, addr);
if (vma)
vmacache_update(addr, vma);
return vma;
}
EXPORT_SYMBOL(find_vma);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
struct vm_area_struct *get_vma(struct mm_struct *mm, unsigned long addr)
{
struct vm_area_struct *vma = NULL;
read_lock(&mm->mm_rb_lock);
vma = __find_vma(mm, addr);
/*
* If there is a concurrent fast mremap, bail out since the entire
* PMD/PUD subtree may have been remapped.
*
* This is usually safe for conventional mremap since it takes the
* PTE locks as does SPF. However fast mremap only takes the lock
* at the PMD/PUD level which is ok as it is done with the mmap
* write lock held. But since SPF, as the term implies forgoes,
* taking the mmap read lock and also cannot take PTL lock at the
* larger PMD/PUD granualrity, since it would introduce huge
* contention in the page fault path; fall back to regular fault
* handling.
*/
if (vma && !atomic_inc_unless_negative(&vma->vm_ref_count))
vma = NULL;
read_unlock(&mm->mm_rb_lock);
return vma;
}
#endif
/*
* Same as find_vma, but also return a pointer to the previous VMA in *pprev.
*/
@ -2589,8 +2494,8 @@ int expand_downwards(struct vm_area_struct *vma,
mm->locked_vm += grow;
vm_stat_account(mm, vma->vm_flags, grow);
anon_vma_interval_tree_pre_update_vma(vma);
WRITE_ONCE(vma->vm_start, address);
WRITE_ONCE(vma->vm_pgoff, vma->vm_pgoff - grow);
vma->vm_start = address;
vma->vm_pgoff -= grow;
anon_vma_interval_tree_post_update_vma(vma);
vma_gap_update(vma);
spin_unlock(&mm->page_table_lock);
@ -2753,7 +2658,7 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
insertion_point = (prev ? &prev->vm_next : &mm->mmap);
vma->vm_prev = NULL;
do {
vma_rb_erase(vma, mm);
vma_rb_erase(vma, &mm->mm_rb);
mm->map_count--;
tail_vma = vma;
vma = vma->vm_next;
@ -3259,9 +3164,10 @@ void exit_mmap(struct mm_struct *mm)
(void)__oom_reap_task_mm(mm);
set_bit(MMF_OOM_SKIP, &mm->flags);
down_write(&mm->mmap_sem);
up_write(&mm->mmap_sem);
}
down_write(&mm->mmap_sem);
if (mm->locked_vm) {
vma = mm->mmap;
while (vma) {
@ -3274,11 +3180,8 @@ void exit_mmap(struct mm_struct *mm)
arch_exit_mmap(mm);
vma = mm->mmap;
if (!vma) {
/* Can happen if dup_mmap() received an OOM */
up_write(&mm->mmap_sem);;
if (!vma) /* Can happen if dup_mmap() received an OOM */
return;
}
lru_add_drain();
flush_cache_mm(mm);
@ -3289,14 +3192,16 @@ void exit_mmap(struct mm_struct *mm)
free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
tlb_finish_mmu(&tlb, 0, -1);
/* Walk the list again, actually closing and freeing it. */
/*
* Walk the list again, actually closing and freeing it,
* with preemption enabled, without holding any MM locks.
*/
while (vma) {
if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += vma_pages(vma);
vma = remove_vma(vma);
cond_resched();
}
up_write(&mm->mmap_sem);
vm_unacct_memory(nr_accounted);
}
@ -3363,21 +3268,9 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
return NULL; /* should never get here */
/* There is 3 cases to manage here in
* AAAA AAAA AAAA AAAA
* PPPP.... PPPP......NNNN PPPP....NNNN PP........NN
* PPPPPPPP(A) PPPP..NNNNNNNN(B) PPPPPPPPPPPP(1) NULL
* PPPPPPPPNNNN(2)
* PPPPNNNNNNNN(3)
*
* new_vma == prev in case A,1,2
* new_vma == next in case B,3
*/
new_vma = __vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
vma->anon_vma, vma->vm_file, pgoff,
vma_policy(vma), vma->vm_userfaultfd_ctx,
vma_get_anon_name(vma), true);
new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, vma_get_anon_name(vma));
if (new_vma) {
/*
* Source vma may have been merged into new_vma
@ -3415,15 +3308,6 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
get_file(new_vma->vm_file);
if (new_vma->vm_ops && new_vma->vm_ops->open)
new_vma->vm_ops->open(new_vma);
/*
* As the VMA is linked right now, it may be hit by the
* speculative page fault handler. But we don't want it to
* to start mapping page in this area until the caller has
* potentially move the pte from the moved VMA. To prevent
* that we protect it right now, and let the caller unprotect
* it once the move is done.
*/
vm_write_begin(new_vma);
vma_link(mm, new_vma, prev, rb_link, rb_parent);
*need_rmap_locks = false;
}

4
mm/mprotect.c
View file

@ -455,14 +455,12 @@ success:
* vm_flags and vm_page_prot are protected by the mmap_sem
* held in write mode.
*/
vm_write_begin(vma);
WRITE_ONCE(vma->vm_flags, vma_pad_fixup_flags(vma, newflags));
vma->vm_flags = vma_pad_fixup_flags(vma, newflags);
dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot);
vma_set_page_prot(vma);
change_protection(vma, start, end, vma->vm_page_prot,
dirty_accountable, 0);
vm_write_end(vma);
/*
* Private VM_LOCKED VMA becoming writable: trigger COW to avoid major

54
mm/mremap.c
View file

@ -210,38 +210,6 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
drop_rmap_locks(vma);
}
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
static inline bool trylock_vma_ref_count(struct vm_area_struct *vma)
{
/*
* If we have the only reference, swap the refcount to -1. This
* will prevent other concurrent references by get_vma() for SPFs.
*/
return atomic_cmpxchg(&vma->vm_ref_count, 1, -1) == 1;
}
/*
* Restore the VMA reference count to 1 after a fast mremap.
*/
static inline void unlock_vma_ref_count(struct vm_area_struct *vma)
{
/*
* This should only be called after a corresponding,
* successful trylock_vma_ref_count().
*/
VM_BUG_ON_VMA(atomic_cmpxchg(&vma->vm_ref_count, -1, 1) != -1,
vma);
}
#else /* !CONFIG_SPECULATIVE_PAGE_FAULT */
static inline bool trylock_vma_ref_count(struct vm_area_struct *vma)
{
return true;
}
static inline void unlock_vma_ref_count(struct vm_area_struct *vma)
{
}
#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
#ifdef CONFIG_HAVE_MOVE_PMD
static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
unsigned long new_addr, unsigned long old_end,
@ -262,14 +230,6 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
if (WARN_ON(!pmd_none(*new_pmd)))
return false;
/*
* We hold both exclusive mmap_lock and rmap_lock at this point and
* cannot block. If we cannot immediately take exclusive ownership
* of the VMA fallback to the move_ptes().
*/
if (!trylock_vma_ref_count(vma))
return false;
/*
* We don't have to worry about the ordering of src and dst
* ptlocks because exclusive mmap_sem prevents deadlock.
@ -292,7 +252,6 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
spin_unlock(new_ptl);
spin_unlock(old_ptl);
unlock_vma_ref_count(vma);
return true;
}
#else
@ -568,14 +527,6 @@ static unsigned long move_vma(struct vm_area_struct *vma,
return -ENOMEM;
}
/* new_vma is returned protected by copy_vma, to prevent speculative
* page fault to be done in the destination area before we move the pte.
* Now, we must also protect the source VMA since we don't want pages
* to be mapped in our back while we are copying the PTEs.
*/
if (vma != new_vma)
vm_write_begin(vma);
moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
need_rmap_locks);
if (moved_len < old_len) {
@ -592,8 +543,6 @@ static unsigned long move_vma(struct vm_area_struct *vma,
*/
move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
true);
if (vma != new_vma)
vm_write_end(vma);
vma = new_vma;
old_len = new_len;
old_addr = new_addr;
@ -602,10 +551,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
mremap_userfaultfd_prep(new_vma, uf);
arch_remap(mm, old_addr, old_addr + old_len,
new_addr, new_addr + new_len);
if (vma != new_vma)
vm_write_end(vma);
}
vm_write_end(new_vma);
/* Conceal VM_ACCOUNT so old reservation is not undone */
if (vm_flags & VM_ACCOUNT && !(flags & MREMAP_DONTUNMAP)) {

5
mm/rmap.c
View file

@ -1150,7 +1150,7 @@ void do_page_add_anon_rmap(struct page *page,
}
/**
* __page_add_new_anon_rmap - add pte mapping to a new anonymous page
* page_add_new_anon_rmap - add pte mapping to a new anonymous page
* @page: the page to add the mapping to
* @vma: the vm area in which the mapping is added
* @address: the user virtual address mapped
@ -1160,11 +1160,12 @@ void do_page_add_anon_rmap(struct page *page,
* This means the inc-and-test can be bypassed.
* Page does not have to be locked.
*/
void __page_add_new_anon_rmap(struct page *page,
void page_add_new_anon_rmap(struct page *page,
struct vm_area_struct *vma, unsigned long address, bool compound)
{
int nr = compound ? hpage_nr_pages(page) : 1;
VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
__SetPageSwapBacked(page);
if (compound) {
VM_BUG_ON_PAGE(!PageTransHuge(page), page);

4
mm/shmem.c
View file

@ -2072,10 +2072,10 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf)
sgp = SGP_CACHE;
if ((vmf->vma_flags & VM_NOHUGEPAGE) ||
if ((vma->vm_flags & VM_NOHUGEPAGE) ||
test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
sgp = SGP_NOHUGE;
else if (vmf->vma_flags & VM_HUGEPAGE)
else if (vma->vm_flags & VM_HUGEPAGE)
sgp = SGP_HUGE;
err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,

6
mm/swap.c
View file

@ -452,12 +452,12 @@ void lru_cache_add(struct page *page)
* directly back onto it's zone's unevictable list, it does NOT use a
* per cpu pagevec.
*/
void __lru_cache_add_active_or_unevictable(struct page *page,
unsigned long vma_flags)
void lru_cache_add_active_or_unevictable(struct page *page,
struct vm_area_struct *vma)
{
VM_BUG_ON_PAGE(PageLRU(page), page);
if (likely((vma_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
SetPageActive(page);
else if (!TestSetPageMlocked(page)) {
/*

10
mm/swap_state.c
View file

@ -537,11 +537,7 @@ static unsigned long swapin_nr_pages(unsigned long offset)
* This has been extended to use the NUMA policies from the mm triggering
* the readahead.
*
* Caller must hold down_read on the vma->vm_mm if vmf->vma is not NULL.
* This is needed to ensure the VMA will not be freed in our back. In the case
* of the speculative page fault handler, this cannot happen, even if we don't
* hold the mmap_sem. Callees are assumed to take care of reading VMA's fields
* using READ_ONCE() to read consistent values.
* Caller must hold read mmap_sem if vmf->vma is not NULL.
*/
struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
struct vm_fault *vmf)
@ -638,9 +634,9 @@ static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma,
unsigned long *start,
unsigned long *end)
{
*start = max3(lpfn, PFN_DOWN(READ_ONCE(vma->vm_start)),
*start = max3(lpfn, PFN_DOWN(vma->vm_start),
PFN_DOWN(faddr & PMD_MASK));
*end = min3(rpfn, PFN_DOWN(READ_ONCE(vma->vm_end)),
*end = min3(rpfn, PFN_DOWN(vma->vm_end),
PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE));
}

6
mm/vmstat.c
View file

@ -1300,11 +1300,7 @@ const char * const vmstat_text[] = {
"swap_ra",
"swap_ra_hit",
#endif
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
"speculative_pgfault_anon",
"speculative_pgfault_file",
#endif
#endif /* CONFIG_VM_EVENT_COUNTERS */
#endif /* CONFIG_VM_EVENTS_COUNTERS */
};
#endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */