A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-25 22:23:14 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge "kernel: Add snapshot of changes to support cpu isolation"

Browse source
This commit is contained in:
qctecmdr 2019-12-14 06:22:08 -08:00 • committed by Gerrit - the friendly Code Review server
commit 18b2bd0259
42 changed files with 2533 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

29
drivers/base/arch_topology.c
View file

@ -471,6 +471,26 @@ const struct cpumask *cpu_coregroup_mask(int cpu)
return core_mask;
}
#ifdef CONFIG_SCHED_WALT
void update_possible_siblings_masks(unsigned int cpuid)
{
struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
int cpu;
if (cpuid_topo->package_id == -1)
return;
for_each_possible_cpu(cpu) {
cpu_topo = &cpu_topology[cpu];
if (cpuid_topo->package_id != cpu_topo->package_id)
continue;
cpumask_set_cpu(cpuid, &cpu_topo->core_possible_sibling);
cpumask_set_cpu(cpu, &cpuid_topo->core_possible_sibling);
}
}
#endif
void update_siblings_masks(unsigned int cpuid)
{
struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
@ -550,6 +570,9 @@ __weak int __init parse_acpi_topology(void)
#if defined(CONFIG_ARM64) || defined(CONFIG_RISCV)
void __init init_cpu_topology(void)
{
#ifdef CONFIG_SCHED_WALT
int cpu;
#endif
reset_cpu_topology();
/*
@ -560,5 +583,11 @@ void __init init_cpu_topology(void)
reset_cpu_topology();
else if (of_have_populated_dt() && parse_dt_topology())
reset_cpu_topology();
#ifdef CONFIG_SCHED_WALT
else {
for_each_possible_cpu(cpu)
update_possible_siblings_masks(cpu);
}
#endif
}
#endif

5
drivers/base/core.c
View file

@ -1156,6 +1156,11 @@ int lock_device_hotplug_sysfs(void)
return restart_syscall();
}
void lock_device_hotplug_assert(void)
{
lockdep_assert_held(&device_hotplug_lock);
}
#ifdef CONFIG_BLOCK
static inline int device_is_not_partition(struct device *dev)
{

93
drivers/base/cpu.c
View file

@ -183,9 +183,94 @@ static struct attribute_group crash_note_cpu_attr_group = {
};
#endif
#ifdef CONFIG_HOTPLUG_CPU
static ssize_t isolate_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cpu *cpu = container_of(dev, struct cpu, dev);
ssize_t rc;
int cpuid = cpu->dev.id;
unsigned int isolated = cpu_isolated(cpuid);
rc = scnprintf(buf, PAGE_SIZE-2, "%d\n", isolated);
return rc;
}
static DEVICE_ATTR_RO(isolate);
static struct attribute *cpu_isolated_attrs[] = {
&dev_attr_isolate.attr,
NULL
};
static struct attribute_group cpu_isolated_attr_group = {
.attrs = cpu_isolated_attrs,
};
#endif
#ifdef CONFIG_SCHED_WALT
static ssize_t sched_load_boost_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
ssize_t rc;
int boost;
struct cpu *cpu = container_of(dev, struct cpu, dev);
int cpuid = cpu->dev.id;
boost = per_cpu(sched_load_boost, cpuid);
rc = scnprintf(buf, PAGE_SIZE-2, "%d\n", boost);
return rc;
}
static ssize_t __ref sched_load_boost_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
int err;
int boost;
struct cpu *cpu = container_of(dev, struct cpu, dev);
int cpuid = cpu->dev.id;
err = kstrtoint(strstrip((char *)buf), 0, &boost);
if (err)
return err;
/*
* -100 is low enough to cancel out CPU's load and make it near zro.
* 1000 is close to the maximum value that cpu_util_freq_{walt,pelt}
* can take without overflow.
*/
if (boost < -100 || boost > 1000)
return -EINVAL;
per_cpu(sched_load_boost, cpuid) = boost;
return count;
}
static DEVICE_ATTR_RW(sched_load_boost);
static struct attribute *sched_cpu_attrs[] = {
&dev_attr_sched_load_boost.attr,
NULL
};
static struct attribute_group sched_cpu_attr_group = {
.attrs = sched_cpu_attrs,
};
#endif
static const struct attribute_group *common_cpu_attr_groups[] = {
#ifdef CONFIG_KEXEC
&crash_note_cpu_attr_group,
#endif
#ifdef CONFIG_HOTPLUG_CPU
&cpu_isolated_attr_group,
#endif
#ifdef CONFIG_SCHED_WALT
&sched_cpu_attr_group,
#endif
NULL
};
@ -193,6 +278,12 @@ static const struct attribute_group *common_cpu_attr_groups[] = {
static const struct attribute_group *hotplugable_cpu_attr_groups[] = {
#ifdef CONFIG_KEXEC
&crash_note_cpu_attr_group,
#endif
#ifdef CONFIG_HOTPLUG_CPU
&cpu_isolated_attr_group,
#endif
#ifdef CONFIG_SCHED_WALT
&sched_cpu_attr_group,
#endif
NULL
};
@ -223,6 +314,7 @@ static struct cpu_attr cpu_attrs[] = {
_CPU_ATTR(online, &__cpu_online_mask),
_CPU_ATTR(possible, &__cpu_possible_mask),
_CPU_ATTR(present, &__cpu_present_mask),
_CPU_ATTR(core_ctl_isolated, &__cpu_isolated_mask),
};
/*
@ -472,6 +564,7 @@ static struct attribute *cpu_root_attrs[] = {
&cpu_attrs[0].attr.attr,
&cpu_attrs[1].attr.attr,
&cpu_attrs[2].attr.attr,
&cpu_attrs[3].attr.attr,
&dev_attr_kernel_max.attr,
&dev_attr_offline.attr,
&dev_attr_isolated.attr,

57
fs/proc/base.c
View file

@ -1459,6 +1459,56 @@ static const struct file_operations proc_pid_sched_operations = {
#endif
/*
* Print out various scheduling related per-task fields:
*/
#ifdef CONFIG_SCHED_WALT
extern int __weak sched_wake_up_idle_show(struct seq_file *m, void *v);
extern ssize_t __weak sched_wake_up_idle_write(struct file *file,
const char __user *buf, size_t count, loff_t *offset);
extern int __weak sched_wake_up_idle_open(struct inode *inode,
struct file *filp);
static const struct file_operations proc_pid_sched_wake_up_idle_operations = {
.open = sched_wake_up_idle_open,
.read = seq_read,
.write = sched_wake_up_idle_write,
.llseek = seq_lseek,
.release = single_release,
};
extern int __weak sched_init_task_load_show(struct seq_file *m, void *v);
extern ssize_t __weak
sched_init_task_load_write(struct file *file, const char __user *buf,
size_t count, loff_t *offset);
extern int __weak
sched_init_task_load_open(struct inode *inode, struct file *filp);
static const struct file_operations proc_pid_sched_init_task_load_operations = {
.open = sched_init_task_load_open,
.read = seq_read,
.write = sched_init_task_load_write,
.llseek = seq_lseek,
.release = single_release,
};
extern int __weak sched_group_id_show(struct seq_file *m, void *v);
extern ssize_t __weak
sched_group_id_write(struct file *file, const char __user *buf,
size_t count, loff_t *offset);
extern int __weak sched_group_id_open(struct inode *inode, struct file *filp);
static const struct file_operations proc_pid_sched_group_id_operations = {
.open = sched_group_id_open,
.read = seq_read,
.write = sched_group_id_write,
.llseek = seq_lseek,
.release = single_release,
};
#endif /* CONFIG_SCHED_WALT */
#ifdef CONFIG_SCHED_AUTOGROUP
/*
* Print out autogroup related information:
@ -3011,6 +3061,13 @@ static const struct pid_entry tgid_base_stuff[] = {
ONE("status", S_IRUGO, proc_pid_status),
ONE("personality", S_IRUSR, proc_pid_personality),
ONE("limits", S_IRUGO, proc_pid_limits),
#ifdef CONFIG_SCHED_WALT
REG("sched_wake_up_idle", 00644,
proc_pid_sched_wake_up_idle_operations),
REG("sched_init_task_load", 00644,
proc_pid_sched_init_task_load_operations),
REG("sched_group_id", 00666, proc_pid_sched_group_id_operations),
#endif
#ifdef CONFIG_SCHED_DEBUG
REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations),
#endif

7
include/linux/arch_topology.h
View file

@ -48,6 +48,9 @@ struct cpu_topology {
int llc_id;
cpumask_t thread_sibling;
cpumask_t core_sibling;
#ifdef CONFIG_SCHED_WALT
cpumask_t core_possible_sibling;
#endif
cpumask_t llc_sibling;
};
@ -59,6 +62,10 @@ extern struct cpu_topology cpu_topology[NR_CPUS];
#define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling)
#define topology_sibling_cpumask(cpu) (&cpu_topology[cpu].thread_sibling)
#define topology_llc_cpumask(cpu) (&cpu_topology[cpu].llc_sibling)
#ifdef CONFIG_SCHED_WALT
#define topology_possible_sibling_cpumask(cpu) \
(&cpu_topology[cpu].core_possible_sibling)
#endif
void init_cpu_topology(void);
void store_cpu_topology(unsigned int cpuid);
const struct cpumask *cpu_coregroup_mask(int cpu);

3
include/linux/cpuhotplug.h
View file

@ -69,6 +69,9 @@ enum cpuhp_state {
CPUHP_SLAB_PREPARE,
CPUHP_MD_RAID5_PREPARE,
CPUHP_RCUTREE_PREP,
#ifdef CONFIG_SCHED_WALT
CPUHP_CORE_CTL_ISOLATION_DEAD,
#endif
CPUHP_CPUIDLE_COUPLED_PREPARE,
CPUHP_POWERPC_PMAC_PREPARE,
CPUHP_POWERPC_MMU_CTX_PREPARE,

25
include/linux/cpumask.h
View file

@ -55,6 +55,7 @@ extern unsigned int nr_cpu_ids;
* cpu_present_mask - has bit 'cpu' set iff cpu is populated
* cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
* cpu_active_mask - has bit 'cpu' set iff cpu available to migration
* cpu_isolated_mask- has bit 'cpu' set iff cpu isolated
*
* If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
*
@ -91,10 +92,12 @@ extern struct cpumask __cpu_possible_mask;
extern struct cpumask __cpu_online_mask;
extern struct cpumask __cpu_present_mask;
extern struct cpumask __cpu_active_mask;
extern struct cpumask __cpu_isolated_mask;
#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
#define cpu_isolated_mask ((const struct cpumask *)&__cpu_isolated_mask)
extern atomic_t __num_online_cpus;
@ -114,19 +117,31 @@ static inline unsigned int num_online_cpus(void)
#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
#define num_isolated_cpus() cpumask_weight(cpu_isolated_mask)
#define num_online_uniso_cpus() \
({ \
cpumask_t mask; \
\
cpumask_andnot(&mask, cpu_online_mask, cpu_isolated_mask); \
cpumask_weight(&mask); \
})
#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
#define cpu_isolated(cpu) cpumask_test_cpu((cpu), cpu_isolated_mask)
#else
#define num_online_cpus() 1U
#define num_possible_cpus() 1U
#define num_present_cpus() 1U
#define num_active_cpus() 1U
#define num_isolated_cpus() 0U
#define num_online_uniso_cpus() 1U
#define cpu_online(cpu) ((cpu) == 0)
#define cpu_possible(cpu) ((cpu) == 0)
#define cpu_present(cpu) ((cpu) == 0)
#define cpu_active(cpu) ((cpu) == 0)
#define cpu_isolated(cpu) ((cpu) != 0)
#endif
extern cpumask_t cpus_booted_once_mask;
@ -806,6 +821,7 @@ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
#define for_each_isolated_cpu(cpu) for_each_cpu((cpu), cpu_isolated_mask)
/* Wrappers for arch boot code to manipulate normally-constant masks */
void init_cpu_present(const struct cpumask *src);
@ -846,6 +862,15 @@ set_cpu_active(unsigned int cpu, bool active)
cpumask_clear_cpu(cpu, &__cpu_active_mask);
}
static inline void
set_cpu_isolated(unsigned int cpu, bool isolated)
{
if (isolated)
cpumask_set_cpu(cpu, &__cpu_isolated_mask);
else
cpumask_clear_cpu(cpu, &__cpu_isolated_mask);
}
/**
* to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *

1
include/linux/device.h
View file

@ -1551,6 +1551,7 @@ static inline bool device_supports_offline(struct device *dev)
extern void lock_device_hotplug(void);
extern void unlock_device_hotplug(void);
extern int lock_device_hotplug_sysfs(void);
extern void lock_device_hotplug_assert(void);
extern int device_offline(struct device *dev);
extern int device_online(struct device *dev);
extern void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode);

17
include/linux/nmi.h
View file

@ -13,6 +13,9 @@
#ifdef CONFIG_LOCKUP_DETECTOR
void lockup_detector_init(void);
extern void watchdog_enable(unsigned int cpu);
extern void watchdog_disable(unsigned int cpu);
extern bool watchdog_configured(unsigned int cpu);
void lockup_detector_soft_poweroff(void);
void lockup_detector_cleanup(void);
bool is_hardlockup(void);
@ -37,6 +40,20 @@ extern int sysctl_hardlockup_all_cpu_backtrace;
static inline void lockup_detector_init(void) { }
static inline void lockup_detector_soft_poweroff(void) { }
static inline void lockup_detector_cleanup(void) { }
static inline void watchdog_enable(unsigned int cpu)
{
}
static inline void watchdog_disable(unsigned int cpu)
{
}
static inline bool watchdog_configured(unsigned int cpu)
{
/*
* Pretend the watchdog is always configured.
* We will be waiting for the watchdog to be enabled in core isolation
*/
return true;
}
#endif /* !CONFIG_LOCKUP_DETECTOR */
#ifdef CONFIG_SOFTLOCKUP_DETECTOR

178
include/linux/sched.h
View file

@ -117,6 +117,18 @@ struct task_group;
(task->flags & PF_FROZEN) == 0 && \
(task->state & TASK_NOLOAD) == 0)
/*
* Enum for display driver to provide varying refresh rates
*/
enum fps {
FPS0 = 0,
FPS30 = 30,
FPS48 = 48,
FPS60 = 60,
FPS90 = 90,
FPS120 = 120,
};
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
/*
@ -212,6 +224,42 @@ struct task_group;
/* Task command name length: */
#define TASK_COMM_LEN 16
enum task_event {
PUT_PREV_TASK = 0,
PICK_NEXT_TASK = 1,
TASK_WAKE = 2,
TASK_MIGRATE = 3,
TASK_UPDATE = 4,
IRQ_UPDATE = 5,
};
/* Note: this need to be in sync with migrate_type_names array */
enum migrate_types {
GROUP_TO_RQ,
RQ_TO_GROUP,
};
#ifdef CONFIG_HOTPLUG_CPU
extern int __weak sched_isolate_cpu(int cpu);
extern int __weak sched_unisolate_cpu(int cpu);
extern int __weak sched_unisolate_cpu_unlocked(int cpu);
#else
static inline int sched_isolate_cpu(int cpu)
{
return 0;
}
static inline int sched_unisolate_cpu(int cpu)
{
return 0;
}
static inline int sched_unisolate_cpu_unlocked(int cpu)
{
return 0;
}
#endif
extern void scheduler_tick(void);
#define MAX_SCHEDULE_TIMEOUT LONG_MAX
@ -478,6 +526,89 @@ struct sched_entity {
#endif
};
struct cpu_cycle_counter_cb {
u64 (*get_cpu_cycle_counter)(int cpu);
};
DECLARE_PER_CPU_READ_MOSTLY(int, sched_load_boost);
#ifdef CONFIG_SCHED_WALT
extern void __weak sched_exit(struct task_struct *p);
extern int __weak
register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb);
extern void __weak
sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax);
extern void __weak free_task_load_ptrs(struct task_struct *p);
extern void __weak sched_set_refresh_rate(enum fps fps);
#define RAVG_HIST_SIZE_MAX 5
#define NUM_BUSY_BUCKETS 10
/* ravg represents frequency scaled cpu-demand of tasks */
struct ravg {
/*
* 'mark_start' marks the beginning of an event (task waking up, task
* starting to execute, task being preempted) within a window
*
* 'sum' represents how runnable a task has been within current
* window. It incorporates both running time and wait time and is
* frequency scaled.
*
* 'sum_history' keeps track of history of 'sum' seen over previous
* RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are
* ignored.
*
* 'demand' represents maximum sum seen over previous
* sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency
* demand for tasks.
*
* 'curr_window_cpu' represents task's contribution to cpu busy time on
* various CPUs in the current window
*
* 'prev_window_cpu' represents task's contribution to cpu busy time on
* various CPUs in the previous window
*
* 'curr_window' represents the sum of all entries in curr_window_cpu
*
* 'prev_window' represents the sum of all entries in prev_window_cpu
*
* 'pred_demand' represents task's current predicted cpu busy time
*
* 'busy_buckets' groups historical busy time into different buckets
* used for prediction
*
* 'demand_scaled' represents task's demand scaled to 1024
*/
u64 mark_start;
u32 sum, demand;
u32 coloc_demand;
u32 sum_history[RAVG_HIST_SIZE_MAX];
u32 *curr_window_cpu, *prev_window_cpu;
u32 curr_window, prev_window;
u32 pred_demand;
u8 busy_buckets[NUM_BUSY_BUCKETS];
u16 demand_scaled;
u16 pred_demand_scaled;
u64 active_time;
u64 last_win_size;
};
#else
static inline void sched_exit(struct task_struct *p) { }
static inline int
register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)
{
return 0;
}
static inline void free_task_load_ptrs(struct task_struct *p) { }
static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus,
u32 fmin, u32 fmax) { }
static inline void sched_set_refresh_rate(enum fps fps) { }
#endif /* CONFIG_SCHED_WALT */
struct sched_rt_entity {
struct list_head run_list;
unsigned long timeout;
@ -675,6 +806,20 @@ struct task_struct {
const struct sched_class *sched_class;
struct sched_entity se;
struct sched_rt_entity rt;
#ifdef CONFIG_SCHED_WALT
u64 last_sleep_ts;
bool wake_up_idle;
struct ravg ravg;
u32 init_load_pct;
u64 last_wake_ts;
u64 last_enqueued_ts;
struct related_thread_group *grp;
struct list_head grp_list;
u64 cpu_cycles;
bool misfit;
u8 unfilter;
#endif
#ifdef CONFIG_CGROUP_SCHED
struct task_group *sched_task_group;
#endif
@ -2000,4 +2145,37 @@ int sched_trace_rq_cpu(struct rq *rq);
const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
#ifdef CONFIG_SCHED_WALT
#define PF_WAKE_UP_IDLE 1
static inline u32 sched_get_wake_up_idle(struct task_struct *p)
{
return p->wake_up_idle;
}
static inline int sched_set_wake_up_idle(struct task_struct *p,
int wake_up_idle)
{
p->wake_up_idle = !!wake_up_idle;
return 0;
}
static inline void set_wake_up_idle(bool enabled)
{
current->wake_up_idle = enabled;
}
#else
static inline u32 sched_get_wake_up_idle(struct task_struct *p)
{
return 0;
}
static inline int sched_set_wake_up_idle(struct task_struct *p,
int wake_up_idle)
{
return 0;
}
static inline void set_wake_up_idle(bool enabled) {}
#endif
#endif

31
include/linux/sched/core_ctl.h Normal file
View file

@ -0,0 +1,31 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016, 2019, The Linux Foundation. All rights reserved.
*/
#ifndef __CORE_CTL_H
#define __CORE_CTL_H
#define MAX_CPUS_PER_CLUSTER 6
#define MAX_CLUSTERS 3
struct core_ctl_notif_data {
unsigned int nr_big;
unsigned int coloc_load_pct;
unsigned int ta_util_pct[MAX_CLUSTERS];
unsigned int cur_cap_pct[MAX_CLUSTERS];
};
#ifdef CONFIG_SCHED_WALT
extern int __weak core_ctl_set_boost(bool boost);
extern void __weak core_ctl_notifier_register(struct notifier_block *n);
extern void __weak core_ctl_notifier_unregister(struct notifier_block *n);
#else
static inline int core_ctl_set_boost(bool boost)
{
return 0;
}
static inline void core_ctl_notifier_register(struct notifier_block *n) {}
static inline void core_ctl_notifier_unregister(struct notifier_block *n) {}
#endif
#endif

5
include/linux/sched/cpufreq.h
View file

@ -10,6 +10,11 @@
#define SCHED_CPUFREQ_IOWAIT (1U << 0)
#define SCHED_CPUFREQ_MIGRATION (1U << 1)
#define SCHED_CPUFREQ_INTERCLUSTER_MIG (1U << 3)
#define SCHED_CPUFREQ_WALT (1U << 4)
#define SCHED_CPUFREQ_PL (1U << 5)
#define SCHED_CPUFREQ_EARLY_DET (1U << 6)
#define SCHED_CPUFREQ_CONTINUE (1U << 8)
#ifdef CONFIG_CPU_FREQ
struct update_util_data {

12
include/linux/sched/isolation.h
View file

@ -28,7 +28,15 @@ extern void __init housekeeping_init(void);
static inline int housekeeping_any_cpu(enum hk_flags flags)
{
return smp_processor_id();
cpumask_t available;
int cpu;
cpumask_andnot(&available, cpu_online_mask, cpu_isolated_mask);
cpu = cpumask_any(&available);
if (cpu >= nr_cpu_ids)
cpu = smp_processor_id();
return cpu;
}
static inline const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
@ -52,7 +60,7 @@ static inline bool housekeeping_cpu(int cpu, enum hk_flags flags)
if (static_branch_unlikely(&housekeeping_overridden))
return housekeeping_test_cpu(cpu, flags);
#endif
return true;
return !cpu_isolated(cpu);
}
#endif /* _LINUX_SCHED_ISOLATION_H */

22
include/linux/sched/stat.h
View file

@ -21,6 +21,28 @@ extern bool single_task_running(void);
extern unsigned long nr_iowait(void);
extern unsigned long nr_iowait_cpu(int cpu);
#ifdef CONFIG_SCHED_WALT
extern void __weak sched_update_nr_prod(int cpu, long delta, bool inc);
extern unsigned int __weak sched_get_cpu_util(int cpu);
extern void __weak sched_update_hyst_times(void);
extern u64 __weak sched_lpm_disallowed_time(int cpu);
#else
static inline void sched_update_nr_prod(int cpu, long delta, bool inc) {}
static inline unsigned int sched_get_cpu_util(int cpu)
{
return 0;
}
static inline u64 sched_get_cpu_last_busy_time(int cpu)
{
return 0;
}
static inline void sched_update_hyst_times(void) {}
static inline u64 sched_lpm_disallowed_time(int cpu)
{
return 0;
}
#endif
static inline int sched_info_on(void)
{
#ifdef CONFIG_SCHEDSTATS

40
include/linux/sched/sysctl.h
View file

@ -24,6 +24,42 @@ extern unsigned int sysctl_sched_latency;
extern unsigned int sysctl_sched_min_granularity;
extern unsigned int sysctl_sched_wakeup_granularity;
extern unsigned int sysctl_sched_child_runs_first;
#ifdef CONFIG_SCHED_WALT
extern unsigned int __weak sysctl_sched_user_hint;
extern const int __weak sched_user_hint_max;
extern unsigned int __weak sysctl_sched_cpu_high_irqload;
extern unsigned int __weak sysctl_sched_boost;
extern unsigned int __weak sysctl_sched_group_upmigrate_pct;
extern unsigned int __weak sysctl_sched_group_downmigrate_pct;
extern unsigned int __weak sysctl_sched_conservative_pl;
extern unsigned int __weak sysctl_sched_walt_rotate_big_tasks;
extern unsigned int __weak sysctl_sched_min_task_util_for_boost;
extern unsigned int __weak sysctl_sched_min_task_util_for_colocation;
extern unsigned int __weak sysctl_sched_asym_cap_sibling_freq_match_pct;
extern unsigned int __weak sysctl_sched_coloc_downmigrate_ns;
extern unsigned int __weak sysctl_sched_task_unfilter_nr_windows;
extern unsigned int __weak sysctl_sched_busy_hyst_enable_cpus;
extern unsigned int __weak sysctl_sched_busy_hyst;
extern unsigned int __weak sysctl_sched_coloc_busy_hyst_enable_cpus;
extern unsigned int __weak sysctl_sched_coloc_busy_hyst;
extern unsigned int __weak sysctl_sched_coloc_busy_hyst_max_ms;
extern unsigned int __weak sysctl_sched_window_stats_policy;
extern unsigned int __weak sysctl_sched_ravg_window_nr_ticks;
extern int __weak
walt_proc_group_thresholds_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos);
extern int __weak
walt_proc_user_hint_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos);
extern int __weak
sched_ravg_window_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos);
#endif
enum sched_tunable_scaling {
SCHED_TUNABLESCALING_NONE,
@ -47,6 +83,10 @@ int sched_proc_update_handler(struct ctl_table *table, int write,
loff_t *ppos);
#endif
#ifdef CONFIG_SCHED_WALT
extern int __weak sched_boost_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos);
#endif
/*
* control realtime throttling:
*

3
include/linux/sysctl.h
View file

@ -73,6 +73,9 @@ extern int proc_do_large_bitmap(struct ctl_table *, int,
extern int proc_do_static_key(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos);
extern int proc_douintvec_ravg_window(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos);
/*
* Register a set of sysctl names by calling register_sysctl_table

3
include/linux/topology.h
View file

@ -214,6 +214,9 @@ static inline int cpu_to_mem(int cpu)
#ifndef topology_die_cpumask
#define topology_die_cpumask(cpu) cpumask_of(cpu)
#endif
#ifndef topology_possible_sibling_cpumask
#define topology_possible_sibling_cpumask(cpu) cpumask_of(cpu)
#endif
#ifdef CONFIG_SCHED_SMT
static inline const struct cpumask *cpu_smt_mask(int cpu)

46
include/trace/events/sched.h
View file

@ -51,6 +51,52 @@ TRACE_EVENT(sched_kthread_stop_ret,
TP_printk("ret=%d", __entry->ret)
);
/*
* Tracepoint for task enqueue/dequeue:
*/
TRACE_EVENT(sched_enq_deq_task,
TP_PROTO(struct task_struct *p, bool enqueue,
unsigned int cpus_allowed),
TP_ARGS(p, enqueue, cpus_allowed),
TP_STRUCT__entry(
__array(char, comm, TASK_COMM_LEN)
__field(pid_t, pid)
__field(int, prio)
__field(int, cpu)
__field(bool, enqueue)
__field(unsigned int, nr_running)
__field(unsigned int, rt_nr_running)
__field(unsigned int, cpus_allowed)
__field(unsigned int, demand)
__field(unsigned int, pred_demand)
),
TP_fast_assign(
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->prio = p->prio;
__entry->cpu = task_cpu(p);
__entry->enqueue = enqueue;
__entry->nr_running = task_rq(p)->nr_running;
__entry->rt_nr_running = task_rq(p)->rt.rt_nr_running;
__entry->cpus_allowed = cpus_allowed;
__entry->demand = task_load(p);
__entry->pred_demand = task_pl(p);
),
TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u",
__entry->cpu,
__entry->enqueue ? "enqueue" : "dequeue",
__entry->comm, __entry->pid,
__entry->prio, __entry->nr_running,
__entry->rt_nr_running,
__entry->cpus_allowed, __entry->demand,
__entry->pred_demand)
);
/*
* Tracepoint for waking up a task:
*/

9
init/Kconfig
View file

@ -470,6 +470,15 @@ config HAVE_SCHED_AVG_IRQ
depends on IRQ_TIME_ACCOUNTING || PARAVIRT_TIME_ACCOUNTING
depends on SMP
config SCHED_WALT
bool "Support window based load tracking"
depends on SMP
help
This feature will allow the scheduler to maintain a tunable window
based set of metrics for tasks and runqueues. These metrics can be
used to guide task placement as well as task frequency requirements
for cpufreq governors.
config BSD_PROCESS_ACCT
bool "BSD Process Accounting"
depends on MULTIUSER

3
init/init_task.c
View file

@ -91,6 +91,9 @@ struct task_struct init_task
#endif
#ifdef CONFIG_CGROUP_SCHED
.sched_task_group = &root_task_group,
#endif
#ifdef CONFIG_SCHED_WALT
.wake_up_idle = false,
#endif
.ptraced = LIST_HEAD_INIT(init_task.ptraced),
.ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry),

11
kernel/cpu.c
View file

@ -990,6 +990,9 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
if (!cpu_present(cpu))
return -EINVAL;
if (!tasks_frozen && !cpu_isolated(cpu) && num_online_uniso_cpus() == 1)
return -EBUSY;
cpus_write_lock();
if (trace_cpuhp_latency_enabled())
start_time = sched_clock();
@ -2377,6 +2380,9 @@ EXPORT_SYMBOL(__cpu_present_mask);
struct cpumask __cpu_active_mask __read_mostly;
EXPORT_SYMBOL(__cpu_active_mask);
struct cpumask __cpu_isolated_mask __read_mostly;
EXPORT_SYMBOL(__cpu_isolated_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
@ -2395,6 +2401,11 @@ void init_cpu_online(const struct cpumask *src)
cpumask_copy(&__cpu_online_mask, src);
}
void init_cpu_isolated(const struct cpumask *src)
{
cpumask_copy(&__cpu_isolated_mask, src);
}
void set_cpu_online(unsigned int cpu, bool online)
{
/*

1
kernel/exit.c
View file

@ -761,6 +761,7 @@ void __noreturn do_exit(long code)
}
exit_signals(tsk); /* sets PF_EXITING */
sched_exit(tsk);
/*
* Ensure that all new tsk->pi_lock acquisitions must observe
* PF_EXITING. Serializes against futex.c:attach_to_pi_owner().

1
kernel/fork.c
View file

@ -2283,6 +2283,7 @@ bad_fork_cleanup_perf:
perf_event_free_task(p);
bad_fork_cleanup_policy:
lockdep_free_task(p);
free_task_load_ptrs(p);
#ifdef CONFIG_NUMA
mpol_put(p->mempolicy);
bad_fork_cleanup_threadgroup_lock:

37
kernel/irq/cpuhotplug.c
View file

@ -12,6 +12,7 @@
#include <linux/interrupt.h>
#include <linux/ratelimit.h>
#include <linux/irq.h>
#include <linux/cpumask.h>
#include "internals.h"
@ -57,6 +58,7 @@ static bool migrate_one_irq(struct irq_desc *desc)
const struct cpumask *affinity;
bool brokeaff = false;
int err;
struct cpumask available_cpus;
/*
* IRQ chip might be already torn down, but the irq descriptor is
@ -109,6 +111,10 @@ static bool migrate_one_irq(struct irq_desc *desc)
if (maskchip && chip->irq_mask)
chip->irq_mask(d);
cpumask_copy(&available_cpus, affinity);
cpumask_andnot(&available_cpus, &available_cpus, cpu_isolated_mask);
affinity = &available_cpus;
if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
/*
* If the interrupt is managed, then shut it down and leave
@ -119,16 +125,41 @@ static bool migrate_one_irq(struct irq_desc *desc)
irq_shutdown_and_deactivate(desc);
return false;
}
affinity = cpu_online_mask;
/*
* The order of preference for selecting a fallback CPU is
*
* (1) online and un-isolated CPU from default affinity
* (2) online and un-isolated CPU
* (3) online CPU
*/
cpumask_andnot(&available_cpus, cpu_online_mask,
cpu_isolated_mask);
if (cpumask_intersects(&available_cpus, irq_default_affinity))
cpumask_and(&available_cpus, &available_cpus,
irq_default_affinity);
else if (cpumask_empty(&available_cpus))
affinity = cpu_online_mask;
/*
* We are overriding the affinity with all online and
* un-isolated cpus. irq_set_affinity_locked() call
* below notify this mask to PM QOS affinity listener.
* That results in applying the CPU_DMA_LATENCY QOS
* to all the CPUs specified in the mask. But the low
* level irqchip driver sets the affinity of an irq
* to only one CPU. So pick only one CPU from the
* prepared mask while overriding the user affinity.
*/
affinity = cpumask_of(cpumask_any(affinity));
brokeaff = true;
}
/*
* Do not set the force argument of irq_do_set_affinity() as this
* Do not set the force argument of irq_set_affinity_locked() as this
* disables the masking of offline CPUs from the supplied affinity
* mask and therefore might keep/reassign the irq to the outgoing
* CPU.
*/
err = irq_do_set_affinity(d, affinity, false);
err = irq_set_affinity_locked(d, affinity, false);
if (err) {
pr_warn_ratelimited("IRQ%u: set affinity failed(%d).\n",
d->irq, err);

5
kernel/irq/proc.c
View file

@ -132,6 +132,11 @@ static ssize_t write_irq_affinity(int type, struct file *file,
if (err)
goto free_cpumask;
if (cpumask_subset(new_value, cpu_isolated_mask)) {
err = -EINVAL;
goto free_cpumask;
}
/*
* Do not allow disabling IRQs completely - it's a too easy
* way to make the system unusable accidentally :-) At least

409
kernel/sched/core.c
View file

@ -11,6 +11,7 @@
#include <linux/nospec.h>
#include <linux/kcov.h>
#include <linux/irq.h>
#include <asm/switch_to.h>
#include <asm/tlb.h>
@ -19,6 +20,7 @@
#include "../smpboot.h"
#include "pelt.h"
#include "walt.h"
#define CREATE_TRACE_POINTS
#include <trace/events/sched.h>
@ -1298,6 +1300,8 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
uclamp_rq_inc(rq, p);
p->sched_class->enqueue_task(rq, p, flags);
walt_update_last_enqueue(p);
trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0]);
}
static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
@ -1312,6 +1316,11 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
uclamp_rq_dec(rq, p);
p->sched_class->dequeue_task(rq, p, flags);
#ifdef CONFIG_SCHED_WALT
if (p == rq->ed_task)
early_detection_notify(rq, sched_ktime_clock());
#endif
trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]);
}
void activate_task(struct rq *rq, struct task_struct *p, int flags)
@ -1331,6 +1340,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
if (task_contributes_to_load(p))
rq->nr_uninterruptible++;
#ifdef CONFIG_SCHED_WALT
if (flags & DEQUEUE_SLEEP)
clear_ed_task(p, rq);
#endif
dequeue_task(rq, p, flags);
}
@ -1492,8 +1506,11 @@ static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
dequeue_task(rq, p, DEQUEUE_NOCLOCK);
double_lock_balance(rq, cpu_rq(new_cpu));
if (!(rq->clock_update_flags & RQCF_UPDATED))
update_rq_clock(rq);
set_task_cpu(p, new_cpu);
rq_unlock(rq, rf);
double_rq_unlock(cpu_rq(new_cpu), rq);
rq = cpu_rq(new_cpu);
@ -1633,6 +1650,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
struct rq_flags rf;
struct rq *rq;
int ret = 0;
cpumask_t allowed_mask;
rq = task_rq_lock(p, &rf);
update_rq_clock(rq);
@ -1656,10 +1674,17 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
if (cpumask_equal(p->cpus_ptr, new_mask))
goto out;
dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
cpumask_andnot(&allowed_mask, new_mask, cpu_isolated_mask);
cpumask_and(&allowed_mask, &allowed_mask, cpu_valid_mask);
dest_cpu = cpumask_any(&allowed_mask);
if (dest_cpu >= nr_cpu_ids) {
ret = -EINVAL;
goto out;
cpumask_and(&allowed_mask, cpu_valid_mask, new_mask);
dest_cpu = cpumask_any(&allowed_mask);
if (!cpumask_intersects(new_mask, cpu_valid_mask)) {
ret = -EINVAL;
goto out;
}
}
do_set_cpus_allowed(p, new_mask);
@ -1675,7 +1700,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
}
/* Can the task run on the task's current CPU? If so, we're done */
if (cpumask_test_cpu(task_cpu(p), new_mask))
if (cpumask_test_cpu(task_cpu(p), &allowed_mask))
goto out;
if (task_running(rq, p) || p->state == TASK_WAKING) {
@ -1750,12 +1775,13 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
p->se.nr_migrations++;
rseq_migrate(p);
perf_event_task_migrate(p);
fixup_busy_time(p, new_cpu);
}
__set_task_cpu(p, new_cpu);
}
#ifdef CONFIG_NUMA_BALANCING
static void __migrate_swap_task(struct task_struct *p, int cpu)
{
if (task_on_rq_queued(p)) {
@ -1870,7 +1896,6 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p,
out:
return ret;
}
#endif /* CONFIG_NUMA_BALANCING */
/*
* wait_task_inactive - wait for a thread to unschedule.
@ -2027,12 +2052,13 @@ EXPORT_SYMBOL_GPL(kick_process);
* select_task_rq() below may allow selection of !active CPUs in order
* to satisfy the above rules.
*/
static int select_fallback_rq(int cpu, struct task_struct *p)
static int select_fallback_rq(int cpu, struct task_struct *p, bool allow_iso)
{
int nid = cpu_to_node(cpu);
const struct cpumask *nodemask = NULL;
enum { cpuset, possible, fail } state = cpuset;
enum { cpuset, possible, fail, bug } state = cpuset;
int dest_cpu;
int isolated_candidate = -1;
/*
* If the node that the CPU is on has been offlined, cpu_to_node()
@ -2046,6 +2072,8 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
for_each_cpu(dest_cpu, nodemask) {
if (!cpu_active(dest_cpu))
continue;
if (cpu_isolated(dest_cpu))
continue;
if (cpumask_test_cpu(dest_cpu, p->cpus_ptr))
return dest_cpu;
}
@ -2056,7 +2084,16 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
for_each_cpu(dest_cpu, p->cpus_ptr) {
if (!is_cpu_allowed(p, dest_cpu))
continue;
if (cpu_isolated(dest_cpu)) {
if (allow_iso)
isolated_candidate = dest_cpu;
continue;
}
goto out;
}
if (isolated_candidate != -1) {
dest_cpu = isolated_candidate;
goto out;
}
@ -2075,6 +2112,11 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
break;
case fail:
allow_iso = true;
state = bug;
break;
case bug:
BUG();
break;
}
@ -2102,6 +2144,8 @@ out:
static inline
int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
{
bool allow_isolated = (p->flags & PF_KTHREAD);
lockdep_assert_held(&p->pi_lock);
if (p->nr_cpus_allowed > 1)
@ -2119,8 +2163,9 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
* [ this allows ->select_task() to simply return task_cpu(p) and
* not worry about this generic constraint ]
*/
if (unlikely(!is_cpu_allowed(p, cpu)))
cpu = select_fallback_rq(task_cpu(p), p);
if (unlikely(!is_cpu_allowed(p, cpu)) ||
(cpu_isolated(cpu) && !allow_isolated))
cpu = select_fallback_rq(task_cpu(p), p, allow_isolated);
return cpu;
}
@ -2311,6 +2356,7 @@ void sched_ttwu_pending(void)
void scheduler_ipi(void)
{
int cpu = smp_processor_id();
/*
* Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
* TIF_NEED_RESCHED remotely (for the first time) will also send
@ -2340,7 +2386,7 @@ void scheduler_ipi(void)
/*
* Check if someone kicked us for doing the nohz idle load balance.
*/
if (unlikely(got_nohz_idle_kick())) {
if (unlikely(got_nohz_idle_kick()) && !cpu_isolated(cpu)) {
this_rq()->idle_balance = 1;
raise_softirq_irqoff(SCHED_SOFTIRQ);
}
@ -2616,6 +2662,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
*/
smp_cond_load_acquire(&p->on_cpu, !VAL);
walt_try_to_wake_up(p);
p->sched_contributes_to_load = !!task_contributes_to_load(p);
p->state = TASK_WAKING;
@ -2644,6 +2692,17 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
unlock:
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
out:
#ifdef CONFIG_SCHED_WALT
if (success && sched_predl) {
raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags);
if (do_pl_notif(cpu_rq(cpu)))
cpufreq_update_util(cpu_rq(cpu),
SCHED_CPUFREQ_WALT |
SCHED_CPUFREQ_PL);
raw_spin_unlock_irqrestore(&cpu_rq(cpu)->lock, flags);
}
#endif
if (success)
ttwu_stat(p, cpu, wake_flags);
preempt_enable();
@ -2689,6 +2748,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
p->se.prev_sum_exec_runtime = 0;
p->se.nr_migrations = 0;
p->se.vruntime = 0;
#ifdef CONFIG_SCHED_WALT
p->last_sleep_ts = 0;
p->wake_up_idle = false;
#endif
INIT_LIST_HEAD(&p->se.group_node);
#ifdef CONFIG_FAIR_GROUP_SCHED
@ -2840,6 +2903,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
{
unsigned long flags;
init_new_task_load(p);
__sched_fork(clone_flags, p);
/*
* We mark the process as NEW here. This guarantees that
@ -2945,7 +3009,9 @@ void wake_up_new_task(struct task_struct *p)
struct rq_flags rf;
struct rq *rq;
add_new_task_to_grp(p);
raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
p->state = TASK_RUNNING;
#ifdef CONFIG_SMP
/*
@ -2963,6 +3029,7 @@ void wake_up_new_task(struct task_struct *p)
update_rq_clock(rq);
post_init_entity_util_avg(p);
mark_task_starting(p);
activate_task(rq, p, ENQUEUE_NOCLOCK);
trace_sched_wakeup_new(p);
check_preempt_curr(rq, p, WF_FORK);
@ -3497,12 +3564,15 @@ void sched_exec(void)
unsigned long flags;
int dest_cpu;
if (sched_energy_enabled())
return;
raw_spin_lock_irqsave(&p->pi_lock, flags);
dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
if (dest_cpu == smp_processor_id())
goto unlock;
if (likely(cpu_active(dest_cpu))) {
if (likely(cpu_active(dest_cpu) && likely(!cpu_isolated(dest_cpu)))) {
struct migration_arg arg = { p, dest_cpu };
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
@ -3592,16 +3662,30 @@ void scheduler_tick(void)
struct rq *rq = cpu_rq(cpu);
struct task_struct *curr = rq->curr;
struct rq_flags rf;
u64 wallclock;
bool early_notif;
u32 old_load;
struct related_thread_group *grp;
unsigned int flag = 0;
sched_clock_tick();
rq_lock(rq, &rf);
old_load = task_load(curr);
set_window_start(rq);
wallclock = sched_ktime_clock();
walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
update_rq_clock(rq);
curr->sched_class->task_tick(rq, curr, 0);
calc_global_load_tick(rq);
psi_task_tick(rq);
early_notif = early_detection_notify(rq, wallclock);
if (early_notif)
flag = SCHED_CPUFREQ_WALT | SCHED_CPUFREQ_EARLY_DET;
cpufreq_update_util(rq, flag);
rq_unlock(rq, &rf);
perf_event_task_tick();
@ -3610,6 +3694,15 @@ void scheduler_tick(void)
rq->idle_balance = idle_cpu(cpu);
trigger_load_balance(rq);
#endif
rcu_read_lock();
grp = task_related_thread_group(curr);
if (update_preferred_cluster(grp, curr, old_load, true))
set_preferred_cluster(grp);
rcu_read_unlock();
if (curr->sched_class == &fair_sched_class)
check_for_migration(rq, curr);
}
#ifdef CONFIG_NO_HZ_FULL
@ -4005,6 +4098,7 @@ static void __sched notrace __schedule(bool preempt)
struct rq_flags rf;
struct rq *rq;
int cpu;
u64 wallclock;
cpu = smp_processor_id();
rq = cpu_rq(cpu);
@ -4052,7 +4146,15 @@ static void __sched notrace __schedule(bool preempt)
clear_tsk_need_resched(prev);
clear_preempt_need_resched();
wallclock = sched_ktime_clock();
if (likely(prev != next)) {
#ifdef CONFIG_SCHED_WALT
if (!prev->on_rq)
prev->last_sleep_ts = wallclock;
#endif
walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0);
walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0);
rq->nr_switches++;
/*
* RCU users of rcu_dereference(rq->curr) may not see
@ -4080,6 +4182,7 @@ static void __sched notrace __schedule(bool preempt)
/* Also unlocks the rq: */
rq = context_switch(rq, prev, next, &rf);
} else {
walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0);
rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
rq_unlock_irq(rq, &rf);
}
@ -4669,7 +4772,7 @@ struct task_struct *idle_task(int cpu)
*
* The task of @pid, if found. %NULL otherwise.
*/
static struct task_struct *find_process_by_pid(pid_t pid)
struct task_struct *find_process_by_pid(pid_t pid)
{
return pid ? find_task_by_vpid(pid) : current;
}
@ -5390,6 +5493,8 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
cpumask_var_t cpus_allowed, new_mask;
struct task_struct *p;
int retval;
int dest_cpu;
cpumask_t allowed_mask;
rcu_read_lock();
@ -5451,20 +5556,26 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
}
#endif
again:
retval = __set_cpus_allowed_ptr(p, new_mask, true);
if (!retval) {
cpuset_cpus_allowed(p, cpus_allowed);
if (!cpumask_subset(new_mask, cpus_allowed)) {
/*
* We must have raced with a concurrent cpuset
* update. Just reset the cpus_allowed to the
* cpuset's cpus_allowed
*/
cpumask_copy(new_mask, cpus_allowed);
goto again;
cpumask_andnot(&allowed_mask, new_mask, cpu_isolated_mask);
dest_cpu = cpumask_any_and(cpu_active_mask, &allowed_mask);
if (dest_cpu < nr_cpu_ids) {
retval = __set_cpus_allowed_ptr(p, new_mask, true);
if (!retval) {
cpuset_cpus_allowed(p, cpus_allowed);
if (!cpumask_subset(new_mask, cpus_allowed)) {
/*
* We must have raced with a concurrent cpuset
* update. Just reset the cpus_allowed to the
* cpuset's cpus_allowed
*/
cpumask_copy(new_mask, cpus_allowed);
goto again;
}
}
} else {
retval = -EINVAL;
}
out_free_new_mask:
free_cpumask_var(new_mask);
out_free_cpus_allowed:
@ -5582,6 +5693,14 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
raw_spin_lock_irqsave(&p->pi_lock, flags);
cpumask_and(mask, &p->cpus_mask, cpu_active_mask);
/* The userspace tasks are forbidden to run on
* isolated CPUs. So exclude isolated CPUs from
* the getaffinity.
*/
if (!(p->flags & PF_KTHREAD))
cpumask_andnot(mask, mask, cpu_isolated_mask);
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
out_unlock:
@ -6253,7 +6372,7 @@ void idle_task_exit(void)
*
* Also see the comment "Global load-average calculations".
*/
static void calc_load_migrate(struct rq *rq)
void calc_load_migrate(struct rq *rq)
{
long delta = calc_load_fold_active(rq, 1);
if (delta)
@ -6278,19 +6397,25 @@ static struct task_struct *__pick_migrate_task(struct rq *rq)
}
/*
* Migrate all tasks from the rq, sleeping tasks will be migrated by
* try_to_wake_up()->select_task_rq().
* Migrate all tasks (not pinned if pinned argument say so) from the rq,
* sleeping tasks will be migrated by try_to_wake_up()->select_task_rq().
*
* Called with rq->lock held even though we'er in stop_machine() and
* there's no concurrency possible, we hold the required locks anyway
* because of lock validation efforts.
*/
static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf,
bool migrate_pinned_tasks)
{
struct rq *rq = dead_rq;
struct task_struct *next, *stop = rq->stop;
struct rq_flags orf = *rf;
int dest_cpu;
unsigned int num_pinned_kthreads = 1; /* this thread */
LIST_HEAD(tasks);
cpumask_t avail_cpus;
cpumask_andnot(&avail_cpus, cpu_online_mask, cpu_isolated_mask);
/*
* Fudge the rq selection such that the below task selection loop
@ -6313,13 +6438,20 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
for (;;) {
/*
* There's this thread running, bail when that's the only
* remaining thread:
* remaining thread.
*/
if (rq->nr_running == 1)
break;
next = __pick_migrate_task(rq);
if (!migrate_pinned_tasks && next->flags & PF_KTHREAD &&
!cpumask_intersects(&avail_cpus, &next->cpus_mask)) {
detach_one_task_core(next, rq, &tasks);
num_pinned_kthreads += 1;
continue;
}
/*
* Rules for changing task_struct::cpus_mask are holding
* both pi_lock and rq->lock, such that holding either
@ -6332,31 +6464,43 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
rq_unlock(rq, rf);
raw_spin_lock(&next->pi_lock);
rq_relock(rq, rf);
if (!(rq->clock_update_flags & RQCF_UPDATED))
update_rq_clock(rq);
/*
* Since we're inside stop-machine, _nothing_ should have
* changed the task, WARN if weird stuff happened, because in
* that case the above rq->lock drop is a fail too.
* However, during cpu isolation the load balancer might have
* interferred since we don't stop all CPUs. Ignore warning for
* this case.
*/
if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {
if (task_rq(next) != rq || !task_on_rq_queued(next)) {
WARN_ON(migrate_pinned_tasks);
raw_spin_unlock(&next->pi_lock);
continue;
}
/* Find suitable destination for @next, with force if needed. */
dest_cpu = select_fallback_rq(dead_rq->cpu, next);
dest_cpu = select_fallback_rq(dead_rq->cpu, next, false);
rq = __migrate_task(rq, rf, next, dest_cpu);
if (rq != dead_rq) {
rq_unlock(rq, rf);
rq = dead_rq;
*rf = orf;
rq_relock(rq, rf);
if (!(rq->clock_update_flags & RQCF_UPDATED))
update_rq_clock(rq);
}
raw_spin_unlock(&next->pi_lock);
}
rq->stop = stop;
if (num_pinned_kthreads > 1)
attach_tasks_core(&tasks, rq);
}
#endif /* CONFIG_HOTPLUG_CPU */
void set_rq_online(struct rq *rq)
@ -6512,6 +6656,11 @@ int sched_cpu_deactivate(unsigned int cpu)
static void sched_rq_cpu_starting(unsigned int cpu)
{
struct rq *rq = cpu_rq(cpu);
unsigned long flags;
raw_spin_lock_irqsave(&rq->lock, flags);
set_window_start(rq);
raw_spin_unlock_irqrestore(&rq->lock, flags);
rq->calc_load_update = calc_load_update;
update_max_interval();
@ -6521,6 +6670,7 @@ int sched_cpu_starting(unsigned int cpu)
{
sched_rq_cpu_starting(cpu);
sched_tick_start(cpu);
clear_walt_request(cpu);
return 0;
}
@ -6535,14 +6685,17 @@ int sched_cpu_dying(unsigned int cpu)
sched_tick_stop(cpu);
rq_lock_irqsave(rq, &rf);
if (rq->rd) {
BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
set_rq_offline(rq);
}
migrate_tasks(rq, &rf);
migrate_tasks(rq, &rf, true);
BUG_ON(rq->nr_running != 1);
rq_unlock_irqrestore(rq, &rf);
clear_walt_request(cpu);
calc_load_migrate(rq);
update_max_interval();
nohz_balance_exit_idle(rq);
@ -6564,6 +6717,8 @@ void __init sched_init_smp(void)
sched_init_domains(cpu_active_mask);
mutex_unlock(&sched_domains_mutex);
update_cluster_topology();
/* Move init over to a non-isolated CPU */
if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0)
BUG();
@ -6618,6 +6773,8 @@ void __init sched_init(void)
wait_bit_init();
init_clusters();
#ifdef CONFIG_FAIR_GROUP_SCHED
ptr += 2 * nr_cpu_ids * sizeof(void **);
#endif
@ -6729,6 +6886,7 @@ void __init sched_init(void)
rq->idle_stamp = 0;
rq->avg_idle = 2*sysctl_sched_migration_cost;
rq->max_idle_balance_cost = sysctl_sched_migration_cost;
walt_sched_init_rq(rq);
INIT_LIST_HEAD(&rq->cfs_tasks);
@ -6743,6 +6901,8 @@ void __init sched_init(void)
atomic_set(&rq->nr_iowait, 0);
}
BUG_ON(alloc_related_thread_groups());
set_load_weight(&init_task, false);
/*
@ -6758,6 +6918,7 @@ void __init sched_init(void)
* when this runqueue becomes "idle".
*/
init_idle(current, smp_processor_id());
init_new_task_load(current);
calc_load_update = jiffies + LOAD_FREQ;
@ -6972,6 +7133,97 @@ void ia64_set_curr_task(int cpu, struct task_struct *p)
/* task_group_lock serializes the addition/removal of task groups */
static DEFINE_SPINLOCK(task_group_lock);
#if defined(CONFIG_SCHED_WALT) && defined(CONFIG_UCLAMP_TASK_GROUP)
static inline void walt_init_sched_boost(struct task_group *tg)
{
tg->sched_boost_no_override = false;
tg->sched_boost_enabled = true;
tg->colocate = false;
tg->colocate_update_disabled = false;
}
void update_cgroup_boost_settings(void)
{
struct task_group *tg;
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list) {
if (tg->sched_boost_no_override)
continue;
tg->sched_boost_enabled = false;
}
rcu_read_unlock();
}
void restore_cgroup_boost_settings(void)
{
struct task_group *tg;
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list)
tg->sched_boost_enabled = true;
rcu_read_unlock();
}
static void walt_schedgp_attach(struct cgroup_taskset *tset)
{
struct task_struct *task;
struct cgroup_subsys_state *css;
bool colocate;
cgroup_taskset_first(tset, &css);
tg = css_tg(css);
colocate = tg->colocate;
cgroup_taskset_for_each(task, css, tset)
sync_cgroup_colocation(task, colocate);
}
static u64
sched_boost_override_read(struct cgroup_subsys_state *css,
struct cftype *cft)
{
struct task_group *tg = css_tg(css);
return (u64) tg->sched_boost_no_override;
}
static int sched_boost_override_write(struct cgroup_subsys_state *css,
struct cftype *cft, u64 override)
{
struct task_group *tg = css_tg(css);
tg->sched_boost_no_override = !!override;
return 0;
}
static u64 sched_colocate_read(struct cgroup_subsys_state *css,
struct cftype *cft)
{
struct task_group *tg = css_tg(css);
return (u64) tg->colocate;
}
static int sched_colocate_write(struct cgroup_subsys_state *css,
struct cftype *cft, u64 colocate)
{
struct task_group *tg = css_tg(css);
if (tg->colocate_update_disabled)
return -EPERM;
tg->colocate = !!colocate;
tg->colocate_update_disabled = true;
return 0;
}
#else
static inline void walt_init_sched_boost(struct task_group *tg) { }
static void walt_schedgp_attach(struct cgroup_taskset *tset) { }
#endif /* CONFIG_SCHED_WALT */
static inline void alloc_uclamp_sched_group(struct task_group *tg,
struct task_group *parent)
{
@ -7139,6 +7391,7 @@ cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
if (IS_ERR(tg))
return ERR_PTR(-ENOMEM);
walt_init_sched_boost(tg);
return &tg->css;
}
@ -7225,6 +7478,8 @@ static void cpu_cgroup_attach(struct cgroup_taskset *tset)
cgroup_taskset_for_each(task, css, tset)
sched_move_task(task);
walt_schedgp_attach(tset);
}
#ifdef CONFIG_UCLAMP_TASK_GROUP
@ -7784,7 +8039,21 @@ static struct cftype cpu_legacy_files[] = {
.read_u64 = cpu_uclamp_ls_read_u64,
.write_u64 = cpu_uclamp_ls_write_u64,
},
#endif
#ifdef CONFIG_SCHED_WALT
{
.name = "uclamp.sched_boost_no_override",
.flags = CFTYPE_NOT_ON_ROOT,
.read_u64 = sched_boost_override_read,
.write_u64 = sched_boost_override_write,
},
{
.name = "uclamp.colocate",
.flags = CFTYPE_NOT_ON_ROOT,
.read_u64 = sched_colocate_read,
.write_u64 = sched_colocate_write,
},
#endif /* CONFIG_SCHED_WALT */
#endif /* CONFIG_UCLAMP_TASK_GROUP */
{ } /* Terminate */
};
@ -7971,7 +8240,21 @@ static struct cftype cpu_files[] = {
.read_u64 = cpu_uclamp_ls_read_u64,
.write_u64 = cpu_uclamp_ls_write_u64,
},
#endif
#ifdef CONFIG_SCHED_WALT
{
.name = "uclamp.sched_boost_no_override",
.flags = CFTYPE_NOT_ON_ROOT,
.read_u64 = sched_boost_override_read,
.write_u64 = sched_boost_override_write,
},
{
.name = "uclamp.colocate",
.flags = CFTYPE_NOT_ON_ROOT,
.read_u64 = sched_colocate_read,
.write_u64 = sched_colocate_write,
},
#endif /* CONFIG_SCHED_WALT */
#endif /* CONFIG_UCLAMP_TASK_GROUP */
{ } /* terminate */
};
@ -8040,3 +8323,57 @@ const u32 sched_prio_to_wmult[40] = {
};
#undef CREATE_TRACE_POINTS
__read_mostly bool sched_predl = 1;
void enqueue_task_core(struct rq *rq, struct task_struct *p, int flags)
{
enqueue_task(rq, p, 0);
}
void dequeue_task_core(struct rq *rq, struct task_struct *p, int flags)
{
dequeue_task(rq, p, 0);
}
#ifdef CONFIG_SCHED_WALT
void sched_account_irqtime(int cpu, struct task_struct *curr,
u64 delta, u64 wallclock)
{
struct rq *rq = cpu_rq(cpu);
unsigned long flags, nr_windows;
u64 cur_jiffies_ts;
raw_spin_lock_irqsave(&rq->lock, flags);
/*
* cputime (wallclock) uses sched_clock so use the same here for
* consistency.
*/
delta += sched_clock() - wallclock;
cur_jiffies_ts = get_jiffies_64();
if (is_idle_task(curr))
walt_update_task_ravg(curr, rq, IRQ_UPDATE, sched_ktime_clock(),
delta);
nr_windows = cur_jiffies_ts - rq->irqload_ts;
if (nr_windows) {
if (nr_windows < 10) {
/* Decay CPU's irqload by 3/4 for each window. */
rq->avg_irqload *= (3 * nr_windows);
rq->avg_irqload = div64_u64(rq->avg_irqload,
4 * nr_windows);
} else {
rq->avg_irqload = 0;
}
rq->avg_irqload += rq->cur_irqload;
rq->cur_irqload = 0;
}
rq->cur_irqload += delta;
rq->irqload_ts = cur_jiffies_ts;
raw_spin_unlock_irqrestore(&rq->lock, flags);
}
#endif

17
kernel/sched/cputime.c
View file

@ -4,6 +4,7 @@
*/
#include <linux/cpufreq_times.h>
#include "sched.h"
#include "walt.h"
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
@ -53,11 +54,18 @@ void irqtime_account_irq(struct task_struct *curr)
struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
s64 delta;
int cpu;
#ifdef CONFIG_SCHED_WALT
u64 wallclock;
bool account = true;
#endif
if (!sched_clock_irqtime)
return;
cpu = smp_processor_id();
#ifdef CONFIG_SCHED_WALT
wallclock = sched_clock_cpu(cpu);
#endif
delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
irqtime->irq_start_time += delta;
@ -71,6 +79,15 @@ void irqtime_account_irq(struct task_struct *curr)
irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
#ifdef CONFIG_SCHED_WALT
else
account = false;
if (account)
sched_account_irqtime(cpu, curr, delta, wallclock);
else if (curr != this_cpu_ksoftirqd())
sched_account_irqstart(cpu, curr, wallclock);
#endif
}
EXPORT_SYMBOL_GPL(irqtime_account_irq);

10
kernel/sched/deadline.c
View file

@ -17,6 +17,7 @@
*/
#include "sched.h"
#include "pelt.h"
#include "walt.h"
struct dl_bandwidth def_dl_bandwidth;
@ -1380,6 +1381,7 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
WARN_ON(!dl_prio(prio));
dl_rq->dl_nr_running++;
add_nr_running(rq_of_dl_rq(dl_rq), 1);
walt_inc_cumulative_runnable_avg(rq_of_dl_rq(dl_rq), dl_task_of(dl_se));
inc_dl_deadline(dl_rq, deadline);
inc_dl_migration(dl_se, dl_rq);
@ -1394,6 +1396,7 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
WARN_ON(!dl_rq->dl_nr_running);
dl_rq->dl_nr_running--;
sub_nr_running(rq_of_dl_rq(dl_rq), 1);
walt_dec_cumulative_runnable_avg(rq_of_dl_rq(dl_rq), dl_task_of(dl_se));
dec_dl_deadline(dl_rq, dl_se->deadline);
dec_dl_migration(dl_se, dl_rq);
@ -2101,7 +2104,9 @@ retry:
}
deactivate_task(rq, next_task, 0);
next_task->on_rq = TASK_ON_RQ_MIGRATING;
set_task_cpu(next_task, later_rq->cpu);
next_task->on_rq = TASK_ON_RQ_QUEUED;
/*
* Update the later_rq clock here, because the clock is used
@ -2195,7 +2200,9 @@ static void pull_dl_task(struct rq *this_rq)
resched = true;
deactivate_task(src_rq, p, 0);
p->on_rq = TASK_ON_RQ_MIGRATING;
set_task_cpu(p, this_cpu);
p->on_rq = TASK_ON_RQ_QUEUED;
activate_task(this_rq, p, 0);
dmin = p->dl.deadline;
@ -2458,6 +2465,9 @@ const struct sched_class dl_sched_class = {
.switched_to = switched_to_dl,
.update_curr = update_curr_dl,
#ifdef CONFIG_SCHED_WALT
.fixup_walt_sched_stats = fixup_walt_sched_stats_common,
#endif
};
int sched_dl_global_validate(void)

21
kernel/sched/debug.c
View file

@ -646,6 +646,19 @@ do { \
SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr)));
PN(clock);
PN(clock_task);
#ifdef CONFIG_SMP
P(cpu_capacity);
#endif
#ifdef CONFIG_SCHED_WALT
P(cluster->max_possible_capacity);
P(cluster->efficiency);
P(cluster->cur_freq);
P(cluster->max_freq);
P(cluster->exec_scale_factor);
P(walt_stats.nr_big_tasks);
SEQ_printf(m, " .%-30s: %llu\n", "walt_stats.cumulative_runnable_avg",
rq->walt_stats.cumulative_runnable_avg_scaled);
#endif
#undef P
#undef PN
@ -724,6 +737,11 @@ static void sched_debug_header(struct seq_file *m)
PN(sysctl_sched_wakeup_granularity);
P(sysctl_sched_child_runs_first);
P(sysctl_sched_features);
#ifdef CONFIG_SCHED_WALT
P(sched_init_task_load_windows);
P(sched_ravg_window);
P(sched_load_granule);
#endif
#undef PN
#undef P
@ -915,6 +933,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
P_SCHEDSTAT(se.statistics.nr_wakeups_passive);
P_SCHEDSTAT(se.statistics.nr_wakeups_idle);
#ifdef CONFIG_SCHED_WALT
P(ravg.demand);
#endif
avg_atom = p->se.sum_exec_runtime;
if (nr_switches)
avg_atom = div64_ul(avg_atom, nr_switches);

235
kernel/sched/fair.c
View file

@ -24,6 +24,12 @@
#include <trace/events/sched.h>
#include "walt.h"
#ifdef CONFIG_SMP
static inline bool task_fits_max(struct task_struct *p, int cpu);
#endif /* CONFIG_SMP */
/*
* Targeted preemption latency for CPU-bound tasks:
*
@ -85,6 +91,7 @@ unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
DEFINE_PER_CPU_READ_MOSTLY(int, sched_load_boost);
#ifdef CONFIG_SMP
/*
@ -118,6 +125,8 @@ int __weak arch_asym_cpu_priority(int cpu)
unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
#endif
unsigned int sched_small_task_threshold = 102;
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
{
lw->weight += inc;
@ -3689,11 +3698,6 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
return cfs_rq->avg.load_avg;
}
static inline unsigned long task_util(struct task_struct *p)
{
return READ_ONCE(p->se.avg.util_avg);
}
static inline unsigned long _task_util_est(struct task_struct *p)
{
struct util_est ue = READ_ONCE(p->se.avg.util_est);
@ -3703,6 +3707,9 @@ static inline unsigned long _task_util_est(struct task_struct *p)
static inline unsigned long task_util_est(struct task_struct *p)
{
#ifdef CONFIG_SCHED_WALT
return p->ravg.demand_scaled;
#endif
return max(task_util(p), _task_util_est(p));
}
@ -4514,13 +4521,16 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
qcfs_rq->h_nr_running -= task_delta;
qcfs_rq->idle_h_nr_running -= idle_task_delta;
walt_dec_throttled_cfs_rq_stats(&qcfs_rq->walt_stats, cfs_rq);
if (qcfs_rq->load.weight)
dequeue = 0;
}
if (!se)
if (!se) {
sub_nr_running(rq, task_delta);
walt_dec_throttled_cfs_rq_stats(&rq->walt_stats, cfs_rq);
}
cfs_rq->throttled = 1;
cfs_rq->throttled_clock = rq_clock(rq);
@ -4554,6 +4564,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
struct sched_entity *se;
int enqueue = 1;
long task_delta, idle_task_delta;
struct cfs_rq *tcfs_rq __maybe_unused = cfs_rq;
se = cfs_rq->tg->se[cpu_of(rq)];
@ -4583,6 +4594,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
cfs_rq->h_nr_running += task_delta;
cfs_rq->idle_h_nr_running += idle_task_delta;
walt_inc_throttled_cfs_rq_stats(&cfs_rq->walt_stats, tcfs_rq);
if (cfs_rq_throttled(cfs_rq))
break;
@ -4590,8 +4602,10 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
assert_list_leaf_cfs_rq(rq);
if (!se)
if (!se) {
add_nr_running(rq, task_delta);
walt_inc_throttled_cfs_rq_stats(&rq->walt_stats, tcfs_rq);
}
/* Determine whether we need to wake up potentially idle CPU: */
if (rq->curr == rq->idle && rq->cfs.nr_running)
@ -4982,6 +4996,7 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
{
cfs_rq->runtime_enabled = 0;
INIT_LIST_HEAD(&cfs_rq->throttled_list);
walt_init_cfs_rq_stats(cfs_rq);
}
void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
@ -5161,8 +5176,6 @@ static inline void hrtick_update(struct rq *rq)
#endif
#ifdef CONFIG_SMP
static inline unsigned long cpu_util(int cpu);
static inline bool cpu_overutilized(int cpu)
{
return !fits_capacity(cpu_util(cpu), capacity_of(cpu));
@ -5223,6 +5236,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
break;
cfs_rq->h_nr_running++;
cfs_rq->idle_h_nr_running += idle_h_nr_running;
walt_inc_cfs_rq_stats(cfs_rq, p);
flags = ENQUEUE_WAKEUP;
}
@ -5231,6 +5245,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
cfs_rq = cfs_rq_of(se);
cfs_rq->h_nr_running++;
cfs_rq->idle_h_nr_running += idle_h_nr_running;
walt_inc_cfs_rq_stats(cfs_rq, p);
if (cfs_rq_throttled(cfs_rq))
break;
@ -5241,6 +5256,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
if (!se) {
add_nr_running(rq, 1);
inc_rq_walt_stats(rq, p);
/*
* Since new tasks are assigned an initial util_avg equal to
* half of the spare capacity of their CPU, tiny tasks have the
@ -5308,6 +5324,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
break;
cfs_rq->h_nr_running--;
cfs_rq->idle_h_nr_running -= idle_h_nr_running;
walt_dec_cfs_rq_stats(cfs_rq, p);
/* Don't dequeue parent if it has other entities besides us */
if (cfs_rq->load.weight) {
@ -5328,6 +5345,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
cfs_rq = cfs_rq_of(se);
cfs_rq->h_nr_running--;
cfs_rq->idle_h_nr_running -= idle_h_nr_running;
walt_dec_cfs_rq_stats(cfs_rq, p);
if (cfs_rq_throttled(cfs_rq))
break;
@ -5336,8 +5354,10 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
update_cfs_group(se);
}
if (!se)
if (!se) {
sub_nr_running(rq, 1);
dec_rq_walt_stats(rq, p);
}
util_est_dequeue(&rq->cfs, p, task_sleep);
hrtick_update(rq);
@ -5375,11 +5395,6 @@ static unsigned long cpu_runnable_load(struct rq *rq)
return cfs_rq_runnable_load_avg(&rq->cfs);
}
static unsigned long capacity_of(int cpu)
{
return cpu_rq(cpu)->cpu_capacity;
}
static unsigned long cpu_avg_load_per_task(int cpu)
{
struct rq *rq = cpu_rq(cpu);
@ -5968,6 +5983,8 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t
return si_cpu;
if (!cpumask_test_cpu(cpu, p->cpus_ptr))
continue;
if (cpu_isolated(cpu))
continue;
if (available_idle_cpu(cpu))
break;
if (si_cpu == -1 && sched_idle_cpu(cpu))
@ -5990,14 +6007,16 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
struct sched_domain *sd;
int i, recent_used_cpu;
if (available_idle_cpu(target) || sched_idle_cpu(target))
if ((available_idle_cpu(target) || sched_idle_cpu(target)) &&
!cpu_isolated(target))
return target;
/*
* If the previous CPU is cache affine and idle, don't be stupid:
*/
if (prev != target && cpus_share_cache(prev, target) &&
(available_idle_cpu(prev) || sched_idle_cpu(prev)))
((available_idle_cpu(prev) || sched_idle_cpu(prev)) &&
!cpu_isolated(prev)))
return prev;
/* Check a recently used CPU as a potential idle candidate: */
@ -6047,58 +6066,6 @@ static unsigned int uclamp_task_util(struct task_struct *p)
#endif
}
/**
* Amount of capacity of a CPU that is (estimated to be) used by CFS tasks
* @cpu: the CPU to get the utilization of
*
* The unit of the return value must be the one of capacity so we can compare
* the utilization with the capacity of the CPU that is available for CFS task
* (ie cpu_capacity).
*
* cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the
* recent utilization of currently non-runnable tasks on a CPU. It represents
* the amount of utilization of a CPU in the range [0..capacity_orig] where
* capacity_orig is the cpu_capacity available at the highest frequency
* (arch_scale_freq_capacity()).
* The utilization of a CPU converges towards a sum equal to or less than the
* current capacity (capacity_curr <= capacity_orig) of the CPU because it is
* the running time on this CPU scaled by capacity_curr.
*
* The estimated utilization of a CPU is defined to be the maximum between its
* cfs_rq.avg.util_avg and the sum of the estimated utilization of the tasks
* currently RUNNABLE on that CPU.
* This allows to properly represent the expected utilization of a CPU which
* has just got a big task running since a long sleep period. At the same time
* however it preserves the benefits of the "blocked utilization" in
* describing the potential for other tasks waking up on the same CPU.
*
* Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even
* higher than capacity_orig because of unfortunate rounding in
* cfs.avg.util_avg or just after migrating tasks and new task wakeups until
* the average stabilizes with the new running time. We need to check that the
* utilization stays within the range of [0..capacity_orig] and cap it if
* necessary. Without utilization capping, a group could be seen as overloaded
* (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of
* available capacity. We allow utilization to overshoot capacity_curr (but not
* capacity_orig) as it useful for predicting the capacity required after task
* migrations (scheduler-driven DVFS).
*
* Return: the (estimated) utilization for the specified CPU
*/
static inline unsigned long cpu_util(int cpu)
{
struct cfs_rq *cfs_rq;
unsigned int util;
cfs_rq = &cpu_rq(cpu)->cfs;
util = READ_ONCE(cfs_rq->avg.util_avg);
if (sched_feat(UTIL_EST))
util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued));
return min_t(unsigned long, util, capacity_orig_of(cpu));
}
/*
* cpu_util_without: compute cpu utilization without any contributions from *p
* @cpu: the CPU which utilization is requested
@ -6114,13 +6081,29 @@ static inline unsigned long cpu_util(int cpu)
*/
static unsigned long cpu_util_without(int cpu, struct task_struct *p)
{
#ifndef CONFIG_SCHED_WALT
struct cfs_rq *cfs_rq;
#endif
unsigned int util;
#ifdef CONFIG_SCHED_WALT
/*
* WALT does not decay idle tasks in the same manner
* as PELT, so it makes little sense to subtract task
* utilization from cpu utilization. Instead just use
* cpu_util for this case.
*/
if (likely(p->state == TASK_WAKING))
return cpu_util(cpu);
#endif
/* Task has no contribution or is new */
if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
return cpu_util(cpu);
#ifdef CONFIG_SCHED_WALT
util = max_t(long, cpu_util(cpu) - task_util(p), 0);
#else
cfs_rq = &cpu_rq(cpu)->cfs;
util = READ_ONCE(cfs_rq->avg.util_avg);
@ -6179,6 +6162,7 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p)
util = max(util, estimated);
}
#endif
/*
* Utilization (estimated) can exceed the CPU capacity, thus let's
@ -6188,6 +6172,18 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p)
return min_t(unsigned long, util, capacity_orig_of(cpu));
}
/*
* Returns the current capacity of cpu after applying both
* cpu and freq scaling.
*/
unsigned long capacity_curr_of(int cpu)
{
unsigned long max_cap = cpu_rq(cpu)->cpu_capacity_orig;
unsigned long scale_freq = arch_scale_freq_capacity(cpu);
return cap_scale(max_cap, scale_freq);
}
/*
* Disable WAKE_AFFINE in the case where task @p doesn't fit in the
* capacity of either the waking CPU @cpu or the previous CPU @prev_cpu.
@ -6344,7 +6340,7 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd)
* other use-cases too. So, until someone finds a better way to solve this,
* let's keep things simple by re-using the existing slow path.
*/
static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync)
int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync)
{
unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
@ -7363,7 +7359,13 @@ static void detach_task(struct task_struct *p, struct lb_env *env)
lockdep_assert_held(&env->src_rq->lock);
deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);
lockdep_off();
double_lock_balance(env->src_rq, env->dst_rq);
if (!(env->src_rq->clock_update_flags & RQCF_UPDATED))
update_rq_clock(env->src_rq);
set_task_cpu(p, env->dst_cpu);
double_unlock_balance(env->src_rq, env->dst_rq);
lockdep_on();
}
/*
@ -7894,6 +7896,8 @@ void update_group_capacity(struct sched_domain *sd, int cpu)
struct sched_group_capacity *sgc;
struct rq *rq = cpu_rq(cpu);
if (cpumask_test_cpu(cpu, cpu_isolated_mask))
continue;
/*
* build_sched_domains() -> init_sched_groups_capacity()
* gets here before we've attached the domains to the
@ -7924,10 +7928,15 @@ void update_group_capacity(struct sched_domain *sd, int cpu)
group = child->groups;
do {
struct sched_group_capacity *sgc = group->sgc;
cpumask_t *cpus = sched_group_span(group);
capacity += sgc->capacity;
min_capacity = min(sgc->min_capacity, min_capacity);
max_capacity = max(sgc->max_capacity, max_capacity);
if (!cpu_isolated(cpumask_first(cpus))) {
capacity += sgc->capacity;
min_capacity = min(sgc->min_capacity,
min_capacity);
max_capacity = max(sgc->max_capacity,
max_capacity);
}
group = group->next;
} while (group != child->groups);
}
@ -8131,6 +8140,9 @@ static inline void update_sg_lb_stats(struct lb_env *env,
for_each_cpu_and(i, sched_group_span(group), env->cpus) {
struct rq *rq = cpu_rq(i);
if (cpu_isolated(i))
continue;
if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false))
env->flags |= LBF_NOHZ_AGAIN;
@ -8162,17 +8174,27 @@ static inline void update_sg_lb_stats(struct lb_env *env,
}
}
/* Adjust by relative CPU capacity of the group */
sgs->group_capacity = group->sgc->capacity;
sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity;
/* Isolated CPU has no weight */
if (!group->group_weight) {
sgs->group_capacity = 0;
sgs->avg_load = 0;
sgs->group_no_capacity = 1;
sgs->group_type = group_other;
sgs->group_weight = group->group_weight;
} else {
/* Adjust by relative CPU capacity of the group */
sgs->group_capacity = group->sgc->capacity;
sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) /
sgs->group_capacity;
sgs->group_weight = group->group_weight;
sgs->group_no_capacity = group_is_overloaded(env, sgs);
sgs->group_type = group_classify(group, sgs);
}
if (sgs->sum_nr_running)
sgs->load_per_task = sgs->group_load / sgs->sum_nr_running;
sgs->group_weight = group->group_weight;
sgs->group_no_capacity = group_is_overloaded(env, sgs);
sgs->group_type = group_classify(group, sgs);
}
/**
@ -8891,8 +8913,6 @@ static int need_active_balance(struct lb_env *env)
return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2);
}
static int active_load_balance_cpu_stop(void *data);
static int should_we_balance(struct lb_env *env)
{
struct sched_group *sg = env->sd->groups;
@ -8914,7 +8934,7 @@ static int should_we_balance(struct lb_env *env)
/* Try to find first idle CPU */
for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) {
if (!idle_cpu(cpu))
if (!idle_cpu(cpu) || cpu_isolated(cpu))
continue;
balance_cpu = cpu;
@ -8922,7 +8942,7 @@ static int should_we_balance(struct lb_env *env)
}
if (balance_cpu == -1)
balance_cpu = group_balance_cpu(sg);
balance_cpu = group_balance_cpu_not_isolated(sg);
/*
* First idle CPU or the first CPU(busiest) in this sched group
@ -9131,7 +9151,8 @@ more_balance:
* ->active_balance_work. Once set, it's cleared
* only after active load balance is finished.
*/
if (!busiest->active_balance) {
if (!busiest->active_balance &&
!cpu_isolated(cpu_of(busiest))) {
busiest->active_balance = 1;
busiest->push_cpu = this_cpu;
active_balance = 1;
@ -9244,7 +9265,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
* least 1 task to be running on each physical CPU where possible, and
* avoids physical / logical imbalances.
*/
static int active_load_balance_cpu_stop(void *data)
int active_load_balance_cpu_stop(void *data)
{
struct rq *busiest_rq = data;
int busiest_cpu = cpu_of(busiest_rq);
@ -9337,7 +9358,13 @@ static DEFINE_SPINLOCK(balancing);
*/
void update_max_interval(void)
{
max_load_balance_interval = HZ*num_online_cpus()/10;
cpumask_t avail_mask;
unsigned int available_cpus;
cpumask_andnot(&avail_mask, cpu_online_mask, cpu_isolated_mask);
available_cpus = cpumask_weight(&avail_mask);
max_load_balance_interval = HZ*available_cpus/10;
}
/*
@ -9514,6 +9541,7 @@ static void nohz_balancer_kick(struct rq *rq)
struct sched_domain *sd;
int nr_busy, i, cpu = rq->cpu;
unsigned int flags = 0;
cpumask_t cpumask;
if (unlikely(rq->idle_balance))
return;
@ -9528,7 +9556,8 @@ static void nohz_balancer_kick(struct rq *rq)
* None are in tickless mode and hence no need for NOHZ idle load
* balancing.
*/
if (likely(!atomic_read(&nohz.nr_cpus)))
cpumask_andnot(&cpumask, nohz.idle_cpus_mask, cpu_isolated_mask);
if (cpumask_empty(&cpumask))
return;
if (READ_ONCE(nohz.has_blocked) &&
@ -9565,7 +9594,7 @@ static void nohz_balancer_kick(struct rq *rq)
* currently idle; in which case, kick the ILB to move tasks
* around.
*/
for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
for_each_cpu_and(i, sched_domain_span(sd), &cpumask) {
if (sched_asym_prefer(i, cpu)) {
flags = NOHZ_KICK_MASK;
goto unlock;
@ -9743,6 +9772,7 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
int balance_cpu;
int ret = false;
struct rq *rq;
cpumask_t cpus;
SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
@ -9762,7 +9792,9 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
*/
smp_mb();
for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
cpumask_andnot(&cpus, nohz.idle_cpus_mask, cpu_isolated_mask);
for_each_cpu(balance_cpu, &cpus) {
if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
continue;
@ -9914,6 +9946,9 @@ int newidle_balance(struct rq *this_rq, struct rq_flags *rf)
int pulled_task = 0;
u64 curr_cost = 0;
if (cpu_isolated(this_cpu))
return 0;
update_misfit_status(NULL, this_rq);
/*
* We must set idle_stamp _before_ calling idle_balance(), such that we
@ -10030,6 +10065,14 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
enum cpu_idle_type idle = this_rq->idle_balance ?
CPU_IDLE : CPU_NOT_IDLE;
/*
* Since core isolation doesn't update nohz.idle_cpus_mask, there
* is a possibility this nohz kicked cpu could be isolated. Hence
* return if the cpu is isolated.
*/
if (cpu_isolated(this_rq->cpu))
return;
/*
* If this CPU has a pending nohz_balance_kick, then do the
* balancing on behalf of the other idle CPUs whose ticks are
@ -10051,8 +10094,10 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
*/
void trigger_load_balance(struct rq *rq)
{
/* Don't need to rebalance while attached to NULL domain */
if (unlikely(on_null_domain(rq)))
/* Don't need to rebalance while attached to NULL domain or
* cpu is isolated.
*/
if (unlikely(on_null_domain(rq)) || cpu_isolated(cpu_of(rq)))
return;
if (time_after_eq(jiffies, rq->next_balance))
@ -10615,6 +10660,10 @@ const struct sched_class fair_sched_class = {
#ifdef CONFIG_UCLAMP_TASK
.uclamp_enabled = 1,
#endif
#ifdef CONFIG_SCHED_WALT
.fixup_walt_sched_stats = walt_fixup_sched_stats_fair,
#endif
};
#ifdef CONFIG_SCHED_DEBUG

6
kernel/sched/idle.c
View file

@ -61,7 +61,8 @@ static noinline int __cpuidle cpu_idle_poll(void)
stop_critical_timings();
while (!tif_need_resched() &&
(cpu_idle_force_poll || tick_check_broadcast_expired()))
(cpu_idle_force_poll || tick_check_broadcast_expired() ||
is_reserved(smp_processor_id())))
cpu_relax();
start_critical_timings();
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
@ -257,7 +258,8 @@ static void do_idle(void)
* broadcast device expired for us, we don't want to go deep
* idle as we know that the IPI is going to arrive right away.
*/
if (cpu_idle_force_poll || tick_check_broadcast_expired()) {
if (cpu_idle_force_poll || tick_check_broadcast_expired() ||
is_reserved(smp_processor_id())) {
tick_nohz_idle_restart_tick();
cpu_idle_poll();
} else {

16
kernel/sched/rt.c
View file

@ -6,6 +6,7 @@
#include "sched.h"
#include "pelt.h"
#include "walt.h"
int sched_rr_timeslice = RR_TIMESLICE;
int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;
@ -262,8 +263,12 @@ static void pull_rt_task(struct rq *this_rq);
static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
{
/* Try to pull RT tasks here if we lower this rq's prio */
return rq->rt.highest_prio.curr > prev->prio;
/*
* Try to pull RT tasks here if we lower this rq's prio and cpu is not
* isolated
*/
return rq->rt.highest_prio.curr > prev->prio &&
!cpu_isolated(cpu_of(rq));
}
static inline int rt_overloaded(struct rq *rq)
@ -2191,7 +2196,8 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p)
* we may need to handle the pulling of RT tasks
* now.
*/
if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)
if (!task_on_rq_queued(p) || rq->rt.rt_nr_running ||
cpu_isolated(cpu_of(rq)))
return;
rt_queue_pull_task(rq);
@ -2388,6 +2394,10 @@ const struct sched_class rt_sched_class = {
#ifdef CONFIG_UCLAMP_TASK
.uclamp_enabled = 1,
#endif
#ifdef CONFIG_SCHED_WALT
.fixup_walt_sched_stats = fixup_walt_sched_stats_common,
#endif
};
#ifdef CONFIG_RT_GROUP_SCHED

785
kernel/sched/sched.h
View file

@ -84,6 +84,73 @@
struct rq;
struct cpuidle_state;
extern __read_mostly bool sched_predl;
struct sched_walt_cpu_load {
unsigned long prev_window_util;
unsigned long nl;
unsigned long pl;
bool rtgb_active;
u64 ws;
};
#ifdef CONFIG_SCHED_WALT
#define DECLARE_BITMAP_ARRAY(name, nr, bits) \
unsigned long name[nr][BITS_TO_LONGS(bits)]
extern unsigned int __weak sched_ravg_window;
struct walt_sched_stats {
int nr_big_tasks;
u64 cumulative_runnable_avg_scaled;
u64 pred_demands_sum_scaled;
};
struct cpu_cycle {
u64 cycles;
u64 time;
};
struct group_cpu_time {
u64 curr_runnable_sum;
u64 prev_runnable_sum;
u64 nt_curr_runnable_sum;
u64 nt_prev_runnable_sum;
};
struct load_subtractions {
u64 window_start;
u64 subs;
u64 new_subs;
};
#define NUM_TRACKED_WINDOWS 2
#define NUM_LOAD_INDICES 1000
struct sched_cluster {
raw_spinlock_t load_lock;
struct list_head list;
struct cpumask cpus;
int id;
int max_power_cost;
int min_power_cost;
int max_possible_capacity;
int efficiency; /* Differentiate cpus with different IPC capability */
unsigned int exec_scale_factor;
/*
* max_freq = user maximum
* max_mitigated_freq = thermal defined maximum
* max_possible_freq = maximum supported by hardware
*/
unsigned int cur_freq, max_freq, max_mitigated_freq, min_freq;
unsigned int max_possible_freq;
bool freq_init_done;
u64 aggr_grp_load;
};
extern __weak cpumask_t asym_cap_sibling_cpus;
#endif /* CONFIG_SCHED_WALT */
/* task_struct::on_rq states: */
#define TASK_ON_RQ_QUEUED 1
#define TASK_ON_RQ_MIGRATING 2
@ -96,6 +163,10 @@ extern atomic_long_t calc_load_tasks;
extern void calc_global_load_tick(struct rq *this_rq);
extern long calc_load_fold_active(struct rq *this_rq, long adjust);
#ifdef CONFIG_SMP
extern void init_sched_groups_capacity(int cpu, struct sched_domain *sd);
#endif
/*
* Helpers for converting nanosecond timing to jiffy resolution
*/
@ -401,7 +472,24 @@ struct task_group {
struct uclamp_se uclamp[UCLAMP_CNT];
/* Latency-sensitive flag used for a task group */
unsigned int latency_sensitive;
#endif
#ifdef CONFIG_SCHED_WALT
/* Toggle ability to override sched boost enabled */
bool sched_boost_no_override;
/*
* Controls whether a cgroup is eligible for sched boost or not. This
* can temporariliy be disabled by the kernel based on the no_override
* flag above.
*/
bool sched_boost_enabled;
/*
* Controls whether tasks of this cgroup should be colocated with each
* other and tasks of other cgroups that have the same flag turned on.
*/
bool colocate;
/* Controls whether further updates are allowed to the colocate flag */
bool colocate_update_disabled;
#endif /* CONFIG_SCHED_WALT */
#endif /* CONFIG_UCLAMP_TASK_GROUP */
};
@ -565,6 +653,10 @@ struct cfs_rq {
struct list_head leaf_cfs_rq_list;
struct task_group *tg; /* group that "owns" this runqueue */
#ifdef CONFIG_SCHED_WALT
struct walt_sched_stats walt_stats;
#endif
#ifdef CONFIG_CFS_BANDWIDTH
int runtime_enabled;
s64 runtime_remaining;
@ -961,6 +1053,41 @@ struct rq {
u64 max_idle_balance_cost;
#endif
#ifdef CONFIG_SCHED_WALT
struct task_struct *push_task;
struct sched_cluster *cluster;
struct cpumask freq_domain_cpumask;
struct walt_sched_stats walt_stats;
u64 window_start;
u32 prev_window_size;
unsigned long walt_flags;
u64 cur_irqload;
u64 avg_irqload;
u64 irqload_ts;
struct task_struct *ed_task;
struct cpu_cycle cc;
u64 old_busy_time, old_busy_time_group;
u64 old_estimated_time;
u64 curr_runnable_sum;
u64 prev_runnable_sum;
u64 nt_curr_runnable_sum;
u64 nt_prev_runnable_sum;
u64 cum_window_demand_scaled;
struct group_cpu_time grp_time;
struct load_subtractions load_subs[NUM_TRACKED_WINDOWS];
DECLARE_BITMAP_ARRAY(top_tasks_bitmap,
NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES);
u8 *top_tasks[NUM_TRACKED_WINDOWS];
u8 curr_table;
int prev_top;
int curr_top;
bool notif_pending;
u64 last_cc_update;
u64 cycles;
#endif /* CONFIG_SCHED_WALT */
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
u64 prev_irq_time;
#endif
@ -1306,8 +1433,6 @@ enum numa_faults_stats {
};
extern void sched_setnuma(struct task_struct *p, int node);
extern int migrate_task_to(struct task_struct *p, int cpu);
extern int migrate_swap(struct task_struct *p, struct task_struct *t,
int cpu, int scpu);
extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p);
#else
static inline void
@ -1316,6 +1441,9 @@ init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
}
#endif /* CONFIG_NUMA_BALANCING */
extern int migrate_swap(struct task_struct *p, struct task_struct *t,
int cpu, int scpu);
#ifdef CONFIG_SMP
static inline void
@ -1782,8 +1910,15 @@ struct sched_class {
#ifdef CONFIG_FAIR_GROUP_SCHED
void (*task_change_group)(struct task_struct *p, int type);
#endif
#ifdef CONFIG_SCHED_WALT
void (*fixup_walt_sched_stats)(struct rq *rq, struct task_struct *p,
u16 updated_demand_scaled,
u16 updated_pred_demand_scaled);
#endif
};
static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
{
WARN_ON_ONCE(rq->curr != prev);
@ -1960,6 +2095,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
{
unsigned prev_nr = rq->nr_running;
sched_update_nr_prod(cpu_of(rq), count, true);
rq->nr_running = prev_nr + count;
#ifdef CONFIG_SMP
@ -1974,6 +2110,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
static inline void sub_nr_running(struct rq *rq, unsigned count)
{
sched_update_nr_prod(cpu_of(rq), count, false);
rq->nr_running -= count;
/* Check if we still need preemption */
sched_update_tick_dependency(rq);
@ -2014,6 +2151,18 @@ static inline int hrtick_enabled(struct rq *rq)
#endif /* CONFIG_SCHED_HRTICK */
#ifdef CONFIG_SCHED_WALT
u64 __weak sched_ktime_clock(void);
unsigned long __weak
cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load);
#else
#define sched_ravg_window TICK_NSEC
static inline u64 sched_ktime_clock(void)
{
return 0;
}
#endif
#ifndef arch_scale_freq_capacity
static __always_inline
unsigned long arch_scale_freq_capacity(int cpu)
@ -2031,8 +2180,127 @@ unsigned long arch_scale_max_freq_capacity(struct sched_domain *sd, int cpu)
}
#endif
unsigned long capacity_curr_of(int cpu);
#ifdef CONFIG_SMP
#ifdef CONFIG_PREEMPTION
static inline unsigned long capacity_of(int cpu)
{
return cpu_rq(cpu)->cpu_capacity;
}
static inline unsigned long capacity_orig_of(int cpu)
{
return cpu_rq(cpu)->cpu_capacity_orig;
}
static inline unsigned long task_util(struct task_struct *p)
{
#ifdef CONFIG_SCHED_WALT
return p->ravg.demand_scaled;
#endif
return READ_ONCE(p->se.avg.util_avg);
}
/**
* Amount of capacity of a CPU that is (estimated to be) used by CFS tasks
* @cpu: the CPU to get the utilization of
*
* The unit of the return value must be the one of capacity so we can compare
* the utilization with the capacity of the CPU that is available for CFS task
* (ie cpu_capacity).
*
* cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the
* recent utilization of currently non-runnable tasks on a CPU. It represents
* the amount of utilization of a CPU in the range [0..capacity_orig] where
* capacity_orig is the cpu_capacity available at the highest frequency
* (arch_scale_freq_capacity()).
* The utilization of a CPU converges towards a sum equal to or less than the
* current capacity (capacity_curr <= capacity_orig) of the CPU because it is
* the running time on this CPU scaled by capacity_curr.
*
* The estimated utilization of a CPU is defined to be the maximum between its
* cfs_rq.avg.util_avg and the sum of the estimated utilization of the tasks
* currently RUNNABLE on that CPU.
* This allows to properly represent the expected utilization of a CPU which
* has just got a big task running since a long sleep period. At the same time
* however it preserves the benefits of the "blocked utilization" in
* describing the potential for other tasks waking up on the same CPU.
*
* Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even
* higher than capacity_orig because of unfortunate rounding in
* cfs.avg.util_avg or just after migrating tasks and new task wakeups until
* the average stabilizes with the new running time. We need to check that the
* utilization stays within the range of [0..capacity_orig] and cap it if
* necessary. Without utilization capping, a group could be seen as overloaded
* (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of
* available capacity. We allow utilization to overshoot capacity_curr (but not
* capacity_orig) as it useful for predicting the capacity required after task
* migrations (scheduler-driven DVFS).
*
* Return: the (estimated) utilization for the specified CPU
*/
static inline unsigned long cpu_util(int cpu)
{
struct cfs_rq *cfs_rq;
unsigned int util;
#ifdef CONFIG_SCHED_WALT
u64 walt_cpu_util =
cpu_rq(cpu)->walt_stats.cumulative_runnable_avg_scaled;
return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu));
#endif
cfs_rq = &cpu_rq(cpu)->cfs;
util = READ_ONCE(cfs_rq->avg.util_avg);
if (sched_feat(UTIL_EST))
util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued));
return min_t(unsigned long, util, capacity_orig_of(cpu));
}
static inline unsigned long cpu_util_cum(int cpu, int delta)
{
u64 util = cpu_rq(cpu)->cfs.avg.util_avg;
unsigned long capacity = capacity_orig_of(cpu);
#ifdef CONFIG_SCHED_WALT
util = cpu_rq(cpu)->cum_window_demand_scaled;
#endif
delta += util;
if (delta < 0)
return 0;
return (delta >= capacity) ? capacity : delta;
}
static inline unsigned long
cpu_util_freq(int cpu, struct sched_walt_cpu_load *walt_load)
{
#ifdef CONFIG_SCHED_WALT
return cpu_util_freq_walt(cpu, walt_load);
#else
return cpu_util(cpu);
#endif
}
extern unsigned int capacity_margin_freq;
static inline unsigned long
add_capacity_margin(unsigned long cpu_capacity, int cpu)
{
cpu_capacity = cpu_capacity * capacity_margin_freq *
(100 + per_cpu(sched_load_boost, cpu));
cpu_capacity /= 100;
cpu_capacity /= SCHED_CAPACITY_SCALE;
return cpu_capacity;
}
#endif /* CONFIG_SMP */
#ifdef CONFIG_SMP
#ifdef CONFIG_PREEMPT
static inline void double_rq_lock(struct rq *rq1, struct rq *rq2);
@ -2345,6 +2613,11 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
{
struct update_util_data *data;
#ifdef CONFIG_SCHED_WALT
if (!(flags & SCHED_CPUFREQ_WALT))
return;
#endif
data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data,
cpu_of(rq)));
if (data)
@ -2432,13 +2705,6 @@ static inline bool uclamp_latency_sensitive(struct task_struct *p)
# define arch_scale_freq_invariant() false
#endif
#ifdef CONFIG_SMP
static inline unsigned long capacity_orig_of(int cpu)
{
return cpu_rq(cpu)->cpu_capacity_orig;
}
#endif
/**
* enum schedutil_type - CPU utilization type
* @FREQUENCY_UTIL: Utilization used to select frequency
@ -2570,3 +2836,500 @@ static inline void membarrier_switch_mm(struct rq *rq,
{
}
#endif
enum sched_boost_policy {
SCHED_BOOST_NONE,
SCHED_BOOST_ON_BIG,
SCHED_BOOST_ON_ALL,
};
#ifdef CONFIG_SCHED_WALT
static inline int cluster_first_cpu(struct sched_cluster *cluster)
{
return cpumask_first(&cluster->cpus);
}
struct related_thread_group {
int id;
raw_spinlock_t lock;
struct list_head tasks;
struct list_head list;
bool skip_min;
struct rcu_head rcu;
u64 last_update;
u64 downmigrate_ts;
u64 start_ts;
};
extern struct sched_cluster *sched_cluster[NR_CPUS];
extern unsigned int __weak sched_disable_window_stats;
extern unsigned int max_possible_freq;
extern unsigned int min_max_freq;
extern unsigned int max_possible_efficiency;
extern unsigned int min_possible_efficiency;
extern unsigned int max_possible_capacity;
extern unsigned int __weak min_max_possible_capacity;
extern unsigned int max_power_cost;
extern unsigned int __read_mostly __weak sched_init_task_load_windows;
extern unsigned int __read_mostly __weak sched_load_granule;
extern int __weak update_preferred_cluster(struct related_thread_group *grp,
struct task_struct *p, u32 old_load, bool from_tick);
extern void __weak set_preferred_cluster(struct related_thread_group *grp);
extern void __weak add_new_task_to_grp(struct task_struct *new);
#define NO_BOOST 0
#define FULL_THROTTLE_BOOST 1
#define CONSERVATIVE_BOOST 2
#define RESTRAINED_BOOST 3
#define FULL_THROTTLE_BOOST_DISABLE -1
#define CONSERVATIVE_BOOST_DISABLE -2
#define RESTRAINED_BOOST_DISABLE -3
#define MAX_NUM_BOOST_TYPE (RESTRAINED_BOOST+1)
static inline int asym_cap_siblings(int cpu1, int cpu2)
{
return (cpumask_test_cpu(cpu1, &asym_cap_sibling_cpus) &&
cpumask_test_cpu(cpu2, &asym_cap_sibling_cpus));
}
static inline int cpu_max_possible_capacity(int cpu)
{
return cpu_rq(cpu)->cluster->max_possible_capacity;
}
static inline unsigned int cluster_max_freq(struct sched_cluster *cluster)
{
/*
* Governor and thermal driver don't know the other party's mitigation
* voting. So struct cluster saves both and return min() for current
* cluster fmax.
*/
return min(cluster->max_mitigated_freq, cluster->max_freq);
}
static inline unsigned int cpu_max_freq(int cpu)
{
return cluster_max_freq(cpu_rq(cpu)->cluster);
}
static inline unsigned int cpu_max_possible_freq(int cpu)
{
return cpu_rq(cpu)->cluster->max_possible_freq;
}
static inline bool hmp_capable(void)
{
return max_possible_capacity != min_max_possible_capacity;
}
static inline bool is_max_capacity_cpu(int cpu)
{
return cpu_max_possible_capacity(cpu) == max_possible_capacity;
}
static inline bool is_min_capacity_cpu(int cpu)
{
return cpu_max_possible_capacity(cpu) == min_max_possible_capacity;
}
static inline unsigned int task_load(struct task_struct *p)
{
return p->ravg.demand;
}
static inline unsigned int task_pl(struct task_struct *p)
{
return p->ravg.pred_demand;
}
static inline bool task_in_related_thread_group(struct task_struct *p)
{
return !!(rcu_access_pointer(p->grp) != NULL);
}
static inline
struct related_thread_group *task_related_thread_group(struct task_struct *p)
{
return rcu_dereference(p->grp);
}
/* Is frequency of two cpus synchronized with each other? */
static inline int same_freq_domain(int src_cpu, int dst_cpu)
{
struct rq *rq = cpu_rq(src_cpu);
if (src_cpu == dst_cpu)
return 1;
if (asym_cap_siblings(src_cpu, dst_cpu))
return 1;
return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask);
}
#define CPU_RESERVED 1
extern enum sched_boost_policy __weak boost_policy;
extern unsigned int __weak sched_task_filter_util;
static inline enum sched_boost_policy sched_boost_policy(void)
{
return boost_policy;
}
extern unsigned int __weak sched_boost_type;
static inline int sched_boost(void)
{
return sched_boost_type;
}
static inline bool rt_boost_on_big(void)
{
return sched_boost() == FULL_THROTTLE_BOOST ?
(sched_boost_policy() == SCHED_BOOST_ON_BIG) : false;
}
static inline bool is_full_throttle_boost(void)
{
return sched_boost() == FULL_THROTTLE_BOOST;
}
extern int __weak preferred_cluster(struct sched_cluster *cluster,
struct task_struct *p);
extern struct sched_cluster *rq_cluster(struct rq *rq);
#ifdef CONFIG_UCLAMP_TASK_GROUP
static inline bool task_sched_boost(struct task_struct *p)
{
struct cgroup_subsys_state *css = task_css(p, cpu_cgrp_id);
struct task_group *tg;
if (!css)
return false;
tg = container_of(css, struct task_group, css);
return tg->sched_boost_enabled;
}
extern int __weak sync_cgroup_colocation(struct task_struct *p, bool insert);
extern void update_cgroup_boost_settings(void);
extern void restore_cgroup_boost_settings(void);
#else
static inline bool
same_schedtg(struct task_struct *tsk1, struct task_struct *tsk2)
{
return true;
}
static inline bool task_sched_boost(struct task_struct *p)
{
return true;
}
static inline void update_cgroup_boost_settings(void) { }
static inline void restore_cgroup_boost_settings(void) { }
#endif
extern int __weak alloc_related_thread_groups(void);
extern void __weak check_for_migration(struct rq *rq, struct task_struct *p);
static inline int is_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
return test_bit(CPU_RESERVED, &rq->walt_flags);
}
static inline int mark_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
return test_and_set_bit(CPU_RESERVED, &rq->walt_flags);
}
static inline void clear_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
clear_bit(CPU_RESERVED, &rq->walt_flags);
}
static inline bool
task_in_cum_window_demand(struct rq *rq, struct task_struct *p)
{
return cpu_of(rq) == task_cpu(p) && (p->on_rq || p->last_sleep_ts >=
rq->window_start);
}
static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta)
{
rq->cum_window_demand_scaled += scaled_delta;
if (unlikely((s64)rq->cum_window_demand_scaled < 0))
rq->cum_window_demand_scaled = 0;
}
extern unsigned long __weak thermal_cap(int cpu);
extern void __weak clear_walt_request(int cpu);
extern enum sched_boost_policy sched_boost_policy(void);
extern void sched_boost_parse_dt(void);
extern void __weak clear_ed_task(struct task_struct *p, struct rq *rq);
extern bool __weak early_detection_notify(struct rq *rq, u64 wallclock);
static inline unsigned int power_cost(int cpu, u64 demand)
{
return cpu_max_possible_capacity(cpu);
}
void __weak note_task_waking(struct task_struct *p, u64 wallclock);
static inline bool task_placement_boost_enabled(struct task_struct *p)
{
if (task_sched_boost(p))
return sched_boost_policy() != SCHED_BOOST_NONE;
return false;
}
static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
{
enum sched_boost_policy policy = task_sched_boost(p) ?
sched_boost_policy() :
SCHED_BOOST_NONE;
if (policy == SCHED_BOOST_ON_BIG) {
/*
* Filter out tasks less than min task util threshold
* under conservative boost.
*/
if (sched_boost() == CONSERVATIVE_BOOST &&
task_util(p) <= sched_task_filter_util)
policy = SCHED_BOOST_NONE;
}
return policy;
}
static inline bool is_min_capacity_cluster(struct sched_cluster *cluster)
{
return is_min_capacity_cpu(cluster_first_cpu(cluster));
}
extern void __weak walt_fixup_sched_stats_fair(struct rq *rq,
struct task_struct *p,
u16 updated_demand_scaled,
u16 updated_pred_demand_scaled);
extern void __weak walt_fixup_nr_big_tasks(struct rq *rq, struct task_struct *p,
int delta, bool inc);
#else /* CONFIG_SCHED_WALT */
struct walt_sched_stats;
struct related_thread_group;
struct sched_cluster;
static inline bool task_sched_boost(struct task_struct *p)
{
return false;
}
static inline bool task_placement_boost_enabled(struct task_struct *p)
{
return false;
}
static inline void check_for_migration(struct rq *rq, struct task_struct *p) { }
static inline int sched_boost(void)
{
return 0;
}
static inline bool rt_boost_on_big(void)
{
return false;
}
static inline bool is_full_throttle_boost(void)
{
return false;
}
static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
{
return SCHED_BOOST_NONE;
}
static inline bool
task_in_cum_window_demand(struct rq *rq, struct task_struct *p)
{
return false;
}
static inline bool hmp_capable(void) { return false; }
static inline bool is_max_capacity_cpu(int cpu) { return true; }
static inline bool is_min_capacity_cpu(int cpu) { return true; }
static inline int
preferred_cluster(struct sched_cluster *cluster, struct task_struct *p)
{
return -1;
}
static inline struct sched_cluster *rq_cluster(struct rq *rq)
{
return NULL;
}
static inline int asym_cap_siblings(int cpu1, int cpu2) { return 0; }
static inline void set_preferred_cluster(struct related_thread_group *grp) { }
static inline bool task_in_related_thread_group(struct task_struct *p)
{
return false;
}
static inline
struct related_thread_group *task_related_thread_group(struct task_struct *p)
{
return NULL;
}
static inline u32 task_load(struct task_struct *p) { return 0; }
static inline u32 task_pl(struct task_struct *p) { return 0; }
static inline int update_preferred_cluster(struct related_thread_group *grp,
struct task_struct *p, u32 old_load, bool from_tick)
{
return 0;
}
static inline void add_new_task_to_grp(struct task_struct *new) {}
static inline int same_freq_domain(int src_cpu, int dst_cpu)
{
return 1;
}
static inline int mark_reserved(int cpu)
{
return 0;
}
static inline void clear_reserved(int cpu) { }
static inline int alloc_related_thread_groups(void) { return 0; }
static inline void walt_fixup_cum_window_demand(struct rq *rq,
s64 scaled_delta) { }
#ifdef CONFIG_SMP
static inline unsigned long thermal_cap(int cpu)
{
return cpu_rq(cpu)->cpu_capacity_orig;
}
#endif
static inline void clear_walt_request(int cpu) { }
static inline int is_reserved(int cpu)
{
return 0;
}
static inline enum sched_boost_policy sched_boost_policy(void)
{
return SCHED_BOOST_NONE;
}
static inline void sched_boost_parse_dt(void) { }
static inline void clear_ed_task(struct task_struct *p, struct rq *rq) { }
static inline bool early_detection_notify(struct rq *rq, u64 wallclock)
{
return 0;
}
#ifdef CONFIG_SMP
static inline unsigned int power_cost(int cpu, u64 demand)
{
return SCHED_CAPACITY_SCALE;
}
#endif
static inline void note_task_waking(struct task_struct *p, u64 wallclock) { }
#endif /* CONFIG_SCHED_WALT */
struct sched_avg_stats {
int nr;
int nr_misfit;
int nr_max;
int nr_scaled;
};
extern void sched_get_nr_running_avg(struct sched_avg_stats *stats);
#if defined(CONFIG_SCHED_WALT) && defined(CONFIG_CFS_BANDWIDTH)
extern void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq);
extern void walt_inc_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p);
extern void walt_dec_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p);
extern void walt_inc_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
struct cfs_rq *cfs_rq);
extern void walt_dec_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
struct cfs_rq *cfs_rq);
#else
static inline void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq) {}
static inline void
walt_inc_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p) {}
static inline void
walt_dec_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p) {}
#define walt_inc_throttled_cfs_rq_stats(...)
#define walt_dec_throttled_cfs_rq_stats(...)
#endif
#ifdef CONFIG_SMP
#ifdef CONFIG_SCHED_WALT
extern int __weak group_balance_cpu_not_isolated(struct sched_group *sg);
#else
static inline int group_balance_cpu_not_isolated(struct sched_group *sg)
{
return group_balance_cpu(sg);
}
#endif /* CONFIG_SCHED_WALT */
#endif /* CONFIG_SMP */
extern int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu,
int sync);
extern int active_load_balance_cpu_stop(void *data);
#ifdef CONFIG_HOTPLUG_CPU
extern void set_rq_online(struct rq *rq);
extern void set_rq_offline(struct rq *rq);
extern void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf,
bool migrate_pinned_tasks);
extern void calc_load_migrate(struct rq *rq);
#ifdef CONFIG_SCHED_WALT
extern void __weak
detach_one_task_core(struct task_struct *p, struct rq *rq,
struct list_head *tasks);
extern void __weak attach_tasks_core(struct list_head *tasks, struct rq *rq);
#else
static inline void
detach_one_task_core(struct task_struct *p, struct rq *rq,
struct list_head *tasks)
{
}
static inline void attach_tasks_core(struct list_head *tasks, struct rq *rq) {}
#endif
#endif
extern struct task_struct *find_process_by_pid(pid_t pid);
extern void enqueue_task_core(struct rq *rq, struct task_struct *p, int flags);
extern void dequeue_task_core(struct rq *rq, struct task_struct *p, int flags);

6
kernel/sched/stop_task.c
View file

@ -8,6 +8,7 @@
* See kernel/stop_machine.c
*/
#include "sched.h"
#include "walt.h"
#ifdef CONFIG_SMP
static int
@ -50,12 +51,14 @@ static void
enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags)
{
add_nr_running(rq, 1);
walt_inc_cumulative_runnable_avg(rq, p);
}
static void
dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags)
{
sub_nr_running(rq, 1);
walt_dec_cumulative_runnable_avg(rq, p);
}
static void yield_task_stop(struct rq *rq)
@ -144,4 +147,7 @@ const struct sched_class stop_sched_class = {
.prio_changed = prio_changed_stop,
.switched_to = switched_to_stop,
.update_curr = update_curr_stop,
#ifdef CONFIG_SCHED_WALT
.fixup_walt_sched_stats = fixup_walt_sched_stats_common,
#endif
};

7
kernel/sched/topology.c
View file

@ -1163,16 +1163,19 @@ build_sched_groups(struct sched_domain *sd, int cpu)
* group having more cpu_capacity will pickup more load compared to the
* group having less cpu_capacity.
*/
static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
{
struct sched_group *sg = sd->groups;
cpumask_t avail_mask;
WARN_ON(!sg);
do {
int cpu, max_cpu = -1;
sg->group_weight = cpumask_weight(sched_group_span(sg));
cpumask_andnot(&avail_mask, sched_group_span(sg),
cpu_isolated_mask);
sg->group_weight = cpumask_weight(&avail_mask);
if (!(sd->flags & SD_ASYM_PACKING))
goto next;

265
kernel/sched/walt.h Normal file
View file

@ -0,0 +1,265 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __WALT_H
#define __WALT_H
#ifdef CONFIG_SCHED_WALT
#include <linux/sched/sysctl.h>
#include <linux/sched/core_ctl.h>
#define EXITING_TASK_MARKER 0xdeaddead
extern void __weak
walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event,
u64 wallclock, u64 irqtime);
static inline void
fixup_cumulative_runnable_avg(struct walt_sched_stats *stats,
s64 demand_scaled_delta,
s64 pred_demand_scaled_delta)
{
if (sched_disable_window_stats)
return;
stats->cumulative_runnable_avg_scaled += demand_scaled_delta;
BUG_ON((s64)stats->cumulative_runnable_avg_scaled < 0);
stats->pred_demands_sum_scaled += pred_demand_scaled_delta;
BUG_ON((s64)stats->pred_demands_sum_scaled < 0);
}
static inline void
walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
{
if (sched_disable_window_stats)
return;
fixup_cumulative_runnable_avg(&rq->walt_stats, p->ravg.demand_scaled,
p->ravg.pred_demand_scaled);
/*
* Add a task's contribution to the cumulative window demand when
*
* (1) task is enqueued with on_rq = 1 i.e migration,
* prio/cgroup/class change.
* (2) task is waking for the first time in this window.
*/
if (p->on_rq || (p->last_sleep_ts < rq->window_start))
walt_fixup_cum_window_demand(rq, p->ravg.demand_scaled);
}
static inline void
walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
{
if (sched_disable_window_stats)
return;
fixup_cumulative_runnable_avg(&rq->walt_stats,
-(s64)p->ravg.demand_scaled,
-(s64)p->ravg.pred_demand_scaled);
/*
* on_rq will be 1 for sleeping tasks. So check if the task
* is migrating or dequeuing in RUNNING state to change the
* prio/cgroup/class.
*/
if (task_on_rq_migrating(p) || p->state == TASK_RUNNING)
walt_fixup_cum_window_demand(rq, -(s64)p->ravg.demand_scaled);
}
extern void __weak
fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p,
u16 updated_demand_scaled,
u16 updated_pred_demand_scaled);
extern void __weak inc_rq_walt_stats(struct rq *rq, struct task_struct *p);
extern void __weak dec_rq_walt_stats(struct rq *rq, struct task_struct *p);
extern void __weak fixup_busy_time(struct task_struct *p, int new_cpu);
extern void __weak init_new_task_load(struct task_struct *p);
extern void __weak mark_task_starting(struct task_struct *p);
extern void __weak set_window_start(struct rq *rq);
extern bool __weak do_pl_notif(struct rq *rq);
#define SCHED_HIGH_IRQ_TIMEOUT 3
static inline u64 sched_irqload(int cpu)
{
struct rq *rq = cpu_rq(cpu);
s64 delta;
delta = get_jiffies_64() - rq->irqload_ts;
/*
* Current context can be preempted by irq and rq->irqload_ts can be
* updated by irq context so that delta can be negative.
* But this is okay and we can safely return as this means there
* was recent irq occurrence.
*/
if (delta < SCHED_HIGH_IRQ_TIMEOUT)
return rq->avg_irqload;
else
return 0;
}
static inline int sched_cpu_high_irqload(int cpu)
{
return sched_irqload(cpu) >= sysctl_sched_cpu_high_irqload;
}
static inline int exiting_task(struct task_struct *p)
{
return (p->ravg.sum_history[0] == EXITING_TASK_MARKER);
}
static inline u64
scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq)
{
return div64_u64(load * (u64)src_freq, (u64)dst_freq);
}
extern void __weak sched_account_irqstart(int cpu, struct task_struct *curr,
u64 wallclock);
static inline unsigned int max_task_load(void)
{
return sched_ravg_window;
}
extern void __weak update_cluster_topology(void);
extern void __weak init_clusters(void);
extern void sched_account_irqtime(int cpu, struct task_struct *curr,
u64 delta, u64 wallclock);
static inline int same_cluster(int src_cpu, int dst_cpu)
{
return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster;
}
void __weak walt_sched_init_rq(struct rq *rq);
static inline void walt_update_last_enqueue(struct task_struct *p)
{
p->last_enqueued_ts = sched_ktime_clock();
}
static inline bool is_suh_max(void)
{
return sysctl_sched_user_hint == sched_user_hint_max;
}
#define DEFAULT_CGROUP_COLOC_ID 1
static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu)
{
struct related_thread_group *rtg = p->grp;
if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID &&
rtg->skip_min && p->unfilter)
return is_min_capacity_cpu(cpu);
return false;
}
extern bool is_rtgb_active(void);
extern u64 get_rtgb_active_time(void);
/* utility function to update walt signals at wakeup */
static inline void walt_try_to_wake_up(struct task_struct *p)
{
struct rq *rq = cpu_rq(task_cpu(p));
struct rq_flags rf;
u64 wallclock;
unsigned int old_load;
struct related_thread_group *grp = NULL;
rq_lock_irqsave(rq, &rf);
old_load = task_load(p);
wallclock = sched_ktime_clock();
walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0);
note_task_waking(p, wallclock);
rq_unlock_irqrestore(rq, &rf);
rcu_read_lock();
grp = task_related_thread_group(p);
if (update_preferred_cluster(grp, p, old_load, false))
set_preferred_cluster(grp);
rcu_read_unlock();
}
#else /* CONFIG_SCHED_WALT */
static inline void walt_sched_init_rq(struct rq *rq) { }
static inline void walt_update_last_enqueue(struct task_struct *p) { }
static inline void walt_update_task_ravg(struct task_struct *p, struct rq *rq,
int event, u64 wallclock, u64 irqtime) { }
static inline void walt_inc_cumulative_runnable_avg(struct rq *rq,
struct task_struct *p)
{
}
static inline void walt_dec_cumulative_runnable_avg(struct rq *rq,
struct task_struct *p)
{
}
static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { }
static inline void init_new_task_load(struct task_struct *p)
{
}
static inline void mark_task_starting(struct task_struct *p) { }
static inline void set_window_start(struct rq *rq) { }
static inline int sched_cpu_high_irqload(int cpu) { return 0; }
static inline void sched_account_irqstart(int cpu, struct task_struct *curr,
u64 wallclock)
{
}
static inline void update_cluster_topology(void) { }
static inline void init_clusters(void) {}
static inline void sched_account_irqtime(int cpu, struct task_struct *curr,
u64 delta, u64 wallclock)
{
}
static inline int same_cluster(int src_cpu, int dst_cpu) { return 1; }
static inline bool do_pl_notif(struct rq *rq) { return false; }
static inline void
inc_rq_walt_stats(struct rq *rq, struct task_struct *p) { }
static inline void
dec_rq_walt_stats(struct rq *rq, struct task_struct *p) { }
static inline void
fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p,
u16 updated_demand_scaled,
u16 updated_pred_demand_scaled)
{
}
static inline u64 sched_irqload(int cpu)
{
return 0;
}
static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu)
{
return false;
}
static inline u64 get_rtgb_active_time(void)
{
return 0;
}
#define walt_try_to_wake_up(a) {}
#endif /* CONFIG_SCHED_WALT */
#endif

3
kernel/smp.c
View file

@ -792,7 +792,8 @@ void wake_up_all_idle_cpus(void)
if (cpu == smp_processor_id())
continue;
wake_up_if_idle(cpu);
if (!cpu_isolated(cpu))
wake_up_if_idle(cpu);
}
preempt_enable();
}

219
kernel/sysctl.c
View file

@ -64,6 +64,7 @@
#include <linux/binfmts.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/coredump.h>
#include <linux/sched/stat.h>
#include <linux/kexec.h>
#include <linux/bpf.h>
#include <linux/mount.h>
@ -126,6 +127,7 @@ static int sixty = 60;
#endif
static int __maybe_unused neg_one = -1;
static int __maybe_unused two = 2;
static int __maybe_unused four = 4;
static unsigned long zero_ul;
@ -140,7 +142,12 @@ static int ten_thousand = 10000;
static int six_hundred_forty_kb = 640 * 1024;
#endif
#ifdef CONFIG_SCHED_WALT
static int neg_three = -3;
static int three = 3;
static int two_hundred_fifty_five = 255;
const int sched_user_hint_max = 1000;
static unsigned int ns_per_sec = NSEC_PER_SEC;
static unsigned int one_hundred_thousand = 100000;
#endif
/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */
@ -231,6 +238,10 @@ static int proc_dostring_coredump(struct ctl_table *table, int write,
#endif
static int proc_dopipe_max_size(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos);
#ifdef CONFIG_SCHED_WALT
static int proc_douintvec_minmax_schedhyst(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos);
#endif
#ifdef CONFIG_MAGIC_SYSRQ
/* Note: sysrq code uses its own private copy */
@ -328,6 +339,172 @@ static struct ctl_table kern_table[] = {
.mode = 0644,
.proc_handler = proc_dointvec,
},
#ifdef CONFIG_SCHED_WALT
{
.procname = "sched_user_hint",
.data = &sysctl_sched_user_hint,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = walt_proc_user_hint_handler,
.extra1 = SYSCTL_ZERO,
.extra2 = (void *)&sched_user_hint_max,
},
{
.procname = "sched_window_stats_policy",
.data = &sysctl_sched_window_stats_policy,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = &four,
},
{
.procname = "sched_cpu_high_irqload",
.data = &sysctl_sched_cpu_high_irqload,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
{
.procname = "sched_group_upmigrate",
.data = &sysctl_sched_group_upmigrate_pct,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = walt_proc_group_thresholds_handler,
.extra1 = &sysctl_sched_group_downmigrate_pct,
},
{
.procname = "sched_group_downmigrate",
.data = &sysctl_sched_group_downmigrate_pct,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = walt_proc_group_thresholds_handler,
.extra1 = SYSCTL_ZERO,
.extra2 = &sysctl_sched_group_upmigrate_pct,
},
{
.procname = "sched_boost",
.data = &sysctl_sched_boost,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = sched_boost_handler,
.extra1 = &neg_three,
.extra2 = &three,
},
{
.procname = "sched_conservative_pl",
.data = &sysctl_sched_conservative_pl,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{
.procname = "sched_walt_rotate_big_tasks",
.data = &sysctl_sched_walt_rotate_big_tasks,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
{
.procname = "sched_min_task_util_for_boost",
.data = &sysctl_sched_min_task_util_for_boost,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = &one_thousand,
},
{
.procname = "sched_min_task_util_for_colocation",
.data = &sysctl_sched_min_task_util_for_colocation,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = &one_thousand,
},
{
.procname = "sched_asym_cap_sibling_freq_match_pct",
.data = &sysctl_sched_asym_cap_sibling_freq_match_pct,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ONE,
.extra2 = &one_hundred,
},
{
.procname = "sched_coloc_downmigrate_ns",
.data = &sysctl_sched_coloc_downmigrate_ns,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax,
},
{
.procname = "sched_task_unfilter_nr_windows",
.data = &sysctl_sched_task_unfilter_nr_windows,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ONE,
.extra2 = &two_hundred_fifty_five,
},
{
.procname = "sched_busy_hysteresis_enable_cpus",
.data = &sysctl_sched_busy_hyst_enable_cpus,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax_schedhyst,
.extra1 = SYSCTL_ZERO,
.extra2 = &two_hundred_fifty_five,
},
{
.procname = "sched_busy_hyst_ns",
.data = &sysctl_sched_busy_hyst,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax_schedhyst,
.extra1 = SYSCTL_ZERO,
.extra2 = &ns_per_sec,
},
{
.procname = "sched_coloc_busy_hysteresis_enable_cpus",
.data = &sysctl_sched_coloc_busy_hyst_enable_cpus,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax_schedhyst,
.extra1 = SYSCTL_ZERO,
.extra2 = &two_hundred_fifty_five,
},
{
.procname = "sched_coloc_busy_hyst_ns",
.data = &sysctl_sched_coloc_busy_hyst,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax_schedhyst,
.extra1 = SYSCTL_ZERO,
.extra2 = &ns_per_sec,
},
{
.procname = "sched_coloc_busy_hyst_max_ms",
.data = &sysctl_sched_coloc_busy_hyst_max_ms,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_douintvec_minmax_schedhyst,
.extra1 = SYSCTL_ZERO,
.extra2 = &one_hundred_thousand,
},
{
.procname = "sched_ravg_window_nr_ticks",
.data = &sysctl_sched_ravg_window_nr_ticks,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = sched_ravg_window_handler,
},
#endif
#ifdef CONFIG_SCHED_DEBUG
{
.procname = "sched_min_granularity_ns",
@ -2874,6 +3051,19 @@ static int proc_dostring_coredump(struct ctl_table *table, int write,
}
#endif
#ifdef CONFIG_SCHED_WALT
static int proc_douintvec_minmax_schedhyst(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
int ret = proc_douintvec_minmax(table, write, buffer, lenp, ppos);
if (!ret && write)
sched_update_hyst_times();
return ret;
}
#endif
static int __do_proc_doulongvec_minmax(void *data, struct ctl_table *table, int write,
void __user *buffer,
size_t *lenp, loff_t *ppos,
@ -3341,6 +3531,29 @@ int proc_do_large_bitmap(struct ctl_table *table, int write,
return err;
}
static int do_proc_douintvec_rwin(bool *negp, unsigned long *lvalp,
int *valp, int write, void *data)
{
if (write) {
if (*lvalp == 0 || *lvalp == 2 || *lvalp == 5)
*valp = *lvalp;
else
return -EINVAL;
} else {
*negp = false;
*lvalp = *valp;
}
return 0;
}
int proc_douintvec_ravg_window(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
return do_proc_dointvec(table, write, buffer, lenp, ppos,
do_proc_douintvec_rwin, NULL);
}
#else /* CONFIG_PROC_SYSCTL */
int proc_dostring(struct ctl_table *table, int write,
@ -3410,6 +3623,12 @@ int proc_do_large_bitmap(struct ctl_table *table, int write,
return -ENOSYS;
}
int proc_douintvec_ravg_window(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
return -ENOSYS;
}
#endif /* CONFIG_PROC_SYSCTL */
#if defined(CONFIG_SYSCTL)

13
kernel/time/hrtimer.c
View file

@ -2014,7 +2014,7 @@ int hrtimers_prepare_cpu(unsigned int cpu)
return 0;
}
#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_CPUSETS)
#ifdef CONFIG_HOTPLUG_CPU
static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
struct hrtimer_clock_base *new_base,
bool remove_pinned)
@ -2023,12 +2023,14 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
struct timerqueue_node *node;
struct timerqueue_head pinned;
int is_pinned;
bool is_hotplug = !cpu_online(old_base->cpu_base->cpu);
timerqueue_init_head(&pinned);
while ((node = timerqueue_getnext(&old_base->active))) {
timer = container_of(node, struct hrtimer, node);
BUG_ON(hrtimer_callback_running(timer));
if (is_hotplug)
BUG_ON(hrtimer_callback_running(timer));
debug_deactivate(timer);
/*
@ -2106,9 +2108,7 @@ static void __migrate_hrtimers(unsigned int scpu, bool remove_pinned)
local_irq_restore(flags);
local_bh_enable();
}
#endif /* CONFIG_HOTPLUG_CPU || CONFIG_CPUSETS */
#ifdef CONFIG_HOTPLUG_CPU
int hrtimers_dead_cpu(unsigned int scpu)
{
BUG_ON(cpu_online(scpu));
@ -2117,14 +2117,13 @@ int hrtimers_dead_cpu(unsigned int scpu)
__migrate_hrtimers(scpu, true);
return 0;
}
#endif /* CONFIG_HOTPLUG_CPU */
#ifdef CONFIG_CPUSETS
void hrtimer_quiesce_cpu(void *cpup)
{
__migrate_hrtimers(*(int *)cpup, false);
}
#endif /* CONFIG_CPUSETS */
#endif /* CONFIG_HOTPLUG_CPU */
void __init hrtimers_init(void)
{

5
kernel/time/timer.c
View file

@ -2038,7 +2038,8 @@ static void __migrate_timers(unsigned int cpu, bool remove_pinned)
*/
forward_timer_base(new_base);
BUG_ON(old_base->running_timer);
if (!cpu_online(cpu))
BUG_ON(old_base->running_timer);
for (i = 0; i < WHEEL_SIZE; i++)
migrate_timer_list(new_base, old_base->vectors + i,
@ -2057,12 +2058,10 @@ int timers_dead_cpu(unsigned int cpu)
return 0;
}
#ifdef CONFIG_CPUSETS
void timer_quiesce_cpu(void *cpup)
{
__migrate_timers(*(unsigned int *)cpup, false);
}
#endif /* CONFIG_CPUSETS */
#endif /* CONFIG_HOTPLUG_CPU */

34
kernel/watchdog.c
View file

@ -14,6 +14,7 @@
#include <linux/mm.h>
#include <linux/cpu.h>
#include <linux/device.h>
#include <linux/nmi.h>
#include <linux/init.h>
#include <linux/module.h>
@ -170,6 +171,7 @@ static u64 __read_mostly sample_period;
static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
static DEFINE_PER_CPU(unsigned int, watchdog_en);
static DEFINE_PER_CPU(bool, softlockup_touch_sync);
static DEFINE_PER_CPU(bool, soft_watchdog_warn);
static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
@ -476,16 +478,20 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
return HRTIMER_RESTART;
}
static void watchdog_enable(unsigned int cpu)
void watchdog_enable(unsigned int cpu)
{
struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
struct completion *done = this_cpu_ptr(&softlockup_completion);
unsigned int *enabled = this_cpu_ptr(&watchdog_en);
WARN_ON_ONCE(cpu != smp_processor_id());
init_completion(done);
complete(done);
if (*enabled)
return;
/*
* Start the timer first to prevent the NMI watchdog triggering
* before the timer has a chance to fire.
@ -500,11 +506,24 @@ static void watchdog_enable(unsigned int cpu)
/* Enable the perf event */
if (watchdog_enabled & NMI_WATCHDOG_ENABLED)
watchdog_nmi_enable(cpu);
/*
* Need to ensure above operations are observed by other CPUs before
* indicating that timer is enabled. This is to synchronize core
* isolation and hotplug. Core isolation will wait for this flag to be
* set.
*/
mb();
*enabled = 1;
}
static void watchdog_disable(unsigned int cpu)
void watchdog_disable(unsigned int cpu)
{
struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
unsigned int *enabled = this_cpu_ptr(&watchdog_en);
if (!*enabled)
return;
WARN_ON_ONCE(cpu != smp_processor_id());
@ -516,6 +535,17 @@ static void watchdog_disable(unsigned int cpu)
watchdog_nmi_disable(cpu);
hrtimer_cancel(hrtimer);
wait_for_completion(this_cpu_ptr(&softlockup_completion));
/*
* No need for barrier here since disabling the watchdog is
* synchronized with hotplug lock
*/
*enabled = 0;
}
bool watchdog_configured(unsigned int cpu)
{
return *per_cpu_ptr(&watchdog_en, cpu);
}
static int softlockup_stop_fn(void *data)

5
mm/vmstat.c
View file

@ -1807,7 +1807,7 @@ int vmstat_refresh(struct ctl_table *table, int write,
static void vmstat_update(struct work_struct *w)
{
if (refresh_cpu_vm_stats(true)) {
if (refresh_cpu_vm_stats(true) && !cpu_isolated(smp_processor_id())) {
/*
* Counters were updated so we expect more updates
* to occur in the future. Keep on running the
@ -1899,7 +1899,8 @@ static void vmstat_shepherd(struct work_struct *w)
for_each_online_cpu(cpu) {
struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
if (!delayed_work_pending(dw) && need_update(cpu))
if (!delayed_work_pending(dw) && need_update(cpu) &&
!cpu_isolated(cpu))
queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0);
}
put_online_cpus();