diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index c0cf9ef5c2f5..bbe3b0237773 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -471,6 +471,26 @@ const struct cpumask *cpu_coregroup_mask(int cpu) return core_mask; } +#ifdef CONFIG_SCHED_WALT +void update_possible_siblings_masks(unsigned int cpuid) +{ + struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; + int cpu; + + if (cpuid_topo->package_id == -1) + return; + + for_each_possible_cpu(cpu) { + cpu_topo = &cpu_topology[cpu]; + + if (cpuid_topo->package_id != cpu_topo->package_id) + continue; + cpumask_set_cpu(cpuid, &cpu_topo->core_possible_sibling); + cpumask_set_cpu(cpu, &cpuid_topo->core_possible_sibling); + } +} +#endif + void update_siblings_masks(unsigned int cpuid) { struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; @@ -550,6 +570,9 @@ __weak int __init parse_acpi_topology(void) #if defined(CONFIG_ARM64) || defined(CONFIG_RISCV) void __init init_cpu_topology(void) { +#ifdef CONFIG_SCHED_WALT + int cpu; +#endif reset_cpu_topology(); /* @@ -560,5 +583,11 @@ void __init init_cpu_topology(void) reset_cpu_topology(); else if (of_have_populated_dt() && parse_dt_topology()) reset_cpu_topology(); +#ifdef CONFIG_SCHED_WALT + else { + for_each_possible_cpu(cpu) + update_possible_siblings_masks(cpu); + } +#endif } #endif diff --git a/drivers/base/core.c b/drivers/base/core.c index 42a672456432..5c5948963db0 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -1156,6 +1156,11 @@ int lock_device_hotplug_sysfs(void) return restart_syscall(); } +void lock_device_hotplug_assert(void) +{ + lockdep_assert_held(&device_hotplug_lock); +} + #ifdef CONFIG_BLOCK static inline int device_is_not_partition(struct device *dev) { diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 6265871a4af2..49018ab906c8 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -183,9 +183,94 @@ static struct attribute_group crash_note_cpu_attr_group = { }; #endif +#ifdef CONFIG_HOTPLUG_CPU +static ssize_t isolate_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + ssize_t rc; + int cpuid = cpu->dev.id; + unsigned int isolated = cpu_isolated(cpuid); + + rc = scnprintf(buf, PAGE_SIZE-2, "%d\n", isolated); + + return rc; +} + +static DEVICE_ATTR_RO(isolate); + +static struct attribute *cpu_isolated_attrs[] = { + &dev_attr_isolate.attr, + NULL +}; + +static struct attribute_group cpu_isolated_attr_group = { + .attrs = cpu_isolated_attrs, +}; +#endif + +#ifdef CONFIG_SCHED_WALT +static ssize_t sched_load_boost_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + ssize_t rc; + int boost; + struct cpu *cpu = container_of(dev, struct cpu, dev); + int cpuid = cpu->dev.id; + + boost = per_cpu(sched_load_boost, cpuid); + rc = scnprintf(buf, PAGE_SIZE-2, "%d\n", boost); + + return rc; +} + +static ssize_t __ref sched_load_boost_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int err; + int boost; + struct cpu *cpu = container_of(dev, struct cpu, dev); + int cpuid = cpu->dev.id; + + err = kstrtoint(strstrip((char *)buf), 0, &boost); + if (err) + return err; + + /* + * -100 is low enough to cancel out CPU's load and make it near zro. + * 1000 is close to the maximum value that cpu_util_freq_{walt,pelt} + * can take without overflow. + */ + if (boost < -100 || boost > 1000) + return -EINVAL; + + per_cpu(sched_load_boost, cpuid) = boost; + + return count; +} + +static DEVICE_ATTR_RW(sched_load_boost); + +static struct attribute *sched_cpu_attrs[] = { + &dev_attr_sched_load_boost.attr, + NULL +}; + +static struct attribute_group sched_cpu_attr_group = { + .attrs = sched_cpu_attrs, +}; +#endif + static const struct attribute_group *common_cpu_attr_groups[] = { #ifdef CONFIG_KEXEC &crash_note_cpu_attr_group, +#endif +#ifdef CONFIG_HOTPLUG_CPU + &cpu_isolated_attr_group, +#endif +#ifdef CONFIG_SCHED_WALT + &sched_cpu_attr_group, #endif NULL }; @@ -193,6 +278,12 @@ static const struct attribute_group *common_cpu_attr_groups[] = { static const struct attribute_group *hotplugable_cpu_attr_groups[] = { #ifdef CONFIG_KEXEC &crash_note_cpu_attr_group, +#endif +#ifdef CONFIG_HOTPLUG_CPU + &cpu_isolated_attr_group, +#endif +#ifdef CONFIG_SCHED_WALT + &sched_cpu_attr_group, #endif NULL }; @@ -223,6 +314,7 @@ static struct cpu_attr cpu_attrs[] = { _CPU_ATTR(online, &__cpu_online_mask), _CPU_ATTR(possible, &__cpu_possible_mask), _CPU_ATTR(present, &__cpu_present_mask), + _CPU_ATTR(core_ctl_isolated, &__cpu_isolated_mask), }; /* @@ -472,6 +564,7 @@ static struct attribute *cpu_root_attrs[] = { &cpu_attrs[0].attr.attr, &cpu_attrs[1].attr.attr, &cpu_attrs[2].attr.attr, + &cpu_attrs[3].attr.attr, &dev_attr_kernel_max.attr, &dev_attr_offline.attr, &dev_attr_isolated.attr, diff --git a/fs/proc/base.c b/fs/proc/base.c index 7d13501910ee..ed38e70f475a 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1459,6 +1459,56 @@ static const struct file_operations proc_pid_sched_operations = { #endif +/* + * Print out various scheduling related per-task fields: + */ + +#ifdef CONFIG_SCHED_WALT +extern int __weak sched_wake_up_idle_show(struct seq_file *m, void *v); +extern ssize_t __weak sched_wake_up_idle_write(struct file *file, + const char __user *buf, size_t count, loff_t *offset); +extern int __weak sched_wake_up_idle_open(struct inode *inode, + struct file *filp); + +static const struct file_operations proc_pid_sched_wake_up_idle_operations = { + .open = sched_wake_up_idle_open, + .read = seq_read, + .write = sched_wake_up_idle_write, + .llseek = seq_lseek, + .release = single_release, +}; + +extern int __weak sched_init_task_load_show(struct seq_file *m, void *v); +extern ssize_t __weak +sched_init_task_load_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset); +extern int __weak +sched_init_task_load_open(struct inode *inode, struct file *filp); + +static const struct file_operations proc_pid_sched_init_task_load_operations = { + .open = sched_init_task_load_open, + .read = seq_read, + .write = sched_init_task_load_write, + .llseek = seq_lseek, + .release = single_release, +}; + +extern int __weak sched_group_id_show(struct seq_file *m, void *v); +extern ssize_t __weak +sched_group_id_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset); +extern int __weak sched_group_id_open(struct inode *inode, struct file *filp); + +static const struct file_operations proc_pid_sched_group_id_operations = { + .open = sched_group_id_open, + .read = seq_read, + .write = sched_group_id_write, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_SCHED_WALT */ + #ifdef CONFIG_SCHED_AUTOGROUP /* * Print out autogroup related information: @@ -3011,6 +3061,13 @@ static const struct pid_entry tgid_base_stuff[] = { ONE("status", S_IRUGO, proc_pid_status), ONE("personality", S_IRUSR, proc_pid_personality), ONE("limits", S_IRUGO, proc_pid_limits), +#ifdef CONFIG_SCHED_WALT + REG("sched_wake_up_idle", 00644, + proc_pid_sched_wake_up_idle_operations), + REG("sched_init_task_load", 00644, + proc_pid_sched_init_task_load_operations), + REG("sched_group_id", 00666, proc_pid_sched_group_id_operations), +#endif #ifdef CONFIG_SCHED_DEBUG REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations), #endif diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index 5402bc0e2b1e..883fba81796c 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -48,6 +48,9 @@ struct cpu_topology { int llc_id; cpumask_t thread_sibling; cpumask_t core_sibling; +#ifdef CONFIG_SCHED_WALT + cpumask_t core_possible_sibling; +#endif cpumask_t llc_sibling; }; @@ -59,6 +62,10 @@ extern struct cpu_topology cpu_topology[NR_CPUS]; #define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling) #define topology_sibling_cpumask(cpu) (&cpu_topology[cpu].thread_sibling) #define topology_llc_cpumask(cpu) (&cpu_topology[cpu].llc_sibling) +#ifdef CONFIG_SCHED_WALT +#define topology_possible_sibling_cpumask(cpu) \ + (&cpu_topology[cpu].core_possible_sibling) +#endif void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 068793a619ca..f4c5aae47659 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -69,6 +69,9 @@ enum cpuhp_state { CPUHP_SLAB_PREPARE, CPUHP_MD_RAID5_PREPARE, CPUHP_RCUTREE_PREP, +#ifdef CONFIG_SCHED_WALT + CPUHP_CORE_CTL_ISOLATION_DEAD, +#endif CPUHP_CPUIDLE_COUPLED_PREPARE, CPUHP_POWERPC_PMAC_PREPARE, CPUHP_POWERPC_MMU_CTX_PREPARE, diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 78a73eba64dd..5d20635bcb59 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -55,6 +55,7 @@ extern unsigned int nr_cpu_ids; * cpu_present_mask - has bit 'cpu' set iff cpu is populated * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler * cpu_active_mask - has bit 'cpu' set iff cpu available to migration + * cpu_isolated_mask- has bit 'cpu' set iff cpu isolated * * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. * @@ -91,10 +92,12 @@ extern struct cpumask __cpu_possible_mask; extern struct cpumask __cpu_online_mask; extern struct cpumask __cpu_present_mask; extern struct cpumask __cpu_active_mask; +extern struct cpumask __cpu_isolated_mask; #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) +#define cpu_isolated_mask ((const struct cpumask *)&__cpu_isolated_mask) extern atomic_t __num_online_cpus; @@ -114,19 +117,31 @@ static inline unsigned int num_online_cpus(void) #define num_possible_cpus() cpumask_weight(cpu_possible_mask) #define num_present_cpus() cpumask_weight(cpu_present_mask) #define num_active_cpus() cpumask_weight(cpu_active_mask) +#define num_isolated_cpus() cpumask_weight(cpu_isolated_mask) +#define num_online_uniso_cpus() \ +({ \ + cpumask_t mask; \ + \ + cpumask_andnot(&mask, cpu_online_mask, cpu_isolated_mask); \ + cpumask_weight(&mask); \ +}) #define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask) #define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask) #define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask) #define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask) +#define cpu_isolated(cpu) cpumask_test_cpu((cpu), cpu_isolated_mask) #else #define num_online_cpus() 1U #define num_possible_cpus() 1U #define num_present_cpus() 1U #define num_active_cpus() 1U +#define num_isolated_cpus() 0U +#define num_online_uniso_cpus() 1U #define cpu_online(cpu) ((cpu) == 0) #define cpu_possible(cpu) ((cpu) == 0) #define cpu_present(cpu) ((cpu) == 0) #define cpu_active(cpu) ((cpu) == 0) +#define cpu_isolated(cpu) ((cpu) != 0) #endif extern cpumask_t cpus_booted_once_mask; @@ -806,6 +821,7 @@ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) +#define for_each_isolated_cpu(cpu) for_each_cpu((cpu), cpu_isolated_mask) /* Wrappers for arch boot code to manipulate normally-constant masks */ void init_cpu_present(const struct cpumask *src); @@ -846,6 +862,15 @@ set_cpu_active(unsigned int cpu, bool active) cpumask_clear_cpu(cpu, &__cpu_active_mask); } +static inline void +set_cpu_isolated(unsigned int cpu, bool isolated) +{ + if (isolated) + cpumask_set_cpu(cpu, &__cpu_isolated_mask); + else + cpumask_clear_cpu(cpu, &__cpu_isolated_mask); +} + /** * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask * diff --git a/include/linux/device.h b/include/linux/device.h index d1bcc8f122f6..4babcf8f5d51 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1551,6 +1551,7 @@ static inline bool device_supports_offline(struct device *dev) extern void lock_device_hotplug(void); extern void unlock_device_hotplug(void); extern int lock_device_hotplug_sysfs(void); +extern void lock_device_hotplug_assert(void); extern int device_offline(struct device *dev); extern int device_online(struct device *dev); extern void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode); diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 9003e29cde46..48c92b309952 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -13,6 +13,9 @@ #ifdef CONFIG_LOCKUP_DETECTOR void lockup_detector_init(void); +extern void watchdog_enable(unsigned int cpu); +extern void watchdog_disable(unsigned int cpu); +extern bool watchdog_configured(unsigned int cpu); void lockup_detector_soft_poweroff(void); void lockup_detector_cleanup(void); bool is_hardlockup(void); @@ -37,6 +40,20 @@ extern int sysctl_hardlockup_all_cpu_backtrace; static inline void lockup_detector_init(void) { } static inline void lockup_detector_soft_poweroff(void) { } static inline void lockup_detector_cleanup(void) { } +static inline void watchdog_enable(unsigned int cpu) +{ +} +static inline void watchdog_disable(unsigned int cpu) +{ +} +static inline bool watchdog_configured(unsigned int cpu) +{ + /* + * Pretend the watchdog is always configured. + * We will be waiting for the watchdog to be enabled in core isolation + */ + return true; +} #endif /* !CONFIG_LOCKUP_DETECTOR */ #ifdef CONFIG_SOFTLOCKUP_DETECTOR diff --git a/include/linux/sched.h b/include/linux/sched.h index 2f35afc13de3..5e1f1c6d630d 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -117,6 +117,18 @@ struct task_group; (task->flags & PF_FROZEN) == 0 && \ (task->state & TASK_NOLOAD) == 0) +/* + * Enum for display driver to provide varying refresh rates + */ +enum fps { + FPS0 = 0, + FPS30 = 30, + FPS48 = 48, + FPS60 = 60, + FPS90 = 90, + FPS120 = 120, +}; + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP /* @@ -212,6 +224,42 @@ struct task_group; /* Task command name length: */ #define TASK_COMM_LEN 16 +enum task_event { + PUT_PREV_TASK = 0, + PICK_NEXT_TASK = 1, + TASK_WAKE = 2, + TASK_MIGRATE = 3, + TASK_UPDATE = 4, + IRQ_UPDATE = 5, +}; + +/* Note: this need to be in sync with migrate_type_names array */ +enum migrate_types { + GROUP_TO_RQ, + RQ_TO_GROUP, +}; + +#ifdef CONFIG_HOTPLUG_CPU +extern int __weak sched_isolate_cpu(int cpu); +extern int __weak sched_unisolate_cpu(int cpu); +extern int __weak sched_unisolate_cpu_unlocked(int cpu); +#else +static inline int sched_isolate_cpu(int cpu) +{ + return 0; +} + +static inline int sched_unisolate_cpu(int cpu) +{ + return 0; +} + +static inline int sched_unisolate_cpu_unlocked(int cpu) +{ + return 0; +} +#endif + extern void scheduler_tick(void); #define MAX_SCHEDULE_TIMEOUT LONG_MAX @@ -478,6 +526,89 @@ struct sched_entity { #endif }; +struct cpu_cycle_counter_cb { + u64 (*get_cpu_cycle_counter)(int cpu); +}; + +DECLARE_PER_CPU_READ_MOSTLY(int, sched_load_boost); + +#ifdef CONFIG_SCHED_WALT +extern void __weak sched_exit(struct task_struct *p); +extern int __weak +register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb); +extern void __weak +sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax); +extern void __weak free_task_load_ptrs(struct task_struct *p); +extern void __weak sched_set_refresh_rate(enum fps fps); + +#define RAVG_HIST_SIZE_MAX 5 +#define NUM_BUSY_BUCKETS 10 + +/* ravg represents frequency scaled cpu-demand of tasks */ +struct ravg { + /* + * 'mark_start' marks the beginning of an event (task waking up, task + * starting to execute, task being preempted) within a window + * + * 'sum' represents how runnable a task has been within current + * window. It incorporates both running time and wait time and is + * frequency scaled. + * + * 'sum_history' keeps track of history of 'sum' seen over previous + * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are + * ignored. + * + * 'demand' represents maximum sum seen over previous + * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency + * demand for tasks. + * + * 'curr_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the current window + * + * 'prev_window_cpu' represents task's contribution to cpu busy time on + * various CPUs in the previous window + * + * 'curr_window' represents the sum of all entries in curr_window_cpu + * + * 'prev_window' represents the sum of all entries in prev_window_cpu + * + * 'pred_demand' represents task's current predicted cpu busy time + * + * 'busy_buckets' groups historical busy time into different buckets + * used for prediction + * + * 'demand_scaled' represents task's demand scaled to 1024 + */ + u64 mark_start; + u32 sum, demand; + u32 coloc_demand; + u32 sum_history[RAVG_HIST_SIZE_MAX]; + u32 *curr_window_cpu, *prev_window_cpu; + u32 curr_window, prev_window; + u32 pred_demand; + u8 busy_buckets[NUM_BUSY_BUCKETS]; + u16 demand_scaled; + u16 pred_demand_scaled; + u64 active_time; + u64 last_win_size; +}; +#else +static inline void sched_exit(struct task_struct *p) { } + +static inline int +register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb) +{ + return 0; +} + +static inline void free_task_load_ptrs(struct task_struct *p) { } + +static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus, + u32 fmin, u32 fmax) { } + +static inline void sched_set_refresh_rate(enum fps fps) { } +#endif /* CONFIG_SCHED_WALT */ + struct sched_rt_entity { struct list_head run_list; unsigned long timeout; @@ -675,6 +806,20 @@ struct task_struct { const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; +#ifdef CONFIG_SCHED_WALT + u64 last_sleep_ts; + bool wake_up_idle; + struct ravg ravg; + u32 init_load_pct; + u64 last_wake_ts; + u64 last_enqueued_ts; + struct related_thread_group *grp; + struct list_head grp_list; + u64 cpu_cycles; + bool misfit; + u8 unfilter; +#endif + #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif @@ -2000,4 +2145,37 @@ int sched_trace_rq_cpu(struct rq *rq); const struct cpumask *sched_trace_rd_span(struct root_domain *rd); +#ifdef CONFIG_SCHED_WALT +#define PF_WAKE_UP_IDLE 1 +static inline u32 sched_get_wake_up_idle(struct task_struct *p) +{ + return p->wake_up_idle; +} + +static inline int sched_set_wake_up_idle(struct task_struct *p, + int wake_up_idle) +{ + p->wake_up_idle = !!wake_up_idle; + return 0; +} + +static inline void set_wake_up_idle(bool enabled) +{ + current->wake_up_idle = enabled; +} +#else +static inline u32 sched_get_wake_up_idle(struct task_struct *p) +{ + return 0; +} + +static inline int sched_set_wake_up_idle(struct task_struct *p, + int wake_up_idle) +{ + return 0; +} + +static inline void set_wake_up_idle(bool enabled) {} +#endif + #endif diff --git a/include/linux/sched/core_ctl.h b/include/linux/sched/core_ctl.h new file mode 100644 index 000000000000..20ba464c1170 --- /dev/null +++ b/include/linux/sched/core_ctl.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016, 2019, The Linux Foundation. All rights reserved. + */ + +#ifndef __CORE_CTL_H +#define __CORE_CTL_H + +#define MAX_CPUS_PER_CLUSTER 6 +#define MAX_CLUSTERS 3 + +struct core_ctl_notif_data { + unsigned int nr_big; + unsigned int coloc_load_pct; + unsigned int ta_util_pct[MAX_CLUSTERS]; + unsigned int cur_cap_pct[MAX_CLUSTERS]; +}; + +#ifdef CONFIG_SCHED_WALT +extern int __weak core_ctl_set_boost(bool boost); +extern void __weak core_ctl_notifier_register(struct notifier_block *n); +extern void __weak core_ctl_notifier_unregister(struct notifier_block *n); +#else +static inline int core_ctl_set_boost(bool boost) +{ + return 0; +} +static inline void core_ctl_notifier_register(struct notifier_block *n) {} +static inline void core_ctl_notifier_unregister(struct notifier_block *n) {} +#endif +#endif diff --git a/include/linux/sched/cpufreq.h b/include/linux/sched/cpufreq.h index afa940cd50dc..145c4bbb2ff8 100644 --- a/include/linux/sched/cpufreq.h +++ b/include/linux/sched/cpufreq.h @@ -10,6 +10,11 @@ #define SCHED_CPUFREQ_IOWAIT (1U << 0) #define SCHED_CPUFREQ_MIGRATION (1U << 1) +#define SCHED_CPUFREQ_INTERCLUSTER_MIG (1U << 3) +#define SCHED_CPUFREQ_WALT (1U << 4) +#define SCHED_CPUFREQ_PL (1U << 5) +#define SCHED_CPUFREQ_EARLY_DET (1U << 6) +#define SCHED_CPUFREQ_CONTINUE (1U << 8) #ifdef CONFIG_CPU_FREQ struct update_util_data { diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index 6c8512d3be88..7765668d556e 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -28,7 +28,15 @@ extern void __init housekeeping_init(void); static inline int housekeeping_any_cpu(enum hk_flags flags) { - return smp_processor_id(); + cpumask_t available; + int cpu; + + cpumask_andnot(&available, cpu_online_mask, cpu_isolated_mask); + cpu = cpumask_any(&available); + if (cpu >= nr_cpu_ids) + cpu = smp_processor_id(); + + return cpu; } static inline const struct cpumask *housekeeping_cpumask(enum hk_flags flags) @@ -52,7 +60,7 @@ static inline bool housekeeping_cpu(int cpu, enum hk_flags flags) if (static_branch_unlikely(&housekeeping_overridden)) return housekeeping_test_cpu(cpu, flags); #endif - return true; + return !cpu_isolated(cpu); } #endif /* _LINUX_SCHED_ISOLATION_H */ diff --git a/include/linux/sched/stat.h b/include/linux/sched/stat.h index 568286411b43..1eed4a2526a5 100644 --- a/include/linux/sched/stat.h +++ b/include/linux/sched/stat.h @@ -21,6 +21,28 @@ extern bool single_task_running(void); extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); +#ifdef CONFIG_SCHED_WALT +extern void __weak sched_update_nr_prod(int cpu, long delta, bool inc); +extern unsigned int __weak sched_get_cpu_util(int cpu); +extern void __weak sched_update_hyst_times(void); +extern u64 __weak sched_lpm_disallowed_time(int cpu); +#else +static inline void sched_update_nr_prod(int cpu, long delta, bool inc) {} +static inline unsigned int sched_get_cpu_util(int cpu) +{ + return 0; +} +static inline u64 sched_get_cpu_last_busy_time(int cpu) +{ + return 0; +} +static inline void sched_update_hyst_times(void) {} +static inline u64 sched_lpm_disallowed_time(int cpu) +{ + return 0; +} +#endif + static inline int sched_info_on(void) { #ifdef CONFIG_SCHEDSTATS diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index f9fb44edf4c7..05bc4afc548d 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -24,6 +24,42 @@ extern unsigned int sysctl_sched_latency; extern unsigned int sysctl_sched_min_granularity; extern unsigned int sysctl_sched_wakeup_granularity; extern unsigned int sysctl_sched_child_runs_first; +#ifdef CONFIG_SCHED_WALT +extern unsigned int __weak sysctl_sched_user_hint; +extern const int __weak sched_user_hint_max; +extern unsigned int __weak sysctl_sched_cpu_high_irqload; +extern unsigned int __weak sysctl_sched_boost; +extern unsigned int __weak sysctl_sched_group_upmigrate_pct; +extern unsigned int __weak sysctl_sched_group_downmigrate_pct; +extern unsigned int __weak sysctl_sched_conservative_pl; +extern unsigned int __weak sysctl_sched_walt_rotate_big_tasks; +extern unsigned int __weak sysctl_sched_min_task_util_for_boost; +extern unsigned int __weak sysctl_sched_min_task_util_for_colocation; +extern unsigned int __weak sysctl_sched_asym_cap_sibling_freq_match_pct; +extern unsigned int __weak sysctl_sched_coloc_downmigrate_ns; +extern unsigned int __weak sysctl_sched_task_unfilter_nr_windows; +extern unsigned int __weak sysctl_sched_busy_hyst_enable_cpus; +extern unsigned int __weak sysctl_sched_busy_hyst; +extern unsigned int __weak sysctl_sched_coloc_busy_hyst_enable_cpus; +extern unsigned int __weak sysctl_sched_coloc_busy_hyst; +extern unsigned int __weak sysctl_sched_coloc_busy_hyst_max_ms; +extern unsigned int __weak sysctl_sched_window_stats_policy; +extern unsigned int __weak sysctl_sched_ravg_window_nr_ticks; + +extern int __weak +walt_proc_group_thresholds_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); +extern int __weak +walt_proc_user_hint_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); + +extern int __weak +sched_ravg_window_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); +#endif enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, @@ -47,6 +83,10 @@ int sched_proc_update_handler(struct ctl_table *table, int write, loff_t *ppos); #endif +#ifdef CONFIG_SCHED_WALT +extern int __weak sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +#endif /* * control realtime throttling: * diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 6df477329b76..39d10afbdce6 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -73,6 +73,9 @@ extern int proc_do_large_bitmap(struct ctl_table *, int, extern int proc_do_static_key(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +extern int proc_douintvec_ravg_window(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos); /* * Register a set of sysctl names by calling register_sysctl_table diff --git a/include/linux/topology.h b/include/linux/topology.h index eb2fe6edd73c..831ad69caee8 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -214,6 +214,9 @@ static inline int cpu_to_mem(int cpu) #ifndef topology_die_cpumask #define topology_die_cpumask(cpu) cpumask_of(cpu) #endif +#ifndef topology_possible_sibling_cpumask +#define topology_possible_sibling_cpumask(cpu) cpumask_of(cpu) +#endif #ifdef CONFIG_SCHED_SMT static inline const struct cpumask *cpu_smt_mask(int cpu) diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 9fea72f19d85..6a4eeafd7797 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -51,6 +51,52 @@ TRACE_EVENT(sched_kthread_stop_ret, TP_printk("ret=%d", __entry->ret) ); +/* + * Tracepoint for task enqueue/dequeue: + */ +TRACE_EVENT(sched_enq_deq_task, + + TP_PROTO(struct task_struct *p, bool enqueue, + unsigned int cpus_allowed), + + TP_ARGS(p, enqueue, cpus_allowed), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(pid_t, pid) + __field(int, prio) + __field(int, cpu) + __field(bool, enqueue) + __field(unsigned int, nr_running) + __field(unsigned int, rt_nr_running) + __field(unsigned int, cpus_allowed) + __field(unsigned int, demand) + __field(unsigned int, pred_demand) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->cpu = task_cpu(p); + __entry->enqueue = enqueue; + __entry->nr_running = task_rq(p)->nr_running; + __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; + __entry->cpus_allowed = cpus_allowed; + __entry->demand = task_load(p); + __entry->pred_demand = task_pl(p); + ), + + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u rt_nr_running=%u affine=%x demand=%u pred_demand=%u", + __entry->cpu, + __entry->enqueue ? "enqueue" : "dequeue", + __entry->comm, __entry->pid, + __entry->prio, __entry->nr_running, + __entry->rt_nr_running, + __entry->cpus_allowed, __entry->demand, + __entry->pred_demand) +); + /* * Tracepoint for waking up a task: */ diff --git a/init/Kconfig b/init/Kconfig index e381021cc2a5..ec2127b69683 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -470,6 +470,15 @@ config HAVE_SCHED_AVG_IRQ depends on IRQ_TIME_ACCOUNTING || PARAVIRT_TIME_ACCOUNTING depends on SMP +config SCHED_WALT + bool "Support window based load tracking" + depends on SMP + help + This feature will allow the scheduler to maintain a tunable window + based set of metrics for tasks and runqueues. These metrics can be + used to guide task placement as well as task frequency requirements + for cpufreq governors. + config BSD_PROCESS_ACCT bool "BSD Process Accounting" depends on MULTIUSER diff --git a/init/init_task.c b/init/init_task.c index 9e5cbe5eab7b..af4dc7dcf246 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -91,6 +91,9 @@ struct task_struct init_task #endif #ifdef CONFIG_CGROUP_SCHED .sched_task_group = &root_task_group, +#endif +#ifdef CONFIG_SCHED_WALT + .wake_up_idle = false, #endif .ptraced = LIST_HEAD_INIT(init_task.ptraced), .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry), diff --git a/kernel/cpu.c b/kernel/cpu.c index 8512a3d690cd..5a3a429ea9cd 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -990,6 +990,9 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, if (!cpu_present(cpu)) return -EINVAL; + if (!tasks_frozen && !cpu_isolated(cpu) && num_online_uniso_cpus() == 1) + return -EBUSY; + cpus_write_lock(); if (trace_cpuhp_latency_enabled()) start_time = sched_clock(); @@ -2377,6 +2380,9 @@ EXPORT_SYMBOL(__cpu_present_mask); struct cpumask __cpu_active_mask __read_mostly; EXPORT_SYMBOL(__cpu_active_mask); +struct cpumask __cpu_isolated_mask __read_mostly; +EXPORT_SYMBOL(__cpu_isolated_mask); + atomic_t __num_online_cpus __read_mostly; EXPORT_SYMBOL(__num_online_cpus); @@ -2395,6 +2401,11 @@ void init_cpu_online(const struct cpumask *src) cpumask_copy(&__cpu_online_mask, src); } +void init_cpu_isolated(const struct cpumask *src) +{ + cpumask_copy(&__cpu_isolated_mask, src); +} + void set_cpu_online(unsigned int cpu, bool online) { /* diff --git a/kernel/exit.c b/kernel/exit.c index a46a50d67002..b26826cdd597 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -761,6 +761,7 @@ void __noreturn do_exit(long code) } exit_signals(tsk); /* sets PF_EXITING */ + sched_exit(tsk); /* * Ensure that all new tsk->pi_lock acquisitions must observe * PF_EXITING. Serializes against futex.c:attach_to_pi_owner(). diff --git a/kernel/fork.c b/kernel/fork.c index 1a38712da4cd..e549047d0990 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2283,6 +2283,7 @@ bad_fork_cleanup_perf: perf_event_free_task(p); bad_fork_cleanup_policy: lockdep_free_task(p); + free_task_load_ptrs(p); #ifdef CONFIG_NUMA mpol_put(p->mempolicy); bad_fork_cleanup_threadgroup_lock: diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index 6c7ca2e983a5..ef18c39f78c1 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "internals.h" @@ -57,6 +58,7 @@ static bool migrate_one_irq(struct irq_desc *desc) const struct cpumask *affinity; bool brokeaff = false; int err; + struct cpumask available_cpus; /* * IRQ chip might be already torn down, but the irq descriptor is @@ -109,6 +111,10 @@ static bool migrate_one_irq(struct irq_desc *desc) if (maskchip && chip->irq_mask) chip->irq_mask(d); + cpumask_copy(&available_cpus, affinity); + cpumask_andnot(&available_cpus, &available_cpus, cpu_isolated_mask); + affinity = &available_cpus; + if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) { /* * If the interrupt is managed, then shut it down and leave @@ -119,16 +125,41 @@ static bool migrate_one_irq(struct irq_desc *desc) irq_shutdown_and_deactivate(desc); return false; } - affinity = cpu_online_mask; + /* + * The order of preference for selecting a fallback CPU is + * + * (1) online and un-isolated CPU from default affinity + * (2) online and un-isolated CPU + * (3) online CPU + */ + cpumask_andnot(&available_cpus, cpu_online_mask, + cpu_isolated_mask); + if (cpumask_intersects(&available_cpus, irq_default_affinity)) + cpumask_and(&available_cpus, &available_cpus, + irq_default_affinity); + else if (cpumask_empty(&available_cpus)) + affinity = cpu_online_mask; + + /* + * We are overriding the affinity with all online and + * un-isolated cpus. irq_set_affinity_locked() call + * below notify this mask to PM QOS affinity listener. + * That results in applying the CPU_DMA_LATENCY QOS + * to all the CPUs specified in the mask. But the low + * level irqchip driver sets the affinity of an irq + * to only one CPU. So pick only one CPU from the + * prepared mask while overriding the user affinity. + */ + affinity = cpumask_of(cpumask_any(affinity)); brokeaff = true; } /* - * Do not set the force argument of irq_do_set_affinity() as this + * Do not set the force argument of irq_set_affinity_locked() as this * disables the masking of offline CPUs from the supplied affinity * mask and therefore might keep/reassign the irq to the outgoing * CPU. */ - err = irq_do_set_affinity(d, affinity, false); + err = irq_set_affinity_locked(d, affinity, false); if (err) { pr_warn_ratelimited("IRQ%u: set affinity failed(%d).\n", d->irq, err); diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index cfc4f088a0e7..47896c9831d3 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -132,6 +132,11 @@ static ssize_t write_irq_affinity(int type, struct file *file, if (err) goto free_cpumask; + if (cpumask_subset(new_value, cpu_isolated_mask)) { + err = -EINVAL; + goto free_cpumask; + } + /* * Do not allow disabling IRQs completely - it's a too easy * way to make the system unusable accidentally :-) At least diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d2b880f2ff09..4af0bd4d1524 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ #include "../smpboot.h" #include "pelt.h" +#include "walt.h" #define CREATE_TRACE_POINTS #include @@ -1298,6 +1300,8 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) uclamp_rq_inc(rq, p); p->sched_class->enqueue_task(rq, p, flags); + walt_update_last_enqueue(p); + trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_mask)[0]); } static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) @@ -1312,6 +1316,11 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) uclamp_rq_dec(rq, p); p->sched_class->dequeue_task(rq, p, flags); +#ifdef CONFIG_SCHED_WALT + if (p == rq->ed_task) + early_detection_notify(rq, sched_ktime_clock()); +#endif + trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]); } void activate_task(struct rq *rq, struct task_struct *p, int flags) @@ -1331,6 +1340,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) if (task_contributes_to_load(p)) rq->nr_uninterruptible++; +#ifdef CONFIG_SCHED_WALT + if (flags & DEQUEUE_SLEEP) + clear_ed_task(p, rq); +#endif + dequeue_task(rq, p, flags); } @@ -1492,8 +1506,11 @@ static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING); dequeue_task(rq, p, DEQUEUE_NOCLOCK); + double_lock_balance(rq, cpu_rq(new_cpu)); + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); set_task_cpu(p, new_cpu); - rq_unlock(rq, rf); + double_rq_unlock(cpu_rq(new_cpu), rq); rq = cpu_rq(new_cpu); @@ -1633,6 +1650,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, struct rq_flags rf; struct rq *rq; int ret = 0; + cpumask_t allowed_mask; rq = task_rq_lock(p, &rf); update_rq_clock(rq); @@ -1656,10 +1674,17 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, if (cpumask_equal(p->cpus_ptr, new_mask)) goto out; - dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask); + cpumask_andnot(&allowed_mask, new_mask, cpu_isolated_mask); + cpumask_and(&allowed_mask, &allowed_mask, cpu_valid_mask); + + dest_cpu = cpumask_any(&allowed_mask); if (dest_cpu >= nr_cpu_ids) { - ret = -EINVAL; - goto out; + cpumask_and(&allowed_mask, cpu_valid_mask, new_mask); + dest_cpu = cpumask_any(&allowed_mask); + if (!cpumask_intersects(new_mask, cpu_valid_mask)) { + ret = -EINVAL; + goto out; + } } do_set_cpus_allowed(p, new_mask); @@ -1675,7 +1700,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, } /* Can the task run on the task's current CPU? If so, we're done */ - if (cpumask_test_cpu(task_cpu(p), new_mask)) + if (cpumask_test_cpu(task_cpu(p), &allowed_mask)) goto out; if (task_running(rq, p) || p->state == TASK_WAKING) { @@ -1750,12 +1775,13 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) p->se.nr_migrations++; rseq_migrate(p); perf_event_task_migrate(p); + + fixup_busy_time(p, new_cpu); } __set_task_cpu(p, new_cpu); } -#ifdef CONFIG_NUMA_BALANCING static void __migrate_swap_task(struct task_struct *p, int cpu) { if (task_on_rq_queued(p)) { @@ -1870,7 +1896,6 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p, out: return ret; } -#endif /* CONFIG_NUMA_BALANCING */ /* * wait_task_inactive - wait for a thread to unschedule. @@ -2027,12 +2052,13 @@ EXPORT_SYMBOL_GPL(kick_process); * select_task_rq() below may allow selection of !active CPUs in order * to satisfy the above rules. */ -static int select_fallback_rq(int cpu, struct task_struct *p) +static int select_fallback_rq(int cpu, struct task_struct *p, bool allow_iso) { int nid = cpu_to_node(cpu); const struct cpumask *nodemask = NULL; - enum { cpuset, possible, fail } state = cpuset; + enum { cpuset, possible, fail, bug } state = cpuset; int dest_cpu; + int isolated_candidate = -1; /* * If the node that the CPU is on has been offlined, cpu_to_node() @@ -2046,6 +2072,8 @@ static int select_fallback_rq(int cpu, struct task_struct *p) for_each_cpu(dest_cpu, nodemask) { if (!cpu_active(dest_cpu)) continue; + if (cpu_isolated(dest_cpu)) + continue; if (cpumask_test_cpu(dest_cpu, p->cpus_ptr)) return dest_cpu; } @@ -2056,7 +2084,16 @@ static int select_fallback_rq(int cpu, struct task_struct *p) for_each_cpu(dest_cpu, p->cpus_ptr) { if (!is_cpu_allowed(p, dest_cpu)) continue; + if (cpu_isolated(dest_cpu)) { + if (allow_iso) + isolated_candidate = dest_cpu; + continue; + } + goto out; + } + if (isolated_candidate != -1) { + dest_cpu = isolated_candidate; goto out; } @@ -2075,6 +2112,11 @@ static int select_fallback_rq(int cpu, struct task_struct *p) break; case fail: + allow_iso = true; + state = bug; + break; + + case bug: BUG(); break; } @@ -2102,6 +2144,8 @@ out: static inline int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) { + bool allow_isolated = (p->flags & PF_KTHREAD); + lockdep_assert_held(&p->pi_lock); if (p->nr_cpus_allowed > 1) @@ -2119,8 +2163,9 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) * [ this allows ->select_task() to simply return task_cpu(p) and * not worry about this generic constraint ] */ - if (unlikely(!is_cpu_allowed(p, cpu))) - cpu = select_fallback_rq(task_cpu(p), p); + if (unlikely(!is_cpu_allowed(p, cpu)) || + (cpu_isolated(cpu) && !allow_isolated)) + cpu = select_fallback_rq(task_cpu(p), p, allow_isolated); return cpu; } @@ -2311,6 +2356,7 @@ void sched_ttwu_pending(void) void scheduler_ipi(void) { + int cpu = smp_processor_id(); /* * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting * TIF_NEED_RESCHED remotely (for the first time) will also send @@ -2340,7 +2386,7 @@ void scheduler_ipi(void) /* * Check if someone kicked us for doing the nohz idle load balance. */ - if (unlikely(got_nohz_idle_kick())) { + if (unlikely(got_nohz_idle_kick()) && !cpu_isolated(cpu)) { this_rq()->idle_balance = 1; raise_softirq_irqoff(SCHED_SOFTIRQ); } @@ -2616,6 +2662,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); + walt_try_to_wake_up(p); + p->sched_contributes_to_load = !!task_contributes_to_load(p); p->state = TASK_WAKING; @@ -2644,6 +2692,17 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) unlock: raw_spin_unlock_irqrestore(&p->pi_lock, flags); out: +#ifdef CONFIG_SCHED_WALT + if (success && sched_predl) { + raw_spin_lock_irqsave(&cpu_rq(cpu)->lock, flags); + if (do_pl_notif(cpu_rq(cpu))) + cpufreq_update_util(cpu_rq(cpu), + SCHED_CPUFREQ_WALT | + SCHED_CPUFREQ_PL); + raw_spin_unlock_irqrestore(&cpu_rq(cpu)->lock, flags); + } +#endif + if (success) ttwu_stat(p, cpu, wake_flags); preempt_enable(); @@ -2689,6 +2748,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->se.prev_sum_exec_runtime = 0; p->se.nr_migrations = 0; p->se.vruntime = 0; +#ifdef CONFIG_SCHED_WALT + p->last_sleep_ts = 0; + p->wake_up_idle = false; +#endif INIT_LIST_HEAD(&p->se.group_node); #ifdef CONFIG_FAIR_GROUP_SCHED @@ -2840,6 +2903,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) { unsigned long flags; + init_new_task_load(p); __sched_fork(clone_flags, p); /* * We mark the process as NEW here. This guarantees that @@ -2945,7 +3009,9 @@ void wake_up_new_task(struct task_struct *p) struct rq_flags rf; struct rq *rq; + add_new_task_to_grp(p); raw_spin_lock_irqsave(&p->pi_lock, rf.flags); + p->state = TASK_RUNNING; #ifdef CONFIG_SMP /* @@ -2963,6 +3029,7 @@ void wake_up_new_task(struct task_struct *p) update_rq_clock(rq); post_init_entity_util_avg(p); + mark_task_starting(p); activate_task(rq, p, ENQUEUE_NOCLOCK); trace_sched_wakeup_new(p); check_preempt_curr(rq, p, WF_FORK); @@ -3497,12 +3564,15 @@ void sched_exec(void) unsigned long flags; int dest_cpu; + if (sched_energy_enabled()) + return; + raw_spin_lock_irqsave(&p->pi_lock, flags); dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); if (dest_cpu == smp_processor_id()) goto unlock; - if (likely(cpu_active(dest_cpu))) { + if (likely(cpu_active(dest_cpu) && likely(!cpu_isolated(dest_cpu)))) { struct migration_arg arg = { p, dest_cpu }; raw_spin_unlock_irqrestore(&p->pi_lock, flags); @@ -3592,16 +3662,30 @@ void scheduler_tick(void) struct rq *rq = cpu_rq(cpu); struct task_struct *curr = rq->curr; struct rq_flags rf; + u64 wallclock; + bool early_notif; + u32 old_load; + struct related_thread_group *grp; + unsigned int flag = 0; sched_clock_tick(); rq_lock(rq, &rf); + old_load = task_load(curr); + set_window_start(rq); + wallclock = sched_ktime_clock(); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); update_rq_clock(rq); curr->sched_class->task_tick(rq, curr, 0); calc_global_load_tick(rq); psi_task_tick(rq); + early_notif = early_detection_notify(rq, wallclock); + if (early_notif) + flag = SCHED_CPUFREQ_WALT | SCHED_CPUFREQ_EARLY_DET; + + cpufreq_update_util(rq, flag); rq_unlock(rq, &rf); perf_event_task_tick(); @@ -3610,6 +3694,15 @@ void scheduler_tick(void) rq->idle_balance = idle_cpu(cpu); trigger_load_balance(rq); #endif + + rcu_read_lock(); + grp = task_related_thread_group(curr); + if (update_preferred_cluster(grp, curr, old_load, true)) + set_preferred_cluster(grp); + rcu_read_unlock(); + + if (curr->sched_class == &fair_sched_class) + check_for_migration(rq, curr); } #ifdef CONFIG_NO_HZ_FULL @@ -4005,6 +4098,7 @@ static void __sched notrace __schedule(bool preempt) struct rq_flags rf; struct rq *rq; int cpu; + u64 wallclock; cpu = smp_processor_id(); rq = cpu_rq(cpu); @@ -4052,7 +4146,15 @@ static void __sched notrace __schedule(bool preempt) clear_tsk_need_resched(prev); clear_preempt_need_resched(); + wallclock = sched_ktime_clock(); if (likely(prev != next)) { +#ifdef CONFIG_SCHED_WALT + if (!prev->on_rq) + prev->last_sleep_ts = wallclock; +#endif + + walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); + walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); rq->nr_switches++; /* * RCU users of rcu_dereference(rq->curr) may not see @@ -4080,6 +4182,7 @@ static void __sched notrace __schedule(bool preempt) /* Also unlocks the rq: */ rq = context_switch(rq, prev, next, &rf); } else { + walt_update_task_ravg(prev, rq, TASK_UPDATE, wallclock, 0); rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP); rq_unlock_irq(rq, &rf); } @@ -4669,7 +4772,7 @@ struct task_struct *idle_task(int cpu) * * The task of @pid, if found. %NULL otherwise. */ -static struct task_struct *find_process_by_pid(pid_t pid) +struct task_struct *find_process_by_pid(pid_t pid) { return pid ? find_task_by_vpid(pid) : current; } @@ -5390,6 +5493,8 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) cpumask_var_t cpus_allowed, new_mask; struct task_struct *p; int retval; + int dest_cpu; + cpumask_t allowed_mask; rcu_read_lock(); @@ -5451,20 +5556,26 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) } #endif again: - retval = __set_cpus_allowed_ptr(p, new_mask, true); - - if (!retval) { - cpuset_cpus_allowed(p, cpus_allowed); - if (!cpumask_subset(new_mask, cpus_allowed)) { - /* - * We must have raced with a concurrent cpuset - * update. Just reset the cpus_allowed to the - * cpuset's cpus_allowed - */ - cpumask_copy(new_mask, cpus_allowed); - goto again; + cpumask_andnot(&allowed_mask, new_mask, cpu_isolated_mask); + dest_cpu = cpumask_any_and(cpu_active_mask, &allowed_mask); + if (dest_cpu < nr_cpu_ids) { + retval = __set_cpus_allowed_ptr(p, new_mask, true); + if (!retval) { + cpuset_cpus_allowed(p, cpus_allowed); + if (!cpumask_subset(new_mask, cpus_allowed)) { + /* + * We must have raced with a concurrent cpuset + * update. Just reset the cpus_allowed to the + * cpuset's cpus_allowed + */ + cpumask_copy(new_mask, cpus_allowed); + goto again; + } } + } else { + retval = -EINVAL; } + out_free_new_mask: free_cpumask_var(new_mask); out_free_cpus_allowed: @@ -5582,6 +5693,14 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask) raw_spin_lock_irqsave(&p->pi_lock, flags); cpumask_and(mask, &p->cpus_mask, cpu_active_mask); + + /* The userspace tasks are forbidden to run on + * isolated CPUs. So exclude isolated CPUs from + * the getaffinity. + */ + if (!(p->flags & PF_KTHREAD)) + cpumask_andnot(mask, mask, cpu_isolated_mask); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); out_unlock: @@ -6253,7 +6372,7 @@ void idle_task_exit(void) * * Also see the comment "Global load-average calculations". */ -static void calc_load_migrate(struct rq *rq) +void calc_load_migrate(struct rq *rq) { long delta = calc_load_fold_active(rq, 1); if (delta) @@ -6278,19 +6397,25 @@ static struct task_struct *__pick_migrate_task(struct rq *rq) } /* - * Migrate all tasks from the rq, sleeping tasks will be migrated by - * try_to_wake_up()->select_task_rq(). + * Migrate all tasks (not pinned if pinned argument say so) from the rq, + * sleeping tasks will be migrated by try_to_wake_up()->select_task_rq(). * * Called with rq->lock held even though we'er in stop_machine() and * there's no concurrency possible, we hold the required locks anyway * because of lock validation efforts. */ -static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) +void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, + bool migrate_pinned_tasks) { struct rq *rq = dead_rq; struct task_struct *next, *stop = rq->stop; struct rq_flags orf = *rf; int dest_cpu; + unsigned int num_pinned_kthreads = 1; /* this thread */ + LIST_HEAD(tasks); + cpumask_t avail_cpus; + + cpumask_andnot(&avail_cpus, cpu_online_mask, cpu_isolated_mask); /* * Fudge the rq selection such that the below task selection loop @@ -6313,13 +6438,20 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) for (;;) { /* * There's this thread running, bail when that's the only - * remaining thread: + * remaining thread. */ if (rq->nr_running == 1) break; next = __pick_migrate_task(rq); + if (!migrate_pinned_tasks && next->flags & PF_KTHREAD && + !cpumask_intersects(&avail_cpus, &next->cpus_mask)) { + detach_one_task_core(next, rq, &tasks); + num_pinned_kthreads += 1; + continue; + } + /* * Rules for changing task_struct::cpus_mask are holding * both pi_lock and rq->lock, such that holding either @@ -6332,31 +6464,43 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf) rq_unlock(rq, rf); raw_spin_lock(&next->pi_lock); rq_relock(rq, rf); + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); /* * Since we're inside stop-machine, _nothing_ should have * changed the task, WARN if weird stuff happened, because in * that case the above rq->lock drop is a fail too. + * However, during cpu isolation the load balancer might have + * interferred since we don't stop all CPUs. Ignore warning for + * this case. */ - if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) { + if (task_rq(next) != rq || !task_on_rq_queued(next)) { + WARN_ON(migrate_pinned_tasks); raw_spin_unlock(&next->pi_lock); continue; } /* Find suitable destination for @next, with force if needed. */ - dest_cpu = select_fallback_rq(dead_rq->cpu, next); + dest_cpu = select_fallback_rq(dead_rq->cpu, next, false); rq = __migrate_task(rq, rf, next, dest_cpu); if (rq != dead_rq) { rq_unlock(rq, rf); rq = dead_rq; *rf = orf; rq_relock(rq, rf); + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); } raw_spin_unlock(&next->pi_lock); } rq->stop = stop; + + if (num_pinned_kthreads > 1) + attach_tasks_core(&tasks, rq); } + #endif /* CONFIG_HOTPLUG_CPU */ void set_rq_online(struct rq *rq) @@ -6512,6 +6656,11 @@ int sched_cpu_deactivate(unsigned int cpu) static void sched_rq_cpu_starting(unsigned int cpu) { struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + raw_spin_lock_irqsave(&rq->lock, flags); + set_window_start(rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); rq->calc_load_update = calc_load_update; update_max_interval(); @@ -6521,6 +6670,7 @@ int sched_cpu_starting(unsigned int cpu) { sched_rq_cpu_starting(cpu); sched_tick_start(cpu); + clear_walt_request(cpu); return 0; } @@ -6535,14 +6685,17 @@ int sched_cpu_dying(unsigned int cpu) sched_tick_stop(cpu); rq_lock_irqsave(rq, &rf); + if (rq->rd) { BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); set_rq_offline(rq); } - migrate_tasks(rq, &rf); + migrate_tasks(rq, &rf, true); BUG_ON(rq->nr_running != 1); rq_unlock_irqrestore(rq, &rf); + clear_walt_request(cpu); + calc_load_migrate(rq); update_max_interval(); nohz_balance_exit_idle(rq); @@ -6564,6 +6717,8 @@ void __init sched_init_smp(void) sched_init_domains(cpu_active_mask); mutex_unlock(&sched_domains_mutex); + update_cluster_topology(); + /* Move init over to a non-isolated CPU */ if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0) BUG(); @@ -6618,6 +6773,8 @@ void __init sched_init(void) wait_bit_init(); + init_clusters(); + #ifdef CONFIG_FAIR_GROUP_SCHED ptr += 2 * nr_cpu_ids * sizeof(void **); #endif @@ -6729,6 +6886,7 @@ void __init sched_init(void) rq->idle_stamp = 0; rq->avg_idle = 2*sysctl_sched_migration_cost; rq->max_idle_balance_cost = sysctl_sched_migration_cost; + walt_sched_init_rq(rq); INIT_LIST_HEAD(&rq->cfs_tasks); @@ -6743,6 +6901,8 @@ void __init sched_init(void) atomic_set(&rq->nr_iowait, 0); } + BUG_ON(alloc_related_thread_groups()); + set_load_weight(&init_task, false); /* @@ -6758,6 +6918,7 @@ void __init sched_init(void) * when this runqueue becomes "idle". */ init_idle(current, smp_processor_id()); + init_new_task_load(current); calc_load_update = jiffies + LOAD_FREQ; @@ -6972,6 +7133,97 @@ void ia64_set_curr_task(int cpu, struct task_struct *p) /* task_group_lock serializes the addition/removal of task groups */ static DEFINE_SPINLOCK(task_group_lock); +#if defined(CONFIG_SCHED_WALT) && defined(CONFIG_UCLAMP_TASK_GROUP) +static inline void walt_init_sched_boost(struct task_group *tg) +{ + tg->sched_boost_no_override = false; + tg->sched_boost_enabled = true; + tg->colocate = false; + tg->colocate_update_disabled = false; +} + +void update_cgroup_boost_settings(void) +{ + struct task_group *tg; + + rcu_read_lock(); + list_for_each_entry_rcu(tg, &task_groups, list) { + if (tg->sched_boost_no_override) + continue; + + tg->sched_boost_enabled = false; + } + rcu_read_unlock(); +} + +void restore_cgroup_boost_settings(void) +{ + struct task_group *tg; + + rcu_read_lock(); + list_for_each_entry_rcu(tg, &task_groups, list) + tg->sched_boost_enabled = true; + rcu_read_unlock(); +} + +static void walt_schedgp_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + bool colocate; + + cgroup_taskset_first(tset, &css); + tg = css_tg(css); + + colocate = tg->colocate; + + cgroup_taskset_for_each(task, css, tset) + sync_cgroup_colocation(task, colocate); +} + +static u64 +sched_boost_override_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return (u64) tg->sched_boost_no_override; +} + +static int sched_boost_override_write(struct cgroup_subsys_state *css, + struct cftype *cft, u64 override) +{ + struct task_group *tg = css_tg(css); + + tg->sched_boost_no_override = !!override; + return 0; +} + +static u64 sched_colocate_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return (u64) tg->colocate; +} + +static int sched_colocate_write(struct cgroup_subsys_state *css, + struct cftype *cft, u64 colocate) +{ + struct task_group *tg = css_tg(css); + + if (tg->colocate_update_disabled) + return -EPERM; + + tg->colocate = !!colocate; + tg->colocate_update_disabled = true; + return 0; +} +#else +static inline void walt_init_sched_boost(struct task_group *tg) { } +static void walt_schedgp_attach(struct cgroup_taskset *tset) { } +#endif /* CONFIG_SCHED_WALT */ + static inline void alloc_uclamp_sched_group(struct task_group *tg, struct task_group *parent) { @@ -7139,6 +7391,7 @@ cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) if (IS_ERR(tg)) return ERR_PTR(-ENOMEM); + walt_init_sched_boost(tg); return &tg->css; } @@ -7225,6 +7478,8 @@ static void cpu_cgroup_attach(struct cgroup_taskset *tset) cgroup_taskset_for_each(task, css, tset) sched_move_task(task); + + walt_schedgp_attach(tset); } #ifdef CONFIG_UCLAMP_TASK_GROUP @@ -7784,7 +8039,21 @@ static struct cftype cpu_legacy_files[] = { .read_u64 = cpu_uclamp_ls_read_u64, .write_u64 = cpu_uclamp_ls_write_u64, }, -#endif +#ifdef CONFIG_SCHED_WALT + { + .name = "uclamp.sched_boost_no_override", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = sched_boost_override_read, + .write_u64 = sched_boost_override_write, + }, + { + .name = "uclamp.colocate", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = sched_colocate_read, + .write_u64 = sched_colocate_write, + }, +#endif /* CONFIG_SCHED_WALT */ +#endif /* CONFIG_UCLAMP_TASK_GROUP */ { } /* Terminate */ }; @@ -7971,7 +8240,21 @@ static struct cftype cpu_files[] = { .read_u64 = cpu_uclamp_ls_read_u64, .write_u64 = cpu_uclamp_ls_write_u64, }, -#endif +#ifdef CONFIG_SCHED_WALT + { + .name = "uclamp.sched_boost_no_override", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = sched_boost_override_read, + .write_u64 = sched_boost_override_write, + }, + { + .name = "uclamp.colocate", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = sched_colocate_read, + .write_u64 = sched_colocate_write, + }, +#endif /* CONFIG_SCHED_WALT */ +#endif /* CONFIG_UCLAMP_TASK_GROUP */ { } /* terminate */ }; @@ -8040,3 +8323,57 @@ const u32 sched_prio_to_wmult[40] = { }; #undef CREATE_TRACE_POINTS + +__read_mostly bool sched_predl = 1; + +void enqueue_task_core(struct rq *rq, struct task_struct *p, int flags) +{ + enqueue_task(rq, p, 0); +} + +void dequeue_task_core(struct rq *rq, struct task_struct *p, int flags) +{ + dequeue_task(rq, p, 0); +} + +#ifdef CONFIG_SCHED_WALT +void sched_account_irqtime(int cpu, struct task_struct *curr, + u64 delta, u64 wallclock) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags, nr_windows; + u64 cur_jiffies_ts; + + raw_spin_lock_irqsave(&rq->lock, flags); + + /* + * cputime (wallclock) uses sched_clock so use the same here for + * consistency. + */ + delta += sched_clock() - wallclock; + cur_jiffies_ts = get_jiffies_64(); + + if (is_idle_task(curr)) + walt_update_task_ravg(curr, rq, IRQ_UPDATE, sched_ktime_clock(), + delta); + + nr_windows = cur_jiffies_ts - rq->irqload_ts; + + if (nr_windows) { + if (nr_windows < 10) { + /* Decay CPU's irqload by 3/4 for each window. */ + rq->avg_irqload *= (3 * nr_windows); + rq->avg_irqload = div64_u64(rq->avg_irqload, + 4 * nr_windows); + } else { + rq->avg_irqload = 0; + } + rq->avg_irqload += rq->cur_irqload; + rq->cur_irqload = 0; + } + + rq->cur_irqload += delta; + rq->irqload_ts = cur_jiffies_ts; + raw_spin_unlock_irqrestore(&rq->lock, flags); +} +#endif diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index fc49c1e169d9..c42e77415e4f 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -4,6 +4,7 @@ */ #include #include "sched.h" +#include "walt.h" #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -53,11 +54,18 @@ void irqtime_account_irq(struct task_struct *curr) struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); s64 delta; int cpu; +#ifdef CONFIG_SCHED_WALT + u64 wallclock; + bool account = true; +#endif if (!sched_clock_irqtime) return; cpu = smp_processor_id(); +#ifdef CONFIG_SCHED_WALT + wallclock = sched_clock_cpu(cpu); +#endif delta = sched_clock_cpu(cpu) - irqtime->irq_start_time; irqtime->irq_start_time += delta; @@ -71,6 +79,15 @@ void irqtime_account_irq(struct task_struct *curr) irqtime_account_delta(irqtime, delta, CPUTIME_IRQ); else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ); +#ifdef CONFIG_SCHED_WALT + else + account = false; + + if (account) + sched_account_irqtime(cpu, curr, delta, wallclock); + else if (curr != this_cpu_ksoftirqd()) + sched_account_irqstart(cpu, curr, wallclock); +#endif } EXPORT_SYMBOL_GPL(irqtime_account_irq); diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index a8a08030a8f7..0947361dc7d0 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -17,6 +17,7 @@ */ #include "sched.h" #include "pelt.h" +#include "walt.h" struct dl_bandwidth def_dl_bandwidth; @@ -1380,6 +1381,7 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) WARN_ON(!dl_prio(prio)); dl_rq->dl_nr_running++; add_nr_running(rq_of_dl_rq(dl_rq), 1); + walt_inc_cumulative_runnable_avg(rq_of_dl_rq(dl_rq), dl_task_of(dl_se)); inc_dl_deadline(dl_rq, deadline); inc_dl_migration(dl_se, dl_rq); @@ -1394,6 +1396,7 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) WARN_ON(!dl_rq->dl_nr_running); dl_rq->dl_nr_running--; sub_nr_running(rq_of_dl_rq(dl_rq), 1); + walt_dec_cumulative_runnable_avg(rq_of_dl_rq(dl_rq), dl_task_of(dl_se)); dec_dl_deadline(dl_rq, dl_se->deadline); dec_dl_migration(dl_se, dl_rq); @@ -2101,7 +2104,9 @@ retry: } deactivate_task(rq, next_task, 0); + next_task->on_rq = TASK_ON_RQ_MIGRATING; set_task_cpu(next_task, later_rq->cpu); + next_task->on_rq = TASK_ON_RQ_QUEUED; /* * Update the later_rq clock here, because the clock is used @@ -2195,7 +2200,9 @@ static void pull_dl_task(struct rq *this_rq) resched = true; deactivate_task(src_rq, p, 0); + p->on_rq = TASK_ON_RQ_MIGRATING; set_task_cpu(p, this_cpu); + p->on_rq = TASK_ON_RQ_QUEUED; activate_task(this_rq, p, 0); dmin = p->dl.deadline; @@ -2458,6 +2465,9 @@ const struct sched_class dl_sched_class = { .switched_to = switched_to_dl, .update_curr = update_curr_dl, +#ifdef CONFIG_SCHED_WALT + .fixup_walt_sched_stats = fixup_walt_sched_stats_common, +#endif }; int sched_dl_global_validate(void) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index f7e4579e746c..d7959f96820f 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -646,6 +646,19 @@ do { \ SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr))); PN(clock); PN(clock_task); +#ifdef CONFIG_SMP + P(cpu_capacity); +#endif +#ifdef CONFIG_SCHED_WALT + P(cluster->max_possible_capacity); + P(cluster->efficiency); + P(cluster->cur_freq); + P(cluster->max_freq); + P(cluster->exec_scale_factor); + P(walt_stats.nr_big_tasks); + SEQ_printf(m, " .%-30s: %llu\n", "walt_stats.cumulative_runnable_avg", + rq->walt_stats.cumulative_runnable_avg_scaled); +#endif #undef P #undef PN @@ -724,6 +737,11 @@ static void sched_debug_header(struct seq_file *m) PN(sysctl_sched_wakeup_granularity); P(sysctl_sched_child_runs_first); P(sysctl_sched_features); +#ifdef CONFIG_SCHED_WALT + P(sched_init_task_load_windows); + P(sched_ravg_window); + P(sched_load_granule); +#endif #undef PN #undef P @@ -915,6 +933,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(se.statistics.nr_wakeups_passive); P_SCHEDSTAT(se.statistics.nr_wakeups_idle); +#ifdef CONFIG_SCHED_WALT + P(ravg.demand); +#endif avg_atom = p->se.sum_exec_runtime; if (nr_switches) avg_atom = div64_ul(avg_atom, nr_switches); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index dc342f8094c2..8d36c9cc28f8 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -24,6 +24,12 @@ #include +#include "walt.h" + +#ifdef CONFIG_SMP +static inline bool task_fits_max(struct task_struct *p, int cpu); +#endif /* CONFIG_SMP */ + /* * Targeted preemption latency for CPU-bound tasks: * @@ -85,6 +91,7 @@ unsigned int sysctl_sched_wakeup_granularity = 1000000UL; static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL; const_debug unsigned int sysctl_sched_migration_cost = 500000UL; +DEFINE_PER_CPU_READ_MOSTLY(int, sched_load_boost); #ifdef CONFIG_SMP /* @@ -118,6 +125,8 @@ int __weak arch_asym_cpu_priority(int cpu) unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +unsigned int sched_small_task_threshold = 102; + static inline void update_load_add(struct load_weight *lw, unsigned long inc) { lw->weight += inc; @@ -3689,11 +3698,6 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) return cfs_rq->avg.load_avg; } -static inline unsigned long task_util(struct task_struct *p) -{ - return READ_ONCE(p->se.avg.util_avg); -} - static inline unsigned long _task_util_est(struct task_struct *p) { struct util_est ue = READ_ONCE(p->se.avg.util_est); @@ -3703,6 +3707,9 @@ static inline unsigned long _task_util_est(struct task_struct *p) static inline unsigned long task_util_est(struct task_struct *p) { +#ifdef CONFIG_SCHED_WALT + return p->ravg.demand_scaled; +#endif return max(task_util(p), _task_util_est(p)); } @@ -4514,13 +4521,16 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); qcfs_rq->h_nr_running -= task_delta; qcfs_rq->idle_h_nr_running -= idle_task_delta; + walt_dec_throttled_cfs_rq_stats(&qcfs_rq->walt_stats, cfs_rq); if (qcfs_rq->load.weight) dequeue = 0; } - if (!se) + if (!se) { sub_nr_running(rq, task_delta); + walt_dec_throttled_cfs_rq_stats(&rq->walt_stats, cfs_rq); + } cfs_rq->throttled = 1; cfs_rq->throttled_clock = rq_clock(rq); @@ -4554,6 +4564,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) struct sched_entity *se; int enqueue = 1; long task_delta, idle_task_delta; + struct cfs_rq *tcfs_rq __maybe_unused = cfs_rq; se = cfs_rq->tg->se[cpu_of(rq)]; @@ -4583,6 +4594,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); cfs_rq->h_nr_running += task_delta; cfs_rq->idle_h_nr_running += idle_task_delta; + walt_inc_throttled_cfs_rq_stats(&cfs_rq->walt_stats, tcfs_rq); if (cfs_rq_throttled(cfs_rq)) break; @@ -4590,8 +4602,10 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) assert_list_leaf_cfs_rq(rq); - if (!se) + if (!se) { add_nr_running(rq, task_delta); + walt_inc_throttled_cfs_rq_stats(&rq->walt_stats, tcfs_rq); + } /* Determine whether we need to wake up potentially idle CPU: */ if (rq->curr == rq->idle && rq->cfs.nr_running) @@ -4982,6 +4996,7 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) { cfs_rq->runtime_enabled = 0; INIT_LIST_HEAD(&cfs_rq->throttled_list); + walt_init_cfs_rq_stats(cfs_rq); } void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) @@ -5161,8 +5176,6 @@ static inline void hrtick_update(struct rq *rq) #endif #ifdef CONFIG_SMP -static inline unsigned long cpu_util(int cpu); - static inline bool cpu_overutilized(int cpu) { return !fits_capacity(cpu_util(cpu), capacity_of(cpu)); @@ -5223,6 +5236,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) break; cfs_rq->h_nr_running++; cfs_rq->idle_h_nr_running += idle_h_nr_running; + walt_inc_cfs_rq_stats(cfs_rq, p); flags = ENQUEUE_WAKEUP; } @@ -5231,6 +5245,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running++; cfs_rq->idle_h_nr_running += idle_h_nr_running; + walt_inc_cfs_rq_stats(cfs_rq, p); if (cfs_rq_throttled(cfs_rq)) break; @@ -5241,6 +5256,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!se) { add_nr_running(rq, 1); + inc_rq_walt_stats(rq, p); /* * Since new tasks are assigned an initial util_avg equal to * half of the spare capacity of their CPU, tiny tasks have the @@ -5308,6 +5324,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) break; cfs_rq->h_nr_running--; cfs_rq->idle_h_nr_running -= idle_h_nr_running; + walt_dec_cfs_rq_stats(cfs_rq, p); /* Don't dequeue parent if it has other entities besides us */ if (cfs_rq->load.weight) { @@ -5328,6 +5345,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running--; cfs_rq->idle_h_nr_running -= idle_h_nr_running; + walt_dec_cfs_rq_stats(cfs_rq, p); if (cfs_rq_throttled(cfs_rq)) break; @@ -5336,8 +5354,10 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_cfs_group(se); } - if (!se) + if (!se) { sub_nr_running(rq, 1); + dec_rq_walt_stats(rq, p); + } util_est_dequeue(&rq->cfs, p, task_sleep); hrtick_update(rq); @@ -5375,11 +5395,6 @@ static unsigned long cpu_runnable_load(struct rq *rq) return cfs_rq_runnable_load_avg(&rq->cfs); } -static unsigned long capacity_of(int cpu) -{ - return cpu_rq(cpu)->cpu_capacity; -} - static unsigned long cpu_avg_load_per_task(int cpu) { struct rq *rq = cpu_rq(cpu); @@ -5968,6 +5983,8 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t return si_cpu; if (!cpumask_test_cpu(cpu, p->cpus_ptr)) continue; + if (cpu_isolated(cpu)) + continue; if (available_idle_cpu(cpu)) break; if (si_cpu == -1 && sched_idle_cpu(cpu)) @@ -5990,14 +6007,16 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) struct sched_domain *sd; int i, recent_used_cpu; - if (available_idle_cpu(target) || sched_idle_cpu(target)) + if ((available_idle_cpu(target) || sched_idle_cpu(target)) && + !cpu_isolated(target)) return target; /* * If the previous CPU is cache affine and idle, don't be stupid: */ if (prev != target && cpus_share_cache(prev, target) && - (available_idle_cpu(prev) || sched_idle_cpu(prev))) + ((available_idle_cpu(prev) || sched_idle_cpu(prev)) && + !cpu_isolated(prev))) return prev; /* Check a recently used CPU as a potential idle candidate: */ @@ -6047,58 +6066,6 @@ static unsigned int uclamp_task_util(struct task_struct *p) #endif } -/** - * Amount of capacity of a CPU that is (estimated to be) used by CFS tasks - * @cpu: the CPU to get the utilization of - * - * The unit of the return value must be the one of capacity so we can compare - * the utilization with the capacity of the CPU that is available for CFS task - * (ie cpu_capacity). - * - * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the - * recent utilization of currently non-runnable tasks on a CPU. It represents - * the amount of utilization of a CPU in the range [0..capacity_orig] where - * capacity_orig is the cpu_capacity available at the highest frequency - * (arch_scale_freq_capacity()). - * The utilization of a CPU converges towards a sum equal to or less than the - * current capacity (capacity_curr <= capacity_orig) of the CPU because it is - * the running time on this CPU scaled by capacity_curr. - * - * The estimated utilization of a CPU is defined to be the maximum between its - * cfs_rq.avg.util_avg and the sum of the estimated utilization of the tasks - * currently RUNNABLE on that CPU. - * This allows to properly represent the expected utilization of a CPU which - * has just got a big task running since a long sleep period. At the same time - * however it preserves the benefits of the "blocked utilization" in - * describing the potential for other tasks waking up on the same CPU. - * - * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even - * higher than capacity_orig because of unfortunate rounding in - * cfs.avg.util_avg or just after migrating tasks and new task wakeups until - * the average stabilizes with the new running time. We need to check that the - * utilization stays within the range of [0..capacity_orig] and cap it if - * necessary. Without utilization capping, a group could be seen as overloaded - * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of - * available capacity. We allow utilization to overshoot capacity_curr (but not - * capacity_orig) as it useful for predicting the capacity required after task - * migrations (scheduler-driven DVFS). - * - * Return: the (estimated) utilization for the specified CPU - */ -static inline unsigned long cpu_util(int cpu) -{ - struct cfs_rq *cfs_rq; - unsigned int util; - - cfs_rq = &cpu_rq(cpu)->cfs; - util = READ_ONCE(cfs_rq->avg.util_avg); - - if (sched_feat(UTIL_EST)) - util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); - - return min_t(unsigned long, util, capacity_orig_of(cpu)); -} - /* * cpu_util_without: compute cpu utilization without any contributions from *p * @cpu: the CPU which utilization is requested @@ -6114,13 +6081,29 @@ static inline unsigned long cpu_util(int cpu) */ static unsigned long cpu_util_without(int cpu, struct task_struct *p) { +#ifndef CONFIG_SCHED_WALT struct cfs_rq *cfs_rq; +#endif unsigned int util; +#ifdef CONFIG_SCHED_WALT + /* + * WALT does not decay idle tasks in the same manner + * as PELT, so it makes little sense to subtract task + * utilization from cpu utilization. Instead just use + * cpu_util for this case. + */ + if (likely(p->state == TASK_WAKING)) + return cpu_util(cpu); +#endif + /* Task has no contribution or is new */ if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) return cpu_util(cpu); +#ifdef CONFIG_SCHED_WALT + util = max_t(long, cpu_util(cpu) - task_util(p), 0); +#else cfs_rq = &cpu_rq(cpu)->cfs; util = READ_ONCE(cfs_rq->avg.util_avg); @@ -6179,6 +6162,7 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) util = max(util, estimated); } +#endif /* * Utilization (estimated) can exceed the CPU capacity, thus let's @@ -6188,6 +6172,18 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) return min_t(unsigned long, util, capacity_orig_of(cpu)); } +/* + * Returns the current capacity of cpu after applying both + * cpu and freq scaling. + */ +unsigned long capacity_curr_of(int cpu) +{ + unsigned long max_cap = cpu_rq(cpu)->cpu_capacity_orig; + unsigned long scale_freq = arch_scale_freq_capacity(cpu); + + return cap_scale(max_cap, scale_freq); +} + /* * Disable WAKE_AFFINE in the case where task @p doesn't fit in the * capacity of either the waking CPU @cpu or the previous CPU @prev_cpu. @@ -6344,7 +6340,7 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) * other use-cases too. So, until someone finds a better way to solve this, * let's keep things simple by re-using the existing slow path. */ -static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync) +int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sync) { unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; struct root_domain *rd = cpu_rq(smp_processor_id())->rd; @@ -7363,7 +7359,13 @@ static void detach_task(struct task_struct *p, struct lb_env *env) lockdep_assert_held(&env->src_rq->lock); deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); + lockdep_off(); + double_lock_balance(env->src_rq, env->dst_rq); + if (!(env->src_rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(env->src_rq); set_task_cpu(p, env->dst_cpu); + double_unlock_balance(env->src_rq, env->dst_rq); + lockdep_on(); } /* @@ -7894,6 +7896,8 @@ void update_group_capacity(struct sched_domain *sd, int cpu) struct sched_group_capacity *sgc; struct rq *rq = cpu_rq(cpu); + if (cpumask_test_cpu(cpu, cpu_isolated_mask)) + continue; /* * build_sched_domains() -> init_sched_groups_capacity() * gets here before we've attached the domains to the @@ -7924,10 +7928,15 @@ void update_group_capacity(struct sched_domain *sd, int cpu) group = child->groups; do { struct sched_group_capacity *sgc = group->sgc; + cpumask_t *cpus = sched_group_span(group); - capacity += sgc->capacity; - min_capacity = min(sgc->min_capacity, min_capacity); - max_capacity = max(sgc->max_capacity, max_capacity); + if (!cpu_isolated(cpumask_first(cpus))) { + capacity += sgc->capacity; + min_capacity = min(sgc->min_capacity, + min_capacity); + max_capacity = max(sgc->max_capacity, + max_capacity); + } group = group->next; } while (group != child->groups); } @@ -8131,6 +8140,9 @@ static inline void update_sg_lb_stats(struct lb_env *env, for_each_cpu_and(i, sched_group_span(group), env->cpus) { struct rq *rq = cpu_rq(i); + if (cpu_isolated(i)) + continue; + if ((env->flags & LBF_NOHZ_STATS) && update_nohz_stats(rq, false)) env->flags |= LBF_NOHZ_AGAIN; @@ -8162,17 +8174,27 @@ static inline void update_sg_lb_stats(struct lb_env *env, } } - /* Adjust by relative CPU capacity of the group */ - sgs->group_capacity = group->sgc->capacity; - sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity; + /* Isolated CPU has no weight */ + if (!group->group_weight) { + sgs->group_capacity = 0; + sgs->avg_load = 0; + sgs->group_no_capacity = 1; + sgs->group_type = group_other; + sgs->group_weight = group->group_weight; + } else { + /* Adjust by relative CPU capacity of the group */ + sgs->group_capacity = group->sgc->capacity; + sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / + sgs->group_capacity; + + sgs->group_weight = group->group_weight; + + sgs->group_no_capacity = group_is_overloaded(env, sgs); + sgs->group_type = group_classify(group, sgs); + } if (sgs->sum_nr_running) sgs->load_per_task = sgs->group_load / sgs->sum_nr_running; - - sgs->group_weight = group->group_weight; - - sgs->group_no_capacity = group_is_overloaded(env, sgs); - sgs->group_type = group_classify(group, sgs); } /** @@ -8891,8 +8913,6 @@ static int need_active_balance(struct lb_env *env) return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); } -static int active_load_balance_cpu_stop(void *data); - static int should_we_balance(struct lb_env *env) { struct sched_group *sg = env->sd->groups; @@ -8914,7 +8934,7 @@ static int should_we_balance(struct lb_env *env) /* Try to find first idle CPU */ for_each_cpu_and(cpu, group_balance_mask(sg), env->cpus) { - if (!idle_cpu(cpu)) + if (!idle_cpu(cpu) || cpu_isolated(cpu)) continue; balance_cpu = cpu; @@ -8922,7 +8942,7 @@ static int should_we_balance(struct lb_env *env) } if (balance_cpu == -1) - balance_cpu = group_balance_cpu(sg); + balance_cpu = group_balance_cpu_not_isolated(sg); /* * First idle CPU or the first CPU(busiest) in this sched group @@ -9131,7 +9151,8 @@ more_balance: * ->active_balance_work. Once set, it's cleared * only after active load balance is finished. */ - if (!busiest->active_balance) { + if (!busiest->active_balance && + !cpu_isolated(cpu_of(busiest))) { busiest->active_balance = 1; busiest->push_cpu = this_cpu; active_balance = 1; @@ -9244,7 +9265,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance) * least 1 task to be running on each physical CPU where possible, and * avoids physical / logical imbalances. */ -static int active_load_balance_cpu_stop(void *data) +int active_load_balance_cpu_stop(void *data) { struct rq *busiest_rq = data; int busiest_cpu = cpu_of(busiest_rq); @@ -9337,7 +9358,13 @@ static DEFINE_SPINLOCK(balancing); */ void update_max_interval(void) { - max_load_balance_interval = HZ*num_online_cpus()/10; + cpumask_t avail_mask; + unsigned int available_cpus; + + cpumask_andnot(&avail_mask, cpu_online_mask, cpu_isolated_mask); + available_cpus = cpumask_weight(&avail_mask); + + max_load_balance_interval = HZ*available_cpus/10; } /* @@ -9514,6 +9541,7 @@ static void nohz_balancer_kick(struct rq *rq) struct sched_domain *sd; int nr_busy, i, cpu = rq->cpu; unsigned int flags = 0; + cpumask_t cpumask; if (unlikely(rq->idle_balance)) return; @@ -9528,7 +9556,8 @@ static void nohz_balancer_kick(struct rq *rq) * None are in tickless mode and hence no need for NOHZ idle load * balancing. */ - if (likely(!atomic_read(&nohz.nr_cpus))) + cpumask_andnot(&cpumask, nohz.idle_cpus_mask, cpu_isolated_mask); + if (cpumask_empty(&cpumask)) return; if (READ_ONCE(nohz.has_blocked) && @@ -9565,7 +9594,7 @@ static void nohz_balancer_kick(struct rq *rq) * currently idle; in which case, kick the ILB to move tasks * around. */ - for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { + for_each_cpu_and(i, sched_domain_span(sd), &cpumask) { if (sched_asym_prefer(i, cpu)) { flags = NOHZ_KICK_MASK; goto unlock; @@ -9743,6 +9772,7 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, int balance_cpu; int ret = false; struct rq *rq; + cpumask_t cpus; SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); @@ -9762,7 +9792,9 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, */ smp_mb(); - for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { + cpumask_andnot(&cpus, nohz.idle_cpus_mask, cpu_isolated_mask); + + for_each_cpu(balance_cpu, &cpus) { if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) continue; @@ -9914,6 +9946,9 @@ int newidle_balance(struct rq *this_rq, struct rq_flags *rf) int pulled_task = 0; u64 curr_cost = 0; + if (cpu_isolated(this_cpu)) + return 0; + update_misfit_status(NULL, this_rq); /* * We must set idle_stamp _before_ calling idle_balance(), such that we @@ -10030,6 +10065,14 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h) enum cpu_idle_type idle = this_rq->idle_balance ? CPU_IDLE : CPU_NOT_IDLE; + /* + * Since core isolation doesn't update nohz.idle_cpus_mask, there + * is a possibility this nohz kicked cpu could be isolated. Hence + * return if the cpu is isolated. + */ + if (cpu_isolated(this_rq->cpu)) + return; + /* * If this CPU has a pending nohz_balance_kick, then do the * balancing on behalf of the other idle CPUs whose ticks are @@ -10051,8 +10094,10 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h) */ void trigger_load_balance(struct rq *rq) { - /* Don't need to rebalance while attached to NULL domain */ - if (unlikely(on_null_domain(rq))) + /* Don't need to rebalance while attached to NULL domain or + * cpu is isolated. + */ + if (unlikely(on_null_domain(rq)) || cpu_isolated(cpu_of(rq))) return; if (time_after_eq(jiffies, rq->next_balance)) @@ -10615,6 +10660,10 @@ const struct sched_class fair_sched_class = { #ifdef CONFIG_UCLAMP_TASK .uclamp_enabled = 1, #endif + +#ifdef CONFIG_SCHED_WALT + .fixup_walt_sched_stats = walt_fixup_sched_stats_fair, +#endif }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index ee68dea4b11a..27a5c1e62c72 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -61,7 +61,8 @@ static noinline int __cpuidle cpu_idle_poll(void) stop_critical_timings(); while (!tif_need_resched() && - (cpu_idle_force_poll || tick_check_broadcast_expired())) + (cpu_idle_force_poll || tick_check_broadcast_expired() || + is_reserved(smp_processor_id()))) cpu_relax(); start_critical_timings(); trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id()); @@ -257,7 +258,8 @@ static void do_idle(void) * broadcast device expired for us, we don't want to go deep * idle as we know that the IPI is going to arrive right away. */ - if (cpu_idle_force_poll || tick_check_broadcast_expired()) { + if (cpu_idle_force_poll || tick_check_broadcast_expired() || + is_reserved(smp_processor_id())) { tick_nohz_idle_restart_tick(); cpu_idle_poll(); } else { diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 9b8adc01be3d..80a260d83111 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -6,6 +6,7 @@ #include "sched.h" #include "pelt.h" +#include "walt.h" int sched_rr_timeslice = RR_TIMESLICE; int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE; @@ -262,8 +263,12 @@ static void pull_rt_task(struct rq *this_rq); static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) { - /* Try to pull RT tasks here if we lower this rq's prio */ - return rq->rt.highest_prio.curr > prev->prio; + /* + * Try to pull RT tasks here if we lower this rq's prio and cpu is not + * isolated + */ + return rq->rt.highest_prio.curr > prev->prio && + !cpu_isolated(cpu_of(rq)); } static inline int rt_overloaded(struct rq *rq) @@ -2191,7 +2196,8 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p) * we may need to handle the pulling of RT tasks * now. */ - if (!task_on_rq_queued(p) || rq->rt.rt_nr_running) + if (!task_on_rq_queued(p) || rq->rt.rt_nr_running || + cpu_isolated(cpu_of(rq))) return; rt_queue_pull_task(rq); @@ -2388,6 +2394,10 @@ const struct sched_class rt_sched_class = { #ifdef CONFIG_UCLAMP_TASK .uclamp_enabled = 1, #endif + +#ifdef CONFIG_SCHED_WALT + .fixup_walt_sched_stats = fixup_walt_sched_stats_common, +#endif }; #ifdef CONFIG_RT_GROUP_SCHED diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e83bdf6bce23..2ccf94a75c07 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -84,6 +84,73 @@ struct rq; struct cpuidle_state; +extern __read_mostly bool sched_predl; + +struct sched_walt_cpu_load { + unsigned long prev_window_util; + unsigned long nl; + unsigned long pl; + bool rtgb_active; + u64 ws; +}; + +#ifdef CONFIG_SCHED_WALT +#define DECLARE_BITMAP_ARRAY(name, nr, bits) \ + unsigned long name[nr][BITS_TO_LONGS(bits)] + +extern unsigned int __weak sched_ravg_window; + +struct walt_sched_stats { + int nr_big_tasks; + u64 cumulative_runnable_avg_scaled; + u64 pred_demands_sum_scaled; +}; + +struct cpu_cycle { + u64 cycles; + u64 time; +}; + +struct group_cpu_time { + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; +}; + +struct load_subtractions { + u64 window_start; + u64 subs; + u64 new_subs; +}; + +#define NUM_TRACKED_WINDOWS 2 +#define NUM_LOAD_INDICES 1000 + +struct sched_cluster { + raw_spinlock_t load_lock; + struct list_head list; + struct cpumask cpus; + int id; + int max_power_cost; + int min_power_cost; + int max_possible_capacity; + int efficiency; /* Differentiate cpus with different IPC capability */ + unsigned int exec_scale_factor; + /* + * max_freq = user maximum + * max_mitigated_freq = thermal defined maximum + * max_possible_freq = maximum supported by hardware + */ + unsigned int cur_freq, max_freq, max_mitigated_freq, min_freq; + unsigned int max_possible_freq; + bool freq_init_done; + u64 aggr_grp_load; +}; + +extern __weak cpumask_t asym_cap_sibling_cpus; +#endif /* CONFIG_SCHED_WALT */ + /* task_struct::on_rq states: */ #define TASK_ON_RQ_QUEUED 1 #define TASK_ON_RQ_MIGRATING 2 @@ -96,6 +163,10 @@ extern atomic_long_t calc_load_tasks; extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq, long adjust); +#ifdef CONFIG_SMP +extern void init_sched_groups_capacity(int cpu, struct sched_domain *sd); +#endif + /* * Helpers for converting nanosecond timing to jiffy resolution */ @@ -401,7 +472,24 @@ struct task_group { struct uclamp_se uclamp[UCLAMP_CNT]; /* Latency-sensitive flag used for a task group */ unsigned int latency_sensitive; -#endif +#ifdef CONFIG_SCHED_WALT + /* Toggle ability to override sched boost enabled */ + bool sched_boost_no_override; + /* + * Controls whether a cgroup is eligible for sched boost or not. This + * can temporariliy be disabled by the kernel based on the no_override + * flag above. + */ + bool sched_boost_enabled; + /* + * Controls whether tasks of this cgroup should be colocated with each + * other and tasks of other cgroups that have the same flag turned on. + */ + bool colocate; + /* Controls whether further updates are allowed to the colocate flag */ + bool colocate_update_disabled; +#endif /* CONFIG_SCHED_WALT */ +#endif /* CONFIG_UCLAMP_TASK_GROUP */ }; @@ -565,6 +653,10 @@ struct cfs_rq { struct list_head leaf_cfs_rq_list; struct task_group *tg; /* group that "owns" this runqueue */ +#ifdef CONFIG_SCHED_WALT + struct walt_sched_stats walt_stats; +#endif + #ifdef CONFIG_CFS_BANDWIDTH int runtime_enabled; s64 runtime_remaining; @@ -961,6 +1053,41 @@ struct rq { u64 max_idle_balance_cost; #endif +#ifdef CONFIG_SCHED_WALT + struct task_struct *push_task; + struct sched_cluster *cluster; + struct cpumask freq_domain_cpumask; + struct walt_sched_stats walt_stats; + + u64 window_start; + u32 prev_window_size; + unsigned long walt_flags; + + u64 cur_irqload; + u64 avg_irqload; + u64 irqload_ts; + struct task_struct *ed_task; + struct cpu_cycle cc; + u64 old_busy_time, old_busy_time_group; + u64 old_estimated_time; + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; + u64 cum_window_demand_scaled; + struct group_cpu_time grp_time; + struct load_subtractions load_subs[NUM_TRACKED_WINDOWS]; + DECLARE_BITMAP_ARRAY(top_tasks_bitmap, + NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES); + u8 *top_tasks[NUM_TRACKED_WINDOWS]; + u8 curr_table; + int prev_top; + int curr_top; + bool notif_pending; + u64 last_cc_update; + u64 cycles; +#endif /* CONFIG_SCHED_WALT */ + #ifdef CONFIG_IRQ_TIME_ACCOUNTING u64 prev_irq_time; #endif @@ -1306,8 +1433,6 @@ enum numa_faults_stats { }; extern void sched_setnuma(struct task_struct *p, int node); extern int migrate_task_to(struct task_struct *p, int cpu); -extern int migrate_swap(struct task_struct *p, struct task_struct *t, - int cpu, int scpu); extern void init_numa_balancing(unsigned long clone_flags, struct task_struct *p); #else static inline void @@ -1316,6 +1441,9 @@ init_numa_balancing(unsigned long clone_flags, struct task_struct *p) } #endif /* CONFIG_NUMA_BALANCING */ +extern int migrate_swap(struct task_struct *p, struct task_struct *t, + int cpu, int scpu); + #ifdef CONFIG_SMP static inline void @@ -1782,8 +1910,15 @@ struct sched_class { #ifdef CONFIG_FAIR_GROUP_SCHED void (*task_change_group)(struct task_struct *p, int type); #endif + +#ifdef CONFIG_SCHED_WALT + void (*fixup_walt_sched_stats)(struct rq *rq, struct task_struct *p, + u16 updated_demand_scaled, + u16 updated_pred_demand_scaled); +#endif }; + static inline void put_prev_task(struct rq *rq, struct task_struct *prev) { WARN_ON_ONCE(rq->curr != prev); @@ -1960,6 +2095,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) { unsigned prev_nr = rq->nr_running; + sched_update_nr_prod(cpu_of(rq), count, true); rq->nr_running = prev_nr + count; #ifdef CONFIG_SMP @@ -1974,6 +2110,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) static inline void sub_nr_running(struct rq *rq, unsigned count) { + sched_update_nr_prod(cpu_of(rq), count, false); rq->nr_running -= count; /* Check if we still need preemption */ sched_update_tick_dependency(rq); @@ -2014,6 +2151,18 @@ static inline int hrtick_enabled(struct rq *rq) #endif /* CONFIG_SCHED_HRTICK */ +#ifdef CONFIG_SCHED_WALT +u64 __weak sched_ktime_clock(void); +unsigned long __weak +cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load); +#else +#define sched_ravg_window TICK_NSEC +static inline u64 sched_ktime_clock(void) +{ + return 0; +} +#endif + #ifndef arch_scale_freq_capacity static __always_inline unsigned long arch_scale_freq_capacity(int cpu) @@ -2031,8 +2180,127 @@ unsigned long arch_scale_max_freq_capacity(struct sched_domain *sd, int cpu) } #endif +unsigned long capacity_curr_of(int cpu); + #ifdef CONFIG_SMP -#ifdef CONFIG_PREEMPTION +static inline unsigned long capacity_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity; +} + +static inline unsigned long capacity_orig_of(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity_orig; +} + +static inline unsigned long task_util(struct task_struct *p) +{ +#ifdef CONFIG_SCHED_WALT + return p->ravg.demand_scaled; +#endif + return READ_ONCE(p->se.avg.util_avg); +} + +/** + * Amount of capacity of a CPU that is (estimated to be) used by CFS tasks + * @cpu: the CPU to get the utilization of + * + * The unit of the return value must be the one of capacity so we can compare + * the utilization with the capacity of the CPU that is available for CFS task + * (ie cpu_capacity). + * + * cfs_rq.avg.util_avg is the sum of running time of runnable tasks plus the + * recent utilization of currently non-runnable tasks on a CPU. It represents + * the amount of utilization of a CPU in the range [0..capacity_orig] where + * capacity_orig is the cpu_capacity available at the highest frequency + * (arch_scale_freq_capacity()). + * The utilization of a CPU converges towards a sum equal to or less than the + * current capacity (capacity_curr <= capacity_orig) of the CPU because it is + * the running time on this CPU scaled by capacity_curr. + * + * The estimated utilization of a CPU is defined to be the maximum between its + * cfs_rq.avg.util_avg and the sum of the estimated utilization of the tasks + * currently RUNNABLE on that CPU. + * This allows to properly represent the expected utilization of a CPU which + * has just got a big task running since a long sleep period. At the same time + * however it preserves the benefits of the "blocked utilization" in + * describing the potential for other tasks waking up on the same CPU. + * + * Nevertheless, cfs_rq.avg.util_avg can be higher than capacity_curr or even + * higher than capacity_orig because of unfortunate rounding in + * cfs.avg.util_avg or just after migrating tasks and new task wakeups until + * the average stabilizes with the new running time. We need to check that the + * utilization stays within the range of [0..capacity_orig] and cap it if + * necessary. Without utilization capping, a group could be seen as overloaded + * (CPU0 utilization at 121% + CPU1 utilization at 80%) whereas CPU1 has 20% of + * available capacity. We allow utilization to overshoot capacity_curr (but not + * capacity_orig) as it useful for predicting the capacity required after task + * migrations (scheduler-driven DVFS). + * + * Return: the (estimated) utilization for the specified CPU + */ +static inline unsigned long cpu_util(int cpu) +{ + struct cfs_rq *cfs_rq; + unsigned int util; + +#ifdef CONFIG_SCHED_WALT + u64 walt_cpu_util = + cpu_rq(cpu)->walt_stats.cumulative_runnable_avg_scaled; + + return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu)); +#endif + + cfs_rq = &cpu_rq(cpu)->cfs; + util = READ_ONCE(cfs_rq->avg.util_avg); + + if (sched_feat(UTIL_EST)) + util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued)); + + return min_t(unsigned long, util, capacity_orig_of(cpu)); +} + +static inline unsigned long cpu_util_cum(int cpu, int delta) +{ + u64 util = cpu_rq(cpu)->cfs.avg.util_avg; + unsigned long capacity = capacity_orig_of(cpu); + +#ifdef CONFIG_SCHED_WALT + util = cpu_rq(cpu)->cum_window_demand_scaled; +#endif + delta += util; + if (delta < 0) + return 0; + + return (delta >= capacity) ? capacity : delta; +} + +static inline unsigned long +cpu_util_freq(int cpu, struct sched_walt_cpu_load *walt_load) +{ +#ifdef CONFIG_SCHED_WALT + return cpu_util_freq_walt(cpu, walt_load); +#else + return cpu_util(cpu); +#endif +} + +extern unsigned int capacity_margin_freq; + +static inline unsigned long +add_capacity_margin(unsigned long cpu_capacity, int cpu) +{ + cpu_capacity = cpu_capacity * capacity_margin_freq * + (100 + per_cpu(sched_load_boost, cpu)); + cpu_capacity /= 100; + cpu_capacity /= SCHED_CAPACITY_SCALE; + return cpu_capacity; +} + +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_SMP +#ifdef CONFIG_PREEMPT static inline void double_rq_lock(struct rq *rq1, struct rq *rq2); @@ -2345,6 +2613,11 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) { struct update_util_data *data; +#ifdef CONFIG_SCHED_WALT + if (!(flags & SCHED_CPUFREQ_WALT)) + return; +#endif + data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data, cpu_of(rq))); if (data) @@ -2432,13 +2705,6 @@ static inline bool uclamp_latency_sensitive(struct task_struct *p) # define arch_scale_freq_invariant() false #endif -#ifdef CONFIG_SMP -static inline unsigned long capacity_orig_of(int cpu) -{ - return cpu_rq(cpu)->cpu_capacity_orig; -} -#endif - /** * enum schedutil_type - CPU utilization type * @FREQUENCY_UTIL: Utilization used to select frequency @@ -2570,3 +2836,500 @@ static inline void membarrier_switch_mm(struct rq *rq, { } #endif + +enum sched_boost_policy { + SCHED_BOOST_NONE, + SCHED_BOOST_ON_BIG, + SCHED_BOOST_ON_ALL, +}; + +#ifdef CONFIG_SCHED_WALT + +static inline int cluster_first_cpu(struct sched_cluster *cluster) +{ + return cpumask_first(&cluster->cpus); +} + +struct related_thread_group { + int id; + raw_spinlock_t lock; + struct list_head tasks; + struct list_head list; + bool skip_min; + struct rcu_head rcu; + u64 last_update; + u64 downmigrate_ts; + u64 start_ts; +}; + +extern struct sched_cluster *sched_cluster[NR_CPUS]; + +extern unsigned int __weak sched_disable_window_stats; +extern unsigned int max_possible_freq; +extern unsigned int min_max_freq; +extern unsigned int max_possible_efficiency; +extern unsigned int min_possible_efficiency; +extern unsigned int max_possible_capacity; +extern unsigned int __weak min_max_possible_capacity; +extern unsigned int max_power_cost; +extern unsigned int __read_mostly __weak sched_init_task_load_windows; +extern unsigned int __read_mostly __weak sched_load_granule; + +extern int __weak update_preferred_cluster(struct related_thread_group *grp, + struct task_struct *p, u32 old_load, bool from_tick); +extern void __weak set_preferred_cluster(struct related_thread_group *grp); +extern void __weak add_new_task_to_grp(struct task_struct *new); + +#define NO_BOOST 0 +#define FULL_THROTTLE_BOOST 1 +#define CONSERVATIVE_BOOST 2 +#define RESTRAINED_BOOST 3 +#define FULL_THROTTLE_BOOST_DISABLE -1 +#define CONSERVATIVE_BOOST_DISABLE -2 +#define RESTRAINED_BOOST_DISABLE -3 +#define MAX_NUM_BOOST_TYPE (RESTRAINED_BOOST+1) + +static inline int asym_cap_siblings(int cpu1, int cpu2) +{ + return (cpumask_test_cpu(cpu1, &asym_cap_sibling_cpus) && + cpumask_test_cpu(cpu2, &asym_cap_sibling_cpus)); +} + +static inline int cpu_max_possible_capacity(int cpu) +{ + return cpu_rq(cpu)->cluster->max_possible_capacity; +} + +static inline unsigned int cluster_max_freq(struct sched_cluster *cluster) +{ + /* + * Governor and thermal driver don't know the other party's mitigation + * voting. So struct cluster saves both and return min() for current + * cluster fmax. + */ + return min(cluster->max_mitigated_freq, cluster->max_freq); +} + +static inline unsigned int cpu_max_freq(int cpu) +{ + return cluster_max_freq(cpu_rq(cpu)->cluster); +} + +static inline unsigned int cpu_max_possible_freq(int cpu) +{ + return cpu_rq(cpu)->cluster->max_possible_freq; +} + +static inline bool hmp_capable(void) +{ + return max_possible_capacity != min_max_possible_capacity; +} + +static inline bool is_max_capacity_cpu(int cpu) +{ + return cpu_max_possible_capacity(cpu) == max_possible_capacity; +} + +static inline bool is_min_capacity_cpu(int cpu) +{ + return cpu_max_possible_capacity(cpu) == min_max_possible_capacity; +} + +static inline unsigned int task_load(struct task_struct *p) +{ + return p->ravg.demand; +} + +static inline unsigned int task_pl(struct task_struct *p) +{ + return p->ravg.pred_demand; +} + +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + return !!(rcu_access_pointer(p->grp) != NULL); +} + +static inline +struct related_thread_group *task_related_thread_group(struct task_struct *p) +{ + return rcu_dereference(p->grp); +} + +/* Is frequency of two cpus synchronized with each other? */ +static inline int same_freq_domain(int src_cpu, int dst_cpu) +{ + struct rq *rq = cpu_rq(src_cpu); + + if (src_cpu == dst_cpu) + return 1; + + if (asym_cap_siblings(src_cpu, dst_cpu)) + return 1; + + return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask); +} + +#define CPU_RESERVED 1 + +extern enum sched_boost_policy __weak boost_policy; +extern unsigned int __weak sched_task_filter_util; +static inline enum sched_boost_policy sched_boost_policy(void) +{ + return boost_policy; +} + +extern unsigned int __weak sched_boost_type; +static inline int sched_boost(void) +{ + return sched_boost_type; +} + +static inline bool rt_boost_on_big(void) +{ + return sched_boost() == FULL_THROTTLE_BOOST ? + (sched_boost_policy() == SCHED_BOOST_ON_BIG) : false; +} + +static inline bool is_full_throttle_boost(void) +{ + return sched_boost() == FULL_THROTTLE_BOOST; +} + +extern int __weak preferred_cluster(struct sched_cluster *cluster, + struct task_struct *p); +extern struct sched_cluster *rq_cluster(struct rq *rq); + +#ifdef CONFIG_UCLAMP_TASK_GROUP +static inline bool task_sched_boost(struct task_struct *p) +{ + struct cgroup_subsys_state *css = task_css(p, cpu_cgrp_id); + struct task_group *tg; + + if (!css) + return false; + tg = container_of(css, struct task_group, css); + + return tg->sched_boost_enabled; +} + +extern int __weak sync_cgroup_colocation(struct task_struct *p, bool insert); +extern void update_cgroup_boost_settings(void); +extern void restore_cgroup_boost_settings(void); +#else +static inline bool +same_schedtg(struct task_struct *tsk1, struct task_struct *tsk2) +{ + return true; +} + +static inline bool task_sched_boost(struct task_struct *p) +{ + return true; +} + +static inline void update_cgroup_boost_settings(void) { } +static inline void restore_cgroup_boost_settings(void) { } +#endif + +extern int __weak alloc_related_thread_groups(void); + +extern void __weak check_for_migration(struct rq *rq, struct task_struct *p); + +static inline int is_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + return test_bit(CPU_RESERVED, &rq->walt_flags); +} + +static inline int mark_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + return test_and_set_bit(CPU_RESERVED, &rq->walt_flags); +} + +static inline void clear_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + clear_bit(CPU_RESERVED, &rq->walt_flags); +} + +static inline bool +task_in_cum_window_demand(struct rq *rq, struct task_struct *p) +{ + return cpu_of(rq) == task_cpu(p) && (p->on_rq || p->last_sleep_ts >= + rq->window_start); +} + +static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta) +{ + rq->cum_window_demand_scaled += scaled_delta; + if (unlikely((s64)rq->cum_window_demand_scaled < 0)) + rq->cum_window_demand_scaled = 0; +} + +extern unsigned long __weak thermal_cap(int cpu); + +extern void __weak clear_walt_request(int cpu); + +extern enum sched_boost_policy sched_boost_policy(void); +extern void sched_boost_parse_dt(void); +extern void __weak clear_ed_task(struct task_struct *p, struct rq *rq); +extern bool __weak early_detection_notify(struct rq *rq, u64 wallclock); + +static inline unsigned int power_cost(int cpu, u64 demand) +{ + return cpu_max_possible_capacity(cpu); +} + +void __weak note_task_waking(struct task_struct *p, u64 wallclock); + +static inline bool task_placement_boost_enabled(struct task_struct *p) +{ + if (task_sched_boost(p)) + return sched_boost_policy() != SCHED_BOOST_NONE; + + return false; +} + +static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) +{ + enum sched_boost_policy policy = task_sched_boost(p) ? + sched_boost_policy() : + SCHED_BOOST_NONE; + if (policy == SCHED_BOOST_ON_BIG) { + /* + * Filter out tasks less than min task util threshold + * under conservative boost. + */ + if (sched_boost() == CONSERVATIVE_BOOST && + task_util(p) <= sched_task_filter_util) + policy = SCHED_BOOST_NONE; + } + + return policy; +} + +static inline bool is_min_capacity_cluster(struct sched_cluster *cluster) +{ + return is_min_capacity_cpu(cluster_first_cpu(cluster)); +} + +extern void __weak walt_fixup_sched_stats_fair(struct rq *rq, + struct task_struct *p, + u16 updated_demand_scaled, + u16 updated_pred_demand_scaled); +extern void __weak walt_fixup_nr_big_tasks(struct rq *rq, struct task_struct *p, + int delta, bool inc); +#else /* CONFIG_SCHED_WALT */ + +struct walt_sched_stats; +struct related_thread_group; +struct sched_cluster; + +static inline bool task_sched_boost(struct task_struct *p) +{ + return false; +} + +static inline bool task_placement_boost_enabled(struct task_struct *p) +{ + return false; +} + +static inline void check_for_migration(struct rq *rq, struct task_struct *p) { } + +static inline int sched_boost(void) +{ + return 0; +} + +static inline bool rt_boost_on_big(void) +{ + return false; +} + +static inline bool is_full_throttle_boost(void) +{ + return false; +} + +static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) +{ + return SCHED_BOOST_NONE; +} + +static inline bool +task_in_cum_window_demand(struct rq *rq, struct task_struct *p) +{ + return false; +} + +static inline bool hmp_capable(void) { return false; } +static inline bool is_max_capacity_cpu(int cpu) { return true; } +static inline bool is_min_capacity_cpu(int cpu) { return true; } + +static inline int +preferred_cluster(struct sched_cluster *cluster, struct task_struct *p) +{ + return -1; +} + +static inline struct sched_cluster *rq_cluster(struct rq *rq) +{ + return NULL; +} + +static inline int asym_cap_siblings(int cpu1, int cpu2) { return 0; } + +static inline void set_preferred_cluster(struct related_thread_group *grp) { } + +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + return false; +} + +static inline +struct related_thread_group *task_related_thread_group(struct task_struct *p) +{ + return NULL; +} + +static inline u32 task_load(struct task_struct *p) { return 0; } +static inline u32 task_pl(struct task_struct *p) { return 0; } + +static inline int update_preferred_cluster(struct related_thread_group *grp, + struct task_struct *p, u32 old_load, bool from_tick) +{ + return 0; +} + +static inline void add_new_task_to_grp(struct task_struct *new) {} + +static inline int same_freq_domain(int src_cpu, int dst_cpu) +{ + return 1; +} + +static inline int mark_reserved(int cpu) +{ + return 0; +} + +static inline void clear_reserved(int cpu) { } +static inline int alloc_related_thread_groups(void) { return 0; } + +static inline void walt_fixup_cum_window_demand(struct rq *rq, + s64 scaled_delta) { } + +#ifdef CONFIG_SMP +static inline unsigned long thermal_cap(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity_orig; +} +#endif + +static inline void clear_walt_request(int cpu) { } + +static inline int is_reserved(int cpu) +{ + return 0; +} + +static inline enum sched_boost_policy sched_boost_policy(void) +{ + return SCHED_BOOST_NONE; +} + +static inline void sched_boost_parse_dt(void) { } + +static inline void clear_ed_task(struct task_struct *p, struct rq *rq) { } + +static inline bool early_detection_notify(struct rq *rq, u64 wallclock) +{ + return 0; +} + +#ifdef CONFIG_SMP +static inline unsigned int power_cost(int cpu, u64 demand) +{ + return SCHED_CAPACITY_SCALE; +} +#endif + +static inline void note_task_waking(struct task_struct *p, u64 wallclock) { } +#endif /* CONFIG_SCHED_WALT */ + +struct sched_avg_stats { + int nr; + int nr_misfit; + int nr_max; + int nr_scaled; +}; +extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); + +#if defined(CONFIG_SCHED_WALT) && defined(CONFIG_CFS_BANDWIDTH) + +extern void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq); +extern void walt_inc_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p); +extern void walt_dec_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p); +extern void walt_inc_throttled_cfs_rq_stats(struct walt_sched_stats *stats, + struct cfs_rq *cfs_rq); +extern void walt_dec_throttled_cfs_rq_stats(struct walt_sched_stats *stats, + struct cfs_rq *cfs_rq); +#else +static inline void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq) {} +static inline void +walt_inc_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p) {} +static inline void +walt_dec_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p) {} + +#define walt_inc_throttled_cfs_rq_stats(...) +#define walt_dec_throttled_cfs_rq_stats(...) + +#endif + + +#ifdef CONFIG_SMP +#ifdef CONFIG_SCHED_WALT +extern int __weak group_balance_cpu_not_isolated(struct sched_group *sg); +#else +static inline int group_balance_cpu_not_isolated(struct sched_group *sg) +{ + return group_balance_cpu(sg); +} +#endif /* CONFIG_SCHED_WALT */ +#endif /* CONFIG_SMP */ + +extern int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, + int sync); +extern int active_load_balance_cpu_stop(void *data); + +#ifdef CONFIG_HOTPLUG_CPU +extern void set_rq_online(struct rq *rq); +extern void set_rq_offline(struct rq *rq); +extern void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, + bool migrate_pinned_tasks); +extern void calc_load_migrate(struct rq *rq); +#ifdef CONFIG_SCHED_WALT +extern void __weak +detach_one_task_core(struct task_struct *p, struct rq *rq, + struct list_head *tasks); +extern void __weak attach_tasks_core(struct list_head *tasks, struct rq *rq); +#else +static inline void +detach_one_task_core(struct task_struct *p, struct rq *rq, + struct list_head *tasks) +{ +} +static inline void attach_tasks_core(struct list_head *tasks, struct rq *rq) {} +#endif +#endif + +extern struct task_struct *find_process_by_pid(pid_t pid); + +extern void enqueue_task_core(struct rq *rq, struct task_struct *p, int flags); +extern void dequeue_task_core(struct rq *rq, struct task_struct *p, int flags); diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index c0640739e05e..e58457b4377a 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -8,6 +8,7 @@ * See kernel/stop_machine.c */ #include "sched.h" +#include "walt.h" #ifdef CONFIG_SMP static int @@ -50,12 +51,14 @@ static void enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) { add_nr_running(rq, 1); + walt_inc_cumulative_runnable_avg(rq, p); } static void dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags) { sub_nr_running(rq, 1); + walt_dec_cumulative_runnable_avg(rq, p); } static void yield_task_stop(struct rq *rq) @@ -144,4 +147,7 @@ const struct sched_class stop_sched_class = { .prio_changed = prio_changed_stop, .switched_to = switched_to_stop, .update_curr = update_curr_stop, +#ifdef CONFIG_SCHED_WALT + .fixup_walt_sched_stats = fixup_walt_sched_stats_common, +#endif }; diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 7c5d8556f374..323def239081 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1163,16 +1163,19 @@ build_sched_groups(struct sched_domain *sd, int cpu) * group having more cpu_capacity will pickup more load compared to the * group having less cpu_capacity. */ -static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) +void init_sched_groups_capacity(int cpu, struct sched_domain *sd) { struct sched_group *sg = sd->groups; + cpumask_t avail_mask; WARN_ON(!sg); do { int cpu, max_cpu = -1; - sg->group_weight = cpumask_weight(sched_group_span(sg)); + cpumask_andnot(&avail_mask, sched_group_span(sg), + cpu_isolated_mask); + sg->group_weight = cpumask_weight(&avail_mask); if (!(sd->flags & SD_ASYM_PACKING)) goto next; diff --git a/kernel/sched/walt.h b/kernel/sched/walt.h new file mode 100644 index 000000000000..22544c970bff --- /dev/null +++ b/kernel/sched/walt.h @@ -0,0 +1,265 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + */ + +#ifndef __WALT_H +#define __WALT_H + +#ifdef CONFIG_SCHED_WALT + +#include +#include + +#define EXITING_TASK_MARKER 0xdeaddead + +extern void __weak +walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime); + +static inline void +fixup_cumulative_runnable_avg(struct walt_sched_stats *stats, + s64 demand_scaled_delta, + s64 pred_demand_scaled_delta) +{ + if (sched_disable_window_stats) + return; + + stats->cumulative_runnable_avg_scaled += demand_scaled_delta; + BUG_ON((s64)stats->cumulative_runnable_avg_scaled < 0); + + stats->pred_demands_sum_scaled += pred_demand_scaled_delta; + BUG_ON((s64)stats->pred_demands_sum_scaled < 0); +} + +static inline void +walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) +{ + if (sched_disable_window_stats) + return; + + fixup_cumulative_runnable_avg(&rq->walt_stats, p->ravg.demand_scaled, + p->ravg.pred_demand_scaled); + + /* + * Add a task's contribution to the cumulative window demand when + * + * (1) task is enqueued with on_rq = 1 i.e migration, + * prio/cgroup/class change. + * (2) task is waking for the first time in this window. + */ + if (p->on_rq || (p->last_sleep_ts < rq->window_start)) + walt_fixup_cum_window_demand(rq, p->ravg.demand_scaled); +} + +static inline void +walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) +{ + if (sched_disable_window_stats) + return; + + fixup_cumulative_runnable_avg(&rq->walt_stats, + -(s64)p->ravg.demand_scaled, + -(s64)p->ravg.pred_demand_scaled); + + /* + * on_rq will be 1 for sleeping tasks. So check if the task + * is migrating or dequeuing in RUNNING state to change the + * prio/cgroup/class. + */ + if (task_on_rq_migrating(p) || p->state == TASK_RUNNING) + walt_fixup_cum_window_demand(rq, -(s64)p->ravg.demand_scaled); +} + +extern void __weak +fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, + u16 updated_demand_scaled, + u16 updated_pred_demand_scaled); +extern void __weak inc_rq_walt_stats(struct rq *rq, struct task_struct *p); +extern void __weak dec_rq_walt_stats(struct rq *rq, struct task_struct *p); +extern void __weak fixup_busy_time(struct task_struct *p, int new_cpu); +extern void __weak init_new_task_load(struct task_struct *p); +extern void __weak mark_task_starting(struct task_struct *p); +extern void __weak set_window_start(struct rq *rq); +extern bool __weak do_pl_notif(struct rq *rq); + +#define SCHED_HIGH_IRQ_TIMEOUT 3 +static inline u64 sched_irqload(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + s64 delta; + + delta = get_jiffies_64() - rq->irqload_ts; + /* + * Current context can be preempted by irq and rq->irqload_ts can be + * updated by irq context so that delta can be negative. + * But this is okay and we can safely return as this means there + * was recent irq occurrence. + */ + + if (delta < SCHED_HIGH_IRQ_TIMEOUT) + return rq->avg_irqload; + else + return 0; +} + +static inline int sched_cpu_high_irqload(int cpu) +{ + return sched_irqload(cpu) >= sysctl_sched_cpu_high_irqload; +} + +static inline int exiting_task(struct task_struct *p) +{ + return (p->ravg.sum_history[0] == EXITING_TASK_MARKER); +} + +static inline u64 +scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) +{ + return div64_u64(load * (u64)src_freq, (u64)dst_freq); +} + +extern void __weak sched_account_irqstart(int cpu, struct task_struct *curr, + u64 wallclock); + +static inline unsigned int max_task_load(void) +{ + return sched_ravg_window; +} + +extern void __weak update_cluster_topology(void); + +extern void __weak init_clusters(void); + +extern void sched_account_irqtime(int cpu, struct task_struct *curr, + u64 delta, u64 wallclock); + +static inline int same_cluster(int src_cpu, int dst_cpu) +{ + return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster; +} + +void __weak walt_sched_init_rq(struct rq *rq); + +static inline void walt_update_last_enqueue(struct task_struct *p) +{ + p->last_enqueued_ts = sched_ktime_clock(); +} + +static inline bool is_suh_max(void) +{ + return sysctl_sched_user_hint == sched_user_hint_max; +} + +#define DEFAULT_CGROUP_COLOC_ID 1 +static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu) +{ + struct related_thread_group *rtg = p->grp; + + if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID && + rtg->skip_min && p->unfilter) + return is_min_capacity_cpu(cpu); + + return false; +} + +extern bool is_rtgb_active(void); +extern u64 get_rtgb_active_time(void); + +/* utility function to update walt signals at wakeup */ +static inline void walt_try_to_wake_up(struct task_struct *p) +{ + struct rq *rq = cpu_rq(task_cpu(p)); + struct rq_flags rf; + u64 wallclock; + unsigned int old_load; + struct related_thread_group *grp = NULL; + + rq_lock_irqsave(rq, &rf); + old_load = task_load(p); + wallclock = sched_ktime_clock(); + walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + walt_update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); + note_task_waking(p, wallclock); + rq_unlock_irqrestore(rq, &rf); + + rcu_read_lock(); + grp = task_related_thread_group(p); + if (update_preferred_cluster(grp, p, old_load, false)) + set_preferred_cluster(grp); + rcu_read_unlock(); +} + +#else /* CONFIG_SCHED_WALT */ + +static inline void walt_sched_init_rq(struct rq *rq) { } +static inline void walt_update_last_enqueue(struct task_struct *p) { } +static inline void walt_update_task_ravg(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) { } +static inline void walt_inc_cumulative_runnable_avg(struct rq *rq, + struct task_struct *p) +{ +} + +static inline void walt_dec_cumulative_runnable_avg(struct rq *rq, + struct task_struct *p) +{ +} + +static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { } +static inline void init_new_task_load(struct task_struct *p) +{ +} + +static inline void mark_task_starting(struct task_struct *p) { } +static inline void set_window_start(struct rq *rq) { } +static inline int sched_cpu_high_irqload(int cpu) { return 0; } + +static inline void sched_account_irqstart(int cpu, struct task_struct *curr, + u64 wallclock) +{ +} + +static inline void update_cluster_topology(void) { } +static inline void init_clusters(void) {} +static inline void sched_account_irqtime(int cpu, struct task_struct *curr, + u64 delta, u64 wallclock) +{ +} + +static inline int same_cluster(int src_cpu, int dst_cpu) { return 1; } +static inline bool do_pl_notif(struct rq *rq) { return false; } + +static inline void +inc_rq_walt_stats(struct rq *rq, struct task_struct *p) { } + +static inline void +dec_rq_walt_stats(struct rq *rq, struct task_struct *p) { } + +static inline void +fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, + u16 updated_demand_scaled, + u16 updated_pred_demand_scaled) +{ +} + +static inline u64 sched_irqload(int cpu) +{ + return 0; +} + +static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu) +{ + return false; +} + +static inline u64 get_rtgb_active_time(void) +{ + return 0; +} + +#define walt_try_to_wake_up(a) {} + +#endif /* CONFIG_SCHED_WALT */ + +#endif diff --git a/kernel/smp.c b/kernel/smp.c index 4d14b8328456..d0a5f693be91 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -792,7 +792,8 @@ void wake_up_all_idle_cpus(void) if (cpu == smp_processor_id()) continue; - wake_up_if_idle(cpu); + if (!cpu_isolated(cpu)) + wake_up_if_idle(cpu); } preempt_enable(); } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index f14c9f30c394..e28fb2d87ca4 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -126,6 +127,7 @@ static int sixty = 60; #endif static int __maybe_unused neg_one = -1; + static int __maybe_unused two = 2; static int __maybe_unused four = 4; static unsigned long zero_ul; @@ -140,7 +142,12 @@ static int ten_thousand = 10000; static int six_hundred_forty_kb = 640 * 1024; #endif #ifdef CONFIG_SCHED_WALT +static int neg_three = -3; +static int three = 3; static int two_hundred_fifty_five = 255; +const int sched_user_hint_max = 1000; +static unsigned int ns_per_sec = NSEC_PER_SEC; +static unsigned int one_hundred_thousand = 100000; #endif /* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */ @@ -231,6 +238,10 @@ static int proc_dostring_coredump(struct ctl_table *table, int write, #endif static int proc_dopipe_max_size(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +#ifdef CONFIG_SCHED_WALT +static int proc_douintvec_minmax_schedhyst(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); +#endif #ifdef CONFIG_MAGIC_SYSRQ /* Note: sysrq code uses its own private copy */ @@ -328,6 +339,172 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, +#ifdef CONFIG_SCHED_WALT + { + .procname = "sched_user_hint", + .data = &sysctl_sched_user_hint, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_user_hint_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = (void *)&sched_user_hint_max, + }, + { + .procname = "sched_window_stats_policy", + .data = &sysctl_sched_window_stats_policy, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &four, + }, + { + .procname = "sched_cpu_high_irqload", + .data = &sysctl_sched_cpu_high_irqload, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_group_upmigrate", + .data = &sysctl_sched_group_upmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_group_thresholds_handler, + .extra1 = &sysctl_sched_group_downmigrate_pct, + }, + { + .procname = "sched_group_downmigrate", + .data = &sysctl_sched_group_downmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = walt_proc_group_thresholds_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = &sysctl_sched_group_upmigrate_pct, + }, + { + .procname = "sched_boost", + .data = &sysctl_sched_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_boost_handler, + .extra1 = &neg_three, + .extra2 = &three, + }, + { + .procname = "sched_conservative_pl", + .data = &sysctl_sched_conservative_pl, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_walt_rotate_big_tasks", + .data = &sysctl_sched_walt_rotate_big_tasks, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sched_min_task_util_for_boost", + .data = &sysctl_sched_min_task_util_for_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_min_task_util_for_colocation", + .data = &sysctl_sched_min_task_util_for_colocation, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_thousand, + }, + { + .procname = "sched_asym_cap_sibling_freq_match_pct", + .data = &sysctl_sched_asym_cap_sibling_freq_match_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &one_hundred, + }, + { + .procname = "sched_coloc_downmigrate_ns", + .data = &sysctl_sched_coloc_downmigrate_ns, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + }, + { + .procname = "sched_task_unfilter_nr_windows", + .data = &sysctl_sched_task_unfilter_nr_windows, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax_schedhyst, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_busy_hyst_ns", + .data = &sysctl_sched_busy_hyst, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax_schedhyst, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_coloc_busy_hysteresis_enable_cpus", + .data = &sysctl_sched_coloc_busy_hyst_enable_cpus, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax_schedhyst, + .extra1 = SYSCTL_ZERO, + .extra2 = &two_hundred_fifty_five, + }, + { + .procname = "sched_coloc_busy_hyst_ns", + .data = &sysctl_sched_coloc_busy_hyst, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax_schedhyst, + .extra1 = SYSCTL_ZERO, + .extra2 = &ns_per_sec, + }, + { + .procname = "sched_coloc_busy_hyst_max_ms", + .data = &sysctl_sched_coloc_busy_hyst_max_ms, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax_schedhyst, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred_thousand, + }, + { + .procname = "sched_ravg_window_nr_ticks", + .data = &sysctl_sched_ravg_window_nr_ticks, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_ravg_window_handler, + }, +#endif #ifdef CONFIG_SCHED_DEBUG { .procname = "sched_min_granularity_ns", @@ -2874,6 +3051,19 @@ static int proc_dostring_coredump(struct ctl_table *table, int write, } #endif +#ifdef CONFIG_SCHED_WALT +static int proc_douintvec_minmax_schedhyst(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = proc_douintvec_minmax(table, write, buffer, lenp, ppos); + + if (!ret && write) + sched_update_hyst_times(); + + return ret; +} +#endif + static int __do_proc_doulongvec_minmax(void *data, struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos, @@ -3341,6 +3531,29 @@ int proc_do_large_bitmap(struct ctl_table *table, int write, return err; } +static int do_proc_douintvec_rwin(bool *negp, unsigned long *lvalp, + int *valp, int write, void *data) +{ + if (write) { + if (*lvalp == 0 || *lvalp == 2 || *lvalp == 5) + *valp = *lvalp; + else + return -EINVAL; + } else { + *negp = false; + *lvalp = *valp; + } + + return 0; +} + +int proc_douintvec_ravg_window(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + return do_proc_dointvec(table, write, buffer, lenp, ppos, + do_proc_douintvec_rwin, NULL); +} + #else /* CONFIG_PROC_SYSCTL */ int proc_dostring(struct ctl_table *table, int write, @@ -3410,6 +3623,12 @@ int proc_do_large_bitmap(struct ctl_table *table, int write, return -ENOSYS; } +int proc_douintvec_ravg_window(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + #endif /* CONFIG_PROC_SYSCTL */ #if defined(CONFIG_SYSCTL) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 2feb5db24220..f53495f36177 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2014,7 +2014,7 @@ int hrtimers_prepare_cpu(unsigned int cpu) return 0; } -#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_CPUSETS) +#ifdef CONFIG_HOTPLUG_CPU static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, struct hrtimer_clock_base *new_base, bool remove_pinned) @@ -2023,12 +2023,14 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, struct timerqueue_node *node; struct timerqueue_head pinned; int is_pinned; + bool is_hotplug = !cpu_online(old_base->cpu_base->cpu); timerqueue_init_head(&pinned); while ((node = timerqueue_getnext(&old_base->active))) { timer = container_of(node, struct hrtimer, node); - BUG_ON(hrtimer_callback_running(timer)); + if (is_hotplug) + BUG_ON(hrtimer_callback_running(timer)); debug_deactivate(timer); /* @@ -2106,9 +2108,7 @@ static void __migrate_hrtimers(unsigned int scpu, bool remove_pinned) local_irq_restore(flags); local_bh_enable(); } -#endif /* CONFIG_HOTPLUG_CPU || CONFIG_CPUSETS */ -#ifdef CONFIG_HOTPLUG_CPU int hrtimers_dead_cpu(unsigned int scpu) { BUG_ON(cpu_online(scpu)); @@ -2117,14 +2117,13 @@ int hrtimers_dead_cpu(unsigned int scpu) __migrate_hrtimers(scpu, true); return 0; } -#endif /* CONFIG_HOTPLUG_CPU */ -#ifdef CONFIG_CPUSETS void hrtimer_quiesce_cpu(void *cpup) { __migrate_hrtimers(*(int *)cpup, false); } -#endif /* CONFIG_CPUSETS */ + +#endif /* CONFIG_HOTPLUG_CPU */ void __init hrtimers_init(void) { diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 7f772cdeca5d..0c1122058d51 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2038,7 +2038,8 @@ static void __migrate_timers(unsigned int cpu, bool remove_pinned) */ forward_timer_base(new_base); - BUG_ON(old_base->running_timer); + if (!cpu_online(cpu)) + BUG_ON(old_base->running_timer); for (i = 0; i < WHEEL_SIZE; i++) migrate_timer_list(new_base, old_base->vectors + i, @@ -2057,12 +2058,10 @@ int timers_dead_cpu(unsigned int cpu) return 0; } -#ifdef CONFIG_CPUSETS void timer_quiesce_cpu(void *cpup) { __migrate_timers(*(unsigned int *)cpup, false); } -#endif /* CONFIG_CPUSETS */ #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/kernel/watchdog.c b/kernel/watchdog.c index f41334ef0971..3154a257c2bd 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -170,6 +171,7 @@ static u64 __read_mostly sample_period; static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts); static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer); +static DEFINE_PER_CPU(unsigned int, watchdog_en); static DEFINE_PER_CPU(bool, softlockup_touch_sync); static DEFINE_PER_CPU(bool, soft_watchdog_warn); static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts); @@ -476,16 +478,20 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) return HRTIMER_RESTART; } -static void watchdog_enable(unsigned int cpu) +void watchdog_enable(unsigned int cpu) { struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); struct completion *done = this_cpu_ptr(&softlockup_completion); + unsigned int *enabled = this_cpu_ptr(&watchdog_en); WARN_ON_ONCE(cpu != smp_processor_id()); init_completion(done); complete(done); + if (*enabled) + return; + /* * Start the timer first to prevent the NMI watchdog triggering * before the timer has a chance to fire. @@ -500,11 +506,24 @@ static void watchdog_enable(unsigned int cpu) /* Enable the perf event */ if (watchdog_enabled & NMI_WATCHDOG_ENABLED) watchdog_nmi_enable(cpu); + + /* + * Need to ensure above operations are observed by other CPUs before + * indicating that timer is enabled. This is to synchronize core + * isolation and hotplug. Core isolation will wait for this flag to be + * set. + */ + mb(); + *enabled = 1; } -static void watchdog_disable(unsigned int cpu) +void watchdog_disable(unsigned int cpu) { struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); + unsigned int *enabled = this_cpu_ptr(&watchdog_en); + + if (!*enabled) + return; WARN_ON_ONCE(cpu != smp_processor_id()); @@ -516,6 +535,17 @@ static void watchdog_disable(unsigned int cpu) watchdog_nmi_disable(cpu); hrtimer_cancel(hrtimer); wait_for_completion(this_cpu_ptr(&softlockup_completion)); + + /* + * No need for barrier here since disabling the watchdog is + * synchronized with hotplug lock + */ + *enabled = 0; +} + +bool watchdog_configured(unsigned int cpu) +{ + return *per_cpu_ptr(&watchdog_en, cpu); } static int softlockup_stop_fn(void *data) diff --git a/mm/vmstat.c b/mm/vmstat.c index a8222041bd44..9c1e5fa99032 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1807,7 +1807,7 @@ int vmstat_refresh(struct ctl_table *table, int write, static void vmstat_update(struct work_struct *w) { - if (refresh_cpu_vm_stats(true)) { + if (refresh_cpu_vm_stats(true) && !cpu_isolated(smp_processor_id())) { /* * Counters were updated so we expect more updates * to occur in the future. Keep on running the @@ -1899,7 +1899,8 @@ static void vmstat_shepherd(struct work_struct *w) for_each_online_cpu(cpu) { struct delayed_work *dw = &per_cpu(vmstat_work, cpu); - if (!delayed_work_pending(dw) && need_update(cpu)) + if (!delayed_work_pending(dw) && need_update(cpu) && + !cpu_isolated(cpu)) queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); } put_online_cpus();