diff --git a/fs/proc/base.c b/fs/proc/base.c index b057ccdbd9f9..b61a22d99ab7 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1464,12 +1464,6 @@ static const struct file_operations proc_pid_sched_operations = { */ #ifdef CONFIG_SCHED_WALT -extern int __weak sched_wake_up_idle_show(struct seq_file *m, void *v); -extern ssize_t __weak sched_wake_up_idle_write(struct file *file, - const char __user *buf, size_t count, loff_t *offset); -extern int __weak sched_wake_up_idle_open(struct inode *inode, - struct file *filp); - static const struct file_operations proc_pid_sched_wake_up_idle_operations = { .open = sched_wake_up_idle_open, .read = seq_read, @@ -1478,13 +1472,6 @@ static const struct file_operations proc_pid_sched_wake_up_idle_operations = { .release = single_release, }; -extern int __weak sched_init_task_load_show(struct seq_file *m, void *v); -extern ssize_t __weak -sched_init_task_load_write(struct file *file, const char __user *buf, - size_t count, loff_t *offset); -extern int __weak -sched_init_task_load_open(struct inode *inode, struct file *filp); - static const struct file_operations proc_pid_sched_init_task_load_operations = { .open = sched_init_task_load_open, .read = seq_read, @@ -1493,12 +1480,6 @@ static const struct file_operations proc_pid_sched_init_task_load_operations = { .release = single_release, }; -extern int __weak sched_group_id_show(struct seq_file *m, void *v); -extern ssize_t __weak -sched_group_id_write(struct file *file, const char __user *buf, - size_t count, loff_t *offset); -extern int __weak sched_group_id_open(struct inode *inode, struct file *filp); - static const struct file_operations proc_pid_sched_group_id_operations = { .open = sched_group_id_open, .read = seq_read, diff --git a/include/linux/sched.h b/include/linux/sched.h index dfc6940f024f..83a3fc23dfea 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -248,9 +248,9 @@ enum migrate_types { }; #ifdef CONFIG_HOTPLUG_CPU -extern int __weak sched_isolate_cpu(int cpu); -extern int __weak sched_unisolate_cpu(int cpu); -extern int __weak sched_unisolate_cpu_unlocked(int cpu); +extern int sched_isolate_cpu(int cpu); +extern int sched_unisolate_cpu(int cpu); +extern int sched_unisolate_cpu_unlocked(int cpu); #else static inline int sched_isolate_cpu(int cpu) { @@ -541,13 +541,13 @@ struct cpu_cycle_counter_cb { DECLARE_PER_CPU_READ_MOSTLY(int, sched_load_boost); #ifdef CONFIG_SCHED_WALT -extern void __weak sched_exit(struct task_struct *p); +extern void sched_exit(struct task_struct *p); extern int __weak register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb); extern void __weak sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax); -extern void __weak free_task_load_ptrs(struct task_struct *p); -extern void __weak sched_set_refresh_rate(enum fps fps); +extern void free_task_load_ptrs(struct task_struct *p); +extern void sched_set_refresh_rate(enum fps fps); extern int set_task_boost(int boost, u64 period); #define RAVG_HIST_SIZE_MAX 5 diff --git a/include/linux/sched/core_ctl.h b/include/linux/sched/core_ctl.h index 83c742ddf549..edfd87aa69b1 100644 --- a/include/linux/sched/core_ctl.h +++ b/include/linux/sched/core_ctl.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2016, 2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2016, 2019-2020, The Linux Foundation. All rights reserved. */ #ifndef __CORE_CTL_H @@ -21,9 +21,9 @@ struct core_ctl_notif_data { struct notifier_block; #ifdef CONFIG_SCHED_WALT -extern int __weak core_ctl_set_boost(bool boost); -extern void __weak core_ctl_notifier_register(struct notifier_block *n); -extern void __weak core_ctl_notifier_unregister(struct notifier_block *n); +extern int core_ctl_set_boost(bool boost); +extern void core_ctl_notifier_register(struct notifier_block *n); +extern void core_ctl_notifier_unregister(struct notifier_block *n); #else static inline int core_ctl_set_boost(bool boost) { diff --git a/include/linux/sched/stat.h b/include/linux/sched/stat.h index 1eed4a2526a5..60cf021ddff6 100644 --- a/include/linux/sched/stat.h +++ b/include/linux/sched/stat.h @@ -22,20 +22,35 @@ extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); #ifdef CONFIG_SCHED_WALT -extern void __weak sched_update_nr_prod(int cpu, long delta, bool inc); -extern unsigned int __weak sched_get_cpu_util(int cpu); -extern void __weak sched_update_hyst_times(void); -extern u64 __weak sched_lpm_disallowed_time(int cpu); +extern void sched_update_nr_prod(int cpu, long delta, bool inc); +extern unsigned int sched_get_cpu_util(int cpu); +extern void sched_update_hyst_times(void); +extern u64 sched_lpm_disallowed_time(int cpu); + +extern int sched_wake_up_idle_show(struct seq_file *m, void *v); +extern ssize_t sched_wake_up_idle_write(struct file *file, + const char __user *buf, size_t count, loff_t *offset); +extern int sched_wake_up_idle_open(struct inode *inode, + struct file *filp); + +extern int sched_init_task_load_show(struct seq_file *m, void *v); +extern ssize_t +sched_init_task_load_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset); +extern int +sched_init_task_load_open(struct inode *inode, struct file *filp); + +extern int sched_group_id_show(struct seq_file *m, void *v); +extern ssize_t +sched_group_id_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset); +extern int sched_group_id_open(struct inode *inode, struct file *filp); #else static inline void sched_update_nr_prod(int cpu, long delta, bool inc) {} static inline unsigned int sched_get_cpu_util(int cpu) { return 0; } -static inline u64 sched_get_cpu_last_busy_time(int cpu) -{ - return 0; -} static inline void sched_update_hyst_times(void) {} static inline u64 sched_lpm_disallowed_time(int cpu) { diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index aac23569af5f..c0aad38a993d 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -54,24 +54,27 @@ extern unsigned int __weak sysctl_sched_ravg_window_nr_ticks; extern unsigned int __weak sysctl_sched_many_wakeup_threshold; extern unsigned int __weak sysctl_sched_dynamic_ravg_window_enable; -extern int __weak +extern int walt_proc_group_thresholds_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); -extern int __weak +extern int walt_proc_user_hint_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); -extern int __weak +extern int sched_updown_migrate_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); -extern int __weak +extern int sched_ravg_window_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); + +extern int sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); #endif #if defined(CONFIG_PREEMPTIRQ_EVENTS) || defined(CONFIG_PREEMPT_TRACER) @@ -103,10 +106,6 @@ int sched_proc_update_handler(struct ctl_table *table, int write, loff_t *ppos); #endif -#ifdef CONFIG_SCHED_WALT -extern int __weak sched_boost_handler(struct ctl_table *table, int write, - void __user *buffer, size_t *lenp, loff_t *ppos); -#endif /* * control realtime throttling: * diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 21fb5a5662b5..1175ed6ccf4d 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -20,6 +20,7 @@ obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle.o fair.o rt.o deadline.o obj-y += wait.o wait_bit.o swait.o completion.o +obj-$(CONFIG_SCHED_WALT) += walt.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o pelt.o obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o obj-$(CONFIG_SCHEDSTATS) += stats.o diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 51ac859e2c3f..b4d5a06f1076 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2162,8 +2162,8 @@ static inline int hrtick_enabled(struct rq *rq) #endif /* CONFIG_SCHED_HRTICK */ #ifdef CONFIG_SCHED_WALT -u64 __weak sched_ktime_clock(void); -unsigned long __weak +u64 sched_ktime_clock(void); +unsigned long cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load); #else #define sched_ravg_window TICK_NSEC @@ -2908,10 +2908,10 @@ extern unsigned int max_power_cost; extern unsigned int __read_mostly __weak sched_init_task_load_windows; extern unsigned int __read_mostly __weak sched_load_granule; -extern int __weak update_preferred_cluster(struct related_thread_group *grp, +extern int update_preferred_cluster(struct related_thread_group *grp, struct task_struct *p, u32 old_load, bool from_tick); -extern void __weak set_preferred_cluster(struct related_thread_group *grp); -extern void __weak add_new_task_to_grp(struct task_struct *new); +extern void set_preferred_cluster(struct related_thread_group *grp); +extern void add_new_task_to_grp(struct task_struct *new); #define NO_BOOST 0 #define FULL_THROTTLE_BOOST 1 @@ -3058,7 +3058,7 @@ static inline bool is_full_throttle_boost(void) return sched_boost() == FULL_THROTTLE_BOOST; } -extern int __weak preferred_cluster(struct sched_cluster *cluster, +extern int preferred_cluster(struct sched_cluster *cluster, struct task_struct *p); extern struct sched_cluster *rq_cluster(struct rq *rq); @@ -3075,7 +3075,7 @@ static inline bool task_sched_boost(struct task_struct *p) return tg->sched_boost_enabled; } -extern int __weak sync_cgroup_colocation(struct task_struct *p, bool insert); +extern int sync_cgroup_colocation(struct task_struct *p, bool insert); extern void update_cgroup_boost_settings(void); extern void restore_cgroup_boost_settings(void); #else @@ -3094,9 +3094,9 @@ static inline void update_cgroup_boost_settings(void) { } static inline void restore_cgroup_boost_settings(void) { } #endif -extern int __weak alloc_related_thread_groups(void); +extern int alloc_related_thread_groups(void); -extern void __weak check_for_migration(struct rq *rq, struct task_struct *p); +extern void check_for_migration(struct rq *rq, struct task_struct *p); static inline int is_reserved(int cpu) { @@ -3133,21 +3133,21 @@ static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta) rq->cum_window_demand_scaled = 0; } -extern unsigned long __weak thermal_cap(int cpu); +extern unsigned long thermal_cap(int cpu); -extern void __weak clear_walt_request(int cpu); +extern void clear_walt_request(int cpu); extern enum sched_boost_policy sched_boost_policy(void); extern void sched_boost_parse_dt(void); -extern void __weak clear_ed_task(struct task_struct *p, struct rq *rq); -extern bool __weak early_detection_notify(struct rq *rq, u64 wallclock); +extern void clear_ed_task(struct task_struct *p, struct rq *rq); +extern bool early_detection_notify(struct rq *rq, u64 wallclock); static inline unsigned int power_cost(int cpu, u64 demand) { return cpu_max_possible_capacity(cpu); } -void __weak note_task_waking(struct task_struct *p, u64 wallclock); +void note_task_waking(struct task_struct *p, u64 wallclock); static inline bool task_placement_boost_enabled(struct task_struct *p) { @@ -3180,11 +3180,11 @@ static inline bool is_min_capacity_cluster(struct sched_cluster *cluster) return is_min_capacity_cpu(cluster_first_cpu(cluster)); } -extern void __weak walt_fixup_sched_stats_fair(struct rq *rq, +extern void walt_fixup_sched_stats_fair(struct rq *rq, struct task_struct *p, u16 updated_demand_scaled, u16 updated_pred_demand_scaled); -extern void __weak walt_fixup_nr_big_tasks(struct rq *rq, struct task_struct *p, +extern void walt_fixup_nr_big_tasks(struct rq *rq, struct task_struct *p, int delta, bool inc); #else /* CONFIG_SCHED_WALT */ @@ -3362,7 +3362,7 @@ walt_dec_cfs_rq_stats(struct cfs_rq *cfs_rq, struct task_struct *p) {} #ifdef CONFIG_SMP #ifdef CONFIG_SCHED_WALT -extern int __weak group_balance_cpu_not_isolated(struct sched_group *sg); +extern int group_balance_cpu_not_isolated(struct sched_group *sg); #else static inline int group_balance_cpu_not_isolated(struct sched_group *sg) { @@ -3382,10 +3382,10 @@ extern void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, bool migrate_pinned_tasks); extern void calc_load_migrate(struct rq *rq); #ifdef CONFIG_SCHED_WALT -extern void __weak +extern void detach_one_task_core(struct task_struct *p, struct rq *rq, struct list_head *tasks); -extern void __weak attach_tasks_core(struct list_head *tasks, struct rq *rq); +extern void attach_tasks_core(struct list_head *tasks, struct rq *rq); #else static inline void detach_one_task_core(struct task_struct *p, struct rq *rq, diff --git a/kernel/sched/walt.c b/kernel/sched/walt.c new file mode 100644 index 000000000000..f68e0a393451 --- /dev/null +++ b/kernel/sched/walt.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved. + */ + +#include "sched.h" +#include "walt.h" + +int __weak sched_wake_up_idle_show(struct seq_file *m, void *v) +{ + return -EPERM; +} + +ssize_t __weak sched_wake_up_idle_write(struct file *file, + const char __user *buf, size_t count, loff_t *offset) +{ + return -EPERM; +} + +int __weak sched_wake_up_idle_open(struct inode *inode, struct file *filp) +{ + return -EPERM; +} + +int __weak sched_init_task_load_show(struct seq_file *m, void *v) +{ + return -EPERM; +} + +ssize_t __weak +sched_init_task_load_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + return -EPERM; +} + +int __weak sched_init_task_load_open(struct inode *inode, struct file *filp) +{ + return -EPERM; +} + +int __weak sched_group_id_show(struct seq_file *m, void *v) +{ + return -EPERM; +} + +ssize_t __weak sched_group_id_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + return -EPERM; +} + +int __weak sched_group_id_open(struct inode *inode, struct file *filp) +{ + return -EPERM; +} + +int __weak sched_isolate_cpu(int cpu) { return 0; } + +int __weak sched_unisolate_cpu(int cpu) { return 0; } + +int __weak sched_unisolate_cpu_unlocked(int cpu) { return 0; } + +void __weak sched_exit(struct task_struct *p) { } + +int __weak register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb) +{ + return 0; +} + +void __weak sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, + u32 fmax) { } + +void __weak free_task_load_ptrs(struct task_struct *p) { } + +void __weak sched_set_refresh_rate(enum fps fps) { } + +int __weak core_ctl_set_boost(bool boost) { return 0; } + +void __weak core_ctl_notifier_register(struct notifier_block *n) { } + +void __weak core_ctl_notifier_unregister(struct notifier_block *n) { } + +void __weak sched_update_nr_prod(int cpu, long delta, bool inc) { } + +unsigned int __weak sched_get_cpu_util(int cpu) { return 0; } + +void __weak sched_update_hyst_times(void) { } + +u64 __weak sched_lpm_disallowed_time(int cpu) { return 0; } + +int __weak +walt_proc_group_thresholds_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + +int __weak +walt_proc_user_hint_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + +int __weak +sched_updown_migrate_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + +int __weak +sched_ravg_window_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + +int __weak sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + return -ENOSYS; +} + +u64 __weak sched_ktime_clock(void) { return 0; } + +unsigned long __weak +cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load) +{ + return cpu_util(cpu); +} + +int __weak update_preferred_cluster(struct related_thread_group *grp, + struct task_struct *p, u32 old_load, bool from_tick) +{ + return 0; +} + +void __weak set_preferred_cluster(struct related_thread_group *grp) { } + +void __weak add_new_task_to_grp(struct task_struct *new) { } + +int __weak +preferred_cluster(struct sched_cluster *cluster, struct task_struct *p) +{ + return -1; +} + +int __weak sync_cgroup_colocation(struct task_struct *p, bool insert) +{ + return 0; +} + +int __weak alloc_related_thread_groups(void) { return 0; } + +void __weak check_for_migration(struct rq *rq, struct task_struct *p) { } + +unsigned long __weak thermal_cap(int cpu) +{ + return cpu_rq(cpu)->cpu_capacity_orig; +} + +void __weak clear_walt_request(int cpu) { } + +void __weak clear_ed_task(struct task_struct *p, struct rq *rq) { } + +bool __weak early_detection_notify(struct rq *rq, u64 wallclock) +{ + return 0; +} + +void __weak note_task_waking(struct task_struct *p, u64 wallclock) { } + +void __weak walt_fixup_sched_stats_fair(struct rq *rq, struct task_struct *p, + u16 updated_demand_scaled, u16 updated_pred_demand_scaled) { } + +void __weak walt_fixup_nr_big_tasks(struct rq *rq, struct task_struct *p, + int delta, bool inc) { } + +int __weak group_balance_cpu_not_isolated(struct sched_group *sg) +{ + return group_balance_cpu(sg); +} + +void __weak detach_one_task_core(struct task_struct *p, struct rq *rq, + struct list_head *tasks) { } + +void __weak attach_tasks_core(struct list_head *tasks, struct rq *rq) { } + +void __weak walt_update_task_ravg(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) { } + +void __weak fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, + u16 updated_demand_scaled, u16 updated_pred_demand_scaled) { } + +void __weak inc_rq_walt_stats(struct rq *rq, struct task_struct *p) { } + +void __weak dec_rq_walt_stats(struct rq *rq, struct task_struct *p) { } + +void __weak fixup_busy_time(struct task_struct *p, int new_cpu) { } + +void __weak init_new_task_load(struct task_struct *p) { } + +void __weak mark_task_starting(struct task_struct *p) { } + +void __weak set_window_start(struct rq *rq) { } + +bool __weak do_pl_notif(struct rq *rq) { return false; } + +void __weak sched_account_irqstart(int cpu, struct task_struct *curr, + u64 wallclock) { } + +void __weak update_cluster_topology(void) { } + +void __weak init_clusters(void) { } + +void __weak walt_sched_init_rq(struct rq *rq) { } diff --git a/kernel/sched/walt.h b/kernel/sched/walt.h index d670cd053e6b..594466d44f29 100644 --- a/kernel/sched/walt.h +++ b/kernel/sched/walt.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved. */ #ifndef __WALT_H @@ -15,7 +15,7 @@ extern unsigned int __weak walt_rotation_enabled; -extern void __weak +extern void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, u64 wallclock, u64 irqtime); @@ -73,17 +73,17 @@ walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) walt_fixup_cum_window_demand(rq, -(s64)p->ravg.demand_scaled); } -extern void __weak +extern void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, u16 updated_demand_scaled, u16 updated_pred_demand_scaled); -extern void __weak inc_rq_walt_stats(struct rq *rq, struct task_struct *p); -extern void __weak dec_rq_walt_stats(struct rq *rq, struct task_struct *p); -extern void __weak fixup_busy_time(struct task_struct *p, int new_cpu); -extern void __weak init_new_task_load(struct task_struct *p); -extern void __weak mark_task_starting(struct task_struct *p); -extern void __weak set_window_start(struct rq *rq); -extern bool __weak do_pl_notif(struct rq *rq); +extern void inc_rq_walt_stats(struct rq *rq, struct task_struct *p); +extern void dec_rq_walt_stats(struct rq *rq, struct task_struct *p); +extern void fixup_busy_time(struct task_struct *p, int new_cpu); +extern void init_new_task_load(struct task_struct *p); +extern void mark_task_starting(struct task_struct *p); +extern void set_window_start(struct rq *rq); +extern bool do_pl_notif(struct rq *rq); /* * This is only for tracepoints to print the avg irq load. For @@ -125,7 +125,7 @@ scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) return div64_u64(load * (u64)src_freq, (u64)dst_freq); } -extern void __weak sched_account_irqstart(int cpu, struct task_struct *curr, +extern void sched_account_irqstart(int cpu, struct task_struct *curr, u64 wallclock); static inline unsigned int max_task_load(void) @@ -133,9 +133,9 @@ static inline unsigned int max_task_load(void) return sched_ravg_window; } -extern void __weak update_cluster_topology(void); +extern void update_cluster_topology(void); -extern void __weak init_clusters(void); +extern void init_clusters(void); extern void sched_account_irqtime(int cpu, struct task_struct *curr, u64 delta, u64 wallclock); @@ -145,7 +145,7 @@ static inline int same_cluster(int src_cpu, int dst_cpu) return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster; } -void __weak walt_sched_init_rq(struct rq *rq); +void walt_sched_init_rq(struct rq *rq); static inline void walt_update_last_enqueue(struct task_struct *p) {