diff --git a/include/linux/sched.h b/include/linux/sched.h index 41d4221170fa..e41c704d2fe5 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -619,7 +619,7 @@ struct walt_task_struct { u32 unfilter; u64 last_wake_ts; u64 last_enqueued_ts; - struct related_thread_group __rcu *grp; + struct walt_related_thread_group __rcu *grp; struct list_head grp_list; u64 cpu_cycles; cpumask_t cpus_requested; diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index eb26ee52d94b..73a5fa7a6368 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -475,11 +475,11 @@ TRACE_EVENT(sched_load_balance_skip_tasks, TP_fast_assign( __entry->scpu = scpu; __entry->src_util_cum = - cpu_rq(scpu)->cum_window_demand_scaled; + cpu_rq(scpu)->wrq.cum_window_demand_scaled; __entry->grp_type = grp_type; __entry->dcpu = dcpu; __entry->dst_util_cum = - cpu_rq(dcpu)->cum_window_demand_scaled; + cpu_rq(dcpu)->wrq.cum_window_demand_scaled; __entry->pid = pid; __entry->affinity = affinity; __entry->task_util = task_util; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index dd29df7beee1..df909a91870c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1338,7 +1338,7 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) uclamp_rq_dec(rq, p); p->sched_class->dequeue_task(rq, p, flags); #ifdef CONFIG_SCHED_WALT - if (p == rq->ed_task) + if (p == rq->wrq.ed_task) early_detection_notify(rq, sched_ktime_clock()); #endif trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]); @@ -3697,7 +3697,7 @@ void scheduler_tick(void) u64 wallclock; bool early_notif; u32 old_load; - struct related_thread_group *grp; + struct walt_related_thread_group *grp; unsigned int flag = 0; sched_clock_tick(); @@ -7202,7 +7202,7 @@ static void walt_schedgp_attach(struct cgroup_taskset *tset) cgroup_taskset_first(tset, &css); tg = css_tg(css); - colocate = tg->colocate; + colocate = tg->wtg.colocate; cgroup_taskset_for_each(task, css, tset) sync_cgroup_colocation(task, colocate); @@ -7214,7 +7214,7 @@ sched_boost_override_read(struct cgroup_subsys_state *css, { struct task_group *tg = css_tg(css); - return (u64) tg->sched_boost_no_override; + return (u64) tg->wtg.sched_boost_no_override; } static int sched_boost_override_write(struct cgroup_subsys_state *css, @@ -7222,7 +7222,7 @@ static int sched_boost_override_write(struct cgroup_subsys_state *css, { struct task_group *tg = css_tg(css); - tg->sched_boost_no_override = !!override; + tg->wtg.sched_boost_no_override = !!override; return 0; } @@ -7231,7 +7231,7 @@ static u64 sched_colocate_read(struct cgroup_subsys_state *css, { struct task_group *tg = css_tg(css); - return (u64) tg->colocate; + return (u64) tg->wtg.colocate; } static int sched_colocate_write(struct cgroup_subsys_state *css, @@ -7239,11 +7239,11 @@ static int sched_colocate_write(struct cgroup_subsys_state *css, { struct task_group *tg = css_tg(css); - if (tg->colocate_update_disabled) + if (tg->wtg.colocate_update_disabled) return -EPERM; - tg->colocate = !!colocate; - tg->colocate_update_disabled = true; + tg->wtg.colocate = !!colocate; + tg->wtg.colocate_update_disabled = true; return 0; } #else @@ -8412,25 +8412,25 @@ void sched_account_irqtime(int cpu, struct task_struct *curr, walt_update_task_ravg(curr, rq, IRQ_UPDATE, sched_ktime_clock(), delta); - nr_ticks = cur_jiffies_ts - rq->irqload_ts; + nr_ticks = cur_jiffies_ts - rq->wrq.irqload_ts; if (nr_ticks) { if (nr_ticks < 10) { /* Decay CPU's irqload by 3/4 for each window. */ - rq->avg_irqload *= (3 * nr_ticks); - rq->avg_irqload = div64_u64(rq->avg_irqload, + rq->wrq.avg_irqload *= (3 * nr_ticks); + rq->wrq.avg_irqload = div64_u64(rq->wrq.avg_irqload, 4 * nr_ticks); } else { - rq->avg_irqload = 0; + rq->wrq.avg_irqload = 0; } - rq->avg_irqload += rq->cur_irqload; - rq->high_irqload = (rq->avg_irqload >= + rq->wrq.avg_irqload += rq->wrq.cur_irqload; + rq->wrq.high_irqload = (rq->wrq.avg_irqload >= sysctl_sched_cpu_high_irqload); - rq->cur_irqload = 0; + rq->wrq.cur_irqload = 0; } - rq->cur_irqload += delta; - rq->irqload_ts = cur_jiffies_ts; + rq->wrq.cur_irqload += delta; + rq->wrq.irqload_ts = cur_jiffies_ts; raw_spin_unlock_irqrestore(&rq->lock, flags); } #endif diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 91d615110187..0fc9aafb04cc 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -65,7 +65,7 @@ struct sugov_cpu { unsigned int iowait_boost; u64 last_update; - struct sched_walt_cpu_load walt_load; + struct walt_cpu_load walt_load; unsigned long util; unsigned int flags; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 4870c042ad37..9b3d8f536270 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -650,10 +650,10 @@ do { \ P(cpu_capacity); #endif #ifdef CONFIG_SCHED_WALT - P(cluster->cur_freq); - P(walt_stats.nr_big_tasks); + P(wrq.cluster->cur_freq); + P(wrq.walt_stats.nr_big_tasks); SEQ_printf(m, " .%-30s: %llu\n", "walt_stats.cumulative_runnable_avg", - rq->walt_stats.cumulative_runnable_avg_scaled); + rq->wrq.walt_stats.cumulative_runnable_avg_scaled); #endif #undef P #undef PN diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 34c872ef6875..6590539ea6f3 100755 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6417,7 +6417,7 @@ unsigned long capacity_curr_of(int cpu) #ifdef CONFIG_SCHED_WALT static inline bool walt_get_rtg_status(struct task_struct *p) { - struct related_thread_group *grp; + struct walt_related_thread_group *grp; bool ret = false; rcu_read_lock(); @@ -6708,7 +6708,7 @@ static inline unsigned long cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu) { unsigned long util = - cpu_rq(cpu)->walt_stats.cumulative_runnable_avg_scaled; + cpu_rq(cpu)->wrq.walt_stats.cumulative_runnable_avg_scaled; bool queued = task_on_rq_queued(p); /* @@ -8106,7 +8106,8 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) } if (env->flags & LBF_IGNORE_PREFERRED_CLUSTER_TASKS && - !preferred_cluster(cpu_rq(env->dst_cpu)->cluster, p)) + !preferred_cluster( + cpu_rq(env->dst_cpu)->wrq.cluster, p)) return 0; /* Don't detach task if it doesn't fit on the destination */ @@ -8115,7 +8116,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) return 0; /* Don't detach task if it is under active migration */ - if (env->src_rq->push_task == p) + if (env->src_rq->wrq.push_task == p) return 0; #endif @@ -10418,7 +10419,7 @@ int active_load_balance_cpu_stop(void *data) BUG_ON(busiest_rq == target_rq); #ifdef CONFIG_SCHED_WALT - push_task = busiest_rq->push_task; + push_task = busiest_rq->wrq.push_task; target_cpu = busiest_rq->push_cpu; if (push_task) { if (task_on_rq_queued(push_task) && @@ -10474,14 +10475,14 @@ int active_load_balance_cpu_stop(void *data) out_unlock: busiest_rq->active_balance = 0; #ifdef CONFIG_SCHED_WALT - push_task = busiest_rq->push_task; + push_task = busiest_rq->wrq.push_task; #endif target_cpu = busiest_rq->push_cpu; clear_reserved(target_cpu); #ifdef CONFIG_SCHED_WALT if (push_task) - busiest_rq->push_task = NULL; + busiest_rq->wrq.push_task = NULL; #endif rq_unlock(busiest_rq, &rf); @@ -11180,7 +11181,7 @@ static bool silver_has_big_tasks(void) for_each_possible_cpu(cpu) { if (!is_min_capacity_cpu(cpu)) break; - if (cpu_rq(cpu)->walt_stats.nr_big_tasks) + if (cpu_rq(cpu)->wrq.walt_stats.nr_big_tasks) return true; } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index ed2b039fa7ae..3b80b97a5a44 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1809,7 +1809,7 @@ static int rt_energy_aware_wake_cpu(struct task_struct *task) rcu_read_lock(); - cpu = cpu_rq(smp_processor_id())->rd->min_cap_orig_cpu; + cpu = cpu_rq(smp_processor_id())->rd->wrd.min_cap_orig_cpu; if (cpu < 0) goto unlock; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ead0ef8e74e9..e17df709a3b6 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -88,7 +88,7 @@ extern __read_mostly bool sched_predl; extern unsigned int sched_capacity_margin_up[NR_CPUS]; extern unsigned int sched_capacity_margin_down[NR_CPUS]; -struct sched_walt_cpu_load { +struct walt_cpu_load { unsigned long nl; unsigned long pl; bool rtgb_active; @@ -107,33 +107,95 @@ struct walt_sched_stats { u64 pred_demands_sum_scaled; }; -struct group_cpu_time { - u64 curr_runnable_sum; - u64 prev_runnable_sum; - u64 nt_curr_runnable_sum; - u64 nt_prev_runnable_sum; +struct walt_task_group { + /* Toggle ability to override sched boost enabled */ + bool sched_boost_no_override; + /* + * Controls whether a cgroup is eligible for sched boost or not. This + * can temporariliy be disabled by the kernel based on the no_override + * flag above. + */ + bool sched_boost_enabled; + /* + * Controls whether tasks of this cgroup should be colocated with each + * other and tasks of other cgroups that have the same flag turned on. + */ + bool colocate; + /* Controls whether further updates are allowed to the colocate flag */ + bool colocate_update_disabled; }; -struct load_subtractions { - u64 window_start; - u64 subs; - u64 new_subs; +struct walt_root_domain { + /* First cpu with maximum and minimum original capacity */ + int max_cap_orig_cpu, min_cap_orig_cpu; + /* First cpu with mid capacity */ + int mid_cap_orig_cpu; }; + #define NUM_TRACKED_WINDOWS 2 #define NUM_LOAD_INDICES 1000 -struct sched_cluster { - raw_spinlock_t load_lock; - struct list_head list; - struct cpumask cpus; - int id; +struct group_cpu_time { + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; +}; + +struct load_subtractions { + u64 window_start; + u64 subs; + u64 new_subs; +}; + +struct walt_rq { + struct task_struct *push_task; + struct walt_sched_cluster *cluster; + struct cpumask freq_domain_cpumask; + struct walt_sched_stats walt_stats; + + u64 window_start; + u32 prev_window_size; + unsigned long walt_flags; + + u64 cur_irqload; + u64 avg_irqload; + u64 irqload_ts; + bool high_irqload; + struct task_struct *ed_task; + u64 task_exec_scale; + u64 old_busy_time; + u64 old_estimated_time; + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; + u64 cum_window_demand_scaled; + struct group_cpu_time grp_time; + struct load_subtractions load_subs[NUM_TRACKED_WINDOWS]; + DECLARE_BITMAP_ARRAY(top_tasks_bitmap, + NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES); + u8 *top_tasks[NUM_TRACKED_WINDOWS]; + u8 curr_table; + int prev_top; + int curr_top; + bool notif_pending; + u64 last_cc_update; + u64 cycles; +}; + +struct walt_sched_cluster { + raw_spinlock_t load_lock; + struct list_head list; + struct cpumask cpus; + int id; /* * max_possible_freq = maximum supported by hardware */ - unsigned int cur_freq; - unsigned int max_possible_freq; - u64 aggr_grp_load; + unsigned int cur_freq; + unsigned int max_possible_freq; + u64 aggr_grp_load; }; extern __weak cpumask_t asym_cap_sibling_cpus; @@ -461,21 +523,7 @@ struct task_group { /* Latency-sensitive flag used for a task group */ unsigned int latency_sensitive; #ifdef CONFIG_SCHED_WALT - /* Toggle ability to override sched boost enabled */ - bool sched_boost_no_override; - /* - * Controls whether a cgroup is eligible for sched boost or not. This - * can temporariliy be disabled by the kernel based on the no_override - * flag above. - */ - bool sched_boost_enabled; - /* - * Controls whether tasks of this cgroup should be colocated with each - * other and tasks of other cgroups that have the same flag turned on. - */ - bool colocate; - /* Controls whether further updates are allowed to the colocate flag */ - bool colocate_update_disabled; + struct walt_task_group wtg; #endif /* CONFIG_SCHED_WALT */ #endif /* CONFIG_UCLAMP_TASK_GROUP */ @@ -873,10 +921,7 @@ struct root_domain { struct perf_domain __rcu *pd; #ifdef CONFIG_SCHED_WALT - /* First cpu with maximum and minimum original capacity */ - int max_cap_orig_cpu, min_cap_orig_cpu; - /* First cpu with mid capacity */ - int mid_cap_orig_cpu; + struct walt_root_domain wrd; #endif }; @@ -1049,39 +1094,7 @@ struct rq { #endif #ifdef CONFIG_SCHED_WALT - struct task_struct *push_task; - struct sched_cluster *cluster; - struct cpumask freq_domain_cpumask; - struct walt_sched_stats walt_stats; - - u64 window_start; - u32 prev_window_size; - unsigned long walt_flags; - - u64 cur_irqload; - u64 avg_irqload; - u64 irqload_ts; - bool high_irqload; - struct task_struct *ed_task; - u64 task_exec_scale; - u64 old_busy_time, old_busy_time_group; - u64 old_estimated_time; - u64 curr_runnable_sum; - u64 prev_runnable_sum; - u64 nt_curr_runnable_sum; - u64 nt_prev_runnable_sum; - u64 cum_window_demand_scaled; - struct group_cpu_time grp_time; - struct load_subtractions load_subs[NUM_TRACKED_WINDOWS]; - DECLARE_BITMAP_ARRAY(top_tasks_bitmap, - NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES); - u8 *top_tasks[NUM_TRACKED_WINDOWS]; - u8 curr_table; - int prev_top; - int curr_top; - bool notif_pending; - u64 last_cc_update; - u64 cycles; + struct walt_rq wrq; #endif /* CONFIG_SCHED_WALT */ #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -2146,7 +2159,7 @@ static inline int hrtick_enabled(struct rq *rq) #ifdef CONFIG_SCHED_WALT u64 sched_ktime_clock(void); unsigned long -cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load); +cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load); #else #define sched_ravg_window TICK_NSEC static inline u64 sched_ktime_clock(void) @@ -2257,7 +2270,7 @@ static inline unsigned long cpu_util(int cpu) #ifdef CONFIG_SCHED_WALT u64 walt_cpu_util = - cpu_rq(cpu)->walt_stats.cumulative_runnable_avg_scaled; + cpu_rq(cpu)->wrq.walt_stats.cumulative_runnable_avg_scaled; return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu)); #endif @@ -2277,7 +2290,7 @@ static inline unsigned long cpu_util_cum(int cpu, int delta) unsigned long capacity = capacity_orig_of(cpu); #ifdef CONFIG_SCHED_WALT - util = cpu_rq(cpu)->cum_window_demand_scaled; + util = cpu_rq(cpu)->wrq.cum_window_demand_scaled; #endif delta += util; if (delta < 0) @@ -2846,12 +2859,12 @@ enum sched_boost_policy { #ifdef CONFIG_SCHED_WALT -static inline int cluster_first_cpu(struct sched_cluster *cluster) +static inline int cluster_first_cpu(struct walt_sched_cluster *cluster) { return cpumask_first(&cluster->cpus); } -struct related_thread_group { +struct walt_related_thread_group { int id; raw_spinlock_t lock; struct list_head tasks; @@ -2863,16 +2876,16 @@ struct related_thread_group { u64 start_ts; }; -extern struct sched_cluster *sched_cluster[NR_CPUS]; +extern struct walt_sched_cluster *sched_cluster[NR_CPUS]; extern unsigned int max_possible_capacity; extern unsigned int __weak min_max_possible_capacity; extern unsigned int __read_mostly __weak sched_init_task_load_windows; extern unsigned int __read_mostly __weak sched_load_granule; -extern int update_preferred_cluster(struct related_thread_group *grp, +extern int update_preferred_cluster(struct walt_related_thread_group *grp, struct task_struct *p, u32 old_load, bool from_tick); -extern void set_preferred_cluster(struct related_thread_group *grp); +extern void set_preferred_cluster(struct walt_related_thread_group *grp); extern void add_new_task_to_grp(struct task_struct *new); #define NO_BOOST 0 @@ -2921,7 +2934,7 @@ static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin) static inline unsigned int cpu_max_possible_freq(int cpu) { - return cpu_rq(cpu)->cluster->max_possible_freq; + return cpu_rq(cpu)->wrq.cluster->max_possible_freq; } static inline unsigned int cpu_max_freq(int cpu) @@ -2960,8 +2973,8 @@ static inline bool task_in_related_thread_group(struct task_struct *p) return (rcu_access_pointer(p->wts.grp) != NULL); } -static inline -struct related_thread_group *task_related_thread_group(struct task_struct *p) +static inline struct walt_related_thread_group +*task_related_thread_group(struct task_struct *p) { return rcu_dereference(p->wts.grp); } @@ -2977,7 +2990,7 @@ static inline int same_freq_domain(int src_cpu, int dst_cpu) if (asym_cap_siblings(src_cpu, dst_cpu)) return 1; - return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask); + return cpumask_test_cpu(dst_cpu, &rq->wrq.freq_domain_cpumask); } #define CPU_RESERVED 1 @@ -3006,9 +3019,9 @@ static inline bool is_full_throttle_boost(void) return sched_boost() == FULL_THROTTLE_BOOST; } -extern int preferred_cluster(struct sched_cluster *cluster, +extern int preferred_cluster(struct walt_sched_cluster *cluster, struct task_struct *p); -extern struct sched_cluster *rq_cluster(struct rq *rq); +extern struct walt_sched_cluster *rq_cluster(struct rq *rq); #ifdef CONFIG_UCLAMP_TASK_GROUP static inline bool task_sched_boost(struct task_struct *p) @@ -3020,7 +3033,7 @@ static inline bool task_sched_boost(struct task_struct *p) return false; tg = container_of(css, struct task_group, css); - return tg->sched_boost_enabled; + return tg->wtg.sched_boost_enabled; } extern int sync_cgroup_colocation(struct task_struct *p, bool insert); @@ -3046,35 +3059,35 @@ static inline int is_reserved(int cpu) { struct rq *rq = cpu_rq(cpu); - return test_bit(CPU_RESERVED, &rq->walt_flags); + return test_bit(CPU_RESERVED, &rq->wrq.walt_flags); } static inline int mark_reserved(int cpu) { struct rq *rq = cpu_rq(cpu); - return test_and_set_bit(CPU_RESERVED, &rq->walt_flags); + return test_and_set_bit(CPU_RESERVED, &rq->wrq.walt_flags); } static inline void clear_reserved(int cpu) { struct rq *rq = cpu_rq(cpu); - clear_bit(CPU_RESERVED, &rq->walt_flags); + clear_bit(CPU_RESERVED, &rq->wrq.walt_flags); } static inline bool task_in_cum_window_demand(struct rq *rq, struct task_struct *p) { return cpu_of(rq) == task_cpu(p) && (p->on_rq || - p->wts.last_sleep_ts >= rq->window_start); + p->wts.last_sleep_ts >= rq->wrq.window_start); } static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta) { - rq->cum_window_demand_scaled += scaled_delta; - if (unlikely((s64)rq->cum_window_demand_scaled < 0)) - rq->cum_window_demand_scaled = 0; + rq->wrq.cum_window_demand_scaled += scaled_delta; + if (unlikely((s64)rq->wrq.cum_window_demand_scaled < 0)) + rq->wrq.cum_window_demand_scaled = 0; } extern unsigned long thermal_cap(int cpu); @@ -3114,15 +3127,15 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) return policy; } -static inline bool is_min_capacity_cluster(struct sched_cluster *cluster) +static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster) { return is_min_capacity_cpu(cluster_first_cpu(cluster)); } #else /* CONFIG_SCHED_WALT */ struct walt_sched_stats; -struct related_thread_group; -struct sched_cluster; +struct walt_related_thread_group; +struct walt_sched_cluster; static inline bool task_sched_boost(struct task_struct *p) { @@ -3167,12 +3180,12 @@ static inline bool is_max_capacity_cpu(int cpu) { return true; } static inline bool is_min_capacity_cpu(int cpu) { return true; } static inline int -preferred_cluster(struct sched_cluster *cluster, struct task_struct *p) +preferred_cluster(struct walt_sched_cluster *cluster, struct task_struct *p) { return -1; } -static inline struct sched_cluster *rq_cluster(struct rq *rq) +static inline struct walt_sched_cluster *rq_cluster(struct rq *rq) { return NULL; } @@ -3184,15 +3197,16 @@ static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin) return false; } -static inline void set_preferred_cluster(struct related_thread_group *grp) { } +static inline void +set_preferred_cluster(struct walt_related_thread_group *grp) { } static inline bool task_in_related_thread_group(struct task_struct *p) { return false; } -static inline -struct related_thread_group *task_related_thread_group(struct task_struct *p) +static inline struct walt_related_thread_group *task_related_thread_group( +struct task_struct *p) { return NULL; } @@ -3200,8 +3214,9 @@ struct related_thread_group *task_related_thread_group(struct task_struct *p) static inline u32 task_load(struct task_struct *p) { return 0; } static inline u32 task_pl(struct task_struct *p) { return 0; } -static inline int update_preferred_cluster(struct related_thread_group *grp, - struct task_struct *p, u32 old_load, bool from_tick) +static inline int +update_preferred_cluster(struct walt_related_thread_group *grp, + struct task_struct *p, u32 old_load, bool from_tick) { return 0; } diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index af0809fc5425..25229e24c3ea 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -495,8 +495,8 @@ static int init_rootdomain(struct root_domain *rd) goto free_cpudl; #ifdef CONFIG_SCHED_WALT - rd->max_cap_orig_cpu = rd->min_cap_orig_cpu = -1; - rd->mid_cap_orig_cpu = -1; + rd->wrd.max_cap_orig_cpu = rd->wrd.min_cap_orig_cpu = -1; + rd->wrd.mid_cap_orig_cpu = -1; #endif init_max_cpu_capacity(&rd->max_cpu_capacity); @@ -2053,8 +2053,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att rcu_read_lock(); for_each_cpu(i, cpu_map) { #ifdef CONFIG_SCHED_WALT - int max_cpu = READ_ONCE(d.rd->max_cap_orig_cpu); - int min_cpu = READ_ONCE(d.rd->min_cap_orig_cpu); + int max_cpu = READ_ONCE(d.rd->wrd.max_cap_orig_cpu); + int min_cpu = READ_ONCE(d.rd->wrd.min_cap_orig_cpu); #endif sd = *per_cpu_ptr(d.sd, i); @@ -2062,11 +2062,11 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att #ifdef CONFIG_SCHED_WALT if ((max_cpu < 0) || (arch_scale_cpu_capacity(i) > arch_scale_cpu_capacity(max_cpu))) - WRITE_ONCE(d.rd->max_cap_orig_cpu, i); + WRITE_ONCE(d.rd->wrd.max_cap_orig_cpu, i); if ((min_cpu < 0) || (arch_scale_cpu_capacity(i) < arch_scale_cpu_capacity(min_cpu))) - WRITE_ONCE(d.rd->min_cap_orig_cpu, i); + WRITE_ONCE(d.rd->wrd.min_cap_orig_cpu, i); #endif cpu_attach_domain(sd, d.rd, i); @@ -2075,14 +2075,14 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att #ifdef CONFIG_SCHED_WALT /* set the mid capacity cpu (assumes only 3 capacities) */ for_each_cpu(i, cpu_map) { - int max_cpu = READ_ONCE(d.rd->max_cap_orig_cpu); - int min_cpu = READ_ONCE(d.rd->min_cap_orig_cpu); + int max_cpu = READ_ONCE(d.rd->wrd.max_cap_orig_cpu); + int min_cpu = READ_ONCE(d.rd->wrd.min_cap_orig_cpu); if ((arch_scale_cpu_capacity(i) - != arch_scale_cpu_capacity(min_cpu)) && + != arch_scale_cpu_capacity(min_cpu)) && (arch_scale_cpu_capacity(i) - != arch_scale_cpu_capacity(max_cpu))) { - WRITE_ONCE(d.rd->mid_cap_orig_cpu, i); + != arch_scale_cpu_capacity(max_cpu))) { + WRITE_ONCE(d.rd->wrd.mid_cap_orig_cpu, i); break; } } @@ -2092,10 +2092,10 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att * change dynamically. So update the max cap CPU and its capacity * here. */ - if (d.rd->max_cap_orig_cpu != -1) { - d.rd->max_cpu_capacity.cpu = d.rd->max_cap_orig_cpu; + if (d.rd->wrd.max_cap_orig_cpu != -1) { + d.rd->max_cpu_capacity.cpu = d.rd->wrd.max_cap_orig_cpu; d.rd->max_cpu_capacity.val = arch_scale_cpu_capacity( - d.rd->max_cap_orig_cpu); + d.rd->wrd.max_cap_orig_cpu); } #endif diff --git a/kernel/sched/walt.c b/kernel/sched/walt.c index 7903b81d526d..8b34b0880fc7 100644 --- a/kernel/sched/walt.c +++ b/kernel/sched/walt.c @@ -132,23 +132,23 @@ int __weak sched_busy_hyst_handler(struct ctl_table *table, int write, u64 __weak sched_ktime_clock(void) { return 0; } unsigned long __weak -cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load) +cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load) { return cpu_util(cpu); } -int __weak update_preferred_cluster(struct related_thread_group *grp, +int __weak update_preferred_cluster(struct walt_related_thread_group *grp, struct task_struct *p, u32 old_load, bool from_tick) { return 0; } -void __weak set_preferred_cluster(struct related_thread_group *grp) { } +void __weak set_preferred_cluster(struct walt_related_thread_group *grp) { } void __weak add_new_task_to_grp(struct task_struct *new) { } int __weak -preferred_cluster(struct sched_cluster *cluster, struct task_struct *p) +preferred_cluster(struct walt_sched_cluster *cluster, struct task_struct *p) { return -1; } diff --git a/kernel/sched/walt.h b/kernel/sched/walt.h index 4ed3b086f670..609ee8e2b0a2 100644 --- a/kernel/sched/walt.h +++ b/kernel/sched/walt.h @@ -35,7 +35,7 @@ fixup_cumulative_runnable_avg(struct walt_sched_stats *stats, static inline void walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) { - fixup_cumulative_runnable_avg(&rq->walt_stats, p->wts.demand_scaled, + fixup_cumulative_runnable_avg(&rq->wrq.walt_stats, p->wts.demand_scaled, p->wts.pred_demand_scaled); /* @@ -45,14 +45,14 @@ walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) * prio/cgroup/class change. * (2) task is waking for the first time in this window. */ - if (p->on_rq || (p->wts.last_sleep_ts < rq->window_start)) + if (p->on_rq || (p->wts.last_sleep_ts < rq->wrq.window_start)) walt_fixup_cum_window_demand(rq, p->wts.demand_scaled); } static inline void walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) { - fixup_cumulative_runnable_avg(&rq->walt_stats, + fixup_cumulative_runnable_avg(&rq->wrq.walt_stats, -(s64)p->wts.demand_scaled, -(s64)p->wts.pred_demand_scaled); @@ -68,15 +68,15 @@ walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p) static inline void walt_adjust_nr_big_tasks(struct rq *rq, int delta, bool inc) { sched_update_nr_prod(cpu_of(rq), 0, true); - rq->walt_stats.nr_big_tasks += inc ? delta : -delta; + rq->wrq.walt_stats.nr_big_tasks += inc ? delta : -delta; - BUG_ON(rq->walt_stats.nr_big_tasks < 0); + BUG_ON(rq->wrq.walt_stats.nr_big_tasks < 0); } static inline void inc_rq_walt_stats(struct rq *rq, struct task_struct *p) { if (p->wts.misfit) - rq->walt_stats.nr_big_tasks++; + rq->wrq.walt_stats.nr_big_tasks++; walt_inc_cumulative_runnable_avg(rq, p); } @@ -84,9 +84,9 @@ static inline void inc_rq_walt_stats(struct rq *rq, struct task_struct *p) static inline void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) { if (p->wts.misfit) - rq->walt_stats.nr_big_tasks--; + rq->wrq.walt_stats.nr_big_tasks--; - BUG_ON(rq->walt_stats.nr_big_tasks < 0); + BUG_ON(rq->wrq.walt_stats.nr_big_tasks < 0); walt_dec_cumulative_runnable_avg(rq, p); } @@ -107,23 +107,23 @@ static inline u64 sched_irqload(int cpu) struct rq *rq = cpu_rq(cpu); s64 delta; - delta = get_jiffies_64() - rq->irqload_ts; + delta = get_jiffies_64() - rq->wrq.irqload_ts; /* - * Current context can be preempted by irq and rq->irqload_ts can be + * Current context can be preempted by irq and rq->wrq.irqload_ts can be * updated by irq context so that delta can be negative. * But this is okay and we can safely return as this means there * was recent irq occurrence. */ if (delta < SCHED_HIGH_IRQ_TIMEOUT) - return rq->avg_irqload; + return rq->wrq.avg_irqload; else return 0; } static inline int sched_cpu_high_irqload(int cpu) { - return cpu_rq(cpu)->high_irqload; + return cpu_rq(cpu)->wrq.high_irqload; } static inline int exiting_task(struct task_struct *p) @@ -152,7 +152,7 @@ extern void sched_account_irqtime(int cpu, struct task_struct *curr, static inline int same_cluster(int src_cpu, int dst_cpu) { - return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster; + return cpu_rq(src_cpu)->wrq.cluster == cpu_rq(dst_cpu)->wrq.cluster; } void walt_sched_init_rq(struct rq *rq); @@ -170,7 +170,7 @@ static inline bool is_suh_max(void) #define DEFAULT_CGROUP_COLOC_ID 1 static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu) { - struct related_thread_group *rtg = p->wts.grp; + struct walt_related_thread_group *rtg = p->wts.grp; if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID && rtg->skip_min && p->wts.unfilter) @@ -189,7 +189,7 @@ static inline void walt_try_to_wake_up(struct task_struct *p) struct rq_flags rf; u64 wallclock; unsigned int old_load; - struct related_thread_group *grp = NULL; + struct walt_related_thread_group *grp = NULL; rq_lock_irqsave(rq, &rf); old_load = task_load(p);