mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-10 06:09:23 -04:00
sched/walt: Consolidate WALT parameters
Create specific structures to hold all WALT related parameters used in the scheduler. This is to separate WALT functionality from scheduler code for GKI purposes. Change-Id: I6013883c00f2089dba3e5d0890c9abf5f57b3da6 Signed-off-by: Sai Harshini Nimmala <snimmala@codeaurora.org>
This commit is contained in:
Sai Harshini Nimmala
parent
3804c2a592
commit
e2ac811691
11 changed files with 185 additions and 169 deletions
|
|
@ -619,7 +619,7 @@ struct walt_task_struct {
|
|||
u32 unfilter;
|
||||
u64 last_wake_ts;
|
||||
u64 last_enqueued_ts;
|
||||
struct related_thread_group __rcu *grp;
|
||||
struct walt_related_thread_group __rcu *grp;
|
||||
struct list_head grp_list;
|
||||
u64 cpu_cycles;
|
||||
cpumask_t cpus_requested;
|
||||
|
|
|
|||
|
|
@ -475,11 +475,11 @@ TRACE_EVENT(sched_load_balance_skip_tasks,
|
|||
TP_fast_assign(
|
||||
__entry->scpu = scpu;
|
||||
__entry->src_util_cum =
|
||||
cpu_rq(scpu)->cum_window_demand_scaled;
|
||||
cpu_rq(scpu)->wrq.cum_window_demand_scaled;
|
||||
__entry->grp_type = grp_type;
|
||||
__entry->dcpu = dcpu;
|
||||
__entry->dst_util_cum =
|
||||
cpu_rq(dcpu)->cum_window_demand_scaled;
|
||||
cpu_rq(dcpu)->wrq.cum_window_demand_scaled;
|
||||
__entry->pid = pid;
|
||||
__entry->affinity = affinity;
|
||||
__entry->task_util = task_util;
|
||||
|
|
|
|||
|
|
@ -1338,7 +1338,7 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
|
|||
uclamp_rq_dec(rq, p);
|
||||
p->sched_class->dequeue_task(rq, p, flags);
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
if (p == rq->ed_task)
|
||||
if (p == rq->wrq.ed_task)
|
||||
early_detection_notify(rq, sched_ktime_clock());
|
||||
#endif
|
||||
trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_mask)[0]);
|
||||
|
|
@ -3697,7 +3697,7 @@ void scheduler_tick(void)
|
|||
u64 wallclock;
|
||||
bool early_notif;
|
||||
u32 old_load;
|
||||
struct related_thread_group *grp;
|
||||
struct walt_related_thread_group *grp;
|
||||
unsigned int flag = 0;
|
||||
|
||||
sched_clock_tick();
|
||||
|
|
@ -7202,7 +7202,7 @@ static void walt_schedgp_attach(struct cgroup_taskset *tset)
|
|||
cgroup_taskset_first(tset, &css);
|
||||
tg = css_tg(css);
|
||||
|
||||
colocate = tg->colocate;
|
||||
colocate = tg->wtg.colocate;
|
||||
|
||||
cgroup_taskset_for_each(task, css, tset)
|
||||
sync_cgroup_colocation(task, colocate);
|
||||
|
|
@ -7214,7 +7214,7 @@ sched_boost_override_read(struct cgroup_subsys_state *css,
|
|||
{
|
||||
struct task_group *tg = css_tg(css);
|
||||
|
||||
return (u64) tg->sched_boost_no_override;
|
||||
return (u64) tg->wtg.sched_boost_no_override;
|
||||
}
|
||||
|
||||
static int sched_boost_override_write(struct cgroup_subsys_state *css,
|
||||
|
|
@ -7222,7 +7222,7 @@ static int sched_boost_override_write(struct cgroup_subsys_state *css,
|
|||
{
|
||||
struct task_group *tg = css_tg(css);
|
||||
|
||||
tg->sched_boost_no_override = !!override;
|
||||
tg->wtg.sched_boost_no_override = !!override;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -7231,7 +7231,7 @@ static u64 sched_colocate_read(struct cgroup_subsys_state *css,
|
|||
{
|
||||
struct task_group *tg = css_tg(css);
|
||||
|
||||
return (u64) tg->colocate;
|
||||
return (u64) tg->wtg.colocate;
|
||||
}
|
||||
|
||||
static int sched_colocate_write(struct cgroup_subsys_state *css,
|
||||
|
|
@ -7239,11 +7239,11 @@ static int sched_colocate_write(struct cgroup_subsys_state *css,
|
|||
{
|
||||
struct task_group *tg = css_tg(css);
|
||||
|
||||
if (tg->colocate_update_disabled)
|
||||
if (tg->wtg.colocate_update_disabled)
|
||||
return -EPERM;
|
||||
|
||||
tg->colocate = !!colocate;
|
||||
tg->colocate_update_disabled = true;
|
||||
tg->wtg.colocate = !!colocate;
|
||||
tg->wtg.colocate_update_disabled = true;
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
|
|
@ -8412,25 +8412,25 @@ void sched_account_irqtime(int cpu, struct task_struct *curr,
|
|||
walt_update_task_ravg(curr, rq, IRQ_UPDATE, sched_ktime_clock(),
|
||||
delta);
|
||||
|
||||
nr_ticks = cur_jiffies_ts - rq->irqload_ts;
|
||||
nr_ticks = cur_jiffies_ts - rq->wrq.irqload_ts;
|
||||
|
||||
if (nr_ticks) {
|
||||
if (nr_ticks < 10) {
|
||||
/* Decay CPU's irqload by 3/4 for each window. */
|
||||
rq->avg_irqload *= (3 * nr_ticks);
|
||||
rq->avg_irqload = div64_u64(rq->avg_irqload,
|
||||
rq->wrq.avg_irqload *= (3 * nr_ticks);
|
||||
rq->wrq.avg_irqload = div64_u64(rq->wrq.avg_irqload,
|
||||
4 * nr_ticks);
|
||||
} else {
|
||||
rq->avg_irqload = 0;
|
||||
rq->wrq.avg_irqload = 0;
|
||||
}
|
||||
rq->avg_irqload += rq->cur_irqload;
|
||||
rq->high_irqload = (rq->avg_irqload >=
|
||||
rq->wrq.avg_irqload += rq->wrq.cur_irqload;
|
||||
rq->wrq.high_irqload = (rq->wrq.avg_irqload >=
|
||||
sysctl_sched_cpu_high_irqload);
|
||||
rq->cur_irqload = 0;
|
||||
rq->wrq.cur_irqload = 0;
|
||||
}
|
||||
|
||||
rq->cur_irqload += delta;
|
||||
rq->irqload_ts = cur_jiffies_ts;
|
||||
rq->wrq.cur_irqload += delta;
|
||||
rq->wrq.irqload_ts = cur_jiffies_ts;
|
||||
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ struct sugov_cpu {
|
|||
unsigned int iowait_boost;
|
||||
u64 last_update;
|
||||
|
||||
struct sched_walt_cpu_load walt_load;
|
||||
struct walt_cpu_load walt_load;
|
||||
|
||||
unsigned long util;
|
||||
unsigned int flags;
|
||||
|
|
|
|||
|
|
@ -650,10 +650,10 @@ do { \
|
|||
P(cpu_capacity);
|
||||
#endif
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
P(cluster->cur_freq);
|
||||
P(walt_stats.nr_big_tasks);
|
||||
P(wrq.cluster->cur_freq);
|
||||
P(wrq.walt_stats.nr_big_tasks);
|
||||
SEQ_printf(m, " .%-30s: %llu\n", "walt_stats.cumulative_runnable_avg",
|
||||
rq->walt_stats.cumulative_runnable_avg_scaled);
|
||||
rq->wrq.walt_stats.cumulative_runnable_avg_scaled);
|
||||
#endif
|
||||
#undef P
|
||||
#undef PN
|
||||
|
|
|
|||
|
|
@ -6417,7 +6417,7 @@ unsigned long capacity_curr_of(int cpu)
|
|||
#ifdef CONFIG_SCHED_WALT
|
||||
static inline bool walt_get_rtg_status(struct task_struct *p)
|
||||
{
|
||||
struct related_thread_group *grp;
|
||||
struct walt_related_thread_group *grp;
|
||||
bool ret = false;
|
||||
|
||||
rcu_read_lock();
|
||||
|
|
@ -6708,7 +6708,7 @@ static inline unsigned long
|
|||
cpu_util_next_walt(int cpu, struct task_struct *p, int dst_cpu)
|
||||
{
|
||||
unsigned long util =
|
||||
cpu_rq(cpu)->walt_stats.cumulative_runnable_avg_scaled;
|
||||
cpu_rq(cpu)->wrq.walt_stats.cumulative_runnable_avg_scaled;
|
||||
bool queued = task_on_rq_queued(p);
|
||||
|
||||
/*
|
||||
|
|
@ -8106,7 +8106,8 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
|
|||
}
|
||||
|
||||
if (env->flags & LBF_IGNORE_PREFERRED_CLUSTER_TASKS &&
|
||||
!preferred_cluster(cpu_rq(env->dst_cpu)->cluster, p))
|
||||
!preferred_cluster(
|
||||
cpu_rq(env->dst_cpu)->wrq.cluster, p))
|
||||
return 0;
|
||||
|
||||
/* Don't detach task if it doesn't fit on the destination */
|
||||
|
|
@ -8115,7 +8116,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
|
|||
return 0;
|
||||
|
||||
/* Don't detach task if it is under active migration */
|
||||
if (env->src_rq->push_task == p)
|
||||
if (env->src_rq->wrq.push_task == p)
|
||||
return 0;
|
||||
#endif
|
||||
|
||||
|
|
@ -10418,7 +10419,7 @@ int active_load_balance_cpu_stop(void *data)
|
|||
BUG_ON(busiest_rq == target_rq);
|
||||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
push_task = busiest_rq->push_task;
|
||||
push_task = busiest_rq->wrq.push_task;
|
||||
target_cpu = busiest_rq->push_cpu;
|
||||
if (push_task) {
|
||||
if (task_on_rq_queued(push_task) &&
|
||||
|
|
@ -10474,14 +10475,14 @@ int active_load_balance_cpu_stop(void *data)
|
|||
out_unlock:
|
||||
busiest_rq->active_balance = 0;
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
push_task = busiest_rq->push_task;
|
||||
push_task = busiest_rq->wrq.push_task;
|
||||
#endif
|
||||
target_cpu = busiest_rq->push_cpu;
|
||||
clear_reserved(target_cpu);
|
||||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
if (push_task)
|
||||
busiest_rq->push_task = NULL;
|
||||
busiest_rq->wrq.push_task = NULL;
|
||||
#endif
|
||||
|
||||
rq_unlock(busiest_rq, &rf);
|
||||
|
|
@ -11180,7 +11181,7 @@ static bool silver_has_big_tasks(void)
|
|||
for_each_possible_cpu(cpu) {
|
||||
if (!is_min_capacity_cpu(cpu))
|
||||
break;
|
||||
if (cpu_rq(cpu)->walt_stats.nr_big_tasks)
|
||||
if (cpu_rq(cpu)->wrq.walt_stats.nr_big_tasks)
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1809,7 +1809,7 @@ static int rt_energy_aware_wake_cpu(struct task_struct *task)
|
|||
|
||||
rcu_read_lock();
|
||||
|
||||
cpu = cpu_rq(smp_processor_id())->rd->min_cap_orig_cpu;
|
||||
cpu = cpu_rq(smp_processor_id())->rd->wrd.min_cap_orig_cpu;
|
||||
if (cpu < 0)
|
||||
goto unlock;
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ extern __read_mostly bool sched_predl;
|
|||
extern unsigned int sched_capacity_margin_up[NR_CPUS];
|
||||
extern unsigned int sched_capacity_margin_down[NR_CPUS];
|
||||
|
||||
struct sched_walt_cpu_load {
|
||||
struct walt_cpu_load {
|
||||
unsigned long nl;
|
||||
unsigned long pl;
|
||||
bool rtgb_active;
|
||||
|
|
@ -107,33 +107,95 @@ struct walt_sched_stats {
|
|||
u64 pred_demands_sum_scaled;
|
||||
};
|
||||
|
||||
struct group_cpu_time {
|
||||
u64 curr_runnable_sum;
|
||||
u64 prev_runnable_sum;
|
||||
u64 nt_curr_runnable_sum;
|
||||
u64 nt_prev_runnable_sum;
|
||||
struct walt_task_group {
|
||||
/* Toggle ability to override sched boost enabled */
|
||||
bool sched_boost_no_override;
|
||||
/*
|
||||
* Controls whether a cgroup is eligible for sched boost or not. This
|
||||
* can temporariliy be disabled by the kernel based on the no_override
|
||||
* flag above.
|
||||
*/
|
||||
bool sched_boost_enabled;
|
||||
/*
|
||||
* Controls whether tasks of this cgroup should be colocated with each
|
||||
* other and tasks of other cgroups that have the same flag turned on.
|
||||
*/
|
||||
bool colocate;
|
||||
/* Controls whether further updates are allowed to the colocate flag */
|
||||
bool colocate_update_disabled;
|
||||
};
|
||||
|
||||
struct load_subtractions {
|
||||
u64 window_start;
|
||||
u64 subs;
|
||||
u64 new_subs;
|
||||
struct walt_root_domain {
|
||||
/* First cpu with maximum and minimum original capacity */
|
||||
int max_cap_orig_cpu, min_cap_orig_cpu;
|
||||
/* First cpu with mid capacity */
|
||||
int mid_cap_orig_cpu;
|
||||
};
|
||||
|
||||
|
||||
#define NUM_TRACKED_WINDOWS 2
|
||||
#define NUM_LOAD_INDICES 1000
|
||||
|
||||
struct sched_cluster {
|
||||
raw_spinlock_t load_lock;
|
||||
struct list_head list;
|
||||
struct cpumask cpus;
|
||||
int id;
|
||||
struct group_cpu_time {
|
||||
u64 curr_runnable_sum;
|
||||
u64 prev_runnable_sum;
|
||||
u64 nt_curr_runnable_sum;
|
||||
u64 nt_prev_runnable_sum;
|
||||
};
|
||||
|
||||
struct load_subtractions {
|
||||
u64 window_start;
|
||||
u64 subs;
|
||||
u64 new_subs;
|
||||
};
|
||||
|
||||
struct walt_rq {
|
||||
struct task_struct *push_task;
|
||||
struct walt_sched_cluster *cluster;
|
||||
struct cpumask freq_domain_cpumask;
|
||||
struct walt_sched_stats walt_stats;
|
||||
|
||||
u64 window_start;
|
||||
u32 prev_window_size;
|
||||
unsigned long walt_flags;
|
||||
|
||||
u64 cur_irqload;
|
||||
u64 avg_irqload;
|
||||
u64 irqload_ts;
|
||||
bool high_irqload;
|
||||
struct task_struct *ed_task;
|
||||
u64 task_exec_scale;
|
||||
u64 old_busy_time;
|
||||
u64 old_estimated_time;
|
||||
u64 curr_runnable_sum;
|
||||
u64 prev_runnable_sum;
|
||||
u64 nt_curr_runnable_sum;
|
||||
u64 nt_prev_runnable_sum;
|
||||
u64 cum_window_demand_scaled;
|
||||
struct group_cpu_time grp_time;
|
||||
struct load_subtractions load_subs[NUM_TRACKED_WINDOWS];
|
||||
DECLARE_BITMAP_ARRAY(top_tasks_bitmap,
|
||||
NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES);
|
||||
u8 *top_tasks[NUM_TRACKED_WINDOWS];
|
||||
u8 curr_table;
|
||||
int prev_top;
|
||||
int curr_top;
|
||||
bool notif_pending;
|
||||
u64 last_cc_update;
|
||||
u64 cycles;
|
||||
};
|
||||
|
||||
struct walt_sched_cluster {
|
||||
raw_spinlock_t load_lock;
|
||||
struct list_head list;
|
||||
struct cpumask cpus;
|
||||
int id;
|
||||
/*
|
||||
* max_possible_freq = maximum supported by hardware
|
||||
*/
|
||||
unsigned int cur_freq;
|
||||
unsigned int max_possible_freq;
|
||||
u64 aggr_grp_load;
|
||||
unsigned int cur_freq;
|
||||
unsigned int max_possible_freq;
|
||||
u64 aggr_grp_load;
|
||||
};
|
||||
|
||||
extern __weak cpumask_t asym_cap_sibling_cpus;
|
||||
|
|
@ -461,21 +523,7 @@ struct task_group {
|
|||
/* Latency-sensitive flag used for a task group */
|
||||
unsigned int latency_sensitive;
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
/* Toggle ability to override sched boost enabled */
|
||||
bool sched_boost_no_override;
|
||||
/*
|
||||
* Controls whether a cgroup is eligible for sched boost or not. This
|
||||
* can temporariliy be disabled by the kernel based on the no_override
|
||||
* flag above.
|
||||
*/
|
||||
bool sched_boost_enabled;
|
||||
/*
|
||||
* Controls whether tasks of this cgroup should be colocated with each
|
||||
* other and tasks of other cgroups that have the same flag turned on.
|
||||
*/
|
||||
bool colocate;
|
||||
/* Controls whether further updates are allowed to the colocate flag */
|
||||
bool colocate_update_disabled;
|
||||
struct walt_task_group wtg;
|
||||
#endif /* CONFIG_SCHED_WALT */
|
||||
#endif /* CONFIG_UCLAMP_TASK_GROUP */
|
||||
|
||||
|
|
@ -873,10 +921,7 @@ struct root_domain {
|
|||
struct perf_domain __rcu *pd;
|
||||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
/* First cpu with maximum and minimum original capacity */
|
||||
int max_cap_orig_cpu, min_cap_orig_cpu;
|
||||
/* First cpu with mid capacity */
|
||||
int mid_cap_orig_cpu;
|
||||
struct walt_root_domain wrd;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
|
@ -1049,39 +1094,7 @@ struct rq {
|
|||
#endif
|
||||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
struct task_struct *push_task;
|
||||
struct sched_cluster *cluster;
|
||||
struct cpumask freq_domain_cpumask;
|
||||
struct walt_sched_stats walt_stats;
|
||||
|
||||
u64 window_start;
|
||||
u32 prev_window_size;
|
||||
unsigned long walt_flags;
|
||||
|
||||
u64 cur_irqload;
|
||||
u64 avg_irqload;
|
||||
u64 irqload_ts;
|
||||
bool high_irqload;
|
||||
struct task_struct *ed_task;
|
||||
u64 task_exec_scale;
|
||||
u64 old_busy_time, old_busy_time_group;
|
||||
u64 old_estimated_time;
|
||||
u64 curr_runnable_sum;
|
||||
u64 prev_runnable_sum;
|
||||
u64 nt_curr_runnable_sum;
|
||||
u64 nt_prev_runnable_sum;
|
||||
u64 cum_window_demand_scaled;
|
||||
struct group_cpu_time grp_time;
|
||||
struct load_subtractions load_subs[NUM_TRACKED_WINDOWS];
|
||||
DECLARE_BITMAP_ARRAY(top_tasks_bitmap,
|
||||
NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES);
|
||||
u8 *top_tasks[NUM_TRACKED_WINDOWS];
|
||||
u8 curr_table;
|
||||
int prev_top;
|
||||
int curr_top;
|
||||
bool notif_pending;
|
||||
u64 last_cc_update;
|
||||
u64 cycles;
|
||||
struct walt_rq wrq;
|
||||
#endif /* CONFIG_SCHED_WALT */
|
||||
|
||||
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
|
||||
|
|
@ -2146,7 +2159,7 @@ static inline int hrtick_enabled(struct rq *rq)
|
|||
#ifdef CONFIG_SCHED_WALT
|
||||
u64 sched_ktime_clock(void);
|
||||
unsigned long
|
||||
cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load);
|
||||
cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load);
|
||||
#else
|
||||
#define sched_ravg_window TICK_NSEC
|
||||
static inline u64 sched_ktime_clock(void)
|
||||
|
|
@ -2257,7 +2270,7 @@ static inline unsigned long cpu_util(int cpu)
|
|||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
u64 walt_cpu_util =
|
||||
cpu_rq(cpu)->walt_stats.cumulative_runnable_avg_scaled;
|
||||
cpu_rq(cpu)->wrq.walt_stats.cumulative_runnable_avg_scaled;
|
||||
|
||||
return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu));
|
||||
#endif
|
||||
|
|
@ -2277,7 +2290,7 @@ static inline unsigned long cpu_util_cum(int cpu, int delta)
|
|||
unsigned long capacity = capacity_orig_of(cpu);
|
||||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
util = cpu_rq(cpu)->cum_window_demand_scaled;
|
||||
util = cpu_rq(cpu)->wrq.cum_window_demand_scaled;
|
||||
#endif
|
||||
delta += util;
|
||||
if (delta < 0)
|
||||
|
|
@ -2846,12 +2859,12 @@ enum sched_boost_policy {
|
|||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
|
||||
static inline int cluster_first_cpu(struct sched_cluster *cluster)
|
||||
static inline int cluster_first_cpu(struct walt_sched_cluster *cluster)
|
||||
{
|
||||
return cpumask_first(&cluster->cpus);
|
||||
}
|
||||
|
||||
struct related_thread_group {
|
||||
struct walt_related_thread_group {
|
||||
int id;
|
||||
raw_spinlock_t lock;
|
||||
struct list_head tasks;
|
||||
|
|
@ -2863,16 +2876,16 @@ struct related_thread_group {
|
|||
u64 start_ts;
|
||||
};
|
||||
|
||||
extern struct sched_cluster *sched_cluster[NR_CPUS];
|
||||
extern struct walt_sched_cluster *sched_cluster[NR_CPUS];
|
||||
|
||||
extern unsigned int max_possible_capacity;
|
||||
extern unsigned int __weak min_max_possible_capacity;
|
||||
extern unsigned int __read_mostly __weak sched_init_task_load_windows;
|
||||
extern unsigned int __read_mostly __weak sched_load_granule;
|
||||
|
||||
extern int update_preferred_cluster(struct related_thread_group *grp,
|
||||
extern int update_preferred_cluster(struct walt_related_thread_group *grp,
|
||||
struct task_struct *p, u32 old_load, bool from_tick);
|
||||
extern void set_preferred_cluster(struct related_thread_group *grp);
|
||||
extern void set_preferred_cluster(struct walt_related_thread_group *grp);
|
||||
extern void add_new_task_to_grp(struct task_struct *new);
|
||||
|
||||
#define NO_BOOST 0
|
||||
|
|
@ -2921,7 +2934,7 @@ static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin)
|
|||
|
||||
static inline unsigned int cpu_max_possible_freq(int cpu)
|
||||
{
|
||||
return cpu_rq(cpu)->cluster->max_possible_freq;
|
||||
return cpu_rq(cpu)->wrq.cluster->max_possible_freq;
|
||||
}
|
||||
|
||||
static inline unsigned int cpu_max_freq(int cpu)
|
||||
|
|
@ -2960,8 +2973,8 @@ static inline bool task_in_related_thread_group(struct task_struct *p)
|
|||
return (rcu_access_pointer(p->wts.grp) != NULL);
|
||||
}
|
||||
|
||||
static inline
|
||||
struct related_thread_group *task_related_thread_group(struct task_struct *p)
|
||||
static inline struct walt_related_thread_group
|
||||
*task_related_thread_group(struct task_struct *p)
|
||||
{
|
||||
return rcu_dereference(p->wts.grp);
|
||||
}
|
||||
|
|
@ -2977,7 +2990,7 @@ static inline int same_freq_domain(int src_cpu, int dst_cpu)
|
|||
if (asym_cap_siblings(src_cpu, dst_cpu))
|
||||
return 1;
|
||||
|
||||
return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask);
|
||||
return cpumask_test_cpu(dst_cpu, &rq->wrq.freq_domain_cpumask);
|
||||
}
|
||||
|
||||
#define CPU_RESERVED 1
|
||||
|
|
@ -3006,9 +3019,9 @@ static inline bool is_full_throttle_boost(void)
|
|||
return sched_boost() == FULL_THROTTLE_BOOST;
|
||||
}
|
||||
|
||||
extern int preferred_cluster(struct sched_cluster *cluster,
|
||||
extern int preferred_cluster(struct walt_sched_cluster *cluster,
|
||||
struct task_struct *p);
|
||||
extern struct sched_cluster *rq_cluster(struct rq *rq);
|
||||
extern struct walt_sched_cluster *rq_cluster(struct rq *rq);
|
||||
|
||||
#ifdef CONFIG_UCLAMP_TASK_GROUP
|
||||
static inline bool task_sched_boost(struct task_struct *p)
|
||||
|
|
@ -3020,7 +3033,7 @@ static inline bool task_sched_boost(struct task_struct *p)
|
|||
return false;
|
||||
tg = container_of(css, struct task_group, css);
|
||||
|
||||
return tg->sched_boost_enabled;
|
||||
return tg->wtg.sched_boost_enabled;
|
||||
}
|
||||
|
||||
extern int sync_cgroup_colocation(struct task_struct *p, bool insert);
|
||||
|
|
@ -3046,35 +3059,35 @@ static inline int is_reserved(int cpu)
|
|||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
|
||||
return test_bit(CPU_RESERVED, &rq->walt_flags);
|
||||
return test_bit(CPU_RESERVED, &rq->wrq.walt_flags);
|
||||
}
|
||||
|
||||
static inline int mark_reserved(int cpu)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
|
||||
return test_and_set_bit(CPU_RESERVED, &rq->walt_flags);
|
||||
return test_and_set_bit(CPU_RESERVED, &rq->wrq.walt_flags);
|
||||
}
|
||||
|
||||
static inline void clear_reserved(int cpu)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
|
||||
clear_bit(CPU_RESERVED, &rq->walt_flags);
|
||||
clear_bit(CPU_RESERVED, &rq->wrq.walt_flags);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
task_in_cum_window_demand(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
return cpu_of(rq) == task_cpu(p) && (p->on_rq ||
|
||||
p->wts.last_sleep_ts >= rq->window_start);
|
||||
p->wts.last_sleep_ts >= rq->wrq.window_start);
|
||||
}
|
||||
|
||||
static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta)
|
||||
{
|
||||
rq->cum_window_demand_scaled += scaled_delta;
|
||||
if (unlikely((s64)rq->cum_window_demand_scaled < 0))
|
||||
rq->cum_window_demand_scaled = 0;
|
||||
rq->wrq.cum_window_demand_scaled += scaled_delta;
|
||||
if (unlikely((s64)rq->wrq.cum_window_demand_scaled < 0))
|
||||
rq->wrq.cum_window_demand_scaled = 0;
|
||||
}
|
||||
|
||||
extern unsigned long thermal_cap(int cpu);
|
||||
|
|
@ -3114,15 +3127,15 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
|
|||
return policy;
|
||||
}
|
||||
|
||||
static inline bool is_min_capacity_cluster(struct sched_cluster *cluster)
|
||||
static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster)
|
||||
{
|
||||
return is_min_capacity_cpu(cluster_first_cpu(cluster));
|
||||
}
|
||||
#else /* CONFIG_SCHED_WALT */
|
||||
|
||||
struct walt_sched_stats;
|
||||
struct related_thread_group;
|
||||
struct sched_cluster;
|
||||
struct walt_related_thread_group;
|
||||
struct walt_sched_cluster;
|
||||
|
||||
static inline bool task_sched_boost(struct task_struct *p)
|
||||
{
|
||||
|
|
@ -3167,12 +3180,12 @@ static inline bool is_max_capacity_cpu(int cpu) { return true; }
|
|||
static inline bool is_min_capacity_cpu(int cpu) { return true; }
|
||||
|
||||
static inline int
|
||||
preferred_cluster(struct sched_cluster *cluster, struct task_struct *p)
|
||||
preferred_cluster(struct walt_sched_cluster *cluster, struct task_struct *p)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline struct sched_cluster *rq_cluster(struct rq *rq)
|
||||
static inline struct walt_sched_cluster *rq_cluster(struct rq *rq)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -3184,15 +3197,16 @@ static inline bool asym_cap_sibling_group_has_capacity(int dst_cpu, int margin)
|
|||
return false;
|
||||
}
|
||||
|
||||
static inline void set_preferred_cluster(struct related_thread_group *grp) { }
|
||||
static inline void
|
||||
set_preferred_cluster(struct walt_related_thread_group *grp) { }
|
||||
|
||||
static inline bool task_in_related_thread_group(struct task_struct *p)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline
|
||||
struct related_thread_group *task_related_thread_group(struct task_struct *p)
|
||||
static inline struct walt_related_thread_group *task_related_thread_group(
|
||||
struct task_struct *p)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -3200,8 +3214,9 @@ struct related_thread_group *task_related_thread_group(struct task_struct *p)
|
|||
static inline u32 task_load(struct task_struct *p) { return 0; }
|
||||
static inline u32 task_pl(struct task_struct *p) { return 0; }
|
||||
|
||||
static inline int update_preferred_cluster(struct related_thread_group *grp,
|
||||
struct task_struct *p, u32 old_load, bool from_tick)
|
||||
static inline int
|
||||
update_preferred_cluster(struct walt_related_thread_group *grp,
|
||||
struct task_struct *p, u32 old_load, bool from_tick)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -495,8 +495,8 @@ static int init_rootdomain(struct root_domain *rd)
|
|||
goto free_cpudl;
|
||||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
rd->max_cap_orig_cpu = rd->min_cap_orig_cpu = -1;
|
||||
rd->mid_cap_orig_cpu = -1;
|
||||
rd->wrd.max_cap_orig_cpu = rd->wrd.min_cap_orig_cpu = -1;
|
||||
rd->wrd.mid_cap_orig_cpu = -1;
|
||||
#endif
|
||||
|
||||
init_max_cpu_capacity(&rd->max_cpu_capacity);
|
||||
|
|
@ -2053,8 +2053,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
rcu_read_lock();
|
||||
for_each_cpu(i, cpu_map) {
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
int max_cpu = READ_ONCE(d.rd->max_cap_orig_cpu);
|
||||
int min_cpu = READ_ONCE(d.rd->min_cap_orig_cpu);
|
||||
int max_cpu = READ_ONCE(d.rd->wrd.max_cap_orig_cpu);
|
||||
int min_cpu = READ_ONCE(d.rd->wrd.min_cap_orig_cpu);
|
||||
#endif
|
||||
|
||||
sd = *per_cpu_ptr(d.sd, i);
|
||||
|
|
@ -2062,11 +2062,11 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
#ifdef CONFIG_SCHED_WALT
|
||||
if ((max_cpu < 0) || (arch_scale_cpu_capacity(i) >
|
||||
arch_scale_cpu_capacity(max_cpu)))
|
||||
WRITE_ONCE(d.rd->max_cap_orig_cpu, i);
|
||||
WRITE_ONCE(d.rd->wrd.max_cap_orig_cpu, i);
|
||||
|
||||
if ((min_cpu < 0) || (arch_scale_cpu_capacity(i) <
|
||||
arch_scale_cpu_capacity(min_cpu)))
|
||||
WRITE_ONCE(d.rd->min_cap_orig_cpu, i);
|
||||
WRITE_ONCE(d.rd->wrd.min_cap_orig_cpu, i);
|
||||
#endif
|
||||
|
||||
cpu_attach_domain(sd, d.rd, i);
|
||||
|
|
@ -2075,14 +2075,14 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
#ifdef CONFIG_SCHED_WALT
|
||||
/* set the mid capacity cpu (assumes only 3 capacities) */
|
||||
for_each_cpu(i, cpu_map) {
|
||||
int max_cpu = READ_ONCE(d.rd->max_cap_orig_cpu);
|
||||
int min_cpu = READ_ONCE(d.rd->min_cap_orig_cpu);
|
||||
int max_cpu = READ_ONCE(d.rd->wrd.max_cap_orig_cpu);
|
||||
int min_cpu = READ_ONCE(d.rd->wrd.min_cap_orig_cpu);
|
||||
|
||||
if ((arch_scale_cpu_capacity(i)
|
||||
!= arch_scale_cpu_capacity(min_cpu)) &&
|
||||
!= arch_scale_cpu_capacity(min_cpu)) &&
|
||||
(arch_scale_cpu_capacity(i)
|
||||
!= arch_scale_cpu_capacity(max_cpu))) {
|
||||
WRITE_ONCE(d.rd->mid_cap_orig_cpu, i);
|
||||
!= arch_scale_cpu_capacity(max_cpu))) {
|
||||
WRITE_ONCE(d.rd->wrd.mid_cap_orig_cpu, i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -2092,10 +2092,10 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
|
|||
* change dynamically. So update the max cap CPU and its capacity
|
||||
* here.
|
||||
*/
|
||||
if (d.rd->max_cap_orig_cpu != -1) {
|
||||
d.rd->max_cpu_capacity.cpu = d.rd->max_cap_orig_cpu;
|
||||
if (d.rd->wrd.max_cap_orig_cpu != -1) {
|
||||
d.rd->max_cpu_capacity.cpu = d.rd->wrd.max_cap_orig_cpu;
|
||||
d.rd->max_cpu_capacity.val = arch_scale_cpu_capacity(
|
||||
d.rd->max_cap_orig_cpu);
|
||||
d.rd->wrd.max_cap_orig_cpu);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -132,23 +132,23 @@ int __weak sched_busy_hyst_handler(struct ctl_table *table, int write,
|
|||
u64 __weak sched_ktime_clock(void) { return 0; }
|
||||
|
||||
unsigned long __weak
|
||||
cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load)
|
||||
cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load)
|
||||
{
|
||||
return cpu_util(cpu);
|
||||
}
|
||||
|
||||
int __weak update_preferred_cluster(struct related_thread_group *grp,
|
||||
int __weak update_preferred_cluster(struct walt_related_thread_group *grp,
|
||||
struct task_struct *p, u32 old_load, bool from_tick)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
void __weak set_preferred_cluster(struct related_thread_group *grp) { }
|
||||
void __weak set_preferred_cluster(struct walt_related_thread_group *grp) { }
|
||||
|
||||
void __weak add_new_task_to_grp(struct task_struct *new) { }
|
||||
|
||||
int __weak
|
||||
preferred_cluster(struct sched_cluster *cluster, struct task_struct *p)
|
||||
preferred_cluster(struct walt_sched_cluster *cluster, struct task_struct *p)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ fixup_cumulative_runnable_avg(struct walt_sched_stats *stats,
|
|||
static inline void
|
||||
walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
fixup_cumulative_runnable_avg(&rq->walt_stats, p->wts.demand_scaled,
|
||||
fixup_cumulative_runnable_avg(&rq->wrq.walt_stats, p->wts.demand_scaled,
|
||||
p->wts.pred_demand_scaled);
|
||||
|
||||
/*
|
||||
|
|
@ -45,14 +45,14 @@ walt_inc_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
|
|||
* prio/cgroup/class change.
|
||||
* (2) task is waking for the first time in this window.
|
||||
*/
|
||||
if (p->on_rq || (p->wts.last_sleep_ts < rq->window_start))
|
||||
if (p->on_rq || (p->wts.last_sleep_ts < rq->wrq.window_start))
|
||||
walt_fixup_cum_window_demand(rq, p->wts.demand_scaled);
|
||||
}
|
||||
|
||||
static inline void
|
||||
walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
fixup_cumulative_runnable_avg(&rq->walt_stats,
|
||||
fixup_cumulative_runnable_avg(&rq->wrq.walt_stats,
|
||||
-(s64)p->wts.demand_scaled,
|
||||
-(s64)p->wts.pred_demand_scaled);
|
||||
|
||||
|
|
@ -68,15 +68,15 @@ walt_dec_cumulative_runnable_avg(struct rq *rq, struct task_struct *p)
|
|||
static inline void walt_adjust_nr_big_tasks(struct rq *rq, int delta, bool inc)
|
||||
{
|
||||
sched_update_nr_prod(cpu_of(rq), 0, true);
|
||||
rq->walt_stats.nr_big_tasks += inc ? delta : -delta;
|
||||
rq->wrq.walt_stats.nr_big_tasks += inc ? delta : -delta;
|
||||
|
||||
BUG_ON(rq->walt_stats.nr_big_tasks < 0);
|
||||
BUG_ON(rq->wrq.walt_stats.nr_big_tasks < 0);
|
||||
}
|
||||
|
||||
static inline void inc_rq_walt_stats(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
if (p->wts.misfit)
|
||||
rq->walt_stats.nr_big_tasks++;
|
||||
rq->wrq.walt_stats.nr_big_tasks++;
|
||||
|
||||
walt_inc_cumulative_runnable_avg(rq, p);
|
||||
}
|
||||
|
|
@ -84,9 +84,9 @@ static inline void inc_rq_walt_stats(struct rq *rq, struct task_struct *p)
|
|||
static inline void dec_rq_walt_stats(struct rq *rq, struct task_struct *p)
|
||||
{
|
||||
if (p->wts.misfit)
|
||||
rq->walt_stats.nr_big_tasks--;
|
||||
rq->wrq.walt_stats.nr_big_tasks--;
|
||||
|
||||
BUG_ON(rq->walt_stats.nr_big_tasks < 0);
|
||||
BUG_ON(rq->wrq.walt_stats.nr_big_tasks < 0);
|
||||
|
||||
walt_dec_cumulative_runnable_avg(rq, p);
|
||||
}
|
||||
|
|
@ -107,23 +107,23 @@ static inline u64 sched_irqload(int cpu)
|
|||
struct rq *rq = cpu_rq(cpu);
|
||||
s64 delta;
|
||||
|
||||
delta = get_jiffies_64() - rq->irqload_ts;
|
||||
delta = get_jiffies_64() - rq->wrq.irqload_ts;
|
||||
/*
|
||||
* Current context can be preempted by irq and rq->irqload_ts can be
|
||||
* Current context can be preempted by irq and rq->wrq.irqload_ts can be
|
||||
* updated by irq context so that delta can be negative.
|
||||
* But this is okay and we can safely return as this means there
|
||||
* was recent irq occurrence.
|
||||
*/
|
||||
|
||||
if (delta < SCHED_HIGH_IRQ_TIMEOUT)
|
||||
return rq->avg_irqload;
|
||||
return rq->wrq.avg_irqload;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int sched_cpu_high_irqload(int cpu)
|
||||
{
|
||||
return cpu_rq(cpu)->high_irqload;
|
||||
return cpu_rq(cpu)->wrq.high_irqload;
|
||||
}
|
||||
|
||||
static inline int exiting_task(struct task_struct *p)
|
||||
|
|
@ -152,7 +152,7 @@ extern void sched_account_irqtime(int cpu, struct task_struct *curr,
|
|||
|
||||
static inline int same_cluster(int src_cpu, int dst_cpu)
|
||||
{
|
||||
return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster;
|
||||
return cpu_rq(src_cpu)->wrq.cluster == cpu_rq(dst_cpu)->wrq.cluster;
|
||||
}
|
||||
|
||||
void walt_sched_init_rq(struct rq *rq);
|
||||
|
|
@ -170,7 +170,7 @@ static inline bool is_suh_max(void)
|
|||
#define DEFAULT_CGROUP_COLOC_ID 1
|
||||
static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu)
|
||||
{
|
||||
struct related_thread_group *rtg = p->wts.grp;
|
||||
struct walt_related_thread_group *rtg = p->wts.grp;
|
||||
|
||||
if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID &&
|
||||
rtg->skip_min && p->wts.unfilter)
|
||||
|
|
@ -189,7 +189,7 @@ static inline void walt_try_to_wake_up(struct task_struct *p)
|
|||
struct rq_flags rf;
|
||||
u64 wallclock;
|
||||
unsigned int old_load;
|
||||
struct related_thread_group *grp = NULL;
|
||||
struct walt_related_thread_group *grp = NULL;
|
||||
|
||||
rq_lock_irqsave(rq, &rf);
|
||||
old_load = task_load(p);
|
||||
|
|
|
|||
Loading…
Reference in a new issue