diff --git a/init/Kconfig b/init/Kconfig index fb3a0f1e47f0..6a6e169b998a 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -924,7 +924,7 @@ config FAIR_GROUP_SCHED config CFS_BANDWIDTH bool "CPU bandwidth provisioning for FAIR_GROUP_SCHED" - depends on FAIR_GROUP_SCHED + depends on FAIR_GROUP_SCHED && !SCHED_WALT default n help This option allows users to define CPU bandwidth rates (limits) for diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ffb00fe1a0ce..3e1d159cb002 100755 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4674,16 +4674,13 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); qcfs_rq->h_nr_running -= task_delta; qcfs_rq->idle_h_nr_running -= idle_task_delta; - walt_dec_throttled_cfs_rq_stats(&qcfs_rq->walt_stats, cfs_rq); if (qcfs_rq->load.weight) dequeue = 0; } - if (!se) { + if (!se) sub_nr_running(rq, task_delta); - walt_dec_throttled_cfs_rq_stats(&rq->walt_stats, cfs_rq); - } cfs_rq->throttled = 1; cfs_rq->throttled_clock = rq_clock(rq); @@ -4717,7 +4714,6 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) struct sched_entity *se; int enqueue = 1; long task_delta, idle_task_delta; - struct cfs_rq *tcfs_rq __maybe_unused = cfs_rq; se = cfs_rq->tg->se[cpu_of(rq)]; @@ -4747,7 +4743,6 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); cfs_rq->h_nr_running += task_delta; cfs_rq->idle_h_nr_running += idle_task_delta; - walt_inc_throttled_cfs_rq_stats(&cfs_rq->walt_stats, tcfs_rq); if (cfs_rq_throttled(cfs_rq)) break; @@ -4755,10 +4750,8 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) assert_list_leaf_cfs_rq(rq); - if (!se) { + if (!se) add_nr_running(rq, task_delta); - walt_inc_throttled_cfs_rq_stats(&rq->walt_stats, tcfs_rq); - } /* Determine whether we need to wake up potentially idle CPU: */ if (rq->curr == rq->idle && rq->cfs.nr_running) @@ -5149,7 +5142,6 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) { cfs_rq->runtime_enabled = 0; INIT_LIST_HEAD(&cfs_rq->throttled_list); - walt_init_cfs_rq_stats(cfs_rq); } void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) @@ -5401,9 +5393,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) */ util_est_enqueue(&rq->cfs, p); -#ifdef CONFIG_SCHED_WALT - p->misfit = !task_fits_max(p, rq->cpu); -#endif /* * If in_iowait is set, the code below may not trigger any cpufreq * utilization updates, so do it here explicitly with the IOWAIT flag @@ -5428,7 +5417,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) break; cfs_rq->h_nr_running++; cfs_rq->idle_h_nr_running += idle_h_nr_running; - walt_inc_cfs_rq_stats(cfs_rq, p); flags = ENQUEUE_WAKEUP; } @@ -5437,7 +5425,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running++; cfs_rq->idle_h_nr_running += idle_h_nr_running; - walt_inc_cfs_rq_stats(cfs_rq, p); if (cfs_rq_throttled(cfs_rq)) break; @@ -5448,6 +5435,9 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!se) { add_nr_running(rq, 1); +#ifdef CONFIG_SCHED_WALT + p->misfit = !task_fits_max(p, rq->cpu); +#endif inc_rq_walt_stats(rq, p); /* * Since new tasks are assigned an initial util_avg equal to @@ -11439,7 +11429,7 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) misfit = rq->misfit_task_load; if (old_misfit != misfit) { - walt_fixup_nr_big_tasks(rq, curr, 1, misfit); + walt_adjust_nr_big_tasks(rq, 1, misfit); curr->misfit = misfit; } #endif @@ -11961,7 +11951,7 @@ const struct sched_class fair_sched_class = { #endif #ifdef CONFIG_SCHED_WALT - .fixup_walt_sched_stats = walt_fixup_sched_stats_fair, + .fixup_walt_sched_stats = fixup_walt_sched_stats_common, #endif }; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a9f81b3cd5c0..32fb5b41ab24 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3134,13 +3134,6 @@ static inline bool is_min_capacity_cluster(struct sched_cluster *cluster) { return is_min_capacity_cpu(cluster_first_cpu(cluster)); } - -extern void walt_fixup_sched_stats_fair(struct rq *rq, - struct task_struct *p, - u16 updated_demand_scaled, - u16 updated_pred_demand_scaled); -extern void walt_fixup_nr_big_tasks(struct rq *rq, struct task_struct *p, - int delta, bool inc); #else /* CONFIG_SCHED_WALT */ struct walt_sched_stats; diff --git a/kernel/sched/walt.c b/kernel/sched/walt.c index 6b76826f20c2..c0ac25dbe5c3 100644 --- a/kernel/sched/walt.c +++ b/kernel/sched/walt.c @@ -172,12 +172,6 @@ bool __weak early_detection_notify(struct rq *rq, u64 wallclock) void __weak note_task_waking(struct task_struct *p, u64 wallclock) { } -void __weak walt_fixup_sched_stats_fair(struct rq *rq, struct task_struct *p, - u16 updated_demand_scaled, u16 updated_pred_demand_scaled) { } - -void __weak walt_fixup_nr_big_tasks(struct rq *rq, struct task_struct *p, - int delta, bool inc) { } - int __weak group_balance_cpu_not_isolated(struct sched_group *sg) { return group_balance_cpu(sg); @@ -194,10 +188,6 @@ void __weak walt_update_task_ravg(struct task_struct *p, struct rq *rq, void __weak fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, u16 updated_demand_scaled, u16 updated_pred_demand_scaled) { } -void __weak inc_rq_walt_stats(struct rq *rq, struct task_struct *p) { } - -void __weak dec_rq_walt_stats(struct rq *rq, struct task_struct *p) { } - void __weak fixup_busy_time(struct task_struct *p, int new_cpu) { } void __weak init_new_task_load(struct task_struct *p) { } diff --git a/kernel/sched/walt.h b/kernel/sched/walt.h index acf247231738..21f91c0b2962 100644 --- a/kernel/sched/walt.h +++ b/kernel/sched/walt.h @@ -69,8 +69,33 @@ extern void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, u16 updated_demand_scaled, u16 updated_pred_demand_scaled); -extern void inc_rq_walt_stats(struct rq *rq, struct task_struct *p); -extern void dec_rq_walt_stats(struct rq *rq, struct task_struct *p); + +static inline void walt_adjust_nr_big_tasks(struct rq *rq, int delta, bool inc) +{ + sched_update_nr_prod(cpu_of(rq), 0, true); + rq->walt_stats.nr_big_tasks += inc ? delta : -delta; + + BUG_ON(rq->walt_stats.nr_big_tasks < 0); +} + +static inline void inc_rq_walt_stats(struct rq *rq, struct task_struct *p) +{ + if (p->misfit) + rq->walt_stats.nr_big_tasks++; + + walt_inc_cumulative_runnable_avg(rq, p); +} + +static inline void dec_rq_walt_stats(struct rq *rq, struct task_struct *p) +{ + if (p->misfit) + rq->walt_stats.nr_big_tasks--; + + BUG_ON(rq->walt_stats.nr_big_tasks < 0); + + walt_dec_cumulative_runnable_avg(rq, p); +} + extern void fixup_busy_time(struct task_struct *p, int new_cpu); extern void init_new_task_load(struct task_struct *p); extern void mark_task_starting(struct task_struct *p);