From 408e7407731ce17ece079830b821fc6f1135812c Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Mon, 14 Oct 2019 16:47:10 +0530 Subject: [PATCH] sched: use the CPU true capacity while sorting the min/mid/max CPUs The current code uses the rq->cpu_capacity_orig while sorting the min/mid/max CPUs while rebuilding the scheduler domains. This rq->cpu_capacity_orig is subjected to changed when the scaling max frequency is clipped for that CPU's frequency domain. Since we don't recompute the min/mid/max CPUs when the capacity is changed, the sorting becomes incorrect later when the frequency limits are lifted. The task CPU selection algorithm depends heavily on min/mid/max CPUs and changing them on the fly results in incorrect task placement. Hence use the true capacity of CPUs while sorting the min/mid/max CPUs. This means that the sorting gets changed only when CPUs are hotplugged out, otherwise these reflect the correct topology all the time. The max_cpu_capacity struct in root domain also maintains the max CPU and its capacity. Since we now use true capacity, there is no need to compute this on the fly. So move this evaluation from the periodic load balancer to the scheduler domain rebuilding. The rq->cpu_capacity_orig is still subjected to change upon frequency or thermal limits. So we still identify the cases where tasks not fitting on the max/mid CPUs when the capacity is reduced. Change-Id: I42735accd079b2ece1eb58d1ebcf322d454e33a2 Signed-off-by: Pavankumar Kondeti [satyap@codeaurora.org: trivial changes to arch_scale_cpu_capacity()] Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/fair.c | 24 ------------------------ kernel/sched/topology.c | 28 ++++++++++++++++++++-------- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3a5e8ba90f3c..150c3df8b7b0 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8641,10 +8641,6 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu) { unsigned long capacity = arch_scale_cpu_capacity(cpu); struct sched_group *sdg = sd->groups; - struct max_cpu_capacity *mcc; - unsigned long max_capacity; - int max_cap_cpu; - unsigned long flags; capacity *= arch_scale_max_freq_capacity(sd, cpu); capacity >>= SCHED_CAPACITY_SHIFT; @@ -8652,26 +8648,6 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu) capacity = min(capacity, thermal_cap(cpu)); cpu_rq(cpu)->cpu_capacity_orig = capacity; - mcc = &cpu_rq(cpu)->rd->max_cpu_capacity; - - raw_spin_lock_irqsave(&mcc->lock, flags); - max_capacity = mcc->val; - max_cap_cpu = mcc->cpu; - - if ((max_capacity > capacity && max_cap_cpu == cpu) || - (max_capacity < capacity)) { - mcc->val = capacity; - mcc->cpu = cpu; -#ifdef CONFIG_SCHED_DEBUG - raw_spin_unlock_irqrestore(&mcc->lock, flags); - printk_deferred(KERN_INFO "CPU%d: update max cpu_capacity %lu\n", - cpu, capacity); - goto skip_unlock; -#endif - } - raw_spin_unlock_irqrestore(&mcc->lock, flags); - -skip_unlock: __attribute__ ((unused)); capacity = scale_rt_capacity(cpu, capacity); if (!capacity) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index d686dd418194..dd72f9740f14 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -2047,12 +2047,12 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att sd = *per_cpu_ptr(d.sd, i); #ifdef CONFIG_SCHED_WALT - if ((max_cpu < 0) || (cpu_rq(i)->cpu_capacity_orig > - cpu_rq(max_cpu)->cpu_capacity_orig)) + if ((max_cpu < 0) || (arch_scale_cpu_capacity(i) > + arch_scale_cpu_capacity(max_cpu))) WRITE_ONCE(d.rd->max_cap_orig_cpu, i); - if ((min_cpu < 0) || (cpu_rq(i)->cpu_capacity_orig < - cpu_rq(min_cpu)->cpu_capacity_orig)) + if ((min_cpu < 0) || (arch_scale_cpu_capacity(i) < + arch_scale_cpu_capacity(min_cpu))) WRITE_ONCE(d.rd->min_cap_orig_cpu, i); #endif @@ -2065,15 +2065,27 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att int max_cpu = READ_ONCE(d.rd->max_cap_orig_cpu); int min_cpu = READ_ONCE(d.rd->min_cap_orig_cpu); - if ((cpu_rq(i)->cpu_capacity_orig - != cpu_rq(min_cpu)->cpu_capacity_orig) && - (cpu_rq(i)->cpu_capacity_orig - != cpu_rq(max_cpu)->cpu_capacity_orig)) { + if ((arch_scale_cpu_capacity(i) + != arch_scale_cpu_capacity(min_cpu)) && + (arch_scale_cpu_capacity(i) + != arch_scale_cpu_capacity(max_cpu))) { WRITE_ONCE(d.rd->mid_cap_orig_cpu, i); break; } } + + /* + * The max_cpu_capacity reflect the original capacity which does not + * change dynamically. So update the max cap CPU and its capacity + * here. + */ + if (d.rd->max_cap_orig_cpu != -1) { + d.rd->max_cpu_capacity.cpu = d.rd->max_cap_orig_cpu; + d.rd->max_cpu_capacity.val = arch_scale_cpu_capacity( + d.rd->max_cap_orig_cpu); + } #endif + rcu_read_unlock(); if (has_asym)