From b26dcb23160f7313e9d9414e8bb5ce50d4e30410 Mon Sep 17 00:00:00 2001 From: Satya Durga Srinivasu Prabhala Date: Fri, 16 Nov 2018 16:05:52 -0800 Subject: [PATCH 1/4] sched/rt: Add snapshot of changes to RT class This snapshot is taken from msm-4.19 as of commit 8a6c9078311bc31 ("sched: rt: Fix start cpu for RT energy aware task placement"). Change-Id: I850161a6fa845c9a73c67072299476eb3c3d1888 Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/rt.c | 196 ++++++++++++++++++++++++++++++++++++++++++++-- lib/Kconfig.debug | 9 +++ 2 files changed, 200 insertions(+), 5 deletions(-) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 80a260d83111..78eefc45d2cf 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -910,6 +910,66 @@ static inline int rt_se_prio(struct sched_rt_entity *rt_se) return rt_task_of(rt_se)->prio; } +static void dump_throttled_rt_tasks(struct rt_rq *rt_rq) +{ + struct rt_prio_array *array = &rt_rq->active; + struct sched_rt_entity *rt_se; + char buf[500]; + char *pos = buf; + char *end = buf + sizeof(buf); + int idx; + struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); + + pos += snprintf(pos, sizeof(buf), + "sched: RT throttling activated for rt_rq %pK (cpu %d)\n", + rt_rq, cpu_of(rq_of_rt_rq(rt_rq))); + + pos += snprintf(pos, end - pos, + "rt_period_timer: expires=%lld now=%llu period=%llu\n", + hrtimer_get_expires_ns(&rt_b->rt_period_timer), + ktime_get_ns(), sched_rt_period(rt_rq)); + + if (bitmap_empty(array->bitmap, MAX_RT_PRIO)) + goto out; + + pos += snprintf(pos, end - pos, "potential CPU hogs:\n"); +#ifdef CONFIG_SCHED_INFO + if (sched_info_on()) + pos += snprintf(pos, end - pos, + "current %s (%d) is running for %llu nsec\n", + current->comm, current->pid, + rq_clock(rq_of_rt_rq(rt_rq)) - + current->sched_info.last_arrival); +#endif + + idx = sched_find_first_bit(array->bitmap); + while (idx < MAX_RT_PRIO) { + list_for_each_entry(rt_se, array->queue + idx, run_list) { + struct task_struct *p; + + if (!rt_entity_is_task(rt_se)) + continue; + + p = rt_task_of(rt_se); + if (pos < end) + pos += snprintf(pos, end - pos, "\t%s (%d)\n", + p->comm, p->pid); + } + idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx + 1); + } +out: +#ifdef CONFIG_PANIC_ON_RT_THROTTLING + /* + * Use pr_err() in the BUG() case since printk_sched() will + * not get flushed and deadlock is not a concern. + */ + pr_err("%s\n", buf); + BUG(); +#else + printk_deferred("%s\n", buf); +#endif +} + static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) { u64 runtime = sched_rt_runtime(rt_rq); @@ -933,8 +993,14 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) * but accrue some time due to boosting. */ if (likely(rt_b->rt_runtime)) { + static bool once; + rt_rq->rt_throttled = 1; - printk_deferred_once("sched: RT throttling activated\n"); + + if (!once) { + once = true; + dump_throttled_rt_tasks(rt_rq); + } } else { /* * In case we did anyway, make it go away, @@ -1339,6 +1405,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) rt_se->timeout = 0; enqueue_rt_entity(rt_se, flags); + walt_inc_cumulative_runnable_avg(rq, p); if (!task_current(rq, p) && p->nr_cpus_allowed > 1) enqueue_pushable_task(rq, p); @@ -1350,6 +1417,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) update_curr_rt(rq); dequeue_rt_entity(rt_se, flags); + walt_dec_cumulative_runnable_avg(rq, p); dequeue_pushable_task(rq, p); } @@ -1428,9 +1496,10 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) * This test is optimistic, if we get it wrong the load-balancer * will have to sort it out. */ - if (curr && unlikely(rt_task(curr)) && - (curr->nr_cpus_allowed < 2 || - curr->prio <= p->prio)) { + if (sched_energy_enabled() || + (curr && unlikely(rt_task(curr)) && + (curr->nr_cpus_allowed < 2 || + curr->prio <= p->prio))) { int target = find_lowest_rq(p); /* @@ -1634,12 +1703,119 @@ static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu) static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); +#ifdef CONFIG_SCHED_WALT +static int rt_energy_aware_wake_cpu(struct task_struct *task) +{ + struct sched_domain *sd; + struct sched_group *sg; + struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask); + int cpu, best_cpu = -1; + unsigned long best_capacity = ULONG_MAX; + unsigned long util, best_cpu_util = ULONG_MAX; + unsigned long best_cpu_util_cum = ULONG_MAX; + unsigned long util_cum; + unsigned long tutil = task_util(task); + int best_cpu_idle_idx = INT_MAX; + int cpu_idle_idx = -1; + bool boost_on_big = rt_boost_on_big(); + + rcu_read_lock(); + + cpu = cpu_rq(smp_processor_id())->rd->min_cap_orig_cpu; + if (cpu < 0) + goto unlock; + + sd = rcu_dereference(*per_cpu_ptr(&sd_asym_cpucapacity, cpu)); + if (!sd) + goto unlock; + +retry: + sg = sd->groups; + do { + int fcpu = group_first_cpu(sg); + int capacity_orig = capacity_orig_of(fcpu); + + if (boost_on_big) { + if (is_min_capacity_cpu(fcpu)) + continue; + } else { + if (capacity_orig > best_capacity) + continue; + } + + for_each_cpu_and(cpu, lowest_mask, sched_group_span(sg)) { + if (cpu_isolated(cpu)) + continue; + + if (sched_cpu_high_irqload(cpu)) + continue; + + util = cpu_util(cpu); + + if (__cpu_overutilized(cpu, util + tutil)) + continue; + + /* Find the least loaded CPU */ + if (util > best_cpu_util) + continue; + + /* + * If the previous CPU has same load, keep it as + * best_cpu. + */ + if (best_cpu_util == util && best_cpu == task_cpu(task)) + continue; + + /* + * If candidate CPU is the previous CPU, select it. + * Otherwise, if its load is same with best_cpu and in + * a shallower C-state, select it. If all above + * conditions are same, select the least cumulative + * window demand CPU. + */ + cpu_idle_idx = idle_get_state_idx(cpu_rq(cpu)); + + util_cum = cpu_util_cum(cpu, 0); + if (cpu != task_cpu(task) && best_cpu_util == util) { + if (best_cpu_idle_idx < cpu_idle_idx) + continue; + + if (best_cpu_idle_idx == cpu_idle_idx && + best_cpu_util_cum < util_cum) + continue; + } + + best_cpu_idle_idx = cpu_idle_idx; + best_cpu_util_cum = util_cum; + best_cpu_util = util; + best_cpu = cpu; + best_capacity = capacity_orig; + } + + } while (sg = sg->next, sg != sd->groups); + + if (unlikely(boost_on_big) && best_cpu == -1) { + boost_on_big = false; + goto retry; + } + +unlock: + rcu_read_unlock(); + return best_cpu; +} +#else +static inline int rt_energy_aware_wake_cpu(struct task_struct *task) +{ + return -1; +} +#endif + static int find_lowest_rq(struct task_struct *task) { struct sched_domain *sd; struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask); int this_cpu = smp_processor_id(); - int cpu = task_cpu(task); + int cpu = -1; /* Make sure the mask is initialized first */ if (unlikely(!lowest_mask)) @@ -1651,6 +1827,12 @@ static int find_lowest_rq(struct task_struct *task) if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask)) return -1; /* No targets found */ + if (sched_energy_enabled()) + cpu = rt_energy_aware_wake_cpu(task); + + if (cpu == -1) + cpu = task_cpu(task); + /* * At this point we have built a mask of CPUs representing the * lowest priority tasks in the system. Now we want to elect @@ -1857,7 +2039,9 @@ retry: } deactivate_task(rq, next_task, 0); + next_task->on_rq = TASK_ON_RQ_MIGRATING; set_task_cpu(next_task, lowest_rq->cpu); + next_task->on_rq = TASK_ON_RQ_QUEUED; activate_task(lowest_rq, next_task, 0); ret = 1; @@ -2129,7 +2313,9 @@ static void pull_rt_task(struct rq *this_rq) resched = true; deactivate_task(src_rq, p, 0); + p->on_rq = TASK_ON_RQ_MIGRATING; set_task_cpu(p, this_cpu); + p->on_rq = TASK_ON_RQ_QUEUED; activate_task(this_rq, p, 0); /* * We continue with the search, just in diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8d41569c67d4..9bb59dbc8639 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1022,6 +1022,15 @@ config PANIC_ON_SCHED_BUG Say N if unsure. +config PANIC_ON_RT_THROTTLING + bool "Panic on RT throttling" + help + Say Y here to enable the kernel to panic when a realtime + runqueue is throttled. This may be useful for detecting + and debugging RT throttling issues. + + Say N if unsure. + config SCHEDSTATS bool "Collect scheduler statistics" depends on DEBUG_KERNEL && PROC_FS From 6e250f90ece87217e32ec23def6022a39ac24eb7 Mon Sep 17 00:00:00 2001 From: John Dias Date: Thu, 15 Sep 2016 08:52:27 -0700 Subject: [PATCH 2/4] sched: avoid scheduling RT threads on cores currently handling softirqs Bug: 31501544 Change-Id: I99dd7aaa12c11270b28dbabea484bcc8fb8ba0c1 Git-commit: 080ea011fd9f47315e1fc53185872ef813b59d00 Git-repo: https://android.googlesource.com/kernel/msm [pkondeti@codeaurora.org: resolved minor merge conflicts and fixed checkpatch warnings] Signed-off-by: Pavankumar Kondeti [satyap@codeaurora.org: resolve trivial merge conflicts] Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/sched/cpupri.c | 35 ++++++++++++++++++++++++++++++++-- kernel/sched/rt.c | 44 +++++++++++++++++++++++++++++++++++++------ kernel/sched/sched.h | 5 +++++ 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index b7abca987d94..0f5b73be3faf 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -41,6 +41,27 @@ static int convert_prio(int prio) return cpupri; } +/** + * drop_nopreempt_cpus - remove a cpu from the mask if it is likely + * non-preemptible + * @lowest_mask: mask with selected CPUs (non-NULL) + */ +static void +drop_nopreempt_cpus(struct cpumask *lowest_mask) +{ + unsigned int cpu = cpumask_first(lowest_mask); + + while (cpu < nr_cpu_ids) { + /* unlocked access */ + struct task_struct *task = READ_ONCE(cpu_rq(cpu)->curr); + + if (task_may_not_preempt(task, cpu)) + cpumask_clear_cpu(cpu, lowest_mask); + + cpu = cpumask_next(cpu, lowest_mask); + } +} + /** * cpupri_find - find the best (lowest-pri) CPU in the system * @cp: The cpupri context @@ -61,9 +82,11 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p, { int idx = 0; int task_pri = convert_prio(p->prio); + bool drop_nopreempts = task_pri <= MAX_RT_PRIO; BUG_ON(task_pri >= CPUPRI_NR_PRIORITIES); +retry: for (idx = 0; idx < task_pri; idx++) { struct cpupri_vec *vec = &cp->pri_to_cpu[idx]; int skip = 0; @@ -99,7 +122,8 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p, if (lowest_mask) { cpumask_and(lowest_mask, p->cpus_ptr, vec->mask); - + if (drop_nopreempts) + drop_nopreempt_cpus(lowest_mask); /* * We have to ensure that we have at least one bit * still set in the array, since the map could have @@ -114,7 +138,14 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p, return 1; } - + /* + * If we can't find any non-preemptible cpu's, retry so we can + * find the lowest priority target and avoid priority inversion. + */ + if (drop_nopreempts) { + drop_nopreempts = false; + goto retry; + } return 0; } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 78eefc45d2cf..bedb98fd186d 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -6,6 +6,9 @@ #include "sched.h" #include "pelt.h" + +#include + #include "walt.h" int sched_rr_timeslice = RR_TIMESLICE; @@ -1459,11 +1462,26 @@ static void yield_task_rt(struct rq *rq) #ifdef CONFIG_SMP static int find_lowest_rq(struct task_struct *task); +/* + * Return whether the task on the given cpu is currently non-preemptible + * while handling a softirq or is likely to block preemptions soon because + * it is a ksoftirq thread. + */ +bool +task_may_not_preempt(struct task_struct *task, int cpu) +{ + struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu); + + return (task_thread_info(task)->preempt_count & SOFTIRQ_MASK) || + task == cpu_ksoftirqd; +} + static int select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) { struct task_struct *curr; struct rq *rq; + bool may_not_preempt; /* For anything but wake ups, just return the task_cpu */ if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) @@ -1475,7 +1493,17 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) curr = READ_ONCE(rq->curr); /* unlocked access */ /* - * If the current task on @p's runqueue is an RT task, then + * If the current task on @p's runqueue is a softirq task, + * it may run without preemption for a time that is + * ill-suited for a waiting RT task. Therefore, try to + * wake this RT task on another runqueue. + * + * Also, if the current task on @p's runqueue is an RT task, then + * it may run without preemption for a time that is + * ill-suited for a waiting RT task. Therefore, try to + * wake this RT task on another runqueue. + * + * Also, if the current task on @p's runqueue is an RT task, then * try to see if we can wake this RT task up on another * runqueue. Otherwise simply start this RT task * on its current runqueue. @@ -1496,18 +1524,22 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) * This test is optimistic, if we get it wrong the load-balancer * will have to sort it out. */ - if (sched_energy_enabled() || - (curr && unlikely(rt_task(curr)) && + may_not_preempt = task_may_not_preempt(curr, cpu); + if (sched_energy_enabled() || may_not_preempt || + (unlikely(rt_task(curr)) && (curr->nr_cpus_allowed < 2 || curr->prio <= p->prio))) { int target = find_lowest_rq(p); /* - * Don't bother moving it if the destination CPU is - * not running a lower priority task. + * If cpu is non-preemptible, prefer remote cpu + * even if it's running a higher-prio task. + * Otherwise: Don't bother moving it if the + * destination CPU is not running a lower priority task. */ if (target != -1 && - p->prio < cpu_rq(target)->rt.highest_prio.curr) + (may_not_preempt || + p->prio < cpu_rq(target)->rt.highest_prio.curr)) cpu = target; } rcu_read_unlock(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1587a51e1038..4e3dd5190b26 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2460,6 +2460,11 @@ extern void set_rq_online (struct rq *rq); extern void set_rq_offline(struct rq *rq); extern bool sched_smp_initialized; +/* + * task_may_not_preempt - check whether a task may not be preemptible soon + */ +extern bool task_may_not_preempt(struct task_struct *task, int cpu); + #else /* CONFIG_SMP */ /* From 68efde1ad11dabda38df2084db0c508450246492 Mon Sep 17 00:00:00 2001 From: John Dias Date: Wed, 5 Oct 2016 15:11:40 -0700 Subject: [PATCH 3/4] sched: avoid migrating when softint on tgt cpu should be short The scheduling change (bug 31501544) to avoid putting RT threads on cores that are handling softint's was catching cases where there was no reason to believe the softint would take a long time, resulting in unnecessary migration overhead. This patch reduces the migration to cases where the core has a softint that is actually likely to take a long time, as opposed to the RCU, SCHED, and TIMER softints that are rather quick. Bug: 31752786 Change-Id: Ib4e179f1e15c736b2fdba31070494e357e9fbbe2 Git-commit: ce05770bd37b8065b61ef650108ecef2b97b148b Git-repo: https://android.googlesource.com/kernel/msm [pkondeti@codeaurora.org: resolved minor merge conflicts] Signed-off-by: Pavankumar Kondeti [satyap@codeaurora.org: resolve trivial merge conflicts] Signed-off-by: Satya Durga Srinivasu Prabhala --- include/linux/interrupt.h | 7 +++++++ kernel/sched/rt.c | 12 ++++++++---- kernel/softirq.c | 9 +++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 89fc59dab57d..2ccad949658b 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -528,6 +528,12 @@ enum }; #define SOFTIRQ_STOP_IDLE_MASK (~(1 << RCU_SOFTIRQ)) +/* Softirq's where the handling might be long: */ +#define LONG_SOFTIRQ_MASK ((1 << NET_TX_SOFTIRQ) | \ + (1 << NET_RX_SOFTIRQ) | \ + (1 << BLOCK_SOFTIRQ) | \ + (1 << IRQ_POLL_SOFTIRQ) | \ + (1 << TASKLET_SOFTIRQ)) /* map softirq index to softirq name. update 'softirq_to_name' in * kernel/softirq.c when adding a new softirq. @@ -563,6 +569,7 @@ extern void raise_softirq_irqoff(unsigned int nr); extern void raise_softirq(unsigned int nr); DECLARE_PER_CPU(struct task_struct *, ksoftirqd); +DECLARE_PER_CPU(__u32, active_softirqs); static inline struct task_struct *this_cpu_ksoftirqd(void) { diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index bedb98fd186d..3a9a757bfe03 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1464,16 +1464,20 @@ static int find_lowest_rq(struct task_struct *task); /* * Return whether the task on the given cpu is currently non-preemptible - * while handling a softirq or is likely to block preemptions soon because - * it is a ksoftirq thread. + * while handling a potentially long softint, or if the task is likely + * to block preemptions soon because it is a ksoftirq thread that is + * handling slow softints. */ bool task_may_not_preempt(struct task_struct *task, int cpu) { + __u32 softirqs = per_cpu(active_softirqs, cpu) | + __IRQ_STAT(cpu, __softirq_pending); struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu); - return (task_thread_info(task)->preempt_count & SOFTIRQ_MASK) || - task == cpu_ksoftirqd; + return ((softirqs & LONG_SOFTIRQ_MASK) && + (task == cpu_ksoftirqd || + task_thread_info(task)->preempt_count & SOFTIRQ_MASK)); } static int diff --git a/kernel/softirq.c b/kernel/softirq.c index 0427a86743a4..a459e8f0f1b0 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -56,6 +56,13 @@ static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp DEFINE_PER_CPU(struct task_struct *, ksoftirqd); +/* + * active_softirqs -- per cpu, a mask of softirqs that are being handled, + * with the expectation that approximate answers are acceptable and therefore + * no synchronization. + */ +DEFINE_PER_CPU(__u32, active_softirqs); + const char * const softirq_to_name[NR_SOFTIRQS] = { "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL", "TASKLET", "SCHED", "HRTIMER", "RCU" @@ -272,6 +279,7 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) restart: /* Reset the pending bitmask before enabling irqs */ set_softirq_pending(0); + __this_cpu_write(active_softirqs, pending); local_irq_enable(); @@ -301,6 +309,7 @@ restart: pending >>= softirq_bit; } + __this_cpu_write(active_softirqs, 0); if (__this_cpu_read(ksoftirqd) == current) rcu_softirq_qs(); local_irq_disable(); From db6940d546f073c2230076e68b00f924a56085de Mon Sep 17 00:00:00 2001 From: Pavankumar Kondeti Date: Wed, 28 Jun 2017 12:00:31 +0530 Subject: [PATCH 4/4] softirq: defer softirq processing to ksoftirqd if CPU is busy with RT Defer the softirq processing to ksoftirqd if a RT task is running or queued on the current CPU. This complements the RT task placement algorithm which tries to find a CPU that is not currently busy with softirqs. Currently NET_TX, NET_RX, BLOCK and TASKLET softirqs are only deferred as they can potentially run for long time. Change-Id: Id7665244af6bbd5a96d9e591cf26154e9eaa860c Signed-off-by: Pavankumar Kondeti [satyap@codeaurora.org: trivial merge conflict resolution.] Signed-off-by: Satya Durga Srinivasu Prabhala --- include/linux/sched.h | 5 +++++ kernel/sched/cpupri.c | 11 +++++++++++ kernel/softirq.c | 5 ++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 5e1f1c6d630d..19a379613a94 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1721,6 +1721,7 @@ extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_ #ifdef CONFIG_SMP extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); +extern bool cpupri_check_rt(void); #else static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) { @@ -1731,6 +1732,10 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpuma return -EINVAL; return 0; } +static inline bool cpupri_check_rt(void) +{ + return false; +} #endif extern int yield_to(struct task_struct *p, bool preempt); diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index 0f5b73be3faf..47ba4a9dcd11 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -266,3 +266,14 @@ void cpupri_cleanup(struct cpupri *cp) for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) free_cpumask_var(cp->pri_to_cpu[i].mask); } + +/* + * cpupri_check_rt - check if CPU has a RT task + * should be called from rcu-sched read section. + */ +bool cpupri_check_rt(void) +{ + int cpu = raw_smp_processor_id(); + + return cpu_rq(cpu)->rd->cpupri.cpu_to_pri[cpu] > CPUPRI_NORMAL; +} diff --git a/kernel/softirq.c b/kernel/softirq.c index a459e8f0f1b0..ef2a7b2bf0a4 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -253,6 +253,8 @@ static inline bool lockdep_softirq_start(void) { return false; } static inline void lockdep_softirq_end(bool in_hardirq) { } #endif +#define long_softirq_pending() (local_softirq_pending() & LONG_SOFTIRQ_MASK) +#define defer_for_rt() (long_softirq_pending() && cpupri_check_rt()) asmlinkage __visible void __softirq_entry __do_softirq(void) { unsigned long end = jiffies + MAX_SOFTIRQ_TIME; @@ -317,6 +319,7 @@ restart: pending = local_softirq_pending(); if (pending) { if (time_before(jiffies, end) && !need_resched() && + !defer_for_rt() && --max_restart) goto restart; @@ -372,7 +375,7 @@ static inline void invoke_softirq(void) if (ksoftirqd_running(local_softirq_pending())) return; - if (!force_irqthreads) { + if (!force_irqthreads && !defer_for_rt()) { #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK /* * We can safely execute softirq on the current stack if