diff --git a/include/linux/timer.h b/include/linux/timer.h index 532e4ba2cfe3..84aa69c3fdbc 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -161,6 +161,9 @@ extern int del_timer(struct timer_list * timer); extern int mod_timer(struct timer_list *timer, unsigned long expires); extern int mod_timer_pending(struct timer_list *timer, unsigned long expires); extern int timer_reduce(struct timer_list *timer, unsigned long expires); +#ifdef CONFIG_SMP +extern bool check_pending_deferrable_timers(int cpu); +#endif /* * The jiffies value which is added to now, when there is no timer diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index f4ee1a3428ae..907f8b265860 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -932,6 +933,11 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts) ktime_t expires; int cpu = smp_processor_id(); +#ifdef CONFIG_SMP + if (check_pending_deferrable_timers(cpu)) + raise_softirq_irqoff(TIMER_SOFTIRQ); +#endif + /* * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the * tick timer expiration time is known already. diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 65af3b1d8120..74bf5dde73a8 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -207,6 +207,7 @@ struct timer_base { static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]); struct timer_base timer_base_deferrable; +static atomic_t deferrable_pending; #ifdef CONFIG_NO_HZ_COMMON @@ -1518,6 +1519,31 @@ static u64 cmp_next_hrtimer_event(u64 basem, u64 expires) return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC; } + +#ifdef CONFIG_SMP +/* + * check_pending_deferrable_timers - Check for unbound deferrable timer expiry + * @cpu - Current CPU + * + * The function checks whether any global deferrable pending timers + * are exipired or not. This function does not check cpu bounded + * diferrable pending timers expiry. + * + * The function returns true when a cpu unbounded deferrable timer is expired. + */ +bool check_pending_deferrable_timers(int cpu) +{ + if (cpu == tick_do_timer_cpu || + tick_do_timer_cpu == TICK_DO_TIMER_NONE) { + if (time_after_eq(jiffies, timer_base_deferrable.clk) + && !atomic_cmpxchg(&deferrable_pending, 0, 1)) { + return true; + } + } + return false; +} +#endif + /** * get_next_timer_interrupt - return the time (clock mono) of the next timer * @basej: base time jiffies @@ -1704,9 +1730,13 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h) __run_timers(base); if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) { - __run_timers(&timer_base_deferrable); __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF])); } + + if ((atomic_cmpxchg(&deferrable_pending, 1, 0) && + tick_do_timer_cpu == TICK_DO_TIMER_NONE) || + tick_do_timer_cpu == smp_processor_id()) + __run_timers(&timer_base_deferrable); } /* @@ -1942,10 +1972,19 @@ static void __init init_timer_cpu(int cpu) } } +static inline void init_timer_deferrable_global(void) +{ + timer_base_deferrable.cpu = nr_cpu_ids; + raw_spin_lock_init(&timer_base_deferrable.lock); + timer_base_deferrable.clk = jiffies; +} + static void __init init_timer_cpus(void) { int cpu; + init_timer_deferrable_global(); + for_each_possible_cpu(cpu) init_timer_cpu(cpu); }