diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 58da31bac5c5..c91b534125e6 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -374,6 +374,9 @@ static inline void hrtimer_cancel_wait_running(struct hrtimer *timer) /* Exported timer functions: */ +/* To be used from cpusets, only */ +extern void hrtimer_quiesce_cpu(void *cpup); + /* Initialize timers: */ extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, enum hrtimer_mode mode); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index af9c2d855008..df8cdd458302 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2013,13 +2013,17 @@ int hrtimers_prepare_cpu(unsigned int cpu) return 0; } -#ifdef CONFIG_HOTPLUG_CPU - +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_CPUSETS) static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, - struct hrtimer_clock_base *new_base) + struct hrtimer_clock_base *new_base, + bool remove_pinned) { struct hrtimer *timer; struct timerqueue_node *node; + struct timerqueue_head pinned; + int is_pinned; + + timerqueue_init_head(&pinned); while ((node = timerqueue_getnext(&old_base->active))) { timer = container_of(node, struct hrtimer, node); @@ -2032,6 +2036,13 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, * under us on another CPU */ __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0); + + is_pinned = timer->state & HRTIMER_STATE_PINNED; + if (!remove_pinned && is_pinned) { + timerqueue_add(&pinned, &timer->node); + continue; + } + timer->base = new_base; /* * Enqueue the timers on the new cpu. This does not @@ -2043,23 +2054,29 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, */ enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS); } + + /* Re-queue pinned timers for non-hotplug usecase */ + while ((node = timerqueue_getnext(&pinned))) { + timer = container_of(node, struct hrtimer, node); + + timerqueue_del(&pinned, &timer->node); + enqueue_hrtimer(timer, old_base, HRTIMER_MODE_ABS); + } } -int hrtimers_dead_cpu(unsigned int scpu) +static void __migrate_hrtimers(unsigned int scpu, bool remove_pinned) { struct hrtimer_cpu_base *old_base, *new_base; + unsigned long flags; int i; - BUG_ON(cpu_online(scpu)); - tick_cancel_sched_timer(scpu); - /* * this BH disable ensures that raise_softirq_irqoff() does * not wakeup ksoftirqd (and acquire the pi-lock) while * holding the cpu_base lock */ local_bh_disable(); - local_irq_disable(); + local_irq_save(flags); old_base = &per_cpu(hrtimer_bases, scpu); new_base = this_cpu_ptr(&hrtimer_bases); /* @@ -2071,7 +2088,7 @@ int hrtimers_dead_cpu(unsigned int scpu) for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { migrate_hrtimer_list(&old_base->clock_base[i], - &new_base->clock_base[i]); + &new_base->clock_base[i], remove_pinned); } /* @@ -2085,13 +2102,29 @@ int hrtimers_dead_cpu(unsigned int scpu) /* Check, if we got expired work to do */ __hrtimer_peek_ahead_timers(); - local_irq_enable(); + local_irq_restore(flags); local_bh_enable(); +} +#endif /* CONFIG_HOTPLUG_CPU || CONFIG_CPUSETS */ + +#ifdef CONFIG_HOTPLUG_CPU +int hrtimers_dead_cpu(unsigned int scpu) +{ + BUG_ON(cpu_online(scpu)); + tick_cancel_sched_timer(scpu); + + __migrate_hrtimers(scpu, true); return 0; } - #endif /* CONFIG_HOTPLUG_CPU */ +#ifdef CONFIG_CPUSETS +void hrtimer_quiesce_cpu(void *cpup) +{ + __migrate_hrtimers(*(int *)cpup, false); +} +#endif /* CONFIG_CPUSETS */ + void __init hrtimers_init(void) { hrtimers_prepare_cpu(smp_processor_id());