From e2e45baa03ae8a7e5e86ab3703983ca18670f869 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 11:57:45 +0530 Subject: [PATCH 1/3] hrtimer: update timer->state with 'pinned' information 'Pinned' information would be required in migrate_hrtimers() now, as we can migrate non-pinned timers away without a hotplug (i.e. with cpuset.quiesce). And so we may need to identify pinned timers now, as we can't migrate them. This patch reuses the timer->state variable for setting this flag as there were enough number of free bits available in this variable. And there is no point increasing size of this struct by adding another field. Change-Id: Id2cdd74903598e31fab6999c4cbab9c2fd17fe23 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4] Git-commit: 62feaf1ed0b64c04868d143d8bdb92d60dc3189b Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan [markivx: Port to 4.14, fix merge conflicts due to lack of timer stats code] Signed-off-by: Vikram Mulukutla [satyap@codeaurora.org: Port to 4.19, fix merge conflicts] Signed-off-by: Satya Durga Srinivasu Prabhala --- include/linux/hrtimer.h | 3 +++ kernel/time/hrtimer.c | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 1b9a51a1bccb..58da31bac5c5 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -73,6 +73,7 @@ enum hrtimer_restart { * * 0x00 inactive * 0x01 enqueued into rbtree + * 0x02 timer is pinned to a cpu * * The callback state is not part of the timer->state because clearing it would * mean touching the timer after the callback, this makes it impossible to free @@ -92,6 +93,8 @@ enum hrtimer_restart { */ #define HRTIMER_STATE_INACTIVE 0x00 #define HRTIMER_STATE_ENQUEUED 0x01 +#define HRTIMER_PINNED_SHIFT 1 +#define HRTIMER_STATE_PINNED (1 << HRTIMER_PINNED_SHIFT) /** * struct hrtimer - the basic hrtimer structure diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 65605530ee34..af9c2d855008 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -966,7 +966,7 @@ static int enqueue_hrtimer(struct hrtimer *timer, base->cpu_base->active_bases |= 1 << base->index; - timer->state = HRTIMER_STATE_ENQUEUED; + timer->state |= HRTIMER_STATE_ENQUEUED; return timerqueue_add(&base->active, &timer->node); } @@ -986,11 +986,9 @@ static void __remove_hrtimer(struct hrtimer *timer, u8 newstate, int reprogram) { struct hrtimer_cpu_base *cpu_base = base->cpu_base; - u8 state = timer->state; - timer->state = newstate; - if (!(state & HRTIMER_STATE_ENQUEUED)) - return; + if (!(timer->state & HRTIMER_STATE_ENQUEUED)) + goto out; if (!timerqueue_del(&base->active, &timer->node)) cpu_base->active_bases &= ~(1 << base->index); @@ -1005,6 +1003,13 @@ static void __remove_hrtimer(struct hrtimer *timer, */ if (reprogram && timer == cpu_base->next_timer) hrtimer_force_reprogram(cpu_base, 1); + +out: + /* + * We need to preserve PINNED state here, otherwise we may end up + * migrating pinned hrtimers as well. + */ + timer->state = newstate | (timer->state & HRTIMER_STATE_PINNED); } /* @@ -1097,6 +1102,10 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, /* Switch the timer base, if necessary: */ new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); + /* Update pinned state */ + timer->state &= ~HRTIMER_STATE_PINNED; + timer->state |= !!(mode & HRTIMER_MODE_PINNED) << HRTIMER_PINNED_SHIFT; + return enqueue_hrtimer(timer, new_base, mode); } From a13bd9b94091243c24b0469adc2c449d6edea984 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 12:07:31 +0530 Subject: [PATCH 2/3] hrtimer: create hrtimer_quiesce_cpu() to isolate CPU from hrtimers To isolate CPUs (isolate from hrtimers) from sysfs using cpusets, we need some support from the hrtimer core. i.e. A routine hrtimer_quiesce_cpu() which would migrate away all the unpinned hrtimers, but shouldn't touch the pinned ones. This patch creates this routine. Change-Id: Icf93622bdeed9c6c28eb1aca1637bc5b9522a533 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla [ohaugan@codeaurora.org: Port to 4.4] Git-commit: d4d50a0ddc35e58ee95137ba4d14e74fea8b682f Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan [rameezmustafa@codeaurora.org: Port to msm-4.9] Signed-off-by: Syed Rameez Mustafa [satyap@codeaurora.org: Port to msm-4.19 and fix merge conflicts] Signed-off-by: Satya Durga Srinivasu Prabhala --- include/linux/hrtimer.h | 3 +++ kernel/time/hrtimer.c | 55 ++++++++++++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 58da31bac5c5..c91b534125e6 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -374,6 +374,9 @@ static inline void hrtimer_cancel_wait_running(struct hrtimer *timer) /* Exported timer functions: */ +/* To be used from cpusets, only */ +extern void hrtimer_quiesce_cpu(void *cpup); + /* Initialize timers: */ extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, enum hrtimer_mode mode); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index af9c2d855008..df8cdd458302 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2013,13 +2013,17 @@ int hrtimers_prepare_cpu(unsigned int cpu) return 0; } -#ifdef CONFIG_HOTPLUG_CPU - +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_CPUSETS) static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, - struct hrtimer_clock_base *new_base) + struct hrtimer_clock_base *new_base, + bool remove_pinned) { struct hrtimer *timer; struct timerqueue_node *node; + struct timerqueue_head pinned; + int is_pinned; + + timerqueue_init_head(&pinned); while ((node = timerqueue_getnext(&old_base->active))) { timer = container_of(node, struct hrtimer, node); @@ -2032,6 +2036,13 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, * under us on another CPU */ __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0); + + is_pinned = timer->state & HRTIMER_STATE_PINNED; + if (!remove_pinned && is_pinned) { + timerqueue_add(&pinned, &timer->node); + continue; + } + timer->base = new_base; /* * Enqueue the timers on the new cpu. This does not @@ -2043,23 +2054,29 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, */ enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS); } + + /* Re-queue pinned timers for non-hotplug usecase */ + while ((node = timerqueue_getnext(&pinned))) { + timer = container_of(node, struct hrtimer, node); + + timerqueue_del(&pinned, &timer->node); + enqueue_hrtimer(timer, old_base, HRTIMER_MODE_ABS); + } } -int hrtimers_dead_cpu(unsigned int scpu) +static void __migrate_hrtimers(unsigned int scpu, bool remove_pinned) { struct hrtimer_cpu_base *old_base, *new_base; + unsigned long flags; int i; - BUG_ON(cpu_online(scpu)); - tick_cancel_sched_timer(scpu); - /* * this BH disable ensures that raise_softirq_irqoff() does * not wakeup ksoftirqd (and acquire the pi-lock) while * holding the cpu_base lock */ local_bh_disable(); - local_irq_disable(); + local_irq_save(flags); old_base = &per_cpu(hrtimer_bases, scpu); new_base = this_cpu_ptr(&hrtimer_bases); /* @@ -2071,7 +2088,7 @@ int hrtimers_dead_cpu(unsigned int scpu) for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { migrate_hrtimer_list(&old_base->clock_base[i], - &new_base->clock_base[i]); + &new_base->clock_base[i], remove_pinned); } /* @@ -2085,13 +2102,29 @@ int hrtimers_dead_cpu(unsigned int scpu) /* Check, if we got expired work to do */ __hrtimer_peek_ahead_timers(); - local_irq_enable(); + local_irq_restore(flags); local_bh_enable(); +} +#endif /* CONFIG_HOTPLUG_CPU || CONFIG_CPUSETS */ + +#ifdef CONFIG_HOTPLUG_CPU +int hrtimers_dead_cpu(unsigned int scpu) +{ + BUG_ON(cpu_online(scpu)); + tick_cancel_sched_timer(scpu); + + __migrate_hrtimers(scpu, true); return 0; } - #endif /* CONFIG_HOTPLUG_CPU */ +#ifdef CONFIG_CPUSETS +void hrtimer_quiesce_cpu(void *cpup) +{ + __migrate_hrtimers(*(int *)cpup, false); +} +#endif /* CONFIG_CPUSETS */ + void __init hrtimers_init(void) { hrtimers_prepare_cpu(smp_processor_id()); From 55efbbcc5fa1b2fe19e882657f233fd3645ae2d5 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 25 Mar 2015 12:57:46 +0530 Subject: [PATCH 3/3] hrtimer: make sure PINNED flag is cleared after removing hrtimer Change-Id: I1a3211bc0d5d310c7fa7b92dc44ec6c4722dd7b9 Signed-off-by: Viresh Kumar [forward port to 3.18] Signed-off-by: Santosh Shukla Git-commit: d6c894e515b4cd93c3a08e7c60cce0aa5118c656 Git-repo: git://git.linaro.org/people/mike.holmes/santosh.shukla/lng-isol.git Signed-off-by: Olav Haugan Signed-off-by: Satya Durga Srinivasu Prabhala --- kernel/time/hrtimer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index df8cdd458302..66885f6f8d4b 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1037,6 +1037,7 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool rest state = HRTIMER_STATE_INACTIVE; __remove_hrtimer(timer, base, state, reprogram); + timer->state &= ~HRTIMER_STATE_PINNED; return 1; } return 0; @@ -1104,7 +1105,7 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, /* Update pinned state */ timer->state &= ~HRTIMER_STATE_PINNED; - timer->state |= !!(mode & HRTIMER_MODE_PINNED) << HRTIMER_PINNED_SHIFT; + timer->state |= (!!(mode & HRTIMER_MODE_PINNED)) << HRTIMER_PINNED_SHIFT; return enqueue_hrtimer(timer, new_base, mode); }