From ffedc6cbb5816b164758ec62e7334ed30fac306c Mon Sep 17 00:00:00 2001 From: Maulik Shah Date: Wed, 20 Feb 2019 17:11:20 +0530 Subject: [PATCH] cpuidle: lpm-levels: Track and predict next rescheduling ipi Add changes to track and predict next rescheduling ipi based on past history. Add a module param to control enabling it. Change-Id: Ie495d8906288ee410708693ee15ed51643aefb44 Signed-off-by: Maulik Shah --- arch/arm/kernel/smp.c | 3 + arch/arm64/kernel/smp.c | 2 + drivers/cpuidle/lpm-levels-of.c | 3 + drivers/cpuidle/lpm-levels.c | 146 ++++++++++++++++++++++---------- drivers/cpuidle/lpm-levels.h | 5 +- include/soc/qcom/lpm_levels.h | 2 + 6 files changed, 113 insertions(+), 48 deletions(-) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index c6e3551972cf..50c0cf671411 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -49,6 +49,8 @@ #include #include +#include + #define CREATE_TRACE_POINTS #include @@ -719,6 +721,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs) void smp_send_reschedule(int cpu) { + update_ipi_history(cpu); smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index e87f7a0b7fc4..1b87e919d686 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -51,6 +51,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -985,6 +986,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs) void smp_send_reschedule(int cpu) { + update_ipi_history(cpu); smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } diff --git a/drivers/cpuidle/lpm-levels-of.c b/drivers/cpuidle/lpm-levels-of.c index b16b552cdb94..f384fa4c74a3 100644 --- a/drivers/cpuidle/lpm-levels-of.c +++ b/drivers/cpuidle/lpm-levels-of.c @@ -557,6 +557,9 @@ static int parse_cpu_levels(struct device_node *dn, struct lpm_cluster *c) if (ret) return ret; + cpu->ipi_prediction = !(of_property_read_bool(dn, + "qcom,disable-ipi-prediction")); + cpu->lpm_prediction = !(of_property_read_bool(dn, "qcom,disable-prediction")); diff --git a/drivers/cpuidle/lpm-levels.c b/drivers/cpuidle/lpm-levels.c index 18527e1799ff..00cd13682f4c 100644 --- a/drivers/cpuidle/lpm-levels.c +++ b/drivers/cpuidle/lpm-levels.c @@ -82,6 +82,9 @@ struct lpm_cluster *lpm_root_node; static bool lpm_prediction = true; module_param_named(lpm_prediction, lpm_prediction, bool, 0664); +static bool lpm_ipi_prediction = true; +module_param_named(lpm_ipi_prediction, lpm_ipi_prediction, bool, 0664); + struct lpm_history { uint32_t resi[MAXSAMPLES]; int mode[MAXSAMPLES]; @@ -92,8 +95,14 @@ struct lpm_history { int64_t stime; }; -static DEFINE_PER_CPU(struct lpm_history, hist); +struct ipi_history { + uint32_t interval[MAXSAMPLES]; + uint32_t current_ptr; + ktime_t cpu_idle_resched_ts; +}; +static DEFINE_PER_CPU(struct lpm_history, hist); +static DEFINE_PER_CPU(struct ipi_history, cpu_ipi_history); static DEFINE_PER_CPU(struct lpm_cpu*, cpu_lpm); static bool suspend_in_progress; static DEFINE_PER_CPU(struct hrtimer, histtimer); @@ -322,14 +331,63 @@ static void clusttimer_start(struct lpm_cluster *cluster, uint32_t time_us) HRTIMER_MODE_REL_PINNED); } -static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev, - struct lpm_cpu *cpu, int *idx_restrict, - uint32_t *idx_restrict_time) +static uint64_t find_deviation(int *interval, uint32_t ref_stddev, + int64_t *stime) { - int i, j, divisor; + int divisor, i; uint64_t max, avg, stddev; int64_t thresh = LLONG_MAX; + + do { + max = avg = divisor = stddev = 0; + for (i = 0; i < MAXSAMPLES; i++) { + int64_t value = interval[i]; + + if (value <= thresh) { + avg += value; + divisor++; + if (value > max) + max = value; + } + } + do_div(avg, divisor); + + for (i = 0; i < MAXSAMPLES; i++) { + int64_t value = interval[i]; + + if (value <= thresh) { + int64_t diff = value - avg; + + stddev += diff * diff; + } + } + do_div(stddev, divisor); + stddev = int_sqrt(stddev); + + /* + * If the deviation is less, return the average, else + * ignore one maximum sample and retry + */ + if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1))) + || stddev <= ref_stddev) { + *stime = ktime_to_us(ktime_get()) + avg; + return avg; + } + thresh = max - 1; + + } while (divisor > (MAXSAMPLES - 1)); + + return 0; +} + +static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev, + struct lpm_cpu *cpu, int *idx_restrict, + uint32_t *idx_restrict_time, uint32_t *ipi_predicted) +{ + int i, j; + uint64_t avg; struct lpm_history *history = &per_cpu(hist, dev->cpu); + struct ipi_history *ipi_history = &per_cpu(cpu_ipi_history, dev->cpu); if (!lpm_prediction || !cpu->lpm_prediction) return 0; @@ -360,44 +418,9 @@ static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev, * that mode. */ -again: - max = avg = divisor = stddev = 0; - for (i = 0; i < MAXSAMPLES; i++) { - int64_t value = history->resi[i]; - - if (value <= thresh) { - avg += value; - divisor++; - if (value > max) - max = value; - } - } - do_div(avg, divisor); - - for (i = 0; i < MAXSAMPLES; i++) { - int64_t value = history->resi[i]; - - if (value <= thresh) { - int64_t diff = value - avg; - - stddev += diff * diff; - } - } - do_div(stddev, divisor); - stddev = int_sqrt(stddev); - - /* - * If the deviation is less, return the average, else - * ignore one maximum sample and retry - */ - if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1))) - || stddev <= cpu->ref_stddev) { - history->stime = ktime_to_us(ktime_get()) + avg; + avg = find_deviation(history->resi, cpu->ref_stddev, &(history->stime)); + if (avg) return avg; - } else if (divisor > (MAXSAMPLES - 1)) { - thresh = max - 1; - goto again; - } /* * Find the number of premature exits for each of the mode, @@ -440,6 +463,18 @@ again: } } } + + if (*idx_restrict_time || !cpu->ipi_prediction || !lpm_ipi_prediction) + return 0; + + avg = find_deviation(ipi_history->interval, cpu->ref_stddev + + DEFAULT_IPI_STDDEV, + &(history->stime)); + if (avg) { + *ipi_predicted = 1; + return avg; + } + return 0; } @@ -519,7 +554,7 @@ static int cpu_power_select(struct cpuidle_device *dev, int i, idx_restrict; uint32_t lvl_latency_us = 0; uint64_t predicted = 0; - uint32_t htime = 0, idx_restrict_time = 0; + uint32_t htime = 0, idx_restrict_time = 0, ipi_predicted = 0; uint32_t next_wakeup_us = (uint32_t)sleep_us; uint32_t min_residency, max_residency; struct power_params *pwr_params; @@ -549,7 +584,8 @@ static int cpu_power_select(struct cpuidle_device *dev, */ if (next_wakeup_us > max_residency) { predicted = lpm_cpuidle_predict(dev, cpu, - &idx_restrict, &idx_restrict_time); + &idx_restrict, &idx_restrict_time, + &ipi_predicted); if (predicted && (predicted < min_residency)) predicted = min_residency; } else @@ -578,7 +614,9 @@ static int cpu_power_select(struct cpuidle_device *dev, if ((predicted || (idx_restrict != cpu->nlevels + 1)) && (best_level < (cpu->nlevels-1))) { htime = predicted + cpu->tmr_add; - if (htime == cpu->tmr_add) + if (lpm_ipi_prediction && cpu->ipi_prediction) + htime += DEFAULT_IPI_TIMER_ADD; + if (!predicted) htime = idx_restrict_time; else if (htime > max_residency) htime = max_residency; @@ -591,8 +629,8 @@ static int cpu_power_select(struct cpuidle_device *dev, done_select: trace_cpu_power_select(best_level, sleep_us, latency_us, 0); - trace_cpu_pred_select(idx_restrict_time ? 2 : (predicted ? 1 : 0), - predicted, htime); + trace_cpu_pred_select(idx_restrict_time ? 2 : (ipi_predicted ? + 3 : (predicted ? 1 : 0)), predicted, htime); return best_level; } @@ -1217,6 +1255,20 @@ static int lpm_cpuidle_select(struct cpuidle_driver *drv, return cpu_power_select(dev, cpu); } +void update_ipi_history(int cpu) +{ + struct ipi_history *history = &per_cpu(cpu_ipi_history, cpu); + ktime_t now = ktime_get(); + + history->interval[history->current_ptr] = + ktime_to_us(ktime_sub(now, + history->cpu_idle_resched_ts)); + (history->current_ptr)++; + if (history->current_ptr >= MAXSAMPLES) + history->current_ptr = 0; + history->cpu_idle_resched_ts = now; +} + static void update_history(struct cpuidle_device *dev, int idx) { struct lpm_history *history = &per_cpu(hist, dev->cpu); diff --git a/drivers/cpuidle/lpm-levels.h b/drivers/cpuidle/lpm-levels.h index e58786dddd3e..2bab7c911d88 100644 --- a/drivers/cpuidle/lpm-levels.h +++ b/drivers/cpuidle/lpm-levels.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2014-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2014-2020, The Linux Foundation. All rights reserved. */ #ifndef __LPM_LEVELS_H__ @@ -14,7 +14,9 @@ #define CLUST_SMPL_INVLD_TIME 40000 #define DEFAULT_PREMATURE_CNT 3 #define DEFAULT_STDDEV 100 +#define DEFAULT_IPI_STDDEV 400 #define DEFAULT_TIMER_ADD 100 +#define DEFAULT_IPI_TIMER_ADD 900 #define TIMER_ADD_LOW 100 #define TIMER_ADD_HIGH 1500 #define STDDEV_LOW 100 @@ -48,6 +50,7 @@ struct lpm_cpu { uint32_t ref_premature_cnt; uint32_t tmr_add; bool lpm_prediction; + bool ipi_prediction; struct cpuidle_driver *drv; struct lpm_cluster *parent; ktime_t next_hrtimer; diff --git a/include/soc/qcom/lpm_levels.h b/include/soc/qcom/lpm_levels.h index 1fcf312b88c6..4d1fd439058d 100644 --- a/include/soc/qcom/lpm_levels.h +++ b/include/soc/qcom/lpm_levels.h @@ -21,9 +21,11 @@ struct system_pm_ops { #ifdef CONFIG_MSM_PM uint32_t register_system_pm_ops(struct system_pm_ops *pm_ops); +void update_ipi_history(int cpu); #else static inline uint32_t register_system_pm_ops(struct system_pm_ops *pm_ops) { return -ENODEV; } +static inline void update_ipi_history(int cpu) {} #endif #endif