From 791a09d4451f0bb74132ff96534e14f4efff628b Mon Sep 17 00:00:00 2001 From: Rama Aparna Mallavarapu Date: Wed, 30 Oct 2019 15:06:31 -0700 Subject: [PATCH] devfreq: memlat: Add CPU Hotplug and Idle awareness in memlat Register with CPU Hotplug and Idle notifiers so that memlat gets notified when CPU is being hotplugged in/out or when CPU enters Idle state. The perf events should be recreated when the cpu is brought back online. Also do not update the counts when the cpu is hotplugged out or when it enters Idle state. Change-Id: I35a05be0a43a4519282d5787e6c87d92e5bec1de Signed-off-by: Rama Aparna Mallavarapu --- drivers/devfreq/arm-memlat-mon.c | 251 +++++++++++++++++++++++++++++-- 1 file changed, 241 insertions(+), 10 deletions(-) diff --git a/drivers/devfreq/arm-memlat-mon.c b/drivers/devfreq/arm-memlat-mon.c index 6e229ebeda96..0d1134a9bb8d 100644 --- a/drivers/devfreq/arm-memlat-mon.c +++ b/drivers/devfreq/arm-memlat-mon.c @@ -26,6 +26,8 @@ #include #include #include +#include +#include enum common_ev_idx { INST_IDX, @@ -47,6 +49,7 @@ struct event_data { struct perf_event *pevent; unsigned long prev_count; unsigned long last_delta; + u64 cached_total_count; }; struct cpu_data { @@ -123,6 +126,7 @@ struct memlat_cpu_grp { unsigned int num_active_mons; struct memlat_mon *mons; struct mutex mons_lock; + spinlock_t mon_active_lock; }; struct memlat_mon_spec { @@ -138,20 +142,28 @@ struct memlat_mon_spec { #define to_mon(hwmon) container_of(hwmon, struct memlat_mon, hw) static struct workqueue_struct *memlat_wq; +static DEFINE_PER_CPU(struct memlat_cpu_grp *, per_cpu_grp); +static DEFINE_MUTEX(notify_lock); +static int hp_idle_register_cnt; +static DEFINE_PER_CPU(bool, cpu_is_idle); +static DEFINE_PER_CPU(bool, cpu_is_hp); #define MAX_COUNT_LIM 0xFFFFFFFFFFFFFFFF static inline void read_event(struct event_data *event) { - unsigned long ev_count = 0; u64 total, enabled, running; if (!event->pevent) return; - total = perf_event_read_value(event->pevent, &enabled, &running); - ev_count = total - event->prev_count; + if (!per_cpu(cpu_is_idle, event->pevent->cpu) && + !per_cpu(cpu_is_hp, event->pevent->cpu)) + total = perf_event_read_value(event->pevent, &enabled, + &running); + else + total = event->cached_total_count; + event->last_delta = total - event->prev_count; event->prev_count = total; - event->last_delta = ev_count; } static void update_counts(struct memlat_cpu_grp *cpu_grp) @@ -210,12 +222,13 @@ static unsigned long get_cnt(struct memlat_hwmon *hw) devstats->freq = cpu_data->freq; devstats->stall_pct = cpu_data->stall_pct; - devstats->inst_count = common_evs[INST_IDX].last_delta; - if (mon->miss_ev) + if (mon->miss_ev) { + devstats->inst_count = + common_evs[INST_IDX].last_delta; devstats->mem_count = mon->miss_ev[mon_idx].last_delta; - else { + } else { devstats->inst_count = 0; devstats->mem_count = 1; } @@ -226,7 +239,9 @@ static unsigned long get_cnt(struct memlat_hwmon *hw) static void delete_event(struct event_data *event) { - event->prev_count = event->last_delta = 0; + event->prev_count = 0; + event->last_delta = 0; + event->cached_total_count = 0; if (event->pevent) { perf_event_release_kernel(event->pevent); event->pevent = NULL; @@ -276,6 +291,8 @@ static int init_common_evs(struct memlat_cpu_grp *cpu_grp, for_each_cpu(cpu, &cpu_grp->cpus) { struct event_data *common_evs = to_common_evs(cpu_grp, cpu); + if (per_cpu(cpu_is_hp, cpu)) + continue; for (i = 0; i < NUM_COMMON_EVS; i++) { ret = set_event(&common_evs[i], cpu, cpu_grp->common_ev_ids[i], attr); @@ -334,6 +351,173 @@ unlock_out: mutex_unlock(&cpu_grp->mons_lock); } +static int memlat_hp_restart_events(unsigned int cpu, bool cpu_up) +{ + struct perf_event_attr *attr = alloc_attr(); + struct memlat_mon *mon; + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + int i, ret = 0; + unsigned int idx; + struct event_data *common_evs; + + if (!cpu_grp) + goto exit; + + common_evs = to_common_evs(cpu_grp, cpu); + for (i = 0; i < NUM_COMMON_EVS; i++) { + if (cpu_up) { + ret = set_event(&common_evs[i], cpu, + cpu_grp->common_ev_ids[i], attr); + if (ret) { + pr_err("event %d not set for cpu %d ret %d\n", + cpu_grp->common_ev_ids[i], cpu, ret); + goto exit; + } + } else { + delete_event(&common_evs[i]); + } + } + + for (i = 0; i < cpu_grp->num_mons; i++) { + mon = &cpu_grp->mons[i]; + if (!mon->is_active || !mon->miss_ev || + !cpumask_test_cpu(cpu, &mon->cpus)) + continue; + + idx = cpu - cpumask_first(&mon->cpus); + if (cpu_up) { + ret = set_event(&mon->miss_ev[idx], cpu, + mon->miss_ev_id, attr); + if (ret) { + pr_err("event %d not set for cpu %d ret %d\n", + mon->miss_ev[idx], cpu, ret); + goto exit; + } + } else { + delete_event(&mon->miss_ev[idx]); + } + } + +exit: + kfree(attr); + return ret; +} + +static int memlat_idle_read_events(unsigned int cpu) +{ + struct memlat_mon *mon; + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + int i, ret = 0; + unsigned int idx; + struct event_data *common_evs; + unsigned long flags; + + if (!cpu_grp) + return 0; + + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); + if (!cpu_grp->num_active_mons) + goto exit; + + common_evs = to_common_evs(cpu_grp, cpu); + for (i = 0; i < NUM_COMMON_EVS; i++) { + if (common_evs[i].pevent) + ret = perf_event_read_local(common_evs[i].pevent, + &common_evs[i].cached_total_count, NULL, NULL); + } + + for (i = 0; i < cpu_grp->num_mons; i++) { + mon = &cpu_grp->mons[i]; + if (!mon->is_active || !mon->miss_ev || + !cpumask_test_cpu(cpu, &mon->cpus)) { + continue; + } + + idx = cpu - cpumask_first(&mon->cpus); + if (mon->miss_ev[idx].pevent) + ret = perf_event_read_local(mon->miss_ev[idx].pevent, + &mon->miss_ev[idx].cached_total_count, NULL, NULL); + } +exit: + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); + return ret; +} + +#ifdef CONFIG_HOTPLUG_CPU +static int memlat_event_hotplug_coming_up(unsigned int cpu) +{ + int ret = 0; + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + + if (!cpu_grp) + return -ENODEV; + mutex_lock(&cpu_grp->mons_lock); + ret = memlat_hp_restart_events(cpu, true); + per_cpu(cpu_is_hp, cpu) = false; + mutex_unlock(&cpu_grp->mons_lock); + + return ret; +} + +static int memlat_event_hotplug_going_down(unsigned int cpu) +{ + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + unsigned int ret = 0; + + if (!cpu_grp) + return -ENODEV; + /* avoid race between cpu hotplug and update_counts */ + mutex_lock(&cpu_grp->mons_lock); + per_cpu(cpu_is_hp, cpu) = true; + ret = memlat_hp_restart_events(cpu, false); + mutex_unlock(&cpu_grp->mons_lock); + + return ret; +} + +static int memlat_event_cpu_hp_init(void) +{ + int ret = 0; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "MEMLAT_EVENT", + memlat_event_hotplug_coming_up, + memlat_event_hotplug_going_down); + if (ret < 0) + pr_err("memlat: failed to register CPU hotplug notifier: %d\n", + ret); + else + ret = 0; + return ret; +} +#else +static int memlat_event_cpu_hp_init(void) { return 0; } +#endif +static int memlat_idle_notif(struct notifier_block *nb, unsigned long action, + void *data) +{ + int ret = NOTIFY_OK; + int cpu = smp_processor_id(); + + switch (action) { + case IDLE_START: + __this_cpu_write(cpu_is_idle, true); + if (per_cpu(cpu_is_hp, cpu)) + goto idle_exit; + else + ret = memlat_idle_read_events(cpu); + break; + case IDLE_END: + __this_cpu_write(cpu_is_idle, false); + break; + } +idle_exit: + return ret; +} +static struct notifier_block memlat_event_idle_nb = { + .notifier_call = memlat_idle_notif, +}; + static int start_hwmon(struct memlat_hwmon *hw) { int ret = 0; @@ -342,13 +526,31 @@ static int start_hwmon(struct memlat_hwmon *hw) struct memlat_cpu_grp *cpu_grp = mon->cpu_grp; bool should_init_cpu_grp; struct perf_event_attr *attr = alloc_attr(); + unsigned long flags; if (!attr) return -ENOMEM; mutex_lock(&cpu_grp->mons_lock); - should_init_cpu_grp = !(cpu_grp->num_active_mons++); + should_init_cpu_grp = !(cpu_grp->num_active_mons); if (should_init_cpu_grp) { + mutex_lock(¬ify_lock); + if (!hp_idle_register_cnt) { + get_online_cpus(); + for_each_cpu(cpu, cpu_possible_mask) { + if (!cpumask_test_cpu(cpu, cpu_online_mask)) + per_cpu(cpu_is_hp, cpu) = true; + } + ret = memlat_event_cpu_hp_init(); + put_online_cpus(); + if (ret < 0) { + mutex_unlock(¬ify_lock); + goto unlock_out; + } + idle_notifier_register(&memlat_event_idle_nb); + } + hp_idle_register_cnt++; + mutex_unlock(¬ify_lock); ret = init_common_evs(cpu_grp, attr); if (ret < 0) goto unlock_out; @@ -360,6 +562,8 @@ static int start_hwmon(struct memlat_hwmon *hw) for_each_cpu(cpu, &mon->cpus) { unsigned int idx = cpu - cpumask_first(&mon->cpus); + if (per_cpu(cpu_is_hp, cpu)) + continue; ret = set_event(&mon->miss_ev[idx], cpu, mon->miss_ev_id, attr); if (ret < 0) @@ -367,7 +571,11 @@ static int start_hwmon(struct memlat_hwmon *hw) } } + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); + cpu_grp->num_active_mons++; mon->is_active = true; + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); + if (should_init_cpu_grp) queue_delayed_work(memlat_wq, &cpu_grp->work, @@ -385,10 +593,13 @@ static void stop_hwmon(struct memlat_hwmon *hw) unsigned int cpu; struct memlat_mon *mon = to_mon(hw); struct memlat_cpu_grp *cpu_grp = mon->cpu_grp; + unsigned long flags; mutex_lock(&cpu_grp->mons_lock); + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); mon->is_active = false; cpu_grp->num_active_mons--; + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); for_each_cpu(cpu, &mon->cpus) { unsigned int idx = cpu - cpumask_first(&mon->cpus); @@ -405,7 +616,20 @@ static void stop_hwmon(struct memlat_hwmon *hw) if (!cpu_grp->num_active_mons) { cancel_delayed_work(&cpu_grp->work); free_common_evs(cpu_grp); + mutex_lock(¬ify_lock); + hp_idle_register_cnt--; + mutex_unlock(¬ify_lock); } + mutex_lock(¬ify_lock); + if (!hp_idle_register_cnt) { + cpuhp_remove_state_nocalls(CPUHP_AP_ONLINE_DYN); + idle_notifier_unregister(&memlat_event_idle_nb); + for_each_cpu(cpu, cpu_possible_mask) { + per_cpu(cpu_is_hp, cpu) = false; + per_cpu(cpu_is_idle, cpu) = false; + } + } + mutex_unlock(¬ify_lock); mutex_unlock(&cpu_grp->mons_lock); } @@ -508,7 +732,7 @@ static int memlat_cpu_grp_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct memlat_cpu_grp *cpu_grp; int ret = 0; - unsigned int event_id, num_cpus, num_mons; + unsigned int event_id, num_cpus, num_mons, cpu; cpu_grp = devm_kzalloc(dev, sizeof(*cpu_grp), GFP_KERNEL); if (!cpu_grp) @@ -565,8 +789,12 @@ static int memlat_cpu_grp_probe(struct platform_device *pdev) return -ENOMEM; mutex_init(&cpu_grp->mons_lock); + spin_lock_init(&cpu_grp->mon_active_lock); cpu_grp->update_ms = DEFAULT_UPDATE_MS; + for_each_cpu(cpu, &cpu_grp->cpus) { + per_cpu(per_cpu_grp, cpu) = cpu_grp; + } dev_set_drvdata(dev, cpu_grp); return 0; @@ -580,6 +808,7 @@ static int memlat_mon_probe(struct platform_device *pdev, bool is_compute) struct memlat_mon *mon; struct memlat_hwmon *hw; unsigned int event_id, num_cpus, cpu; + unsigned long flags; if (!memlat_wq) memlat_wq = create_freezable_workqueue("memlat_wq"); @@ -597,7 +826,9 @@ static int memlat_mon_probe(struct platform_device *pdev, bool is_compute) mutex_lock(&cpu_grp->mons_lock); mon = &cpu_grp->mons[cpu_grp->num_inited_mons]; + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); mon->is_active = false; + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); mon->requested_update_ms = 0; mon->cpu_grp = cpu_grp;