diff --git a/drivers/devfreq/arm-memlat-mon.c b/drivers/devfreq/arm-memlat-mon.c index 6e229ebeda96..0d1134a9bb8d 100644 --- a/drivers/devfreq/arm-memlat-mon.c +++ b/drivers/devfreq/arm-memlat-mon.c @@ -26,6 +26,8 @@ #include #include #include +#include +#include enum common_ev_idx { INST_IDX, @@ -47,6 +49,7 @@ struct event_data { struct perf_event *pevent; unsigned long prev_count; unsigned long last_delta; + u64 cached_total_count; }; struct cpu_data { @@ -123,6 +126,7 @@ struct memlat_cpu_grp { unsigned int num_active_mons; struct memlat_mon *mons; struct mutex mons_lock; + spinlock_t mon_active_lock; }; struct memlat_mon_spec { @@ -138,20 +142,28 @@ struct memlat_mon_spec { #define to_mon(hwmon) container_of(hwmon, struct memlat_mon, hw) static struct workqueue_struct *memlat_wq; +static DEFINE_PER_CPU(struct memlat_cpu_grp *, per_cpu_grp); +static DEFINE_MUTEX(notify_lock); +static int hp_idle_register_cnt; +static DEFINE_PER_CPU(bool, cpu_is_idle); +static DEFINE_PER_CPU(bool, cpu_is_hp); #define MAX_COUNT_LIM 0xFFFFFFFFFFFFFFFF static inline void read_event(struct event_data *event) { - unsigned long ev_count = 0; u64 total, enabled, running; if (!event->pevent) return; - total = perf_event_read_value(event->pevent, &enabled, &running); - ev_count = total - event->prev_count; + if (!per_cpu(cpu_is_idle, event->pevent->cpu) && + !per_cpu(cpu_is_hp, event->pevent->cpu)) + total = perf_event_read_value(event->pevent, &enabled, + &running); + else + total = event->cached_total_count; + event->last_delta = total - event->prev_count; event->prev_count = total; - event->last_delta = ev_count; } static void update_counts(struct memlat_cpu_grp *cpu_grp) @@ -210,12 +222,13 @@ static unsigned long get_cnt(struct memlat_hwmon *hw) devstats->freq = cpu_data->freq; devstats->stall_pct = cpu_data->stall_pct; - devstats->inst_count = common_evs[INST_IDX].last_delta; - if (mon->miss_ev) + if (mon->miss_ev) { + devstats->inst_count = + common_evs[INST_IDX].last_delta; devstats->mem_count = mon->miss_ev[mon_idx].last_delta; - else { + } else { devstats->inst_count = 0; devstats->mem_count = 1; } @@ -226,7 +239,9 @@ static unsigned long get_cnt(struct memlat_hwmon *hw) static void delete_event(struct event_data *event) { - event->prev_count = event->last_delta = 0; + event->prev_count = 0; + event->last_delta = 0; + event->cached_total_count = 0; if (event->pevent) { perf_event_release_kernel(event->pevent); event->pevent = NULL; @@ -276,6 +291,8 @@ static int init_common_evs(struct memlat_cpu_grp *cpu_grp, for_each_cpu(cpu, &cpu_grp->cpus) { struct event_data *common_evs = to_common_evs(cpu_grp, cpu); + if (per_cpu(cpu_is_hp, cpu)) + continue; for (i = 0; i < NUM_COMMON_EVS; i++) { ret = set_event(&common_evs[i], cpu, cpu_grp->common_ev_ids[i], attr); @@ -334,6 +351,173 @@ unlock_out: mutex_unlock(&cpu_grp->mons_lock); } +static int memlat_hp_restart_events(unsigned int cpu, bool cpu_up) +{ + struct perf_event_attr *attr = alloc_attr(); + struct memlat_mon *mon; + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + int i, ret = 0; + unsigned int idx; + struct event_data *common_evs; + + if (!cpu_grp) + goto exit; + + common_evs = to_common_evs(cpu_grp, cpu); + for (i = 0; i < NUM_COMMON_EVS; i++) { + if (cpu_up) { + ret = set_event(&common_evs[i], cpu, + cpu_grp->common_ev_ids[i], attr); + if (ret) { + pr_err("event %d not set for cpu %d ret %d\n", + cpu_grp->common_ev_ids[i], cpu, ret); + goto exit; + } + } else { + delete_event(&common_evs[i]); + } + } + + for (i = 0; i < cpu_grp->num_mons; i++) { + mon = &cpu_grp->mons[i]; + if (!mon->is_active || !mon->miss_ev || + !cpumask_test_cpu(cpu, &mon->cpus)) + continue; + + idx = cpu - cpumask_first(&mon->cpus); + if (cpu_up) { + ret = set_event(&mon->miss_ev[idx], cpu, + mon->miss_ev_id, attr); + if (ret) { + pr_err("event %d not set for cpu %d ret %d\n", + mon->miss_ev[idx], cpu, ret); + goto exit; + } + } else { + delete_event(&mon->miss_ev[idx]); + } + } + +exit: + kfree(attr); + return ret; +} + +static int memlat_idle_read_events(unsigned int cpu) +{ + struct memlat_mon *mon; + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + int i, ret = 0; + unsigned int idx; + struct event_data *common_evs; + unsigned long flags; + + if (!cpu_grp) + return 0; + + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); + if (!cpu_grp->num_active_mons) + goto exit; + + common_evs = to_common_evs(cpu_grp, cpu); + for (i = 0; i < NUM_COMMON_EVS; i++) { + if (common_evs[i].pevent) + ret = perf_event_read_local(common_evs[i].pevent, + &common_evs[i].cached_total_count, NULL, NULL); + } + + for (i = 0; i < cpu_grp->num_mons; i++) { + mon = &cpu_grp->mons[i]; + if (!mon->is_active || !mon->miss_ev || + !cpumask_test_cpu(cpu, &mon->cpus)) { + continue; + } + + idx = cpu - cpumask_first(&mon->cpus); + if (mon->miss_ev[idx].pevent) + ret = perf_event_read_local(mon->miss_ev[idx].pevent, + &mon->miss_ev[idx].cached_total_count, NULL, NULL); + } +exit: + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); + return ret; +} + +#ifdef CONFIG_HOTPLUG_CPU +static int memlat_event_hotplug_coming_up(unsigned int cpu) +{ + int ret = 0; + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + + if (!cpu_grp) + return -ENODEV; + mutex_lock(&cpu_grp->mons_lock); + ret = memlat_hp_restart_events(cpu, true); + per_cpu(cpu_is_hp, cpu) = false; + mutex_unlock(&cpu_grp->mons_lock); + + return ret; +} + +static int memlat_event_hotplug_going_down(unsigned int cpu) +{ + struct memlat_cpu_grp *cpu_grp = per_cpu(per_cpu_grp, cpu); + unsigned int ret = 0; + + if (!cpu_grp) + return -ENODEV; + /* avoid race between cpu hotplug and update_counts */ + mutex_lock(&cpu_grp->mons_lock); + per_cpu(cpu_is_hp, cpu) = true; + ret = memlat_hp_restart_events(cpu, false); + mutex_unlock(&cpu_grp->mons_lock); + + return ret; +} + +static int memlat_event_cpu_hp_init(void) +{ + int ret = 0; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "MEMLAT_EVENT", + memlat_event_hotplug_coming_up, + memlat_event_hotplug_going_down); + if (ret < 0) + pr_err("memlat: failed to register CPU hotplug notifier: %d\n", + ret); + else + ret = 0; + return ret; +} +#else +static int memlat_event_cpu_hp_init(void) { return 0; } +#endif +static int memlat_idle_notif(struct notifier_block *nb, unsigned long action, + void *data) +{ + int ret = NOTIFY_OK; + int cpu = smp_processor_id(); + + switch (action) { + case IDLE_START: + __this_cpu_write(cpu_is_idle, true); + if (per_cpu(cpu_is_hp, cpu)) + goto idle_exit; + else + ret = memlat_idle_read_events(cpu); + break; + case IDLE_END: + __this_cpu_write(cpu_is_idle, false); + break; + } +idle_exit: + return ret; +} +static struct notifier_block memlat_event_idle_nb = { + .notifier_call = memlat_idle_notif, +}; + static int start_hwmon(struct memlat_hwmon *hw) { int ret = 0; @@ -342,13 +526,31 @@ static int start_hwmon(struct memlat_hwmon *hw) struct memlat_cpu_grp *cpu_grp = mon->cpu_grp; bool should_init_cpu_grp; struct perf_event_attr *attr = alloc_attr(); + unsigned long flags; if (!attr) return -ENOMEM; mutex_lock(&cpu_grp->mons_lock); - should_init_cpu_grp = !(cpu_grp->num_active_mons++); + should_init_cpu_grp = !(cpu_grp->num_active_mons); if (should_init_cpu_grp) { + mutex_lock(¬ify_lock); + if (!hp_idle_register_cnt) { + get_online_cpus(); + for_each_cpu(cpu, cpu_possible_mask) { + if (!cpumask_test_cpu(cpu, cpu_online_mask)) + per_cpu(cpu_is_hp, cpu) = true; + } + ret = memlat_event_cpu_hp_init(); + put_online_cpus(); + if (ret < 0) { + mutex_unlock(¬ify_lock); + goto unlock_out; + } + idle_notifier_register(&memlat_event_idle_nb); + } + hp_idle_register_cnt++; + mutex_unlock(¬ify_lock); ret = init_common_evs(cpu_grp, attr); if (ret < 0) goto unlock_out; @@ -360,6 +562,8 @@ static int start_hwmon(struct memlat_hwmon *hw) for_each_cpu(cpu, &mon->cpus) { unsigned int idx = cpu - cpumask_first(&mon->cpus); + if (per_cpu(cpu_is_hp, cpu)) + continue; ret = set_event(&mon->miss_ev[idx], cpu, mon->miss_ev_id, attr); if (ret < 0) @@ -367,7 +571,11 @@ static int start_hwmon(struct memlat_hwmon *hw) } } + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); + cpu_grp->num_active_mons++; mon->is_active = true; + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); + if (should_init_cpu_grp) queue_delayed_work(memlat_wq, &cpu_grp->work, @@ -385,10 +593,13 @@ static void stop_hwmon(struct memlat_hwmon *hw) unsigned int cpu; struct memlat_mon *mon = to_mon(hw); struct memlat_cpu_grp *cpu_grp = mon->cpu_grp; + unsigned long flags; mutex_lock(&cpu_grp->mons_lock); + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); mon->is_active = false; cpu_grp->num_active_mons--; + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); for_each_cpu(cpu, &mon->cpus) { unsigned int idx = cpu - cpumask_first(&mon->cpus); @@ -405,7 +616,20 @@ static void stop_hwmon(struct memlat_hwmon *hw) if (!cpu_grp->num_active_mons) { cancel_delayed_work(&cpu_grp->work); free_common_evs(cpu_grp); + mutex_lock(¬ify_lock); + hp_idle_register_cnt--; + mutex_unlock(¬ify_lock); } + mutex_lock(¬ify_lock); + if (!hp_idle_register_cnt) { + cpuhp_remove_state_nocalls(CPUHP_AP_ONLINE_DYN); + idle_notifier_unregister(&memlat_event_idle_nb); + for_each_cpu(cpu, cpu_possible_mask) { + per_cpu(cpu_is_hp, cpu) = false; + per_cpu(cpu_is_idle, cpu) = false; + } + } + mutex_unlock(¬ify_lock); mutex_unlock(&cpu_grp->mons_lock); } @@ -508,7 +732,7 @@ static int memlat_cpu_grp_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct memlat_cpu_grp *cpu_grp; int ret = 0; - unsigned int event_id, num_cpus, num_mons; + unsigned int event_id, num_cpus, num_mons, cpu; cpu_grp = devm_kzalloc(dev, sizeof(*cpu_grp), GFP_KERNEL); if (!cpu_grp) @@ -565,8 +789,12 @@ static int memlat_cpu_grp_probe(struct platform_device *pdev) return -ENOMEM; mutex_init(&cpu_grp->mons_lock); + spin_lock_init(&cpu_grp->mon_active_lock); cpu_grp->update_ms = DEFAULT_UPDATE_MS; + for_each_cpu(cpu, &cpu_grp->cpus) { + per_cpu(per_cpu_grp, cpu) = cpu_grp; + } dev_set_drvdata(dev, cpu_grp); return 0; @@ -580,6 +808,7 @@ static int memlat_mon_probe(struct platform_device *pdev, bool is_compute) struct memlat_mon *mon; struct memlat_hwmon *hw; unsigned int event_id, num_cpus, cpu; + unsigned long flags; if (!memlat_wq) memlat_wq = create_freezable_workqueue("memlat_wq"); @@ -597,7 +826,9 @@ static int memlat_mon_probe(struct platform_device *pdev, bool is_compute) mutex_lock(&cpu_grp->mons_lock); mon = &cpu_grp->mons[cpu_grp->num_inited_mons]; + spin_lock_irqsave(&cpu_grp->mon_active_lock, flags); mon->is_active = false; + spin_unlock_irqrestore(&cpu_grp->mon_active_lock, flags); mon->requested_update_ms = 0; mon->cpu_grp = cpu_grp;