diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c index 82a0788795b9..bf795f86c459 100644 --- a/drivers/mmc/host/sdhci-msm.c +++ b/drivers/mmc/host/sdhci-msm.c @@ -14,6 +14,11 @@ #include #include #include +#include +#include +#include +#include +#include #include "sdhci-pltfm.h" #include "cqhci.h" @@ -126,6 +131,7 @@ #define CMUX_SHIFT_PHASE_MASK (7 << CMUX_SHIFT_PHASE_SHIFT) #define MSM_MMC_AUTOSUSPEND_DELAY_MS 50 +#define MSM_PMQOS_UNVOTING_DELAY_MS 10 /* msec */ /* Timeout value to avoid infinite waiting for pwr_irq */ #define MSM_PWR_IRQ_TIMEOUT_MS 5000 @@ -359,6 +365,30 @@ struct sdhci_msm_vreg_data { struct sdhci_msm_reg_data *vdd_io_data; }; +/* Per cpu cluster qos group */ +struct qos_cpu_group { + cpumask_t mask; /* CPU mask of cluster */ + unsigned int *votes; /* Different votes for cluster */ + struct dev_pm_qos_request *qos_req; /* Pointer to host qos request*/ + bool voted; + struct sdhci_msm_host *host; + bool initialized; + bool curr_vote; +}; + +/* Per host qos request structure */ +struct sdhci_msm_qos_req { + struct qos_cpu_group *qcg; /* CPU group per host */ + unsigned int num_groups; /* Number of groups */ + unsigned int active_mask; /* Active affine irq mask */ +}; + +enum constraint { + QOS_PERF, + QOS_POWER, + QOS_MAX, +}; + struct sdhci_msm_host { struct platform_device *pdev; void __iomem *core_mem; /* MSM SDCC mapped address */ @@ -389,15 +419,22 @@ struct sdhci_msm_host { bool skip_bus_bw_voting; struct sdhci_msm_bus_vote_data *bus_vote_data; struct delayed_work bus_vote_work; + struct delayed_work pmqos_unvote_work; bool pltfm_init_done; bool core_3_0v_support; bool use_7nm_dll; struct sdhci_msm_dll_hsr *dll_hsr; struct sdhci_msm_regs_restore regs_restore; + struct workqueue_struct *workq; /* QoS work queue */ + struct sdhci_msm_qos_req *sdhci_qos; + struct irq_affinity_notify affinity_notify; }; static struct sdhci_msm_host *sdhci_slot[2]; +static int sdhci_msm_update_qos_constraints(struct qos_cpu_group *qcg, + enum constraint type); + static void sdhci_msm_bus_voting(struct sdhci_host *host, bool enable); static int sdhci_msm_dt_get_array(struct device *dev, const char *prop_name, @@ -3067,6 +3104,343 @@ static void sdhci_set_default_hw_caps(struct sdhci_msm_host *msm_host, msm_host->use_7nm_dll = true; } +/* Find cpu group qos from a given cpu */ +static struct qos_cpu_group *cpu_to_group(struct sdhci_msm_qos_req *r, int cpu) +{ + int i; + struct qos_cpu_group *g = r->qcg; + + if (cpu < 0 || cpu > num_possible_cpus()) + return NULL; + + for (i = 0; i < r->num_groups; i++, g++) { + if (cpumask_test_cpu(cpu, &g->mask)) + return &r->qcg[i]; + } + + return NULL; +} + +/* + * Function to put qos vote. This takes qos cpu group of + * host and type of vote as input + */ +static int sdhci_msm_update_qos_constraints(struct qos_cpu_group *qcg, + enum constraint type) +{ + unsigned int vote; + int cpu, err; + struct dev_pm_qos_request *qos_req = qcg->qos_req; + + if (type == QOS_MAX) + vote = S32_MAX; + else + vote = qcg->votes[type]; + + if (qcg->curr_vote == vote) + return 0; + + for_each_cpu(cpu, &qcg->mask) { + err = dev_pm_qos_update_request(qos_req, vote); + if (err < 0) + return err; + ++qos_req; + } + + if (type == QOS_MAX) + qcg->voted = false; + else + qcg->voted = true; + + qcg->curr_vote = vote; + + return 0; +} + +/* Unregister pm qos requests */ +static int remove_group_qos(struct qos_cpu_group *qcg) +{ + int err, cpu; + struct dev_pm_qos_request *qos_req = qcg->qos_req; + + for_each_cpu(cpu, &qcg->mask) { + if (!dev_pm_qos_request_active(qos_req)) { + ++qos_req; + continue; + } + err = dev_pm_qos_remove_request(qos_req); + if (err < 0) + return err; + qos_req++; + } + + return 0; +} + +/* Register pm qos request */ +static int add_group_qos(struct qos_cpu_group *qcg, enum constraint type) +{ + int cpu, err; + struct dev_pm_qos_request *qos_req = qcg->qos_req; + + for_each_cpu(cpu, &qcg->mask) { + memset(qos_req, 0, + sizeof(struct dev_pm_qos_request)); + err = dev_pm_qos_add_request(get_cpu_device(cpu), + qos_req, + DEV_PM_QOS_RESUME_LATENCY, + type); + if (err < 0) + return err; + qos_req++; + } + return 0; +} + +/* Function to remove pm qos vote */ +static void sdhci_msm_unvote_qos_all(struct work_struct *work) +{ + struct sdhci_msm_host *msm_host = container_of(work, + struct sdhci_msm_host, pmqos_unvote_work.work); + struct sdhci_msm_qos_req *qos_req = msm_host->sdhci_qos; + struct qos_cpu_group *qcg; + int i, err; + + if (!qos_req) + return; + qcg = qos_req->qcg; + for (i = 0; ((i < qos_req->num_groups) && qcg->initialized); i++, + qcg++) { + err = sdhci_msm_update_qos_constraints(qcg, QOS_MAX); + if (err) + dev_err(&msm_host->pdev->dev, + "Failed (%d) removing qos vote(%d)\n", err, i); + } +} + +/* Function to vote pmqos from sdcc. */ +static void sdhci_msm_vote_pmqos(struct mmc_host *mmc, int cpu) +{ + struct sdhci_host *host = mmc_priv(mmc); + struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); + struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); + struct qos_cpu_group *qcg; + + qcg = cpu_to_group(msm_host->sdhci_qos, cpu); + if (!qcg) { + dev_dbg(&msm_host->pdev->dev, "QoS group is undefined\n"); + return; + } + + if (qcg->voted) + return; + + if (sdhci_msm_update_qos_constraints(qcg, QOS_PERF)) + dev_err(&qcg->host->pdev->dev, "%s: update qos - failed\n", + __func__); + dev_dbg(&msm_host->pdev->dev, "Voted pmqos - cpu: %d\n", cpu); +} + +/** + * sdhci_msm_irq_affinity_notify - Callback for affinity changes + * @notify: context as to what irq was changed + * @mask: the new affinity mask + * + * This is a callback function used by the irq_set_affinity_notifier function + * so that we may register to receive changes to the irq affinity masks. + */ +static void +sdhci_msm_irq_affinity_notify(struct irq_affinity_notify *notify, + const cpumask_t *mask) +{ + struct sdhci_msm_host *msm_host = + container_of(notify, struct sdhci_msm_host, affinity_notify); + struct platform_device *pdev = msm_host->pdev; + struct sdhci_msm_qos_req *qos_req = msm_host->sdhci_qos; + struct qos_cpu_group *qcg; + int i, err; + + if (!qos_req) + return; + /* + * If device is in suspend mode, just update the active mask, + * vote would be taken care when device resumes. + */ + msm_host->sdhci_qos->active_mask = cpumask_first(mask); + if (pm_runtime_status_suspended(&pdev->dev)) + return; + + /* Cancel previous scheduled work and unvote votes */ + qcg = qos_req->qcg; + for (i = 0; i < qos_req->num_groups; i++, qcg++) { + err = sdhci_msm_update_qos_constraints(qcg, QOS_MAX); + if (err) + pr_err("%s: Failed (%d) removing qos vote of grp(%d)\n", + mmc_hostname(msm_host->mmc), err, i); + } + + cancel_delayed_work_sync(&msm_host->pmqos_unvote_work); + sdhci_msm_vote_pmqos(msm_host->mmc, + msm_host->sdhci_qos->active_mask); +} + +/** + * sdhci_msm_irq_affinity_release - Callback for affinity notifier release + * @ref: internal core kernel usage + * + * This is a callback function used by the irq_set_affinity_notifier function + * to inform the current notification subscriber that they will no longer + * receive notifications. + */ +static void +inline sdhci_msm_irq_affinity_release(struct kref __always_unused *ref) +{ } + +/* Function for settig up qos based on parsed dt entries */ +static int sdhci_msm_setup_qos(struct sdhci_msm_host *msm_host) +{ + struct platform_device *pdev = msm_host->pdev; + struct sdhci_msm_qos_req *qr = msm_host->sdhci_qos; + struct qos_cpu_group *qcg = qr->qcg; + struct mmc_host *mmc = msm_host->mmc; + struct sdhci_host *host = mmc_priv(mmc); + int i, err; + + if (!msm_host->sdhci_qos) + return 0; + + /* Affine irq to first set of mask */ + WARN_ON(irq_set_affinity_hint(host->irq, &qcg->mask)); + + /* Setup notifier for case of affinity change/migration */ + msm_host->affinity_notify.notify = sdhci_msm_irq_affinity_notify; + msm_host->affinity_notify.release = sdhci_msm_irq_affinity_release; + irq_set_affinity_notifier(host->irq, &msm_host->affinity_notify); + + for (i = 0; i < qr->num_groups; i++, qcg++) { + qcg->qos_req = kcalloc(cpumask_weight(&qcg->mask), + sizeof(struct dev_pm_qos_request), + GFP_KERNEL); + if (!qcg->qos_req) { + dev_err(&pdev->dev, "Memory allocation failed\n"); + if (!i) + return -ENOMEM; + goto free_mem; + } + err = add_group_qos(qcg, S32_MAX); + if (err < 0) { + dev_err(&pdev->dev, "Fail (%d) add qos-req: grp-%d\n", + err, i); + if (!i) { + kfree(qcg->qos_req); + return err; + } + goto free_mem; + } + qcg->initialized = true; + dev_dbg(&pdev->dev, "%s: qcg: 0x%08x | mask: 0x%08x\n", + __func__, qcg, qcg->mask); + } + + INIT_DELAYED_WORK(&msm_host->pmqos_unvote_work, + sdhci_msm_unvote_qos_all); + + /* Vote pmqos during setup for first set of mask*/ + sdhci_msm_update_qos_constraints(qr->qcg, QOS_PERF); + qr->active_mask = cpumask_first(&qr->qcg->mask); + return 0; + +free_mem: + while (i--) { + kfree(qcg->qos_req); + qcg--; + } + + return err; +} + +/* + * QoS init function. It parses dt entries and intializes data + * structures. + */ +static void sdhci_msm_qos_init(struct sdhci_msm_host *msm_host) +{ + struct platform_device *pdev = msm_host->pdev; + struct device_node *np = pdev->dev.of_node; + struct device_node *group_node; + struct sdhci_msm_qos_req *qr; + struct qos_cpu_group *qcg; + int i, err, mask = 0; + + qr = kzalloc(sizeof(*qr), GFP_KERNEL); + if (!qr) + return; + + msm_host->sdhci_qos = qr; + + /* find numbers of qos child node present */ + qr->num_groups = of_get_available_child_count(np); + dev_dbg(&pdev->dev, "num-groups: %d\n", qr->num_groups); + if (!qr->num_groups) { + dev_err(&pdev->dev, "QoS groups undefined\n"); + kfree(qr); + msm_host->sdhci_qos = NULL; + return; + } + qcg = kzalloc(sizeof(*qcg) * qr->num_groups, GFP_KERNEL); + if (!qcg) { + msm_host->sdhci_qos = NULL; + kfree(qr); + return; + } + + /* + * Assign qos cpu group/cluster to host qos request and + * read child entries of qos node + */ + qr->qcg = qcg; + for_each_available_child_of_node(np, group_node) { + err = of_property_read_u32(group_node, "mask", &mask); + if (err) { + dev_dbg(&pdev->dev, "Error reading group mask: %d\n", + err); + continue; + } + qcg->mask.bits[0] = mask; + if (!cpumask_subset(&qcg->mask, cpu_possible_mask)) { + dev_err(&pdev->dev, "Invalid group mask\n"); + goto out_vote_err; + } + + err = of_property_count_u32_elems(group_node, "vote"); + if (err <= 0) { + dev_err(&pdev->dev, "1 vote is needed, bailing out\n"); + goto out_vote_err; + } + qcg->votes = kmalloc(sizeof(*qcg->votes) * err, GFP_KERNEL); + if (!qcg->votes) + goto out_vote_err; + for (i = 0; i < err; i++) { + if (of_property_read_u32_index(group_node, "vote", i, + &qcg->votes[i])) + goto out_vote_err; + } + qcg->host = msm_host; + ++qcg; + } + err = sdhci_msm_setup_qos(msm_host); + if (!err) + return; + dev_err(&pdev->dev, "Failed to setup PM QoS.\n"); + +out_vote_err: + for (i = 0, qcg = qr->qcg; i < qr->num_groups; i++, qcg++) + kfree(qcg->votes); + kfree(qr->qcg); + kfree(qr); + msm_host->sdhci_qos = NULL; +} + static int sdhci_msm_probe(struct platform_device *pdev) { struct sdhci_host *host; @@ -3312,6 +3686,14 @@ static int sdhci_msm_probe(struct platform_device *pdev) pm_runtime_use_autosuspend(&pdev->dev); host->mmc_host_ops.execute_tuning = sdhci_msm_execute_tuning; + + msm_host->workq = create_workqueue("sdhci_msm_generic_swq"); + if (!msm_host->workq) + dev_err(&pdev->dev, "Generic swq creation failed\n"); + + /* Initialize pmqos */ + sdhci_msm_qos_init(msm_host); + if (of_property_read_bool(node, "supports-cqe")) ret = sdhci_msm_cqe_add_host(host, pdev); else @@ -3366,6 +3748,9 @@ static int sdhci_msm_remove(struct platform_device *pdev) struct sdhci_host *host = platform_get_drvdata(pdev); struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); + struct sdhci_msm_qos_req *r = msm_host->sdhci_qos; + struct qos_cpu_group *qcg; + int i; int dead = (readl_relaxed(host->ioaddr + SDHCI_INT_STATUS) == 0xffffffff); @@ -3374,6 +3759,19 @@ static int sdhci_msm_remove(struct platform_device *pdev) sdhci_msm_vreg_init(&pdev->dev, msm_host, false); pm_runtime_get_sync(&pdev->dev); + + /* Add delay to complete resume where qos vote is scheduled */ + if (!r) + goto skip_removing_qos; + qcg = r->qcg; + msleep(50); + for (i = 0; i < r->num_groups; i++, qcg++) { + sdhci_msm_update_qos_constraints(qcg, QOS_MAX); + remove_group_qos(qcg); + } + destroy_workqueue(msm_host->workq); + +skip_removing_qos: pm_runtime_disable(&pdev->dev); pm_runtime_put_noidle(&pdev->dev); @@ -3394,7 +3792,15 @@ static __maybe_unused int sdhci_msm_runtime_suspend(struct device *dev) struct sdhci_host *host = dev_get_drvdata(dev); struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); + struct sdhci_msm_qos_req *qos_req = msm_host->sdhci_qos; + if (!qos_req) + goto skip_qos; + queue_delayed_work(msm_host->workq, + &msm_host->pmqos_unvote_work, + msecs_to_jiffies(MSM_PMQOS_UNVOTING_DELAY_MS)); + +skip_qos: sdhci_msm_registers_save(host); clk_bulk_disable_unprepare(ARRAY_SIZE(msm_host->bulk_clks), msm_host->bulk_clks); @@ -3407,10 +3813,11 @@ static __maybe_unused int sdhci_msm_runtime_resume(struct device *dev) struct sdhci_host *host = dev_get_drvdata(dev); struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host); + struct sdhci_msm_qos_req *qos_req = msm_host->sdhci_qos; int ret; ret = clk_bulk_prepare_enable(ARRAY_SIZE(msm_host->bulk_clks), - msm_host->bulk_clks); + msm_host->bulk_clks); if (ret) return ret; @@ -3420,8 +3827,15 @@ static __maybe_unused int sdhci_msm_runtime_resume(struct device *dev) * restore the SDR DLL settings when the clock is ungated. */ if (msm_host->restore_dll_config && msm_host->clk_rate) - return sdhci_msm_restore_sdr_dll_config(host); + sdhci_msm_restore_sdr_dll_config(host); + if (!qos_req) + goto skip_qos_vote; + cancel_delayed_work_sync(&msm_host->pmqos_unvote_work); + sdhci_msm_vote_pmqos(msm_host->mmc, + msm_host->sdhci_qos->active_mask); + +skip_qos_vote: sdhci_msm_bus_voting(host, true); return 0; }