From bf1d7d92f8cb6e77f0df53aec29045aa261fad05 Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Wed, 22 Jul 2020 15:36:35 -0600 Subject: [PATCH] msm: kgsl: Add asynchronous processing of acks This adds the initial bits of handling acks asynchronously. It is assumed that there is only one outstanding ack since we wait for ack while holding the device mutex. GMU handles acks inline until start_msg hfi and beyond that, each hfi will be acked asynchronously. Change-Id: I15e930d22f1154db377599a0c0409640821fb868 Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_gmu.c | 35 ++-- drivers/gpu/msm/adreno_a6xx_gmu.h | 8 - drivers/gpu/msm/adreno_a6xx_hfi.c | 20 +-- drivers/gpu/msm/adreno_a6xx_hfi.h | 27 +-- drivers/gpu/msm/adreno_a6xx_hwsched.c | 113 +++++++++++- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 208 +++++++++++++++++++++- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h | 25 ++- drivers/gpu/msm/kgsl_gmu_core.h | 1 + 8 files changed, 382 insertions(+), 55 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index ce7d9867bac9..dd71faf813ac 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -1688,7 +1688,7 @@ static void a6xx_gmu_pwrctrl_suspend(struct adreno_device *adreno_dev) * a6xx_gmu_notify_slumber() - initiate request to GMU to prepare to slumber * @device: Pointer to KGSL device */ -int a6xx_gmu_notify_slumber(struct adreno_device *adreno_dev) +static int a6xx_gmu_notify_slumber(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; @@ -2639,7 +2639,6 @@ int a6xx_gmu_probe(struct kgsl_device *device, { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); - struct a6xx_hfi *hfi = &gmu->hfi; struct resource *res; int ret; @@ -2706,14 +2705,6 @@ int a6xx_gmu_probe(struct kgsl_device *device, device->gmu_core.dev_ops = &a6xx_gmudev; - /* Initialize HFI and GMU interrupts */ - hfi->irq = kgsl_request_irq(gmu->pdev, "kgsl_hfi_irq", - a6xx_hfi_irq_handler, device); - if (hfi->irq < 0) { - ret = hfi->irq; - goto error; - } - gmu->irq = kgsl_request_irq(gmu->pdev, "kgsl_gmu_irq", a6xx_gmu_irq_handler, device); @@ -2725,7 +2716,6 @@ int a6xx_gmu_probe(struct kgsl_device *device, /* Don't enable GMU interrupts until GMU started */ /* We cannot use irq_disable because it writes registers */ disable_irq(gmu->irq); - disable_irq(gmu->hfi.irq); return 0; @@ -3367,8 +3357,29 @@ int a6xx_gmu_restart(struct kgsl_device *device) static int a6xx_gmu_bind(struct device *dev, struct device *master, void *data) { struct kgsl_device *device = dev_get_drvdata(master); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(ADRENO_DEVICE(device)); + struct a6xx_hfi *hfi = &gmu->hfi; + int ret; - return a6xx_gmu_probe(device, to_platform_device(dev)); + ret = a6xx_gmu_probe(device, to_platform_device(dev)); + if (ret) + return ret; + + /* + * a6xx_gmu_probe() is also called by hwscheduling probe. However, + * since HFI interrupts are handled differently in hwscheduling, move + * out HFI interrupt setup from a6xx_gmu_probe(). + */ + hfi->irq = kgsl_request_irq(gmu->pdev, "kgsl_hfi_irq", + a6xx_hfi_irq_handler, device); + if (hfi->irq < 0) { + a6xx_gmu_remove(device); + return hfi->irq; + } + + disable_irq(gmu->hfi.irq); + + return 0; } static void a6xx_gmu_unbind(struct device *dev, struct device *master, diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.h b/drivers/gpu/msm/adreno_a6xx_gmu.h index 4b40b394826b..4c58f20d798e 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.h +++ b/drivers/gpu/msm/adreno_a6xx_gmu.h @@ -446,14 +446,6 @@ void a6xx_gmu_oob_clear(struct kgsl_device *device, enum oob_request oob); */ int a6xx_gmu_wait_for_lowest_idle(struct adreno_device *adreno_dev); -/** - * a6xx_gmu_notify_slumber - Send NOTIFY_SLUMBER hfi to gmu - * @adreno_dev: Pointer to the adreno device - * - * Return: 0 on success or negative error on failure - */ -int a6xx_gmu_notify_slumber(struct adreno_device *adreno_dev); - /** * a6xx_gmu_wait_for_idle - Wait for gmu to become idle * @adreno_dev: Pointer to the adreno device diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.c b/drivers/gpu/msm/adreno_a6xx_hfi.c index f1f25c314f57..fa25862a4d93 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hfi.c @@ -21,9 +21,6 @@ #define HOST_QUEUE_START_ADDR(hfi_mem, i) \ ((hfi_mem)->hostptr + HFI_QUEUE_OFFSET(i)) -static int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, - uint32_t queue_idx, struct pending_cmd *ret_cmd); - struct a6xx_hfi *to_a6xx_hfi(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -220,9 +217,6 @@ int a6xx_hfi_init(struct adreno_device *adreno_dev) return PTR_ERR_OR_ZERO(hfi->hfi_mem); } -#define HDR_CMP_SEQNUM(out_hdr, in_hdr) \ - (MSG_HDR_GET_SEQNUM(out_hdr) == MSG_HDR_GET_SEQNUM(in_hdr)) - int a6xx_receive_ack_cmd(struct a6xx_gmu_device *gmu, void *rcvd, struct pending_cmd *ret_cmd) { @@ -286,8 +280,8 @@ static int poll_gmu_reg(struct adreno_device *adreno_dev, return -ETIMEDOUT; } -int a6xx_hfi_send_cmd(struct adreno_device *adreno_dev, uint32_t queue_idx, - void *data, struct pending_cmd *ret_cmd) +static int a6xx_hfi_send_cmd_wait_inline(struct adreno_device *adreno_dev, + uint32_t queue_idx, void *data, struct pending_cmd *ret_cmd) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); @@ -336,7 +330,7 @@ int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, memset(&ret_cmd, 0, sizeof(ret_cmd)); - rc = a6xx_hfi_send_cmd(adreno_dev, queue, cmd, &ret_cmd); + rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, queue, cmd, &ret_cmd); if (!rc && ret_cmd.results[2] == HFI_ACK_ERROR) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -378,7 +372,8 @@ static int a6xx_hfi_get_fw_version(struct adreno_device *adreno_dev, memset(&ret_cmd, 0, sizeof(ret_cmd)); - rc = a6xx_hfi_send_cmd(adreno_dev, HFI_CMD_ID, &cmd, &ret_cmd); + rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, HFI_CMD_ID, &cmd, + &ret_cmd); if (rc) return rc; @@ -473,7 +468,8 @@ static int a6xx_hfi_send_get_value(struct adreno_device *adreno_dev, cmd->hdr = CMD_MSG_HDR(H2F_MSG_GET_VALUE, sizeof(*cmd)); - rc = a6xx_hfi_send_cmd(adreno_dev, HFI_CMD_ID, cmd, &ret_cmd); + rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, HFI_CMD_ID, cmd, + &ret_cmd); if (rc) return rc; @@ -535,7 +531,7 @@ static void a6xx_hfi_v1_receiver(struct a6xx_gmu_device *gmu, uint32_t *rcvd, } } -static int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, +int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, uint32_t queue_idx, struct pending_cmd *ret_cmd) { uint32_t rcvd[MAX_RCVD_SIZE]; diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index 929193bffd32..9df7dc3c9f2f 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -156,6 +156,12 @@ struct hfi_queue_table { #define MSG_HDR_GET_TYPE(hdr) (((hdr) >> 16) & 0xF) #define MSG_HDR_GET_SEQNUM(hdr) (((hdr) >> 20) & 0xFFF) +#define MSG_HDR_GET_SIZE(hdr) (((hdr) >> 8) & 0xFF) +#define MSG_HDR_GET_SEQNUM(hdr) (((hdr) >> 20) & 0xFFF) + +#define HDR_CMP_SEQNUM(out_hdr, in_hdr) \ + (MSG_HDR_GET_SEQNUM(out_hdr) == MSG_HDR_GET_SEQNUM(in_hdr)) + #define MSG_HDR_SET_SEQNUM(hdr, num) \ (((hdr) & 0xFFFFF) | ((num) << 20)) @@ -514,12 +520,14 @@ struct hfi_context_bad_reply_cmd { /** * struct pending_cmd - data structure to track outstanding HFI * command messages - * @sent_hdr: copy of outgoing header for response comparison - * @results: the payload of received return message (ACK) */ struct pending_cmd { - uint32_t sent_hdr; - uint32_t results[MAX_RCVD_SIZE]; + /** @sent_hdr: Header of the un-ack'd hfi packet */ + u32 sent_hdr; + /** @results: Array to store the ack packet */ + u32 results[MAX_RCVD_SIZE]; + /** @complete: Completion to signal hfi ack has been received */ + struct completion complete; }; /** @@ -677,14 +685,13 @@ int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, int a6xx_hfi_send_bcl_feature_ctrl(struct adreno_device *adreno_dev); /* - * a6xx_hfi_send_cmd - Send and wait for a hfi packet - * @adreno_dev: Pointer to the adreno device - * @queue_idx: Destination queue id - * @data: Pointer to hfi packet header and data + * a6xx_hfi_process_queue - Check hfi queue for messages from gmu + * @gmu: Pointer to the a6xx gmu device + * @queue_idx: queue id to be processed * @ret_cmd: Container for data needed for waiting for the ack * * Return: 0 on success or negative error on failure */ -int a6xx_hfi_send_cmd(struct adreno_device *adreno_dev, u32 queue_idx, - void *data, struct pending_cmd *ret_cmd); +int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, + u32 queue_idx, struct pending_cmd *ret_cmd); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index b49704337cab..adb03fecc1ff 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -177,6 +177,24 @@ static void a6xx_hwsched_active_count_put(struct adreno_device *adreno_dev) wake_up(&device->active_cnt_wq); } +static int a6xx_hwsched_notify_slumber(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct hfi_prep_slumber_cmd req; + + req.hdr = CMD_MSG_HDR(H2F_MSG_PREPARE_SLUMBER, sizeof(req)); + req.freq = gmu->hfi.dcvs_table.gpu_level_num - + pwr->default_pwrlevel - 1; + req.bw = pwr->pwrlevels[pwr->default_pwrlevel].bus_freq; + + /* Disable the power counter so that the GMU is not busy */ + gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0); + + return a6xx_hfi_send_cmd_async(adreno_dev, &req); + +} static int a6xx_hwsched_gmu_power_off(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); @@ -191,7 +209,7 @@ static int a6xx_hwsched_gmu_power_off(struct adreno_device *adreno_dev) if (ret) goto error; - ret = a6xx_gmu_notify_slumber(adreno_dev); + ret = a6xx_hwsched_notify_slumber(adreno_dev); if (ret) goto error; @@ -549,12 +567,92 @@ static int a6xx_hwsched_active_count_get(struct adreno_device *adreno_dev) return ret; } +static int a6xx_hwsched_dcvs_set(struct adreno_device *adreno_dev, + int gpu_pwrlevel, int bus_level) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct hfi_dcvstable_cmd *table = &gmu->hfi.dcvs_table; + struct hfi_gx_bw_perf_vote_cmd req = { + .hdr = CMD_MSG_HDR(H2F_MSG_GX_BW_PERF_VOTE, sizeof(req)), + .ack_type = DCVS_ACK_BLOCK, + .freq = INVALID_DCVS_IDX, + .bw = INVALID_DCVS_IDX, + }; + int ret = 0; + + if (!test_bit(GMU_PRIV_HFI_STARTED, &gmu->flags)) + return 0; + + /* Do not set to XO and lower GPU clock vote from GMU */ + if ((gpu_pwrlevel != INVALID_DCVS_IDX) && + (gpu_pwrlevel >= table->gpu_level_num - 1)) { + dev_err(&gmu->pdev->dev, "Invalid gpu dcvs request: %d\n", + gpu_pwrlevel); + return -EINVAL; + } + + if (gpu_pwrlevel < table->gpu_level_num - 1) + req.freq = table->gpu_level_num - gpu_pwrlevel - 1; + + if (bus_level < pwr->ddr_table_count && bus_level > 0) + req.bw = bus_level; + + /* GMU will vote for slumber levels through the sleep sequence */ + if ((req.freq == INVALID_DCVS_IDX) && (req.bw == INVALID_DCVS_IDX)) + return 0; + + ret = a6xx_hfi_send_cmd_async(adreno_dev, &req); + + if (ret) + dev_err_ratelimited(&gmu->pdev->dev, + "Failed to set GPU perf idx %d, bw idx %d\n", + req.freq, req.bw); + + return ret; +} + +static int a6xx_hwsched_clock_set(struct adreno_device *adreno_dev, + u32 pwrlevel) +{ + return a6xx_hwsched_dcvs_set(adreno_dev, pwrlevel, INVALID_DCVS_IDX); +} + +static int a6xx_hwsched_bus_set(struct adreno_device *adreno_dev, int buslevel, + u32 ab) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + int ret = 0; + + if (buslevel != pwr->cur_buslevel) { + ret = a6xx_hwsched_dcvs_set(adreno_dev, INVALID_DCVS_IDX, + buslevel); + if (ret) + return ret; + + pwr->cur_buslevel = buslevel; + + trace_kgsl_buslevel(device, pwr->active_pwrlevel, buslevel); + } + + if (ab != pwr->cur_ab) { + icc_set_bw(pwr->icc_path, MBps_to_icc(ab), 0); + pwr->cur_ab = ab; + } + + return ret; +} + const struct adreno_power_ops a6xx_hwsched_power_ops = { .first_open = a6xx_hwsched_first_open, .last_close = a6xx_hwsched_power_off, .active_count_get = a6xx_hwsched_active_count_get, .active_count_put = a6xx_hwsched_active_count_put, .touch_wakeup = a6xx_hwsched_touch_wakeup, + .gpu_clock_set = a6xx_hwsched_clock_set, + .gpu_bus_set = a6xx_hwsched_bus_set, }; int a6xx_hwsched_probe(struct platform_device *pdev, @@ -589,8 +687,19 @@ static int a6xx_hwsched_bind(struct device *dev, struct device *master, void *data) { struct kgsl_device *device = dev_get_drvdata(master); + int ret; - return a6xx_gmu_probe(device, to_platform_device(dev)); + ret = a6xx_gmu_probe(device, to_platform_device(dev)); + if (ret) + goto error; + + ret = a6xx_hwsched_hfi_probe(ADRENO_DEVICE(device)); + +error: + if (ret) + a6xx_gmu_remove(device); + + return ret; } static void a6xx_hwsched_unbind(struct device *dev, struct device *master, diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index 8fdda837f075..f838c8ab46de 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -61,6 +61,156 @@ static struct a6xx_hwsched_hfi *to_a6xx_hwsched_hfi( return &a6xx_hwsched->hwsched_hfi; } +static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd, + struct pending_cmd *ret_cmd) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + u32 *ack = rcvd; + u32 hdr = ack[0]; + u32 req_hdr = ack[1]; + u32 size_bytes = MSG_HDR_GET_SIZE(hdr) << 2; + + trace_kgsl_hfi_receive(MSG_HDR_GET_ID(req_hdr), + MSG_HDR_GET_SIZE(req_hdr), MSG_HDR_GET_SEQNUM(req_hdr)); + + if (size_bytes > sizeof(ret_cmd->results)) + dev_err(&gmu->pdev->dev, + "Ack result too big: %d Truncating to: %d\n", + size_bytes, sizeof(ret_cmd->results)); + + if (HDR_CMP_SEQNUM(ret_cmd->sent_hdr, req_hdr)) { + memcpy(ret_cmd->results, ack, + min_t(u32, size_bytes, sizeof(ret_cmd->results))); + complete(&ret_cmd->complete); + return; + } + + /* Didn't find the sender, list the waiter */ + dev_err_ratelimited(&gmu->pdev->dev, + "Unexpectedly got id %d seqnum %d while waiting for id %d seqnum %d\n", + MSG_HDR_GET_ID(req_hdr), MSG_HDR_GET_SEQNUM(req_hdr), + MSG_HDR_GET_ID(ret_cmd->sent_hdr), + MSG_HDR_GET_SEQNUM(ret_cmd->sent_hdr)); +} + +static void process_msgq_irq(struct adreno_device *adreno_dev) +{ + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + u32 rcvd[MAX_RCVD_SIZE]; + + if (a6xx_hfi_queue_read(to_a6xx_gmu(adreno_dev), + HFI_MSG_ID, rcvd, sizeof(rcvd)) <= 0) + return; + + /* + * We are assuming that there is only one outstanding ack + * because hfi sending thread waits for completion while + * holding the device mutex + */ + if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) + a6xx_receive_ack_async(adreno_dev, rcvd, &hfi->pending_ack); +} + +/* HFI interrupt handler */ +static irqreturn_t a6xx_hwsched_hfi_handler(int irq, void *data) +{ + struct adreno_device *adreno_dev = data; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 status = 0; + + gmu_core_regread(device, A6XX_GMU_GMU2HOST_INTR_INFO, &status); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, hfi->irq_mask); + + if (status & HFI_IRQ_MSGQ_MASK) + process_msgq_irq(adreno_dev); + if (status & HFI_IRQ_DBGQ_MASK) + a6xx_hfi_process_queue(gmu, HFI_DBG_ID, NULL); + if (status & HFI_IRQ_CM3_FAULT_MASK) { + atomic_set(&gmu->cm3_fault, 1); + + /* make sure other CPUs see the update */ + smp_wmb(); + + dev_err_ratelimited(&gmu->pdev->dev, + "GMU CM3 fault interrupt received\n"); + } + + /* Ignore OOB bits */ + status &= GENMASK(31, 31 - (oob_max - 1)); + + if (status & ~hfi->irq_mask) + dev_err_ratelimited(&gmu->pdev->dev, + "Unhandled HFI interrupts 0x%lx\n", + status & ~hfi->irq_mask); + + return IRQ_HANDLED; +} + +static int wait_ack_completion(struct adreno_device *adreno_dev, u32 *cmd) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + int rc; + + rc = wait_for_completion_timeout(&hfi->pending_ack.complete, + HFI_RSP_TIMEOUT); + if (!rc) { + dev_err(&gmu->pdev->dev, + "Ack timeout for id:%d sequence=%d\n", + MSG_HDR_GET_ID(*cmd), + MSG_HDR_GET_SEQNUM(*cmd)); + gmu_fault_snapshot(KGSL_DEVICE(adreno_dev)); + return -ETIMEDOUT; + } + + return 0; +} + +static int check_ack_failure(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct pending_cmd *cmd = &hfi->pending_ack; + int rc = cmd->results[2] ? -EINVAL : 0; + + if (cmd->results[2] == 0xffffffff) + dev_err(&gmu->pdev->dev, + "HFI ACK failure: Req 0x%8.8x\n", + cmd->results[1]); + + /* reset the ack */ + memset(cmd->results, 0x0, sizeof(cmd->results)); + reinit_completion(&hfi->pending_ack.complete); + cmd->sent_hdr = 0; + + return rc; +} + +int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + u32 *cmd = data; + u32 seqnum = atomic_inc_return(&gmu->hfi.seqnum); + int rc; + + *cmd = MSG_HDR_SET_SEQNUM(*cmd, seqnum); + + hfi->pending_ack.sent_hdr = cmd[0]; + + rc = a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, cmd); + if (rc) + return rc; + + rc = wait_ack_completion(adreno_dev, cmd); + if (rc) + return rc; + + return check_ack_failure(adreno_dev); +} + static void init_queues(struct a6xx_hfi *hfi) { u32 gmuaddr = hfi->hfi_mem->gmuaddr; @@ -183,11 +333,11 @@ static int gmu_import_buffer(struct adreno_device *adreno_dev, static struct mem_alloc_entry *lookup_mem_alloc_table( struct adreno_device *adreno_dev, struct hfi_mem_alloc_desc *desc) { - struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct a6xx_hwsched_hfi *hw_hfi = to_a6xx_hwsched_hfi(adreno_dev); int i; - for (i = 0; i < hfi->mem_alloc_entries; i++) { - struct mem_alloc_entry *entry = &hfi->mem_alloc_table[i]; + for (i = 0; i < hw_hfi->mem_alloc_entries; i++) { + struct mem_alloc_entry *entry = &hw_hfi->mem_alloc_table[i]; if ((entry->desc.mem_kind == desc->mem_kind) && (entry->desc.gmu_mem_handle == desc->gmu_mem_handle) && @@ -305,8 +455,8 @@ static int send_start_msg(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); unsigned int seqnum = atomic_inc_return(&gmu->hfi.seqnum); - struct pending_cmd ret_cmd = {0}; int rc = 0; struct hfi_start_cmd cmd; u32 rcvd[MAX_RCVD_SIZE]; @@ -314,7 +464,7 @@ static int send_start_msg(struct adreno_device *adreno_dev) cmd.hdr = CMD_MSG_HDR(H2F_MSG_START, sizeof(cmd)); cmd.hdr = MSG_HDR_SET_SEQNUM(cmd.hdr, seqnum); - ret_cmd.sent_hdr = cmd.hdr; + hfi->pending_ack.sent_hdr = cmd.hdr; rc = a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, (u32 *)&cmd); if (rc) @@ -342,8 +492,13 @@ poll: return -EINVAL; } - if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) - return a6xx_receive_ack_cmd(gmu, rcvd, &ret_cmd); + if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) { + rc = a6xx_receive_ack_cmd(gmu, rcvd, &hfi->pending_ack); + if (rc) + return rc; + + return check_ack_failure(adreno_dev); + } if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_MEM_ALLOC) { rc = mem_alloc_reply(adreno_dev, rcvd); @@ -357,6 +512,7 @@ poll: "MSG_START: unexpected response id:%d, type:%d\n", MSG_HDR_GET_ID(rcvd[0]), MSG_HDR_GET_TYPE(rcvd[0])); + gmu_fault_snapshot(device); return rc; @@ -389,6 +545,9 @@ static void reset_hfi_queues(struct adreno_device *adreno_dev) void a6xx_hwsched_hfi_stop(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + + hfi->irq_mask &= ~HFI_IRQ_MSGQ_MASK; reset_hfi_queues(adreno_dev); @@ -398,6 +557,18 @@ void a6xx_hwsched_hfi_stop(struct adreno_device *adreno_dev) } +static void enable_async_hfi(struct adreno_device *adreno_dev) +{ + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + + hfi->irq_mask |= HFI_IRQ_MSGQ_MASK; + + gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_GMU2HOST_INTR_MASK, + (u32)~hfi->irq_mask); + + init_completion(&hfi->pending_ack.complete); +} + int a6xx_hwsched_hfi_start(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -442,6 +613,8 @@ int a6xx_hwsched_hfi_start(struct adreno_device *adreno_dev) if (ret) goto err; + enable_async_hfi(adreno_dev); + set_bit(GMU_PRIV_HFI_STARTED, &gmu->flags); /* Request default DCVS level */ @@ -462,10 +635,9 @@ err: static int submit_raw_cmds(struct adreno_device *adreno_dev, void *cmds, const char *str) { - struct pending_cmd ret_cmd = {0}; int ret; - ret = a6xx_hfi_send_cmd(adreno_dev, HFI_CMD_ID, cmds, &ret_cmd); + ret = a6xx_hfi_send_cmd_async(adreno_dev, cmds); if (ret) return ret; @@ -524,3 +696,21 @@ int a6xx_hwsched_cp_init(struct adreno_device *adreno_dev) return ret; } + +int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hw_hfi = to_a6xx_hwsched_hfi(adreno_dev); + + gmu->hfi.irq = kgsl_request_irq(gmu->pdev, "kgsl_hfi_irq", + a6xx_hwsched_hfi_handler, adreno_dev); + + if (gmu->hfi.irq < 0) + return gmu->hfi.irq; + + hw_hfi->irq_mask = HFI_IRQ_MASK; + + disable_irq(gmu->hfi.irq); + + return 0; +} diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index 9bbf9a520fbc..d1c6907d6760 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -107,10 +107,19 @@ struct mem_alloc_entry { struct a6xx_hwsched_hfi { struct mem_alloc_entry mem_alloc_table[32]; u32 mem_alloc_entries; + /** @pending_ack: To track un-ack'd hfi packet */ + struct pending_cmd pending_ack; + /** @irq_mask: Store the hfi interrupt mask */ + u32 irq_mask; }; -/* Interrupt handler for hfi interrupts */ -irqreturn_t a6xx_hfi_handler(int irq, void *data); +/** + * a6xx_hwsched_hfi_probe - Probe hwsched hfi resources + * @adreno_dev: Pointer to adreno device structure + * + * Return: 0 on success and negative error on failure. + */ +int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev); /** * a6xx_hwsched_hfi_init - Initialize hfi resources @@ -151,4 +160,16 @@ void a6xx_hwsched_hfi_stop(struct adreno_device *adreno_dev); * Return: 0 on success and negative error on failure. */ int a6xx_hwsched_cp_init(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_send_cmd_async - Send an hfi packet + * @adreno_dev: Pointer to adreno device structure + * @data: Data to be sent in the hfi packet + * + * Send data in the form of an HFI packet to gmu and wait for + * it's ack asynchronously + * + * Return: 0 on success and negative error on failure. + */ +int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data); #endif diff --git a/drivers/gpu/msm/kgsl_gmu_core.h b/drivers/gpu/msm/kgsl_gmu_core.h index 91d53fa77ea0..8ed31effca3c 100644 --- a/drivers/gpu/msm/kgsl_gmu_core.h +++ b/drivers/gpu/msm/kgsl_gmu_core.h @@ -54,6 +54,7 @@ enum oob_request { oob_perfcntr = 1, oob_boot_slumber = 6, /* reserved special case */ oob_dcvs = 7, /* reserved special case */ + oob_max, }; enum gmu_pwrctrl_mode {