From 019b8504baf50b973cbcf7651b1e4b07e09951ed Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Thu, 23 Jul 2020 09:37:41 -0600 Subject: [PATCH 01/14] msm: kgsl: Add support for multiple ack waiters It is possible that there may be multiple host to firmware messages waiting to be ACKed. So add support for putting them in a list and traversing the list when an ack is received. The primary use case for this feature is the context unregister hfi of which there can be many, and they may get ACKed out of order. Change-Id: I56087dab0814daf7e8ba720db0105e4762614b5e Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_hfi.h | 2 + drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 133 ++++++++++++++-------- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h | 7 +- 3 files changed, 92 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index a2371b1d7fc6..17a6ab432339 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -539,6 +539,8 @@ struct pending_cmd { u32 results[MAX_RCVD_SIZE]; /** @complete: Completion to signal hfi ack has been received */ struct completion complete; + /** @node: to add it to the list of hfi packets waiting for ack */ + struct list_head node; }; /** diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index e25daccd49cb..7c1da23afdd9 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -61,10 +61,32 @@ static struct a6xx_hwsched_hfi *to_a6xx_hwsched_hfi( return &a6xx_hwsched->hwsched_hfi; } -static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd, - struct pending_cmd *ret_cmd) +static void add_waiter(struct a6xx_hwsched_hfi *hfi, u32 hdr, + struct pending_cmd *ack) +{ + memset(ack, 0x0, sizeof(*ack)); + + init_completion(&ack->complete); + write_lock_irq(&hfi->msglock); + list_add_tail(&ack->node, &hfi->msglist); + write_unlock_irq(&hfi->msglock); + + ack->sent_hdr = hdr; +} + +static void del_waiter(struct a6xx_hwsched_hfi *hfi, struct pending_cmd *ack) +{ + write_lock_irq(&hfi->msglock); + list_del(&ack->node); + write_unlock_irq(&hfi->msglock); +} + +static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct pending_cmd *cmd = NULL; + u32 waiters[64], num_waiters = 0, i; u32 *ack = rcvd; u32 hdr = ack[0]; u32 req_hdr = ack[1]; @@ -73,35 +95,47 @@ static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd, trace_kgsl_hfi_receive(MSG_HDR_GET_ID(req_hdr), MSG_HDR_GET_SIZE(req_hdr), MSG_HDR_GET_SEQNUM(req_hdr)); - if (size_bytes > sizeof(ret_cmd->results)) - dev_err(&gmu->pdev->dev, + if (size_bytes > sizeof(cmd->results)) + dev_err_ratelimited(&gmu->pdev->dev, "Ack result too big: %d Truncating to: %d\n", - size_bytes, sizeof(ret_cmd->results)); + size_bytes, sizeof(cmd->results)); - if (HDR_CMP_SEQNUM(ret_cmd->sent_hdr, req_hdr)) { - memcpy(ret_cmd->results, ack, - min_t(u32, size_bytes, sizeof(ret_cmd->results))); - complete(&ret_cmd->complete); - return; + read_lock(&hfi->msglock); + + list_for_each_entry(cmd, &hfi->msglist, node) { + if (HDR_CMP_SEQNUM(cmd->sent_hdr, req_hdr)) { + memcpy(cmd->results, ack, + min_t(u32, size_bytes, + sizeof(cmd->results))); + complete(&cmd->complete); + read_unlock(&hfi->msglock); + return; + } + + if (num_waiters < ARRAY_SIZE(waiters)) + waiters[num_waiters++] = cmd->sent_hdr; } + read_unlock(&hfi->msglock); + /* Didn't find the sender, list the waiter */ dev_err_ratelimited(&gmu->pdev->dev, - "Unexpectedly got id %d seqnum %d while waiting for id %d seqnum %d\n", + "Unexpectedly got id %d seqnum %d. Total waiters: %d Top %d Waiters:\n", MSG_HDR_GET_ID(req_hdr), MSG_HDR_GET_SEQNUM(req_hdr), - MSG_HDR_GET_ID(ret_cmd->sent_hdr), - MSG_HDR_GET_SEQNUM(ret_cmd->sent_hdr)); + num_waiters, min_t(u32, num_waiters, 5)); + + for (i = 0; i < num_waiters && i < 5; i++) + dev_err_ratelimited(&gmu->pdev->dev, + " id %d seqnum %d\n", + MSG_HDR_GET_ID(waiters[i]), + MSG_HDR_GET_SEQNUM(waiters[i])); } static void process_msgq_irq(struct adreno_device *adreno_dev) { - struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); u32 rcvd[MAX_RCVD_SIZE]; - if (a6xx_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd)) <= 0) - return; - while (a6xx_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd)) > 0) { /* @@ -110,8 +144,7 @@ static void process_msgq_irq(struct adreno_device *adreno_dev) * holding the device mutex */ if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) - a6xx_receive_ack_async(adreno_dev, rcvd, - &hfi->pending_ack); + a6xx_receive_ack_async(adreno_dev, rcvd); } } @@ -155,19 +188,19 @@ static irqreturn_t a6xx_hwsched_hfi_handler(int irq, void *data) #define HFI_IRQ_MSGQ_MASK BIT(0) #define HFI_RSP_TIMEOUT 100 /* msec */ -static int wait_ack_completion(struct adreno_device *adreno_dev, u32 *cmd) +static int wait_ack_completion(struct adreno_device *adreno_dev, + struct pending_cmd *ack) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); - struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); int rc; - rc = wait_for_completion_timeout(&hfi->pending_ack.complete, + rc = wait_for_completion_timeout(&ack->complete, HFI_RSP_TIMEOUT); if (!rc) { dev_err(&gmu->pdev->dev, "Ack timeout for id:%d sequence=%d\n", - MSG_HDR_GET_ID(*cmd), - MSG_HDR_GET_SEQNUM(*cmd)); + MSG_HDR_GET_ID(ack->sent_hdr), + MSG_HDR_GET_SEQNUM(ack->sent_hdr)); gmu_fault_snapshot(KGSL_DEVICE(adreno_dev)); return -ETIMEDOUT; } @@ -175,24 +208,20 @@ static int wait_ack_completion(struct adreno_device *adreno_dev, u32 *cmd) return 0; } -static int check_ack_failure(struct adreno_device *adreno_dev) +static int check_ack_failure(struct adreno_device *adreno_dev, + struct pending_cmd *ack) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); - struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); - struct pending_cmd *cmd = &hfi->pending_ack; - int rc = cmd->results[2] ? -EINVAL : 0; - if (cmd->results[2] == 0xffffffff) - dev_err(&gmu->pdev->dev, - "HFI ACK failure: Req 0x%8.8x\n", - cmd->results[1]); + if (ack->results[2] != 0xffffffff) + return 0; - /* reset the ack */ - memset(cmd->results, 0x0, sizeof(cmd->results)); - reinit_completion(&hfi->pending_ack.complete); - cmd->sent_hdr = 0; + dev_err(&gmu->pdev->dev, + "ACK error: sender id %d seqnum %d\n", + MSG_HDR_GET_ID(ack->sent_hdr), + MSG_HDR_GET_SEQNUM(ack->sent_hdr)); - return rc; + return -EINVAL; } int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data) @@ -202,20 +231,26 @@ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data) u32 *cmd = data; u32 seqnum = atomic_inc_return(&gmu->hfi.seqnum); int rc; + struct pending_cmd pending_ack; *cmd = MSG_HDR_SET_SEQNUM(*cmd, seqnum); - hfi->pending_ack.sent_hdr = cmd[0]; + add_waiter(hfi, *cmd, &pending_ack); rc = a6xx_hfi_cmdq_write(adreno_dev, cmd); if (rc) - return rc; + goto done; - rc = wait_ack_completion(adreno_dev, cmd); + rc = wait_ack_completion(adreno_dev, &pending_ack); if (rc) - return rc; + goto done; - return check_ack_failure(adreno_dev); + rc = check_ack_failure(adreno_dev, &pending_ack); + +done: + del_waiter(hfi, &pending_ack); + + return rc; } static void init_queues(struct a6xx_hfi *hfi) @@ -462,16 +497,16 @@ static int send_start_msg(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); unsigned int seqnum = atomic_inc_return(&gmu->hfi.seqnum); int rc = 0; struct hfi_start_cmd cmd; u32 rcvd[MAX_RCVD_SIZE]; + struct pending_cmd pending_ack = {0}; cmd.hdr = CMD_MSG_HDR(H2F_MSG_START, sizeof(cmd)); cmd.hdr = MSG_HDR_SET_SEQNUM(cmd.hdr, seqnum); - hfi->pending_ack.sent_hdr = cmd.hdr; + pending_ack.sent_hdr = cmd.hdr; rc = a6xx_hfi_cmdq_write(adreno_dev, (u32 *)&cmd); if (rc) @@ -500,11 +535,11 @@ poll: } if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) { - rc = a6xx_receive_ack_cmd(gmu, rcvd, &hfi->pending_ack); + rc = a6xx_receive_ack_cmd(gmu, rcvd, &pending_ack); if (rc) return rc; - return check_ack_failure(adreno_dev); + return check_ack_failure(adreno_dev, &pending_ack); } if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_MEM_ALLOC) { @@ -572,8 +607,6 @@ static void enable_async_hfi(struct adreno_device *adreno_dev) gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_GMU2HOST_INTR_MASK, (u32)~hfi->irq_mask); - - init_completion(&hfi->pending_ack.complete); } int a6xx_hwsched_hfi_start(struct adreno_device *adreno_dev) @@ -717,6 +750,10 @@ int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev) disable_irq(gmu->hfi.irq); + rwlock_init(&hw_hfi->msglock); + + INIT_LIST_HEAD(&hw_hfi->msglist); + return 0; } diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index d314a0bd9961..b679dc1af8cf 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -107,10 +107,13 @@ struct mem_alloc_entry { struct a6xx_hwsched_hfi { struct mem_alloc_entry mem_alloc_table[32]; u32 mem_alloc_entries; - /** @pending_ack: To track un-ack'd hfi packet */ - struct pending_cmd pending_ack; /** @irq_mask: Store the hfi interrupt mask */ u32 irq_mask; + /** @msglock: To protect the list of un-ACKed hfi packets */ + rwlock_t msglock; + /** @msglist: List of un-ACKed hfi packets */ + struct list_head msglist; + }; struct kgsl_drawobj_cmd; From a8002ec26303cfc88d51bc939f120ae6aa3c71aa Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Thu, 23 Jul 2020 12:46:05 -0600 Subject: [PATCH 02/14] msm: kgsl: Add context unregister HFI This is a special sequence because we don't want to wait for the ack back while holding the device mutex. This means, we have to define a new sequence of unlocking the device mutex before we begin our wait for ack. Change-Id: I3ea27bdccd66cdaafd8cff6cc1366848c1a4bea6 Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno.h | 2 +- drivers/gpu/msm/adreno_a6xx.c | 7 +- drivers/gpu/msm/adreno_a6xx_hwsched.c | 27 ++++- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 81 ++++++++++++++ drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h | 9 ++ drivers/gpu/msm/adreno_a6xx_preempt.c | 14 --- drivers/gpu/msm/adreno_drawctxt.c | 124 ++++++++++++---------- drivers/gpu/msm/kgsl_gmu_core.h | 1 + 8 files changed, 187 insertions(+), 78 deletions(-) diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 6d6b2efc6d13..9a97d1627b67 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -806,7 +806,7 @@ struct adreno_gpudev { int (*preemption_init)(struct adreno_device *adreno_dev); void (*preemption_schedule)(struct adreno_device *adreno_dev); int (*preemption_context_init)(struct kgsl_context *context); - void (*preemption_context_destroy)(struct kgsl_context *context); + void (*context_detach)(struct adreno_context *drawctxt); void (*clk_set_options)(struct adreno_device *adreno_dev, const char *name, struct clk *clk, bool on); void (*pre_reset)(struct adreno_device *adreno_dev); diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c index e533930aba74..fb9cda3c9713 100644 --- a/drivers/gpu/msm/adreno_a6xx.c +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -2801,7 +2801,6 @@ struct adreno_gpudev adreno_a6xx_gpudev = { .preemption_schedule = a6xx_preemption_schedule, .set_marker = a6xx_set_marker, .preemption_context_init = a6xx_preemption_context_init, - .preemption_context_destroy = a6xx_preemption_context_destroy, .sptprac_is_on = a6xx_sptprac_is_on, .ccu_invalidate = a6xx_ccu_invalidate, .perfcounter_update = a6xx_perfcounter_update, @@ -2824,7 +2823,7 @@ struct adreno_gpudev adreno_a6xx_hwsched_gpudev = { .read_throttling_counters = a6xx_read_throttling_counters, .iommu_fault_block = a6xx_iommu_fault_block, .preemption_context_init = a6xx_preemption_context_init, - .preemption_context_destroy = a6xx_preemption_context_destroy, + .context_detach = a6xx_hwsched_context_detach, .perfcounter_update = a6xx_perfcounter_update, #ifdef CONFIG_QCOM_KGSL_CORESIGHT .coresight = {&a6xx_coresight, &a6xx_coresight_cx}, @@ -2856,7 +2855,6 @@ struct adreno_gpudev adreno_a6xx_gmu_gpudev = { .preemption_schedule = a6xx_preemption_schedule, .set_marker = a6xx_set_marker, .preemption_context_init = a6xx_preemption_context_init, - .preemption_context_destroy = a6xx_preemption_context_destroy, .sptprac_is_on = a6xx_sptprac_is_on, .ccu_invalidate = a6xx_ccu_invalidate, .perfcounter_update = a6xx_perfcounter_update, @@ -2890,7 +2888,6 @@ struct adreno_gpudev adreno_a6xx_rgmu_gpudev = { .preemption_schedule = a6xx_preemption_schedule, .set_marker = a6xx_set_marker, .preemption_context_init = a6xx_preemption_context_init, - .preemption_context_destroy = a6xx_preemption_context_destroy, .sptprac_is_on = a6xx_sptprac_is_on, .ccu_invalidate = a6xx_ccu_invalidate, .perfcounter_update = a6xx_perfcounter_update, @@ -2924,7 +2921,6 @@ struct adreno_gpudev adreno_a619_holi_gpudev = { .preemption_schedule = a6xx_preemption_schedule, .set_marker = a6xx_set_marker, .preemption_context_init = a6xx_preemption_context_init, - .preemption_context_destroy = a6xx_preemption_context_destroy, .sptprac_is_on = a6xx_sptprac_is_on, .ccu_invalidate = a6xx_ccu_invalidate, .perfcounter_update = a6xx_perfcounter_update, @@ -2961,7 +2957,6 @@ struct adreno_gpudev adreno_a630_gpudev = { .preemption_schedule = a6xx_preemption_schedule, .set_marker = a6xx_set_marker, .preemption_context_init = a6xx_preemption_context_init, - .preemption_context_destroy = a6xx_preemption_context_destroy, .sptprac_is_on = a6xx_sptprac_is_on, .ccu_invalidate = a6xx_ccu_invalidate, .perfcounter_update = a6xx_perfcounter_update, diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index adb03fecc1ff..a6c9f5b85fbb 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -177,6 +177,21 @@ static void a6xx_hwsched_active_count_put(struct adreno_device *adreno_dev) wake_up(&device->active_cnt_wq); } +static int unregister_context_hwsched(int id, void *ptr, void *data) +{ + struct kgsl_context *context = ptr; + + /* + * We don't need to send the unregister hfi packet because + * we are anyway going to lose the gmu state of registered + * contexts. So just reset the flag so that the context + * registers with gmu on its first submission post slumber. + */ + context->gmu_registered = false; + + return 0; +} + static int a6xx_hwsched_notify_slumber(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); @@ -477,6 +492,10 @@ no_gx_power: a6xx_hwsched_gmu_power_off(adreno_dev); + read_lock(&device->context_lock); + idr_for_each(&device->context_idr, unregister_context_hwsched, NULL); + read_unlock(&device->context_lock); + if (!IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice)) llcc_slice_deactivate(adreno_dev->gpu_llc_slice); @@ -695,9 +714,13 @@ static int a6xx_hwsched_bind(struct device *dev, struct device *master, ret = a6xx_hwsched_hfi_probe(ADRENO_DEVICE(device)); + if (!ret) { + set_bit(GMU_DISPATCH, &device->gmu_core.flags); + return 0; + } + error: - if (ret) - a6xx_gmu_remove(device); + a6xx_gmu_remove(device); return ret; } diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index 7c1da23afdd9..a56e3d9c6eb1 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -911,3 +911,84 @@ free: return ret; } + +static int send_context_unregister_hfi(struct adreno_device *adreno_dev, + u32 ctxt_id, u32 ts) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct pending_cmd pending_ack; + struct hfi_unregister_ctxt_cmd cmd; + u32 seqnum; + int rc; + + cmd.hdr = CMD_MSG_HDR(H2F_MSG_UNREGISTER_CONTEXT, sizeof(cmd)); + cmd.ctxt_id = ctxt_id, + cmd.ts = ts, + + seqnum = atomic_inc_return(&gmu->hfi.seqnum); + cmd.hdr = MSG_HDR_SET_SEQNUM(cmd.hdr, seqnum); + + add_waiter(hfi, cmd.hdr, &pending_ack); + + rc = a6xx_hfi_cmdq_write(adreno_dev, (u32 *)&cmd); + if (rc) + goto done; + + mutex_unlock(&device->mutex); + + rc = wait_for_completion_timeout(&pending_ack.complete, + msecs_to_jiffies(30 * 1000)); + if (!rc) { + dev_err(&gmu->pdev->dev, + "Ack timeout for context unregister seq: %d ctx: %d ts: %d\n", + MSG_HDR_GET_SEQNUM(pending_ack.sent_hdr), + ctxt_id, ts); + rc = -ETIMEDOUT; + mutex_lock(&device->mutex); + gmu_fault_snapshot(device); + goto done; + } + + mutex_lock(&device->mutex); + + rc = check_ack_failure(adreno_dev, &pending_ack); +done: + del_waiter(hfi, &pending_ack); + + return rc; +} + +void a6xx_hwsched_context_detach(struct adreno_context *drawctxt) +{ + struct kgsl_context *context = &drawctxt->base; + struct kgsl_device *device = context->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret = 0; + + mutex_lock(&device->mutex); + + /* Only send HFI if device is not in SLUMBER */ + if (context->gmu_registered && + test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) + ret = send_context_unregister_hfi(adreno_dev, context->id, + drawctxt->internal_timestamp); + + if (!ret) { + kgsl_sharedmem_writel(device->memstore, + KGSL_MEMSTORE_OFFSET(context->id, soptimestamp), + drawctxt->timestamp); + + kgsl_sharedmem_writel(device->memstore, + KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp), + drawctxt->timestamp); + + adreno_profile_process_results(adreno_dev); + } + + context->gmu_registered = false; + + mutex_unlock(&device->mutex); +} diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index b679dc1af8cf..a82c0a68ad14 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -191,4 +191,13 @@ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data); */ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags, struct kgsl_drawobj_cmd *cmdobj); + +/** + * a6xx_hwsched_context_detach - Unregister a context with GMU + * @drawctxt: Pointer to the adreno context + * + * This function sends context unregister HFI and waits for the ack + * to ensure all submissions from this context have retired + */ +void a6xx_hwsched_context_detach(struct adreno_context *drawctxt); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_preempt.c b/drivers/gpu/msm/adreno_a6xx_preempt.c index 33236c8bfa05..d6868274dbf9 100644 --- a/drivers/gpu/msm/adreno_a6xx_preempt.c +++ b/drivers/gpu/msm/adreno_a6xx_preempt.c @@ -714,20 +714,6 @@ int a6xx_preemption_init(struct adreno_device *adreno_dev) return 0; } -void a6xx_preemption_context_destroy(struct kgsl_context *context) -{ - struct kgsl_device *device = context->device; - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - - if (!adreno_is_preemption_enabled(adreno_dev)) - return; - - gpumem_free_entry(context->user_ctxt_record); - - /* Put the extra ref from gpumem_alloc_entry() */ - kgsl_mem_entry_put(context->user_ctxt_record); -} - int a6xx_preemption_context_init(struct kgsl_context *context) { struct kgsl_device *device = context->device; diff --git a/drivers/gpu/msm/adreno_drawctxt.c b/drivers/gpu/msm/adreno_drawctxt.c index b0b4cd26cb8b..622593f2c37a 100644 --- a/drivers/gpu/msm/adreno_drawctxt.c +++ b/drivers/gpu/msm/adreno_drawctxt.c @@ -381,9 +381,6 @@ adreno_drawctxt_create(struct kgsl_device_private *dev_priv, (drawctxt->base.flags & KGSL_CONTEXT_PRIORITY_MASK) >> KGSL_CONTEXT_PRIORITY_SHIFT; - /* set the context ringbuffer */ - drawctxt->rb = adreno_ctx_get_rb(adreno_dev, drawctxt); - /* * Now initialize the common part of the context. This allocates the * context id, and then possibly another thread could look it up. @@ -405,8 +402,6 @@ adreno_drawctxt_create(struct kgsl_device_private *dev_priv, adreno_context_debugfs_init(ADRENO_DEVICE(device), drawctxt); - INIT_LIST_HEAD(&drawctxt->active_node); - if (gpudev->preemption_context_init) { ret = gpudev->preemption_context_init(&drawctxt->base); if (ret != 0) { @@ -417,57 +412,23 @@ adreno_drawctxt_create(struct kgsl_device_private *dev_priv, /* copy back whatever flags we dediced were valid */ *flags = drawctxt->base.flags; + + if (!test_bit(GMU_DISPATCH, &device->gmu_core.flags)) { + /* set the context ringbuffer */ + drawctxt->rb = adreno_ctx_get_rb(adreno_dev, drawctxt); + + INIT_LIST_HEAD(&drawctxt->active_node); + } + return &drawctxt->base; } -/** - * adreno_drawctxt_detach(): detach a context from the GPU - * @context: Generic KGSL context container for the context - * - */ -void adreno_drawctxt_detach(struct kgsl_context *context) +static void wait_for_timestamp_rb(struct kgsl_device *device, + struct adreno_context *drawctxt) { - struct kgsl_device *device; - struct adreno_device *adreno_dev; - struct adreno_gpudev *gpudev; - struct adreno_context *drawctxt; - struct adreno_ringbuffer *rb; - int ret, count, i; - struct kgsl_drawobj *list[ADRENO_CONTEXT_DRAWQUEUE_SIZE]; - - if (context == NULL) - return; - - device = context->device; - adreno_dev = ADRENO_DEVICE(device); - gpudev = ADRENO_GPU_DEVICE(adreno_dev); - drawctxt = ADRENO_CONTEXT(context); - rb = drawctxt->rb; - - spin_lock(&drawctxt->lock); - - spin_lock(&adreno_dev->active_list_lock); - list_del_init(&drawctxt->active_node); - spin_unlock(&adreno_dev->active_list_lock); - - - count = drawctxt_detach_drawobjs(drawctxt, list); - spin_unlock(&drawctxt->lock); - - for (i = 0; i < count; i++) { - /* - * If the context is deteached while we are waiting for - * the next command in GFT SKIP CMD, print the context - * detached status here. - */ - adreno_fault_skipcmd_detached(adreno_dev, drawctxt, list[i]); - kgsl_drawobj_destroy(list[i]); - } - - debugfs_remove_recursive(drawctxt->debug_root); - /* The debugfs file has a reference, release it */ - if (drawctxt->debug_root) - kgsl_context_put(context); + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_context *context = &drawctxt->base; + int ret; /* * internal_timestamp is set in adreno_ringbuffer_addcmds, @@ -482,7 +443,7 @@ void adreno_drawctxt_detach(struct kgsl_context *context) * commands to retire will be greater than 10s. 30s should be sufficient * time to wait for the commands even if a hang happens. */ - ret = adreno_drawctxt_wait_rb(adreno_dev, context, + ret = adreno_drawctxt_wait_rb(adreno_dev, &drawctxt->base, drawctxt->internal_timestamp, 30 * 1000); /* @@ -525,9 +486,62 @@ void adreno_drawctxt_detach(struct kgsl_context *context) adreno_profile_process_results(adreno_dev); mutex_unlock(&device->mutex); +} - if (gpudev->preemption_context_destroy) - gpudev->preemption_context_destroy(context); +void adreno_drawctxt_detach(struct kgsl_context *context) +{ + struct kgsl_device *device; + struct adreno_device *adreno_dev; + struct adreno_gpudev *gpudev; + struct adreno_context *drawctxt; + int count, i; + struct kgsl_drawobj *list[ADRENO_CONTEXT_DRAWQUEUE_SIZE]; + + if (context == NULL) + return; + + device = context->device; + adreno_dev = ADRENO_DEVICE(device); + gpudev = ADRENO_GPU_DEVICE(adreno_dev); + drawctxt = ADRENO_CONTEXT(context); + + spin_lock(&drawctxt->lock); + + if (!test_bit(GMU_DISPATCH, &device->gmu_core.flags)) { + spin_lock(&adreno_dev->active_list_lock); + list_del_init(&drawctxt->active_node); + spin_unlock(&adreno_dev->active_list_lock); + } + + count = drawctxt_detach_drawobjs(drawctxt, list); + spin_unlock(&drawctxt->lock); + + for (i = 0; i < count; i++) { + /* + * If the context is detached while we are waiting for + * the next command in GFT SKIP CMD, print the context + * detached status here. + */ + adreno_fault_skipcmd_detached(adreno_dev, drawctxt, list[i]); + kgsl_drawobj_destroy(list[i]); + } + + debugfs_remove_recursive(drawctxt->debug_root); + /* The debugfs file has a reference, release it */ + if (drawctxt->debug_root) + kgsl_context_put(context); + + if (gpudev->context_detach) + gpudev->context_detach(drawctxt); + else + wait_for_timestamp_rb(device, drawctxt); + + if (context->user_ctxt_record) { + gpumem_free_entry(context->user_ctxt_record); + + /* Put the extra ref from gpumem_alloc_entry() */ + kgsl_mem_entry_put(context->user_ctxt_record); + } /* wake threads waiting to submit commands from this context */ wake_up_all(&drawctxt->waiting); diff --git a/drivers/gpu/msm/kgsl_gmu_core.h b/drivers/gpu/msm/kgsl_gmu_core.h index 8ed31effca3c..37d30cc32599 100644 --- a/drivers/gpu/msm/kgsl_gmu_core.h +++ b/drivers/gpu/msm/kgsl_gmu_core.h @@ -40,6 +40,7 @@ enum gmu_core_flags { GMU_ENABLED, GMU_RSCC_SLEEP_SEQ_DONE, GMU_DISABLE_SLUMBER, + GMU_DISPATCH, }; /* From ea24bd99f655c859c1cb230dbe9a7a3892942e08 Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Thu, 23 Jul 2020 13:11:50 -0600 Subject: [PATCH 03/14] msm: kgsl: Add the hwsched dispatcher We need a dispatcher which can keep track of inflight submissions so that in case of a fault, the remaining submissions can be re- submitted after reset. Also, we need a dispatcher to hold queued submissions until the input fences are signaled. Change-Id: I59f5cf0fed6d443cf8a55fd0f40e3f16600d9a6f Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/Makefile | 1 + drivers/gpu/msm/adreno.c | 22 +- drivers/gpu/msm/adreno_a6xx_hwsched.c | 6 + drivers/gpu/msm/adreno_a6xx_hwsched.h | 3 + drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 20 +- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h | 3 +- drivers/gpu/msm/adreno_hwsched.c | 1017 +++++++++++++++++++++ drivers/gpu/msm/adreno_hwsched.h | 82 ++ 8 files changed, 1142 insertions(+), 12 deletions(-) create mode 100644 drivers/gpu/msm/adreno_hwsched.c create mode 100644 drivers/gpu/msm/adreno_hwsched.h diff --git a/drivers/gpu/msm/Makefile b/drivers/gpu/msm/Makefile index 857095463c80..4940cd5878ce 100644 --- a/drivers/gpu/msm/Makefile +++ b/drivers/gpu/msm/Makefile @@ -44,6 +44,7 @@ msm_kgsl-y += \ adreno_cp_parser.o \ adreno_dispatch.o \ adreno_drawctxt.o \ + adreno_hwsched.o \ adreno_ioctl.o \ adreno_perfcounter.o \ adreno_ringbuffer.o \ diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index ca7ed4155555..be8f4b78f9bb 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -22,6 +22,7 @@ #include "adreno_a5xx.h" #include "adreno_a6xx.h" #include "adreno_compat.h" +#include "adreno_hwsched.h" #include "adreno_iommu.h" #include "adreno_trace.h" #include "kgsl_bus.h" @@ -1522,10 +1523,15 @@ static void adreno_unbind(struct device *dev) kgsl_pwrscale_close(device); - adreno_dispatcher_close(adreno_dev); - adreno_ringbuffer_close(adreno_dev); + if (test_bit(GMU_DISPATCH, &device->gmu_core.flags)) + adreno_hwsched_dispatcher_close(adreno_dev); + else { + adreno_dispatcher_close(adreno_dev); - adreno_fault_detect_stop(adreno_dev); + adreno_ringbuffer_close(adreno_dev); + + adreno_fault_detect_stop(adreno_dev); + } kfree(adreno_ft_regs); adreno_ft_regs = NULL; @@ -3607,6 +3613,12 @@ static int adreno_queue_cmds(struct kgsl_device_private *dev_priv, struct kgsl_context *context, struct kgsl_drawobj *drawobj[], u32 count, u32 *timestamp) { + struct kgsl_device *device = dev_priv->device; + + if (test_bit(GMU_DISPATCH, &device->gmu_core.flags)) + return adreno_hwsched_queue_cmds(dev_priv, context, drawobj, + count, timestamp); + return adreno_dispatcher_queue_cmds(dev_priv, context, drawobj, count, timestamp); } @@ -3614,6 +3626,10 @@ static int adreno_queue_cmds(struct kgsl_device_private *dev_priv, static void adreno_drawctxt_sched(struct kgsl_device *device, struct kgsl_context *context) { + if (test_bit(GMU_DISPATCH, &device->gmu_core.flags)) + return adreno_hwsched_queue_context(device, + ADRENO_CONTEXT(context)); + adreno_dispatcher_queue_context(device, ADRENO_CONTEXT(context)); } diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index a6c9f5b85fbb..52017b2df964 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -393,6 +393,8 @@ static int a6xx_hwsched_boot(struct adreno_device *adreno_dev) if (ret) return ret; + adreno_hwsched_start(adreno_dev); + mod_timer(&device->idle_timer, jiffies + device->pwrctrl.interval_timeout); @@ -437,6 +439,10 @@ static int a6xx_hwsched_first_boot(struct adreno_device *adreno_dev) if (ret) return ret; + adreno_hwsched_init(adreno_dev); + + adreno_hwsched_start(adreno_dev); + adreno_get_bus_counters(adreno_dev); adreno_dev->cooperative_reset = ADRENO_FEATURE(adreno_dev, diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.h b/drivers/gpu/msm/adreno_a6xx_hwsched.h index ec985908b2fd..1ef0cb6fd531 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.h @@ -7,6 +7,7 @@ #define _ADRENO_A6XX_HWSCHED_H_ #include "adreno_a6xx_hwsched_hfi.h" +#include "adreno_hwsched.h" /** * struct a6xx_hwsched_device - Container for the a6xx hwscheduling device @@ -16,6 +17,8 @@ struct a6xx_hwsched_device { struct a6xx_device a6xx_dev; /** @hwsched_hfi: Container for hwscheduling specific hfi resources */ struct a6xx_hwsched_hfi hwsched_hfi; + /** @hwsched: Container for the hardware dispatcher */ + struct adreno_hwsched hwsched; }; /** diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index a56e3d9c6eb1..b61c54197366 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -849,7 +849,7 @@ static int hfi_context_register(struct adreno_device *adreno_dev, #define DISPQ_IRQ_BIT(_idx) BIT((_idx) + HFI_DSP_IRQ_BASE) -int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags, +int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, struct kgsl_drawobj_cmd *cmdobj) { struct a6xx_hfi *hfi = to_a6xx_hfi(adreno_dev); @@ -868,6 +868,10 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags, list_for_each_entry(ib, &cmdobj->cmdlist, node) numibs++; + /* We need to dispatch a marker object but not execute it on the GPU */ + if (test_bit(CMDOBJ_SKIP, &cmdobj->priv)) + numibs = 0; + /* Add a *issue_ib struct for each IB */ cmd_sizebytes = sizeof(*cmd) + (sizeof(*issue_ib) * numibs); @@ -880,16 +884,18 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags, atomic_inc_return(&hfi->seqnum)); cmd->ctxt_id = drawobj->context->id; - cmd->flags = flags; + cmd->flags = CTXT_FLAG_NOTIFY; cmd->ts = drawobj->timestamp; cmd->numibs = numibs; - issue_ib = (struct hfi_issue_ib *)&cmd[1]; + if (numibs) { + issue_ib = (struct hfi_issue_ib *)&cmd[1]; - list_for_each_entry(ib, &cmdobj->cmdlist, node) { - issue_ib->addr = ib->gpuaddr; - issue_ib->size = ib->size; - issue_ib++; + list_for_each_entry(ib, &cmdobj->cmdlist, node) { + issue_ib->addr = ib->gpuaddr; + issue_ib->size = ib->size; + issue_ib++; + } } ret = a6xx_hfi_queue_write(adreno_dev, HFI_DSP_ID_0, (u32 *)cmd); diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index a82c0a68ad14..7a8e1ec48850 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -181,7 +181,6 @@ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data); /** * a6xx_hwsched_submit_cmdobj - Dispatch IBs to dispatch queues * @adreno_dev: Pointer to adreno device structure - * @flags: Flags associated with the submission * @cmdobj: The command object which needs to be submitted * * This function is used to register the context if needed and submit @@ -189,7 +188,7 @@ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data); * Return: 0 on success and negative error on failure */ -int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags, +int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, struct kgsl_drawobj_cmd *cmdobj); /** diff --git a/drivers/gpu/msm/adreno_hwsched.c b/drivers/gpu/msm/adreno_hwsched.c new file mode 100644 index 000000000000..8d30e49b8bfc --- /dev/null +++ b/drivers/gpu/msm/adreno_hwsched.c @@ -0,0 +1,1017 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#include "adreno.h" +#include "adreno_a6xx.h" +#include "adreno_a6xx_hwsched.h" +#include "adreno_trace.h" + +/* This structure represents inflight command object */ +struct cmd_list_obj { + /** @cmdobj: Handle to the command object */ + struct kgsl_drawobj_cmd *cmdobj; + /** @node: List node to put it in the list of inflight commands */ + struct list_head node; +}; + +/* Number of milliseconds to wait for the context queue to clear */ +static unsigned int _context_queue_wait = 10000; + +/* Use a kmem cache to speed up allocations for dispatcher jobs */ +static struct kmem_cache *jobs_cache; +/* Use a kmem cache to speed up allocations for inflight command objects */ +static struct kmem_cache *obj_cache; + +static struct adreno_hwsched *to_hwsched(struct adreno_device *adreno_dev) +{ + struct a6xx_device *a6xx_dev = container_of(adreno_dev, + struct a6xx_device, adreno_dev); + struct a6xx_hwsched_device *a6xx_hwsched = container_of(a6xx_dev, + struct a6xx_hwsched_device, a6xx_dev); + + return &a6xx_hwsched->hwsched; +} + +static struct adreno_device *hwsched_to_adreno(struct adreno_hwsched *hwsched) +{ + struct a6xx_hwsched_device *a6xx_hwsched = container_of(hwsched, + struct a6xx_hwsched_device, hwsched); + + return &a6xx_hwsched->a6xx_dev.adreno_dev; +} + +static bool _check_context_queue(struct adreno_context *drawctxt) +{ + bool ret; + + spin_lock(&drawctxt->lock); + + /* + * Wake up if there is room in the context or if the whole thing got + * invalidated while we were asleep + */ + + if (kgsl_context_invalid(&drawctxt->base)) + ret = false; + else + ret = drawctxt->queued < (ADRENO_CONTEXT_DRAWQUEUE_SIZE - 1); + + spin_unlock(&drawctxt->lock); + + return ret; +} + +static void _pop_drawobj(struct adreno_context *drawctxt) +{ + drawctxt->drawqueue_head = DRAWQUEUE_NEXT(drawctxt->drawqueue_head, + ADRENO_CONTEXT_DRAWQUEUE_SIZE); + drawctxt->queued--; +} + +static int _retire_syncobj(struct kgsl_drawobj_sync *syncobj, + struct adreno_context *drawctxt) +{ + if (!kgsl_drawobj_events_pending(syncobj)) { + _pop_drawobj(drawctxt); + kgsl_drawobj_destroy(DRAWOBJ(syncobj)); + return 0; + } + + /* + * If we got here, there are pending events for sync object. + * Start the canary timer if it hasnt been started already. + */ + if (!syncobj->timeout_jiffies) { + syncobj->timeout_jiffies = jiffies + msecs_to_jiffies(5000); + mod_timer(&syncobj->timer, syncobj->timeout_jiffies); + } + + return -EAGAIN; +} + +static bool _marker_expired(struct kgsl_drawobj_cmd *markerobj) +{ + struct kgsl_drawobj *drawobj = DRAWOBJ(markerobj); + + return (drawobj->flags & KGSL_DRAWOBJ_MARKER) && + kgsl_check_timestamp(drawobj->device, drawobj->context, + markerobj->marker_timestamp); +} + +static void _retire_timestamp(struct kgsl_drawobj *drawobj) +{ + struct kgsl_context *context = drawobj->context; + struct kgsl_device *device = context->device; + + /* + * Write the start and end timestamp to the memstore to keep the + * accounting sane + */ + kgsl_sharedmem_writel(device->memstore, + KGSL_MEMSTORE_OFFSET(context->id, soptimestamp), + drawobj->timestamp); + + kgsl_sharedmem_writel(device->memstore, + KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp), + drawobj->timestamp); + + /* Retire pending GPU events for the object */ + kgsl_process_event_group(device, &context->events); + + kgsl_drawobj_destroy(drawobj); +} + +static int _retire_markerobj(struct kgsl_drawobj_cmd *cmdobj, + struct adreno_context *drawctxt) +{ + if (_marker_expired(cmdobj)) { + _pop_drawobj(drawctxt); + _retire_timestamp(DRAWOBJ(cmdobj)); + return 0; + } + + /* + * If the marker isn't expired but the SKIP bit + * is set then there are real commands following + * this one in the queue. This means that we + * need to dispatch the command so that we can + * keep the timestamp accounting correct. If + * skip isn't set then we block this queue + * until the dependent timestamp expires + */ + + return test_bit(CMDOBJ_SKIP, &cmdobj->priv) ? 1 : -EAGAIN; +} + +/* + * Retires all expired marker and sync objs from the context + * queue and returns one of the below + * a) next drawobj that needs to be sent to ringbuffer + * b) -EAGAIN for syncobj with syncpoints pending. + * c) -EAGAIN for markerobj whose marker timestamp has not expired yet. + * c) NULL for no commands remaining in drawqueue. + */ +static struct kgsl_drawobj *_process_drawqueue_get_next_drawobj( + struct adreno_context *drawctxt) +{ + struct kgsl_drawobj *drawobj; + unsigned int i = drawctxt->drawqueue_head; + int ret = 0; + + if (drawctxt->drawqueue_head == drawctxt->drawqueue_tail) + return NULL; + + for (i = drawctxt->drawqueue_head; i != drawctxt->drawqueue_tail; + i = DRAWQUEUE_NEXT(i, ADRENO_CONTEXT_DRAWQUEUE_SIZE)) { + + drawobj = drawctxt->drawqueue[i]; + + if (drawobj == NULL) + return NULL; + + if (drawobj->type == CMDOBJ_TYPE) + return drawobj; + else if (drawobj->type == SYNCOBJ_TYPE) + ret = _retire_syncobj(SYNCOBJ(drawobj), drawctxt); + else if (drawobj->type == MARKEROBJ_TYPE) { + ret = _retire_markerobj(CMDOBJ(drawobj), drawctxt); + /* Special case where marker needs to be sent to GPU */ + if (ret == 1) + return drawobj; + } else { + return ERR_PTR(-EINVAL); + } + + if (ret == -EAGAIN) + return ERR_PTR(-EAGAIN); + } + + return NULL; +} + +/** + * hwsched_dispatcher_requeue_cmdobj() - Put a command back on the context + * queue + * @drawctxt: Pointer to the adreno draw context + * @cmdobj: Pointer to the KGSL command object to requeue + * + * Failure to submit a command to the ringbuffer isn't the fault of the command + * being submitted so if a failure happens, push it back on the head of the + * context queue to be reconsidered again unless the context got detached. + */ +static inline int hwsched_dispatcher_requeue_cmdobj( + struct adreno_context *drawctxt, + struct kgsl_drawobj_cmd *cmdobj) +{ + unsigned int prev; + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + + spin_lock(&drawctxt->lock); + + if (kgsl_context_detached(&drawctxt->base) || + kgsl_context_invalid(&drawctxt->base)) { + spin_unlock(&drawctxt->lock); + /* get rid of this drawobj since the context is bad */ + kgsl_drawobj_destroy(drawobj); + return -ENOENT; + } + + prev = drawctxt->drawqueue_head == 0 ? + (ADRENO_CONTEXT_DRAWQUEUE_SIZE - 1) : + (drawctxt->drawqueue_head - 1); + + /* + * The maximum queue size always needs to be one less then the size of + * the ringbuffer queue so there is "room" to put the drawobj back in + */ + + WARN_ON(prev == drawctxt->drawqueue_tail); + + drawctxt->drawqueue[prev] = drawobj; + drawctxt->queued++; + + /* Reset the command queue head to reflect the newly requeued change */ + drawctxt->drawqueue_head = prev; + spin_unlock(&drawctxt->lock); + return 0; +} + +/** + * hwsched_queue_context() - Queue a context in the dispatcher list of jobs + * @adreno_dev: Pointer to the adreno device structure + * @drawctxt: Pointer to the adreno draw context + * + * Add a context to the dispatcher list of jobs. + */ +static int hwsched_queue_context(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct adreno_dispatch_job *job; + + /* Refuse to queue a detached context */ + if (kgsl_context_detached(&drawctxt->base)) + return 0; + + if (!_kgsl_context_get(&drawctxt->base)) + return 0; + + job = kmem_cache_alloc(jobs_cache, GFP_ATOMIC); + if (!job) { + kgsl_context_put(&drawctxt->base); + return -ENOMEM; + } + + job->drawctxt = drawctxt; + + trace_dispatch_queue_context(drawctxt); + llist_add(&job->node, &hwsched->jobs[drawctxt->base.priority]); + + return 0; +} + +/** + * sendcmd() - Send a drawobj to the GPU hardware + * @dispatcher: Pointer to the adreno dispatcher struct + * @drawobj: Pointer to the KGSL drawobj being sent + * + * Send a KGSL drawobj to the GPU hardware + */ +static int hwsched_sendcmd(struct adreno_device *adreno_dev, + struct kgsl_drawobj_cmd *cmdobj) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + struct kgsl_context *context = drawobj->context; + struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context); + int ret; + struct cmd_list_obj *obj; + + obj = kmem_cache_alloc(obj_cache, GFP_KERNEL); + if (!obj) + return -ENOMEM; + + mutex_lock(&device->mutex); + + if (kgsl_context_detached(context)) { + mutex_unlock(&device->mutex); + kmem_cache_free(obj_cache, obj); + return -ENOENT; + } + + hwsched->inflight++; + + if (hwsched->inflight == 1 && + !test_bit(ADRENO_HWSCHED_POWER, &hwsched->flags)) { + ret = adreno_active_count_get(adreno_dev); + if (ret) { + hwsched->inflight--; + mutex_unlock(&device->mutex); + kmem_cache_free(obj_cache, obj); + return ret; + } + set_bit(ADRENO_HWSCHED_POWER, &hwsched->flags); + } + + ret = a6xx_hwsched_submit_cmdobj(adreno_dev, cmdobj); + if (ret) { + /* + * If the first submission failed, then put back the active + * count to relinquish active vote + */ + if (hwsched->inflight == 1) { + adreno_active_count_put(adreno_dev); + clear_bit(ADRENO_HWSCHED_POWER, &hwsched->flags); + } + + hwsched->inflight--; + kmem_cache_free(obj_cache, obj); + mutex_unlock(&device->mutex); + return ret; + } + + drawctxt->internal_timestamp = drawobj->timestamp; + + obj->cmdobj = cmdobj; + list_add_tail(&obj->node, &hwsched->cmd_list); + mutex_unlock(&device->mutex); + + return 0; +} + +/** + * hwsched_sendcmds() - Send commands from a context to the GPU + * @adreno_dev: Pointer to the adreno device struct + * @drawctxt: Pointer to the adreno context to dispatch commands from + * + * Dequeue and send a burst of commands from the specified context to the GPU + * Returns postive if the context needs to be put back on the pending queue + * 0 if the context is empty or detached and negative on error + */ +static int hwsched_sendcmds(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt) +{ + int count = 0; + int ret = 0; + unsigned int timestamp; + + while (1) { + struct kgsl_drawobj *drawobj; + struct kgsl_drawobj_cmd *cmdobj; + + spin_lock(&drawctxt->lock); + drawobj = _process_drawqueue_get_next_drawobj(drawctxt); + + /* + * adreno_context_get_drawobj returns -EAGAIN if the current + * drawobj has pending sync points so no more to do here. + * When the sync points are satisfied then the context will get + * reqeueued + */ + + if (IS_ERR_OR_NULL(drawobj)) { + if (IS_ERR(drawobj)) + ret = PTR_ERR(drawobj); + spin_unlock(&drawctxt->lock); + break; + } + _pop_drawobj(drawctxt); + spin_unlock(&drawctxt->lock); + + timestamp = drawobj->timestamp; + cmdobj = CMDOBJ(drawobj); + ret = hwsched_sendcmd(adreno_dev, cmdobj); + + /* + * On error from hwsched_sendcmd() try to requeue the cmdobj + * unless we got back -ENOENT which means that the context has + * been detached and there will be no more deliveries from here + */ + if (ret != 0) { + /* Destroy the cmdobj on -ENOENT */ + if (ret == -ENOENT) + kgsl_drawobj_destroy(drawobj); + else { + /* + * If we couldn't put it on dispatch queue + * then return it to the context queue + */ + int r = hwsched_dispatcher_requeue_cmdobj( + drawctxt, cmdobj); + if (r) + ret = r; + } + + break; + } + + drawctxt->submitted_timestamp = timestamp; + + count++; + } + + /* + * Wake up any snoozing threads if we have consumed any real commands + * or marker commands and we have room in the context queue. + */ + + if (_check_context_queue(drawctxt)) + wake_up_all(&drawctxt->wq); + + if (!ret) + ret = count; + + /* Return error or the number of commands queued */ + return ret; +} + +static bool adreno_drawctxt_bad(struct adreno_context *drawctxt) +{ + return (kgsl_context_detached(&drawctxt->base) || + kgsl_context_invalid(&drawctxt->base)); +} + + +static void hwsched_handle_jobs_list(struct adreno_device *adreno_dev, + int id, unsigned long *map, struct llist_node *list) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct adreno_dispatch_job *job, *next; + + if (!list) + return; + + /* Reverse the list so we deal with oldest submitted contexts first */ + list = llist_reverse_order(list); + + llist_for_each_entry_safe(job, next, list, node) { + int ret; + + if (adreno_drawctxt_bad(job->drawctxt)) { + kgsl_context_put(&job->drawctxt->base); + kmem_cache_free(jobs_cache, job); + continue; + } + + /* + * Due to the nature of the lockless queue the same context + * might have multiple jobs on the list. We allow this so we + * don't have to query the list on the producer side but on the + * consumer side we only want each context to be considered + * once. Use a bitmap to remember which contexts we've already + * seen and quietly discard duplicate jobs + */ + if (test_and_set_bit(job->drawctxt->base.id, map)) { + kgsl_context_put(&job->drawctxt->base); + kmem_cache_free(jobs_cache, job); + continue; + } + + ret = hwsched_sendcmds(adreno_dev, job->drawctxt); + + /* + * If the context had nothing queued or the context has been + * destroyed then drop the job + */ + if (!ret || ret == -ENOENT) { + kgsl_context_put(&job->drawctxt->base); + kmem_cache_free(jobs_cache, job); + continue; + } + + /* + * If the dispatch queue is full then requeue the job to be + * considered first next time. Otherwise the context + * either successfully submmitted to the GPU or another error + * happened and it should go back on the regular queue + */ + if (ret == -ENOSPC) + llist_add(&job->node, &hwsched->requeue[id]); + else + llist_add(&job->node, &hwsched->jobs[id]); + } +} + +static void hwsched_handle_jobs(struct adreno_device *adreno_dev, int id) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + unsigned long map[BITS_TO_LONGS(KGSL_MEMSTORE_MAX)]; + struct llist_node *requeue, *jobs; + + memset(map, 0, sizeof(map)); + + requeue = llist_del_all(&hwsched->requeue[id]); + jobs = llist_del_all(&hwsched->jobs[id]); + + hwsched_handle_jobs_list(adreno_dev, id, map, requeue); + hwsched_handle_jobs_list(adreno_dev, id, map, jobs); +} + +/** + * hwsched_issuecmds() - Issue commmands from pending contexts + * @adreno_dev: Pointer to the adreno device struct + * + * Issue as many commands as possible (up to inflight) from the pending contexts + * This function assumes the dispatcher mutex has been locked. + */ +static void hwsched_issuecmds(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + int i; + + for (i = 0; i < ARRAY_SIZE(hwsched->jobs); i++) + hwsched_handle_jobs(adreno_dev, i); +} + +static void adreno_hwsched_trigger(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + + kthread_queue_work(&kgsl_driver.worker, &hwsched->work); +} + +/** + * adreno_hwsched_issuecmds() - Issue commmands from pending contexts + * @adreno_dev: Pointer to the adreno device struct + * + * Lock the dispatcher and call hwsched_issuecmds + */ +static void adreno_hwsched_issuecmds(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + + /* If the dispatcher is busy then schedule the work for later */ + if (!mutex_trylock(&hwsched->mutex)) { + adreno_hwsched_trigger(adreno_dev); + return; + } + + hwsched_issuecmds(adreno_dev); + + mutex_unlock(&hwsched->mutex); +} + +/** + * get_timestamp() - Return the next timestamp for the context + * @drawctxt - Pointer to an adreno draw context struct + * @drawobj - Pointer to a drawobj + * @timestamp - Pointer to a timestamp value possibly passed from the user + * @user_ts - user generated timestamp + * + * Assign a timestamp based on the settings of the draw context and the command + * batch. + */ +static int get_timestamp(struct adreno_context *drawctxt, + struct kgsl_drawobj *drawobj, unsigned int *timestamp, + unsigned int user_ts) +{ + + if (drawctxt->base.flags & KGSL_CONTEXT_USER_GENERATED_TS) { + /* + * User specified timestamps need to be greater than the last + * issued timestamp in the context + */ + if (timestamp_cmp(drawctxt->timestamp, user_ts) >= 0) + return -ERANGE; + + drawctxt->timestamp = user_ts; + } else + drawctxt->timestamp++; + + *timestamp = drawctxt->timestamp; + drawobj->timestamp = *timestamp; + return 0; +} + +static inline int _check_context_state(struct kgsl_context *context) +{ + if (kgsl_context_invalid(context)) + return -EDEADLK; + + if (kgsl_context_detached(context)) + return -ENOENT; + + return 0; +} + +static inline bool _verify_ib(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_memobj_node *ib) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_process_private *private = dev_priv->process_priv; + + /* The maximum allowable size for an IB in the CP is 0xFFFFF dwords */ + if (ib->size == 0 || ((ib->size >> 2) > 0xFFFFF)) { + pr_context(device, context, "ctxt %d invalid ib size %lld\n", + context->id, ib->size); + return false; + } + + /* Make sure that the address is mapped */ + if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, ib->gpuaddr)) { + pr_context(device, context, "ctxt %d invalid ib gpuaddr %llX\n", + context->id, ib->gpuaddr); + return false; + } + + return true; +} + +static inline int _verify_cmdobj(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj[], + uint32_t count) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_memobj_node *ib; + unsigned int i; + + for (i = 0; i < count; i++) { + /* Verify the IBs before they get queued */ + if (drawobj[i]->type == CMDOBJ_TYPE) { + struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj[i]); + + list_for_each_entry(ib, &cmdobj->cmdlist, node) + if (!_verify_ib(dev_priv, + &ADRENO_CONTEXT(context)->base, ib)) + return -EINVAL; + + /* + * Clear the wake on touch bit to indicate an IB has + * been submitted since the last time we set it. + * But only clear it when we have rendering commands. + */ + device->flags &= ~KGSL_FLAG_WAKE_ON_TOUCH; + } + } + + return 0; +} + +static inline int _wait_for_room_in_context_queue( + struct adreno_context *drawctxt) +{ + int ret = 0; + + /* Wait for room in the context queue */ + while (drawctxt->queued >= ADRENO_CONTEXT_DRAWQUEUE_SIZE - 2) { + trace_adreno_drawctxt_sleep(drawctxt); + spin_unlock(&drawctxt->lock); + + ret = wait_event_interruptible_timeout(drawctxt->wq, + _check_context_queue(drawctxt), + msecs_to_jiffies(_context_queue_wait)); + + spin_lock(&drawctxt->lock); + trace_adreno_drawctxt_wake(drawctxt); + + if (ret <= 0) + return (ret == 0) ? -ETIMEDOUT : (int) ret; + } + + return 0; +} + +static unsigned int _check_context_state_to_queue_cmds( + struct adreno_context *drawctxt) +{ + int ret = _check_context_state(&drawctxt->base); + + if (ret) + return ret; + + ret = _wait_for_room_in_context_queue(drawctxt); + if (ret) + return ret; + + /* + * Account for the possiblity that the context got invalidated + * while we were sleeping + */ + return _check_context_state(&drawctxt->base); +} + +static void _queue_drawobj(struct adreno_context *drawctxt, + struct kgsl_drawobj *drawobj) +{ + /* Put the command into the queue */ + drawctxt->drawqueue[drawctxt->drawqueue_tail] = drawobj; + drawctxt->drawqueue_tail = (drawctxt->drawqueue_tail + 1) % + ADRENO_CONTEXT_DRAWQUEUE_SIZE; + drawctxt->queued++; + trace_adreno_cmdbatch_queued(drawobj, drawctxt->queued); +} + +static int _queue_cmdobj(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, struct kgsl_drawobj_cmd *cmdobj, + uint32_t *timestamp, unsigned int user_ts) +{ + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + u32 j; + int ret; + + ret = get_timestamp(drawctxt, drawobj, timestamp, user_ts); + if (ret) + return ret; + + /* + * If this is a real command then we need to force any markers + * queued before it to dispatch to keep time linear - set the + * skip bit so the commands get NOPed. + */ + j = drawctxt->drawqueue_head; + + while (j != drawctxt->drawqueue_tail) { + if (drawctxt->drawqueue[j]->type == MARKEROBJ_TYPE) { + struct kgsl_drawobj_cmd *markerobj = + CMDOBJ(drawctxt->drawqueue[j]); + + set_bit(CMDOBJ_SKIP, &markerobj->priv); + } + + j = DRAWQUEUE_NEXT(j, ADRENO_CONTEXT_DRAWQUEUE_SIZE); + } + + drawctxt->queued_timestamp = *timestamp; + + _queue_drawobj(drawctxt, drawobj); + + return 0; +} + +static void _queue_syncobj(struct adreno_context *drawctxt, + struct kgsl_drawobj_sync *syncobj, uint32_t *timestamp) +{ + struct kgsl_drawobj *drawobj = DRAWOBJ(syncobj); + + *timestamp = 0; + drawobj->timestamp = 0; + + _queue_drawobj(drawctxt, drawobj); +} + +static int _queue_markerobj(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, struct kgsl_drawobj_cmd *markerobj, + u32 *timestamp, u32 user_ts) +{ + struct kgsl_drawobj *drawobj = DRAWOBJ(markerobj); + int ret; + + ret = get_timestamp(drawctxt, drawobj, timestamp, user_ts); + if (ret) + return ret; + + /* + * See if we can fastpath this thing - if nothing is queued + * and nothing is inflight retire without bothering the GPU + */ + if (!drawctxt->queued && kgsl_check_timestamp(drawobj->device, + drawobj->context, drawctxt->queued_timestamp)) { + _retire_timestamp(drawobj); + return 1; + } + + /* + * Remember the last queued timestamp - the marker will block + * until that timestamp is expired (unless another command + * comes along and forces the marker to execute) + */ + markerobj->marker_timestamp = drawctxt->queued_timestamp; + drawctxt->queued_timestamp = *timestamp; + + _queue_drawobj(drawctxt, drawobj); + + return 0; +} + +int adreno_hwsched_queue_cmds(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj[], + u32 count, u32 *timestamp) + +{ + struct kgsl_device *device = dev_priv->device; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct adreno_context *drawctxt = ADRENO_CONTEXT(context); + int ret; + unsigned int i, user_ts; + + if (!count) + return -EINVAL; + + ret = _check_context_state(&drawctxt->base); + if (ret) + return ret; + + ret = _verify_cmdobj(dev_priv, context, drawobj, count); + if (ret) + return ret; + + /* wait for the suspend gate */ + wait_for_completion(&device->halt_gate); + + spin_lock(&drawctxt->lock); + + ret = _check_context_state_to_queue_cmds(drawctxt); + if (ret) { + spin_unlock(&drawctxt->lock); + return ret; + } + + user_ts = *timestamp; + + for (i = 0; i < count; i++) { + + switch (drawobj[i]->type) { + case MARKEROBJ_TYPE: + ret = _queue_markerobj(adreno_dev, drawctxt, + CMDOBJ(drawobj[i]), + timestamp, user_ts); + if (ret == 1) { + spin_unlock(&drawctxt->lock); + return 0; + } else if (ret) { + spin_unlock(&drawctxt->lock); + return ret; + } + break; + case CMDOBJ_TYPE: + ret = _queue_cmdobj(adreno_dev, drawctxt, + CMDOBJ(drawobj[i]), + timestamp, user_ts); + if (ret) { + spin_unlock(&drawctxt->lock); + return ret; + } + break; + case SYNCOBJ_TYPE: + _queue_syncobj(drawctxt, SYNCOBJ(drawobj[i]), + timestamp); + break; + default: + spin_unlock(&drawctxt->lock); + return -EINVAL; + } + + } + + spin_unlock(&drawctxt->lock); + + /* Add the context to the dispatcher pending list */ + ret = hwsched_queue_context(adreno_dev, drawctxt); + if (ret) + return ret; + + adreno_hwsched_issuecmds(adreno_dev); + + return 0; +} + +static void retire_cmdobj(struct kgsl_drawobj_cmd *cmdobj) +{ + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + struct kgsl_mem_entry *entry; + struct kgsl_drawobj_profiling_buffer *profile_buffer; + + if (cmdobj != NULL) { + entry = cmdobj->profiling_buf_entry; + if (entry) { + profile_buffer = kgsl_gpuaddr_to_vaddr(&entry->memdesc, + cmdobj->profiling_buffer_gpuaddr); + + if (profile_buffer == NULL) + return; + + kgsl_memdesc_unmap(&entry->memdesc); + } + } + + kgsl_drawobj_destroy(drawobj); +} + +static int retire_cmd_list(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int count = 0; + struct cmd_list_obj *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) { + struct kgsl_drawobj_cmd *cmdobj = obj->cmdobj; + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + + if (!kgsl_check_timestamp(device, drawobj->context, + drawobj->timestamp)) + continue; + + retire_cmdobj(cmdobj); + + list_del_init(&obj->node); + + kmem_cache_free(obj_cache, obj); + + hwsched->inflight--; + + count++; + } + + return count; +} + +/* Take down the dispatcher and release any power states */ +static void hwsched_power_down(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + + mutex_lock(&device->mutex); + + if (test_bit(ADRENO_HWSCHED_POWER, &hwsched->flags)) { + adreno_active_count_put(adreno_dev); + clear_bit(ADRENO_HWSCHED_POWER, &hwsched->flags); + } + + mutex_unlock(&device->mutex); +} + +void adreno_hwsched_queue_context(struct kgsl_device *device, + struct adreno_context *drawctxt) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + hwsched_queue_context(adreno_dev, drawctxt); + adreno_hwsched_trigger(adreno_dev); +} + +void adreno_hwsched_start(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + complete_all(&device->halt_gate); + + adreno_hwsched_trigger(adreno_dev); +} + +void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev) +{ + kmem_cache_destroy(jobs_cache); + kmem_cache_destroy(obj_cache); +} + +static void adreno_hwsched_work(struct kthread_work *work) +{ + struct adreno_hwsched *hwsched = container_of(work, + struct adreno_hwsched, work); + struct adreno_device *adreno_dev = hwsched_to_adreno(hwsched); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int count = 0; + + mutex_lock(&hwsched->mutex); + + /* + * As long as there are inflight commands, process retired comamnds from + * all drawqueues + */ + count += retire_cmd_list(adreno_dev); + + /* Signal fences */ + kgsl_process_event_groups(device); + + /* Run the scheduler for to dispatch new commands */ + hwsched_issuecmds(adreno_dev); + + if (hwsched->inflight == 0) { + hwsched_power_down(adreno_dev); + } else { + mutex_lock(&device->mutex); + kgsl_pwrscale_update(device); + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + mutex_unlock(&device->mutex); + } + + mutex_unlock(&hwsched->mutex); +} + +void adreno_hwsched_init(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + int i; + + memset(hwsched, 0, sizeof(*hwsched)); + + mutex_init(&hwsched->mutex); + + kthread_init_work(&hwsched->work, adreno_hwsched_work); + + jobs_cache = KMEM_CACHE(adreno_dispatch_job, 0); + obj_cache = KMEM_CACHE(cmd_list_obj, 0); + + INIT_LIST_HEAD(&hwsched->cmd_list); + + for (i = 0; i < ARRAY_SIZE(hwsched->jobs); i++) { + init_llist_head(&hwsched->jobs[i]); + init_llist_head(&hwsched->requeue[i]); + } +} diff --git a/drivers/gpu/msm/adreno_hwsched.h b/drivers/gpu/msm/adreno_hwsched.h new file mode 100644 index 000000000000..49d261ef77d0 --- /dev/null +++ b/drivers/gpu/msm/adreno_hwsched.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#ifndef _ADRENO_HWSCHED_H_ +#define _ADRENO_HWSCHED_H_ + +/** + * struct adreno_hwsched - Container for the hardware scheduler + */ +struct adreno_hwsched { + /** @mutex: Mutex needed to run dispatcher function */ + struct mutex mutex; + /** @flags: Container for the dispatcher internal flags */ + unsigned long flags; + /** @inflight: Number of active submissions to the dispatch queues */ + u32 inflight; + /** @jobs - Array of dispatch job lists for each priority level */ + struct llist_head jobs[16]; + /** @requeue - Array of lists for dispatch jobs that got requeued */ + struct llist_head requeue[16]; + /** @work: The work structure to execute dispatcher function */ + struct kthread_work work; + /** @cmd_list: List of objects submitted to dispatch queues */ + struct list_head cmd_list; +}; + +enum adreno_hwsched_flags { + ADRENO_HWSCHED_POWER = 0, +}; + +/** + * adreno_hwsched_queue_cmds() - Queue a new draw object in the context + * @dev_priv: Pointer to the device private struct + * @context: Pointer to the kgsl draw context + * @drawobj: Pointer to the array of drawobj's being submitted + * @count: Number of drawobj's being submitted + * @timestamp: Pointer to the requested timestamp + * + * Queue a command in the context - if there isn't any room in the queue, then + * block until there is + * + * Return: 0 on success and negative error on failure to queue + */ +int adreno_hwsched_queue_cmds(struct kgsl_device_private *dev_priv, + struct kgsl_context *context, struct kgsl_drawobj *drawobj[], + u32 count, u32 *timestamp); +/** + * adreno_hwsched_queue_context() - schedule a drawctxt in the hw dispatcher + * @device: pointer to the KGSL device + * @drawctxt: pointer to the drawctxt to schedule + * + * Put a draw context on the dispatcher job listse and schedule the + * dispatcher. This is used to reschedule changes that might have been blocked + * for sync points or other concerns + */ +void adreno_hwsched_queue_context(struct kgsl_device *device, + struct adreno_context *drawctxt); +/** + * adreno_hwsched_start() - activate the hwsched dispatcher + * @adreno_dev: pointer to the adreno device + * + * Enable dispatcher thread to execute + */ +void adreno_hwsched_start(struct adreno_device *adreno_dev); +/** + * adreno_hwsched_dispatcher_init() - Initialize the hwsched dispatcher + * @adreno_dev: pointer to the adreno device + * + * Set up the dispatcher resources + */ +void adreno_hwsched_init(struct adreno_device *adreno_dev); + +/** + * adreno_hwsched_dispatcher_close() - close the hwsched dispatcher + * @adreno_dev: pointer to the adreno device structure + * + * Free the dispatcher resources + */ +void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev); +#endif From 0eb8eb4dccbd47876db97f96eafbf95c3bb745ff Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Thu, 23 Jul 2020 13:19:24 -0600 Subject: [PATCH 04/14] msm: kgsl: Modify the a6xx interrupt mask for hwsched With hardware scheduling, we only need to enable a subset of interrupts as the rest of them are completely owned by GMU. Change-Id: I32afc4420299688b9a4322804ce972c79f1a7a22 Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx.c | 20 ++------------------ drivers/gpu/msm/adreno_a6xx.h | 25 +++++++++++++++++++++++++ drivers/gpu/msm/adreno_a6xx_gmu.c | 2 ++ drivers/gpu/msm/adreno_a6xx_hwsched.c | 2 ++ drivers/gpu/msm/adreno_a6xx_rgmu.c | 2 ++ 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c index fb9cda3c9713..e1698a5c03f4 100644 --- a/drivers/gpu/msm/adreno_a6xx.c +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -17,22 +17,6 @@ #include "adreno_trace.h" #include "kgsl_trace.h" -#define A6XX_INT_MASK \ - ((1 << A6XX_INT_CP_AHB_ERROR) | \ - (1 << A6XX_INT_ATB_ASYNCFIFO_OVERFLOW) | \ - (1 << A6XX_INT_RBBM_GPC_ERROR) | \ - (1 << A6XX_INT_CP_SW) | \ - (1 << A6XX_INT_CP_HW_ERROR) | \ - (1 << A6XX_INT_CP_IB2) | \ - (1 << A6XX_INT_CP_IB1) | \ - (1 << A6XX_INT_CP_RB) | \ - (1 << A6XX_INT_CP_CACHE_FLUSH_TS) | \ - (1 << A6XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ - (1 << A6XX_INT_RBBM_HANG_DETECT) | \ - (1 << A6XX_INT_UCHE_OOB_ACCESS) | \ - (1 << A6XX_INT_UCHE_TRAP_INTR) | \ - (1 << A6XX_INT_TSB_WRITE_ERROR)) - /* IFPC & Preemption static powerup restore list */ static u32 a6xx_pwrup_reglist[] = { A6XX_VSC_ADDR_MODE_CNTL, @@ -439,8 +423,6 @@ void a6xx_start(struct adreno_device *adreno_dev) unsigned int rgb565_predicator = 0; static bool patch_reglist; - adreno_dev->irq_mask = A6XX_INT_MASK; - /* enable hardware clockgating */ a6xx_hwcg_set(adreno_dev, true); @@ -2532,6 +2514,8 @@ static int a6xx_probe(struct platform_device *pdev, INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + adreno_dev->irq_mask = A6XX_INT_MASK; + return 0; } diff --git a/drivers/gpu/msm/adreno_a6xx.h b/drivers/gpu/msm/adreno_a6xx.h index 5fa428139b4f..c8895ef8d9d7 100644 --- a/drivers/gpu/msm/adreno_a6xx.h +++ b/drivers/gpu/msm/adreno_a6xx.h @@ -186,6 +186,31 @@ struct cpu_gpu_lock { /* Size of the CP_INIT pm4 stream in dwords */ #define A6XX_CP_INIT_DWORDS 12 +#define A6XX_INT_MASK \ + ((1 << A6XX_INT_CP_AHB_ERROR) | \ + (1 << A6XX_INT_ATB_ASYNCFIFO_OVERFLOW) | \ + (1 << A6XX_INT_RBBM_GPC_ERROR) | \ + (1 << A6XX_INT_CP_SW) | \ + (1 << A6XX_INT_CP_HW_ERROR) | \ + (1 << A6XX_INT_CP_IB2) | \ + (1 << A6XX_INT_CP_IB1) | \ + (1 << A6XX_INT_CP_RB) | \ + (1 << A6XX_INT_CP_CACHE_FLUSH_TS) | \ + (1 << A6XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A6XX_INT_RBBM_HANG_DETECT) | \ + (1 << A6XX_INT_UCHE_OOB_ACCESS) | \ + (1 << A6XX_INT_UCHE_TRAP_INTR) | \ + (1 << A6XX_INT_TSB_WRITE_ERROR)) + +#define A6XX_HWSCHED_INT_MASK \ + ((1 << A6XX_INT_CP_AHB_ERROR) | \ + (1 << A6XX_INT_ATB_ASYNCFIFO_OVERFLOW) | \ + (1 << A6XX_INT_RBBM_GPC_ERROR) | \ + (1 << A6XX_INT_RBBM_ATB_BUS_OVERFLOW) | \ + (1 << A6XX_INT_UCHE_OOB_ACCESS) | \ + (1 << A6XX_INT_UCHE_TRAP_INTR) | \ + (1 << A6XX_INT_TSB_WRITE_ERROR)) + /** * to_a6xx_core - return the a6xx specific GPU core struct * @adreno_dev: An Adreno GPU device handle diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index dd71faf813ac..e25b409f1f83 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -3333,6 +3333,8 @@ int a6xx_gmu_device_probe(struct platform_device *pdev, timer_setup(&device->idle_timer, gmu_idle_timer, 0); + adreno_dev->irq_mask = A6XX_INT_MASK; + return 0; } diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index 52017b2df964..127facf3d300 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -705,6 +705,8 @@ int a6xx_hwsched_probe(struct platform_device *pdev, timer_setup(&device->idle_timer, hwsched_idle_timer, 0); + adreno_dev->irq_mask = A6XX_HWSCHED_INT_MASK; + return 0; } diff --git a/drivers/gpu/msm/adreno_a6xx_rgmu.c b/drivers/gpu/msm/adreno_a6xx_rgmu.c index dd312269c080..0cb54da191c3 100644 --- a/drivers/gpu/msm/adreno_a6xx_rgmu.c +++ b/drivers/gpu/msm/adreno_a6xx_rgmu.c @@ -1310,6 +1310,8 @@ int a6xx_rgmu_device_probe(struct platform_device *pdev, timer_setup(&device->idle_timer, rgmu_idle_timer, 0); + adreno_dev->irq_mask = A6XX_INT_MASK; + return 0; } From 473b816ca65f6bff25c7e97c35bd91c26941934c Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Thu, 23 Jul 2020 13:26:58 -0600 Subject: [PATCH 05/14] msm: kgsl: Assign dispatch queue based on context priority Based on context priority, assign a dispatch queue to a context at context registration time. The dispatch queue id is decided in a round robin fashion. Change-Id: I8f9265aa99c22d941397fc204b0a6d8a2b25ec92 Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 49 ++++++++++++++++++++--- drivers/gpu/msm/kgsl_device.h | 5 +++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index b61c54197366..42c741e6a0f0 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -31,6 +31,20 @@ .write_index = 0, \ } +static struct dq_info { + /** @max_dq: Maximum number of dispatch queues per RB level */ + u32 max_dq; + /** @base_dq_id: Base dqid for level */ + u32 base_dq_id; + /** @offset: Next dqid to use for roundrobin context assignment */ + u32 offset; +} a6xx_hfi_dqs[KGSL_PRIORITY_MAX_RB_LEVELS] = { + { 4, 0, }, /* RB0 */ + { 4, 4, }, /* RB1 */ + { 3, 8, }, /* RB2 */ + { 3, 11, }, /* RB3 */ +}; + static const char * const memkind_strings[] = { [MEMKIND_GENERIC] = "GMU GENERIC", [MEMKIND_RB] = "GMU RB", @@ -781,6 +795,25 @@ int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev) #define CTXT_FLAG_PREEMPT_STYLE_RB 1 #define CTXT_FLAG_PREEMPT_STYLE_FG 2 +static u32 get_next_dq(u32 priority) +{ + struct dq_info *info = &a6xx_hfi_dqs[priority]; + u32 next = info->base_dq_id + info->offset; + + info->offset = (info->offset + 1) % info->max_dq; + + return next; +} + +static u32 get_dq_id(u32 priority) +{ + u32 level = priority / KGSL_PRIORITY_MAX_RB_LEVELS; + + level = min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1); + + return get_next_dq(level); +} + static int send_context_register(struct adreno_device *adreno_dev, struct kgsl_context *context) { @@ -834,15 +867,17 @@ static int hfi_context_register(struct adreno_device *adreno_dev, } ret = send_context_pointers(adreno_dev, context); - if (ret) + if (ret) { dev_err(&gmu->pdev->dev, "Unable to register context %d pointers: %d\n", context->id, ret); + return ret; + } - if (!ret) - context->gmu_registered = true; + context->gmu_registered = true; + context->gmu_dispatch_queue = get_dq_id(context->priority); - return ret; + return 0; } #define HFI_DSP_IRQ_BASE 2 @@ -898,7 +933,9 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, } } - ret = a6xx_hfi_queue_write(adreno_dev, HFI_DSP_ID_0, (u32 *)cmd); + ret = a6xx_hfi_queue_write(adreno_dev, + HFI_DSP_ID_0 + drawobj->context->gmu_dispatch_queue, + (u32 *)cmd); if (ret) goto free; @@ -910,7 +947,7 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, /* Send interrupt to GMU to receive the message */ gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_HOST2GMU_INTR_SET, - DISPQ_IRQ_BIT(0)); + DISPQ_IRQ_BIT(drawobj->context->gmu_dispatch_queue)); free: kvfree(cmd); diff --git a/drivers/gpu/msm/kgsl_device.h b/drivers/gpu/msm/kgsl_device.h index 836b49c81c57..f0fac46d2dba 100644 --- a/drivers/gpu/msm/kgsl_device.h +++ b/drivers/gpu/msm/kgsl_device.h @@ -397,6 +397,11 @@ struct kgsl_context { unsigned int total_fault_count; unsigned int last_faulted_cmd_ts; bool gmu_registered; + /** + * @gmu_dispatch_queue: dispatch queue id to which this context will be + * submitted + */ + u32 gmu_dispatch_queue; }; #define _context_comm(_c) \ From e6245a54c34cbd24871cfb8d0f9abf0b8071dd61 Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Thu, 23 Jul 2020 13:39:39 -0600 Subject: [PATCH 06/14] msm: kgsl: Add the f2h daemon This daemon is a bottom half handler for asynchronous packets originating from GMU for example TS_RETIRE, CONTEXT_BAD and F2H_MSG_GMU_CNTR_REGISTER. The way it works is -the interrupt handler snoops all the packets coming from GMU, processes the acks inline but copies the aforementioned packets to a queue and wakes up the daemon to process those. Change-Id: Ib2c9d470a9c4c64bc44409015201adaff398edd8 Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 75 ++++++++++++++++++++++- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h | 7 ++- drivers/gpu/msm/adreno_hwsched.c | 2 +- drivers/gpu/msm/adreno_hwsched.h | 8 +++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index 42c741e6a0f0..c00d6d1bf554 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -15,6 +15,9 @@ #define HFI_QUEUE_MAX (HFI_QUEUE_DEFAULT_CNT + HFI_QUEUE_DISPATCH_MAX_CNT) +/* Use a kmem cache to speed up allocations for f2h packets */ +static struct kmem_cache *f2h_cache; + #define DEFINE_QHDR(gmuaddr, id, prio) \ {\ .status = 1, \ @@ -145,10 +148,38 @@ static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd) MSG_HDR_GET_SEQNUM(waiters[i])); } +struct f2h_packet { + /** @rcvd: the contents of the fw to host packet */ + u32 rcvd[MAX_RCVD_SIZE]; + /** @node: To add to the fw to host msg list */ + struct llist_node node; +}; + +static void add_f2h_packet(struct adreno_device *adreno_dev, u32 *msg) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct f2h_packet *pkt = kmem_cache_alloc(f2h_cache, GFP_ATOMIC); + u32 size = MSG_HDR_GET_SIZE(msg[0]) << 2; + + if (!pkt) + return; + + if (size > sizeof(pkt->rcvd)) + dev_err_ratelimited(&gmu->pdev->dev, + "f2h packet too big: %d allowed: %d\n", + size, sizeof(pkt->rcvd)); + + memcpy(pkt->rcvd, msg, min_t(u32, size, sizeof(pkt->rcvd))); + + llist_add(&pkt->node, &hfi->f2h_msglist); +} + static void process_msgq_irq(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); u32 rcvd[MAX_RCVD_SIZE]; + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); while (a6xx_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd)) > 0) { @@ -157,8 +188,12 @@ static void process_msgq_irq(struct adreno_device *adreno_dev) * because hfi sending thread waits for completion while * holding the device mutex */ - if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) + if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) { a6xx_receive_ack_async(adreno_dev, rcvd); + } else { + add_f2h_packet(adreno_dev, rcvd); + wake_up_interruptible(&hfi->f2h_wq); + } } } @@ -749,6 +784,36 @@ int a6xx_hwsched_cp_init(struct adreno_device *adreno_dev) return ret; } +static int hfi_f2h_main(void *arg) +{ + struct adreno_device *adreno_dev = arg; + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct llist_node *list; + struct f2h_packet *pkt, *tmp; + + while (!kthread_should_stop()) { + wait_event_interruptible(hfi->f2h_wq, + (!llist_empty(&hfi->f2h_msglist) && + !kthread_should_stop())); + + if (kthread_should_stop()) + break; + + list = llist_del_all(&hfi->f2h_msglist); + + list = llist_reverse_order(list); + + llist_for_each_entry_safe(pkt, tmp, list, node) { + if (MSG_HDR_GET_ID(pkt->rcvd[0]) == F2H_MSG_TS_RETIRE) + adreno_hwsched_trigger(adreno_dev); + + kmem_cache_free(f2h_cache, pkt); + } + } + + return 0; +} + int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -768,6 +833,14 @@ int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev) INIT_LIST_HEAD(&hw_hfi->msglist); + init_llist_head(&hw_hfi->f2h_msglist); + + init_waitqueue_head(&hw_hfi->f2h_wq); + + hw_hfi->f2h_task = kthread_run(hfi_f2h_main, adreno_dev, "gmu_f2h"); + + f2h_cache = KMEM_CACHE(f2h_packet, 0); + return 0; } diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index 7a8e1ec48850..97f62fb68b1b 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -113,7 +113,12 @@ struct a6xx_hwsched_hfi { rwlock_t msglock; /** @msglist: List of un-ACKed hfi packets */ struct list_head msglist; - + /** @f2h_task: Task for processing gmu fw to host packets */ + struct task_struct *f2h_task; + /** @f2h_msglist: List of gmu fw to host packets */ + struct llist_head f2h_msglist; + /** @f2h_wq: Waitqueue for the f2h_task */ + wait_queue_head_t f2h_wq; }; struct kgsl_drawobj_cmd; diff --git a/drivers/gpu/msm/adreno_hwsched.c b/drivers/gpu/msm/adreno_hwsched.c index 8d30e49b8bfc..1f015e4fbde4 100644 --- a/drivers/gpu/msm/adreno_hwsched.c +++ b/drivers/gpu/msm/adreno_hwsched.c @@ -526,7 +526,7 @@ static void hwsched_issuecmds(struct adreno_device *adreno_dev) hwsched_handle_jobs(adreno_dev, i); } -static void adreno_hwsched_trigger(struct adreno_device *adreno_dev) +void adreno_hwsched_trigger(struct adreno_device *adreno_dev) { struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); diff --git a/drivers/gpu/msm/adreno_hwsched.h b/drivers/gpu/msm/adreno_hwsched.h index 49d261ef77d0..7e1256c5e7a0 100644 --- a/drivers/gpu/msm/adreno_hwsched.h +++ b/drivers/gpu/msm/adreno_hwsched.h @@ -30,6 +30,14 @@ enum adreno_hwsched_flags { ADRENO_HWSCHED_POWER = 0, }; +/** + * adreno_hwsched_trigger - Function to schedule the hwsched thread + * @adreno_dev: A handle to adreno device + * + * Schedule the hw dispatcher for retiring and submitting command objects + */ +void adreno_hwsched_trigger(struct adreno_device *adreno_dev); + /** * adreno_hwsched_queue_cmds() - Queue a new draw object in the context * @dev_priv: Pointer to the device private struct From d8dec295ebc1a318c3d4ed94bced3ada472a835f Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Fri, 24 Jul 2020 11:54:16 -0600 Subject: [PATCH 07/14] msm: kgsl: Add support for kernel profiling in hw scheduler Enable kernel command profiling by sending the HFI feature flag to GMU. Add support for the sync event which maps GPU/GMU ticks to ftrace time. Record the GPU ticks when command is submitted to the HFI dispatch queue. The ticks corresponding to submission to the ringbuffer, start, retire on GPU/GMU will be conveyed to kgsl by GMU through the HFI retire packet. Modify the trace events to log this information appropriately. Change-Id: Ib6b5a7858848d8d409925015aad719322de8c12d Signed-off-by: Sushmita Susheelendra Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_hfi.h | 13 ++- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 100 +++++++++++++++++++++- drivers/gpu/msm/adreno_dispatch.c | 65 ++++++++++---- drivers/gpu/msm/adreno_ringbuffer.c | 6 +- drivers/gpu/msm/adreno_trace.h | 90 ++++++++++--------- drivers/gpu/msm/kgsl.h | 42 +++++++++ 6 files changed, 248 insertions(+), 68 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index 17a6ab432339..df52b27fff3b 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -60,6 +60,7 @@ #define HFI_FEATURE_BCL 11 #define HFI_FEATURE_ACD 12 #define HFI_FEATURE_DIDT 13 +#define HFI_FEATURE_KPROF 14 #define HFI_VALUE_FT_POLICY 100 #define HFI_VALUE_RB_MAX_CMDS 101 @@ -481,10 +482,14 @@ struct hfi_ts_notify_cmd { /* F2H */ struct hfi_ts_retire_cmd { - uint32_t hdr; - uint32_t ctxt_id; - uint32_t ts; - uint32_t ret; + u32 hdr; + u32 ctxt_id; + u32 ts; + u32 type; + u64 submitted_to_rb; + u64 sop; + u64 eop; + u64 retired_on_gmu; } __packed; /* H2F */ diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index c00d6d1bf554..75e258deefaa 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -4,11 +4,13 @@ */ #include +#include #include "adreno.h" #include "adreno_a6xx.h" #include "adreno_a6xx_hwsched.h" #include "adreno_pm4types.h" +#include "adreno_trace.h" #include "kgsl_device.h" #include "kgsl_pwrctrl.h" #include "kgsl_trace.h" @@ -148,6 +150,35 @@ static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd) MSG_HDR_GET_SEQNUM(waiters[i])); } +static u32 get_level(u32 priority) +{ + u32 level = priority / KGSL_PRIORITY_MAX_RB_LEVELS; + + return min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1); +} + +static void log_profiling_info(struct adreno_device *adreno_dev, u32 *rcvd) +{ + struct hfi_ts_retire_cmd *cmd = (struct hfi_ts_retire_cmd *)rcvd; + struct kgsl_context *context; + struct retire_info info = {0}; + + context = kgsl_context_get(KGSL_DEVICE(adreno_dev), cmd->ctxt_id); + if (context == NULL) + return; + + info.timestamp = cmd->ts; + info.rb_id = get_level(context->priority); + info.gmu_dispatch_queue = context->gmu_dispatch_queue; + info.submitted_to_rb = cmd->submitted_to_rb; + info.sop = cmd->sop; + info.eop = cmd->eop; + info.retired_on_gmu = cmd->retired_on_gmu; + + trace_adreno_cmdbatch_retired(context, &info, 0, 0, 0); + kgsl_context_put(context); +} + struct f2h_packet { /** @rcvd: the contents of the fw to host packet */ u32 rcvd[MAX_RCVD_SIZE]; @@ -692,6 +723,10 @@ int a6xx_hwsched_hfi_start(struct adreno_device *adreno_dev) if (ret) goto err; + ret = a6xx_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_KPROF, 1, 0); + if (ret) + return ret; + ret = a6xx_hfi_send_core_fw_start(adreno_dev); if (ret) goto err; @@ -784,6 +819,12 @@ int a6xx_hwsched_cp_init(struct adreno_device *adreno_dev) return ret; } +static void process_ts_retire(struct adreno_device *adreno_dev, u32 *rcvd) +{ + log_profiling_info(adreno_dev, rcvd); + adreno_hwsched_trigger(adreno_dev); +} + static int hfi_f2h_main(void *arg) { struct adreno_device *adreno_dev = arg; @@ -805,7 +846,7 @@ static int hfi_f2h_main(void *arg) llist_for_each_entry_safe(pkt, tmp, list, node) { if (MSG_HDR_GET_ID(pkt->rcvd[0]) == F2H_MSG_TS_RETIRE) - adreno_hwsched_trigger(adreno_dev); + process_ts_retire(adreno_dev, pkt->rcvd); kmem_cache_free(f2h_cache, pkt); } @@ -844,6 +885,54 @@ int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev) return 0; } +static void add_profile_events(struct adreno_device *adreno_dev, + struct kgsl_drawobj *drawobj, struct adreno_submit_time *time) +{ + unsigned long flags; + u64 time_in_s; + unsigned long time_in_ns; + struct kgsl_context *context = drawobj->context; + struct submission_info info = {0}; + + /* + * Here we are attempting to create a mapping between the + * GPU time domain (alwayson counter) and the CPU time domain + * (local_clock) by sampling both values as close together as + * possible. This is useful for many types of debugging and + * profiling. In order to make this mapping as accurate as + * possible, we must turn off interrupts to avoid running + * interrupt handlers between the two samples. + */ + + local_irq_save(flags); + + /* Read always on registers */ + time->ticks = a6xx_read_alwayson(adreno_dev); + + /* Trace the GPU time to create a mapping to ftrace time */ + trace_adreno_cmdbatch_sync(context->id, context->priority, + drawobj->timestamp, time->ticks); + + /* Get the kernel clock for time since boot */ + time->ktime = local_clock(); + + /* Get the timeofday for the wall time (for the user) */ + getnstimeofday(&time->utime); + + local_irq_restore(flags); + + /* Return kernel clock time to the client if requested */ + time_in_s = time->ktime; + time_in_ns = do_div(time_in_s, 1000000000); + + info.inflight = -1; + info.rb_id = get_level(context->priority); + info.gmu_dispatch_queue = context->gmu_dispatch_queue; + + trace_adreno_cmdbatch_submitted(drawobj, &info, time->ticks, + (unsigned long) time_in_s, time_in_ns / 1000, 0); +} + #define CTXT_FLAG_PMODE 0x00000001 #define CTXT_FLAG_SWITCH_INTERNAL 0x00000002 #define CTXT_FLAG_SWITCH 0x00000008 @@ -880,9 +969,7 @@ static u32 get_next_dq(u32 priority) static u32 get_dq_id(u32 priority) { - u32 level = priority / KGSL_PRIORITY_MAX_RB_LEVELS; - - level = min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1); + u32 level = get_level(priority); return get_next_dq(level); } @@ -967,6 +1054,7 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); struct hfi_issue_ib *issue_ib; struct hfi_submit_cmd *cmd; + struct adreno_submit_time time = {0}; ret = hfi_context_register(adreno_dev, drawobj->context); if (ret) @@ -1018,6 +1106,10 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, */ wmb(); + add_profile_events(adreno_dev, drawobj, &time); + + cmdobj->submit_ticks = time.ticks; + /* Send interrupt to GMU to receive the message */ gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_HOST2GMU_INTR_SET, DISPQ_IRQ_BIT(drawobj->context->gmu_dispatch_queue)); diff --git a/drivers/gpu/msm/adreno_dispatch.c b/drivers/gpu/msm/adreno_dispatch.c index ed4708215e6a..ccf2925bb11b 100644 --- a/drivers/gpu/msm/adreno_dispatch.c +++ b/drivers/gpu/msm/adreno_dispatch.c @@ -271,6 +271,8 @@ static void _retire_timestamp(struct kgsl_drawobj *drawobj) struct kgsl_context *context = drawobj->context; struct adreno_context *drawctxt = ADRENO_CONTEXT(context); struct kgsl_device *device = context->device; + struct adreno_ringbuffer *rb = drawctxt->rb; + struct retire_info info = {0}; /* * Write the start and end timestamp to the memstore to keep the @@ -284,21 +286,29 @@ static void _retire_timestamp(struct kgsl_drawobj *drawobj) KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp), drawobj->timestamp); - /* Retire pending GPU events for the object */ kgsl_process_event_group(device, &context->events); + info.inflight = -1; + info.rb_id = rb->id; + info.wptr = rb->wptr; + info.timestamp = drawobj->timestamp; + /* * For A3xx we still get the rptr from the CP_RB_RPTR instead of * rptr scratch out address. At this point GPU clocks turned off. * So avoid reading GPU register directly for A3xx. */ - if (adreno_is_a3xx(ADRENO_DEVICE(device))) - trace_adreno_cmdbatch_retired(drawobj, -1, 0, 0, drawctxt->rb, - 0, 0); - else - trace_adreno_cmdbatch_retired(drawobj, -1, 0, 0, drawctxt->rb, - adreno_get_rptr(drawctxt->rb), 0); + if (adreno_is_a3xx(ADRENO_DEVICE(device))) { + trace_adreno_cmdbatch_retired(context, &info, + drawobj->flags, rb->dispatch_q.inflight, 0); + } else { + info.rptr = adreno_get_rptr(rb); + + trace_adreno_cmdbatch_retired(context, &info, + drawobj->flags, rb->dispatch_q.inflight, 0); + } + kgsl_drawobj_destroy(drawobj); } @@ -542,6 +552,7 @@ static int sendcmd(struct adreno_device *adreno_dev, uint64_t secs = 0; unsigned long nsecs = 0; int ret; + struct submission_info info = {0}; mutex_lock(&device->mutex); if (adreno_gpu_halt(adreno_dev) != 0) { @@ -650,9 +661,15 @@ static int sendcmd(struct adreno_device *adreno_dev, dispatch_q->expires = jiffies + msecs_to_jiffies(adreno_drawobj_timeout); - trace_adreno_cmdbatch_submitted(drawobj, (int) dispatcher->inflight, - time.ticks, (unsigned long) secs, nsecs / 1000, drawctxt->rb, - adreno_get_rptr(drawctxt->rb)); + info.inflight = (int) dispatcher->inflight; + info.rb_id = drawctxt->rb->id; + info.rptr = adreno_get_rptr(drawctxt->rb); + info.wptr = drawctxt->rb->wptr; + info.gmu_dispatch_queue = -1; + + trace_adreno_cmdbatch_submitted(drawobj, &info, + time.ticks, (unsigned long) secs, nsecs / 1000, + dispatch_q->inflight); mutex_unlock(&device->mutex); @@ -2335,7 +2352,9 @@ static void retire_cmdobj(struct adreno_device *adreno_dev, struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context); + struct adreno_ringbuffer *rb = drawctxt->rb; uint64_t start = 0, end = 0; + struct retire_info info = {0}; if (cmdobj->fault_recovery != 0) { set_bit(ADRENO_CONTEXT_FAULT, &drawobj->context->priv); @@ -2345,20 +2364,28 @@ static void retire_cmdobj(struct adreno_device *adreno_dev, if (test_bit(CMDOBJ_PROFILE, &cmdobj->priv)) cmdobj_profile_ticks(adreno_dev, cmdobj, &start, &end); + info.inflight = (int)dispatcher->inflight; + info.rb_id = rb->id; + info.wptr = rb->wptr; + info.timestamp = drawobj->timestamp; + info.sop = start; + info.eop = end; + /* * For A3xx we still get the rptr from the CP_RB_RPTR instead of * rptr scratch out address. At this point GPU clocks turned off. * So avoid reading GPU register directly for A3xx. */ - if (adreno_is_a3xx(adreno_dev)) - trace_adreno_cmdbatch_retired(drawobj, - (int) dispatcher->inflight, start, end, - ADRENO_DRAWOBJ_RB(drawobj), 0, cmdobj->fault_recovery); - else - trace_adreno_cmdbatch_retired(drawobj, - (int) dispatcher->inflight, start, end, - ADRENO_DRAWOBJ_RB(drawobj), - adreno_get_rptr(drawctxt->rb), cmdobj->fault_recovery); + if (adreno_is_a3xx(adreno_dev)) { + trace_adreno_cmdbatch_retired(drawobj->context, &info, + drawobj->flags, rb->dispatch_q.inflight, + cmdobj->fault_recovery); + } else { + info.rptr = adreno_get_rptr(rb); + trace_adreno_cmdbatch_retired(drawobj->context, &info, + drawobj->flags, rb->dispatch_q.inflight, + cmdobj->fault_recovery); + } drawctxt->submit_retire_ticks[drawctxt->ticks_index] = end - cmdobj->submit_ticks; diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index fe158328fba7..a15f4188b7f4 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -34,6 +34,9 @@ static void adreno_get_submit_time(struct adreno_device *adreno_dev, { struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); unsigned long flags; + struct adreno_context *drawctxt = rb->drawctxt_active; + struct kgsl_context *context = &drawctxt->base; + /* * Here we are attempting to create a mapping between the * GPU time domain (alwayson counter) and the CPU time domain @@ -49,7 +52,8 @@ static void adreno_get_submit_time(struct adreno_device *adreno_dev, time->ticks = gpudev->read_alwayson(adreno_dev); /* Trace the GPU time to create a mapping to ftrace time */ - trace_adreno_cmdbatch_sync(rb->drawctxt_active, time->ticks); + trace_adreno_cmdbatch_sync(context->id, context->priority, + drawctxt->timestamp, time->ticks); /* Get the kernel clock for time since boot */ time->ktime = local_clock(); diff --git a/drivers/gpu/msm/adreno_trace.h b/drivers/gpu/msm/adreno_trace.h index 5af5f397af60..bd9052e65c64 100644 --- a/drivers/gpu/msm/adreno_trace.h +++ b/drivers/gpu/msm/adreno_trace.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. */ #if !defined(_ADRENO_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) @@ -54,10 +54,10 @@ TRACE_EVENT(adreno_cmdbatch_queued, ); TRACE_EVENT(adreno_cmdbatch_submitted, - TP_PROTO(struct kgsl_drawobj *drawobj, int inflight, uint64_t ticks, - unsigned long secs, unsigned long usecs, - struct adreno_ringbuffer *rb, unsigned int rptr), - TP_ARGS(drawobj, inflight, ticks, secs, usecs, rb, rptr), + TP_PROTO(struct kgsl_drawobj *drawobj, struct submission_info *info, + uint64_t ticks, unsigned long secs, unsigned long usecs, + int q_inflight), + TP_ARGS(drawobj, info, ticks, secs, usecs, q_inflight), TP_STRUCT__entry( __field(unsigned int, id) __field(unsigned int, timestamp) @@ -71,39 +71,40 @@ TRACE_EVENT(adreno_cmdbatch_submitted, __field(unsigned int, rptr) __field(unsigned int, wptr) __field(int, q_inflight) + __field(int, dispatch_queue) ), TP_fast_assign( __entry->id = drawobj->context->id; __entry->timestamp = drawobj->timestamp; - __entry->inflight = inflight; + __entry->inflight = info->inflight; __entry->flags = drawobj->flags; __entry->ticks = ticks; __entry->secs = secs; __entry->usecs = usecs; __entry->prio = drawobj->context->priority; - __entry->rb_id = rb->id; - __entry->rptr = rptr; - __entry->wptr = rb->wptr; - __entry->q_inflight = rb->dispatch_q.inflight; + __entry->rb_id = info->rb_id; + __entry->rptr = info->rptr; + __entry->wptr = info->wptr; + __entry->q_inflight = q_inflight; + __entry->dispatch_queue = info->gmu_dispatch_queue; ), TP_printk( - "ctx=%u ctx_prio=%d ts=%u inflight=%d flags=%s ticks=%lld time=%lu.%0lu rb_id=%d r/w=%x/%x, q_inflight=%d", + "ctx=%u ctx_prio=%d ts=%u inflight=%d flags=%s ticks=%lld time=%lu.%0lu rb_id=%d r/w=%x/%x, q_inflight=%d dq_id=%d", __entry->id, __entry->prio, __entry->timestamp, __entry->inflight, __entry->flags ? __print_flags(__entry->flags, "|", KGSL_DRAWOBJ_FLAGS) : "none", __entry->ticks, __entry->secs, __entry->usecs, __entry->rb_id, __entry->rptr, __entry->wptr, - __entry->q_inflight + __entry->q_inflight, __entry->dispatch_queue ) ); TRACE_EVENT(adreno_cmdbatch_retired, - TP_PROTO(struct kgsl_drawobj *drawobj, int inflight, - uint64_t start, uint64_t retire, - struct adreno_ringbuffer *rb, unsigned int rptr, - unsigned long fault_recovery), - TP_ARGS(drawobj, inflight, start, retire, rb, rptr, fault_recovery), + TP_PROTO(struct kgsl_context *context, struct retire_info *info, + unsigned int flags, int q_inflight, + unsigned long fault_recovery), + TP_ARGS(context, info, flags, q_inflight, fault_recovery), TP_STRUCT__entry( __field(unsigned int, id) __field(unsigned int, timestamp) @@ -118,41 +119,50 @@ TRACE_EVENT(adreno_cmdbatch_retired, __field(unsigned int, wptr) __field(int, q_inflight) __field(unsigned long, fault_recovery) - ), + __field(unsigned int, dispatch_queue) + __field(uint64_t, submitted_to_rb) + __field(uint64_t, retired_on_gmu) + ), TP_fast_assign( - __entry->id = drawobj->context->id; - __entry->timestamp = drawobj->timestamp; - __entry->inflight = inflight; + __entry->id = context->id; + __entry->timestamp = info->timestamp; + __entry->inflight = info->inflight; __entry->recovery = fault_recovery; - __entry->flags = drawobj->flags; - __entry->start = start; - __entry->retire = retire; - __entry->prio = drawobj->context->priority; - __entry->rb_id = rb->id; - __entry->rptr = rptr; - __entry->wptr = rb->wptr; - __entry->q_inflight = rb->dispatch_q.inflight; - ), + __entry->flags = flags; + __entry->start = info->sop; + __entry->retire = info->eop; + __entry->prio = context->priority; + __entry->rb_id = info->rb_id; + __entry->rptr = info->rptr; + __entry->wptr = info->wptr; + __entry->q_inflight = q_inflight; + __entry->dispatch_queue = info->gmu_dispatch_queue; + __entry->submitted_to_rb = info->submitted_to_rb; + __entry->retired_on_gmu = info->retired_on_gmu; + ), + TP_printk( - "ctx=%u ctx_prio=%d ts=%u inflight=%d recovery=%s flags=%s start=%lld retire=%lld rb_id=%d, r/w=%x/%x, q_inflight=%d", + "ctx=%u prio=%d ts=%u inflight=%d recovery=%s flags=%s start=%llu retire=%llu rb_id=%d, r/w=%x/%x, q_inflight=%d, dq_id=%u, submitted_to_rb=%llu, retired_on_gmu=%llu", __entry->id, __entry->prio, __entry->timestamp, __entry->inflight, __entry->recovery ? - __print_flags(__entry->recovery, "|", + __print_flags(__entry->fault_recovery, "|", ADRENO_FT_TYPES) : "none", __entry->flags ? __print_flags(__entry->flags, "|", KGSL_DRAWOBJ_FLAGS) : "none", __entry->start, __entry->retire, __entry->rb_id, __entry->rptr, __entry->wptr, - __entry->q_inflight - ) + __entry->q_inflight, + __entry->dispatch_queue, + __entry->submitted_to_rb, __entry->retired_on_gmu + ) ); TRACE_EVENT(adreno_cmdbatch_sync, - TP_PROTO(struct adreno_context *drawctxt, - uint64_t ticks), - TP_ARGS(drawctxt, ticks), + TP_PROTO(unsigned int ctx_id, unsigned int ctx_prio, + unsigned int timestamp, uint64_t ticks), + TP_ARGS(ctx_id, ctx_prio, timestamp, ticks), TP_STRUCT__entry( __field(unsigned int, id) __field(unsigned int, timestamp) @@ -160,10 +170,10 @@ TRACE_EVENT(adreno_cmdbatch_sync, __field(int, prio) ), TP_fast_assign( - __entry->id = drawctxt->base.id; - __entry->timestamp = drawctxt->timestamp; + __entry->id = ctx_id; + __entry->timestamp = timestamp; __entry->ticks = ticks; - __entry->prio = drawctxt->base.priority; + __entry->prio = ctx_prio; ), TP_printk( "ctx=%u ctx_prio=%d ts=%u ticks=%lld", diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h index 1be47571ddb0..de0cc2116106 100644 --- a/drivers/gpu/msm/kgsl.h +++ b/drivers/gpu/msm/kgsl.h @@ -349,6 +349,48 @@ struct sparse_bind_object { uint64_t flags; }; +/** + * struct submission_info - Container for submission statistics + * @inflight: Number of commands that are inflight + * @rb_id: id of the ringbuffer to which this submission is made + * @rptr: Read pointer of the ringbuffer + * @wptr: Write pointer of the ringbuffer + * @gmu_dispatch_queue: GMU dispach queue to which this submission is made + */ +struct submission_info { + int inflight; + u32 rb_id; + u32 rptr; + u32 wptr; + u32 gmu_dispatch_queue; +}; + +/** + * struct retire_info - Container for retire statistics + * @inflight: NUmber of commands that are inflight + * @rb_id: id of the ringbuffer to which this submission is made + * @rptr: Read pointer of the ringbuffer + * @wptr: Write pointer of the ringbuffer + * @gmu_dispatch_queue: GMU dispach queue to which this submission is made + * @timestamp: Timestamp of submission that retired + * @submitted_to_rb: AO ticks when GMU put this submission on ringbuffer + * @sop: AO ticks when GPU started procssing this submission + * @eop: AO ticks when GPU finished this submission + * @retired_on_gmu: AO ticks when GMU retired this submission + */ +struct retire_info { + int inflight; + int rb_id; + u32 rptr; + u32 wptr; + u32 gmu_dispatch_queue; + u32 timestamp; + u64 submitted_to_rb; + u64 sop; + u64 eop; + u64 retired_on_gmu; +}; + long kgsl_ioctl_device_getproperty(struct kgsl_device_private *dev_priv, unsigned int cmd, void *data); long kgsl_ioctl_device_setproperty(struct kgsl_device_private *dev_priv, From c7f6786eb936017a85ec9357a878bd4238eb70f0 Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Fri, 24 Jul 2020 12:25:35 -0600 Subject: [PATCH 08/14] msm: kgsl: Add user command profiling with hw scheduling User mode specifies a profiling buffer as part of the ib list. This ib is passed on to the GMU as part of the HFI packet to issue commands. The GMU will in turn create a special user profiling IB to have the GPU write the command start and retire ticks into the user specified buffer. The submit to HFI time is recorded in the user specified buffer by kgsl. Change-Id: I800b4a23876f6f0707fb7a942e97d70a2d712aeb Signed-off-by: Sushmita Susheelendra Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno.c | 39 ++++++++++++++++++++ drivers/gpu/msm/adreno.h | 10 ++++++ drivers/gpu/msm/adreno_a6xx_hfi.h | 2 ++ drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 33 +++++++++++++---- drivers/gpu/msm/adreno_ringbuffer.c | 44 ----------------------- 5 files changed, 77 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index be8f4b78f9bb..5039adfebda6 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -3201,6 +3201,45 @@ void adreno_cx_misc_regrmw(struct adreno_device *adreno_dev, adreno_cx_misc_regwrite(adreno_dev, offsetwords, val | bits); } +void adreno_profile_submit_time(struct adreno_submit_time *time) +{ + struct kgsl_drawobj *drawobj; + struct kgsl_drawobj_cmd *cmdobj; + struct kgsl_mem_entry *entry; + struct kgsl_drawobj_profiling_buffer *profile_buffer; + + drawobj = time->drawobj; + if (drawobj == NULL) + return; + + cmdobj = CMDOBJ(drawobj); + entry = cmdobj->profiling_buf_entry; + if (!entry) + return; + + profile_buffer = kgsl_gpuaddr_to_vaddr(&entry->memdesc, + cmdobj->profiling_buffer_gpuaddr); + + if (profile_buffer == NULL) + return; + + /* Return kernel clock time to the client if requested */ + if (drawobj->flags & KGSL_DRAWOBJ_PROFILING_KTIME) { + u64 secs = time->ktime; + + profile_buffer->wall_clock_ns = + do_div(secs, NSEC_PER_SEC); + profile_buffer->wall_clock_s = secs; + } else { + profile_buffer->wall_clock_s = time->utime.tv_sec; + profile_buffer->wall_clock_ns = time->utime.tv_nsec; + } + + profile_buffer->gpu_ticks_queued = time->ticks; + + kgsl_memdesc_unmap(&entry->memdesc); +} + /** * adreno_waittimestamp - sleep while waiting for the specified timestamp * @device - pointer to a KGSL device structure diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 9a97d1627b67..0b60e4fe848a 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -1925,4 +1925,14 @@ void gmu_fault_snapshot(struct kgsl_device *device); * Return: 0 on success or negative error on failure */ int adreno_suspend_context(struct kgsl_device *device); + +/* + * adreno_profile_submit_time - Populate profiling buffer with timestamps + * @time: Container for the statistics + * + * Populate the draw object user profiling buffer with the timestamps + * recored in the adreno_submit_time structure at the time of draw object + * submission. + */ +void adreno_profile_submit_time(struct adreno_submit_time *time); #endif /*__ADRENO_H */ diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index df52b27fff3b..02c6fb00049a 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -529,6 +529,8 @@ struct hfi_submit_cmd { u32 ctxt_id; u32 flags; u32 ts; + u32 profile_gpuaddr_lo; + u32 profile_gpuaddr_hi; u32 numibs; } __packed; diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index 75e258deefaa..76aaa8ecdcaa 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -1084,16 +1084,32 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, cmd->ts = drawobj->timestamp; cmd->numibs = numibs; - if (numibs) { - issue_ib = (struct hfi_issue_ib *)&cmd[1]; + if (!numibs) + goto skipib; - list_for_each_entry(ib, &cmdobj->cmdlist, node) { - issue_ib->addr = ib->gpuaddr; - issue_ib->size = ib->size; - issue_ib++; - } + if ((drawobj->flags & KGSL_DRAWOBJ_PROFILING) && + !cmdobj->profiling_buf_entry) { + + time.drawobj = drawobj; + + cmd->profile_gpuaddr_lo = + lower_32_bits(cmdobj->profiling_buffer_gpuaddr); + cmd->profile_gpuaddr_hi = + upper_32_bits(cmdobj->profiling_buffer_gpuaddr); + + /* Indicate to GMU to do user profiling for this submission */ + cmd->flags |= BIT(4); } + issue_ib = (struct hfi_issue_ib *)&cmd[1]; + + list_for_each_entry(ib, &cmdobj->cmdlist, node) { + issue_ib->addr = ib->gpuaddr; + issue_ib->size = ib->size; + issue_ib++; + } + +skipib: ret = a6xx_hfi_queue_write(adreno_dev, HFI_DSP_ID_0 + drawobj->context->gmu_dispatch_queue, (u32 *)cmd); @@ -1114,6 +1130,9 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_HOST2GMU_INTR_SET, DISPQ_IRQ_BIT(drawobj->context->gmu_dispatch_queue)); + /* Put the profiling information in the user profiling buffer */ + adreno_profile_submit_time(&time); + free: kvfree(cmd); diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index a15f4188b7f4..b8556bea2a73 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -115,50 +115,6 @@ static void adreno_ringbuffer_wptr(struct adreno_device *adreno_dev, } } -static void adreno_profile_submit_time(struct adreno_submit_time *time) -{ - struct kgsl_drawobj *drawobj; - struct kgsl_drawobj_cmd *cmdobj; - struct kgsl_mem_entry *entry; - - if (time == NULL) - return; - - drawobj = time->drawobj; - - if (drawobj == NULL) - return; - - cmdobj = CMDOBJ(drawobj); - entry = cmdobj->profiling_buf_entry; - - if (entry) { - struct kgsl_drawobj_profiling_buffer *profile_buffer; - - profile_buffer = kgsl_gpuaddr_to_vaddr(&entry->memdesc, - cmdobj->profiling_buffer_gpuaddr); - - if (profile_buffer == NULL) - return; - - /* Return kernel clock time to the the client if requested */ - if (drawobj->flags & KGSL_DRAWOBJ_PROFILING_KTIME) { - uint64_t secs = time->ktime; - - profile_buffer->wall_clock_ns = - do_div(secs, NSEC_PER_SEC); - profile_buffer->wall_clock_s = secs; - } else { - profile_buffer->wall_clock_s = time->utime.tv_sec; - profile_buffer->wall_clock_ns = time->utime.tv_nsec; - } - - profile_buffer->gpu_ticks_queued = time->ticks; - - kgsl_memdesc_unmap(&entry->memdesc); - } -} - void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb, struct adreno_submit_time *time) { From 27f4d2c02bdc5ceea1fa06824d11d8f70d2d4a1f Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Fri, 24 Jul 2020 12:28:16 -0600 Subject: [PATCH 09/14] msm: kgsl: Get rid of a6xx_hfi_send_req function Instead, directly call a6xx_hfi_send_generic_req. Also, get rid of a6xx_hfi_send_get_value() as it is not used. Change-Id: Idee3af33ff0e5e03a9f974c535f8bc1c7579d492 Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_gmu.c | 18 ++++---- drivers/gpu/msm/adreno_a6xx_hfi.c | 69 +------------------------------ drivers/gpu/msm/adreno_a6xx_hfi.h | 11 ----- 3 files changed, 13 insertions(+), 85 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index e25b409f1f83..785cdfa7e155 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -841,9 +841,11 @@ static int a6xx_gmu_hfi_start_msg(struct adreno_device *adreno_dev) * serves as a better means to identify targets that depend on * legacy firmware. */ - if (!ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) - return a6xx_hfi_send_req(adreno_dev, - H2F_MSG_START, &req); + if (!ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) { + req.hdr = CMD_MSG_HDR(H2F_MSG_START, sizeof(req)); + + return a6xx_hfi_send_generic_req(adreno_dev, &req); + } return 0; @@ -1711,8 +1713,9 @@ static int a6xx_gmu_notify_slumber(struct adreno_device *adreno_dev) .bw = bus_level, }; - ret = a6xx_hfi_send_req(adreno_dev, - H2F_MSG_PREPARE_SLUMBER, &req); + req.hdr = CMD_MSG_HDR(H2F_MSG_PREPARE_SLUMBER, sizeof(req)); + + ret = a6xx_hfi_send_generic_req(adreno_dev, &req); goto out; } @@ -1800,11 +1803,12 @@ static int a6xx_gmu_dcvs_set(struct adreno_device *adreno_dev, return 0; } + req.hdr = CMD_MSG_HDR(H2F_MSG_GX_BW_PERF_VOTE, sizeof(req)); + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) ret = a6xx_gmu_dcvs_nohfi(device, req.freq, req.bw); else - ret = a6xx_hfi_send_req(adreno_dev, H2F_MSG_GX_BW_PERF_VOTE, - &req); + ret = a6xx_hfi_send_generic_req(adreno_dev, &req); if (ret) { dev_err_ratelimited(&gmu->pdev->dev, diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.c b/drivers/gpu/msm/adreno_a6xx_hfi.c index c162fe6c13ed..49e7e31f0f5e 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hfi.c @@ -466,27 +466,6 @@ static int a6xx_hfi_send_dcvstbl_v1(struct adreno_device *adreno_dev) return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } -static int a6xx_hfi_send_get_value(struct adreno_device *adreno_dev, - struct hfi_get_value_req *req) -{ - struct hfi_get_value_cmd *cmd = &req->cmd; - struct pending_cmd ret_cmd; - struct hfi_get_value_reply_cmd *reply = - (struct hfi_get_value_reply_cmd *)ret_cmd.results; - int rc; - - cmd->hdr = CMD_MSG_HDR(H2F_MSG_GET_VALUE, sizeof(*cmd)); - - rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, cmd, &ret_cmd); - if (rc) - return rc; - - memset(&req->data, 0, sizeof(req->data)); - memcpy(&req->data, &reply->data, - (MSG_HDR_GET_SIZE(reply->hdr) - 2) << 2); - return 0; -} - static int a6xx_hfi_send_test(struct adreno_device *adreno_dev) { struct hfi_test_cmd cmd = { @@ -644,6 +623,7 @@ int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev) nvmem_cell_read_u32(&device->pdev->dev, "isense_slope", &slope); + req.hdr = CMD_MSG_HDR(H2F_MSG_SET_VALUE, sizeof(req)); req.type = HFI_VALUE_LM_CS0; req.subtype = 0; req.data = slope; @@ -652,7 +632,7 @@ int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev) device->pwrctrl.throttle_mask); if (!ret) - ret = a6xx_hfi_send_req(adreno_dev, H2F_MSG_SET_VALUE, &req); + ret = a6xx_hfi_send_generic_req(adreno_dev, &req); return ret; } @@ -793,51 +773,6 @@ void a6xx_hfi_stop(struct adreno_device *adreno_dev) } -int a6xx_hfi_send_req(struct adreno_device *adreno_dev, unsigned int id, - void *data) -{ - switch (id) { - case H2F_MSG_GX_BW_PERF_VOTE: { - struct hfi_gx_bw_perf_vote_cmd *cmd = data; - - cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - - return a6xx_hfi_send_generic_req(adreno_dev, cmd); - } - case H2F_MSG_PREPARE_SLUMBER: { - struct hfi_prep_slumber_cmd *cmd = data; - - if (cmd->freq >= MAX_GX_LEVELS || cmd->bw >= MAX_GX_LEVELS) - return -EINVAL; - - cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - - return a6xx_hfi_send_generic_req(adreno_dev, cmd); - } - case H2F_MSG_START: { - struct hfi_start_cmd *cmd = data; - - cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - - return a6xx_hfi_send_generic_req(adreno_dev, cmd); - } - case H2F_MSG_GET_VALUE: { - return a6xx_hfi_send_get_value(adreno_dev, data); - } - case H2F_MSG_SET_VALUE: { - struct hfi_set_value_cmd *cmd = data; - - cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - - return a6xx_hfi_send_generic_req(adreno_dev, cmd); - } - default: - break; - } - - return -EINVAL; -} - /* HFI interrupt handler */ irqreturn_t a6xx_hfi_irq_handler(int irq, void *data) { diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index 02c6fb00049a..1982f8b118da 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -601,17 +601,6 @@ void a6xx_hfi_stop(struct adreno_device *adreno_dev); */ int a6xx_hfi_init(struct adreno_device *adreno_dev); -/** - * a6xx_hfi_send_req - Send an HFI packet to GMU - * @adreno_dev: Pointer to the adreno device - * @id: Packet id to be sent - * @data: Container for the data sent as part of this pcket - * - * Return: 0 on success or negative error on failure - */ -int a6xx_hfi_send_req(struct adreno_device *adreno_dev, - unsigned int id, void *data); - /* Helper function to get to a6xx hfi struct from adreno device */ struct a6xx_hfi *to_a6xx_hfi(struct adreno_device *adreno_dev); From 26b7004ddf2d0f67acdca6046479154e218d7f9b Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Fri, 24 Jul 2020 12:42:30 -0600 Subject: [PATCH 10/14] msm: kgsl: Fix check for oversized HFI messages The check for HFI message size is incorrect as it is done after the higher bits of size are masked out while creating the HFI header. Fix the check so that we do not let oversized messages through. Change-Id: Ic2350da328d21f085bd41656d36c74132c5007ac Signed-off-by: Sushmita Susheelendra Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_gmu.c | 9 ++++--- drivers/gpu/msm/adreno_a6xx_hfi.c | 31 +++++++++++------------ drivers/gpu/msm/adreno_a6xx_hfi.h | 10 ++++++-- drivers/gpu/msm/adreno_a6xx_hwsched.c | 6 +++-- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 25 ++++++++++++------ drivers/gpu/msm/adreno_a6xx_rpmh.c | 6 ++--- 6 files changed, 52 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index 785cdfa7e155..9fb8db6f99ca 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -842,7 +842,7 @@ static int a6xx_gmu_hfi_start_msg(struct adreno_device *adreno_dev) * legacy firmware. */ if (!ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) { - req.hdr = CMD_MSG_HDR(H2F_MSG_START, sizeof(req)); + CMD_MSG_HDR(req, H2F_MSG_START); return a6xx_hfi_send_generic_req(adreno_dev, &req); } @@ -1713,7 +1713,7 @@ static int a6xx_gmu_notify_slumber(struct adreno_device *adreno_dev) .bw = bus_level, }; - req.hdr = CMD_MSG_HDR(H2F_MSG_PREPARE_SLUMBER, sizeof(req)); + CMD_MSG_HDR(req, H2F_MSG_PREPARE_SLUMBER); ret = a6xx_hfi_send_generic_req(adreno_dev, &req); goto out; @@ -1803,7 +1803,7 @@ static int a6xx_gmu_dcvs_set(struct adreno_device *adreno_dev, return 0; } - req.hdr = CMD_MSG_HDR(H2F_MSG_GX_BW_PERF_VOTE, sizeof(req)); + CMD_MSG_HDR(req, H2F_MSG_GX_BW_PERF_VOTE); if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) ret = a6xx_gmu_dcvs_nohfi(device, req.freq, req.bw); @@ -2443,7 +2443,8 @@ static void a6xx_gmu_acd_probe(struct kgsl_device *device, if (!ADRENO_FEATURE(adreno_dev, ADRENO_ACD)) return; - cmd->hdr = CMD_MSG_HDR(H2F_MSG_ACD_TBL, sizeof(*cmd)); + cmd->hdr = CREATE_MSG_HDR(H2F_MSG_ACD_TBL, sizeof(*cmd), HFI_MSG_CMD); + cmd->version = 1; cmd->stride = 1; cmd->enable_by_level = 0; diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.c b/drivers/gpu/msm/adreno_a6xx_hfi.c index 49e7e31f0f5e..49f8d1d35d10 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hfi.c @@ -103,13 +103,6 @@ int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, uint32_t queue_idx, if (hdr->status == HFI_QUEUE_STATUS_DISABLED) return -EINVAL; - if (size > HFI_MAX_MSG_SIZE) { - dev_err(&gmu->pdev->dev, - "Message too big to send: sz=%d, id=%d\n", - size, id); - return -EINVAL; - } - queue = HOST_QUEUE_START_ADDR(gmu->hfi.hfi_mem, queue_idx); trace_kgsl_hfi_send(id, size, MSG_HDR_GET_SEQNUM(*msg)); @@ -359,13 +352,14 @@ static int a6xx_hfi_send_gmu_init(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_gmu_init_cmd cmd = { - .hdr = CMD_MSG_HDR(H2F_MSG_INIT, sizeof(cmd)), .seg_id = 0, .dbg_buffer_addr = (unsigned int) gmu->dump_mem->gmuaddr, .dbg_buffer_size = (unsigned int) gmu->dump_mem->size, .boot_state = 0x1, }; + CMD_MSG_HDR(cmd, H2F_MSG_INIT); + return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } @@ -374,12 +368,13 @@ static int a6xx_hfi_get_fw_version(struct adreno_device *adreno_dev, { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_fw_version_cmd cmd = { - .hdr = CMD_MSG_HDR(H2F_MSG_FW_VER, sizeof(cmd)), .supported_ver = expected_ver, }; int rc; struct pending_cmd ret_cmd; + CMD_MSG_HDR(cmd, H2F_MSG_FW_VER); + memset(&ret_cmd, 0, sizeof(ret_cmd)); rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, &cmd, &ret_cmd); @@ -399,10 +394,11 @@ static int a6xx_hfi_get_fw_version(struct adreno_device *adreno_dev, int a6xx_hfi_send_core_fw_start(struct adreno_device *adreno_dev) { struct hfi_core_fw_start_cmd cmd = { - .hdr = CMD_MSG_HDR(H2F_MSG_CORE_FW_START, sizeof(cmd)), .handle = 0x0, }; + CMD_MSG_HDR(cmd, H2F_MSG_CORE_FW_START); + return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } @@ -425,13 +421,14 @@ int a6xx_hfi_send_feature_ctrl(struct adreno_device *adreno_dev, { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_feature_ctrl_cmd cmd = { - .hdr = CMD_MSG_HDR(H2F_MSG_FEATURE_CTRL, sizeof(cmd)), .feature = feature, .enable = enable, .data = data, }; int ret; + CMD_MSG_HDR(cmd, H2F_MSG_FEATURE_CTRL); + ret = a6xx_hfi_send_generic_req(adreno_dev, &cmd); if (ret) dev_err(&gmu->pdev->dev, @@ -447,12 +444,13 @@ static int a6xx_hfi_send_dcvstbl_v1(struct adreno_device *adreno_dev) struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_dcvstable_cmd *table = &gmu->hfi.dcvs_table; struct hfi_dcvstable_v1_cmd cmd = { - .hdr = CMD_MSG_HDR(H2F_MSG_PERF_TBL, sizeof(cmd)), .gpu_level_num = table->gpu_level_num, .gmu_level_num = table->gmu_level_num, }; int i; + CMD_MSG_HDR(cmd, H2F_MSG_PERF_TBL); + for (i = 0; i < table->gpu_level_num; i++) { cmd.gx_votes[i].vote = table->gx_votes[i].vote; cmd.gx_votes[i].freq = table->gx_votes[i].freq; @@ -468,9 +466,9 @@ static int a6xx_hfi_send_dcvstbl_v1(struct adreno_device *adreno_dev) static int a6xx_hfi_send_test(struct adreno_device *adreno_dev) { - struct hfi_test_cmd cmd = { - .hdr = CMD_MSG_HDR(H2F_MSG_TEST, sizeof(cmd)), - }; + struct hfi_test_cmd cmd; + + CMD_MSG_HDR(cmd, H2F_MSG_TEST); return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } @@ -623,7 +621,8 @@ int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev) nvmem_cell_read_u32(&device->pdev->dev, "isense_slope", &slope); - req.hdr = CMD_MSG_HDR(H2F_MSG_SET_VALUE, sizeof(req)); + CMD_MSG_HDR(req, H2F_MSG_SET_VALUE); + req.type = HFI_VALUE_LM_CS0; req.subtype = 0; req.data = slope; diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index 1982f8b118da..85aa36f9dac0 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -8,7 +8,7 @@ #define HFI_QUEUE_SIZE SZ_4K /* bytes, must be base 4dw */ #define MAX_RCVD_PAYLOAD_SIZE 16 /* dwords */ #define MAX_RCVD_SIZE (MAX_RCVD_PAYLOAD_SIZE + 3) /* dwords */ -#define HFI_MAX_MSG_SIZE (SZ_1K>>2) /* dwords */ +#define HFI_MAX_MSG_SIZE (SZ_1K) #define HFI_CMD_ID 0 #define HFI_MSG_ID 1 @@ -133,7 +133,6 @@ struct hfi_queue_header { /* Size is converted from Bytes to DWords */ #define CREATE_MSG_HDR(id, size, type) \ (((type) << 16) | ((((size) >> 2) & 0xFF) << 8) | ((id) & 0xFF)) -#define CMD_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_CMD) #define ACK_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_ACK) #define HFI_QUEUE_DEFAULT_CNT 3 @@ -569,6 +568,13 @@ struct a6xx_hfi { struct hfi_dcvstable_cmd dcvs_table; }; +#define CMD_MSG_HDR(cmd, id) \ + do { \ + if (WARN_ON((sizeof(cmd)) > HFI_MAX_MSG_SIZE)) \ + return -EMSGSIZE; \ + cmd.hdr = CREATE_MSG_HDR((id), (sizeof(cmd)), HFI_MSG_CMD); \ + } while (0) + struct a6xx_gmu_device; /* a6xx_hfi_irq_handler - IRQ handler for HFI interripts */ diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index 127facf3d300..9d3983874a09 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -199,7 +199,8 @@ static int a6xx_hwsched_notify_slumber(struct adreno_device *adreno_dev) struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_prep_slumber_cmd req; - req.hdr = CMD_MSG_HDR(H2F_MSG_PREPARE_SLUMBER, sizeof(req)); + CMD_MSG_HDR(req, H2F_MSG_PREPARE_SLUMBER); + req.freq = gmu->hfi.dcvs_table.gpu_level_num - pwr->default_pwrlevel - 1; req.bw = pwr->pwrlevels[pwr->default_pwrlevel].bus_freq; @@ -600,7 +601,6 @@ static int a6xx_hwsched_dcvs_set(struct adreno_device *adreno_dev, struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_dcvstable_cmd *table = &gmu->hfi.dcvs_table; struct hfi_gx_bw_perf_vote_cmd req = { - .hdr = CMD_MSG_HDR(H2F_MSG_GX_BW_PERF_VOTE, sizeof(req)), .ack_type = DCVS_ACK_BLOCK, .freq = INVALID_DCVS_IDX, .bw = INVALID_DCVS_IDX, @@ -628,6 +628,8 @@ static int a6xx_hwsched_dcvs_set(struct adreno_device *adreno_dev, if ((req.freq == INVALID_DCVS_IDX) && (req.bw == INVALID_DCVS_IDX)) return 0; + CMD_MSG_HDR(req, H2F_MSG_GX_BW_PERF_VOTE); + ret = a6xx_hfi_send_cmd_async(adreno_dev, &req); if (ret) diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index 76aaa8ecdcaa..524f69edc068 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -583,7 +583,8 @@ static int send_start_msg(struct adreno_device *adreno_dev) u32 rcvd[MAX_RCVD_SIZE]; struct pending_cmd pending_ack = {0}; - cmd.hdr = CMD_MSG_HDR(H2F_MSG_START, sizeof(cmd)); + CMD_MSG_HDR(cmd, H2F_MSG_START); + cmd.hdr = MSG_HDR_SET_SEQNUM(cmd.hdr, seqnum); pending_ack.sent_hdr = cmd.hdr; @@ -775,8 +776,9 @@ static int cp_init(struct adreno_device *adreno_dev) { u32 cmds[A6XX_CP_INIT_DWORDS + 1]; - cmds[0] = CMD_MSG_HDR(H2F_MSG_ISSUE_CMD_RAW, - (A6XX_CP_INIT_DWORDS + 1) << 2); + cmds[0] = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD_RAW, + (A6XX_CP_INIT_DWORDS + 1) << 2, HFI_MSG_CMD); + memcpy(&cmds[1], adreno_dev->cp_init_cmds, A6XX_CP_INIT_DWORDS << 2); return submit_raw_cmds(adreno_dev, cmds, @@ -787,7 +789,9 @@ static int send_switch_to_unsecure(struct adreno_device *adreno_dev) { u32 cmds[3]; - cmds[0] = CMD_MSG_HDR(H2F_MSG_ISSUE_CMD_RAW, sizeof(cmds)); + cmds[0] = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD_RAW, sizeof(cmds), + HFI_MSG_CMD); + cmds[1] = cp_type7_packet(CP_SET_SECURE_MODE, 1); cmds[2] = 0; @@ -980,7 +984,8 @@ static int send_context_register(struct adreno_device *adreno_dev, struct hfi_register_ctxt_cmd cmd; struct kgsl_pagetable *pt = context->proc_priv->pagetable; - cmd.hdr = CMD_MSG_HDR(H2F_MSG_REGISTER_CONTEXT, sizeof(cmd)); + CMD_MSG_HDR(cmd, H2F_MSG_REGISTER_CONTEXT); + cmd.ctxt_id = context->id; cmd.flags = CTXT_FLAG_NOTIFY | context->flags; cmd.pt_addr = kgsl_mmu_pagetable_get_ttbr0(pt); @@ -996,7 +1001,7 @@ static int send_context_pointers(struct adreno_device *adreno_dev, struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct hfi_context_pointers_cmd cmd; - cmd.hdr = CMD_MSG_HDR(H2F_MSG_CONTEXT_POINTERS, sizeof(cmd)); + CMD_MSG_HDR(cmd, H2F_MSG_CONTEXT_POINTERS); cmd.ctxt_id = context->id; cmd.sop_addr = MEMSTORE_ID_GPU_ADDR(device, context->id, soptimestamp); cmd.eop_addr = MEMSTORE_ID_GPU_ADDR(device, context->id, eoptimestamp); @@ -1071,11 +1076,15 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, /* Add a *issue_ib struct for each IB */ cmd_sizebytes = sizeof(*cmd) + (sizeof(*issue_ib) * numibs); + if (WARN_ON(cmd_sizebytes > HFI_MAX_MSG_SIZE)) + return -EMSGSIZE; + cmd = kvmalloc(cmd_sizebytes, GFP_KERNEL); if (cmd == NULL) return -ENOMEM; - cmd->hdr = CMD_MSG_HDR(H2F_MSG_ISSUE_CMD, cmd_sizebytes); + cmd->hdr = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD, cmd_sizebytes, + HFI_MSG_CMD); cmd->hdr = MSG_HDR_SET_SEQNUM(cmd->hdr, atomic_inc_return(&hfi->seqnum)); @@ -1150,7 +1159,7 @@ static int send_context_unregister_hfi(struct adreno_device *adreno_dev, u32 seqnum; int rc; - cmd.hdr = CMD_MSG_HDR(H2F_MSG_UNREGISTER_CONTEXT, sizeof(cmd)); + CMD_MSG_HDR(cmd, H2F_MSG_UNREGISTER_CONTEXT); cmd.ctxt_id = ctxt_id, cmd.ts = ts, diff --git a/drivers/gpu/msm/adreno_a6xx_rpmh.c b/drivers/gpu/msm/adreno_a6xx_rpmh.c index fe03df7239f6..0594304d64ee 100644 --- a/drivers/gpu/msm/adreno_a6xx_rpmh.c +++ b/drivers/gpu/msm/adreno_a6xx_rpmh.c @@ -346,8 +346,7 @@ static int build_dcvs_table(struct adreno_device *adreno_dev) struct rpmh_arc_vals gx_arc, cx_arc, mx_arc; int ret; - hfi->dcvs_table.hdr = CMD_MSG_HDR(H2F_MSG_PERF_TBL, - sizeof(hfi->dcvs_table)); + CMD_MSG_HDR(hfi->dcvs_table, H2F_MSG_PERF_TBL); ret = rpmh_arc_cmds(&gx_arc, "gfx.lvl"); if (ret) @@ -388,7 +387,6 @@ static void build_bw_table_cmd(struct hfi_bwtable_cmd *cmd, { u32 i, j; - cmd->hdr = CMD_MSG_HDR(H2F_MSG_BW_VOTE_TBL, sizeof(*cmd)); cmd->bw_level_num = ddr->num_levels; cmd->ddr_cmds_num = ddr->num_cmds; cmd->ddr_wait_bitmask = ddr->wait_bitmask; @@ -442,6 +440,8 @@ static int build_bw_table(struct adreno_device *adreno_dev) return PTR_ERR(cnoc); } + CMD_MSG_HDR(gmu->hfi.bw_table, H2F_MSG_BW_VOTE_TBL); + build_bw_table_cmd(&gmu->hfi.bw_table, ddr, cnoc); free_rpmh_bw_votes(ddr); From 0fc2ce7e400c55a0f782f94a2fa409688be23500 Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Fri, 24 Jul 2020 12:45:10 -0600 Subject: [PATCH 11/14] msm: kgsl: Add suspend and resume for hwsched driver Add the support for performing a driver wide suspend and resume with hwsched driver. Change-Id: I9bc47e714aaafd33f52ea618fdecf14cb512ea61 Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno.c | 4 +- drivers/gpu/msm/adreno_a6xx_gmu.c | 7 +-- drivers/gpu/msm/adreno_a6xx_hwsched.c | 83 +++++++++++++++++++++++++++ drivers/gpu/msm/adreno_a6xx_rgmu.c | 7 +-- drivers/gpu/msm/adreno_hwsched.c | 7 +++ drivers/gpu/msm/kgsl_pwrctrl.c | 13 +---- drivers/gpu/msm/kgsl_pwrctrl.h | 11 +++- 7 files changed, 111 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index 5039adfebda6..fcc21dbf8ced 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -1881,11 +1881,11 @@ static int adreno_last_close(struct kgsl_device *device) * Wait up to 1 second for the active count to go low * and then start complaining about it */ - if (kgsl_active_count_wait(device, 0)) { + if (kgsl_active_count_wait(device, 0, HZ)) { dev_err(device->dev, "Waiting for the active count to become 0\n"); - while (kgsl_active_count_wait(device, 0)) + while (kgsl_active_count_wait(device, 0, HZ)) dev_err(device->dev, "Still waiting for the active count\n"); } diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index 9fb8db6f99ca..b445687146a9 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -3201,7 +3201,7 @@ static int a6xx_gmu_pm_suspend(struct adreno_device *adreno_dev) reinit_completion(&device->halt_gate); /* wait for active count so device can be put in slumber */ - ret = kgsl_active_count_wait(device, 0); + ret = kgsl_active_count_wait(device, 0, HZ); if (ret) { dev_err(device->dev, "Timed out waiting for the active count\n"); @@ -3232,10 +3232,9 @@ static void a6xx_gmu_pm_resume(struct adreno_device *adreno_dev) struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); - if (!test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags)) { - dev_err(device->dev, "resume invoked without a suspend\n"); + if (WARN(!test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags), + "resume invoked without a suspend\n")) return; - } adreno_dispatcher_unhalt(device); diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index 9d3983874a09..edd147109a04 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -672,12 +672,95 @@ static int a6xx_hwsched_bus_set(struct adreno_device *adreno_dev, int buslevel, return ret; } +static int a6xx_hwsched_pm_suspend(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + + if (test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags)) + return 0; + + trace_kgsl_pwr_request_state(device, KGSL_STATE_SUSPEND); + + /* Halt any new submissions */ + reinit_completion(&device->halt_gate); + + mutex_unlock(&device->mutex); + + /* Flush any currently running instances of the dispatcher */ + kthread_flush_worker(&kgsl_driver.worker); + + mutex_lock(&device->mutex); + + /* This ensures that dispatcher doesn't submit any new work */ + adreno_dispatcher_halt(device); + + /** + * Wait for the dispatcher to retire everything by waiting + * for the active count to go to zero. + */ + ret = kgsl_active_count_wait(device, 0, msecs_to_jiffies(100)); + if (ret) { + dev_err(device->dev, "Timed out waiting for the active count\n"); + goto err; + } + + if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) { + unsigned long wait = jiffies + + msecs_to_jiffies(ADRENO_IDLE_TIMEOUT); + + do { + if (a6xx_hw_isidle(adreno_dev)) + break; + } while (time_before(jiffies, wait)); + + if (!a6xx_hw_isidle(adreno_dev)) { + dev_err(device->dev, "Timed out idling the gpu\n"); + ret = -ETIMEDOUT; + goto err; + } + + a6xx_hwsched_power_off(adreno_dev); + } + + set_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags); + + trace_kgsl_pwr_set_state(device, KGSL_STATE_SUSPEND); + + return 0; + +err: + adreno_dispatcher_unhalt(device); + adreno_hwsched_start(adreno_dev); + + return ret; +} + +static void a6xx_hwsched_pm_resume(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + if (WARN(!test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags), + "resume invoked without a suspend\n")) + return; + + adreno_dispatcher_unhalt(device); + + adreno_hwsched_start(adreno_dev); + + clear_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags); +} + const struct adreno_power_ops a6xx_hwsched_power_ops = { .first_open = a6xx_hwsched_first_open, .last_close = a6xx_hwsched_power_off, .active_count_get = a6xx_hwsched_active_count_get, .active_count_put = a6xx_hwsched_active_count_put, .touch_wakeup = a6xx_hwsched_touch_wakeup, + .pm_suspend = a6xx_hwsched_pm_suspend, + .pm_resume = a6xx_hwsched_pm_resume, .gpu_clock_set = a6xx_hwsched_clock_set, .gpu_bus_set = a6xx_hwsched_bus_set, }; diff --git a/drivers/gpu/msm/adreno_a6xx_rgmu.c b/drivers/gpu/msm/adreno_a6xx_rgmu.c index 0cb54da191c3..d0a921febe3c 100644 --- a/drivers/gpu/msm/adreno_a6xx_rgmu.c +++ b/drivers/gpu/msm/adreno_a6xx_rgmu.c @@ -1153,7 +1153,7 @@ static int a6xx_rgmu_pm_suspend(struct adreno_device *adreno_dev) reinit_completion(&device->halt_gate); /* wait for active count so device can be put in slumber */ - ret = kgsl_active_count_wait(device, 0); + ret = kgsl_active_count_wait(device, 0, HZ); if (ret) { dev_err(device->dev, "Timed out waiting for the active count\n"); @@ -1183,10 +1183,9 @@ static void a6xx_rgmu_pm_resume(struct adreno_device *adreno_dev) struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); - if (!test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) { - dev_err(device->dev, "resume invoked without a suspend\n"); + if (WARN(!test_bit(GMU_PRIV_PM_SUSPEND, &rgmu->flags), + "resume invoked without a suspend\n")) return; - } adreno_dispatcher_unhalt(device); diff --git a/drivers/gpu/msm/adreno_hwsched.c b/drivers/gpu/msm/adreno_hwsched.c index 1f015e4fbde4..9184a64d8d5c 100644 --- a/drivers/gpu/msm/adreno_hwsched.c +++ b/drivers/gpu/msm/adreno_hwsched.c @@ -296,6 +296,13 @@ static int hwsched_sendcmd(struct adreno_device *adreno_dev, mutex_lock(&device->mutex); + if (adreno_gpu_halt(adreno_dev) != 0) { + mutex_unlock(&device->mutex); + kmem_cache_free(obj_cache, obj); + return -EBUSY; + } + + if (kgsl_context_detached(context)) { mutex_unlock(&device->mutex); kmem_cache_free(obj_cache, obj); diff --git a/drivers/gpu/msm/kgsl_pwrctrl.c b/drivers/gpu/msm/kgsl_pwrctrl.c index 87245a77ee12..926c13c18100 100644 --- a/drivers/gpu/msm/kgsl_pwrctrl.c +++ b/drivers/gpu/msm/kgsl_pwrctrl.c @@ -2026,7 +2026,7 @@ static int _suspend(struct kgsl_device *device) /* drain to prevent from more commands being submitted */ device->ftbl->drain(device); /* wait for active count so device can be put in slumber */ - ret = kgsl_active_count_wait(device, 0); + ret = kgsl_active_count_wait(device, 0, HZ); if (ret) goto err; @@ -2155,17 +2155,10 @@ static int _check_active_count(struct kgsl_device *device, int count) return atomic_read(&device->active_cnt) > count ? 0 : 1; } -/** - * kgsl_active_count_wait() - Wait for activity to finish. - * @device: Pointer to a KGSL device - * @count: Active count value to wait for - * - * Block until the active_cnt value hits the desired value - */ -int kgsl_active_count_wait(struct kgsl_device *device, int count) +int kgsl_active_count_wait(struct kgsl_device *device, int count, + unsigned long wait_jiffies) { int result = 0; - long wait_jiffies = HZ; if (WARN_ON(!mutex_is_locked(&device->mutex))) return -EINVAL; diff --git a/drivers/gpu/msm/kgsl_pwrctrl.h b/drivers/gpu/msm/kgsl_pwrctrl.h index 6747b7d39555..eb2ce504455b 100644 --- a/drivers/gpu/msm/kgsl_pwrctrl.h +++ b/drivers/gpu/msm/kgsl_pwrctrl.h @@ -198,7 +198,16 @@ kgsl_pwrctrl_active_freq(struct kgsl_pwrctrl *pwr) return pwr->pwrlevels[pwr->active_pwrlevel].gpu_freq; } -int kgsl_active_count_wait(struct kgsl_device *device, int count); +/** + * kgsl_active_count_wait() - Wait for activity to finish. + * @device: Pointer to a KGSL device + * @count: Active count value to wait for + * @wait_jiffies: Jiffies to wait + * + * Block until the active_cnt value hits the desired value + */ +int kgsl_active_count_wait(struct kgsl_device *device, int count, + unsigned long wait_jiffies); void kgsl_pwrctrl_busy_time(struct kgsl_device *device, u64 time, u64 busy); void kgsl_pwrctrl_set_constraint(struct kgsl_device *device, struct kgsl_pwr_constraint *pwrc, uint32_t id); From 44ed382d7f42d2b8da5ef5850a12e80a5fff5e5a Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Mon, 27 Jul 2020 10:46:00 -0600 Subject: [PATCH 12/14] msm: kgsl: Add hwsched based reset and recovery There can be various gmu faults or a context_bad hfi alongside an actively rendering gpu, for example, dcvs timeout, context register or unregister timeout or a perfcounter oob timeout. It is important to request dispatcher to perform reset and recovery so that un-finished inflight submissions can be re-submitted post recovery. Change-Id: I2e66c5b24c45f1af7514044caa74abe1bf84dd0c Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno.h | 16 ++- drivers/gpu/msm/adreno_a6xx_gmu.c | 25 +++++ drivers/gpu/msm/adreno_a6xx_hfi.h | 9 +- drivers/gpu/msm/adreno_a6xx_hwsched.c | 65 +++++++++++- drivers/gpu/msm/adreno_a6xx_hwsched.h | 6 ++ drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 38 ++++++- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h | 3 + drivers/gpu/msm/adreno_dispatch.c | 16 +-- drivers/gpu/msm/adreno_hwsched.c | 115 +++++++++++++++++++++- drivers/gpu/msm/adreno_hwsched.h | 10 ++ 10 files changed, 276 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 0b60e4fe848a..64105f67cf7c 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -1658,12 +1658,8 @@ static inline int adreno_perfcntr_active_oob_get( if (!ret) { ret = gmu_core_dev_oob_set(device, oob_perfcntr); - if (ret) { - adreno_set_gpu_fault(adreno_dev, - ADRENO_GMU_FAULT_SKIP_SNAPSHOT); - adreno_dispatcher_schedule(device); + if (ret) adreno_active_count_put(adreno_dev); - } } return ret; @@ -1935,4 +1931,14 @@ int adreno_suspend_context(struct kgsl_device *device); * submission. */ void adreno_profile_submit_time(struct adreno_submit_time *time); + +/** + * adreno_mark_guilty_context - Mark the given context as guilty + * (failed recovery) + * @device: Pointer to a KGSL device structure + * @id: Context ID of the guilty context (or 0 to mark all as guilty) + * + * Mark the given (or all) context(s) as guilty (failed recovery) + */ +void adreno_mark_guilty_context(struct kgsl_device *device, unsigned int id); #endif /*__ADRENO_H */ diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index b445687146a9..d62e7d753257 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -18,6 +18,7 @@ #include "adreno.h" #include "adreno_a6xx.h" +#include "adreno_hwsched.h" #include "kgsl_bus.h" #include "kgsl_device.h" #include "kgsl_trace.h" @@ -728,6 +729,29 @@ static const char *oob_to_str(enum oob_request req) return "unknown"; } +static void trigger_reset_recovery(struct adreno_device *adreno_dev, + enum oob_request req) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* + * Trigger recovery for perfcounter oob only since only + * perfcounter oob can happen alongside an actively rendering gpu. + */ + if (req != oob_perfcntr) + return; + + if (test_bit(GMU_DISPATCH, &device->gmu_core.flags)) { + adreno_get_gpu_halt(adreno_dev); + + adreno_hwsched_set_fault(adreno_dev); + } else { + adreno_set_gpu_fault(adreno_dev, + ADRENO_GMU_FAULT_SKIP_SNAPSHOT); + adreno_dispatcher_schedule(device); + } +} + int a6xx_gmu_oob_set(struct kgsl_device *device, enum oob_request req) { @@ -762,6 +786,7 @@ int a6xx_gmu_oob_set(struct kgsl_device *device, gmu_fault_snapshot(device); ret = -ETIMEDOUT; WARN(1, "OOB request %s timed out\n", oob_to_str(req)); + trigger_reset_recovery(adreno_dev, req); } gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, check); diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index 85aa36f9dac0..70314b922270 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -510,10 +510,11 @@ struct hfi_context_rule_cmd { /* F2H */ struct hfi_context_bad_cmd { - uint32_t hdr; - uint32_t ctxt_id; - uint32_t status; - uint32_t error; + u32 hdr; + u32 ctxt_id; + u32 policy; + u32 ts; + u32 error; } __packed; /* H2F */ diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index edd147109a04..045025162a6f 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -632,11 +632,24 @@ static int a6xx_hwsched_dcvs_set(struct adreno_device *adreno_dev, ret = a6xx_hfi_send_cmd_async(adreno_dev, &req); - if (ret) + if (ret) { dev_err_ratelimited(&gmu->pdev->dev, "Failed to set GPU perf idx %d, bw idx %d\n", req.freq, req.bw); + /* + * If this was a dcvs request along side an active gpu, request + * dispatcher based reset and recovery. + */ + if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) { + + adreno_get_gpu_halt(adreno_dev); + + adreno_hwsched_set_fault(adreno_dev); + } + + } + return ret; } @@ -753,6 +766,56 @@ static void a6xx_hwsched_pm_resume(struct adreno_device *adreno_dev) clear_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags); } +static void a6xx_hwsched_drain_ctxt_unregister(struct adreno_device *adreno_dev) +{ + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct pending_cmd *cmd = NULL; + + read_lock(&hfi->msglock); + + list_for_each_entry(cmd, &hfi->msglist, node) { + if (MSG_HDR_GET_ID(cmd->sent_hdr) == H2F_MSG_UNREGISTER_CONTEXT) + complete(&cmd->complete); + } + + read_unlock(&hfi->msglock); +} + +void a6xx_hwsched_restart(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* + * Any pending context unregister packets will be lost + * since we hard reset the GMU. This means any threads waiting + * for context unregister hfi ack will timeout. Wake them + * to avoid false positive ack timeout messages later. + */ + a6xx_hwsched_drain_ctxt_unregister(adreno_dev); + + read_lock(&device->context_lock); + idr_for_each(&device->context_idr, unregister_context_hwsched, NULL); + read_unlock(&device->context_lock); + + + if (!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) + return; + + a6xx_hwsched_hfi_stop(adreno_dev); + + a6xx_disable_gpu_irq(adreno_dev); + + a6xx_gmu_suspend(adreno_dev); + + clear_bit(GMU_PRIV_GPU_STARTED, &gmu->flags); + + ret = a6xx_hwsched_boot(adreno_dev); + + BUG_ON(ret); +} + const struct adreno_power_ops a6xx_hwsched_power_ops = { .first_open = a6xx_hwsched_first_open, .last_close = a6xx_hwsched_power_off, diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.h b/drivers/gpu/msm/adreno_a6xx_hwsched.h index 1ef0cb6fd531..9ee2190963ac 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.h @@ -33,4 +33,10 @@ struct a6xx_hwsched_device { */ int a6xx_hwsched_probe(struct platform_device *pdev, u32 chipid, const struct adreno_gpu_core *gpucore); + +/** + * a6xx_hwsched_restart - Restart the gmu and gpu + * @adreno_dev: Pointer to the adreno device + */ +void a6xx_hwsched_restart(struct adreno_device *adreno_dev); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index 524f69edc068..5ed17babbaac 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -9,6 +9,7 @@ #include "adreno.h" #include "adreno_a6xx.h" #include "adreno_a6xx_hwsched.h" +#include "adreno_hwsched.h" #include "adreno_pm4types.h" #include "adreno_trace.h" #include "kgsl_device.h" @@ -69,7 +70,7 @@ static const char * const memkind_strings[] = { [MEMKIND_USER_PROFILE_IBS] = "GMU USER PROFILING", }; -static struct a6xx_hwsched_hfi *to_a6xx_hwsched_hfi( +struct a6xx_hwsched_hfi *to_a6xx_hwsched_hfi( struct adreno_device *adreno_dev) { struct a6xx_device *a6xx_dev = container_of(adreno_dev, @@ -829,6 +830,14 @@ static void process_ts_retire(struct adreno_device *adreno_dev, u32 *rcvd) adreno_hwsched_trigger(adreno_dev); } +static void process_ctx_bad(struct adreno_device *adreno_dev, void *rcvd) +{ + /* Block dispatcher to submit more commands */ + adreno_get_gpu_halt(adreno_dev); + + adreno_hwsched_set_fault(adreno_dev); +} + static int hfi_f2h_main(void *arg) { struct adreno_device *adreno_dev = arg; @@ -852,6 +861,9 @@ static int hfi_f2h_main(void *arg) if (MSG_HDR_GET_ID(pkt->rcvd[0]) == F2H_MSG_TS_RETIRE) process_ts_retire(adreno_dev, pkt->rcvd); + if (MSG_HDR_GET_ID(pkt->rcvd[0]) == F2H_MSG_CONTEXT_BAD) + process_ctx_bad(adreno_dev, pkt->rcvd); + kmem_cache_free(f2h_cache, pkt); } } @@ -1149,7 +1161,7 @@ free: } static int send_context_unregister_hfi(struct adreno_device *adreno_dev, - u32 ctxt_id, u32 ts) + struct kgsl_context *context, u32 ts) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -1160,7 +1172,7 @@ static int send_context_unregister_hfi(struct adreno_device *adreno_dev, int rc; CMD_MSG_HDR(cmd, H2F_MSG_UNREGISTER_CONTEXT); - cmd.ctxt_id = ctxt_id, + cmd.ctxt_id = context->id, cmd.ts = ts, seqnum = atomic_inc_return(&gmu->hfi.seqnum); @@ -1180,10 +1192,26 @@ static int send_context_unregister_hfi(struct adreno_device *adreno_dev, dev_err(&gmu->pdev->dev, "Ack timeout for context unregister seq: %d ctx: %d ts: %d\n", MSG_HDR_GET_SEQNUM(pending_ack.sent_hdr), - ctxt_id, ts); + context->id, ts); rc = -ETIMEDOUT; + mutex_lock(&device->mutex); + gmu_fault_snapshot(device); + + /* + * Trigger dispatcher based reset and recovery. Invalidate the + * context so that any un-finished inflight submissions are not + * replayed after recovery. + */ + adreno_mark_guilty_context(device, context->id); + + adreno_drawctxt_invalidate(device, context); + + adreno_get_gpu_halt(adreno_dev); + + adreno_hwsched_set_fault(adreno_dev); + goto done; } @@ -1209,7 +1237,7 @@ void a6xx_hwsched_context_detach(struct adreno_context *drawctxt) /* Only send HFI if device is not in SLUMBER */ if (context->gmu_registered && test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) - ret = send_context_unregister_hfi(adreno_dev, context->id, + ret = send_context_unregister_hfi(adreno_dev, context, drawctxt->internal_timestamp); if (!ret) { diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index 97f62fb68b1b..70635b60f6ff 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -204,4 +204,7 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, * to ensure all submissions from this context have retired */ void a6xx_hwsched_context_detach(struct adreno_context *drawctxt); + +/* Helper function to get to a6xx hwsched hfi device from adreno device */ +struct a6xx_hwsched_hfi *to_a6xx_hwsched_hfi(struct adreno_device *adreno_dev); #endif diff --git a/drivers/gpu/msm/adreno_dispatch.c b/drivers/gpu/msm/adreno_dispatch.c index ccf2925bb11b..4e0f82e9c917 100644 --- a/drivers/gpu/msm/adreno_dispatch.c +++ b/drivers/gpu/msm/adreno_dispatch.c @@ -1524,14 +1524,7 @@ static int _mark_context(int id, void *ptr, void *data) return 0; } -/** - * mark_guilty_context() - Mark the given context as guilty (failed recovery) - * @device: Pointer to a KGSL device structure - * @id: Context ID of the guilty context (or 0 to mark all as guilty) - * - * Mark the given (or all) context(s) as guilty (failed recovery) - */ -static void mark_guilty_context(struct kgsl_device *device, unsigned int id) +void adreno_mark_guilty_context(struct kgsl_device *device, unsigned int id) { /* Mark the status for all the contexts in the device */ @@ -1953,7 +1946,7 @@ static void process_cmdobj_fault(struct kgsl_device *device, state, drawobj->context->id, drawobj->timestamp); /* Mark the context as failed */ - mark_guilty_context(device, drawobj->context->id); + adreno_mark_guilty_context(device, drawobj->context->id); /* Invalidate the context */ adreno_drawctxt_invalidate(device, drawobj->context); @@ -1991,7 +1984,8 @@ static void recover_dispatch_q(struct kgsl_device *device, dispatch_q->cmd_q[ptr]; struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); - mark_guilty_context(device, drawobj->context->id); + adreno_mark_guilty_context(device, + drawobj->context->id); adreno_drawctxt_invalidate(device, drawobj->context); kgsl_drawobj_destroy(drawobj); @@ -2064,7 +2058,7 @@ replay: replay[i]->base.timestamp); /* Mark this context as guilty (failed recovery) */ - mark_guilty_context(device, + adreno_mark_guilty_context(device, replay[i]->base.context->id); adreno_drawctxt_invalidate(device, diff --git a/drivers/gpu/msm/adreno_hwsched.c b/drivers/gpu/msm/adreno_hwsched.c index 9184a64d8d5c..e5b96150d1b4 100644 --- a/drivers/gpu/msm/adreno_hwsched.c +++ b/drivers/gpu/msm/adreno_hwsched.c @@ -272,6 +272,25 @@ static int hwsched_queue_context(struct adreno_device *adreno_dev, return 0; } +void adreno_hwsched_set_fault(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + + atomic_set(&hwsched->fault, ADRENO_HWSCHED_FAULT_RESTART); + + /* make sure fault is written before triggering dispatcher */ + smp_wmb(); + + adreno_hwsched_trigger(adreno_dev); +} + +static bool hwsched_in_fault(struct adreno_hwsched *hwsched) +{ + /* make sure we're reading the latest value */ + smp_rmb(); + return atomic_read(&hwsched->fault) != 0; +} + /** * sendcmd() - Send a drawobj to the GPU hardware * @dispatcher: Pointer to the adreno dispatcher struct @@ -332,6 +351,14 @@ static int hwsched_sendcmd(struct adreno_device *adreno_dev, if (hwsched->inflight == 1) { adreno_active_count_put(adreno_dev); clear_bit(ADRENO_HWSCHED_POWER, &hwsched->flags); + } else if (device->gmu_fault) { + /* + * If we encountered a GMU fault, then trigger reset and + * recovery. + */ + adreno_get_gpu_halt(adreno_dev); + + adreno_hwsched_set_fault(adreno_dev); } hwsched->inflight--; @@ -556,7 +583,8 @@ static void adreno_hwsched_issuecmds(struct adreno_device *adreno_dev) return; } - hwsched_issuecmds(adreno_dev); + if (!hwsched_in_fault(hwsched)) + hwsched_issuecmds(adreno_dev); mutex_unlock(&hwsched->mutex); } @@ -966,6 +994,85 @@ void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev) kmem_cache_destroy(obj_cache); } +static void adreno_hwsched_init_replay(struct adreno_hwsched *hwsched) +{ + struct cmd_list_obj *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) { + struct kgsl_drawobj_cmd *cmdobj = obj->cmdobj; + + clear_bit(KGSL_FT_REPLAY, &cmdobj->fault_policy); + } +} + +static void adreno_hwsched_complete_replay(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct cmd_list_obj *obj, *tmp; + u32 retired = 0; + + list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) { + struct kgsl_drawobj_cmd *cmdobj = obj->cmdobj; + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + struct kgsl_context *context = drawobj->context; + + /* + * Get rid of retired objects or objects that belong to detached + * or invalidated contexts + */ + if ((kgsl_check_timestamp(device, context, drawobj->timestamp)) + || kgsl_context_invalid(context) + || kgsl_context_detached(context)) { + + retire_cmdobj(cmdobj); + retired++; + list_del_init(&obj->node); + kmem_cache_free(obj_cache, obj); + + continue; + } + + if (!test_and_set_bit(KGSL_FT_REPLAY, &cmdobj->fault_policy)) + a6xx_hwsched_submit_cmdobj(adreno_dev, cmdobj); + } + + /* Signal fences */ + if (retired) + kgsl_process_event_groups(device); + + hwsched->inflight = 0; + + /* + * We have nothing to replay. So clear the fault so that + * dispatcher starts accepting new submissions. + */ + if (list_empty(&hwsched->cmd_list)) { + atomic_set(&hwsched->fault, 0); + adreno_clear_gpu_halt(adreno_dev); + adreno_hwsched_trigger(adreno_dev); + } +} + + +static void adreno_hwsched_recovery(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + + mutex_lock(&device->mutex); + + if ((atomic_cmpxchg(&hwsched->fault, ADRENO_HWSCHED_FAULT_RESTART, + ADRENO_HWSCHED_FAULT_REPLAY) == ADRENO_HWSCHED_FAULT_RESTART)) { + a6xx_hwsched_restart(adreno_dev); + adreno_hwsched_init_replay(hwsched); + } + + adreno_hwsched_complete_replay(adreno_dev); + + mutex_unlock(&device->mutex); +} + static void adreno_hwsched_work(struct kthread_work *work) { struct adreno_hwsched *hwsched = container_of(work, @@ -976,6 +1083,12 @@ static void adreno_hwsched_work(struct kthread_work *work) mutex_lock(&hwsched->mutex); + if (hwsched_in_fault(hwsched)) { + adreno_hwsched_recovery(adreno_dev); + mutex_unlock(&hwsched->mutex); + return; + } + /* * As long as there are inflight commands, process retired comamnds from * all drawqueues diff --git a/drivers/gpu/msm/adreno_hwsched.h b/drivers/gpu/msm/adreno_hwsched.h index 7e1256c5e7a0..8953f85db72d 100644 --- a/drivers/gpu/msm/adreno_hwsched.h +++ b/drivers/gpu/msm/adreno_hwsched.h @@ -24,10 +24,14 @@ struct adreno_hwsched { struct kthread_work work; /** @cmd_list: List of objects submitted to dispatch queues */ struct list_head cmd_list; + /** @fault: Atomic to record a fault */ + atomic_t fault; }; enum adreno_hwsched_flags { ADRENO_HWSCHED_POWER = 0, + ADRENO_HWSCHED_FAULT_RESTART, + ADRENO_HWSCHED_FAULT_REPLAY, }; /** @@ -87,4 +91,10 @@ void adreno_hwsched_init(struct adreno_device *adreno_dev); * Free the dispatcher resources */ void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev); + +/** + * adreno_hwsched_set_fault - Set hwsched fault to request recovery + * @adreno_dev: A handle to adreno device + */ +void adreno_hwsched_set_fault(struct adreno_device *adreno_dev); #endif From b2f0a442ec38bd3aa67d8844710b5774d543970b Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Mon, 27 Jul 2020 10:59:37 -0600 Subject: [PATCH 13/14] msm: kgsl: Add a6xx_hwsched_snapshot function Since GMU tells us the exact submission that faulted via the context bad hfi, look up the command object corresponding to the submission and mark it for snapshot purposes. Create the top level gpudev snapshot function for hw scheduling which first snapshots the faulted command object and then calls into the generic gmu snapshot function. Change-Id: I5c5300b7b147dfcf0c009b44376ba0c6e1f8e25b Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno.h | 7 + drivers/gpu/msm/adreno_a6xx.c | 2 +- drivers/gpu/msm/adreno_a6xx_hwsched.c | 8 ++ drivers/gpu/msm/adreno_a6xx_hwsched.h | 10 ++ drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 17 +-- drivers/gpu/msm/adreno_hwsched.c | 154 +++++++++++++++++++++- drivers/gpu/msm/adreno_hwsched.h | 23 ++++ drivers/gpu/msm/adreno_snapshot.c | 22 ++-- drivers/gpu/msm/adreno_snapshot.h | 20 ++- drivers/gpu/msm/kgsl_drawobj.h | 4 +- 10 files changed, 235 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 64105f67cf7c..75fff13359bf 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -1773,6 +1773,13 @@ static inline void adreno_reg_offset_init(u32 *reg_offsets) } } +static inline u32 adreno_get_level(u32 priority) +{ + u32 level = priority / KGSL_PRIORITY_MAX_RB_LEVELS; + + return min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1); +} + int adreno_gmu_fenced_write(struct adreno_device *adreno_dev, enum adreno_regs offset, unsigned int val, unsigned int fence_mask); diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c index e1698a5c03f4..46d0f0d492b6 100644 --- a/drivers/gpu/msm/adreno_a6xx.c +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -2801,7 +2801,7 @@ struct adreno_gpudev adreno_a6xx_gpudev = { struct adreno_gpudev adreno_a6xx_hwsched_gpudev = { .reg_offsets = a6xx_register_offsets, .probe = a6xx_hwsched_probe, - .snapshot = a6xx_gmu_snapshot, + .snapshot = a6xx_hwsched_snapshot, .irq_handler = a6xx_irq_handler, .perfcounters = &a6xx_perfcounters, .read_throttling_counters = a6xx_read_throttling_counters, diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index 045025162a6f..27466c623a44 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -15,6 +15,14 @@ #include "kgsl_trace.h" #include "kgsl_util.h" +void a6xx_hwsched_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + adreno_hwsched_parse_fault_cmdobj(adreno_dev, snapshot); + + a6xx_gmu_snapshot(adreno_dev, snapshot); +} + static int a6xx_hwsched_gmu_first_boot(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.h b/drivers/gpu/msm/adreno_a6xx_hwsched.h index 9ee2190963ac..66a4fce5ce5b 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.h @@ -39,4 +39,14 @@ int a6xx_hwsched_probe(struct platform_device *pdev, * @adreno_dev: Pointer to the adreno device */ void a6xx_hwsched_restart(struct adreno_device *adreno_dev); + +/** + * a6xx_hwsched_snapshot - take a6xx hwsched snapshot + * @adreno_dev: Pointer to the adreno device + * @snapshot: Pointer to the snapshot instance + * + * Snapshot the faulty ib and then snapshot rest of a6xx gmu things + */ +void a6xx_hwsched_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index 5ed17babbaac..1fd3dc29a921 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -151,13 +151,6 @@ static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd) MSG_HDR_GET_SEQNUM(waiters[i])); } -static u32 get_level(u32 priority) -{ - u32 level = priority / KGSL_PRIORITY_MAX_RB_LEVELS; - - return min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1); -} - static void log_profiling_info(struct adreno_device *adreno_dev, u32 *rcvd) { struct hfi_ts_retire_cmd *cmd = (struct hfi_ts_retire_cmd *)rcvd; @@ -169,7 +162,7 @@ static void log_profiling_info(struct adreno_device *adreno_dev, u32 *rcvd) return; info.timestamp = cmd->ts; - info.rb_id = get_level(context->priority); + info.rb_id = adreno_get_level(context->priority); info.gmu_dispatch_queue = context->gmu_dispatch_queue; info.submitted_to_rb = cmd->submitted_to_rb; info.sop = cmd->sop; @@ -832,10 +825,12 @@ static void process_ts_retire(struct adreno_device *adreno_dev, u32 *rcvd) static void process_ctx_bad(struct adreno_device *adreno_dev, void *rcvd) { + struct hfi_context_bad_cmd *cmd = rcvd; + /* Block dispatcher to submit more commands */ adreno_get_gpu_halt(adreno_dev); - adreno_hwsched_set_fault(adreno_dev); + adreno_hwsched_mark_drawobj(adreno_dev, cmd->ctxt_id, cmd->ts); } static int hfi_f2h_main(void *arg) @@ -942,7 +937,7 @@ static void add_profile_events(struct adreno_device *adreno_dev, time_in_ns = do_div(time_in_s, 1000000000); info.inflight = -1; - info.rb_id = get_level(context->priority); + info.rb_id = adreno_get_level(context->priority); info.gmu_dispatch_queue = context->gmu_dispatch_queue; trace_adreno_cmdbatch_submitted(drawobj, &info, time->ticks, @@ -985,7 +980,7 @@ static u32 get_next_dq(u32 priority) static u32 get_dq_id(u32 priority) { - u32 level = get_level(priority); + u32 level = adreno_get_level(priority); return get_next_dq(level); } diff --git a/drivers/gpu/msm/adreno_hwsched.c b/drivers/gpu/msm/adreno_hwsched.c index e5b96150d1b4..3fa6ac3cf260 100644 --- a/drivers/gpu/msm/adreno_hwsched.c +++ b/drivers/gpu/msm/adreno_hwsched.c @@ -6,6 +6,7 @@ #include "adreno.h" #include "adreno_a6xx.h" #include "adreno_a6xx_hwsched.h" +#include "adreno_snapshot.h" #include "adreno_trace.h" /* This structure represents inflight command object */ @@ -1005,6 +1006,18 @@ static void adreno_hwsched_init_replay(struct adreno_hwsched *hwsched) } } +static void force_retire_timestamp(struct kgsl_device *device, + struct kgsl_drawobj *drawobj) +{ + kgsl_sharedmem_writel(device->memstore, + KGSL_MEMSTORE_OFFSET(drawobj->context->id, soptimestamp), + drawobj->timestamp); + + kgsl_sharedmem_writel(device->memstore, + KGSL_MEMSTORE_OFFSET(drawobj->context->id, eoptimestamp), + drawobj->timestamp); +} + static void adreno_hwsched_complete_replay(struct adreno_device *adreno_dev) { struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); @@ -1054,6 +1067,93 @@ static void adreno_hwsched_complete_replay(struct adreno_device *adreno_dev) } } +static void do_fault_header(struct adreno_device *adreno_dev, + struct kgsl_drawobj *drawobj) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context); + u32 status, rptr, wptr, ib1sz, ib2sz; + u64 ib1base, ib2base; + + adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS, &status); + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &rptr); + adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr); + adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB1_BASE, + ADRENO_REG_CP_IB1_BASE_HI, &ib1base); + adreno_readreg(adreno_dev, ADRENO_REG_CP_IB1_BUFSZ, &ib1sz); + adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB2_BASE, + ADRENO_REG_CP_IB2_BASE_HI, &ib2base); + adreno_readreg(adreno_dev, ADRENO_REG_CP_IB2_BUFSZ, &ib2sz); + + drawobj->context->last_faulted_cmd_ts = drawobj->timestamp; + drawobj->context->total_fault_count++; + + pr_context(device, drawobj->context, + "ctx %d ctx_type %s ts %d status %8.8X dispatch_queue=%d rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", + drawobj->context->id, kgsl_context_type(drawctxt->type), + drawobj->timestamp, status, + drawobj->context->gmu_dispatch_queue, rptr, wptr, + ib1base, ib1sz, ib2base, ib2sz); + + trace_adreno_gpu_fault(drawobj->context->id, drawobj->timestamp, status, + rptr, wptr, ib1base, ib1sz, ib2base, ib2sz, + adreno_get_level(drawobj->context->priority)); +} + +static struct cmd_list_obj *get_fault_cmdobj(struct adreno_device *adreno_dev) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct cmd_list_obj *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) { + + if (test_bit(CMDOBJ_FAULT, &obj->cmdobj->priv)) + return obj; + } + + return NULL; +} + +static void reset_and_snapshot(struct adreno_device *adreno_dev) +{ + struct kgsl_drawobj *drawobj = NULL; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_context *context = NULL; + struct cmd_list_obj *obj = get_fault_cmdobj(adreno_dev); + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + + if (!obj) { + kgsl_device_snapshot(device, NULL, false); + goto done; + } + + drawobj = DRAWOBJ(obj->cmdobj); + + context = drawobj->context; + + do_fault_header(adreno_dev, drawobj); + + kgsl_device_snapshot(device, context, false); + + force_retire_timestamp(device, drawobj); + + if (context->flags & KGSL_CONTEXT_INVALIDATE_ON_FAULT) { + adreno_mark_guilty_context(device, context->id); + adreno_drawctxt_invalidate(device, context); + } + + /* This must always happen after taking snapshot */ + clear_bit(CMDOBJ_FAULT, &obj->cmdobj->priv); + + /* + * Put back the reference which we incremented when marking the faulty + * cmdobj + */ + kgsl_drawobj_put(drawobj); +done: + a6xx_hwsched_restart(adreno_dev); + adreno_hwsched_init_replay(hwsched); +} static void adreno_hwsched_recovery(struct adreno_device *adreno_dev) { @@ -1063,10 +1163,8 @@ static void adreno_hwsched_recovery(struct adreno_device *adreno_dev) mutex_lock(&device->mutex); if ((atomic_cmpxchg(&hwsched->fault, ADRENO_HWSCHED_FAULT_RESTART, - ADRENO_HWSCHED_FAULT_REPLAY) == ADRENO_HWSCHED_FAULT_RESTART)) { - a6xx_hwsched_restart(adreno_dev); - adreno_hwsched_init_replay(hwsched); - } + ADRENO_HWSCHED_FAULT_REPLAY) == ADRENO_HWSCHED_FAULT_RESTART)) + reset_and_snapshot(adreno_dev); adreno_hwsched_complete_replay(adreno_dev); @@ -1135,3 +1233,51 @@ void adreno_hwsched_init(struct adreno_device *adreno_dev) init_llist_head(&hwsched->requeue[i]); } } + +void adreno_hwsched_mark_drawobj(struct adreno_device *adreno_dev, + u32 ctxt_id, u32 ts) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct cmd_list_obj *obj, *tmp; + struct kgsl_drawobj *drawobj = NULL; + + mutex_lock(&hwsched->mutex); + + list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) { + + drawobj = DRAWOBJ(obj->cmdobj); + + if (ctxt_id == drawobj->context->id && + ts == drawobj->timestamp) { + + if (kref_get_unless_zero(&drawobj->refcount)) + set_bit(CMDOBJ_FAULT, &obj->cmdobj->priv); + break; + } + } + + adreno_hwsched_set_fault(adreno_dev); + + mutex_unlock(&hwsched->mutex); +} + +void adreno_hwsched_parse_fault_cmdobj(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct adreno_hwsched *hwsched = to_hwsched(adreno_dev); + struct cmd_list_obj *obj, *tmp; + + list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) { + struct kgsl_drawobj_cmd *cmdobj = obj->cmdobj; + + if (test_bit(CMDOBJ_FAULT, &cmdobj->priv)) { + struct kgsl_memobj_node *ib; + + list_for_each_entry(ib, &cmdobj->cmdlist, node) { + adreno_parse_ib(KGSL_DEVICE(adreno_dev), + snapshot, snapshot->process, + ib->gpuaddr, ib->size >> 2); + } + } + } +} diff --git a/drivers/gpu/msm/adreno_hwsched.h b/drivers/gpu/msm/adreno_hwsched.h index 8953f85db72d..ac319ceca309 100644 --- a/drivers/gpu/msm/adreno_hwsched.h +++ b/drivers/gpu/msm/adreno_hwsched.h @@ -97,4 +97,27 @@ void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev); * @adreno_dev: A handle to adreno device */ void adreno_hwsched_set_fault(struct adreno_device *adreno_dev); + +/** + * adreno_hwsched_mark_drawobj() - Get the drawobj that faulted + * @adreno_dev: pointer to the adreno device + * @ctxt_id: context id of the faulty submission + * @ts: timestamp of the faulty submission + * + * When we get a context bad hfi, use this function to get to the + * faulty submission and mark the submission for snapshot purposes + */ +void adreno_hwsched_mark_drawobj(struct adreno_device *adreno_dev, u32 ctxt_id, + u32 ts); + +/** + * adreno_hwsched_parse_fault_ib - Parse the faulty submission + * @adreno_dev: pointer to the adreno device + * @snapshot: Pointer to the snapshot structure + * + * Walk the list of active submissions to find the one that faulted and + * parse it so that relevant command buffers can be added to the snapshot + */ +void adreno_hwsched_parse_fault_cmdobj(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); #endif diff --git a/drivers/gpu/msm/adreno_snapshot.c b/drivers/gpu/msm/adreno_snapshot.c index 01e2f317d51d..ca2f64496a32 100644 --- a/drivers/gpu/msm/adreno_snapshot.c +++ b/drivers/gpu/msm/adreno_snapshot.c @@ -171,13 +171,7 @@ static int snapshot_freeze_obj_list(struct kgsl_snapshot *snapshot, return ret; } -/* - * We want to store the last executed IB1 and IB2 in the static region to ensure - * that we get at least some information out of the snapshot even if we can't - * access the dynamic data from the sysfs file. Push all other IBs on the - * dynamic list - */ -static inline void parse_ib(struct kgsl_device *device, +void adreno_parse_ib(struct kgsl_device *device, struct kgsl_snapshot *snapshot, struct kgsl_process_private *process, uint64_t gpuaddr, uint64_t dwords) @@ -245,8 +239,8 @@ static void dump_all_ibs(struct kgsl_device *device, ibaddr, ibsize)) continue; - parse_ib(device, snapshot, snapshot->process, ibaddr, - ibsize); + adreno_parse_ib(device, snapshot, snapshot->process, + ibaddr, ibsize); } else index = index + 1; } @@ -398,7 +392,7 @@ static void snapshot_rb_ibs(struct kgsl_device *device, ibaddr, ibsize)) continue; - parse_ib(device, snapshot, snapshot->process, + adreno_parse_ib(device, snapshot, snapshot->process, ibaddr, ibsize); } else index = (index + 1) % KGSL_RB_DWORDS; @@ -561,7 +555,7 @@ static void kgsl_snapshot_add_active_ib_obj_list(struct kgsl_device *device, index = find_object(snapshot->ib2base, snapshot->process); if (index != -ENOENT) - parse_ib(device, snapshot, snapshot->process, + adreno_parse_ib(device, snapshot, snapshot->process, snapshot->ib2base, objbuf[index].size >> 2); } } @@ -830,13 +824,13 @@ void adreno_snapshot(struct kgsl_device *device, struct kgsl_snapshot *snapshot, snapshot_frozen_objsize = 0; + setup_fault_process(device, snapshot, + context ? context->proc_priv : NULL); + /* Add GPU specific sections - registers mainly, but other stuff too */ if (gpudev->snapshot) gpudev->snapshot(adreno_dev, snapshot); - setup_fault_process(device, snapshot, - context ? context->proc_priv : NULL); - snapshot->ib1dumped = false; snapshot->ib2dumped = false; diff --git a/drivers/gpu/msm/adreno_snapshot.h b/drivers/gpu/msm/adreno_snapshot.h index 0800d419ed5c..8b68eec2cf6a 100644 --- a/drivers/gpu/msm/adreno_snapshot.h +++ b/drivers/gpu/msm/adreno_snapshot.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2013-2015,2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2013-2015,2020, The Linux Foundation. All rights reserved. */ #ifndef __ADRENO_SNAPSHOT_H #define __ADRENO_SNAPSHOT_H @@ -39,4 +39,22 @@ void adreno_snapshot_vbif_registers(struct kgsl_device *device, const struct adreno_vbif_snapshot_registers *list, unsigned int count); +/** + * adreno_parse_ib - Parse the given IB + * @device: Pointer to the kgsl device + * @snapshot: Pointer to the snapshot structure + * @process: Process to which this IB belongs + * @gpuaddr: Gpu address of the IB + * @dwords: Size in dwords of the IB + * + * We want to store the last executed IB1 and IB2 in the static region to ensure + * that we get at least some information out of the snapshot even if we can't + * access the dynamic data from the sysfs file. Push all other IBs on the + * dynamic list + */ +void adreno_parse_ib(struct kgsl_device *device, + struct kgsl_snapshot *snapshot, + struct kgsl_process_private *process, + u64 gpuaddr, u64 dwords); + #endif /*__ADRENO_SNAPSHOT_H */ diff --git a/drivers/gpu/msm/kgsl_drawobj.h b/drivers/gpu/msm/kgsl_drawobj.h index 7f3dd2624a5f..1df02ccf746f 100644 --- a/drivers/gpu/msm/kgsl_drawobj.h +++ b/drivers/gpu/msm/kgsl_drawobj.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved. */ #ifndef __KGSL_DRAWOBJ_H @@ -165,6 +165,7 @@ struct kgsl_drawobj_sparse { * command obj * @CMDOBJ_WFI - Force wait-for-idle for the submission * @CMDOBJ_PROFILE - store the start / retire ticks for + * @CMDOBJ_FAULT - Mark the command object as faulted * the command obj in the profiling buffer */ enum kgsl_drawobj_cmd_priv { @@ -172,6 +173,7 @@ enum kgsl_drawobj_cmd_priv { CMDOBJ_FORCE_PREAMBLE, CMDOBJ_WFI, CMDOBJ_PROFILE, + CMDOBJ_FAULT, }; struct kgsl_ibdesc; From 6f79885a61b5daa574e4d71068b31bfb2e1dae4c Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Mon, 27 Jul 2020 14:16:55 -0600 Subject: [PATCH 14/14] msm: kgsl: Capture gpu globals in hwsched snapshot Capture ringbuffers, smmu info record, scratch memory and preemption records into hw scheduling snapshot. Change-Id: I119cc18115f3a74a6a26c976772b7d746e70900e Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx.h | 3 + drivers/gpu/msm/adreno_a6xx_hwsched.c | 113 ++++++++++++++++++++++++- drivers/gpu/msm/adreno_a6xx_snapshot.c | 3 - drivers/gpu/msm/adreno_snapshot.c | 10 +-- drivers/gpu/msm/adreno_snapshot.h | 12 ++- 5 files changed, 131 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx.h b/drivers/gpu/msm/adreno_a6xx.h index c8895ef8d9d7..a12e3d3ce91a 100644 --- a/drivers/gpu/msm/adreno_a6xx.h +++ b/drivers/gpu/msm/adreno_a6xx.h @@ -13,6 +13,9 @@ #include "adreno_a6xx_gmu.h" #include "adreno_a6xx_rgmu.h" +/* Snapshot section size of each CP preemption record for A6XX */ +#define A6XX_SNAPSHOT_CP_CTXRECORD_SIZE_IN_BYTES (64 * 1024) + extern const struct adreno_power_ops a6xx_gmu_power_ops; extern const struct adreno_power_ops a6xx_rgmu_power_ops; extern const struct adreno_power_ops a630_gmu_power_ops; diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index 27466c623a44..6e1d85a156b9 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -11,16 +11,127 @@ #include "adreno.h" #include "adreno_a6xx.h" #include "adreno_a6xx_hwsched.h" +#include "adreno_snapshot.h" #include "kgsl_device.h" #include "kgsl_trace.h" #include "kgsl_util.h" +static size_t adreno_hwsched_snapshot_rb(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_snapshot_rb_v2 *header = (struct kgsl_snapshot_rb_v2 *)buf; + u32 *data = (u32 *)(buf + sizeof(*header)); + struct kgsl_memdesc *rb = (struct kgsl_memdesc *)priv; + + if (remain < rb->size + sizeof(*header)) { + SNAPSHOT_ERR_NOMEM(device, "RB"); + return 0; + } + + header->start = 0; + header->end = rb->size >> 2; + header->rptr = 0; + header->rbsize = rb->size >> 2; + header->count = rb->size >> 2; + header->timestamp_queued = 0; + header->timestamp_retired = 0; + header->gpuaddr = rb->gpuaddr; + header->id = 0; + + memcpy(data, rb->hostptr, rb->size); + + return rb->size + sizeof(*header); +} + +static void a6xx_hwsched_snapshot_preemption_record(struct kgsl_device *device, + struct kgsl_snapshot *snapshot, struct kgsl_memdesc *md, u64 offset) +{ + struct kgsl_snapshot_section_header *section_header = + (struct kgsl_snapshot_section_header *)snapshot->ptr; + u8 *dest = snapshot->ptr + sizeof(*section_header); + struct kgsl_snapshot_gpu_object_v2 *header = + (struct kgsl_snapshot_gpu_object_v2 *)dest; + size_t section_size = sizeof(*section_header) + sizeof(*header) + + A6XX_SNAPSHOT_CP_CTXRECORD_SIZE_IN_BYTES; + + if (snapshot->remain < section_size) { + SNAPSHOT_ERR_NOMEM(device, "PREEMPTION RECORD"); + return; + } + + section_header->magic = SNAPSHOT_SECTION_MAGIC; + section_header->id = KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2; + section_header->size = section_size; + + header->size = A6XX_SNAPSHOT_CP_CTXRECORD_SIZE_IN_BYTES >> 2; + header->gpuaddr = md->gpuaddr + offset; + header->ptbase = + kgsl_mmu_pagetable_get_ttbr0(device->mmu.defaultpagetable); + header->type = SNAPSHOT_GPU_OBJECT_GLOBAL; + + dest += sizeof(*header); + + memcpy(dest, md->hostptr + offset, + A6XX_SNAPSHOT_CP_CTXRECORD_SIZE_IN_BYTES); + + snapshot->ptr += section_header->size; + snapshot->remain -= section_header->size; + snapshot->size += section_header->size; +} + +static void snapshot_preemption_records(struct kgsl_device *device, + struct kgsl_snapshot *snapshot, struct kgsl_memdesc *md) +{ + const struct adreno_a6xx_core *a6xx_core = + to_a6xx_core(ADRENO_DEVICE(device)); + u64 ctxt_record_size = A6XX_CP_CTXRECORD_SIZE_IN_BYTES; + u64 offset; + + if (a6xx_core->ctxt_record_size) + ctxt_record_size = a6xx_core->ctxt_record_size; + + /* All preemption records exist as a single mem alloc entry */ + for (offset = 0; offset < md->size; offset += ctxt_record_size) + a6xx_hwsched_snapshot_preemption_record(device, snapshot, md, + offset); +} + void a6xx_hwsched_snapshot(struct adreno_device *adreno_dev, struct kgsl_snapshot *snapshot) { - adreno_hwsched_parse_fault_cmdobj(adreno_dev, snapshot); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_hwsched_hfi *hw_hfi = to_a6xx_hwsched_hfi(adreno_dev); + u32 i; a6xx_gmu_snapshot(adreno_dev, snapshot); + + for (i = 0; i < hw_hfi->mem_alloc_entries; i++) { + struct mem_alloc_entry *entry = &hw_hfi->mem_alloc_table[i]; + + if (entry->desc.mem_kind == MEMKIND_RB) + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_RB_V2, + snapshot, adreno_hwsched_snapshot_rb, + entry->gpu_md); + + if (entry->desc.mem_kind == MEMKIND_SCRATCH) + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, + snapshot, adreno_snapshot_global, + entry->gpu_md); + + if (entry->desc.mem_kind == MEMKIND_CSW_SMMU_INFO) + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, + snapshot, adreno_snapshot_global, + entry->gpu_md); + + if (entry->desc.mem_kind == MEMKIND_CSW_PRIV_NON_SECURE) + snapshot_preemption_records(device, snapshot, + entry->gpu_md); + } + + adreno_hwsched_parse_fault_cmdobj(adreno_dev, snapshot); } static int a6xx_hwsched_gmu_first_boot(struct adreno_device *adreno_dev) diff --git a/drivers/gpu/msm/adreno_a6xx_snapshot.c b/drivers/gpu/msm/adreno_a6xx_snapshot.c index 1319346b8935..6cf2d66eebc1 100644 --- a/drivers/gpu/msm/adreno_a6xx_snapshot.c +++ b/drivers/gpu/msm/adreno_a6xx_snapshot.c @@ -12,9 +12,6 @@ #define A6XX_NUM_XIN_AXI_BLOCKS 5 #define A6XX_NUM_XIN_CORE_BLOCKS 4 -/* Snapshot section size of each CP preemption record for A6XX */ -#define A6XX_SNAPSHOT_CP_CTXRECORD_SIZE_IN_BYTES (64 * 1024) - static const unsigned int a6xx_gras_cluster[] = { 0x8000, 0x8006, 0x8010, 0x8092, 0x8094, 0x809D, 0x80A0, 0x80A6, 0x80AF, 0x80F1, 0x8100, 0x8107, 0x8109, 0x8109, 0x8110, 0x8110, diff --git a/drivers/gpu/msm/adreno_snapshot.c b/drivers/gpu/msm/adreno_snapshot.c index ca2f64496a32..1f0874a539dc 100644 --- a/drivers/gpu/msm/adreno_snapshot.c +++ b/drivers/gpu/msm/adreno_snapshot.c @@ -736,7 +736,7 @@ done: } /* Snapshot a global memory buffer */ -static size_t snapshot_global(struct kgsl_device *device, u8 *buf, +size_t adreno_snapshot_global(struct kgsl_device *device, u8 *buf, size_t remain, void *priv) { struct kgsl_memdesc *memdesc = priv; @@ -780,12 +780,12 @@ static void adreno_snapshot_iommu(struct kgsl_device *device, struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device); kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, - snapshot, snapshot_global, iommu->setstate); + snapshot, adreno_snapshot_global, iommu->setstate); if (ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION)) kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, - snapshot, snapshot_global, iommu->smmu_info); + snapshot, adreno_snapshot_global, iommu->smmu_info); } static void adreno_snapshot_ringbuffer(struct kgsl_device *device, @@ -848,10 +848,10 @@ void adreno_snapshot(struct kgsl_device *device, struct kgsl_snapshot *snapshot, /* Dump selected global buffers */ kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, - snapshot, snapshot_global, device->memstore); + snapshot, adreno_snapshot_global, device->memstore); kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2, - snapshot, snapshot_global, + snapshot, adreno_snapshot_global, adreno_dev->pwron_fixup); if (kgsl_mmu_get_mmutype(device) == KGSL_MMU_TYPE_IOMMU) diff --git a/drivers/gpu/msm/adreno_snapshot.h b/drivers/gpu/msm/adreno_snapshot.h index 8b68eec2cf6a..6b22141b4ed2 100644 --- a/drivers/gpu/msm/adreno_snapshot.h +++ b/drivers/gpu/msm/adreno_snapshot.h @@ -56,5 +56,15 @@ void adreno_parse_ib(struct kgsl_device *device, struct kgsl_snapshot *snapshot, struct kgsl_process_private *process, u64 gpuaddr, u64 dwords); - +/** + * adreno_snapshot_global - Add global buffer to snapshot + * @device: Pointer to the kgsl device + * @buf: Where the global buffer section is to be written + * @remain: Remaining bytes in snapshot buffer + * @priv: Opaque data + * + * Return: Number of bytes written to the snapshot buffer + */ +size_t adreno_snapshot_global(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv); #endif /*__ADRENO_SNAPSHOT_H */