diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.c b/drivers/gpu/msm/adreno_a6xx_hfi.c index fa25862a4d93..c162fe6c13ed 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hfi.c @@ -138,6 +138,15 @@ int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, uint32_t queue_idx, hdr->write_index = write; + return 0; +} + +int a6xx_hfi_cmdq_write(struct adreno_device *adreno_dev, u32 *msg) +{ + int ret; + + ret = a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, msg); + /* * Memory barrier to make sure packet and write index are written before * an interrupt is raised @@ -145,10 +154,12 @@ int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, uint32_t queue_idx, wmb(); /* Send interrupt to GMU to receive the message */ - gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_HOST2GMU_INTR_SET, - 0x1); + if (!ret) + gmu_core_regwrite(KGSL_DEVICE(adreno_dev), + A6XX_GMU_HOST2GMU_INTR_SET, + 0x1); - return 0; + return ret; } /* Sizes of the queue and message are in unit of dwords */ @@ -281,7 +292,7 @@ static int poll_gmu_reg(struct adreno_device *adreno_dev, } static int a6xx_hfi_send_cmd_wait_inline(struct adreno_device *adreno_dev, - uint32_t queue_idx, void *data, struct pending_cmd *ret_cmd) + void *data, struct pending_cmd *ret_cmd) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); @@ -292,11 +303,11 @@ static int a6xx_hfi_send_cmd_wait_inline(struct adreno_device *adreno_dev, *cmd = MSG_HDR_SET_SEQNUM(*cmd, seqnum); if (ret_cmd == NULL) - return a6xx_hfi_queue_write(adreno_dev, queue_idx, cmd); + return a6xx_hfi_cmdq_write(adreno_dev, cmd); ret_cmd->sent_hdr = cmd[0]; - rc = a6xx_hfi_queue_write(adreno_dev, queue_idx, cmd); + rc = a6xx_hfi_cmdq_write(adreno_dev, cmd); if (rc) return rc; @@ -322,15 +333,14 @@ static int a6xx_hfi_send_cmd_wait_inline(struct adreno_device *adreno_dev, #define HFI_ACK_ERROR 0xffffffff -int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, - uint32_t queue, void *cmd) +int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, void *cmd) { struct pending_cmd ret_cmd; int rc; memset(&ret_cmd, 0, sizeof(ret_cmd)); - rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, queue, cmd, &ret_cmd); + rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, cmd, &ret_cmd); if (!rc && ret_cmd.results[2] == HFI_ACK_ERROR) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -356,7 +366,7 @@ static int a6xx_hfi_send_gmu_init(struct adreno_device *adreno_dev) .boot_state = 0x1, }; - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } static int a6xx_hfi_get_fw_version(struct adreno_device *adreno_dev, @@ -372,8 +382,7 @@ static int a6xx_hfi_get_fw_version(struct adreno_device *adreno_dev, memset(&ret_cmd, 0, sizeof(ret_cmd)); - rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, HFI_CMD_ID, &cmd, - &ret_cmd); + rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, &cmd, &ret_cmd); if (rc) return rc; @@ -394,7 +403,7 @@ int a6xx_hfi_send_core_fw_start(struct adreno_device *adreno_dev) .handle = 0x0, }; - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } static const char * const a6xx_hfi_features[] = { @@ -423,7 +432,7 @@ int a6xx_hfi_send_feature_ctrl(struct adreno_device *adreno_dev, }; int ret; - ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); + ret = a6xx_hfi_send_generic_req(adreno_dev, &cmd); if (ret) dev_err(&gmu->pdev->dev, "Unable to %s feature %s (%d)\n", @@ -454,7 +463,7 @@ static int a6xx_hfi_send_dcvstbl_v1(struct adreno_device *adreno_dev) cmd.cx_votes[1].vote = table->cx_votes[1].vote; cmd.cx_votes[1].freq = table->cx_votes[1].freq; - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } static int a6xx_hfi_send_get_value(struct adreno_device *adreno_dev, @@ -468,8 +477,7 @@ static int a6xx_hfi_send_get_value(struct adreno_device *adreno_dev, cmd->hdr = CMD_MSG_HDR(H2F_MSG_GET_VALUE, sizeof(*cmd)); - rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, HFI_CMD_ID, cmd, - &ret_cmd); + rc = a6xx_hfi_send_cmd_wait_inline(adreno_dev, cmd, &ret_cmd); if (rc) return rc; @@ -485,7 +493,7 @@ static int a6xx_hfi_send_test(struct adreno_device *adreno_dev) .hdr = CMD_MSG_HDR(H2F_MSG_TEST, sizeof(cmd)), }; - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, &cmd); } static void receive_err_req(struct a6xx_gmu_device *gmu, void *rcvd) @@ -655,7 +663,7 @@ int a6xx_hfi_send_acd_feature_ctrl(struct adreno_device *adreno_dev) int ret = 0; if (adreno_dev->acd_enabled) { - ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, + ret = a6xx_hfi_send_generic_req(adreno_dev, &gmu->hfi.acd_table); if (!ret) ret = a6xx_hfi_send_feature_ctrl(adreno_dev, @@ -702,13 +710,12 @@ int a6xx_hfi_start(struct adreno_device *adreno_dev) if (GMU_VER_MAJOR(gmu->ver.hfi) < 2) result = a6xx_hfi_send_dcvstbl_v1(adreno_dev); else - result = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, + result = a6xx_hfi_send_generic_req(adreno_dev, &gmu->hfi.dcvs_table); if (result) goto err; - result = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, - &gmu->hfi.bw_table); + result = a6xx_hfi_send_generic_req(adreno_dev, &gmu->hfi.bw_table); if (result) goto err; @@ -795,7 +802,7 @@ int a6xx_hfi_send_req(struct adreno_device *adreno_dev, unsigned int id, cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, cmd); } case H2F_MSG_PREPARE_SLUMBER: { struct hfi_prep_slumber_cmd *cmd = data; @@ -805,14 +812,14 @@ int a6xx_hfi_send_req(struct adreno_device *adreno_dev, unsigned int id, cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, cmd); } case H2F_MSG_START: { struct hfi_start_cmd *cmd = data; cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, cmd); } case H2F_MSG_GET_VALUE: { return a6xx_hfi_send_get_value(adreno_dev, data); @@ -822,7 +829,7 @@ int a6xx_hfi_send_req(struct adreno_device *adreno_dev, unsigned int id, cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, cmd); } default: break; diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index 9df7dc3c9f2f..a2371b1d7fc6 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -493,6 +493,7 @@ struct hfi_context_pointers_cmd { uint32_t ctxt_id; uint64_t sop_addr; uint64_t eop_addr; + u64 user_ctxt_record_addr; } __packed; /* H2F */ @@ -517,6 +518,16 @@ struct hfi_context_bad_reply_cmd { uint32_t req_hdr; } __packed; +/* H2F */ +struct hfi_submit_cmd { + u32 hdr; + u32 ctxt_id; + u32 flags; + u32 ts; + u32 numibs; +} __packed; + + /** * struct pending_cmd - data structure to track outstanding HFI * command messages @@ -668,13 +679,11 @@ int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev); /** * a6xx_hfi_send_generic_req - Send a generic hfi packet * @adreno_dev: Pointer to the adreno device - * @queue: The destination hfi queue * @cmd: Pointer to the hfi packet header and data * * Return: 0 on success or negative error on failure */ -int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, - u32 queue, void *cmd); +int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, void *cmd); /** * a6xx_hfi_send_bcl_feature_ctrl - Send the bcl feature hfi packet @@ -694,4 +703,13 @@ int a6xx_hfi_send_bcl_feature_ctrl(struct adreno_device *adreno_dev); */ int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, u32 queue_idx, struct pending_cmd *ret_cmd); + +/** + * a6xx_hfi_cmdq_write - Write a command to command queue + * @adreno_dev: Pointer to the adreno device + * @msg: Data to be written to the queue + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_cmdq_write(struct adreno_device *adreno_dev, u32 *msg); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index f838c8ab46de..e25daccd49cb 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -96,19 +96,23 @@ static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd, static void process_msgq_irq(struct adreno_device *adreno_dev) { struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); u32 rcvd[MAX_RCVD_SIZE]; - if (a6xx_hfi_queue_read(to_a6xx_gmu(adreno_dev), - HFI_MSG_ID, rcvd, sizeof(rcvd)) <= 0) + if (a6xx_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd)) <= 0) return; - /* - * We are assuming that there is only one outstanding ack - * because hfi sending thread waits for completion while - * holding the device mutex - */ - if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) - a6xx_receive_ack_async(adreno_dev, rcvd, &hfi->pending_ack); + while (a6xx_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd)) > 0) { + + /* + * We are assuming that there is only one outstanding ack + * because hfi sending thread waits for completion while + * holding the device mutex + */ + if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) + a6xx_receive_ack_async(adreno_dev, rcvd, + &hfi->pending_ack); + } } /* HFI interrupt handler */ @@ -148,6 +152,9 @@ static irqreturn_t a6xx_hwsched_hfi_handler(int irq, void *data) return IRQ_HANDLED; } +#define HFI_IRQ_MSGQ_MASK BIT(0) +#define HFI_RSP_TIMEOUT 100 /* msec */ + static int wait_ack_completion(struct adreno_device *adreno_dev, u32 *cmd) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -200,7 +207,7 @@ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data) hfi->pending_ack.sent_hdr = cmd[0]; - rc = a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, cmd); + rc = a6xx_hfi_cmdq_write(adreno_dev, cmd); if (rc) return rc; @@ -448,7 +455,7 @@ static int mem_alloc_reply(struct adreno_device *adreno_dev, void *rcvd) out.hdr = ACK_MSG_HDR(F2H_MSG_MEM_ALLOC, sizeof(out)); out.req_hdr = in->hdr; - return a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, (u32 *)&out); + return a6xx_hfi_cmdq_write(adreno_dev, (u32 *)&out); } static int send_start_msg(struct adreno_device *adreno_dev) @@ -466,7 +473,7 @@ static int send_start_msg(struct adreno_device *adreno_dev) hfi->pending_ack.sent_hdr = cmd.hdr; - rc = a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, (u32 *)&cmd); + rc = a6xx_hfi_cmdq_write(adreno_dev, (u32 *)&cmd); if (rc) return rc; @@ -579,13 +586,11 @@ int a6xx_hwsched_hfi_start(struct adreno_device *adreno_dev) if (ret) goto err; - ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, - &gmu->hfi.dcvs_table); + ret = a6xx_hfi_send_generic_req(adreno_dev, &gmu->hfi.dcvs_table); if (ret) goto err; - ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, - &gmu->hfi.bw_table); + ret = a6xx_hfi_send_generic_req(adreno_dev, &gmu->hfi.bw_table); if (ret) goto err; @@ -714,3 +719,158 @@ int a6xx_hwsched_hfi_probe(struct adreno_device *adreno_dev) return 0; } + +#define CTXT_FLAG_PMODE 0x00000001 +#define CTXT_FLAG_SWITCH_INTERNAL 0x00000002 +#define CTXT_FLAG_SWITCH 0x00000008 +#define CTXT_FLAG_NOTIFY 0x00000020 +#define CTXT_FLAG_NO_FAULT_TOLERANCE 0x00000200 +#define CTXT_FLAG_PWR_RULE 0x00000800 +#define CTXT_FLAG_PRIORITY_MASK 0x0000f000 +#define CTXT_FLAG_IFH_NOP 0x00010000 +#define CTXT_FLAG_SECURE 0x00020000 +#define CTXT_FLAG_TYPE_MASK 0x01f00000 +#define CTXT_FLAG_TYPE_SHIFT 20 +#define CTXT_FLAG_TYPE_ANY 0 +#define CTXT_FLAG_TYPE_GL 1 +#define CTXT_FLAG_TYPE_CL 2 +#define CTXT_FLAG_TYPE_C2D 3 +#define CTXT_FLAG_TYPE_RS 4 +#define CTXT_FLAG_TYPE_VK 5 +#define CTXT_FLAG_TYPE_UNKNOWN 0x1e +#define CTXT_FLAG_PREEMPT_STYLE_MASK 0x0e000000 +#define CTXT_FLAG_PREEMPT_STYLE_SHIFT 25 +#define CTXT_FLAG_PREEMPT_STYLE_ANY 0 +#define CTXT_FLAG_PREEMPT_STYLE_RB 1 +#define CTXT_FLAG_PREEMPT_STYLE_FG 2 + +static int send_context_register(struct adreno_device *adreno_dev, + struct kgsl_context *context) +{ + struct hfi_register_ctxt_cmd cmd; + struct kgsl_pagetable *pt = context->proc_priv->pagetable; + + cmd.hdr = CMD_MSG_HDR(H2F_MSG_REGISTER_CONTEXT, sizeof(cmd)); + cmd.ctxt_id = context->id; + cmd.flags = CTXT_FLAG_NOTIFY | context->flags; + cmd.pt_addr = kgsl_mmu_pagetable_get_ttbr0(pt); + cmd.ctxt_idr = kgsl_mmu_pagetable_get_contextidr(pt); + cmd.ctxt_bank = kgsl_mmu_pagetable_get_context_bank(pt); + + return a6xx_hfi_send_cmd_async(adreno_dev, &cmd); +} + +static int send_context_pointers(struct adreno_device *adreno_dev, + struct kgsl_context *context) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct hfi_context_pointers_cmd cmd; + + cmd.hdr = CMD_MSG_HDR(H2F_MSG_CONTEXT_POINTERS, sizeof(cmd)); + cmd.ctxt_id = context->id; + cmd.sop_addr = MEMSTORE_ID_GPU_ADDR(device, context->id, soptimestamp); + cmd.eop_addr = MEMSTORE_ID_GPU_ADDR(device, context->id, eoptimestamp); + if (context->user_ctxt_record) + cmd.user_ctxt_record_addr = + context->user_ctxt_record->memdesc.gpuaddr; + else + cmd.user_ctxt_record_addr = 0; + + return a6xx_hfi_send_cmd_async(adreno_dev, &cmd); +} + +static int hfi_context_register(struct adreno_device *adreno_dev, + struct kgsl_context *context) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + + if (context->gmu_registered) + return 0; + + ret = send_context_register(adreno_dev, context); + if (ret) { + dev_err(&gmu->pdev->dev, + "Unable to register context %d: %d\n", + context->id, ret); + return ret; + } + + ret = send_context_pointers(adreno_dev, context); + if (ret) + dev_err(&gmu->pdev->dev, + "Unable to register context %d pointers: %d\n", + context->id, ret); + + if (!ret) + context->gmu_registered = true; + + return ret; +} + +#define HFI_DSP_IRQ_BASE 2 + +#define DISPQ_IRQ_BIT(_idx) BIT((_idx) + HFI_DSP_IRQ_BASE) + +int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags, + struct kgsl_drawobj_cmd *cmdobj) +{ + struct a6xx_hfi *hfi = to_a6xx_hfi(adreno_dev); + struct kgsl_memobj_node *ib; + int ret = 0; + u32 numibs = 0, cmd_sizebytes; + struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + struct hfi_issue_ib *issue_ib; + struct hfi_submit_cmd *cmd; + + ret = hfi_context_register(adreno_dev, drawobj->context); + if (ret) + return ret; + + /* Get the total IBs in the list */ + list_for_each_entry(ib, &cmdobj->cmdlist, node) + numibs++; + + /* Add a *issue_ib struct for each IB */ + cmd_sizebytes = sizeof(*cmd) + (sizeof(*issue_ib) * numibs); + + cmd = kvmalloc(cmd_sizebytes, GFP_KERNEL); + if (cmd == NULL) + return -ENOMEM; + + cmd->hdr = CMD_MSG_HDR(H2F_MSG_ISSUE_CMD, cmd_sizebytes); + cmd->hdr = MSG_HDR_SET_SEQNUM(cmd->hdr, + atomic_inc_return(&hfi->seqnum)); + + cmd->ctxt_id = drawobj->context->id; + cmd->flags = flags; + cmd->ts = drawobj->timestamp; + cmd->numibs = numibs; + + issue_ib = (struct hfi_issue_ib *)&cmd[1]; + + list_for_each_entry(ib, &cmdobj->cmdlist, node) { + issue_ib->addr = ib->gpuaddr; + issue_ib->size = ib->size; + issue_ib++; + } + + ret = a6xx_hfi_queue_write(adreno_dev, HFI_DSP_ID_0, (u32 *)cmd); + if (ret) + goto free; + + /* + * Memory barrier to make sure packet and write index are written before + * an interrupt is raised + */ + wmb(); + + /* Send interrupt to GMU to receive the message */ + gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_HOST2GMU_INTR_SET, + DISPQ_IRQ_BIT(0)); + +free: + kvfree(cmd); + + return ret; +} diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index d1c6907d6760..d314a0bd9961 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -113,6 +113,8 @@ struct a6xx_hwsched_hfi { u32 irq_mask; }; +struct kgsl_drawobj_cmd; + /** * a6xx_hwsched_hfi_probe - Probe hwsched hfi resources * @adreno_dev: Pointer to adreno device structure @@ -172,4 +174,18 @@ int a6xx_hwsched_cp_init(struct adreno_device *adreno_dev); * Return: 0 on success and negative error on failure. */ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data); + +/** + * a6xx_hwsched_submit_cmdobj - Dispatch IBs to dispatch queues + * @adreno_dev: Pointer to adreno device structure + * @flags: Flags associated with the submission + * @cmdobj: The command object which needs to be submitted + * + * This function is used to register the context if needed and submit + * IBs to the hfi dispatch queues. + + * Return: 0 on success and negative error on failure + */ +int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags, + struct kgsl_drawobj_cmd *cmdobj); #endif diff --git a/drivers/gpu/msm/kgsl_device.h b/drivers/gpu/msm/kgsl_device.h index 335321b4f819..836b49c81c57 100644 --- a/drivers/gpu/msm/kgsl_device.h +++ b/drivers/gpu/msm/kgsl_device.h @@ -374,6 +374,7 @@ struct kgsl_process_private; * across preemption * @total_fault_count: number of times gpu faulted in this context * @last_faulted_cmd_ts: last faulted command batch timestamp + * @gmu_registered: whether context is registered with gmu or not */ struct kgsl_context { struct kref refcount; @@ -395,6 +396,7 @@ struct kgsl_context { struct kgsl_mem_entry *user_ctxt_record; unsigned int total_fault_count; unsigned int last_faulted_cmd_ts; + bool gmu_registered; }; #define _context_comm(_c) \