From dde333681cee19a5dea528901ecf536ab9f3ae22 Mon Sep 17 00:00:00 2001 From: Harshdeep Dhatt Date: Thu, 25 Jun 2020 13:00:37 -0600 Subject: [PATCH] msm: kgsl: Add memalloc hfi support Allocate gpu globals based on memalloc hfi requests. Map them in gmu as well. Change-Id: I43ecfedf5be92e39fd079a00f8311fe6aca7191b Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/adreno_a6xx_gmu.c | 13 +- drivers/gpu/msm/adreno_a6xx_gmu.h | 14 + drivers/gpu/msm/adreno_a6xx_hfi.c | 32 +- drivers/gpu/msm/adreno_a6xx_hfi.h | 103 +++++- drivers/gpu/msm/adreno_a6xx_hwsched.c | 27 +- drivers/gpu/msm/adreno_a6xx_hwsched.h | 12 + drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c | 376 +++++++++++++++++++++- drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h | 121 +++++++ 8 files changed, 646 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index d34bc874556e..ce7d9867bac9 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -618,10 +618,10 @@ static int find_vma_block(struct a6xx_gmu_device *gmu, u32 addr, u32 size) return -ENOENT; } -static struct iommu_domain *get_gmu_domain(struct a6xx_gmu_device *gmu, - struct gmu_memdesc *md) +struct iommu_domain *a6xx_get_gmu_domain(struct a6xx_gmu_device *gmu, + u32 gmuaddr, u32 size) { - u32 vma_id = find_vma_block(gmu, md->gmuaddr, md->size); + u32 vma_id = find_vma_block(gmu, gmuaddr, size); if (vma_id == GMU_NONCACHED_USER) return a6xx_gmu_ctx[GMU_CONTEXT_USER].domain; @@ -1390,8 +1390,8 @@ struct gmu_memdesc *reserve_gmu_kernel_block(struct a6xx_gmu_device *gmu, md->gmuaddr = addr; - ret = iommu_map(get_gmu_domain(gmu, md), addr, md->physaddr, md->size, - IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV); + ret = iommu_map(a6xx_get_gmu_domain(gmu, md->gmuaddr, md->size), addr, + md->physaddr, md->size, IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV); if (ret) { dev_err(&gmu->pdev->dev, "Unable to map GMU kernel block: addr:0x%08x size:0x%x :%d\n", @@ -2393,7 +2393,8 @@ static void a6xx_free_gmu_globals(struct a6xx_gmu_device *gmu) if (!md->gmuaddr) continue; - iommu_unmap(get_gmu_domain(gmu, md), md->gmuaddr, md->size); + iommu_unmap(a6xx_get_gmu_domain(gmu, md->gmuaddr, md->size), + md->gmuaddr, md->size); dma_free_attrs(&gmu->pdev->dev, (size_t) md->size, (void *)md->hostptr, md->physaddr, 0); diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.h b/drivers/gpu/msm/adreno_a6xx_gmu.h index e3796da39938..4b40b394826b 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.h +++ b/drivers/gpu/msm/adreno_a6xx_gmu.h @@ -524,4 +524,18 @@ int a6xx_gmu_enable_clks(struct adreno_device *adreno_dev); * Return: 0 on success or negative error on failure */ int a6xx_gmu_enable_gdsc(struct adreno_device *adreno_dev); + +/** + * a6xx_get_gmu_domain - Get the gmu iommu domain for a gmu memory block + * @gmu: Pointer to the a6xx gmu device + * @gmuaddr: Address of the memory block + * @size: Size in bytes of the memory block + * + * Based on the gmu address and size of a gmu memory block, get the gmu iommu + * domain to map the memory block to. + * + * Return: gmu iommu domain to which the given memory block is to be mapped + */ +struct iommu_domain *a6xx_get_gmu_domain(struct a6xx_gmu_device *gmu, + u32 gmuaddr, u32 size); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.c b/drivers/gpu/msm/adreno_a6xx_hfi.c index 3cdc16f34ef3..2fde866540b5 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hfi.c @@ -32,7 +32,7 @@ struct a6xx_hfi *to_a6xx_hfi(struct adreno_device *adreno_dev) } /* Size in below functions are in unit of dwords */ -static int a6xx_hfi_queue_read(struct a6xx_gmu_device *gmu, uint32_t queue_idx, +int a6xx_hfi_queue_read(struct a6xx_gmu_device *gmu, uint32_t queue_idx, unsigned int *output, unsigned int max_size) { struct gmu_memdesc *mem_addr = gmu->hfi.hfi_mem; @@ -91,8 +91,8 @@ done: } /* Size in below functions are in unit of dwords */ -static int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, - uint32_t queue_idx, uint32_t *msg) +int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, uint32_t queue_idx, + uint32_t *msg) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_queue_table *tbl = gmu->hfi.hfi_mem->hostptr; @@ -154,11 +154,6 @@ static int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, return 0; } -#define QUEUE_HDR_TYPE(id, prio, rtype, stype) \ - (((id) & 0xFF) | (((prio) & 0xFF) << 8) | \ - (((rtype) & 0xFF) << 16) | (((stype) & 0xFF) << 24)) - - /* Sizes of the queue and message are in unit of dwords */ static void init_queues(struct adreno_device *adreno_dev) { @@ -228,7 +223,7 @@ int a6xx_hfi_init(struct adreno_device *adreno_dev) #define HDR_CMP_SEQNUM(out_hdr, in_hdr) \ (MSG_HDR_GET_SEQNUM(out_hdr) == MSG_HDR_GET_SEQNUM(in_hdr)) -static int receive_ack_cmd(struct a6xx_gmu_device *gmu, void *rcvd, +int a6xx_receive_ack_cmd(struct a6xx_gmu_device *gmu, void *rcvd, struct pending_cmd *ret_cmd) { struct adreno_device *adreno_dev = a6xx_gmu_to_adreno(gmu); @@ -259,9 +254,6 @@ static int receive_ack_cmd(struct a6xx_gmu_device *gmu, void *rcvd, return -ENODEV; } -#define MSG_HDR_SET_SEQNUM(hdr, num) \ - (((hdr) & 0xFFFFF) | ((num) << 20)) - static int poll_gmu_reg(struct adreno_device *adreno_dev, u32 offsetdwords, unsigned int expected_val, unsigned int mask, unsigned int timeout_ms) @@ -336,7 +328,7 @@ static int a6xx_hfi_send_cmd(struct adreno_device *adreno_dev, #define HFI_ACK_ERROR 0xffffffff -static int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, +int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, uint32_t queue, void *cmd) { struct pending_cmd ret_cmd; @@ -400,7 +392,7 @@ static int a6xx_hfi_get_fw_version(struct adreno_device *adreno_dev, return rc; } -static int a6xx_hfi_send_core_fw_start(struct adreno_device *adreno_dev) +int a6xx_hfi_send_core_fw_start(struct adreno_device *adreno_dev) { struct hfi_core_fw_start_cmd cmd = { .hdr = CMD_MSG_HDR(H2F_MSG_CORE_FW_START, sizeof(cmd)), @@ -424,7 +416,7 @@ static const char *feature_to_string(uint32_t feature) return "unknown"; } -static int a6xx_hfi_send_feature_ctrl(struct adreno_device *adreno_dev, +int a6xx_hfi_send_feature_ctrl(struct adreno_device *adreno_dev, uint32_t feature, uint32_t enable, uint32_t data) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); @@ -523,7 +515,7 @@ static void a6xx_hfi_v1_receiver(struct a6xx_gmu_device *gmu, uint32_t *rcvd, { /* V1 ACK Handler */ if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_V1_MSG_ACK) { - receive_ack_cmd(gmu, rcvd, ret_cmd); + a6xx_receive_ack_cmd(gmu, rcvd, ret_cmd); return; } @@ -557,7 +549,7 @@ static int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, /* V2 ACK Handler */ if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) { - int ret = receive_ack_cmd(gmu, rcvd, ret_cmd); + int ret = a6xx_receive_ack_cmd(gmu, rcvd, ret_cmd); if (ret) return ret; @@ -622,7 +614,7 @@ static int a6xx_hfi_verify_fw_version(struct adreno_device *adreno_dev) return 0; } -static int a6xx_hfi_send_bcl_feature_ctrl(struct adreno_device *adreno_dev) +int a6xx_hfi_send_bcl_feature_ctrl(struct adreno_device *adreno_dev) { int ret; @@ -634,7 +626,7 @@ static int a6xx_hfi_send_bcl_feature_ctrl(struct adreno_device *adreno_dev) return ret; } -static int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev) +int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct hfi_set_value_cmd req; @@ -661,7 +653,7 @@ static int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev) return ret; } -static int a6xx_hfi_send_acd_feature_ctrl(struct adreno_device *adreno_dev) +int a6xx_hfi_send_acd_feature_ctrl(struct adreno_device *adreno_dev) { struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); int ret = 0; diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h index bcd647fbb42d..88ce3f7f7f36 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -35,9 +35,6 @@ #define HFI_DBG_PRI 40 #define HFI_DSP_PRI_0 20 -#define HFI_RSP_TIMEOUT 100 /* msec */ - -#define HFI_IRQ_MSGQ_MASK BIT(0) #define HFI_IRQ_SIDEMSGQ_MASK BIT(1) #define HFI_IRQ_DBGQ_MASK BIT(2) #define HFI_IRQ_CM3_FAULT_MASK BIT(15) @@ -159,6 +156,17 @@ struct hfi_queue_table { #define MSG_HDR_GET_TYPE(hdr) (((hdr) >> 16) & 0xF) #define MSG_HDR_GET_SEQNUM(hdr) (((hdr) >> 20) & 0xFFF) +#define MSG_HDR_SET_SEQNUM(hdr, num) \ + (((hdr) & 0xFFFFF) | ((num) << 20)) + +#define QUEUE_HDR_TYPE(id, prio, rtype, stype) \ + (((id) & 0xFF) | (((prio) & 0xFF) << 8) | \ + (((rtype) & 0xFF) << 16) | (((stype) & 0xFF) << 24)) + +#define HFI_RSP_TIMEOUT 100 /* msec */ + +#define HFI_IRQ_MSGQ_MASK BIT(0) + #define H2F_MSG_INIT 0 #define H2F_MSG_FW_VER 1 #define H2F_MSG_LM_CFG 2 @@ -578,4 +586,93 @@ int a6xx_hfi_send_req(struct adreno_device *adreno_dev, /* Helper function to get to a6xx hfi struct from adreno device */ struct a6xx_hfi *to_a6xx_hfi(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_queue_write - Write a command to hfi queue + * @adreno_dev: Pointer to the adreno device + * @queue_idx: destination queue id + * @msg: Data to be written to the queue + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, u32 queue_idx, + u32 *msg); + +/** + * a6xx_hfi_queue_read - Read data from hfi queue + * @gmu: Pointer to the a6xx gmu device + * @queue_idx: queue id to read from + * @output: Pointer to read the data into + * @max_size: Number of bytes to read from the queue + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_queue_read(struct a6xx_gmu_device *gmu, u32 queue_idx, + u32 *output, u32 max_size); + +/** + * a6xx_receive_ack_cmd - Process ack type packets + * @gmu: Pointer to the a6xx gmu device + * @rcvd: Pointer to the data read from hfi queue + * @ret_cmd: Container for the hfi packet for which this ack is received + * + * Return: 0 on success or negative error on failure + */ +int a6xx_receive_ack_cmd(struct a6xx_gmu_device *gmu, void *rcvd, + struct pending_cmd *ret_cmd); + +/** + * a6xx_hfi_send_feature_ctrl - Enable gmu feature via hfi + * @adreno_dev: Pointer to the adreno device + * @feature: feature to be enabled or disabled + * enable: Set 1 to enable or 0 to disable a feature + * @data: payload for the send feature hfi packet + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_send_feature_ctrl(struct adreno_device *adreno_dev, + u32 feature, u32 enable, u32 data); + +/** + * a6xx_hfi_send_core_fw_start - Send the core fw start hfi + * @adreno_dev: Pointer to the adreno device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_send_core_fw_start(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_send_acd_feature_ctrl - Send the acd table and acd feature + * @adreno_dev: Pointer to the adreno device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_send_acd_feature_ctrl(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_send_lm_feature_ctrl - Send the lm feature hfi packet + * @adreno_dev: Pointer to the adreno device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_send_generic_req - Send a generic hfi packet + * @adreno_dev: Pointer to the adreno device + * @queue: The destination hfi queue + * @cmd: Pointer to the hfi packet header and data + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, + u32 queue, void *cmd); + +/** + * a6xx_hfi_send_bcl_feature_ctrl - Send the bcl feature hfi packet + * @adreno_dev: Pointer to the adreno device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_send_bcl_feature_ctrl(struct adreno_device *adreno_dev); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.c b/drivers/gpu/msm/adreno_a6xx_hwsched.c index 52252d6b7d38..23f826ed7d58 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.c @@ -11,7 +11,6 @@ #include "adreno.h" #include "adreno_a6xx.h" #include "adreno_a6xx_hwsched.h" -#include "adreno_a6xx_hwsched_hfi.h" #include "kgsl_device.h" #include "kgsl_trace.h" #include "kgsl_util.h" @@ -67,11 +66,7 @@ static int a6xx_hwsched_gmu_first_boot(struct adreno_device *adreno_dev) if (ret) goto err; - ret = a6xx_gmu_hfi_start(adreno_dev); - if (ret) - goto err; - - ret = a6xx_hfi_start(adreno_dev); + ret = a6xx_hwsched_hfi_start(adreno_dev); if (ret) goto err; @@ -132,11 +127,7 @@ static int a6xx_hwsched_gmu_boot(struct adreno_device *adreno_dev) if (ret) goto err; - ret = a6xx_gmu_hfi_start(adreno_dev); - if (ret) - goto err; - - ret = a6xx_hfi_start(adreno_dev); + ret = a6xx_hwsched_hfi_start(adreno_dev); if (ret) goto err; @@ -215,7 +206,7 @@ static int a6xx_hwsched_gmu_power_off(struct adreno_device *adreno_dev) a6xx_gmu_irq_disable(adreno_dev); - a6xx_hfi_stop(adreno_dev); + a6xx_hwsched_hfi_stop(adreno_dev); clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks); @@ -226,7 +217,7 @@ static int a6xx_hwsched_gmu_power_off(struct adreno_device *adreno_dev) return ret; error: - a6xx_hfi_stop(adreno_dev); + a6xx_hwsched_hfi_stop(adreno_dev); a6xx_gmu_suspend(adreno_dev); return ret; @@ -565,15 +556,15 @@ int a6xx_hwsched_probe(struct platform_device *pdev, { struct adreno_device *adreno_dev; struct kgsl_device *device; - struct a6xx_device *a6xx_dev; + struct a6xx_hwsched_device *a6xx_hwsched_dev; int ret; - a6xx_dev = devm_kzalloc(&pdev->dev, sizeof(*a6xx_dev), - GFP_KERNEL); - if (!a6xx_dev) + a6xx_hwsched_dev = devm_kzalloc(&pdev->dev, sizeof(*a6xx_hwsched_dev), + GFP_KERNEL); + if (!a6xx_hwsched_dev) return -ENOMEM; - adreno_dev = &a6xx_dev->adreno_dev; + adreno_dev = &a6xx_hwsched_dev->a6xx_dev.adreno_dev; ret = a6xx_probe_common(pdev, adreno_dev, chipid, gpucore); if (ret) diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched.h b/drivers/gpu/msm/adreno_a6xx_hwsched.h index 90568b29893a..ec985908b2fd 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched.h @@ -6,6 +6,18 @@ #ifndef _ADRENO_A6XX_HWSCHED_H_ #define _ADRENO_A6XX_HWSCHED_H_ +#include "adreno_a6xx_hwsched_hfi.h" + +/** + * struct a6xx_hwsched_device - Container for the a6xx hwscheduling device + */ +struct a6xx_hwsched_device { + /** @a6xx_dev: Container for the a6xx device */ + struct a6xx_device a6xx_dev; + /** @hwsched_hfi: Container for hwscheduling specific hfi resources */ + struct a6xx_hwsched_hfi hwsched_hfi; +}; + /** * a6xx_hwsched_probe - Target specific probe for hwsched * @pdev: Pointer to the platform device diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c index ad2e658d2c2d..8db3f9c05b9b 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c @@ -3,20 +3,17 @@ * Copyright (c) 2020, The Linux Foundation. All rights reserved. */ +#include #include "adreno.h" #include "adreno_a6xx.h" -#include "adreno_a6xx_hwsched_hfi.h" +#include "adreno_a6xx_hwsched.h" #include "kgsl_device.h" #include "kgsl_pwrctrl.h" #include "kgsl_trace.h" #define HFI_QUEUE_MAX (HFI_QUEUE_DEFAULT_CNT + HFI_QUEUE_DISPATCH_MAX_CNT) -#define QUEUE_HDR_TYPE(id, prio, rtype, stype) \ - (((id) & 0xFF) | (((prio) & 0xFF) << 8) | \ - (((rtype) & 0xFF) << 16) | (((stype) & 0xFF) << 24)) - #define DEFINE_QHDR(gmuaddr, id, prio) \ {\ .status = 1, \ @@ -33,6 +30,36 @@ .write_index = 0, \ } +static const char * const memkind_strings[] = { + [MEMKIND_GENERIC] = "GMU GENERIC", + [MEMKIND_RB] = "GMU RB", + [MEMKIND_SCRATCH] = "GMU SCRATCH", + [MEMKIND_CSW_SMMU_INFO] = "GMU SMMU INFO", + [MEMKIND_CSW_PRIV_NON_SECURE] = "GMU CSW PRIV NON SECURE", + [MEMKIND_CSW_PRIV_SECURE] = "GMU CSW PRIV SECURE", + [MEMKIND_CSW_NON_PRIV] = "GMU CSW NON PRIV", + [MEMKIND_CSW_COUNTER] = "GMU CSW COUNTER", + [MEMKIND_CTXTREC_PREEMPT_CNTR] = "GMU PREEMPT CNTR", + [MEMKIND_SYS_LOG] = "GMU SYS LOG", + [MEMKIND_CRASH_DUMP] = "GMU CRASHDUMP", + [MEMKIND_MMIO_DPU] = "GMU MMIO DPU", + [MEMKIND_MMIO_TCSR] = "GMU MMIO TCSR", + [MEMKIND_MMIO_QDSS_STM] = "GMU MMIO QDSS STM", + [MEMKIND_PROFILE] = "GMU KERNEL PROFILING", + [MEMKIND_USER_PROFILE_IBS] = "GMU USER PROFILING", +}; + +static struct a6xx_hwsched_hfi *to_a6xx_hwsched_hfi( + struct adreno_device *adreno_dev) +{ + struct a6xx_device *a6xx_dev = container_of(adreno_dev, + struct a6xx_device, adreno_dev); + struct a6xx_hwsched_device *a6xx_hwsched = container_of(a6xx_dev, + struct a6xx_hwsched_device, a6xx_dev); + + return &a6xx_hwsched->hwsched_hfi; +} + static void init_queues(struct a6xx_hfi *hfi) { u32 gmuaddr = hfi->hfi_mem->gmuaddr; @@ -91,3 +118,342 @@ int a6xx_hwsched_hfi_init(struct adreno_device *adreno_dev) return PTR_ERR_OR_ZERO(hfi->hfi_mem); } + +static int get_attrs(u32 flags) +{ + int attrs = IOMMU_READ; + + if (flags & MEMFLAG_GMU_PRIV) + attrs |= IOMMU_PRIV; + + if (flags & MEMFLAG_GMU_WRITEABLE) + attrs |= IOMMU_WRITE; + + return attrs; +} + +static int gmu_import_buffer(struct adreno_device *adreno_dev, + struct mem_alloc_entry *entry, u32 flags) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int attrs = get_attrs(flags); + struct sg_table *sgt; + struct kgsl_memdesc *gpu_md = entry->gpu_md; + int mapped = 0; + struct gmu_vma_entry *vma = &gmu->vma[GMU_NONCACHED_KERNEL]; + struct hfi_mem_alloc_desc *desc = &entry->desc; + + if (flags & MEMFLAG_GMU_CACHEABLE) + vma = &gmu->vma[GMU_CACHE]; + + if ((vma->next_va + desc->size) > (vma->start + vma->size)) { + dev_err(&gmu->pdev->dev, + "GMU mapping too big. available: %d required: %d\n", + vma->next_va - vma->start, desc->size); + return -ENOMEM; + } + + /* Alloc sgt for map and then free it */ + if (gpu_md->pages != NULL) + sgt = kgsl_alloc_sgt_from_pages(gpu_md); + else + sgt = gpu_md->sgt; + + if (IS_ERR(sgt)) + return -ENOMEM; + + desc->gmu_addr = vma->next_va; + + mapped = iommu_map_sg(a6xx_get_gmu_domain(gmu, desc->gmu_addr, + desc->size), + desc->gmu_addr, sgt->sgl, sgt->nents, attrs); + if (mapped == 0) + dev_err(&gmu->pdev->dev, "gmu map sg err: 0x%08x, %d, %x, %zd\n", + desc->gmu_addr, sgt->nents, attrs, mapped); + else + vma->next_va += desc->size; + + if (gpu_md->pages != NULL) + kgsl_free_sgt(sgt); + + return ((mapped == 0) ? -ENOMEM : 0); +} + +static struct mem_alloc_entry *lookup_mem_alloc_table( + struct adreno_device *adreno_dev, struct hfi_mem_alloc_desc *desc) +{ + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + int i; + + for (i = 0; i < hfi->mem_alloc_entries; i++) { + struct mem_alloc_entry *entry = &hfi->mem_alloc_table[i]; + + if ((entry->desc.mem_kind == desc->mem_kind) && + (entry->desc.gmu_mem_handle == desc->gmu_mem_handle) && + entry->gpu_md->gpuaddr) + return entry; + } + + return NULL; +} + +static struct mem_alloc_entry *get_mem_alloc_entry( + struct adreno_device *adreno_dev, struct hfi_mem_alloc_desc *desc) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_hwsched_hfi *hfi = to_a6xx_hwsched_hfi(adreno_dev); + struct mem_alloc_entry *entry = + lookup_mem_alloc_table(adreno_dev, desc); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + u64 flags = 0; + u32 priv = 0; + int ret; + const char *memkind_string = desc->mem_kind < NUM_HFI_MEMKINDS ? + memkind_strings[desc->mem_kind] : "UNKNOWN"; + + if (entry) + return entry; + + if (hfi->mem_alloc_entries == ARRAY_SIZE(hfi->mem_alloc_table)) { + dev_err(&gmu->pdev->dev, + "Reached max mem alloc entries\n"); + return ERR_PTR(-ENOMEM); + } + + entry = &hfi->mem_alloc_table[hfi->mem_alloc_entries]; + + memcpy(&entry->desc, desc, sizeof(*desc)); + + entry->desc.host_mem_handle = desc->gmu_mem_handle; + + if (desc->flags & MEMFLAG_GFX_PRIV) + priv |= KGSL_MEMDESC_PRIVILEGED; + + if (!(desc->flags & MEMFLAG_GFX_WRITEABLE)) + flags |= KGSL_MEMFLAGS_GPUREADONLY; + + entry->gpu_md = kgsl_allocate_global(device, desc->size, flags, priv, + memkind_string); + if (IS_ERR(entry->gpu_md)) { + int ret = PTR_ERR(entry->gpu_md); + + memset(entry, 0, sizeof(*entry)); + return ERR_PTR(ret); + } + + entry->desc.size = entry->gpu_md->size; + + /* + * Map all buffers in GMU. If this fails, then we have to live with + * leaking the gpu global buffer allocated above. + */ + ret = gmu_import_buffer(adreno_dev, entry, desc->flags); + if (ret) { + dev_err(&gmu->pdev->dev, + "gpuaddr: 0x%llx size: %zd bytes lost\n", + entry->gpu_md->gpuaddr, entry->gpu_md->size); + memset(entry, 0, sizeof(*entry)); + return ERR_PTR(ret); + } + + hfi->mem_alloc_entries++; + + return entry; +} + +static int process_mem_alloc(struct adreno_device *adreno_dev, + struct hfi_mem_alloc_desc *mad) +{ + struct mem_alloc_entry *entry; + + entry = get_mem_alloc_entry(adreno_dev, mad); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + mad->gpu_addr = entry->gpu_md->gpuaddr; + mad->gmu_addr = entry->desc.gmu_addr; + + /* + * GMU uses the host_mem_handle to check if this memalloc was + * successful + */ + mad->host_mem_handle = mad->gmu_mem_handle; + + return 0; +} + +static int mem_alloc_reply(struct adreno_device *adreno_dev, void *rcvd) +{ + struct hfi_mem_alloc_cmd *in = (struct hfi_mem_alloc_cmd *)rcvd; + struct hfi_mem_alloc_reply_cmd out = {0}; + int ret; + + ret = process_mem_alloc(adreno_dev, &in->desc); + if (ret) + return ret; + + memcpy(&out.desc, &in->desc, sizeof(out.desc)); + + out.hdr = ACK_MSG_HDR(F2H_MSG_MEM_ALLOC, sizeof(out)); + out.req_hdr = in->hdr; + + return a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, (u32 *)&out); +} + +static int send_start_msg(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int seqnum = atomic_inc_return(&gmu->hfi.seqnum); + struct pending_cmd ret_cmd = {0}; + int rc = 0; + struct hfi_start_cmd cmd; + u32 rcvd[MAX_RCVD_SIZE]; + + cmd.hdr = CMD_MSG_HDR(H2F_MSG_START, sizeof(cmd)); + cmd.hdr = MSG_HDR_SET_SEQNUM(cmd.hdr, seqnum); + + ret_cmd.sent_hdr = cmd.hdr; + + rc = a6xx_hfi_queue_write(adreno_dev, HFI_CMD_ID, (u32 *)&cmd); + if (rc) + return rc; + +poll: + rc = timed_poll_check(device, A6XX_GMU_GMU2HOST_INTR_INFO, + HFI_IRQ_MSGQ_MASK, HFI_RSP_TIMEOUT, HFI_IRQ_MSGQ_MASK); + + if (rc) { + dev_err(&gmu->pdev->dev, + "Timed out processing MSG_START seqnum: %d\n", + seqnum); + gmu_fault_snapshot(device); + return rc; + } + + /* Clear the interrupt */ + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, + HFI_IRQ_MSGQ_MASK); + + if (a6xx_hfi_queue_read(gmu, HFI_MSG_ID, rcvd, sizeof(rcvd)) <= 0) { + dev_err(&gmu->pdev->dev, "MSG_START: no payload\n"); + gmu_fault_snapshot(device); + return -EINVAL; + } + + if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) + return a6xx_receive_ack_cmd(gmu, rcvd, &ret_cmd); + + if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_MEM_ALLOC) { + rc = mem_alloc_reply(adreno_dev, rcvd); + if (rc) + return rc; + + goto poll; + } + + dev_err(&gmu->pdev->dev, + "MSG_START: unexpected response id:%d, type:%d\n", + MSG_HDR_GET_ID(rcvd[0]), + MSG_HDR_GET_TYPE(rcvd[0])); + gmu_fault_snapshot(device); + + return rc; +} + +static void reset_hfi_queues(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct hfi_queue_table *tbl = gmu->hfi.hfi_mem->hostptr; + u32 i; + + /* Flush HFI queues */ + for (i = 0; i < HFI_QUEUE_MAX; i++) { + struct hfi_queue_header *hdr = &tbl->qhdr[i]; + + if (hdr->status == HFI_QUEUE_STATUS_DISABLED) + continue; + + if (hdr->read_index != hdr->write_index) { + dev_err(&gmu->pdev->dev, + "HFI queue[%d] is not empty before close: rd=%d,wt=%d\n", + i, hdr->read_index, hdr->write_index); + hdr->read_index = hdr->write_index; + + gmu_fault_snapshot(KGSL_DEVICE(adreno_dev)); + } + } +} + +void a6xx_hwsched_hfi_stop(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + reset_hfi_queues(adreno_dev); + + kgsl_pwrctrl_axi(KGSL_DEVICE(adreno_dev), KGSL_PWRFLAGS_OFF); + + clear_bit(GMU_PRIV_HFI_STARTED, &gmu->flags); + +} + +int a6xx_hwsched_hfi_start(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = a6xx_gmu_hfi_start(adreno_dev); + if (ret) + goto err; + + ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, + &gmu->hfi.dcvs_table); + if (ret) + goto err; + + ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, + &gmu->hfi.bw_table); + if (ret) + goto err; + + ret = a6xx_hfi_send_acd_feature_ctrl(adreno_dev); + if (ret) + goto err; + + ret = a6xx_hfi_send_lm_feature_ctrl(adreno_dev); + if (ret) + goto err; + + ret = a6xx_hfi_send_bcl_feature_ctrl(adreno_dev); + if (ret) + goto err; + + ret = a6xx_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_HWSCHED, 1, 0); + if (ret) + goto err; + + ret = a6xx_hfi_send_core_fw_start(adreno_dev); + if (ret) + goto err; + + ret = send_start_msg(adreno_dev); + if (ret) + goto err; + + set_bit(GMU_PRIV_HFI_STARTED, &gmu->flags); + + /* Request default DCVS level */ + ret = kgsl_pwrctrl_set_default_gpu_pwrlevel(device); + if (ret) + goto err; + + /* Request default BW vote */ + ret = kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_ON); + +err: + if (ret) + a6xx_hwsched_hfi_stop(adreno_dev); + + return ret; +} diff --git a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h index 5686569ad0fc..e3c37395bec0 100644 --- a/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h @@ -6,6 +6,109 @@ #ifndef _ADRENO_A6XX_HWSCHED_HFI_H_ #define _ADRENO_A6XX_HWSCHED_HFI_H_ +enum mem_kind { + /** @MEMKIND_GENERIC: Used for requesting generic memory */ + MEMKIND_GENERIC = 0, + /** @MEMKIND_RB: Used for requesting ringbuffer memory */ + MEMKIND_RB, + /** @MEMKIND_SCRATCH: Used for requesting scratch memory */ + MEMKIND_SCRATCH, + /** + * @MEMKIND_CSW_SMMU_INFO: Used for requesting SMMU record for + * preemption context switching + */ + MEMKIND_CSW_SMMU_INFO, + /** + * @MEMKIND_CSW_PRIV_NON_SECURE: Used for requesting privileged non + * secure preemption records + */ + MEMKIND_CSW_PRIV_NON_SECURE, + /** + * @MEMKIND_CSW_PRIV_SECURE: Used for requesting privileged secure + * preemption records + */ + MEMKIND_CSW_PRIV_SECURE, + /** + * @MEMKIND_CSW_NON_PRIV: Used for requesting non privileged per context + * preemption buffer + */ + MEMKIND_CSW_NON_PRIV, + /** + * @MEMKIND_CSW_COUNTER: Used for requesting preemption performance + * counter save/restore buffer + */ + MEMKIND_CSW_COUNTER, + /** + * @MEMKIND_CTXTREC_PREEMPT_CNTR: Used for requesting preemption + * counter buffer + */ + MEMKIND_CTXTREC_PREEMPT_CNTR, + /** @MEMKIND_SYSLOG: Used for requesting system log memory */ + MEMKIND_SYS_LOG, + /** @MEMKIND_CRASH_DUMP: Used for requesting carsh dumper memory */ + MEMKIND_CRASH_DUMP, + /** + * @MEMKIND_MMIO_DPU: Used for requesting Display processing unit's + * register space + */ + MEMKIND_MMIO_DPU, + /** + * @MEMKIND_MMIO_TCSR: Used for requesting Top CSR(contains SoC + * doorbells) register space + */ + MEMKIND_MMIO_TCSR, + /** + * @MEMKIND_MMIO_QDSS_STM: Used for requesting QDSS STM register space + */ + MEMKIND_MMIO_QDSS_STM, + /** @MEMKIND_PROFILE: Used for kernel profiling */ + MEMKIND_PROFILE, + /** @MEMKIND_USER_PROFILING_IBS: Used for user profiling */ + MEMKIND_USER_PROFILE_IBS, + NUM_HFI_MEMKINDS, + MEMKIND_NONE = 0x7fffffff, +}; + +/* CP/GFX pipeline can access */ +#define MEMFLAG_GFX_ACC BIT(0) + +/* Buffer has APRIV protection in GFX PTEs */ +#define MEMFLAG_GFX_PRIV BIT(1) + +/* Buffer is read-write for GFX PTEs. A 0 indicates read-only */ +#define MEMFLAG_GFX_WRITEABLE BIT(2) + +/* GMU can access */ +#define MEMFLAG_GMU_ACC BIT(3) + +/* Buffer has APRIV protection in GMU PTEs */ +#define MEMFLAG_GMU_PRIV BIT(4) + +/* Buffer is read-write for GMU PTEs. A 0 indicates read-only */ +#define MEMFLAG_GMU_WRITEABLE BIT(5) + +/* Buffer is located in GMU's non-cached bufferable VA range */ +#define MEMFLAG_GMU_BUFFERABLE BIT(6) + +/* Buffer is located in GMU's cacheable VA range */ +#define MEMFLAG_GMU_CACHEABLE BIT(7) + +/* Host can access */ +#define MEMFLAG_HOST_ACC BIT(8) + +/* Host initializes the buffer */ +#define MEMFLAG_HOST_INIT BIT(9) + +struct mem_alloc_entry { + struct hfi_mem_alloc_desc desc; + struct kgsl_memdesc *gpu_md; +}; + +struct a6xx_hwsched_hfi { + struct mem_alloc_entry mem_alloc_table[32]; + u32 mem_alloc_entries; +}; + /** * a6xx_hwsched_hfi_init - Initialize hfi resources * @adreno_dev - Pointer to adreno device structure @@ -16,4 +119,22 @@ * Return: 0 on success and negative error on failure. */ int a6xx_hwsched_hfi_init(struct adreno_device *adreno_dev); + +/** + * a6xx_hwsched_hfi_start - Start hfi resources + * @adreno_dev: Pointer to adreno device structure + * + * Send the various hfi packets before booting the gpu + * + * Return: 0 on success and negative error on failure. + */ +int a6xx_hwsched_hfi_start(struct adreno_device *adreno_dev); + +/** + * a6xx_hwsched_hfi_stop - Stop the hfi resources + * @adreno_dev: Pointer to the adreno device + * + * This function does the hfi cleanup when powering down the gmu + */ +void a6xx_hwsched_hfi_stop(struct adreno_device *adreno_dev); #endif