diff --git a/drivers/gpu/msm/Makefile b/drivers/gpu/msm/Makefile index 4c3f64d41c97..8d893855b8a5 100644 --- a/drivers/gpu/msm/Makefile +++ b/drivers/gpu/msm/Makefile @@ -9,14 +9,11 @@ msm_kgsl-y = \ kgsl_drawobj.o \ kgsl_events.o \ kgsl_ioctl.o \ - kgsl_gmu.o \ kgsl_gmu_core.o \ - kgsl_hfi.o \ kgsl_mmu.o \ kgsl_pool.o \ kgsl_pwrctrl.o \ kgsl_pwrscale.o \ - kgsl_rgmu.o \ kgsl_sharedmem.o \ kgsl_snapshot.o \ kgsl_trace.o \ @@ -36,8 +33,11 @@ msm_kgsl-y += \ adreno_a5xx_snapshot.o \ adreno_a6xx.o \ adreno_a6xx_gmu.o \ + adreno_a6xx_gmu_snapshot.o \ + adreno_a6xx_hfi.o \ adreno_a6xx_preempt.o \ adreno_a6xx_rgmu.o \ + adreno_a6xx_rpmh.o \ adreno_a6xx_snapshot.o \ adreno_cp_parser.o \ adreno_dispatch.o \ diff --git a/drivers/gpu/msm/a6xx_reg.h b/drivers/gpu/msm/a6xx_reg.h index 47b5faaa316f..b3b7f2e5ce1b 100644 --- a/drivers/gpu/msm/a6xx_reg.h +++ b/drivers/gpu/msm/a6xx_reg.h @@ -971,8 +971,6 @@ #define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H 0x1F84D #define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L 0x1F84E #define A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H 0x1F84F -#define A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_L 0x1F888 -#define A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_H 0x1F889 #define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL 0x1F8C0 #define A6XX_GMU_PWR_COL_INTER_FRAME_HYST 0x1F8C1 #define A6XX_GMU_PWR_COL_SPTPRAC_HYST 0x1F8C2 @@ -1124,7 +1122,7 @@ * now we are using a separate section for RSCC regsiters. Add the * offset for backward compatibility. */ -#define RSCC_OFFSET_LEGACY 0x23400 +#define RSCC_OFFSET_LEGACY 0x23000 /* RGMU(PCC) registers in A6X_GMU_CX_0_NON_CONTEXT_DEC domain */ #define A6XX_RGMU_CX_INTR_GEN_EN 0x1F80F diff --git a/drivers/gpu/msm/adreno-gpulist.h b/drivers/gpu/msm/adreno-gpulist.h index 5ac41d1ffcc0..fd58a9316bcb 100644 --- a/drivers/gpu/msm/adreno-gpulist.h +++ b/drivers/gpu/msm/adreno-gpulist.h @@ -782,7 +782,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a630v2 = { .features = ADRENO_RPMH | ADRENO_IFPC | ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | ADRENO_PREEMPTION, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a630_gpudev, .gmem_base = 0x100000, .gmem_size = SZ_1M, .bus_width = 32, @@ -881,7 +881,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a615 = { .features = ADRENO_RPMH | ADRENO_PREEMPTION | ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | ADRENO_IOCOHERENT, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a630_gpudev, .gmem_base = 0x100000, .gmem_size = SZ_512K, .bus_width = 32, @@ -908,7 +908,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a618 = { .features = ADRENO_RPMH | ADRENO_PREEMPTION | ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | ADRENO_IOCOHERENT, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a630_gpudev, .gmem_base = 0x100000, .gmem_size = SZ_512K, .bus_width = 32, @@ -935,7 +935,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a619 = { .features = ADRENO_RPMH | ADRENO_PREEMPTION | ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | ADRENO_IOCOHERENT, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a630_gpudev, .gmem_size = SZ_512K, .bus_width = 32, }, @@ -1081,7 +1081,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a620 = { ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_ACD | ADRENO_APRIV, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a630_gpudev, .gmem_base = 0, .gmem_size = SZ_512K, .bus_width = 32, @@ -1171,7 +1171,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a640 = { .features = ADRENO_RPMH | ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | ADRENO_IFPC | ADRENO_PREEMPTION, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a6xx_gmu_gpudev, .gmem_base = 0x100000, .gmem_size = SZ_1M, //Verified 1MB .bus_width = 32, @@ -1251,7 +1251,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a650 = { .features = ADRENO_RPMH | ADRENO_GPMU | ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | ADRENO_APRIV, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a6xx_gmu_gpudev, .gmem_base = 0, .gmem_size = SZ_1M + SZ_128K, /* verified 1152kB */ .bus_width = 32, @@ -1280,7 +1280,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a650v2 = { ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | ADRENO_PREEMPTION | ADRENO_ACD | ADRENO_LM | ADRENO_APRIV, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a6xx_gmu_gpudev, .gmem_base = 0, .gmem_size = SZ_1M + SZ_128K, /* verified 1152kB */ .bus_width = 32, @@ -1306,7 +1306,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a680 = { .base = { DEFINE_ADRENO_REV(ADRENO_REV_A680, 6, 8, 0, ANY_ID), .features = ADRENO_RPMH | ADRENO_GPMU, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a6xx_gmu_gpudev, .gmem_base = 0x100000, .gmem_size = SZ_2M, .bus_width = 32, @@ -1383,7 +1383,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a612 = { .features = ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | ADRENO_PREEMPTION | ADRENO_GPMU | ADRENO_IFPC, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a6xx_rgmu_gpudev, .gmem_base = 0x100000, .gmem_size = (SZ_128K + SZ_4K), .bus_width = 32, @@ -1408,7 +1408,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a616 = { .features = ADRENO_RPMH | ADRENO_PREEMPTION | ADRENO_GPMU | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC | ADRENO_IOCOHERENT, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a630_gpudev, .gmem_base = 0x100000, .gmem_size = SZ_512K, .bus_width = 32, @@ -1549,7 +1549,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a660 = { .features = ADRENO_RPMH | ADRENO_GPMU | ADRENO_APRIV | ADRENO_IOCOHERENT | ADRENO_CONTENT_PROTECTION | ADRENO_IFPC, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a6xx_gmu_gpudev, .gmem_base = 0, .gmem_size = SZ_1M + SZ_512K, .bus_width = 32, diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index 96863da68b5b..46fb0735c173 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -321,47 +322,10 @@ void adreno_fault_detect_stop(struct adreno_device *adreno_dev) adreno_dev->fast_hang_detect = 0; } -#define GMU_CM3_CFG_NONMASKINTR_SHIFT 9 - -/* Send an NMI to the GMU */ -void adreno_gmu_send_nmi(struct adreno_device *adreno_dev) +static void adreno_touch_wakeup(struct adreno_device *adreno_dev) { - u32 val; - /* Mask so there's no interrupt caused by NMI */ - adreno_write_gmureg(adreno_dev, - ADRENO_REG_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF); - - /* Make sure the interrupt is masked before causing it */ - wmb(); - if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) - adreno_write_gmureg(adreno_dev, - ADRENO_REG_GMU_NMI_CONTROL_STATUS, 0); - - adreno_read_gmureg(adreno_dev, ADRENO_REG_GMU_CM3_CFG, &val); - val |= 1 << GMU_CM3_CFG_NONMASKINTR_SHIFT; - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_CM3_CFG, val); - - /* Make sure the NMI is invoked before we proceed*/ - wmb(); -} - -/* - * A workqueue callback responsible for actually turning on the GPU after a - * touch event. kgsl_pwrctrl_change_state(ACTIVE) is used without any - * active_count protection to avoid the need to maintain state. Either - * somebody will start using the GPU or the idle timer will fire and put the - * GPU back into slumber. - */ -static void adreno_input_work(struct work_struct *work) -{ - struct adreno_device *adreno_dev = container_of(work, - struct adreno_device, input_work); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - mutex_lock(&device->mutex); - - device->flags |= KGSL_FLAG_WAKE_ON_TOUCH; - /* * Don't schedule adreno_start in a high priority workqueue, we are * already in a workqueue which should be sufficient @@ -376,6 +340,29 @@ static void adreno_input_work(struct work_struct *work) */ mod_timer(&device->idle_timer, jiffies + msecs_to_jiffies(adreno_wake_timeout)); + +} + +/* + * A workqueue callback responsible for actually turning on the GPU after a + * touch event. kgsl_pwrctrl_change_state(ACTIVE) is used without any + * active_count protection to avoid the need to maintain state. Either + * somebody will start using the GPU or the idle timer will fire and put the + * GPU back into slumber. + */ +static void adreno_input_work(struct work_struct *work) +{ + struct adreno_device *adreno_dev = container_of(work, + struct adreno_device, input_work); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + mutex_lock(&device->mutex); + + device->flags |= KGSL_FLAG_WAKE_ON_TOUCH; + + ops->touch_wakeup(adreno_dev); + mutex_unlock(&device->mutex); } @@ -401,6 +388,11 @@ static void adreno_input_event(struct input_handle *handle, unsigned int type, if (device->flags & KGSL_FLAG_WAKE_ON_TOUCH) return; + if (gmu_core_isenabled(device)) { + schedule_work(&adreno_dev->input_work); + return; + } + /* * If the device is in nap, kick the idle timer to make sure that we * don't go into slumber before the first render. If the device is @@ -691,26 +683,64 @@ adreno_get_soc_hw_revision_node(struct platform_device *pdev, u32 hwrev) return NULL; } +static u32 adreno_efuse_read_soc_hw_rev(struct platform_device *pdev) +{ + unsigned int val; + unsigned int soc_hw_rev[3]; + int ret; + + if (of_property_read_u32_array(pdev->dev.of_node, + "qcom,soc-hw-rev-efuse", soc_hw_rev, 3)) + return 0; + + ret = adreno_efuse_map(pdev); + if (ret) { + dev_err(&pdev->dev, + "Unable to map hardware revision fuse: ret=%d\n", ret); + return 0; + } + + ret = adreno_efuse_read_u32(soc_hw_rev[0], &val); + adreno_efuse_unmap(); + + if (ret) { + dev_err(&pdev->dev, + "Unable to read hardware revision fuse: ret=%d\n", ret); + return 0; + } + + return (val >> soc_hw_rev[1]) & soc_hw_rev[2]; +} + +static int adreno_get_chipid(struct platform_device *pdev, u32 *chipid) +{ + u32 hwrev = adreno_efuse_read_soc_hw_rev(pdev); + struct device_node *node; + int ret; + + node = adreno_get_soc_hw_revision_node(pdev, hwrev); + if (!node) + node = of_node_get(pdev->dev.of_node); + + ret = of_property_read_u32(node, "qcom,chipid", chipid); + + of_node_put(node); + return ret; +} + static void adreno_update_soc_hw_revision_quirks(struct adreno_device *adreno_dev, - struct platform_device *pdev, u32 hwrev) + struct platform_device *pdev) { struct device_node *node; int i; + u32 hwrev; + + hwrev = adreno_efuse_read_soc_hw_rev(pdev); node = adreno_get_soc_hw_revision_node(pdev, hwrev); if (node == NULL) - node = pdev->dev.of_node; - - /* get chip id, fall back to parent if revision node does not have it */ - if (of_property_read_u32(node, "qcom,chipid", &adreno_dev->chipid)) { - if (of_property_read_u32(pdev->dev.of_node, - "qcom,chipid", &adreno_dev->chipid)) { - dev_crit(&pdev->dev, "No GPU chip ID was specified\n"); - BUG(); - return; - } - } + node = of_node_get(pdev->dev.of_node); /* update quirk */ for (i = 0; i < ARRAY_SIZE(adreno_quirks); i++) { @@ -723,55 +753,40 @@ adreno_update_soc_hw_revision_quirks(struct adreno_device *adreno_dev, kgsl_mmu_set_feature(KGSL_DEVICE(adreno_dev), KGSL_MMU_SECURE_CB_ALT); - - if (node != pdev->dev.of_node) - of_node_put(node); + of_node_put(node); } -static int adreno_identify_gpu(struct platform_device *pdev, - struct adreno_device *adreno_dev) +static const struct adreno_gpu_core * +adreno_identify_gpu(struct platform_device *pdev, u32 *chipid) { - struct adreno_gpudev *gpudev; - int i; + const struct adreno_gpu_core *gpucore; - adreno_dev->gpucore = _get_gpu_core(pdev, adreno_dev->chipid); + if (adreno_get_chipid(pdev, chipid)) { + dev_crit(&pdev->dev, "Unable to get the GPU chip ID\n"); + return ERR_PTR(-ENODEV); + } - if (adreno_dev->gpucore == NULL) { - dev_crit(&pdev->dev, - "Unknown GPU chip ID %8.8X\n", adreno_dev->chipid); - return -ENODEV; + gpucore = _get_gpu_core(pdev, *chipid); + if (!gpucore) { + dev_crit(&pdev->dev, "Unknown GPU chip ID %8.8x\n", *chipid); + return ERR_PTR(-ENODEV); } /* * Identify non-longer supported targets and spins and print a helpful * message */ - if (adreno_dev->gpucore->features & ADRENO_DEPRECATED) { + if (gpucore->features & ADRENO_DEPRECATED) { dev_err(&pdev->dev, "Support for GPU %d.%d.%d.%d has been deprecated\n", - adreno_dev->gpucore->core, - adreno_dev->gpucore->major, - adreno_dev->gpucore->minor, - adreno_dev->gpucore->patchid); - return -ENODEV; + gpucore->core, + gpucore->major, + gpucore->minor, + gpucore->patchid); + return ERR_PTR(-ENODEV); } - gpudev = ADRENO_GPU_DEVICE(adreno_dev); - - /* - * Initialize uninitialzed gpu registers, only needs to be done once - * Make all offsets that are not initialized to ADRENO_REG_UNUSED - */ - for (i = 0; i < ADRENO_REG_REGISTER_MAX; i++) { - if (!gpudev->reg_offsets[i]) - gpudev->reg_offsets[i] = ADRENO_REG_UNUSED; - } - - /* Do target specific identification */ - if (gpudev->platform_setup != NULL) - gpudev->platform_setup(adreno_dev); - - return 0; + return gpucore; } static const struct of_device_id adreno_match_table[] = { @@ -1095,25 +1110,6 @@ static void adreno_cx_misc_probe(struct kgsl_device *device) res->start, adreno_dev->cx_misc_len); } -static void adreno_rscc_probe(struct kgsl_device *device) -{ - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct resource *res; - - res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, - "rscc"); - - if (res == NULL) - return; - - adreno_dev->rscc_base = res->start - device->reg_phys; - adreno_dev->rscc_len = resource_size(res); - adreno_dev->rscc_virt = devm_ioremap(&device->pdev->dev, res->start, - adreno_dev->rscc_len); - if (adreno_dev->rscc_virt == NULL) - dev_warn(device->dev, "rscc ioremap failed\n"); -} - static void adreno_isense_probe(struct kgsl_device *device) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); @@ -1132,50 +1128,20 @@ static void adreno_isense_probe(struct kgsl_device *device) dev_warn(device->dev, "isense ioremap failed\n"); } -static u32 adreno_efuse_read_soc_hw_rev(struct platform_device *pdev) -{ - unsigned int val; - unsigned int soc_hw_rev[3]; - int ret; - - if (of_property_read_u32_array(pdev->dev.of_node, - "qcom,soc-hw-rev-efuse", soc_hw_rev, 3)) - return 0; - - ret = adreno_efuse_map(pdev); - if (ret) { - dev_err(&pdev->dev, - "Unable to map hardware revision fuse: ret=%d\n", ret); - return 0; - } - - ret = adreno_efuse_read_u32(soc_hw_rev[0], &val); - adreno_efuse_unmap(); - - if (ret) { - dev_err(&pdev->dev, - "Unable to read hardware revision fuse: ret=%d\n", ret); - return 0; - } - - return (val >> soc_hw_rev[1]) & soc_hw_rev[2]; -} - -static bool adreno_is_gpu_disabled(struct adreno_device *adreno_dev) +static bool adreno_is_gpu_disabled(struct platform_device *pdev) { unsigned int row0; unsigned int pte_row0_msb[3]; int ret; - struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - if (of_property_read_u32_array(device->pdev->dev.of_node, + if (of_property_read_u32_array(pdev->dev.of_node, "qcom,gpu-disable-fuse", pte_row0_msb, 3)) return false; /* * Read the fuse value to disable GPU driver if fuse * is blown. By default(fuse value is 0) GPU is enabled. */ - if (adreno_efuse_map(device->pdev)) + if (adreno_efuse_map(pdev)) return false; ret = adreno_efuse_read_u32(pte_row0_msb[0], &row0); @@ -1278,8 +1244,6 @@ static void adreno_setup_device(struct adreno_device *adreno_dev) { u32 i; - memset(adreno_dev, 0, sizeof(*adreno_dev)); - adreno_dev->dev.name = "kgsl-3d0"; adreno_dev->dev.ftbl = &adreno_functable; @@ -1310,35 +1274,23 @@ static void adreno_setup_device(struct adreno_device *adreno_dev) } } -static int adreno_bind(struct device *dev) +int adreno_device_probe(struct platform_device *pdev, + struct adreno_device *adreno_dev) { - struct platform_device *pdev = to_platform_device(dev); - struct adreno_device *adreno_dev; - struct kgsl_device *device; - int status; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct device *dev = &pdev->dev; unsigned int priv = 0; - u32 size, hwrev; - - adreno_dev = (struct adreno_device *) of_device_get_match_data(dev); + int status; + u32 size; /* Initialize the adreno device structure */ adreno_setup_device(adreno_dev); - device = KGSL_DEVICE(adreno_dev); - dev_set_drvdata(dev, device); device->pdev = pdev; - if (adreno_is_gpu_disabled(adreno_dev)) { - dev_err(&pdev->dev, "adreno: GPU is disabled on this device\n"); - return -ENODEV; - } - - /* Identify SOC hardware revision to be used */ - hwrev = adreno_efuse_read_soc_hw_rev(pdev); - - adreno_update_soc_hw_revision_quirks(adreno_dev, pdev, hwrev); + adreno_update_soc_hw_revision_quirks(adreno_dev, pdev); status = adreno_read_speed_bin(pdev); if (status < 0) @@ -1346,10 +1298,6 @@ static int adreno_bind(struct device *dev) device->speed_bin = status; - /* Get the chip ID from the DT and set up target specific parameters */ - if (adreno_identify_gpu(pdev, adreno_dev)) - return -ENODEV; - status = adreno_of_get_power(adreno_dev, pdev); if (status) return status; @@ -1412,8 +1360,6 @@ static int adreno_bind(struct device *dev) /* Probe for the optional CX_MISC block */ adreno_cx_misc_probe(device); - adreno_rscc_probe(device); - adreno_isense_probe(device); /* Allocate the memstore for storing timestamps and other useful info */ @@ -1478,6 +1424,24 @@ err: return status; } +static int adreno_bind(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + const struct adreno_gpu_core *gpucore; + u32 chipid; + + if (adreno_is_gpu_disabled(pdev)) { + dev_err(&pdev->dev, "adreno: GPU is disabled on this device\n"); + return -ENODEV; + } + + gpucore = adreno_identify_gpu(pdev, &chipid); + if (IS_ERR(gpucore)) + return PTR_ERR(gpucore); + + return gpucore->gpudev->probe(pdev, chipid, gpucore); +} + static void _adreno_free_memories(struct adreno_device *adreno_dev) { struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP); @@ -1500,9 +1464,11 @@ static void adreno_unbind(struct device *dev) struct kgsl_device *device; struct adreno_gpudev *gpudev; - adreno_dev = (struct adreno_device *) of_device_get_match_data(dev); + device = dev_get_drvdata(dev); + if (!device) + return; - device = KGSL_DEVICE(adreno_dev); + adreno_dev = ADRENO_DEVICE(device); gpudev = ADRENO_GPU_DEVICE(adreno_dev); if (gpudev->remove != NULL) @@ -1551,11 +1517,10 @@ static void adreno_unbind(struct device *dev) clear_bit(ADRENO_DEVICE_INITIALIZED, &adreno_dev->priv); } -static int adreno_pm_resume(struct device *dev) +static void adreno_resume(struct adreno_device *adreno_dev) { - struct kgsl_device *device = dev_get_drvdata(dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - mutex_lock(&device->mutex); if (device->state == KGSL_STATE_SUSPEND) { adreno_dispatcher_unhalt(device); kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); @@ -1570,19 +1535,41 @@ static int adreno_pm_resume(struct device *dev) kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); dev_err(device->dev, "resume invoked without a suspend\n"); } +} + +static int adreno_pm_resume(struct device *dev) +{ + struct kgsl_device *device = dev_get_drvdata(dev); + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + mutex_lock(&device->mutex); + ops->pm_resume(adreno_dev); mutex_unlock(&device->mutex); + return 0; } +static int adreno_suspend(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int status = kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); + + if (!status && device->state == KGSL_STATE_SUSPEND) + adreno_dispatcher_halt(device); + + return status; +} + static int adreno_pm_suspend(struct device *dev) { struct kgsl_device *device = dev_get_drvdata(dev); + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); int status; mutex_lock(&device->mutex); - status = kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); - if (!status && device->state == KGSL_STATE_SUSPEND) - adreno_dispatcher_halt(device); + status = ops->pm_suspend(adreno_dev); mutex_unlock(&device->mutex); return status; @@ -1638,28 +1625,26 @@ int adreno_clear_pending_transactions(struct kgsl_device *device) if (adreno_has_gbif(adreno_dev)) { - /* This is taken care by GMU firmware if GMU is enabled */ - if (!gmu_core_gpmu_isenabled(device)) { - /* Halt GBIF GX traffic and poll for halt ack */ - if (adreno_is_a615_family(adreno_dev)) { - adreno_writereg(adreno_dev, - ADRENO_REG_RBBM_GPR0_CNTL, - GBIF_HALT_REQUEST); - ret = adreno_wait_for_halt_ack(device, - A6XX_RBBM_VBIF_GX_RESET_STATUS, - VBIF_RESET_ACK_MASK); - } else { - adreno_writereg(adreno_dev, - ADRENO_REG_RBBM_GBIF_HALT, - gpudev->gbif_gx_halt_mask); - ret = adreno_wait_for_halt_ack(device, - ADRENO_REG_RBBM_GBIF_HALT_ACK, - gpudev->gbif_gx_halt_mask); - } - if (ret) - return ret; + /* Halt GBIF GX traffic and poll for halt ack */ + if (adreno_is_a615_family(adreno_dev)) { + adreno_writereg(adreno_dev, + ADRENO_REG_RBBM_GPR0_CNTL, + GBIF_HALT_REQUEST); + ret = adreno_wait_for_halt_ack(device, + A6XX_RBBM_VBIF_GX_RESET_STATUS, + VBIF_RESET_ACK_MASK); + } else { + adreno_writereg(adreno_dev, + ADRENO_REG_RBBM_GBIF_HALT, + gpudev->gbif_gx_halt_mask); + ret = adreno_wait_for_halt_ack(device, + ADRENO_REG_RBBM_GBIF_HALT_ACK, + gpudev->gbif_gx_halt_mask); } + if (ret) + return ret; + /* Halt new client requests */ adreno_writereg(adreno_dev, ADRENO_REG_GBIF_HALT, gpudev->gbif_client_halt_mask); @@ -1726,10 +1711,6 @@ static int adreno_init(struct kgsl_device *device) return ret; } - ret = gmu_core_init(device); - if (ret) - return ret; - /* Put the GPU in a responsive state */ if (ADRENO_GPUREV(adreno_dev) < 600) { /* No need for newer generation architectures */ @@ -1815,7 +1796,7 @@ int adreno_switch_to_unsecure_mode(struct adreno_device *adreno_dev, return ret; } -static void adreno_set_active_ctxs_null(struct adreno_device *adreno_dev) +void adreno_set_active_ctxs_null(struct adreno_device *adreno_dev) { int i; struct adreno_ringbuffer *rb; @@ -1830,14 +1811,15 @@ static void adreno_set_active_ctxs_null(struct adreno_device *adreno_dev) } } -static int adreno_first_open(struct kgsl_device *device) +static int adreno_open(struct adreno_device *adreno_dev) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); int ret; /* * active_cnt special case: we are starting up for the first * time, so use this sequence instead of the kgsl_pwrctrl_wake() - * which will be called by kgsl_active_count_get(). + * which will be called by adreno_active_count_get(). */ atomic_inc(&device->active_cnt); @@ -1851,11 +1833,9 @@ static int adreno_first_open(struct kgsl_device *device) if (ret) goto err; - timer_setup(&device->idle_timer, kgsl_timer, 0); - complete_all(&device->hwaccess_gate); kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); return 0; err: @@ -1865,8 +1845,18 @@ err: return ret; } -static int adreno_last_close(struct kgsl_device *device) +static int adreno_first_open(struct kgsl_device *device) { + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + return ops->first_open(adreno_dev); +} + +static int adreno_close(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + /* * Wait up to 1 second for the active count to go low * and then start complaining about it @@ -1883,6 +1873,172 @@ static int adreno_last_close(struct kgsl_device *device) return kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); } +static int adreno_last_close(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + return ops->last_close(adreno_dev); +} + +static int adreno_pwrctrl_active_count_get(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret = 0; + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return -EINVAL; + + if ((atomic_read(&device->active_cnt) == 0) && + (device->state != KGSL_STATE_ACTIVE)) { + mutex_unlock(&device->mutex); + wait_for_completion(&device->hwaccess_gate); + mutex_lock(&device->mutex); + device->pwrctrl.superfast = true; + ret = kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); + } + if (ret == 0) + atomic_inc(&device->active_cnt); + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + return ret; +} + +static void adreno_pwrctrl_active_count_put(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return; + + if (WARN(atomic_read(&device->active_cnt) == 0, + "Unbalanced get/put calls to KGSL active count\n")) + return; + + if (atomic_dec_and_test(&device->active_cnt)) { + bool nap_on = !(device->pwrctrl.ctrl_flags & + BIT(KGSL_PWRFLAGS_NAP_OFF)); + if (nap_on && device->state == KGSL_STATE_ACTIVE && + device->requested_state == KGSL_STATE_NONE) { + kgsl_pwrctrl_request_state(device, KGSL_STATE_NAP); + kgsl_schedule_work(&device->idle_check_ws); + } else if (!nap_on) { + kgsl_pwrscale_update_stats(device); + kgsl_pwrscale_update(device); + } + + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + } + + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + + wake_up(&device->active_cnt_wq); +} + +int adreno_active_count_get(struct adreno_device *adreno_dev) +{ + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + return ops->active_count_get(adreno_dev); +} + +void adreno_active_count_put(struct adreno_device *adreno_dev) +{ + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + ops->active_count_put(adreno_dev); +} + +void adreno_get_bus_counters(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (!device->pwrctrl.bus_control) + return; + + /* VBIF waiting for RAM */ + if (adreno_dev->starved_ram_lo == 0) + if (adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 0, + &adreno_dev->starved_ram_lo, NULL, + PERFCOUNTER_FLAG_KERNEL)) + goto err; + + /* VBIF DDR cycles */ + if (!adreno_has_gbif(adreno_dev) && (adreno_dev->ram_cycles_lo == 0)) { + if (adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + VBIF_AXI_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo, NULL, + PERFCOUNTER_FLAG_KERNEL)) + goto err; + return; + } + + if (adreno_dev->starved_ram_lo_ch1 == 0) + if (adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 1, + &adreno_dev->starved_ram_lo_ch1, NULL, + PERFCOUNTER_FLAG_KERNEL)) + goto err; + + if (adreno_dev->ram_cycles_lo == 0) + if (adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI0_READ_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo, NULL, + PERFCOUNTER_FLAG_KERNEL)) + goto err; + + if (adreno_dev->ram_cycles_lo_ch1_read == 0) + if (adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI1_READ_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch1_read, + NULL, + PERFCOUNTER_FLAG_KERNEL)) + goto err; + + if (adreno_dev->ram_cycles_lo_ch0_write == 0) + if (adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI0_WRITE_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch0_write, + NULL, + PERFCOUNTER_FLAG_KERNEL)) + goto err; + + if (adreno_dev->ram_cycles_lo_ch1_write == 0) + if (adreno_perfcounter_get(adreno_dev, + KGSL_PERFCOUNTER_GROUP_VBIF, + GBIF_AXI1_WRITE_DATA_TOTAL_BEATS, + &adreno_dev->ram_cycles_lo_ch1_write, + NULL, + PERFCOUNTER_FLAG_KERNEL)) + goto err; + + return; +err: + dev_err(KGSL_DEVICE(adreno_dev)->dev, + "Unable to get perf counters for bus DCVS\n"); + +} + +void adreno_clear_dcvs_counters(struct adreno_device *adreno_dev) +{ + /* Clear the busy_data stats - we're starting over from scratch */ + adreno_dev->busy_data.gpu_busy = 0; + adreno_dev->busy_data.bif_ram_cycles = 0; + adreno_dev->busy_data.bif_ram_cycles_read_ch1 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch0 = 0; + adreno_dev->busy_data.bif_ram_cycles_write_ch1 = 0; + adreno_dev->busy_data.bif_starved_ram = 0; + adreno_dev->busy_data.bif_starved_ram_ch1 = 0; + adreno_dev->busy_data.num_ifpc = 0; +} + /** * _adreno_start - Power up the GPU and prepare to accept commands * @adreno_dev: Pointer to an adreno_device structure @@ -1894,7 +2050,7 @@ static int _adreno_start(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); - int status = -EINVAL, ret; + int status; unsigned int state = device->state; bool regulator_left_on; @@ -1937,135 +2093,11 @@ static int _adreno_start(struct adreno_device *adreno_dev) status = kgsl_mmu_start(device); if (status) - goto error_boot_oob_clear; + goto error_pwr_off; - /* Send OOB request to turn on the GX */ - status = gmu_core_dev_oob_set(device, oob_gpu); - if (status) { - gmu_core_snapshot(device); - goto error_oob_clear; - } + adreno_get_bus_counters(adreno_dev); - status = gmu_core_dev_hfi_start_msg(device); - if (status) { - gmu_core_snapshot(device); - goto error_oob_clear; - } - - if (device->pwrctrl.bus_control) { - /* VBIF waiting for RAM */ - if (adreno_dev->starved_ram_lo == 0) { - ret = adreno_perfcounter_get(adreno_dev, - KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 0, - &adreno_dev->starved_ram_lo, NULL, - PERFCOUNTER_FLAG_KERNEL); - - if (ret) { - dev_err(device->dev, - "Unable to get perf counters for bus DCVS\n"); - adreno_dev->starved_ram_lo = 0; - } - } - - if (adreno_has_gbif(adreno_dev)) { - if (adreno_dev->starved_ram_lo_ch1 == 0) { - ret = adreno_perfcounter_get(adreno_dev, - KGSL_PERFCOUNTER_GROUP_VBIF_PWR, 1, - &adreno_dev->starved_ram_lo_ch1, NULL, - PERFCOUNTER_FLAG_KERNEL); - - if (ret) { - dev_err(device->dev, - "Unable to get perf counters for bus DCVS\n"); - adreno_dev->starved_ram_lo_ch1 = 0; - } - } - - if (adreno_dev->ram_cycles_lo == 0) { - ret = adreno_perfcounter_get(adreno_dev, - KGSL_PERFCOUNTER_GROUP_VBIF, - GBIF_AXI0_READ_DATA_TOTAL_BEATS, - &adreno_dev->ram_cycles_lo, NULL, - PERFCOUNTER_FLAG_KERNEL); - - if (ret) { - dev_err(device->dev, - "Unable to get perf counters for bus DCVS\n"); - adreno_dev->ram_cycles_lo = 0; - } - } - - if (adreno_dev->ram_cycles_lo_ch1_read == 0) { - ret = adreno_perfcounter_get(adreno_dev, - KGSL_PERFCOUNTER_GROUP_VBIF, - GBIF_AXI1_READ_DATA_TOTAL_BEATS, - &adreno_dev->ram_cycles_lo_ch1_read, - NULL, - PERFCOUNTER_FLAG_KERNEL); - - if (ret) { - dev_err(device->dev, - "Unable to get perf counters for bus DCVS\n"); - adreno_dev->ram_cycles_lo_ch1_read = 0; - } - } - - if (adreno_dev->ram_cycles_lo_ch0_write == 0) { - ret = adreno_perfcounter_get(adreno_dev, - KGSL_PERFCOUNTER_GROUP_VBIF, - GBIF_AXI0_WRITE_DATA_TOTAL_BEATS, - &adreno_dev->ram_cycles_lo_ch0_write, - NULL, - PERFCOUNTER_FLAG_KERNEL); - - if (ret) { - dev_err(device->dev, - "Unable to get perf counters for bus DCVS\n"); - adreno_dev->ram_cycles_lo_ch0_write = 0; - } - } - - if (adreno_dev->ram_cycles_lo_ch1_write == 0) { - ret = adreno_perfcounter_get(adreno_dev, - KGSL_PERFCOUNTER_GROUP_VBIF, - GBIF_AXI1_WRITE_DATA_TOTAL_BEATS, - &adreno_dev->ram_cycles_lo_ch1_write, - NULL, - PERFCOUNTER_FLAG_KERNEL); - - if (ret) { - dev_err(device->dev, - "Unable to get perf counters for bus DCVS\n"); - adreno_dev->ram_cycles_lo_ch1_write = 0; - } - } - } else { - /* VBIF DDR cycles */ - if (adreno_dev->ram_cycles_lo == 0) { - ret = adreno_perfcounter_get(adreno_dev, - KGSL_PERFCOUNTER_GROUP_VBIF, - VBIF_AXI_TOTAL_BEATS, - &adreno_dev->ram_cycles_lo, NULL, - PERFCOUNTER_FLAG_KERNEL); - - if (ret) { - dev_err(device->dev, - "Unable to get perf counters for bus DCVS\n"); - adreno_dev->ram_cycles_lo = 0; - } - } - } - } - - /* Clear the busy_data stats - we're starting over from scratch */ - adreno_dev->busy_data.gpu_busy = 0; - adreno_dev->busy_data.bif_ram_cycles = 0; - adreno_dev->busy_data.bif_ram_cycles_read_ch1 = 0; - adreno_dev->busy_data.bif_ram_cycles_write_ch0 = 0; - adreno_dev->busy_data.bif_ram_cycles_write_ch1 = 0; - adreno_dev->busy_data.bif_starved_ram = 0; - adreno_dev->busy_data.bif_starved_ram_ch1 = 0; - adreno_dev->busy_data.num_ifpc = 0; + adreno_clear_dcvs_counters(adreno_dev); /* Restore performance counter registers with saved values */ adreno_perfcounter_restore(adreno_dev); @@ -2085,7 +2117,7 @@ static int _adreno_start(struct adreno_device *adreno_dev) status = gpudev->rb_start(adreno_dev); if (status) - goto error_oob_clear; + goto error_pwr_off; /* * At this point it is safe to assume that we recovered. Setting @@ -2102,22 +2134,8 @@ static int _adreno_start(struct adreno_device *adreno_dev) set_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); - /* Send OOB request to allow IFPC */ - gmu_core_dev_oob_clear(device, oob_gpu); - - /* If we made it this far, the BOOT OOB was sent to the GMU */ - if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) - gmu_core_dev_oob_clear(device, oob_boot_slumber); - return 0; -error_oob_clear: - gmu_core_dev_oob_clear(device, oob_gpu); - -error_boot_oob_clear: - if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) - gmu_core_dev_oob_clear(device, oob_boot_slumber); - error_pwr_off: /* set the state back to original state */ kgsl_pwrctrl_change_state(device, state); @@ -2159,14 +2177,6 @@ static int adreno_stop(struct kgsl_device *device) if (!test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv)) return 0; - error = gmu_core_dev_oob_set(device, oob_gpu); - if (error) { - gmu_core_dev_oob_clear(device, oob_gpu); - gmu_core_snapshot(device); - error = -EINVAL; - goto no_gx_power; - } - kgsl_pwrscale_update_stats(device); adreno_irqctrl(adreno_dev, 0); @@ -2177,24 +2187,8 @@ static int adreno_stop(struct kgsl_device *device) /* Save physical performance counter values before GPU power down*/ adreno_perfcounter_save(adreno_dev); - gmu_core_dev_prepare_stop(device); - gmu_core_dev_oob_clear(device, oob_gpu); - - /* - * Saving perfcounters will use an OOB to put the GMU into - * active state. Before continuing, we should wait for the - * GMU to return to the lowest idle level. This is - * because some idle level transitions require VBIF and MMU. - */ - - if (!error && gmu_core_dev_wait_for_lowest_idle(device)) { - gmu_core_snapshot(device); - error = -EINVAL; - } - adreno_clear_pending_transactions(device); -no_gx_power: adreno_dispatcher_stop(adreno_dev); adreno_ringbuffer_stop(adreno_dev); @@ -2205,16 +2199,6 @@ no_gx_power: if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) llcc_slice_deactivate(adreno_dev->gpuhtw_llc_slice); - /* - * The halt is not cleared in the above function if we have GBIF. - * Clear it here if GMU is enabled as GMU stop needs access to - * system memory to stop. For non-GMU targets, we don't need to - * clear it as it will get cleared automatically once headswitch - * goes OFF immediately after adreno_stop. - */ - if (gmu_core_gpmu_isenabled(device)) - adreno_deassert_gbif_halt(adreno_dev); - adreno_set_active_ctxs_null(adreno_dev); clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); @@ -2239,7 +2223,7 @@ int adreno_reset(struct kgsl_device *device, int fault) int i; if (gpudev->reset) - return gpudev->reset(device, fault); + return gpudev->reset(device); /* * Try soft reset first Do not do soft reset for a IOMMU fault (because @@ -2602,9 +2586,9 @@ static int adreno_setproperty(struct kgsl_device_private *dev_priv, if (enable) { device->pwrctrl.ctrl_flags = 0; - if (!kgsl_active_count_get(device)) { + if (!adreno_active_count_get(adreno_dev)) { adreno_fault_detect_start(adreno_dev); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); } kgsl_pwrscale_enable(device); @@ -2815,15 +2799,7 @@ void adreno_spin_idle_debug(struct adreno_device *adreno_dev, dev_err(device->dev, " hwfault=%8.8X\n", hwfault); - /* - * If CP is stuck, gmu may not perform as expected. So force a gmu - * snapshot which captures entire state as well as sets the gmu fault - * because things need to be reset anyway. - */ - if (gmu_core_isenabled(device)) - gmu_core_snapshot(device); - else - kgsl_device_snapshot(device, NULL, false); + kgsl_device_snapshot(device, NULL, false); } /** @@ -3018,47 +2994,6 @@ static void adreno_regwrite(struct kgsl_device *device, __raw_writel(value, reg); } -/** - * adreno_gmu_clear_and_unmask_irqs() - Clear pending IRQs and Unmask IRQs - * @adreno_dev: Pointer to the Adreno device that owns the GMU - */ -void adreno_gmu_clear_and_unmask_irqs(struct adreno_device *adreno_dev) -{ - struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device); - - /* Clear any pending IRQs before unmasking on GMU */ - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_CLR, - 0xFFFFFFFF); - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, - 0xFFFFFFFF); - - /* Unmask needed IRQs on GMU */ - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_MASK, - (unsigned int) ~(gmu_dev_ops->gmu2host_intr_mask)); - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, - (unsigned int) ~(gmu_dev_ops->gmu_ao_intr_mask)); -} - -/** - * adreno_gmu_mask_and_clear_irqs() - Mask all IRQs and clear pending IRQs - * @adreno_dev: Pointer to the Adreno device that owns the GMU - */ -void adreno_gmu_mask_and_clear_irqs(struct adreno_device *adreno_dev) -{ - /* Mask all IRQs on GMU */ - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, - 0xFFFFFFFF); - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_MASK, - 0xFFFFFFFF); - - /* Clear any pending IRQs before disabling */ - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, - 0xFFFFFFFF); - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_CLR, - 0xFFFFFFFF); -} - /* * adreno_gmu_fenced_write() - Check if there is a GMU and it is enabled * @adreno_dev: Pointer to the Adreno device that owns the GMU @@ -3188,24 +3123,6 @@ void adreno_cx_misc_regread(struct adreno_device *adreno_dev, */ rmb(); } -void adreno_rscc_regread(struct adreno_device *adreno_dev, - unsigned int offsetwords, unsigned int *value) -{ - unsigned int rscc_offset; - - rscc_offset = (offsetwords << 2); - if (!adreno_dev->rscc_virt || - (rscc_offset >= adreno_dev->rscc_len)) - return; - - *value = __raw_readl(adreno_dev->rscc_virt + rscc_offset); - - /* - * ensure this read finishes before the next one. - * i.e. act like normal readl() - */ - rmb(); -} void adreno_isense_regread(struct adreno_device *adreno_dev, unsigned int offsetwords, unsigned int *value) @@ -3667,6 +3584,116 @@ static void adreno_drawctxt_sched(struct kgsl_device *device, adreno_dispatcher_queue_context(device, ADRENO_CONTEXT(context)); } +int adreno_power_cycle(struct adreno_device *adreno_dev, + void (*callback)(struct adreno_device *adreno_dev, void *priv), + void *priv) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + int ret; + + mutex_lock(&device->mutex); + ret = ops->pm_suspend(adreno_dev); + + if (!ret) { + callback(adreno_dev, priv); + ops->pm_resume(adreno_dev); + } + + mutex_unlock(&device->mutex); + + return ret; +} + +int adreno_power_cycle_bool(struct adreno_device *adreno_dev, + bool *flag, bool val) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + int ret; + + mutex_lock(&device->mutex); + ret = ops->pm_suspend(adreno_dev); + + if (!ret) { + *flag = val; + ops->pm_resume(adreno_dev); + } + + mutex_unlock(&device->mutex); + + return ret; +} + +int adreno_power_cycle_u32(struct adreno_device *adreno_dev, + u32 *flag, u32 val) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + int ret; + + mutex_lock(&device->mutex); + ret = ops->pm_suspend(adreno_dev); + + if (!ret) { + *flag = val; + ops->pm_resume(adreno_dev); + } + + mutex_unlock(&device->mutex); + + return ret; +} + +static int adreno_gpu_clock_set(struct kgsl_device *device, u32 pwrlevel) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct kgsl_pwrlevel *pl = &pwr->pwrlevels[pwrlevel]; + int ret; + + if (ops->gpu_clock_set) + return ops->gpu_clock_set(adreno_dev, pwrlevel); + + ret = clk_set_rate(pwr->grp_clks[0], pl->gpu_freq); + if (ret) + dev_err(device->dev, "GPU clk freq set failure: %d\n", ret); + + return ret; +} + +static int adreno_interconnect_bus_set(struct adreno_device *adreno_dev, + int level, u32 ab) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + if ((level == pwr->cur_buslevel) && (ab == pwr->cur_ab)) + return 0; + + pwr->cur_buslevel = level; + pwr->cur_ab = ab; + + icc_set_bw(pwr->icc_path, MBps_to_icc(ab), + kBps_to_icc(pwr->ddr_table[level])); + + trace_kgsl_buslevel(device, pwr->active_pwrlevel, level); + + return 0; +} + +static int adreno_gpu_bus_set(struct kgsl_device *device, int level, u32 ab) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + const struct adreno_power_ops *ops = ADRENO_POWER_OPS(adreno_dev); + + if (ops->gpu_bus_set) + return ops->gpu_bus_set(adreno_dev, level, ab); + + return adreno_interconnect_bus_set(adreno_dev, level, ab); +} + static const struct kgsl_functable adreno_functable = { /* Mandatory functions */ .regread = adreno_regread, @@ -3710,6 +3737,8 @@ static const struct kgsl_functable adreno_functable = { .stop_fault_timer = adreno_dispatcher_stop_fault_timer, .query_property_list = adreno_query_property_list, .is_hwcg_on = adreno_is_hwcg_on, + .gpu_clock_set = adreno_gpu_clock_set, + .gpu_bus_set = adreno_gpu_bus_set, }; static const struct component_master_ops adreno_ops = { @@ -3717,6 +3746,16 @@ static const struct component_master_ops adreno_ops = { .unbind = adreno_unbind, }; +const struct adreno_power_ops adreno_power_operations = { + .first_open = adreno_open, + .last_close = adreno_close, + .active_count_get = adreno_pwrctrl_active_count_get, + .active_count_put = adreno_pwrctrl_active_count_put, + .pm_suspend = adreno_suspend, + .pm_resume = adreno_resume, + .touch_wakeup = adreno_touch_wakeup, +}; + static const struct of_device_id adreno_gmu_match[] = { { .compatible = "qcom,gpu-gmu" }, { .compatible = "qcom,gpu-rgmu" }, diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 912224ca6057..8e5229d7df51 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -27,6 +27,12 @@ /* ADRENO_GPU_DEVICE - Given an adreno device return the GPU specific struct */ #define ADRENO_GPU_DEVICE(_a) ((_a)->gpucore->gpudev) +/* + * ADRENO_POWER_OPS - Given an adreno device return the GPU specific power + * ops + */ +#define ADRENO_POWER_OPS(_a) ((_a)->gpucore->gpudev->power_ops) + #define ADRENO_CHIPID_CORE(_id) (((_id) >> 24) & 0xFF) #define ADRENO_CHIPID_MAJOR(_id) (((_id) >> 16) & 0xFF) #define ADRENO_CHIPID_MINOR(_id) (((_id) >> 8) & 0xFF) @@ -97,14 +103,12 @@ * for droop mitigation */ #define ADRENO_ACD BIT(17) -/* ECP enabled GMU */ -#define ADRENO_ECP BIT(18) /* Cooperative reset enabled GMU */ -#define ADRENO_COOP_RESET BIT(19) +#define ADRENO_COOP_RESET BIT(18) /* Indicates that the specific target is no longer supported */ -#define ADRENO_DEPRECATED BIT(20) +#define ADRENO_DEPRECATED BIT(19) /* The target supports ringbuffer level APRIV */ -#define ADRENO_APRIV BIT(21) +#define ADRENO_APRIV BIT(20) /* * Adreno GPU quirks - control bits for various workarounds */ @@ -324,6 +328,46 @@ struct adreno_reglist { u32 value; }; +/** + * struct adreno_power_ops - Container for target specific power up/down + * sequences + */ +struct adreno_power_ops { + /** + * @first_open: Target specific function triggered when first kgsl + * instance is opened + */ + int (*first_open)(struct adreno_device *adreno_dev); + /** + * @last_close: Target specific function triggered when last kgsl + * instance is closed + */ + int (*last_close)(struct adreno_device *adreno_dev); + /** + * @active_count_get: Target specific function to keep gpu from power + * collapsing + */ + int (*active_count_get)(struct adreno_device *adreno_dev); + /** + * @active_count_put: Target specific function to allow gpu to power + * collapse + */ + void (*active_count_put)(struct adreno_device *adreno_dev); + /** @pm_suspend: Target specific function to suspend the driver */ + int (*pm_suspend)(struct adreno_device *adreno_dev); + /** @pm_resume: Target specific function to resume the driver */ + void (*pm_resume)(struct adreno_device *adreno_dev); + /** + * @touch_wakeup: Target specific function to start gpu on touch event + */ + void (*touch_wakeup)(struct adreno_device *adreno_dev); + /** @gpu_clock_set: Target specific function to set gpu frequency */ + int (*gpu_clock_set)(struct adreno_device *adreno_dev, u32 pwrlevel); + /** @gpu_bus_set: Target specific function to set gpu bandwidth */ + int (*gpu_bus_set)(struct adreno_device *adreno_dev, int bus_level, + u32 ab); +}; + /** * struct adreno_gpu_core - A specific GPU core definition * @gpurev: Unique GPU revision identifier @@ -361,9 +405,6 @@ struct adreno_gpu_core { * @chipid: Chip ID specific to the GPU * @cx_misc_len: Length of the CX MISC register block * @cx_misc_virt: Pointer where the CX MISC block is mapped - * @rscc_base: Base physical address of the RSCC - * @rscc_len: Length of the RSCC register block - * @rscc_virt: Pointer where RSCC block is mapped * @isense_base: Base physical address of isense block * @isense_len: Length of the isense register block * @isense_virt: Pointer where isense block is mapped @@ -443,9 +484,6 @@ struct adreno_device { void __iomem *cx_dbgc_virt; unsigned int cx_misc_len; void __iomem *cx_misc_virt; - unsigned long rscc_base; - unsigned int rscc_len; - void __iomem *rscc_virt; unsigned long isense_base; unsigned int isense_len; void __iomem *isense_virt; @@ -665,24 +703,9 @@ enum adreno_regs { ADRENO_REG_VBIF_VERSION, ADRENO_REG_GBIF_HALT, ADRENO_REG_GBIF_HALT_ACK, - ADRENO_REG_GMU_AO_INTERRUPT_EN, - ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, - ADRENO_REG_GMU_AO_HOST_INTERRUPT_STATUS, ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, - ADRENO_REG_GMU_PWR_COL_KEEPALIVE, ADRENO_REG_GMU_AHB_FENCE_STATUS, - ADRENO_REG_GMU_RPMH_POWER_STATE, - ADRENO_REG_GMU_HFI_CTRL_STATUS, - ADRENO_REG_GMU_HFI_VERSION_INFO, - ADRENO_REG_GMU_HFI_SFR_ADDR, - ADRENO_REG_GMU_GMU2HOST_INTR_CLR, - ADRENO_REG_GMU_GMU2HOST_INTR_INFO, ADRENO_REG_GMU_GMU2HOST_INTR_MASK, - ADRENO_REG_GMU_HOST2GMU_INTR_SET, - ADRENO_REG_GMU_HOST2GMU_INTR_CLR, - ADRENO_REG_GMU_HOST2GMU_INTR_RAW_INFO, - ADRENO_REG_GMU_NMI_CONTROL_STATUS, - ADRENO_REG_GMU_CM3_CFG, ADRENO_REG_GPMU_POWER_COUNTER_ENABLE, ADRENO_REG_REGISTER_MAX, }; @@ -746,9 +769,10 @@ struct adreno_gpudev { unsigned int gbif_arb_halt_mask; unsigned int gbif_gx_halt_mask; /* GPU specific function hooks */ + int (*probe)(struct platform_device *pdev, u32 chipid, + const struct adreno_gpu_core *gpucore); void (*snapshot)(struct adreno_device *adreno_dev, struct kgsl_snapshot *snapshot); - void (*platform_setup)(struct adreno_device *adreno_dev); irqreturn_t (*irq_handler)(struct adreno_device *adreno_dev); int (*init)(struct adreno_device *adreno_dev); void (*remove)(struct adreno_device *adreno_dev); @@ -787,7 +811,7 @@ struct adreno_gpudev { bool (*hw_isidle)(struct adreno_device *adreno_dev); const char *(*iommu_fault_block)(struct kgsl_device *device, unsigned int fsynr1); - int (*reset)(struct kgsl_device *device, int fault); + int (*reset)(struct kgsl_device *device); bool (*sptprac_is_on)(struct adreno_device *adreno_dev); unsigned int (*ccu_invalidate)(struct adreno_device *adreno_dev, unsigned int *cmds); @@ -796,6 +820,11 @@ struct adreno_gpudev { bool update_reg); /** @read_alwayson: Return the current value of the alwayson counter */ u64 (*read_alwayson)(struct adreno_device *adreno_dev); + /** + * @power_ops: Target specific function pointers to power up/down the + * gpu + */ + const struct adreno_power_ops *power_ops; }; /** @@ -857,6 +886,7 @@ struct adreno_ft_perf_counters { unsigned int countable; }; +extern const struct adreno_power_ops adreno_power_operations; extern unsigned int *adreno_ft_regs; extern unsigned int adreno_ft_regs_num; extern unsigned int *adreno_ft_regs_val; @@ -864,6 +894,9 @@ extern unsigned int *adreno_ft_regs_val; extern struct adreno_gpudev adreno_a3xx_gpudev; extern struct adreno_gpudev adreno_a5xx_gpudev; extern struct adreno_gpudev adreno_a6xx_gpudev; +extern struct adreno_gpudev adreno_a6xx_gmu_gpudev; +extern struct adreno_gpudev adreno_a6xx_rgmu_gpudev; +extern struct adreno_gpudev adreno_a630_gpudev; extern int adreno_wake_nice; extern unsigned int adreno_wake_timeout; @@ -939,8 +972,6 @@ void adreno_cx_misc_regwrite(struct adreno_device *adreno_dev, void adreno_cx_misc_regrmw(struct adreno_device *adreno_dev, unsigned int offsetwords, unsigned int mask, unsigned int bits); -void adreno_rscc_regread(struct adreno_device *adreno_dev, - unsigned int offsetwords, unsigned int *value); void adreno_isense_regread(struct adreno_device *adreno_dev, unsigned int offsetwords, unsigned int *value); @@ -952,6 +983,30 @@ void adreno_isense_regread(struct adreno_device *adreno_dev, */ bool adreno_irq_pending(struct adreno_device *adreno_dev); +/** + * adreno_active_count_get - Wrapper for target specific active count get + * @adreno_dev: pointer to the adreno device + * + * Increase the active count for the KGSL device and execute slumber exit + * sequence if this is the first reference. Code paths that need to touch the + * hardware or wait for the hardware to complete an operation must hold an + * active count reference until they are finished. The device mutex must be held + * while calling this function. + * + * Return: 0 on success or negative error on failure to wake up the device + */ +int adreno_active_count_get(struct adreno_device *adreno_dev); + +/** + * adreno_active_count_put - Wrapper for target specific active count put + * @adreno_dev: pointer to the adreno device + * + * Decrease the active or the KGSL device and schedule the idle thread to + * execute the slumber sequence if there are no remaining references. The + * device mutex must be held while calling this function. + */ +void adreno_active_count_put(struct adreno_device *adreno_dev); + #define ADRENO_TARGET(_name, _id) \ static inline int adreno_is_##_name(struct adreno_device *adreno_dev) \ { \ @@ -1558,28 +1613,32 @@ static inline unsigned int counter_delta(struct kgsl_device *device, return ret; } -static inline int adreno_perfcntr_active_oob_get(struct kgsl_device *device) +static inline int adreno_perfcntr_active_oob_get( + struct adreno_device *adreno_dev) { - int ret = kgsl_active_count_get(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret = adreno_active_count_get(adreno_dev); if (!ret) { ret = gmu_core_dev_oob_set(device, oob_perfcntr); if (ret) { - gmu_core_snapshot(device); - adreno_set_gpu_fault(ADRENO_DEVICE(device), + adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT_SKIP_SNAPSHOT); adreno_dispatcher_schedule(device); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); } } return ret; } -static inline void adreno_perfcntr_active_oob_put(struct kgsl_device *device) +static inline void adreno_perfcntr_active_oob_put( + struct adreno_device *adreno_dev) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + gmu_core_dev_oob_clear(device, oob_perfcntr); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); } static inline bool adreno_has_sptprac_gdsc(struct adreno_device *adreno_dev) @@ -1660,13 +1719,31 @@ static inline bool adreno_move_preempt_state(struct adreno_device *adreno_dev, return (atomic_cmpxchg(&adreno_dev->preempt.state, old, new) == old); } -void adreno_gmu_clear_and_unmask_irqs(struct adreno_device *adreno_dev); -void adreno_gmu_mask_and_clear_irqs(struct adreno_device *adreno_dev); +/** + * adreno_reg_offset_init - Helper function to initialize reg_offsets + * @reg_offsets: Pointer to an array of register offsets + * + * Helper function to setup register_offsets for a target. Go through + * and set ADRENO_REG_UNUSED for all unused entries in the list. + */ +static inline void adreno_reg_offset_init(u32 *reg_offsets) +{ + int i; + + /* + * Initialize uninitialzed gpu registers, only needs to be done once. + * Make all offsets that are not initialized to ADRENO_REG_UNUSED + */ + for (i = 0; i < ADRENO_REG_REGISTER_MAX; i++) { + if (!reg_offsets[i]) + reg_offsets[i] = ADRENO_REG_UNUSED; + } +} + int adreno_gmu_fenced_write(struct adreno_device *adreno_dev, enum adreno_regs offset, unsigned int val, unsigned int fence_mask); int adreno_clear_pending_transactions(struct kgsl_device *device); -void adreno_gmu_send_nmi(struct adreno_device *adreno_dev); /** @@ -1706,4 +1783,98 @@ int adreno_zap_shader_load(struct adreno_device *adreno_dev, irqreturn_t adreno_irq_callbacks(struct adreno_device *adreno_dev, struct adreno_irq_funcs *funcs, u32 status); + +/** + * adreno_device_probe - Generic adreno device probe function + * @pdev: Pointer to the platform device + * @adreno_dev: Adreno GPU device handle + * + * Do the generic setup for the Adreno device. Called from the target specific + * probe functions. + * + * Return: 0 on success or negative on failure + */ +int adreno_device_probe(struct platform_device *pdev, + struct adreno_device *adreno_dev); + +/** + * adreno_power_cycle - Suspend and resume the device + * @adreno_dev: Pointer to the adreno device + * @callback: Function that needs to be executed + * @priv: Argument to be passed to the callback + * + * Certain properties that can be set via sysfs need to power + * cycle the device to take effect. This function suspends + * the device, executes the callback, and resumes the device. + * + * Return: 0 on success or negative on failure + */ +int adreno_power_cycle(struct adreno_device *adreno_dev, + void (*callback)(struct adreno_device *adreno_dev, void *priv), + void *priv); + +/** + * adreno_power_cycle_bool - Power cycle the device to change device setting + * @adreno_dev: Pointer to the adreno device + * @flag: Flag that needs to be set + * @val: The value flag should be set to + * + * Certain properties that can be set via sysfs need to power cycle the device + * to take effect. This function suspends the device, sets the flag, and + * resumes the device. + * + * Return: 0 on success or negative on failure + */ +int adreno_power_cycle_bool(struct adreno_device *adreno_dev, + bool *flag, bool val); + +/** + * adreno_power_cycle_u32 - Power cycle the device to change device setting + * @adreno_dev: Pointer to the adreno device + * @flag: Flag that needs to be set + * @val: The value flag should be set to + * + * Certain properties that can be set via sysfs need to power cycle the device + * to take effect. This function suspends the device, sets the flag, and + * resumes the device. + * + * Return: 0 on success or negative on failure + */ +int adreno_power_cycle_u32(struct adreno_device *adreno_dev, + u32 *flag, u32 val); + +/** + * adreno_set_active_ctxs_null - Give up active context refcount + * @adreno_dev: Adreno GPU device handle + * + * This puts back the reference for that last active context on + * each ringbuffer when going in and out of slumber. + */ +void adreno_set_active_ctxs_null(struct adreno_device *adreno_dev); + +/** + * adreno_get_bus_counters - Allocate the bus dcvs counters + * @adreno_dev: Adreno GPU device handle + * + * This function allocates the various gpu counters to measure + * gpu bus usage for bus dcvs + */ +void adreno_get_bus_counters(struct adreno_device *adreno_dev); + +/** + * adreno_clear_dcvs_counters - Clear the dcvs measurements + * @adreno_dev: Adreno GPU device handle + * + * The various dcvs statistics need to be cleared everytime we + * power up the gpu + */ +void adreno_clear_dcvs_counters(struct adreno_device *adreno_dev); + +/** + * gmu_fault_snapshot - Set gmu fault and trigger snapshot + * @device: Pointer to the kgsl device + * + * Set the gmu fault and take snapshot when we hit a gmu fault + */ +void gmu_fault_snapshot(struct kgsl_device *device); #endif /*__ADRENO_H */ diff --git a/drivers/gpu/msm/adreno_a3xx.c b/drivers/gpu/msm/adreno_a3xx.c index 898bf3091515..a1e12f094686 100644 --- a/drivers/gpu/msm/adreno_a3xx.c +++ b/drivers/gpu/msm/adreno_a3xx.c @@ -5,6 +5,7 @@ #include #include +#include #include #include "adreno.h" @@ -567,10 +568,32 @@ static int _a3xx_pwron_fixup(struct adreno_device *adreno_dev) return 0; } -static void a3xx_platform_setup(struct adreno_device *adreno_dev) +static int a3xx_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) { + struct adreno_device *adreno_dev; + struct kgsl_device *device; + + adreno_dev = (struct adreno_device *) + of_device_get_match_data(&pdev->dev); + + memset(adreno_dev, 0, sizeof(*adreno_dev)); + + adreno_dev->gpucore = gpucore; + adreno_dev->chipid = chipid; + + adreno_reg_offset_init(gpucore->gpudev->reg_offsets); + /* Set the GPU busy counter for frequency scaling */ adreno_dev->perfctr_pwr_lo = A3XX_RBBM_PERFCTR_PWR_1_LO; + + device = KGSL_DEVICE(adreno_dev); + + timer_setup(&device->idle_timer, kgsl_timer, 0); + + INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + + return adreno_device_probe(pdev, adreno_dev); } static int a3xx_send_me_init(struct adreno_device *adreno_dev, @@ -1406,7 +1429,7 @@ struct adreno_gpudev adreno_a3xx_gpudev = { .perfcounters = &a3xx_perfcounters, .irq_handler = a3xx_irq_handler, .vbif_xin_halt_ctrl0_mask = A30X_VBIF_XIN_HALT_CTRL0_MASK, - .platform_setup = a3xx_platform_setup, + .probe = a3xx_probe, .rb_start = a3xx_rb_start, .init = a3xx_init, .microcode_read = a3xx_microcode_read, @@ -1418,4 +1441,5 @@ struct adreno_gpudev adreno_a3xx_gpudev = { .clk_set_options = a3xx_clk_set_options, .read_alwayson = a3xx_read_alwayson, .hw_isidle = a3xx_hw_isidle, + .power_ops = &adreno_power_operations, }; diff --git a/drivers/gpu/msm/adreno_a5xx.c b/drivers/gpu/msm/adreno_a5xx.c index 36f5e2ff21f3..cf23b5215fd6 100644 --- a/drivers/gpu/msm/adreno_a5xx.c +++ b/drivers/gpu/msm/adreno_a5xx.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -87,8 +88,22 @@ static void a530_efuse_leakage(struct adreno_device *adreno_dev) adreno_efuse_unmap(); } -static void a5xx_platform_setup(struct adreno_device *adreno_dev) +static int a5xx_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) { + struct adreno_device *adreno_dev; + struct kgsl_device *device; + + adreno_dev = (struct adreno_device *) + of_device_get_match_data(&pdev->dev); + + memset(adreno_dev, 0, sizeof(*adreno_dev)); + + adreno_dev->gpucore = gpucore; + adreno_dev->chipid = chipid; + + adreno_reg_offset_init(gpucore->gpudev->reg_offsets); + adreno_dev->sptp_pc_enabled = ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC); @@ -107,6 +122,14 @@ static void a5xx_platform_setup(struct adreno_device *adreno_dev) if (adreno_is_a530(adreno_dev)) a530_efuse_leakage(adreno_dev); + + device = KGSL_DEVICE(adreno_dev); + + timer_setup(&device->idle_timer, kgsl_timer, 0); + + INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + + return adreno_device_probe(pdev, adreno_dev); } static void _do_fixup(const struct adreno_critical_fixup *fixups, int count, @@ -1433,7 +1456,7 @@ static void a5xx_start(struct adreno_device *adreno_dev) /* Enable ISDB mode if requested */ if (test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv)) { - if (!kgsl_active_count_get(device)) { + if (!adreno_active_count_get(adreno_dev)) { /* * Disable ME/PFP split timeouts when the debugger is * enabled because the CP doesn't know when a shader is @@ -2970,9 +2993,9 @@ struct adreno_gpudev adreno_a5xx_gpudev = { #ifdef CONFIG_QCOM_KGSL_CORESIGHT .coresight = {&a5xx_coresight}, #endif + .probe = a5xx_probe, .start = a5xx_start, .snapshot = a5xx_snapshot, - .platform_setup = a5xx_platform_setup, .init = a5xx_init, .irq_handler = a5xx_irq_handler, .rb_start = a5xx_rb_start, @@ -2995,4 +3018,5 @@ struct adreno_gpudev adreno_a5xx_gpudev = { .clk_set_options = a5xx_clk_set_options, .read_alwayson = a5xx_read_alwayson, .hw_isidle = a5xx_hw_isidle, + .power_ops = &adreno_power_operations, }; diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c index 724e50413bfb..37ea5549fa32 100644 --- a/drivers/gpu/msm/adreno_a6xx.c +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "adreno.h" @@ -119,7 +120,7 @@ static u32 a615_pwrup_reglist[] = { static int a6xx_get_cp_init_cmds(struct adreno_device *adreno_dev); -static int a6xx_init(struct adreno_device *adreno_dev) +int a6xx_init(struct adreno_device *adreno_dev) { const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); @@ -400,13 +401,7 @@ static void a6xx_set_secvid(struct kgsl_device *device) #define A6XX_APRIV_DEFAULT \ ((1 << 6) | (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1)) -/* - * a6xx_start() - Device start - * @adreno_dev: Pointer to adreno device - * - * a6xx device start - */ -static void a6xx_start(struct adreno_device *adreno_dev) +void a6xx_start(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); @@ -617,7 +612,6 @@ static void a6xx_start(struct adreno_device *adreno_dev) * 3. HFI * At this point, we are guaranteed all. */ - gmu_core_dev_enable_lm(device); /* Configure LLCC */ a6xx_llc_configure_gpu_scid(adreno_dev); @@ -861,11 +855,7 @@ static int a6xx_post_start(struct adreno_device *adreno_dev) return ret; } -/* - * a6xx_rb_start() - Start the ringbuffer - * @adreno_dev: Pointer to adreno device - */ -static int a6xx_rb_start(struct adreno_device *adreno_dev) +int a6xx_rb_start(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct adreno_ringbuffer *rb; @@ -1031,11 +1021,7 @@ static bool a6xx_hw_isidle(struct adreno_device *adreno_dev) return (reg & BIT(23)) ? false : true; } -/* - * a6xx_microcode_read() - Read microcode - * @adreno_dev: Pointer to adreno device - */ -static int a6xx_microcode_read(struct adreno_device *adreno_dev) +int a6xx_microcode_read(struct adreno_device *adreno_dev) { struct adreno_firmware *sqe_fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); @@ -1139,25 +1125,16 @@ void a6xx_gx_cpr_toggle(struct kgsl_device *device) /** * a6xx_reset() - Helper function to reset the GPU * @device: Pointer to the KGSL device structure for the GPU - * @fault: Type of fault. Needed to skip soft reset for MMU fault * - * Try to reset the GPU to recover from a fault. First, try to do a low latency - * soft reset. If the soft reset fails for some reason, then bring out the big - * guns and toggle the footswitch. + * Try to reset the GPU to recover from a fault for targets without + * a GMU. */ -static int a6xx_reset(struct kgsl_device *device, int fault) +static int a6xx_reset(struct kgsl_device *device) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); int ret; - /* - * GMU devices transition to KGSL_STATE_RESET, non GMU devices go - * directly to KGSL_STATE_INIT - */ - if (gmu_core_isenabled(device)) - kgsl_pwrctrl_change_state(device, KGSL_STATE_RESET); - else - kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); + kgsl_pwrctrl_change_state(device, KGSL_STATE_INIT); /* since device is officially off now clear start bit */ clear_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv); @@ -2382,9 +2359,16 @@ static struct adreno_perfcounters a6xx_perfcounters = { ARRAY_SIZE(a6xx_perfcounter_groups), }; -static void a6xx_platform_setup(struct adreno_device *adreno_dev) +int a6xx_probe_common(struct platform_device *pdev, + struct adreno_device *adreno_dev, u32 chipid, + const struct adreno_gpu_core *gpucore) { - struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev); + struct adreno_gpudev *gpudev = gpucore->gpudev; + + adreno_dev->gpucore = gpucore; + adreno_dev->chipid = chipid; + + adreno_reg_offset_init(gpudev->reg_offsets); adreno_dev->hwcg_enabled = true; @@ -2421,6 +2405,34 @@ static void a6xx_platform_setup(struct adreno_device *adreno_dev) if (ADRENO_FEATURE(adreno_dev, ADRENO_IFPC)) adreno_dev->perfctr_ifpc_lo = A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L; + + + return adreno_device_probe(pdev, adreno_dev); +} + +static int a6xx_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + int ret; + + adreno_dev = (struct adreno_device *) + of_device_get_match_data(&pdev->dev); + + memset(adreno_dev, 0, sizeof(*adreno_dev)); + + ret = a6xx_probe_common(pdev, adreno_dev, chipid, gpucore); + if (ret) + return ret; + + device = KGSL_DEVICE(adreno_dev); + + timer_setup(&device->idle_timer, kgsl_timer, 0); + + INIT_WORK(&device->idle_check_ws, kgsl_idle_check); + + return 0; } @@ -2516,42 +2528,12 @@ static unsigned int a6xx_register_offsets[ADRENO_REG_REGISTER_MAX] = { A6XX_RBBM_GBIF_HALT_ACK), ADRENO_REG_DEFINE(ADRENO_REG_GBIF_HALT, A6XX_GBIF_HALT), ADRENO_REG_DEFINE(ADRENO_REG_GBIF_HALT_ACK, A6XX_GBIF_HALT_ACK), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_INTERRUPT_EN, - A6XX_GMU_AO_INTERRUPT_EN), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, - A6XX_GMU_AO_HOST_INTERRUPT_CLR), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_HOST_INTERRUPT_STATUS, - A6XX_GMU_AO_HOST_INTERRUPT_STATUS), ADRENO_REG_DEFINE(ADRENO_REG_GMU_AO_HOST_INTERRUPT_MASK, A6XX_GMU_AO_HOST_INTERRUPT_MASK), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_PWR_COL_KEEPALIVE, - A6XX_GMU_GMU_PWR_COL_KEEPALIVE), ADRENO_REG_DEFINE(ADRENO_REG_GMU_AHB_FENCE_STATUS, A6XX_GMU_AHB_FENCE_STATUS), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_HFI_CTRL_STATUS, - A6XX_GMU_HFI_CTRL_STATUS), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_HFI_VERSION_INFO, - A6XX_GMU_HFI_VERSION_INFO), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_HFI_SFR_ADDR, - A6XX_GMU_HFI_SFR_ADDR), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_RPMH_POWER_STATE, - A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_GMU2HOST_INTR_CLR, - A6XX_GMU_GMU2HOST_INTR_CLR), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_GMU2HOST_INTR_INFO, - A6XX_GMU_GMU2HOST_INTR_INFO), ADRENO_REG_DEFINE(ADRENO_REG_GMU_GMU2HOST_INTR_MASK, A6XX_GMU_GMU2HOST_INTR_MASK), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_HOST2GMU_INTR_SET, - A6XX_GMU_HOST2GMU_INTR_SET), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_HOST2GMU_INTR_CLR, - A6XX_GMU_HOST2GMU_INTR_CLR), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_HOST2GMU_INTR_RAW_INFO, - A6XX_GMU_HOST2GMU_INTR_RAW_INFO), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_NMI_CONTROL_STATUS, - A6XX_GMU_NMI_CONTROL_STATUS), - ADRENO_REG_DEFINE(ADRENO_REG_GMU_CM3_CFG, - A6XX_GMU_CM3_CFG), }; static int cpu_gpu_lock(struct cpu_gpu_lock *lock) @@ -2684,9 +2666,9 @@ u64 a6xx_read_alwayson(struct adreno_device *adreno_dev) struct adreno_gpudev adreno_a6xx_gpudev = { .reg_offsets = a6xx_register_offsets, + .probe = a6xx_probe, .start = a6xx_start, .snapshot = a6xx_snapshot, - .platform_setup = a6xx_platform_setup, .init = a6xx_init, .irq_handler = a6xx_irq_handler, .rb_start = a6xx_rb_start, @@ -2714,4 +2696,110 @@ struct adreno_gpudev adreno_a6xx_gpudev = { #endif .clk_set_options = a6xx_clk_set_options, .read_alwayson = a6xx_read_alwayson, + .power_ops = &adreno_power_operations, +}; + +struct adreno_gpudev adreno_a6xx_gmu_gpudev = { + .reg_offsets = a6xx_register_offsets, + .probe = a6xx_gmu_device_probe, + .start = a6xx_start, + .snapshot = a6xx_gmu_snapshot, + .init = a6xx_init, + .irq_handler = a6xx_irq_handler, + .rb_start = a6xx_rb_start, + .regulator_enable = a6xx_sptprac_enable, + .regulator_disable = a6xx_sptprac_disable, + .perfcounters = &a6xx_perfcounters, + .read_throttling_counters = a6xx_read_throttling_counters, + .microcode_read = a6xx_microcode_read, + .gpu_keepalive = a6xx_gpu_keepalive, + .hw_isidle = a6xx_hw_isidle, + .iommu_fault_block = a6xx_iommu_fault_block, + .reset = a6xx_gmu_restart, + .preemption_pre_ibsubmit = a6xx_preemption_pre_ibsubmit, + .preemption_post_ibsubmit = a6xx_preemption_post_ibsubmit, + .preemption_init = a6xx_preemption_init, + .preemption_schedule = a6xx_preemption_schedule, + .set_marker = a6xx_set_marker, + .preemption_context_init = a6xx_preemption_context_init, + .preemption_context_destroy = a6xx_preemption_context_destroy, + .sptprac_is_on = a6xx_sptprac_is_on, + .ccu_invalidate = a6xx_ccu_invalidate, + .perfcounter_update = a6xx_perfcounter_update, +#ifdef CONFIG_QCOM_KGSL_CORESIGHT + .coresight = {&a6xx_coresight, &a6xx_coresight_cx}, +#endif + .clk_set_options = a6xx_clk_set_options, + .read_alwayson = a6xx_read_alwayson, + .power_ops = &a6xx_gmu_power_ops, +}; + +struct adreno_gpudev adreno_a6xx_rgmu_gpudev = { + .reg_offsets = a6xx_register_offsets, + .probe = a6xx_rgmu_device_probe, + .start = a6xx_start, + .snapshot = a6xx_rgmu_snapshot, + .init = a6xx_init, + .irq_handler = a6xx_irq_handler, + .rb_start = a6xx_rb_start, + .regulator_enable = a6xx_sptprac_enable, + .regulator_disable = a6xx_sptprac_disable, + .perfcounters = &a6xx_perfcounters, + .read_throttling_counters = a6xx_read_throttling_counters, + .microcode_read = a6xx_microcode_read, + .gpu_keepalive = a6xx_gpu_keepalive, + .hw_isidle = a6xx_hw_isidle, + .iommu_fault_block = a6xx_iommu_fault_block, + .reset = a6xx_rgmu_restart, + .preemption_pre_ibsubmit = a6xx_preemption_pre_ibsubmit, + .preemption_post_ibsubmit = a6xx_preemption_post_ibsubmit, + .preemption_init = a6xx_preemption_init, + .preemption_schedule = a6xx_preemption_schedule, + .set_marker = a6xx_set_marker, + .preemption_context_init = a6xx_preemption_context_init, + .preemption_context_destroy = a6xx_preemption_context_destroy, + .sptprac_is_on = a6xx_sptprac_is_on, + .ccu_invalidate = a6xx_ccu_invalidate, + .perfcounter_update = a6xx_perfcounter_update, +#ifdef CONFIG_QCOM_KGSL_CORESIGHT + .coresight = {&a6xx_coresight, &a6xx_coresight_cx}, +#endif + .clk_set_options = a6xx_clk_set_options, + .read_alwayson = a6xx_read_alwayson, + .power_ops = &a6xx_rgmu_power_ops, +}; + +struct adreno_gpudev adreno_a630_gpudev = { + .reg_offsets = a6xx_register_offsets, + .probe = a6xx_gmu_device_probe, + .start = a6xx_start, + .snapshot = a6xx_gmu_snapshot, + .init = a6xx_init, + .irq_handler = a6xx_irq_handler, + .rb_start = a6xx_rb_start, + .regulator_enable = a6xx_sptprac_enable, + .regulator_disable = a6xx_sptprac_disable, + .perfcounters = &a6xx_perfcounters, + .read_throttling_counters = a6xx_read_throttling_counters, + .microcode_read = a6xx_microcode_read, + .gpu_keepalive = a6xx_gpu_keepalive, + .hw_isidle = a6xx_hw_isidle, + .iommu_fault_block = a6xx_iommu_fault_block, + .reset = a6xx_gmu_restart, + .preemption_pre_ibsubmit = a6xx_preemption_pre_ibsubmit, + .preemption_post_ibsubmit = a6xx_preemption_post_ibsubmit, + .preemption_init = a6xx_preemption_init, + .preemption_schedule = a6xx_preemption_schedule, + .set_marker = a6xx_set_marker, + .preemption_context_init = a6xx_preemption_context_init, + .preemption_context_destroy = a6xx_preemption_context_destroy, + .sptprac_is_on = a6xx_sptprac_is_on, + .ccu_invalidate = a6xx_ccu_invalidate, + .perfcounter_update = a6xx_perfcounter_update, +#ifdef CONFIG_QCOM_KGSL_CORESIGHT + .coresight = {&a6xx_coresight, &a6xx_coresight_cx}, +#endif + .clk_set_options = a6xx_clk_set_options, + .read_alwayson = a6xx_read_alwayson, + .power_ops = &a630_gmu_power_ops, }; diff --git a/drivers/gpu/msm/adreno_a6xx.h b/drivers/gpu/msm/adreno_a6xx.h index 126c50bcd209..60c155836b7d 100644 --- a/drivers/gpu/msm/adreno_a6xx.h +++ b/drivers/gpu/msm/adreno_a6xx.h @@ -7,8 +7,27 @@ #define _ADRENO_A6XX_H_ #include +#include #include "a6xx_reg.h" +#include "adreno_a6xx_gmu.h" +#include "adreno_a6xx_rgmu.h" + +extern const struct adreno_power_ops a6xx_gmu_power_ops; +extern const struct adreno_power_ops a6xx_rgmu_power_ops; +extern const struct adreno_power_ops a630_gmu_power_ops; + +/** + * struct a6xx_device - Container for the a6xx_device + */ +struct a6xx_device { + /** @gmu: Container for the a6xx GMU device */ + struct a6xx_gmu_device gmu; + /** @rgmu: Container for the a6xx rGMU device */ + struct a6xx_rgmu_device rgmu; + /** @adreno_dev: Container for the generic adreno device */ + struct adreno_device adreno_dev; +}; /** * struct a6xx_protected_regs - container for a protect register span @@ -176,74 +195,35 @@ to_a6xx_core(struct adreno_device *adreno_dev) return container_of(core, struct adreno_a6xx_core, base); } -/* +/** * timed_poll_check() - polling *gmu* register at given offset until * its value changed to match expected value. The function times * out and returns after given duration if register is not updated * as expected. * * @device: Pointer to KGSL device - * @offset: Register offset + * @offset: Register offset in dwords * @expected_ret: expected register value that stops polling - * @timout: number of jiffies to abort the polling + * @timeout_ms: time in milliseconds to poll the register * @mask: bitmask to filter register value to match expected_ret */ static inline int timed_poll_check(struct kgsl_device *device, unsigned int offset, unsigned int expected_ret, - unsigned int timeout, unsigned int mask) + unsigned int timeout_ms, unsigned int mask) { - unsigned long t; - unsigned int value; + u32 val; + void __iomem *addr = device->gmu_core.reg_virt + + ((offset - device->gmu_core.gmu2gpu_offset) << 2); - t = jiffies + msecs_to_jiffies(timeout); + if (WARN(!gmu_core_is_register_offset(device, offset), + "Out of bounds register read: 0x%x\n", offset)) + return -EINVAL; - do { - gmu_core_regread(device, offset, &value); - if ((value & mask) == expected_ret) - return 0; - /* Wait 100us to reduce unnecessary AHB bus traffic */ - usleep_range(10, 100); - } while (!time_after(jiffies, t)); + if (readl_poll_timeout(addr, val, (val & mask) == expected_ret, 100, + timeout_ms * 1000)) + return -ETIMEDOUT; - /* Double check one last time */ - gmu_core_regread(device, offset, &value); - if ((value & mask) == expected_ret) - return 0; - - return -ETIMEDOUT; -} - -static inline int timed_poll_check_rscc(struct kgsl_device *device, - unsigned int offset, unsigned int expected_ret, - unsigned int timeout, unsigned int mask) -{ - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - unsigned long t; - unsigned int value; - - t = jiffies + msecs_to_jiffies(timeout); - - do { - if (adreno_is_a650_family(adreno_dev)) - adreno_rscc_regread(adreno_dev, offset, &value); - else - gmu_core_regread(device, offset + RSCC_OFFSET_LEGACY, - &value); - if ((value & mask) == expected_ret) - return 0; - /* Wait 100us to reduce unnecessary AHB bus traffic */ - usleep_range(10, 100); - } while (!time_after(jiffies, t)); - - /* Double check one last time */ - if (adreno_is_a650_family(adreno_dev)) - adreno_rscc_regread(adreno_dev, offset, &value); - else - gmu_core_regread(device, offset + RSCC_OFFSET_LEGACY, &value); - if ((value & mask) == expected_ret) - return 0; - - return -ETIMEDOUT; + return 0; } /* Preemption functions */ @@ -275,10 +255,67 @@ void a6xx_gmu_sptprac_disable(struct adreno_device *adreno_dev); bool a6xx_gmu_sptprac_is_on(struct adreno_device *adreno_dev); /** - * a6xx_read_alwayson: Read the current always on clock value + * a6xx_read_alwayson - Read the current always on clock value * @adreno_dev: An Adreno GPU handle * * Return: The current value of the GMU always on counter */ u64 a6xx_read_alwayson(struct adreno_device *adreno_dev); + +/** + * a6xx_start - Program a6xx registers + * @adreno_dev: An Adreno GPU handle + * + * This function does all a6xx register programming every + * time we boot the gpu + */ +void a6xx_start(struct adreno_device *adreno_dev); + +/** + * a6xx_init - Initialize a6xx resources + * @adreno_dev: An Adreno GPU handle + * + * This function does a6xx specific one time initialization + * and is invoked when the very first client opens a + * kgsl instance + * + * Return: Zero on success and negative error on failure + */ +int a6xx_init(struct adreno_device *adreno_dev); + +/** + * a6xx_rb_start - A6xx specific ringbuffer setup + * @adreno_dev: An Adreno GPU handle + * + * This function does a6xx specific ringbuffer setup and + * attempts to submit CP INIT and bring GPU out of secure mode + * + * Return: Zero on success and negative error on failure + */ +int a6xx_rb_start(struct adreno_device *adreno_dev); + +/** + * a6xx_microcode_read - Get the cp microcode from the filesystem + * @adreno_dev: An Adreno GPU handle + * + * This function gets the firmware from filesystem and sets up + * the micorocode global buffer + * + * Return: Zero on success and negative error on failure + */ +int a6xx_microcode_read(struct adreno_device *adreno_dev); + +/** + * a6xx_probe_common - Probe common a6xx resources + * @pdev: Pointer to the platform device + * @adreno_dev: Pointer to the adreno device + * @chipid: Chipid of the target + * @gpucore: Pointer to the gpucore strucure + * + * This function sets up the a6xx resources common across all + * a6xx targets + */ +int a6xx_probe_common(struct platform_device *pdev, + struct adreno_device *adreno_dev, u32 chipid, + const struct adreno_gpu_core *gpucore); #endif diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index 84398b5a4618..edd48127fe52 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -3,67 +3,143 @@ * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. */ -/* soc/qcom/cmd-db.h needs types.h */ +#include +#include +#include #include +#include #include -#include +#include +#include #include +#include +#include #include #include "adreno.h" #include "adreno_a6xx.h" -#include "adreno_snapshot.h" -#include "kgsl_gmu.h" +#include "kgsl_bus.h" +#include "kgsl_device.h" #include "kgsl_trace.h" +#include "kgsl_util.h" -static const unsigned int a6xx_gmu_gx_registers[] = { - /* GMU GX */ - 0x1A800, 0x1A800, 0x1A810, 0x1A813, 0x1A816, 0x1A816, 0x1A818, 0x1A81B, - 0x1A81E, 0x1A81E, 0x1A820, 0x1A823, 0x1A826, 0x1A826, 0x1A828, 0x1A82B, - 0x1A82E, 0x1A82E, 0x1A830, 0x1A833, 0x1A836, 0x1A836, 0x1A838, 0x1A83B, - 0x1A83E, 0x1A83E, 0x1A840, 0x1A843, 0x1A846, 0x1A846, 0x1A880, 0x1A884, - 0x1A900, 0x1A92B, 0x1A940, 0x1A940, +struct gmu_iommu_context { + const char *name; + struct platform_device *pdev; + struct iommu_domain *domain; }; -static const unsigned int a6xx_gmu_tcm_registers[] = { - /* ITCM */ - 0x1B400, 0x1C3FF, - /* DTCM */ - 0x1C400, 0x1D3FF, +#define ARC_VOTE_GET_PRI(_v) ((_v) & 0xFF) +#define ARC_VOTE_GET_SEC(_v) (((_v) >> 8) & 0xFF) +#define ARC_VOTE_GET_VLVL(_v) (((_v) >> 16) & 0xFFFF) + +#define ARC_VOTE_SET(pri, sec, vlvl) \ + ((((vlvl) & 0xFFFF) << 16) | (((sec) & 0xFF) << 8) | ((pri) & 0xFF)) + +static struct gmu_vma_entry a6xx_gmu_vma_legacy[] = { + [GMU_ITCM] = { + .start = 0x00000, + .size = SZ_16K + }, + [GMU_ICACHE] = { + .start = 0x04000, + .size = (SZ_256K - SZ_16K), + .next_va = 0x4000 + }, + [GMU_DTCM] = { + .start = 0x40000, + .size = SZ_16K + }, + [GMU_DCACHE] = { + .start = 0x44000, + .size = (SZ_256K - SZ_16K), + .next_va = 0x44000 + }, + [GMU_NONCACHED_KERNEL] = { + .start = 0x60000000, + .size = SZ_512M, + .next_va = 0x60000000 + }, + [GMU_NONCACHED_USER] = { + .start = 0x80000000, + .size = SZ_1G, + .next_va = 0x80000000 + }, }; -static const unsigned int a6xx_gmu_registers[] = { - /* GMU CX */ - 0x1F400, 0x1F407, 0x1F410, 0x1F412, 0x1F500, 0x1F500, 0x1F507, 0x1F50A, - 0x1F800, 0x1F804, 0x1F807, 0x1F808, 0x1F80B, 0x1F80C, 0x1F80F, 0x1F81C, - 0x1F824, 0x1F82A, 0x1F82D, 0x1F830, 0x1F840, 0x1F853, 0x1F887, 0x1F889, - 0x1F8A0, 0x1F8A2, 0x1F8A4, 0x1F8AF, 0x1F8C0, 0x1F8C3, 0x1F8D0, 0x1F8D0, - 0x1F8E4, 0x1F8E4, 0x1F8E8, 0x1F8EC, 0x1F900, 0x1F903, 0x1F940, 0x1F940, - 0x1F942, 0x1F944, 0x1F94C, 0x1F94D, 0x1F94F, 0x1F951, 0x1F954, 0x1F954, - 0x1F957, 0x1F958, 0x1F95D, 0x1F95D, 0x1F962, 0x1F962, 0x1F964, 0x1F965, - 0x1F980, 0x1F986, 0x1F990, 0x1F99E, 0x1F9C0, 0x1F9C0, 0x1F9C5, 0x1F9CC, - 0x1F9E0, 0x1F9E2, 0x1F9F0, 0x1F9F0, 0x1FA00, 0x1FA01, - /* GMU AO */ - 0x23B00, 0x23B16, - /* GPU CC */ - 0x24000, 0x24012, 0x24040, 0x24052, 0x24400, 0x24404, 0x24407, 0x2440B, - 0x24415, 0x2441C, 0x2441E, 0x2442D, 0x2443C, 0x2443D, 0x2443F, 0x24440, - 0x24442, 0x24449, 0x24458, 0x2445A, 0x24540, 0x2455E, 0x24800, 0x24802, - 0x24C00, 0x24C02, 0x25400, 0x25402, 0x25800, 0x25802, 0x25C00, 0x25C02, - 0x26000, 0x26002, - /* GPU CC ACD */ - 0x26400, 0x26416, 0x26420, 0x26427, +static struct gmu_vma_entry a6xx_gmu_vma[] = { + [GMU_ITCM] = { + .start = 0x00000000, + .size = SZ_16K + }, + [GMU_CACHE] = { + .start = SZ_16K, + .size = (SZ_16M - SZ_16K), + .next_va = SZ_16K + }, + [GMU_DTCM] = { + .start = SZ_256M + SZ_16K, + .size = SZ_16K + }, + [GMU_DCACHE] = { + .start = 0x0, + .size = 0x0 + }, + [GMU_NONCACHED_KERNEL] = { + .start = 0x60000000, + .size = SZ_512M, + .next_va = 0x60000000 + }, + [GMU_NONCACHED_USER] = { + .start = 0x80000000, + .size = SZ_1G, + .next_va = 0x80000000 + }, }; -static const unsigned int a660_gmu_registers[] = { - /* GMU CX */ - 0x1F408, 0x1F40D, 0x1F40F, 0x1F40F, 0x1F50B, 0x1F50B, 0x1F860, 0x1F860, - 0x1F870, 0x1F877, 0x1F8C4, 0x1F8C4, 0x1F8F0, 0x1F8F1, 0x1F948, 0x1F94A, - 0x1F966, 0x1F96B, 0x1F970, 0x1F970, 0x1F972, 0x1F979, 0x1F9CD, 0x1F9D4, - 0x1FA02, 0x1FA03, 0x20000, 0x20001, 0x20004, 0x20004, 0x20008, 0x20012, - 0x20018, 0x20018, +static struct gmu_iommu_context a6xx_gmu_ctx[] = { + [GMU_CONTEXT_USER] = { .name = "gmu_user" }, + [GMU_CONTEXT_KERNEL] = { .name = "gmu_kernel" } }; +static int timed_poll_check_rscc(struct kgsl_device *device, + unsigned int offset, unsigned int expected_ret, + unsigned int timeout, unsigned int mask) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + u32 value; + + if (!adreno_is_a650_family(adreno_dev)) + return timed_poll_check(device, offset + RSCC_OFFSET_LEGACY, + expected_ret, timeout, mask); + + return readl_poll_timeout(gmu->rscc_virt + (offset << 2), value, + (value & mask) == expected_ret, 100, timeout * 1000); +} + +void gmu_fault_snapshot(struct kgsl_device *device) +{ + device->gmu_fault = true; + kgsl_device_snapshot(device, NULL, true); +} + +struct a6xx_gmu_device *to_a6xx_gmu(struct adreno_device *adreno_dev) +{ + struct a6xx_device *a6xx_dev = container_of(adreno_dev, + struct a6xx_device, adreno_dev); + + return &a6xx_dev->gmu; +} + +struct adreno_device *a6xx_gmu_to_adreno(struct a6xx_gmu_device *gmu) +{ + struct a6xx_device *a6xx_dev = + container_of(gmu, struct a6xx_device, gmu); + + return &a6xx_dev->adreno_dev; +} + #define RSC_CMD_OFFSET 2 #define PDC_CMD_OFFSET 4 @@ -76,13 +152,14 @@ static void _regwrite(void __iomem *regbase, __raw_writel(value, reg); } -static void a6xx_load_rsc_ucode(struct kgsl_device *device) +static void a6xx_load_rsc_ucode(struct adreno_device *adreno_dev) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); void __iomem *rscc; if (adreno_is_a650_family(adreno_dev)) - rscc = adreno_dev->rscc_virt; + rscc = gmu->rscc_virt; else rscc = device->gmu_core.reg_virt + 0x23000; @@ -120,10 +197,9 @@ static void a6xx_load_rsc_ucode(struct kgsl_device *device) } } -static int a6xx_load_pdc_ucode(struct kgsl_device *device) +static int a6xx_load_pdc_ucode(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct resource *res_pdc, *res_cfg, *res_seq; unsigned int cfg_offset, seq_offset; void __iomem *cfg = NULL, *seq = NULL; @@ -290,12 +366,6 @@ done: return 0; } -static int _load_gmu_rpmh_ucode(struct kgsl_device *device) -{ - a6xx_load_rsc_ucode(device); - return a6xx_load_pdc_ucode(device); -} - /* GMU timeouts */ #define GMU_IDLE_TIMEOUT 100 /* ms */ #define GMU_START_TIMEOUT 100 /* ms */ @@ -315,12 +385,12 @@ static int _load_gmu_rpmh_ucode(struct kgsl_device *device) /* * a6xx_gmu_power_config() - Configure and enable GMU's low power mode * setting based on ADRENO feature flags. - * @device: Pointer to KGSL device + * @adreno_dev: Pointer to adreno device */ -static void a6xx_gmu_power_config(struct kgsl_device *device) +static void a6xx_gmu_power_config(struct adreno_device *adreno_dev) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); /* Configure registers for idle setting. The setting is cumulative */ @@ -365,13 +435,10 @@ static void a6xx_gmu_power_config(struct kgsl_device *device) RPMH_ENABLE_MASK); } -/* - * a6xx_gmu_start() - Start GMU and wait until FW boot up. - * @device: Pointer to KGSL device - */ -static int a6xx_gmu_start(struct kgsl_device *device) +static int a6xx_gmu_device_start(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); u32 val = 0x00000100; u32 mask = 0x000001FF; @@ -399,7 +466,7 @@ static int a6xx_gmu_start(struct kgsl_device *device) gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + (0x3fdc >> 2), &val); dev_err(&gmu->pdev->dev, "GMU doesn't boot: 0x%x\n", val); - + gmu_fault_snapshot(device); return -ETIMEDOUT; } @@ -410,9 +477,10 @@ static int a6xx_gmu_start(struct kgsl_device *device) * a6xx_gmu_hfi_start() - Write registers and start HFI. * @device: Pointer to KGSL device */ -static int a6xx_gmu_hfi_start(struct kgsl_device *device) +static int a6xx_gmu_hfi_start(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); gmu_core_regwrite(device, A6XX_GMU_HFI_CTRL_INIT, 1); @@ -422,22 +490,23 @@ static int a6xx_gmu_hfi_start(struct kgsl_device *device) GMU_START_TIMEOUT, BIT(0))) { dev_err(&gmu->pdev->dev, "GMU HFI init failed\n"); + gmu_fault_snapshot(device); return -ETIMEDOUT; } return 0; } -static int a6xx_rpmh_power_on_gpu(struct kgsl_device *device) +static int a6xx_rscc_wakeup_sequence(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct device *dev = &gmu->pdev->dev; int val; - /* Only trigger wakeup sequence if sleep sequence was done earlier */ - if (!test_bit(GMU_RSCC_SLEEP_SEQ_DONE, &device->gmu_core.flags)) + /* Skip wakeup sequence if we didn't do the sleep sequence */ + if (!test_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags)) return 0; - /* A660 has a replacement register */ if (adreno_is_a660(ADRENO_DEVICE(device))) gmu_core_regread(device, A6XX_GPU_CC_GX_DOMAIN_MISC3, &val); @@ -460,7 +529,7 @@ static int a6xx_rpmh_power_on_gpu(struct kgsl_device *device) GPU_START_TIMEOUT, BIT(1))) { dev_err(dev, "Failed to do GPU RSC power on\n"); - return -EINVAL; + return -ETIMEDOUT; } if (timed_poll_check_rscc(device, @@ -472,25 +541,25 @@ static int a6xx_rpmh_power_on_gpu(struct kgsl_device *device) gmu_core_regwrite(device, A6XX_GMU_RSCC_CONTROL_REQ, 0); - /* Clear sleep sequence flag as wakeup sequence is successful */ - clear_bit(GMU_RSCC_SLEEP_SEQ_DONE, &device->gmu_core.flags); - - /* Enable the power counter because it was disabled before slumber */ - gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1); + clear_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags); return 0; + error_rsc: dev_err(dev, "GPU RSC sequence stuck in waking up GPU\n"); - return -EINVAL; + return -ETIMEDOUT; } -static int a6xx_rpmh_power_off_gpu(struct kgsl_device *device) +static int a6xx_rscc_sleep_sequence(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); int ret; - if (test_bit(GMU_RSCC_SLEEP_SEQ_DONE, &device->gmu_core.flags)) + if (!test_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags)) + return 0; + + if (test_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags)) return 0; gmu_core_regwrite(device, A6XX_GMU_CM3_SYSRESET, 1); @@ -520,12 +589,55 @@ static int a6xx_rpmh_power_off_gpu(struct kgsl_device *device) if (adreno_dev->lm_enabled) gmu_core_regwrite(device, A6XX_GMU_AO_SPARE_CNTL, 0); - set_bit(GMU_RSCC_SLEEP_SEQ_DONE, &device->gmu_core.flags); + set_bit(GMU_PRIV_RSCC_SLEEP_DONE, &gmu->flags); + return 0; } +static struct gmu_memdesc *find_gmu_memdesc(struct a6xx_gmu_device *gmu, + u32 addr, u32 size) +{ + int i; + + for (i = 0; i < gmu->global_entries; i++) { + struct gmu_memdesc *md = &gmu->gmu_globals[i]; + + if ((addr >= md->gmuaddr) && + (((addr + size) <= (md->gmuaddr + md->size)))) + return md; + } + + return NULL; +} + +static int find_vma_block(struct a6xx_gmu_device *gmu, u32 addr, u32 size) +{ + int i; + + for (i = 0; i < GMU_MEM_TYPE_MAX; i++) { + struct gmu_vma_entry *vma = &gmu->vma[i]; + + if ((addr >= vma->start) && + ((addr + size) <= (vma->start + vma->size))) + return i; + } + + return -ENOENT; +} + +static struct iommu_domain *get_gmu_domain(struct a6xx_gmu_device *gmu, + struct gmu_memdesc *md) +{ + u32 vma_id = find_vma_block(gmu, md->gmuaddr, md->size); + + if (vma_id == GMU_NONCACHED_USER) + return a6xx_gmu_ctx[GMU_CONTEXT_USER].domain; + + return a6xx_gmu_ctx[GMU_CONTEXT_KERNEL].domain; +} + static int _load_legacy_gmu_fw(struct kgsl_device *device, - struct gmu_device *gmu) + struct a6xx_gmu_device *gmu) { const struct firmware *fw = gmu->fw_image; @@ -540,49 +652,66 @@ static int _load_legacy_gmu_fw(struct kgsl_device *device, return 0; } -static int load_gmu_fw(struct kgsl_device *device) +static void load_tcm(struct adreno_device *adreno_dev, const u8 *src, + u32 tcm_start, u32 base, const struct gmu_block_header *blk) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - uint8_t *fw = (uint8_t *)gmu->fw_image->data; - int tcm_addr; - struct gmu_block_header *blk; - struct gmu_memdesc *md; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 tcm_offset = tcm_start + ((blk->addr - base)/sizeof(u32)); + void __iomem *addr = device->gmu_core.reg_virt + + ((tcm_offset - device->gmu_core.gmu2gpu_offset) << 2); - if (adreno_is_a630(ADRENO_DEVICE(device)) || - adreno_is_a615_family(ADRENO_DEVICE(device))) - return _load_legacy_gmu_fw(device, gmu); + memcpy_toio(addr, src, blk->size); +} + +static int a6xx_gmu_load_fw(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + const u8 *fw = (const u8 *)gmu->fw_image->data; + + if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) + return _load_legacy_gmu_fw(KGSL_DEVICE(adreno_dev), gmu); + + while (fw < gmu->fw_image->data + gmu->fw_image->size) { + const struct gmu_block_header *blk = + (const struct gmu_block_header *)fw; + int id; - while (fw < (uint8_t *)gmu->fw_image->data + gmu->fw_image->size) { - blk = (struct gmu_block_header *)fw; fw += sizeof(*blk); /* Don't deal with zero size blocks */ if (blk->size == 0) continue; - md = gmu_get_memdesc(gmu, blk->addr, blk->size); - if (md == NULL) { + id = find_vma_block(gmu, blk->addr, blk->size); + + if (id < 0) { dev_err(&gmu->pdev->dev, - "No backing memory for 0x%8.8X\n", - blk->addr); + "Unknown block in GMU FW addr:0x%x size:0x%x\n", + blk->addr, blk->size); return -EINVAL; } - if (md->mem_type == GMU_ITCM || md->mem_type == GMU_DTCM) { - tcm_addr = (blk->addr - (uint32_t)md->gmuaddr) / - sizeof(uint32_t); - - if (md->mem_type == GMU_ITCM) - tcm_addr += A6XX_GMU_CM3_ITCM_START; - else - tcm_addr += A6XX_GMU_CM3_DTCM_START; - - gmu_core_blkwrite(device, tcm_addr, fw, blk->size); + if (id == GMU_ITCM) { + load_tcm(adreno_dev, fw, + A6XX_GMU_CM3_ITCM_START, + gmu->vma[GMU_ITCM].start, blk); + } else if (id == GMU_DTCM) { + load_tcm(adreno_dev, fw, + A6XX_GMU_CM3_DTCM_START, + gmu->vma[GMU_DTCM].start, blk); } else { - uint32_t offset = blk->addr - (uint32_t)md->gmuaddr; + struct gmu_memdesc *md = + find_gmu_memdesc(gmu, blk->addr, blk->size); - /* Copy the memory directly */ - memcpy(md->hostptr + offset, fw, blk->size); + if (!md) { + dev_err(&gmu->pdev->dev, + "No backing memory for GMU FW block addr:0x%x size:0x%x\n", + blk->addr, blk->size); + return -EINVAL; + } + + memcpy(md->hostptr + (blk->addr - md->gmuaddr), fw, + blk->size); } fw += blk->size; @@ -606,16 +735,11 @@ static const char *oob_to_str(enum oob_request req) return "unknown"; } -/* - * a6xx_gmu_oob_set() - Set OOB interrupt to GMU. - * @device: Pointer to kgsl device - * @req: Which of the OOB bits to request - */ static int a6xx_gmu_oob_set(struct kgsl_device *device, enum oob_request req) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); int ret = 0; int set, check; @@ -642,6 +766,7 @@ static int a6xx_gmu_oob_set(struct kgsl_device *device, if (timed_poll_check(device, A6XX_GMU_GMU2HOST_INTR_INFO, check, GPU_START_TIMEOUT, check)) { + gmu_fault_snapshot(device); ret = -ETIMEDOUT; WARN(1, "OOB request %s timed out\n", oob_to_str(req)); } @@ -652,16 +777,11 @@ static int a6xx_gmu_oob_set(struct kgsl_device *device, return ret; } -/* - * a6xx_gmu_oob_clear() - Clear a previously set OOB request. - * @device: Pointer to the kgsl device that has the GMU - * @req: Which of the OOB bits to clear - */ static void a6xx_gmu_oob_clear(struct kgsl_device *device, enum oob_request req) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); int clear; if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) { @@ -679,35 +799,48 @@ static void a6xx_gmu_oob_clear(struct kgsl_device *device, trace_kgsl_gmu_oob_clear(clear); } -static void a6xx_gmu_irq_enable(struct kgsl_device *device) +static void a6xx_gmu_irq_enable(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct kgsl_hfi *hfi = &gmu->hfi; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hfi *hfi = &gmu->hfi; /* Clear pending IRQs and Unmask needed IRQs */ - adreno_gmu_clear_and_unmask_irqs(ADRENO_DEVICE(device)); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, 0xffffffff); + + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, + (unsigned int)~HFI_IRQ_MASK); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK, + (unsigned int)~GMU_AO_INT_MASK); + /* Enable all IRQs on host */ - enable_irq(hfi->hfi_interrupt_num); - enable_irq(gmu->gmu_interrupt_num); + enable_irq(hfi->irq); + enable_irq(gmu->irq); } -static void a6xx_gmu_irq_disable(struct kgsl_device *device) +static void a6xx_gmu_irq_disable(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct kgsl_hfi *hfi = &gmu->hfi; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hfi *hfi = &gmu->hfi; /* Disable all IRQs on host */ - disable_irq(gmu->gmu_interrupt_num); - disable_irq(hfi->hfi_interrupt_num); + disable_irq(gmu->irq); + disable_irq(hfi->irq); /* Mask all IRQs and clear pending IRQs */ - adreno_gmu_mask_and_clear_irqs(ADRENO_DEVICE(device)); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK, 0xffffffff); + + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, 0xffffffff); + } -static int a6xx_gmu_hfi_start_msg(struct kgsl_device *device) +static int a6xx_gmu_hfi_start_msg(struct adreno_device *adreno_dev) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); struct hfi_start_cmd req; /* @@ -716,7 +849,7 @@ static int a6xx_gmu_hfi_start_msg(struct kgsl_device *device) * legacy firmware. */ if (!ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) - return hfi_send_req(KGSL_GMU_DEVICE(device), + return a6xx_hfi_send_req(adreno_dev, H2F_MSG_START, &req); return 0; @@ -726,6 +859,8 @@ static int a6xx_gmu_hfi_start_msg(struct kgsl_device *device) #define FREQ_VOTE(idx, ack) (((idx) & 0xFF) | (((ack) & 0xF) << 28)) #define BW_VOTE(idx) ((((idx) & 0xFFF) << 12) | ((idx) & 0xFFF)) +#define CLKSET_OPTION_ATLEAST 3 + /* * a6xx_gmu_dcvs_nohfi() - request GMU to do DCVS without using HFI * @device: Pointer to KGSL device @@ -754,8 +889,10 @@ static int a6xx_gmu_dcvs_nohfi(struct kgsl_device *device, return ret; } -static int a6xx_complete_rpmh_votes(struct kgsl_device *device) + +static int a6xx_complete_rpmh_votes(struct adreno_device *adreno_dev) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); int ret = 0; ret |= timed_poll_check_rscc(device, A6XX_RSCC_TCS0_DRV0_STATUS, @@ -784,7 +921,7 @@ static int a6xx_complete_rpmh_votes(struct kgsl_device *device) int a6xx_gmu_sptprac_enable(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); if (!gmu_core_gpmu_isenabled(device) || !adreno_has_sptprac_gdsc(adreno_dev)) @@ -799,7 +936,8 @@ int a6xx_gmu_sptprac_enable(struct adreno_device *adreno_dev) SPTPRAC_CTRL_TIMEOUT, SPTPRAC_POWERON_STATUS_MASK)) { dev_err(&gmu->pdev->dev, "power on SPTPRAC fail\n"); - return -EINVAL; + gmu_fault_snapshot(device); + return -ETIMEDOUT; } return 0; @@ -812,7 +950,7 @@ int a6xx_gmu_sptprac_enable(struct adreno_device *adreno_dev) void a6xx_gmu_sptprac_disable(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); if (!gmu_core_gpmu_isenabled(device) || !adreno_has_sptprac_gdsc(adreno_dev)) @@ -838,13 +976,8 @@ void a6xx_gmu_sptprac_disable(struct adreno_device *adreno_dev) #define GX_GDSC_POWER_OFF BIT(6) #define GX_CLK_OFF BIT(7) #define is_on(val) (!(val & (GX_GDSC_POWER_OFF | GX_CLK_OFF))) -/* - * a6xx_gmu_gx_is_on() - Check if GX is on using pwr status register - * @device - Pointer to KGSLxi _device - * This check should only be performed if the keepalive bit is set or it - * can be guaranteed that the power state of the GPU will remain unchanged - */ -static bool a6xx_gmu_gx_is_on(struct kgsl_device *device) + +bool a6xx_gmu_gx_is_on(struct kgsl_device *device) { unsigned int val; @@ -875,13 +1008,14 @@ bool a6xx_gmu_sptprac_is_on(struct adreno_device *adreno_dev) * @device: Pointer to KGSL device * */ -static int a6xx_gmu_gfx_rail_on(struct kgsl_device *device) +static int a6xx_gmu_gfx_rail_on(struct adreno_device *adreno_dev) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - unsigned int perf_idx = gmu->num_gpupwrlevels - + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + u32 perf_idx = gmu->hfi.dcvs_table.gpu_level_num - pwr->default_pwrlevel - 1; - uint32_t default_opp = gmu->rpmh_votes.gx_votes[perf_idx]; + u32 default_opp = gmu->hfi.dcvs_table.gx_votes[perf_idx].vote; gmu_core_regwrite(device, A6XX_GMU_BOOT_SLUMBER_OPTION, OOB_BOOT_OPTION); @@ -929,14 +1063,15 @@ static const char *idle_level_name(int level) return ""; } -static int a6xx_gmu_wait_for_lowest_idle(struct kgsl_device *device) +static int a6xx_gmu_wait_for_lowest_idle(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); unsigned int reg, reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8; unsigned long t; uint64_t ts1, ts2, ts3; - ts1 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts1 = a6xx_read_alwayson(adreno_dev); t = jiffies + msecs_to_jiffies(GMU_IDLE_TIMEOUT); do { @@ -951,7 +1086,7 @@ static int a6xx_gmu_wait_for_lowest_idle(struct kgsl_device *device) usleep_range(10, 100); } while (!time_after(jiffies, t)); - ts2 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts2 = a6xx_read_alwayson(adreno_dev); /* Check one last time */ gmu_core_regread(device, A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE, ®); @@ -960,7 +1095,7 @@ static int a6xx_gmu_wait_for_lowest_idle(struct kgsl_device *device) if (idle_trandition_complete(gmu->idle_level, reg, reg1)) return 0; - ts3 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts3 = a6xx_read_alwayson(adreno_dev); /* Collect abort data to help with debugging */ gmu_core_regread(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, ®2); @@ -999,18 +1134,20 @@ static int a6xx_gmu_wait_for_lowest_idle(struct kgsl_device *device) } WARN_ON(1); + gmu_fault_snapshot(device); return -ETIMEDOUT; } /* Bitmask for GPU idle status check */ #define CXGXCPUBUSYIGNAHB BIT(30) -static int a6xx_gmu_wait_for_idle(struct kgsl_device *device) +static int a6xx_gmu_wait_for_idle(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); unsigned int status2; uint64_t ts1; - ts1 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts1 = a6xx_read_alwayson(adreno_dev); if (timed_poll_check(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, 0, GMU_START_TIMEOUT, CXGXCPUBUSYIGNAHB)) { gmu_core_regread(device, @@ -1019,6 +1156,7 @@ static int a6xx_gmu_wait_for_idle(struct kgsl_device *device) "GMU not idling: status2=0x%x %llx %llx\n", status2, ts1, a6xx_read_alwayson(ADRENO_DEVICE(device))); + gmu_fault_snapshot(device); return -ETIMEDOUT; } @@ -1028,9 +1166,10 @@ static int a6xx_gmu_wait_for_idle(struct kgsl_device *device) /* A6xx GMU FENCE RANGE MASK */ #define GMU_FENCE_RANGE_MASK ((0x1 << 31) | ((0xA << 2) << 18) | (0x8A0)) -static void load_gmu_version_info(struct kgsl_device *device) +static void a6xx_gmu_version_info(struct adreno_device *adreno_dev) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); /* GMU version info is at a fixed offset in the DTCM */ gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + 0xFF8, @@ -1045,64 +1184,81 @@ static void load_gmu_version_info(struct kgsl_device *device) &gmu->ver.hfi); } -/* - * a6xx_gmu_fw_start() - set up GMU and start FW - * @device: Pointer to KGSL device - * @boot_state: State of the GMU being started - */ -static int a6xx_gmu_fw_start(struct kgsl_device *device, - unsigned int boot_state) +static int a6xx_gmu_itcm_shadow(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + u32 i, *dest; + + if (gmu->itcm_shadow) + return 0; + + gmu->itcm_shadow = vzalloc(gmu->vma[GMU_ITCM].size); + if (!gmu->itcm_shadow) + return -ENOMEM; + + dest = (u32 *)gmu->itcm_shadow; + + for (i = 0; i < (gmu->vma[GMU_ITCM].size >> 2); i++) + gmu_core_regread(KGSL_DEVICE(adreno_dev), + A6XX_GMU_CM3_ITCM_START + i, dest++); + + return 0; +} + +static void a6xx_gmu_enable_lm(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 val; + + memset(adreno_dev->busy_data.throttle_cycles, 0, + sizeof(adreno_dev->busy_data.throttle_cycles)); + + if (!adreno_dev->lm_enabled) + return; + + /* + * For throttling, use the following counters for throttled cycles: + * XOCLK1: countable: 0x10 + * XOCLK2: countable: 0x16 for newer hardware / 0x15 for others + * XOCLK3: countable: 0xf for newer hardware / 0x19 for others + * + * POWER_CONTROL_SELECT_0 controls counters 0 - 3, each selector + * is 8 bits wide. + */ + + if (adreno_is_a620(adreno_dev) || adreno_is_a650(adreno_dev)) + val = (0x10 << 8) | (0x16 << 16) | (0x0f << 24); + else + val = (0x10 << 8) | (0x15 << 16) | (0x19 << 24); + + + /* Make sure not to write over XOCLK0 */ + gmu_core_regrmw(device, A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, + 0xffffff00, val); + + gmu_core_regwrite(device, A6XX_GMU_AO_SPARE_CNTL, 1); +} + +static void a6xx_gmu_register_config(struct adreno_device *adreno_dev) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); - uint32_t gmu_log_info; - int ret; - unsigned int chipid = 0; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 gmu_log_info, chipid = 0; /* Vote veto for FAL10 feature if supported*/ if (a6xx_core->veto_fal10) gmu_core_regwrite(device, A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 0x1); - switch (boot_state) { - case GMU_COLD_BOOT: - /* Turn on TCM retention */ - gmu_core_regwrite(device, A6XX_GMU_GENERAL_7, 1); - - if (!test_and_set_bit(GMU_BOOT_INIT_DONE, - &device->gmu_core.flags)) - ret = _load_gmu_rpmh_ucode(device); - else - ret = a6xx_rpmh_power_on_gpu(device); - if (ret) - return ret; - - if (gmu->load_mode == TCM_BOOT) { - /* Load GMU image via AHB bus */ - ret = load_gmu_fw(device); - if (ret) - return ret; - } else { - dev_err(&gmu->pdev->dev, "Unsupported GMU load mode %d\n", - gmu->load_mode); - return -EINVAL; - } - break; - case GMU_WARM_BOOT: - ret = a6xx_rpmh_power_on_gpu(device); - if (ret) - return ret; - break; - default: - break; - } + /* Turn on TCM retention */ + gmu_core_regwrite(device, A6XX_GMU_GENERAL_7, 1); /* Clear init result to make sure we are getting fresh value */ gmu_core_regwrite(device, A6XX_GMU_CM3_FW_INIT_RESULT, 0); - gmu_core_regwrite(device, A6XX_GMU_CM3_BOOT_CONFIG, gmu->load_mode); + gmu_core_regwrite(device, A6XX_GMU_CM3_BOOT_CONFIG, 0x2); gmu_core_regwrite(device, A6XX_GMU_HFI_QTBL_ADDR, - gmu->hfi_mem->gmuaddr); + gmu->hfi.hfi_mem->gmuaddr); gmu_core_regwrite(device, A6XX_GMU_HFI_QTBL_INFO, 1); gmu_core_regwrite(device, A6XX_GMU_AHB_FENCE_RANGE_0, @@ -1120,7 +1276,7 @@ static int a6xx_gmu_fw_start(struct kgsl_device *device, * not get cleared if the gdsc was not reset. So clear it before * attempting GMU boot. */ - if (!adreno_is_a630(ADRENO_DEVICE(device))) + if (!adreno_is_a630(adreno_dev)) kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0); /* Set the log wptr index */ @@ -1149,49 +1305,135 @@ static int a6xx_gmu_fw_start(struct kgsl_device *device, gmu_log_info); /* Configure power control and bring the GMU out of reset */ - a6xx_gmu_power_config(device); + a6xx_gmu_power_config(adreno_dev); - /* Populate the GMU version info before GMU boots */ - load_gmu_version_info(device); + a6xx_gmu_enable_lm(adreno_dev); +} - /* Clear any previously set cm3 fault */ - atomic_set(&gmu->cm3_fault, 0); +struct gmu_memdesc *reserve_gmu_kernel_block(struct a6xx_gmu_device *gmu, + u32 addr, u32 size, u32 vma_id) +{ + int ret; + struct gmu_memdesc *md; + struct gmu_vma_entry *vma = &gmu->vma[vma_id]; - ret = a6xx_gmu_start(device); + if (gmu->global_entries == ARRAY_SIZE(gmu->gmu_globals)) + return ERR_PTR(-ENOMEM); + + md = &gmu->gmu_globals[gmu->global_entries]; + + md->size = PAGE_ALIGN(size); + + md->hostptr = dma_alloc_attrs(&gmu->pdev->dev, (size_t)md->size, + &md->physaddr, GFP_KERNEL, 0); + + if (md->hostptr == NULL) + return ERR_PTR(-ENOMEM); + + memset(md->hostptr, 0x0, size); + + if (!addr) + addr = vma->next_va; + + md->gmuaddr = addr; + + ret = iommu_map(get_gmu_domain(gmu, md), addr, md->physaddr, md->size, + IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV); + if (ret) { + dev_err(&gmu->pdev->dev, + "Unable to map GMU kernel block: addr:0x%08x size:0x%x :%d\n", + md->gmuaddr, md->size, ret); + dma_free_attrs(&gmu->pdev->dev, (size_t)size, + (void *)md->hostptr, md->physaddr, 0); + memset(md, 0, sizeof(*md)); + return ERR_PTR(ret); + } + + vma->next_va = md->gmuaddr + md->size; + + gmu->global_entries++; + + return md; +} + +static int reserve_entire_vma(struct a6xx_gmu_device *gmu, u32 vma_id) +{ + struct gmu_memdesc *md; + u32 start = gmu->vma[vma_id].start, size = gmu->vma[vma_id].size; + + md = find_gmu_memdesc(gmu, start, size); + if (md) + return 0; + + md = reserve_gmu_kernel_block(gmu, start, size, vma_id); + + return PTR_ERR_OR_ZERO(md); +} + +static int a6xx_gmu_cache_finalize(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct gmu_memdesc *md; + int ret; + + /* Preallocations were made so no need to request all this memory */ + if (gmu->preallocations) + return 0; + + ret = reserve_entire_vma(gmu, GMU_ICACHE); if (ret) return ret; - if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) { - ret = a6xx_gmu_gfx_rail_on(device); - if (ret) { - a6xx_gmu_oob_clear(device, oob_boot_slumber); - return ret; - } - } - - if (gmu->idle_level < GPU_HW_SPTP_PC) { - ret = a6xx_gmu_sptprac_enable(adreno_dev); + if (!adreno_is_a650_family(adreno_dev)) { + ret = reserve_entire_vma(gmu, GMU_DCACHE); if (ret) return ret; } - ret = a6xx_gmu_hfi_start(device); - if (ret) - return ret; + md = reserve_gmu_kernel_block(gmu, 0, SZ_4K, GMU_NONCACHED_KERNEL); + if (IS_ERR(md)) + return PTR_ERR(md); - /* Make sure the write to start HFI happens before sending a message */ - wmb(); - return ret; + gmu->preallocations = true; + + return 0; } -/* - * a6xx_gmu_load_firmware() - Load the ucode into the GPMU RAM & PDC/RSC - * @device: Pointer to KGSL device - */ -static int a6xx_gmu_load_firmware(struct kgsl_device *device) +static int a6xx_gmu_process_prealloc(struct a6xx_gmu_device *gmu, + struct gmu_block_header *blk) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct gmu_memdesc *md; + + int id = find_vma_block(gmu, blk->addr, blk->value); + + if (id < 0) { + dev_err(&gmu->pdev->dev, + "Invalid prealloc block addr: 0x%x value:%d\n", + blk->addr, blk->value); + return id; + } + + /* Nothing to do for TCM blocks or user uncached */ + if (id == GMU_ITCM || id == GMU_DTCM || id == GMU_NONCACHED_USER) + return 0; + + /* Check if the block is already allocated */ + md = find_gmu_memdesc(gmu, blk->addr, blk->value); + if (md != NULL) + return 0; + + md = reserve_gmu_kernel_block(gmu, blk->addr, blk->value, id); + if (IS_ERR(md)) + return PTR_ERR(md); + + gmu->preallocations = true; + + return 0; +} + +static int a6xx_gmu_parse_fw(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); struct gmu_block_header *blk; int ret, offset = 0; @@ -1232,14 +1474,54 @@ static int a6xx_gmu_load_firmware(struct kgsl_device *device) if (blk->type == GMU_BLK_TYPE_PREALLOC_REQ || blk->type == GMU_BLK_TYPE_PREALLOC_PERSIST_REQ) - ret = gmu_prealloc_req(device, blk); + ret = a6xx_gmu_process_prealloc(gmu, blk); if (ret) return ret; } + return 0; +} + +static int a6xx_gmu_memory_init(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + /* Allocates & maps GMU crash dump memory */ + if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) { + if (IS_ERR_OR_NULL(gmu->dump_mem)) + gmu->dump_mem = reserve_gmu_kernel_block(gmu, 0, SZ_16K, + GMU_NONCACHED_KERNEL); + if (IS_ERR(gmu->dump_mem)) + return PTR_ERR(gmu->dump_mem); + } + + /* GMU master log */ + if (IS_ERR_OR_NULL(gmu->gmu_log)) + gmu->gmu_log = reserve_gmu_kernel_block(gmu, 0, SZ_4K, + GMU_NONCACHED_KERNEL); + + return PTR_ERR_OR_ZERO(gmu->gmu_log); +} + +static int a6xx_gmu_init(struct adreno_device *adreno_dev) +{ + int ret; + + ret = a6xx_gmu_parse_fw(adreno_dev); + if (ret) + return ret; + /* Request any other cache ranges that might be required */ - return gmu_cache_finalize(device); + ret = a6xx_gmu_cache_finalize(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gmu_memory_init(adreno_dev); + if (ret) + return ret; + + return a6xx_hfi_init(adreno_dev); } #define A6XX_VBIF_XIN_HALT_CTRL1_ACKS (BIT(0) | BIT(1) | BIT(2) | BIT(3)) @@ -1278,11 +1560,11 @@ static void do_gbif_halt(struct kgsl_device *device, u32 reg, u32 ack_reg, dev_err(device->dev, "%s GBIF halt timed out\n", client); } -static int a6xx_gmu_suspend(struct kgsl_device *device) +static void a6xx_gmu_pwrctrl_suspend(struct adreno_device *adreno_dev) { int ret = 0; - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); /* If SPTP_RAC is on, turn off SPTP_RAC HS */ a6xx_gmu_sptprac_disable(adreno_dev); @@ -1290,7 +1572,7 @@ static int a6xx_gmu_suspend(struct kgsl_device *device) /* Disconnect GPU from BUS is not needed if CX GDSC goes off later */ /* Check no outstanding RPMh voting */ - a6xx_complete_rpmh_votes(device); + a6xx_complete_rpmh_votes(adreno_dev); /* Clear the WRITEDROPPED fields and set fence to allow mode */ gmu_core_regwrite(device, A6XX_GMU_AHB_FENCE_STATUS_CLR, 0x7); @@ -1302,19 +1584,16 @@ static int a6xx_gmu_suspend(struct kgsl_device *device) gmu_core_regwrite(device, A6XX_GMU_CM3_SYSRESET, 1); if (adreno_has_gbif(adreno_dev)) { - struct adreno_gpudev *gpudev = - ADRENO_GPU_DEVICE(adreno_dev); - /* Halt GX traffic */ if (a6xx_gmu_gx_is_on(device)) do_gbif_halt(device, A6XX_RBBM_GBIF_HALT, A6XX_RBBM_GBIF_HALT_ACK, - gpudev->gbif_gx_halt_mask, + A6XX_GBIF_GX_HALT_MASK, "GX"); /* Halt CX traffic */ do_gbif_halt(device, A6XX_GBIF_HALT, A6XX_GBIF_HALT_ACK, - gpudev->gbif_arb_halt_mask, "CX"); + A6XX_GBIF_ARB_HALT_MASK, "CX"); } if (a6xx_gmu_gx_is_on(device)) @@ -1350,21 +1629,20 @@ static int a6xx_gmu_suspend(struct kgsl_device *device) "gx is stuck on\n"); } } - - return ret; } /* * a6xx_gmu_notify_slumber() - initiate request to GMU to prepare to slumber * @device: Pointer to KGSL device */ -static int a6xx_gmu_notify_slumber(struct kgsl_device *device) +static int a6xx_gmu_notify_slumber(struct adreno_device *adreno_dev) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); int bus_level = pwr->pwrlevels[pwr->default_pwrlevel].bus_freq; - int perf_idx = gmu->num_gpupwrlevels - pwr->default_pwrlevel - 1; + int perf_idx = gmu->hfi.dcvs_table.gpu_level_num - + pwr->default_pwrlevel - 1; int ret, state; /* Disable the power counter so that the GMU is not busy */ @@ -1380,7 +1658,8 @@ static int a6xx_gmu_notify_slumber(struct kgsl_device *device) .bw = bus_level, }; - ret = hfi_send_req(gmu, H2F_MSG_PREPARE_SLUMBER, &req); + ret = a6xx_hfi_send_req(adreno_dev, + H2F_MSG_PREPARE_SLUMBER, &req); goto out; } @@ -1399,7 +1678,7 @@ static int a6xx_gmu_notify_slumber(struct kgsl_device *device) dev_err(&gmu->pdev->dev, "Failed to prepare for slumber: 0x%x\n", state); - ret = -EINVAL; + ret = -ETIMEDOUT; } } @@ -1409,87 +1688,100 @@ out: return ret; } -/* - * a6xx_rpmh_gpu_pwrctrl() - GPU power control via RPMh/GMU interface - * @adreno_dev: Pointer to adreno device - * @mode: requested power mode - * @arg1: first argument for mode control - * @arg2: second argument for mode control - */ -static int a6xx_gmu_rpmh_gpu_pwrctrl(struct kgsl_device *device, - unsigned int mode, unsigned int arg1, unsigned int arg2) +static void a6xx_gmu_suspend(struct adreno_device *adreno_dev) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - int ret; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - switch (mode) { - case GMU_FW_START: - ret = a6xx_gmu_fw_start(device, arg1); - break; - case GMU_SUSPEND: - ret = a6xx_gmu_suspend(device); - break; - case GMU_FW_STOP: - if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) - a6xx_gmu_oob_clear(device, oob_boot_slumber); - ret = a6xx_rpmh_power_off_gpu(device); - break; - case GMU_DCVS_NOHFI: - ret = a6xx_gmu_dcvs_nohfi(device, arg1, arg2); - break; - case GMU_NOTIFY_SLUMBER: - ret = a6xx_gmu_notify_slumber(device); - break; - default: - dev_err(&gmu->pdev->dev, - "unsupported GMU power ctrl mode:%d\n", mode); - ret = -EINVAL; - break; + a6xx_gmu_irq_disable(adreno_dev); + + a6xx_gmu_pwrctrl_suspend(adreno_dev); + + clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CX_GDSC)) + regulator_set_mode(gmu->cx_gdsc, REGULATOR_MODE_IDLE); + + if (!kgsl_regulator_disable_wait(gmu->cx_gdsc, 5000)) + dev_err(&gmu->pdev->dev, "GMU CX gdsc off timeout\n"); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CX_GDSC)) + regulator_set_mode(gmu->cx_gdsc, REGULATOR_MODE_NORMAL); + + dev_err(&gmu->pdev->dev, "Suspended GMU\n"); + + device->state = KGSL_STATE_NONE; +} + +static int a6xx_gmu_dcvs_set(struct adreno_device *adreno_dev, + int gpu_pwrlevel, int bus_level) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct hfi_dcvstable_cmd *table = &gmu->hfi.dcvs_table; + struct hfi_gx_bw_perf_vote_cmd req = { + .ack_type = DCVS_ACK_BLOCK, + .freq = INVALID_DCVS_IDX, + .bw = INVALID_DCVS_IDX, + }; + int ret = 0; + + if (!test_bit(GMU_PRIV_HFI_STARTED, &gmu->flags)) + return 0; + + /* Do not set to XO and lower GPU clock vote from GMU */ + if ((gpu_pwrlevel != INVALID_DCVS_IDX) && + (gpu_pwrlevel >= table->gpu_level_num - 1)) + return -EINVAL; + + if (gpu_pwrlevel < table->gpu_level_num - 1) + req.freq = table->gpu_level_num - gpu_pwrlevel - 1; + + if (bus_level < pwr->ddr_table_count && bus_level > 0) + req.bw = bus_level; + + /* GMU will vote for slumber levels through the sleep sequence */ + if ((req.freq == INVALID_DCVS_IDX) && + (req.bw == INVALID_DCVS_IDX)) { + return 0; + } + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) + ret = a6xx_gmu_dcvs_nohfi(device, req.freq, req.bw); + else + ret = a6xx_hfi_send_req(adreno_dev, H2F_MSG_GX_BW_PERF_VOTE, + &req); + + if (ret) { + dev_err_ratelimited(&gmu->pdev->dev, + "Failed to set GPU perf idx %d, bw idx %d\n", + req.freq, req.bw); + + /* + * If this was a dcvs request along side an active gpu, request + * dispatcher based reset and recovery. + */ + if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) { + adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT | + ADRENO_GMU_FAULT_SKIP_SNAPSHOT); + adreno_dispatcher_schedule(device); + } } return ret; } -static void a6xx_gmu_enable_lm(struct kgsl_device *device) +static int a6xx_gmu_clock_set(struct adreno_device *adreno_dev, u32 pwrlevel) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - u32 val; - - memset(adreno_dev->busy_data.throttle_cycles, 0, - sizeof(adreno_dev->busy_data.throttle_cycles)); - - if (!adreno_dev->lm_enabled) - return; - - /* - * For throttling, use the following counters for throttled cycles: - * XOCLK1: countable: 0x10 - * XOCLK2: countable: 0x16 for newer hardware / 0x15 for others - * XOCLK3: countable: 0xf for newer hardware / 0x19 for others - * - * POWER_CONTROL_SELECT_0 controls counters 0 - 3, each selector - * is 8 bits wide. - */ - - if (adreno_is_a620(adreno_dev) || adreno_is_a650(adreno_dev)) - val = (0x10 << 8) | (0x16 << 16) | (0x0f << 24); - else - val = (0x10 << 8) | (0x15 << 16) | (0x19 << 24); - - - /* Make sure not to write over XOCLK0 */ - gmu_core_regrmw(device, A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, - 0xffffff00, val); - - gmu_core_regwrite(device, A6XX_GMU_AO_SPARE_CNTL, 1); + return a6xx_gmu_dcvs_set(adreno_dev, pwrlevel, INVALID_DCVS_IDX); } static int a6xx_gmu_ifpc_store(struct kgsl_device *device, unsigned int val) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); unsigned int requested_idle_level; if (!ADRENO_FEATURE(adreno_dev, ADRENO_IFPC)) @@ -1508,248 +1800,46 @@ static int a6xx_gmu_ifpc_store(struct kgsl_device *device, requested_idle_level = GPU_HW_ACTIVE; } - mutex_lock(&device->mutex); - /* Power down the GPU before changing the idle level */ - kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); - gmu->idle_level = requested_idle_level; - kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); - - mutex_unlock(&device->mutex); - - return 0; + return adreno_power_cycle_u32(adreno_dev, &gmu->idle_level, + requested_idle_level); } static unsigned int a6xx_gmu_ifpc_show(struct kgsl_device *device) { - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(ADRENO_DEVICE(device)); return gmu->idle_level >= GPU_HW_IFPC; } -struct gmu_mem_type_desc { - struct gmu_memdesc *memdesc; - uint32_t type; -}; - -static size_t a6xx_snapshot_gmu_mem(struct kgsl_device *device, - u8 *buf, size_t remain, void *priv) +/* Send an NMI to the GMU */ +static void a6xx_gmu_send_nmi(struct adreno_device *adreno_dev) { - struct kgsl_snapshot_gmu_mem *mem_hdr = - (struct kgsl_snapshot_gmu_mem *)buf; - unsigned int *data = (unsigned int *) - (buf + sizeof(*mem_hdr)); - struct gmu_mem_type_desc *desc = priv; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 val; - if (priv == NULL) - return 0; + /* Mask so there's no interrupt caused by NMI */ + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF); - if (remain < desc->memdesc->size + sizeof(*mem_hdr)) { - dev_err(device->dev, - "snapshot: Not enough memory for the gmu section %d\n", - desc->type); - return 0; - } + /* Make sure the interrupt is masked before causing it */ + wmb(); + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) + gmu_core_regwrite(device, + A6XX_GMU_NMI_CONTROL_STATUS, 0); - memset(mem_hdr, 0, sizeof(*mem_hdr)); - mem_hdr->type = desc->type; - mem_hdr->hostaddr = (uintptr_t)desc->memdesc->hostptr; - mem_hdr->gmuaddr = desc->memdesc->gmuaddr; - mem_hdr->gpuaddr = 0; + /* This will cause the GMU to save it's internal state to ddr */ + gmu_core_regread(device, A6XX_GMU_CM3_CFG, &val); + val |= BIT(9); + gmu_core_regwrite(device, A6XX_GMU_CM3_CFG, val); - /* Just copy the ringbuffer, there are no active IBs */ - memcpy(data, desc->memdesc->hostptr, desc->memdesc->size); - - return desc->memdesc->size + sizeof(*mem_hdr); -} - -struct a6xx_tcm_data { - enum gmu_mem_type type; - u32 start; - u32 last; -}; - -static size_t a6xx_snapshot_gmu_tcm(struct kgsl_device *device, - u8 *buf, size_t remain, void *priv) -{ - struct kgsl_snapshot_gmu_mem *mem_hdr = - (struct kgsl_snapshot_gmu_mem *)buf; - unsigned int *data = (unsigned int *)(buf + sizeof(*mem_hdr)); - unsigned int i, bytes; - struct a6xx_tcm_data *tcm = priv; - - bytes = (tcm->last - tcm->start + 1) << 2; - - if (remain < bytes + sizeof(*mem_hdr)) { - SNAPSHOT_ERR_NOMEM(device, "GMU Memory"); - return 0; - } - - mem_hdr->type = SNAPSHOT_GMU_MEM_BIN_BLOCK; - mem_hdr->hostaddr = 0; - mem_hdr->gmuaddr = gmu_get_memtype_base(KGSL_GMU_DEVICE(device), - tcm->type); - mem_hdr->gpuaddr = 0; - - for (i = tcm->start; i <= tcm->last; i++) - kgsl_regread(device, i, data++); - - return bytes + sizeof(*mem_hdr); -} - -static void a6xx_gmu_snapshot_memories(struct kgsl_device *device, - struct kgsl_snapshot *snapshot) -{ - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct gmu_mem_type_desc desc; - struct gmu_memdesc *md; - int i; - - for (i = 0; i < ARRAY_SIZE(gmu->kmem_entries); i++) { - if (!test_bit(i, &gmu->kmem_bitmap)) - continue; - - md = &gmu->kmem_entries[i]; - if (!md->size) - continue; - - desc.memdesc = md; - if (md == gmu->hfi_mem) - desc.type = SNAPSHOT_GMU_MEM_HFI; - else if (md == gmu->gmu_log) - desc.type = SNAPSHOT_GMU_MEM_LOG; - else if (md == gmu->dump_mem) - desc.type = SNAPSHOT_GMU_MEM_DEBUG; - else - desc.type = SNAPSHOT_GMU_MEM_BIN_BLOCK; - - if (md->mem_type == GMU_ITCM) { - struct a6xx_tcm_data tcm = { - .type = md->mem_type, - .start = a6xx_gmu_tcm_registers[0], - .last = a6xx_gmu_tcm_registers[1], - }; - - kgsl_snapshot_add_section(device, - KGSL_SNAPSHOT_SECTION_GMU_MEMORY, - snapshot, a6xx_snapshot_gmu_tcm, &tcm); - } else if (md->mem_type == GMU_DTCM) { - struct a6xx_tcm_data tcm = { - .type = md->mem_type, - .start = a6xx_gmu_tcm_registers[2], - .last = a6xx_gmu_tcm_registers[3], - }; - - kgsl_snapshot_add_section(device, - KGSL_SNAPSHOT_SECTION_GMU_MEMORY, - snapshot, a6xx_snapshot_gmu_tcm, &tcm); - } else { - kgsl_snapshot_add_section(device, - KGSL_SNAPSHOT_SECTION_GMU_MEMORY, - snapshot, a6xx_snapshot_gmu_mem, &desc); - } - } -} - -struct kgsl_snapshot_gmu_version { - uint32_t type; - uint32_t value; -}; - -static size_t a6xx_snapshot_gmu_version(struct kgsl_device *device, - u8 *buf, size_t remain, void *priv) -{ - struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; - uint32_t *data = (uint32_t *) (buf + sizeof(*header)); - struct kgsl_snapshot_gmu_version *ver = priv; - - if (remain < DEBUG_SECTION_SZ(1)) { - SNAPSHOT_ERR_NOMEM(device, "GMU Version"); - return 0; - } - - header->type = ver->type; - header->size = 1; - - *data = ver->value; - - return DEBUG_SECTION_SZ(1); -} - -static void a6xx_gmu_snapshot_versions(struct kgsl_device *device, - struct kgsl_snapshot *snapshot) -{ - int i; - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct kgsl_snapshot_gmu_version gmu_vers[] = { - { .type = SNAPSHOT_DEBUG_GMU_CORE_VERSION, - .value = gmu->ver.core, }, - { .type = SNAPSHOT_DEBUG_GMU_CORE_DEV_VERSION, - .value = gmu->ver.core_dev, }, - { .type = SNAPSHOT_DEBUG_GMU_PWR_VERSION, - .value = gmu->ver.pwr, }, - { .type = SNAPSHOT_DEBUG_GMU_PWR_DEV_VERSION, - .value = gmu->ver.pwr_dev, }, - { .type = SNAPSHOT_DEBUG_GMU_HFI_VERSION, - .value = gmu->ver.hfi, }, - }; - - for (i = 0; i < ARRAY_SIZE(gmu_vers); i++) - kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, - snapshot, a6xx_snapshot_gmu_version, - &gmu_vers[i]); -} - -/* - * a6xx_gmu_snapshot() - A6XX GMU snapshot function - * @device: Device being snapshotted - * @snapshot: Pointer to the snapshot instance - * - * This is where all of the A6XX GMU specific bits and pieces are grabbed - * into the snapshot memory - */ -static void a6xx_gmu_snapshot(struct kgsl_device *device, - struct kgsl_snapshot *snapshot) -{ - unsigned int val; - - a6xx_gmu_snapshot_versions(device, snapshot); - - a6xx_gmu_snapshot_memories(device, snapshot); - - /* Snapshot tcms as registers for legacy targets */ - if (adreno_is_a630(ADRENO_DEVICE(device)) || - adreno_is_a615_family(ADRENO_DEVICE(device))) - adreno_snapshot_registers(device, snapshot, - a6xx_gmu_tcm_registers, - ARRAY_SIZE(a6xx_gmu_tcm_registers) / 2); - - adreno_snapshot_registers(device, snapshot, a6xx_gmu_registers, - ARRAY_SIZE(a6xx_gmu_registers) / 2); - - /* Snapshot A660 specific GMU registers */ - if (adreno_is_a660(ADRENO_DEVICE(device))) - adreno_snapshot_registers(device, snapshot, a660_gmu_registers, - ARRAY_SIZE(a660_gmu_registers) / 2); - - if (a6xx_gmu_gx_is_on(device)) { - /* Set fence to ALLOW mode so registers can be read */ - kgsl_regwrite(device, A6XX_GMU_AO_AHB_FENCE_CTRL, 0); - /* Make sure the previous write posted before reading */ - wmb(); - kgsl_regread(device, A6XX_GMU_AO_AHB_FENCE_CTRL, &val); - - dev_err(device->dev, "set FENCE to ALLOW mode:%x\n", val); - adreno_snapshot_registers(device, snapshot, - a6xx_gmu_gx_registers, - ARRAY_SIZE(a6xx_gmu_gx_registers) / 2); - } + /* Make sure the NMI is invoked before we proceed*/ + wmb(); } static void a6xx_gmu_cooperative_reset(struct kgsl_device *device) { - - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); unsigned int result; gmu_core_regwrite(device, A6XX_GMU_CX_GMU_WDOG_CTRL, 0); @@ -1770,7 +1860,7 @@ static void a6xx_gmu_cooperative_reset(struct kgsl_device *device) * If we dont get a snapshot ready from GMU, trigger NMI * and if we still timeout then we just continue with reset. */ - adreno_gmu_send_nmi(ADRENO_DEVICE(device)); + a6xx_gmu_send_nmi(adreno_dev); udelay(200); gmu_core_regread(device, A6XX_GMU_CM3_FW_INIT_RESULT, &result); if ((result & 0x800) != 0x800) @@ -1782,7 +1872,10 @@ static int a6xx_gmu_wait_for_active_transition( struct kgsl_device *device) { unsigned int reg, num_retries; - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(ADRENO_DEVICE(device)); + + if (!gmu_core_isenabled(device)) + return 0; gmu_core_regread(device, A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE, ®); @@ -1812,30 +1905,1411 @@ static bool a6xx_gmu_scales_bandwidth(struct kgsl_device *device) return (ADRENO_GPUREV(adreno_dev) >= ADRENO_REV_A640); } -static u64 a6xx_gmu_read_alwayson(struct kgsl_device *device) +static irqreturn_t a6xx_gmu_irq_handler(int irq, void *data) { - return a6xx_read_alwayson(ADRENO_DEVICE(device)); + struct kgsl_device *device = data; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + unsigned int mask, status = 0; + + gmu_core_regread(device, A6XX_GMU_AO_HOST_INTERRUPT_STATUS, &status); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, status); + + /* Ignore GMU_INT_RSCC_COMP and GMU_INT_DBD WAKEUP interrupts */ + if (status & GMU_INT_WDOG_BITE) { + /* Temporarily mask the watchdog interrupt to prevent a storm */ + gmu_core_regread(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK, + &mask); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK, + (mask | GMU_INT_WDOG_BITE)); + + /* make sure we're reading the latest cm3_fault */ + smp_rmb(); + + /* + * We should not send NMI if there was a CM3 fault reported + * because we don't want to overwrite the critical CM3 state + * captured by gmu before it sent the CM3 fault interrupt. + */ + if (!atomic_read(&gmu->cm3_fault)) + a6xx_gmu_send_nmi(adreno_dev); + + /* + * There is sufficient delay for the GMU to have finished + * handling the NMI before snapshot is taken, as the fault + * worker is scheduled below. + */ + + dev_err_ratelimited(&gmu->pdev->dev, + "GMU watchdog expired interrupt received\n"); + } + if (status & GMU_INT_HOST_AHB_BUS_ERR) + dev_err_ratelimited(&gmu->pdev->dev, + "AHB bus error interrupt received\n"); + if (status & GMU_INT_FENCE_ERR) { + unsigned int fence_status; + + gmu_core_regread(device, A6XX_GMU_AHB_FENCE_STATUS, + &fence_status); + dev_err_ratelimited(&gmu->pdev->dev, + "FENCE error interrupt received %x\n", fence_status); + } + + if (status & ~GMU_AO_INT_MASK) + dev_err_ratelimited(&gmu->pdev->dev, + "Unhandled GMU interrupts 0x%lx\n", + status & ~GMU_AO_INT_MASK); + + return IRQ_HANDLED; } -struct gmu_dev_ops adreno_a6xx_gmudev = { - .load_firmware = a6xx_gmu_load_firmware, +static void a6xx_gmu_nmi(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + /* No need to nmi if it was a gpu fault */ + if (!device->gmu_fault) + return; + + /* make sure we're reading the latest cm3_fault */ + smp_rmb(); + + /* + * We should not send NMI if there was a CM3 fault reported because we + * don't want to overwrite the critical CM3 state captured by gmu before + * it sent the CM3 fault interrupt. + */ + if (!atomic_read(&gmu->cm3_fault)) { + a6xx_gmu_send_nmi(adreno_dev); + + /* Wait for the NMI to be handled */ + udelay(100); + } +} + +void a6xx_gmu_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + a6xx_gmu_nmi(adreno_dev); + + a6xx_gmu_device_snapshot(device, snapshot); + + a6xx_snapshot(adreno_dev, snapshot); + + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, + 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, + HFI_IRQ_MASK); + +} + +static int a6xx_gmu_aop_send_acd_state(struct mbox_chan *channel, bool flag) +{ + char msg_buf[33]; + struct { + u32 len; + void *msg; + } msg; + + if (IS_ERR_OR_NULL(channel)) + return 0; + + msg.len = scnprintf(msg_buf, sizeof(msg_buf), + "{class: gpu, res: acd, value: %d}", flag); + msg.msg = msg_buf; + + return mbox_send_message(channel, &msg); +} + +static int a6xx_gmu_enable_gdsc(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + + ret = regulator_enable(gmu->cx_gdsc); + if (ret) + dev_err(&gmu->pdev->dev, + "Failed to enable GMU CX gdsc, error %d\n", ret); + + return ret; +} + +static int a6xx_gmu_clk_set_rate(struct a6xx_gmu_device *gmu, const char *id, + unsigned long rate) +{ + struct clk *clk; + + clk = kgsl_of_clk_by_name(gmu->clks, gmu->num_clks, id); + if (!clk) + return -ENODEV; + + return clk_set_rate(clk, rate); +} + +static int a6xx_gmu_enable_clks(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = a6xx_gmu_clk_set_rate(gmu, "gmu_clk", GMU_FREQUENCY); + if (ret) { + dev_err(&gmu->pdev->dev, "Unable to set the GMU clock\n"); + return ret; + } + + ret = a6xx_gmu_clk_set_rate(gmu, "hub_clk", 150000000); + if (ret && ret != ENODEV) { + dev_err(&gmu->pdev->dev, "Unable to set the HUB clock\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(gmu->num_clks, gmu->clks); + if (ret) { + dev_err(&gmu->pdev->dev, "Cannot enable GMU clocks\n"); + return ret; + } + + device->state = KGSL_STATE_AWARE; + + return 0; +} + +static int a6xx_gmu_first_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int level, ret; + + ret = a6xx_gmu_aop_send_acd_state(gmu->mailbox.channel, + adreno_dev->acd_enabled); + if (ret) { + dev_err(&gmu->pdev->dev, + "AOP mbox send message failed: %d\n", ret); + return ret; + } + + ret = a6xx_gmu_enable_gdsc(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gmu_enable_clks(adreno_dev); + if (ret) + goto gdsc_off; + + ret = a6xx_gmu_load_fw(adreno_dev); + if (ret) + goto clks_gdsc_off; + + ret = a6xx_gmu_itcm_shadow(adreno_dev); + if (ret) + goto clks_gdsc_off; + + a6xx_gmu_register_config(adreno_dev); + + a6xx_gmu_version_info(adreno_dev); + + a6xx_gmu_irq_enable(adreno_dev); + + /* Vote for minimal DDR BW for GMU to init */ + level = pwr->pwrlevels[pwr->default_pwrlevel].bus_min; + icc_set_bw(pwr->icc_path, 0, kBps_to_icc(pwr->ddr_table[level])); + + ret = a6xx_gmu_device_start(adreno_dev); + if (ret) + goto err; + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) { + ret = a6xx_gmu_gfx_rail_on(adreno_dev); + if (ret) { + a6xx_gmu_oob_clear(device, oob_boot_slumber); + goto err; + } + } + + if (gmu->idle_level < GPU_HW_SPTP_PC) { + ret = a6xx_gmu_sptprac_enable(adreno_dev); + if (ret) + goto err; + } + + ret = a6xx_gmu_hfi_start(adreno_dev); + if (ret) + goto err; + + ret = a6xx_hfi_start(adreno_dev); + if (ret) + goto err; + + icc_set_bw(pwr->icc_path, 0, 0); + + device->gmu_fault = false; + + return 0; + +err: + if (device->gmu_fault) { + a6xx_gmu_suspend(adreno_dev); + return ret; + } + +clks_gdsc_off: + clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks); + +gdsc_off: + /* Pool to make sure that the CX is off */ + if (!kgsl_regulator_disable_wait(gmu->cx_gdsc, 5000)) + dev_err(&gmu->pdev->dev, "GMU CX gdsc off timeout\n"); + + return ret; +} + +static int a6xx_gmu_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret = 0; + + ret = a6xx_gmu_enable_gdsc(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gmu_enable_clks(adreno_dev); + if (ret) + goto gdsc_off; + + ret = a6xx_rscc_wakeup_sequence(adreno_dev); + if (ret) + goto clks_gdsc_off; + + ret = a6xx_gmu_load_fw(adreno_dev); + if (ret) + goto clks_gdsc_off; + + a6xx_gmu_register_config(adreno_dev); + + a6xx_gmu_irq_enable(adreno_dev); + + ret = a6xx_gmu_device_start(adreno_dev); + if (ret) + goto err; + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) { + ret = a6xx_gmu_gfx_rail_on(adreno_dev); + if (ret) { + a6xx_gmu_oob_clear(device, oob_boot_slumber); + goto err; + } + } + + if (gmu->idle_level < GPU_HW_SPTP_PC) { + ret = a6xx_gmu_sptprac_enable(adreno_dev); + if (ret) + goto err; + } + + ret = a6xx_gmu_hfi_start(adreno_dev); + if (ret) + goto err; + + ret = a6xx_hfi_start(adreno_dev); + if (ret) + goto err; + + device->gmu_fault = false; + + return 0; + +err: + if (device->gmu_fault) { + a6xx_gmu_suspend(adreno_dev); + return ret; + } + +clks_gdsc_off: + clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks); + +gdsc_off: + /* Pool to make sure that the CX is off */ + if (!kgsl_regulator_disable_wait(gmu->cx_gdsc, 5000)) + dev_err(&gmu->pdev->dev, "GMU CX gdsc off timeout\n"); + + return ret; +} + +static void set_acd(struct adreno_device *adreno_dev, void *priv) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + + adreno_dev->acd_enabled = *((bool *)priv); + + ret = a6xx_gmu_aop_send_acd_state(gmu->mailbox.channel, + adreno_dev->acd_enabled); + if (ret) + dev_err(&gmu->pdev->dev, + "AOP mbox send message failed: %d\n", ret); +} + +static int a6xx_gmu_acd_set(struct kgsl_device *device, bool val) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + if (IS_ERR_OR_NULL(gmu->mailbox.channel)) + return -EINVAL; + + /* Don't do any unneeded work if ACD is already in the correct state */ + if (adreno_dev->acd_enabled == val) + return 0; + + /* Power cycle the GPU for changes to take effect */ + return adreno_power_cycle(adreno_dev, set_acd, &val); +} + +static struct gmu_dev_ops a6xx_gmudev = { .oob_set = a6xx_gmu_oob_set, .oob_clear = a6xx_gmu_oob_clear, - .irq_enable = a6xx_gmu_irq_enable, - .irq_disable = a6xx_gmu_irq_disable, - .hfi_start_msg = a6xx_gmu_hfi_start_msg, - .enable_lm = a6xx_gmu_enable_lm, - .rpmh_gpu_pwrctrl = a6xx_gmu_rpmh_gpu_pwrctrl, .gx_is_on = a6xx_gmu_gx_is_on, - .wait_for_lowest_idle = a6xx_gmu_wait_for_lowest_idle, - .wait_for_gmu_idle = a6xx_gmu_wait_for_idle, .ifpc_store = a6xx_gmu_ifpc_store, .ifpc_show = a6xx_gmu_ifpc_show, - .snapshot = a6xx_gmu_snapshot, .cooperative_reset = a6xx_gmu_cooperative_reset, .wait_for_active_transition = a6xx_gmu_wait_for_active_transition, - .read_alwayson = a6xx_gmu_read_alwayson, - .gmu2host_intr_mask = HFI_IRQ_MASK, - .gmu_ao_intr_mask = GMU_AO_INT_MASK, .scales_bandwidth = a6xx_gmu_scales_bandwidth, + .acd_set = a6xx_gmu_acd_set, +}; + +static int a6xx_gmu_bus_set(struct adreno_device *adreno_dev, int buslevel, + u32 ab) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + int ret = 0; + + if (buslevel != pwr->cur_buslevel) { + ret = a6xx_gmu_dcvs_set(adreno_dev, INVALID_DCVS_IDX, buslevel); + if (ret) + return ret; + + pwr->cur_buslevel = buslevel; + + trace_kgsl_buslevel(device, pwr->active_pwrlevel, buslevel); + } + + if (ab != pwr->cur_ab) { + icc_set_bw(pwr->icc_path, MBps_to_icc(ab), 0); + pwr->cur_ab = ab; + } + + return ret; +} + +static void a6xx_gmu_iommu_cb_close(struct gmu_iommu_context *ctx) +{ + if (!ctx->domain) + return; + + iommu_detach_device(ctx->domain, &ctx->pdev->dev); + iommu_domain_free(ctx->domain); + + platform_device_put(ctx->pdev); + ctx->domain = NULL; +} + +static void a6xx_free_gmu_globals(struct a6xx_gmu_device *gmu) +{ + int i; + + for (i = 0; i < gmu->global_entries; i++) { + struct gmu_memdesc *md = &gmu->gmu_globals[i]; + + if (!md->gmuaddr) + continue; + + iommu_unmap(get_gmu_domain(gmu, md), md->gmuaddr, md->size); + + dma_free_attrs(&gmu->pdev->dev, (size_t) md->size, + (void *)md->hostptr, md->physaddr, 0); + + memset(md, 0, sizeof(*md)); + } + + a6xx_gmu_iommu_cb_close(&a6xx_gmu_ctx[GMU_CONTEXT_KERNEL]); + a6xx_gmu_iommu_cb_close(&a6xx_gmu_ctx[GMU_CONTEXT_USER]); + + gmu->global_entries = 0; +} + +static int a6xx_gmu_aop_mailbox_init(struct adreno_device *adreno_dev, + struct a6xx_gmu_device *gmu) +{ + struct kgsl_mailbox *mailbox = &gmu->mailbox; + + mailbox->client.dev = &gmu->pdev->dev; + mailbox->client.tx_block = true; + mailbox->client.tx_tout = 1000; + mailbox->client.knows_txdone = false; + + mailbox->channel = mbox_request_channel(&mailbox->client, 0); + if (IS_ERR(mailbox->channel)) + return PTR_ERR(mailbox->channel); + + adreno_dev->acd_enabled = true; + return 0; +} + +static void a6xx_gmu_acd_probe(struct kgsl_device *device, + struct a6xx_gmu_device *gmu, struct device_node *node) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct hfi_acd_table_cmd *cmd = &gmu->hfi.acd_table; + u32 acd_level, cmd_idx, numlvl = pwr->num_pwrlevels; + int ret, i; + + if (!ADRENO_FEATURE(adreno_dev, ADRENO_ACD)) + return; + + cmd->hdr = CMD_MSG_HDR(H2F_MSG_ACD_TBL, sizeof(*cmd)); + cmd->version = 1; + cmd->stride = 1; + cmd->enable_by_level = 0; + + for (i = 0, cmd_idx = 0; i < numlvl; i++) { + acd_level = pwr->pwrlevels[numlvl - i].acd_level; + if (acd_level) { + cmd->enable_by_level |= (1 << i); + cmd->data[cmd_idx++] = acd_level; + } + } + + if (!cmd->enable_by_level) + return; + + cmd->num_levels = cmd_idx; + + ret = a6xx_gmu_aop_mailbox_init(adreno_dev, gmu); + if (ret) + dev_err(&gmu->pdev->dev, + "AOP mailbox init failed: %d\n", ret); +} + +static int a6xx_gmu_reg_probe(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct resource *res; + + res = platform_get_resource_byname(gmu->pdev, IORESOURCE_MEM, + "kgsl_gmu_reg"); + if (!res) { + dev_err(&gmu->pdev->dev, "The GMU register region isn't defined\n"); + return -ENODEV; + } + + device->gmu_core.gmu2gpu_offset = (res->start - device->reg_phys) >> 2; + device->gmu_core.reg_len = resource_size(res); + + /* + * We can't use devm_ioremap_resource here because we purposely double + * map the gpu_cc registers for debugging purposes + */ + device->gmu_core.reg_virt = devm_ioremap(&gmu->pdev->dev, res->start, + resource_size(res)); + + if (!device->gmu_core.reg_virt) { + dev_err(&gmu->pdev->dev, "Unable to map the GMU registers\n"); + return -ENOMEM; + } + + return 0; +} + +static int a6xx_gmu_regulators_probe(struct a6xx_gmu_device *gmu, + struct platform_device *pdev) +{ + gmu->cx_gdsc = devm_regulator_get(&pdev->dev, "vddcx"); + if (IS_ERR(gmu->cx_gdsc)) { + if (PTR_ERR(gmu->cx_gdsc) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Couldn't get the vddcx gdsc\n"); + return PTR_ERR(gmu->cx_gdsc); + } + + gmu->gx_gdsc = devm_regulator_get(&pdev->dev, "vdd"); + if (IS_ERR(gmu->gx_gdsc)) { + if (PTR_ERR(gmu->gx_gdsc) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Couldn't get the vdd gdsc\n"); + return PTR_ERR(gmu->gx_gdsc); + } + + return 0; +} + +static void a6xx_gmu_remove(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + if (!IS_ERR_OR_NULL(gmu->mailbox.channel)) + mbox_free_channel(gmu->mailbox.channel); + + adreno_dev->acd_enabled = false; + + if (gmu->fw_image) + release_firmware(gmu->fw_image); + + a6xx_free_gmu_globals(gmu); + + vfree(gmu->itcm_shadow); +} + +static int a6xx_gmu_iommu_fault_handler(struct iommu_domain *domain, + struct device *dev, unsigned long addr, int flags, void *token, + const char *name) +{ + char *fault_type = "unknown"; + + if (flags & IOMMU_FAULT_TRANSLATION) + fault_type = "translation"; + else if (flags & IOMMU_FAULT_PERMISSION) + fault_type = "permission"; + else if (flags & IOMMU_FAULT_EXTERNAL) + fault_type = "external"; + else if (flags & IOMMU_FAULT_TRANSACTION_STALLED) + fault_type = "transaction stalled"; + + dev_err(dev, "GMU fault addr = %lX, context=%s (%s %s fault)\n", + addr, name, + (flags & IOMMU_FAULT_WRITE) ? "write" : "read", + fault_type); + + return 0; +} + +static int a6xx_gmu_kernel_fault_handler(struct iommu_domain *domain, + struct device *dev, unsigned long addr, int flags, void *token) +{ + return a6xx_gmu_iommu_fault_handler(domain, dev, addr, flags, token, + "gmu_kernel"); +} + +static int a6xx_gmu_user_fault_handler(struct iommu_domain *domain, + struct device *dev, unsigned long addr, int flags, void *token) +{ + return a6xx_gmu_iommu_fault_handler(domain, dev, addr, flags, token, + "gmu_user"); +} + +static int a6xx_gmu_iommu_cb_probe(struct a6xx_gmu_device *gmu, + struct gmu_iommu_context *ctx, struct device_node *parent, + iommu_fault_handler_t handler) +{ + struct device_node *node = of_get_child_by_name(parent, ctx->name); + struct platform_device *pdev; + int ret; + + if (!node) + return -ENODEV; + + pdev = of_find_device_by_node(node); + ret = of_dma_configure(&pdev->dev, node, true); + of_node_put(node); + + if (ret) { + platform_device_put(pdev); + return ret; + } + + ctx->pdev = pdev; + ctx->domain = iommu_domain_alloc(&platform_bus_type); + if (ctx->domain == NULL) { + dev_err(&gmu->pdev->dev, "gmu iommu fail to alloc %s domain\n", + ctx->name); + platform_device_put(pdev); + return -ENODEV; + } + + ret = iommu_attach_device(ctx->domain, &pdev->dev); + if (!ret) { + iommu_set_fault_handler(ctx->domain, handler, ctx); + return 0; + } + + dev_err(&gmu->pdev->dev, + "gmu iommu fail to attach %s device\n", ctx->name); + iommu_domain_free(ctx->domain); + ctx->domain = NULL; + platform_device_put(pdev); + + return ret; +} + +static int a6xx_gmu_iommu_init(struct a6xx_gmu_device *gmu, + struct device_node *node) +{ + int ret; + + devm_of_platform_populate(&gmu->pdev->dev); + + ret = a6xx_gmu_iommu_cb_probe(gmu, &a6xx_gmu_ctx[GMU_CONTEXT_USER], + node, a6xx_gmu_user_fault_handler); + if (ret) + return ret; + + return a6xx_gmu_iommu_cb_probe(gmu, &a6xx_gmu_ctx[GMU_CONTEXT_KERNEL], + node, a6xx_gmu_kernel_fault_handler); +} + +static int a6xx_gmu_probe(struct kgsl_device *device, + struct platform_device *pdev) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hfi *hfi = &gmu->hfi; + struct resource *res; + int ret; + + gmu->pdev = pdev; + + dma_set_coherent_mask(&gmu->pdev->dev, DMA_BIT_MASK(64)); + gmu->pdev->dev.dma_mask = &gmu->pdev->dev.coherent_dma_mask; + set_dma_ops(&gmu->pdev->dev, NULL); + + res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM, + "rscc"); + if (res) { + gmu->rscc_virt = devm_ioremap(&device->pdev->dev, res->start, + resource_size(res)); + if (gmu->rscc_virt == NULL) { + dev_err(&gmu->pdev->dev, "rscc ioremap failed\n"); + return -ENOMEM; + } + } + + /* Set up GMU regulators */ + ret = a6xx_gmu_regulators_probe(gmu, pdev); + if (ret) + return ret; + + ret = devm_clk_bulk_get_all(&pdev->dev, &gmu->clks); + if (ret < 0) + return ret; + + gmu->num_clks = ret; + + /* Set up GMU IOMMU and shared memory with GMU */ + ret = a6xx_gmu_iommu_init(gmu, pdev->dev.of_node); + if (ret) + goto error; + + if (adreno_is_a650_family(adreno_dev)) + gmu->vma = a6xx_gmu_vma; + else + gmu->vma = a6xx_gmu_vma_legacy; + + /* Map and reserve GMU CSRs registers */ + ret = a6xx_gmu_reg_probe(adreno_dev); + if (ret) + goto error; + + /* Populates RPMh configurations */ + ret = a6xx_build_rpmh_tables(adreno_dev); + if (ret) + goto error; + + /* Set up GMU idle state */ + if (ADRENO_FEATURE(adreno_dev, ADRENO_IFPC)) + gmu->idle_level = GPU_HW_IFPC; + else + gmu->idle_level = GPU_HW_ACTIVE; + + a6xx_gmu_acd_probe(device, gmu, pdev->dev.of_node); + + set_bit(GMU_ENABLED, &device->gmu_core.flags); + + device->gmu_core.dev_ops = &a6xx_gmudev; + + /* Initialize HFI and GMU interrupts */ + hfi->irq = kgsl_request_irq(gmu->pdev, "kgsl_hfi_irq", + a6xx_hfi_irq_handler, device); + if (hfi->irq < 0) { + ret = hfi->irq; + goto error; + } + + gmu->irq = kgsl_request_irq(gmu->pdev, "kgsl_gmu_irq", + a6xx_gmu_irq_handler, device); + + if (gmu->irq < 0) { + ret = gmu->irq; + goto error; + } + + /* Don't enable GMU interrupts until GMU started */ + /* We cannot use irq_disable because it writes registers */ + disable_irq(gmu->irq); + disable_irq(gmu->hfi.irq); + + return 0; + +error: + a6xx_gmu_remove(device); + return ret; +} + +static void a6xx_gmu_active_count_put(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return; + + if (WARN(atomic_read(&device->active_cnt) == 0, + "Unbalanced get/put calls to KGSL active count\n")) + return; + + if (atomic_dec_and_test(&device->active_cnt)) { + kgsl_pwrscale_update_stats(device); + kgsl_pwrscale_update(device); + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + } + + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + + wake_up(&device->active_cnt_wq); +} + +static int halt_gbif(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* Halt new client requests */ + kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_CLIENT_HALT_MASK); + ret = adreno_wait_for_halt_ack(device, + ADRENO_REG_GBIF_HALT_ACK, + A6XX_GBIF_CLIENT_HALT_MASK); + + /* Halt all AXI requests */ + kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_ARB_HALT_MASK); + ret = adreno_wait_for_halt_ack(device, + ADRENO_REG_GBIF_HALT_ACK, + A6XX_GBIF_ARB_HALT_MASK); + + /* De-assert the halts */ + kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0); + + return ret; +} + +static int a6xx_gmu_power_off(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret = 0; + + if (device->gmu_fault) + goto error; + + /* Wait for the lowest idle level we requested */ + ret = a6xx_gmu_wait_for_lowest_idle(adreno_dev); + if (ret) + goto error; + + ret = a6xx_gmu_notify_slumber(adreno_dev); + if (ret) + goto error; + + ret = a6xx_gmu_wait_for_idle(adreno_dev); + if (ret) + goto error; + + ret = a6xx_rscc_sleep_sequence(adreno_dev); + + /* Now that we are done with GMU and GPU, Clear the GBIF */ + if (!adreno_is_a630(adreno_dev)) + ret = halt_gbif(adreno_dev); + + a6xx_gmu_irq_disable(adreno_dev); + + a6xx_hfi_stop(adreno_dev); + + clk_bulk_disable_unprepare(gmu->num_clks, gmu->clks); + + /* Pool to make sure that the CX is off */ + if (!kgsl_regulator_disable_wait(gmu->cx_gdsc, 5000)) + dev_err(&gmu->pdev->dev, "GMU CX gdsc off timeout\n"); + + device->state = KGSL_STATE_NONE; + + return ret; + +error: + a6xx_hfi_stop(adreno_dev); + a6xx_gmu_suspend(adreno_dev); + + return ret; +} + +void a6xx_enable_gpu_irq(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + adreno_irqctrl(adreno_dev, 1); + enable_irq(pwr->interrupt_num); + + trace_kgsl_irq(device, 1); +} + +void a6xx_disable_gpu_irq(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + disable_irq(pwr->interrupt_num); + + if (a6xx_gmu_gx_is_on(device)) + adreno_irqctrl(adreno_dev, 0); + + trace_kgsl_irq(device, 0); +} + +static int a6xx_gpu_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* Clear any GPU faults that might have been left over */ + adreno_clear_gpu_fault(adreno_dev); + + adreno_set_active_ctxs_null(adreno_dev); + + adreno_ringbuffer_set_global(adreno_dev, 0); + + ret = kgsl_mmu_start(device); + if (ret) + goto err; + + ret = a6xx_gmu_oob_set(device, oob_gpu); + if (ret) + goto oob_clear; + + ret = a6xx_gmu_hfi_start_msg(adreno_dev); + if (ret) + goto oob_clear; + + adreno_clear_dcvs_counters(adreno_dev); + + /* Restore performance counter registers with saved values */ + adreno_perfcounter_restore(adreno_dev); + + a6xx_start(adreno_dev); + + /* Re-initialize the coresight registers if applicable */ + adreno_coresight_start(adreno_dev); + + adreno_perfcounter_start(adreno_dev); + + /* Clear FSR here in case it is set from a previous pagefault */ + kgsl_mmu_clear_fsr(&device->mmu); + + a6xx_enable_gpu_irq(adreno_dev); + + ret = a6xx_rb_start(adreno_dev); + if (ret) { + a6xx_disable_gpu_irq(adreno_dev); + goto oob_clear; + } + + /* Start the dispatcher */ + adreno_dispatcher_start(device); + + device->reset_counter++; + + a6xx_gmu_oob_clear(device, oob_gpu); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) + gmu_core_dev_oob_clear(device, oob_boot_slumber); + + return 0; + +oob_clear: + a6xx_gmu_oob_clear(device, oob_gpu); + + if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) + gmu_core_dev_oob_clear(device, oob_boot_slumber); + +err: + a6xx_gmu_power_off(adreno_dev); + + return ret; +} + +static void gmu_idle_timer(struct timer_list *t) +{ + struct kgsl_device *device = container_of(t, struct kgsl_device, + idle_timer); + + kgsl_schedule_work(&device->idle_check_ws); +} + +static int a6xx_boot(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + WARN_ON(test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)); + + ret = a6xx_gmu_boot(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return ret; + + mod_timer(&device->idle_timer, jiffies + + device->pwrctrl.interval_timeout); + + kgsl_pwrscale_wake(device); + + set_bit(GMU_PRIV_GPU_STARTED, &gmu->flags); + device->state = KGSL_STATE_ACTIVE; + + return ret; +} + +static int a6xx_first_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + unsigned long priv = 0; + + if (test_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags)) + return a6xx_boot(adreno_dev); + + ret = adreno_dispatcher_init(adreno_dev); + if (ret) + return ret; + + ret = adreno_ringbuffer_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_microcode_read(adreno_dev); + if (ret) + return ret; + + ret = a6xx_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gmu_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gmu_first_boot(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return ret; + + ret = a6xx_load_pdc_ucode(adreno_dev); + if (ret) { + a6xx_gmu_power_off(adreno_dev); + return ret; + } + + a6xx_load_rsc_ucode(adreno_dev); + + adreno_get_bus_counters(adreno_dev); + + adreno_dev->cooperative_reset = ADRENO_FEATURE(adreno_dev, + ADRENO_COOP_RESET); + + if (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) + priv |= KGSL_MEMDESC_PRIVILEGED; + + adreno_dev->profile_buffer = kgsl_allocate_global(device, PAGE_SIZE, 0, + priv, "alwayson"); + + adreno_dev->profile_index = 0; + + if (!IS_ERR(adreno_dev->profile_buffer)) + set_bit(ADRENO_DEVICE_DRAWOBJ_PROFILE, &adreno_dev->priv); + + set_bit(GMU_PRIV_FIRST_BOOT_DONE, &gmu->flags); + set_bit(GMU_PRIV_GPU_STARTED, &gmu->flags); + + device->state = KGSL_STATE_ACTIVE; + + return 0; +} + +static int a630_vbif_halt(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + kgsl_regwrite(device, A6XX_VBIF_XIN_HALT_CTRL0, + A6XX_VBIF_XIN_HALT_CTRL0_MASK); + ret = adreno_wait_for_halt_ack(device, + ADRENO_REG_VBIF_XIN_HALT_CTRL1, + A6XX_VBIF_XIN_HALT_CTRL0_MASK); + kgsl_regwrite(device, A6XX_VBIF_XIN_HALT_CTRL0, 0); + + return ret; +} + +static int a6xx_power_off(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + + WARN_ON(!test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)); + + ret = a6xx_gmu_oob_set(device, oob_gpu); + if (ret) { + a6xx_gmu_oob_clear(device, oob_gpu); + goto no_gx_power; + } + + kgsl_pwrscale_update_stats(device); + + /* Save active coresight registers if applicable */ + adreno_coresight_stop(adreno_dev); + + /* Save physical performance counter values before GPU power down*/ + adreno_perfcounter_save(adreno_dev); + + /* + * Clear GX halt on non-gbif targets. For targets with GBIF, + * GX halt is handled by the GMU FW. + */ + if (adreno_is_a630(adreno_dev)) + a630_vbif_halt(adreno_dev); + + a6xx_gmu_oob_clear(device, oob_gpu); + +no_gx_power: + a6xx_disable_gpu_irq(adreno_dev); + + a6xx_gmu_power_off(adreno_dev); + + adreno_set_active_ctxs_null(adreno_dev); + + adreno_dispatcher_stop(adreno_dev); + + adreno_ringbuffer_stop(adreno_dev); + + if (!IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpu_llc_slice); + + if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpuhtw_llc_slice); + + clear_bit(GMU_PRIV_GPU_STARTED, &gmu->flags); + + del_timer_sync(&device->idle_timer); + + kgsl_pwrscale_sleep(device); + + return ret; +} + +static void gmu_idle_check(struct work_struct *work) +{ + struct kgsl_device *device = container_of(work, + struct kgsl_device, idle_check_ws); + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + mutex_lock(&device->mutex); + + if (!atomic_read(&device->active_cnt)) { + if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) + a6xx_power_off(adreno_dev); + } else { + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + } + + mutex_unlock(&device->mutex); +} + +static int a6xx_gmu_first_open(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* + * Do the one time settings that need to happen when we + * attempt to boot the gpu the very first time + */ + ret = a6xx_first_boot(adreno_dev); + if (ret) + return ret; + + /* + * A client that does a first_open but never closes the device + * may prevent us from going back to SLUMBER. So trigger the idle + * check by incrementing the active count and immediately releasing it. + */ + atomic_inc(&device->active_cnt); + a6xx_gmu_active_count_put(adreno_dev); + + return 0; +} + +static int a6xx_gmu_last_close(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) + return a6xx_power_off(adreno_dev); + + return 0; +} + +static int a6xx_gmu_active_count_get(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret = 0; + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return -EINVAL; + + if (test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags)) + return -EINVAL; + + if ((atomic_read(&device->active_cnt) == 0) && + !test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) + ret = a6xx_boot(adreno_dev); + + if (ret == 0) + atomic_inc(&device->active_cnt); + + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + + return ret; +} + +static int a6xx_gmu_pm_suspend(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + + if (test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags)) + return 0; + + /* Halt any new submissions */ + reinit_completion(&device->halt_gate); + + /* wait for active count so device can be put in slumber */ + ret = kgsl_active_count_wait(device, 0); + if (ret) { + dev_err(device->dev, + "Timed out waiting for the active count\n"); + goto err; + } + + ret = adreno_idle(device); + if (ret) + goto err; + + if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) + a6xx_power_off(adreno_dev); + + set_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags); + + adreno_dispatcher_halt(device); + + return 0; +err: + adreno_dispatcher_start(device); + return ret; +} + +static void a6xx_gmu_pm_resume(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + if (!test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags)) { + dev_err(device->dev, "resume invoked without a suspend\n"); + return; + } + + adreno_dispatcher_unhalt(device); + + adreno_dispatcher_start(device); + + clear_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags); +} + +static void a6xx_gmu_touch_wakeup(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + int ret; + + /* Do not wake up a suspended device through touch event */ + if (test_bit(GMU_PRIV_PM_SUSPEND, &gmu->flags)) + return; + + if (test_bit(GMU_PRIV_GPU_STARTED, &gmu->flags)) + goto done; + + ret = a6xx_gmu_boot(adreno_dev); + if (ret) + return; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return; + + kgsl_pwrscale_wake(device); + + set_bit(GMU_PRIV_GPU_STARTED, &gmu->flags); + device->state = KGSL_STATE_ACTIVE; + +done: + /* + * When waking up from a touch event we want to stay active long enough + * for the user to send a draw command. The default idle timer timeout + * is shorter than we want so go ahead and push the idle timer out + * further for this special case + */ + mod_timer(&device->idle_timer, jiffies + + msecs_to_jiffies(adreno_wake_timeout)); + +} + +const struct adreno_power_ops a6xx_gmu_power_ops = { + .first_open = a6xx_gmu_first_open, + .last_close = a6xx_gmu_last_close, + .active_count_get = a6xx_gmu_active_count_get, + .active_count_put = a6xx_gmu_active_count_put, + .pm_suspend = a6xx_gmu_pm_suspend, + .pm_resume = a6xx_gmu_pm_resume, + .touch_wakeup = a6xx_gmu_touch_wakeup, + .gpu_clock_set = a6xx_gmu_clock_set, + .gpu_bus_set = a6xx_gmu_bus_set, +}; + +const struct adreno_power_ops a630_gmu_power_ops = { + .first_open = a6xx_gmu_first_open, + .last_close = a6xx_gmu_last_close, + .active_count_get = a6xx_gmu_active_count_get, + .active_count_put = a6xx_gmu_active_count_put, + .pm_suspend = a6xx_gmu_pm_suspend, + .pm_resume = a6xx_gmu_pm_resume, + .touch_wakeup = a6xx_gmu_touch_wakeup, + .gpu_clock_set = a6xx_gmu_clock_set, +}; + +int a6xx_gmu_device_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + struct a6xx_device *a6xx_dev; + int ret; + + a6xx_dev = devm_kzalloc(&pdev->dev, sizeof(*a6xx_dev), + GFP_KERNEL); + if (!a6xx_dev) + return -ENOMEM; + + adreno_dev = &a6xx_dev->adreno_dev; + + ret = a6xx_probe_common(pdev, adreno_dev, chipid, gpucore); + if (ret) + return ret; + + device = KGSL_DEVICE(adreno_dev); + + INIT_WORK(&device->idle_check_ws, gmu_idle_check); + + timer_setup(&device->idle_timer, gmu_idle_timer, 0); + + return 0; +} + +int a6xx_gmu_restart(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + a6xx_hfi_stop(adreno_dev); + + a6xx_disable_gpu_irq(adreno_dev); + + /* Hard reset the gmu and gpu */ + a6xx_gmu_suspend(adreno_dev); + + clear_bit(GMU_PRIV_GPU_STARTED, &gmu->flags); + + /* Attempt to reboot the gmu and gpu */ + return a6xx_boot(adreno_dev); +} + +static int a6xx_gmu_bind(struct device *dev, struct device *master, void *data) +{ + struct kgsl_device *device = dev_get_drvdata(master); + + return a6xx_gmu_probe(device, to_platform_device(dev)); +} + +static void a6xx_gmu_unbind(struct device *dev, struct device *master, + void *data) +{ + struct kgsl_device *device = dev_get_drvdata(master); + + a6xx_gmu_remove(device); +} + +static const struct component_ops a6xx_gmu_component_ops = { + .bind = a6xx_gmu_bind, + .unbind = a6xx_gmu_unbind, +}; + +static int a6xx_gmu_probe_dev(struct platform_device *pdev) +{ + return component_add(&pdev->dev, &a6xx_gmu_component_ops); +} + +static int a6xx_gmu_remove_dev(struct platform_device *pdev) +{ + component_del(&pdev->dev, &a6xx_gmu_component_ops); + return 0; +} + +static const struct of_device_id a6xx_gmu_match_table[] = { + { .compatible = "qcom,gpu-gmu" }, + { }, +}; + +struct platform_driver a6xx_gmu_driver = { + .probe = a6xx_gmu_probe_dev, + .remove = a6xx_gmu_remove_dev, + .driver = { + .name = "adreno-a6xx-gmu", + .of_match_table = a6xx_gmu_match_table, + }, }; diff --git a/drivers/gpu/msm/kgsl_gmu.h b/drivers/gpu/msm/adreno_a6xx_gmu.h similarity index 52% rename from drivers/gpu/msm/kgsl_gmu.h rename to drivers/gpu/msm/adreno_a6xx_gmu.h index 062cb362bfaa..e6bdc5f49d60 100644 --- a/drivers/gpu/msm/kgsl_gmu.h +++ b/drivers/gpu/msm/adreno_a6xx_gmu.h @@ -2,20 +2,19 @@ /* * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. */ -#ifndef __KGSL_GMU_H -#define __KGSL_GMU_H +#ifndef __ADRENO_A6XX_GMU_H +#define __ADRENO_A6XX_GMU_H #include +#include "adreno_a6xx_hfi.h" #include "kgsl_gmu_core.h" -#include "kgsl_hfi.h" + +#define GMU_FREQUENCY 200000000 #define GMU_PWR_LEVELS 2 -#define GMU_FREQUENCY 200000000 #define MAX_GMUFW_SIZE 0x8000 /* in bytes */ -#define BWMEM_SIZE (12 + (4 * NUM_BW_LEVELS)) /*in bytes*/ - #define GMU_VER_MAJOR(ver) (((ver) >> 28) & 0xF) #define GMU_VER_MINOR(ver) (((ver) >> 16) & 0xFFF) #define GMU_VER_STEP(ver) ((ver) & 0xFFFF) @@ -81,9 +80,6 @@ struct gmu_block_header { /* GMU memdesc entries */ #define GMU_KERNEL_ENTRIES 16 -extern struct gmu_dev_ops adreno_a6xx_gmudev; -#define KGSL_GMU_DEVICE(_a) ((struct gmu_device *)((_a)->gmu_core.ptr)) - enum gmu_mem_type { GMU_ITCM = 0, GMU_ICACHE, @@ -96,7 +92,7 @@ enum gmu_mem_type { }; enum gmu_context_index { - GMU_CONTEXT_USER = 0, + GMU_CONTEXT_USER, GMU_CONTEXT_KERNEL, }; @@ -106,16 +102,12 @@ enum gmu_context_index { * @gmuaddr: GPU virtual address * @physaddr: Physical address of the memory object * @size: Size of the memory object - * @mem_type: memory type for this memory - * @ctx_idx: GMU IOMMU context idx */ struct gmu_memdesc { void *hostptr; - uint64_t gmuaddr; + u32 gmuaddr; phys_addr_t physaddr; - uint64_t size; - enum gmu_mem_type mem_type; - enum gmu_context_index ctx_idx; + u32 size; }; struct rpmh_votes_t { @@ -123,14 +115,6 @@ struct rpmh_votes_t { uint32_t cx_votes[MAX_CX_LEVELS]; }; -enum gmu_load_mode { - CACHED_LOAD_BOOT, - CACHED_BOOT, - TCM_BOOT, - TCM_LOAD_BOOT, - INVALID_LOAD -}; - struct kgsl_mailbox { struct mbox_client client; struct mbox_chan *channel; @@ -138,10 +122,27 @@ struct kgsl_mailbox { struct icc_path; +struct gmu_vma_entry { + /** @start: Starting virtual address of the vma */ + u32 start; + /** @size: Size of this vma */ + u32 size; + /** @next_va: Next available virtual address in this vma */ + u32 next_va; +}; + +enum { + GMU_PRIV_FIRST_BOOT_DONE = 0, + GMU_PRIV_GPU_STARTED, + GMU_PRIV_HFI_STARTED, + GMU_PRIV_RSCC_SLEEP_DONE, + GMU_PRIV_PM_SUSPEND, +}; + /** - * struct gmu_device - GMU device structure + * struct a6xx_gmu_device - GMU device structure * @ver: GMU Version information - * @gmu_interrupt_num: GMU interrupt number + * @irq: GMU interrupt number * @fw_image: GMU FW image * @hfi_mem: pointer to HFI shared memory * @dump_mem: pointer to GMU debug dump memory @@ -155,7 +156,6 @@ struct icc_path; subsystem peripherals * @gx_gdsc: GX headswitch that controls power of GPU subsystem * @clks: GPU subsystem clocks required for GMU functionality - * @load_mode: GMU FW load/boot mode * @wakeup_pwrlevel: GPU wake up power/DCVS level in case different * than default power level * @idle_level: Minimal GPU idle power level @@ -163,7 +163,7 @@ struct icc_path; * @mailbox: Messages to AOP for ACD enable/disable go through this * @log_wptr_retention: Store the log wptr offset on slumber */ -struct gmu_device { +struct a6xx_gmu_device { struct { u32 core; u32 core_dev; @@ -172,48 +172,129 @@ struct gmu_device { u32 hfi; } ver; struct platform_device *pdev; - int gmu_interrupt_num; + int irq; const struct firmware *fw_image; - struct gmu_memdesc *hfi_mem; struct gmu_memdesc *dump_mem; struct gmu_memdesc *gmu_log; - struct kgsl_hfi hfi; + struct a6xx_hfi hfi; /** @pwrlevels: Array of GMU power levels */ - struct { - /** @freq: GPU frequency */ - unsigned long freq; - /** @level: Voltage level */ - u32 level; - } pwrlevels[MAX_GX_LEVELS]; - unsigned int num_gpupwrlevels; - unsigned int num_bwlevels; - unsigned int num_cnocbwlevels; - struct rpmh_votes_t rpmh_votes; struct regulator *cx_gdsc; struct regulator *gx_gdsc; struct clk_bulk_data *clks; /** @num_clks: Number of entries in the @clks array */ int num_clks; - enum gmu_load_mode load_mode; - unsigned int wakeup_pwrlevel; unsigned int idle_level; - unsigned int fault_count; struct kgsl_mailbox mailbox; bool preallocations; - struct gmu_memdesc kmem_entries[GMU_KERNEL_ENTRIES]; - unsigned long kmem_bitmap; - const struct gmu_vma_entry *vma; + /** @gmu_globals: Array to store gmu global buffers */ + struct gmu_memdesc gmu_globals[GMU_KERNEL_ENTRIES]; + /** @global_entries: To keep track of number of gmu buffers */ + u32 global_entries; + struct gmu_vma_entry *vma; unsigned int log_wptr_retention; /** @cm3_fault: whether gmu received a cm3 fault interrupt */ atomic_t cm3_fault; + /** + * @itcm_shadow: Copy of the itcm block in firmware binary used for + * snapshot + */ + void *itcm_shadow; + /** @flags: Internal gmu flags */ + unsigned long flags; + /** @rscc_virt: Pointer where RSCC block is mapped */ + void __iomem *rscc_virt; }; -struct gmu_memdesc *gmu_get_memdesc(struct gmu_device *gmu, - unsigned int addr, unsigned int size); -unsigned int gmu_get_memtype_base(struct gmu_device *gmu, - enum gmu_mem_type type); +/* Helper function to get to a6xx gmu device from adreno device */ +struct a6xx_gmu_device *to_a6xx_gmu(struct adreno_device *adreno_dev); -int gmu_prealloc_req(struct kgsl_device *device, struct gmu_block_header *blk); -int gmu_cache_finalize(struct kgsl_device *device); +/* Helper function to get to adreno device from a6xx gmu device */ +struct adreno_device *a6xx_gmu_to_adreno(struct a6xx_gmu_device *gmu); -#endif /* __KGSL_GMU_H */ +/** + * reserve_gmu_kernel_block() - Allocate a gmu buffer + * @gmu: Pointer to the a6xx gmu device + * @addr: Desired gmu virtual address + * @size: Size of the buffer in bytes + * @vma_id: Target gmu vma where this bufer should be mapped + * + * This function allocates a buffer and maps it in + * the desired gmu vma + * + * Return: Pointer to the memory descriptor or error pointer on failure + */ +struct gmu_memdesc *reserve_gmu_kernel_block(struct a6xx_gmu_device *gmu, + u32 addr, u32 size, u32 vma_id); + +/** + * a6xx_build_rpmh_tables - Build the rpmh tables + * @adreno_dev: Pointer to the adreno device + * + * This function creates the gpu dcvs and bw tables + * + * Return: 0 on success and negative error on failure + */ +int a6xx_build_rpmh_tables(struct adreno_device *adreno_dev); + +/** + * a6xx_gmu_gx_is_on - Check if GX is on + * @device: Pointer to KGSL device + * + * This function reads pwr status registers to check if GX + * is on or off + */ +bool a6xx_gmu_gx_is_on(struct kgsl_device *device); + +/** + * a6xx_gmu_device_snapshot - A6XX GMU snapshot function + * @device: Device being snapshotted + * @snapshot: Pointer to the snapshot instance + * + * This is where all of the A6XX GMU specific bits and pieces are grabbed + * into the snapshot memory + */ +void a6xx_gmu_device_snapshot(struct kgsl_device *device, + struct kgsl_snapshot *snapshot); + +/** + * a6xx_gmu_device_probe - A6XX GMU snapshot function + * @pdev: Pointer to the platform device + * @chipid: Chipid of the target + * @gpucore: Pointer to the gpucore + * + * The target specific probe function for gmu based a6xx targets. + */ +int a6xx_gmu_device_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore); + +/** + * a6xx_gmu_restart - Reset and restart the gmu + * @device: Pointer to the kgsl device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_gmu_restart(struct kgsl_device *device); + +/** + * a6xx_enable_gpu_irq - Enable gpu interrupt + * @adreno_dev: Pointer to the adreno device + */ +void a6xx_enable_gpu_irq(struct adreno_device *adreno_dev); + +/** + * a6xx_disable_gpu_irq - Disable gpu interrupt + * @adreno_dev: Pointer to the adreno device + */ +void a6xx_disable_gpu_irq(struct adreno_device *adreno_dev); + +/** + * a6xx_gmu_snapshot- Take snapshot for gmu targets + * @adreno_dev: Pointer to the adreno device + * @snapshot: Pointer to the snapshot structure + * + * Send an NMI to gmu if we hit a gmu fault. Then take gmu + * snapshot and carry on with rest of the a6xx snapshot + */ +void a6xx_gmu_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); +#endif diff --git a/drivers/gpu/msm/adreno_a6xx_gmu_snapshot.c b/drivers/gpu/msm/adreno_a6xx_gmu_snapshot.c new file mode 100644 index 000000000000..8ec0951108f3 --- /dev/null +++ b/drivers/gpu/msm/adreno_a6xx_gmu_snapshot.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#include "a6xx_reg.h" +#include "adreno.h" +#include "adreno_a6xx_gmu.h" +#include "adreno_snapshot.h" +#include "kgsl_device.h" + +static const unsigned int a6xx_gmu_gx_registers[] = { + /* GMU GX */ + 0x1A800, 0x1A800, 0x1A810, 0x1A813, 0x1A816, 0x1A816, 0x1A818, 0x1A81B, + 0x1A81E, 0x1A81E, 0x1A820, 0x1A823, 0x1A826, 0x1A826, 0x1A828, 0x1A82B, + 0x1A82E, 0x1A82E, 0x1A830, 0x1A833, 0x1A836, 0x1A836, 0x1A838, 0x1A83B, + 0x1A83E, 0x1A83E, 0x1A840, 0x1A843, 0x1A846, 0x1A846, 0x1A880, 0x1A884, + 0x1A900, 0x1A92B, 0x1A940, 0x1A940, +}; + +static const unsigned int a6xx_gmu_tcm_registers[] = { + /* ITCM */ + 0x1B400, 0x1C3FF, + /* DTCM */ + 0x1C400, 0x1D3FF, +}; + +static const unsigned int a6xx_gmu_registers[] = { + /* GMU CX */ + 0x1F400, 0x1F407, 0x1F410, 0x1F412, 0x1F500, 0x1F500, 0x1F507, 0x1F50A, + 0x1F800, 0x1F804, 0x1F807, 0x1F808, 0x1F80B, 0x1F80C, 0x1F80F, 0x1F81C, + 0x1F824, 0x1F82A, 0x1F82D, 0x1F830, 0x1F840, 0x1F853, 0x1F887, 0x1F889, + 0x1F8A0, 0x1F8A2, 0x1F8A4, 0x1F8AF, 0x1F8C0, 0x1F8C3, 0x1F8D0, 0x1F8D0, + 0x1F8E4, 0x1F8E4, 0x1F8E8, 0x1F8EC, 0x1F900, 0x1F903, 0x1F940, 0x1F940, + 0x1F942, 0x1F944, 0x1F94C, 0x1F94D, 0x1F94F, 0x1F951, 0x1F954, 0x1F954, + 0x1F957, 0x1F958, 0x1F95D, 0x1F95D, 0x1F962, 0x1F962, 0x1F964, 0x1F965, + 0x1F980, 0x1F986, 0x1F990, 0x1F99E, 0x1F9C0, 0x1F9C0, 0x1F9C5, 0x1F9CC, + 0x1F9E0, 0x1F9E2, 0x1F9F0, 0x1F9F0, 0x1FA00, 0x1FA01, + /* GMU AO */ + 0x23B00, 0x23B16, + /* GPU CC */ + 0x24000, 0x24012, 0x24040, 0x24052, 0x24400, 0x24404, 0x24407, 0x2440B, + 0x24415, 0x2441C, 0x2441E, 0x2442D, 0x2443C, 0x2443D, 0x2443F, 0x24440, + 0x24442, 0x24449, 0x24458, 0x2445A, 0x24540, 0x2455E, 0x24800, 0x24802, + 0x24C00, 0x24C02, 0x25400, 0x25402, 0x25800, 0x25802, 0x25C00, 0x25C02, + 0x26000, 0x26002, + /* GPU CC ACD */ + 0x26400, 0x26416, 0x26420, 0x26427, +}; + +static const unsigned int a660_gmu_registers[] = { + /* GMU CX */ + 0x1F408, 0x1F40D, 0x1F40F, 0x1F40F, 0x1F50B, 0x1F50B, 0x1F860, 0x1F860, + 0x1F870, 0x1F877, 0x1F8C4, 0x1F8C4, 0x1F8F0, 0x1F8F1, 0x1F948, 0x1F94A, + 0x1F966, 0x1F96B, 0x1F970, 0x1F970, 0x1F972, 0x1F979, 0x1F9CD, 0x1F9D4, + 0x1FA02, 0x1FA03, 0x20000, 0x20001, 0x20004, 0x20004, 0x20008, 0x20012, + 0x20018, 0x20018, +}; + +static const unsigned int a630_rscc_snapshot_registers[] = { + 0x23400, 0x23434, 0x23436, 0x23436, 0x23480, 0x23484, 0x23489, 0x2348C, + 0x23491, 0x23494, 0x23499, 0x2349C, 0x234A1, 0x234A4, 0x234A9, 0x234AC, + 0x23500, 0x23502, 0x23504, 0x23507, 0x23514, 0x23519, 0x23524, 0x2352B, + 0x23580, 0x23597, 0x23740, 0x23741, 0x23744, 0x23747, 0x2374C, 0x23787, + 0x237EC, 0x237EF, 0x237F4, 0x2382F, 0x23894, 0x23897, 0x2389C, 0x238D7, + 0x2393C, 0x2393F, 0x23944, 0x2397F, +}; + +static const unsigned int a6xx_rscc_snapshot_registers[] = { + 0x23400, 0x23434, 0x23436, 0x23436, 0x23440, 0x23440, 0x23480, 0x23484, + 0x23489, 0x2348C, 0x23491, 0x23494, 0x23499, 0x2349C, 0x234A1, 0x234A4, + 0x234A9, 0x234AC, 0x23500, 0x23502, 0x23504, 0x23507, 0x23514, 0x23519, + 0x23524, 0x2352B, 0x23580, 0x23597, 0x23740, 0x23741, 0x23744, 0x23747, + 0x2374C, 0x23787, 0x237EC, 0x237EF, 0x237F4, 0x2382F, 0x23894, 0x23897, + 0x2389C, 0x238D7, 0x2393C, 0x2393F, 0x23944, 0x2397F, +}; + +static const unsigned int a650_rscc_registers[] = { + 0x38000, 0x38034, 0x38036, 0x38036, 0x38040, 0x38042, 0x38080, 0x38084, + 0x38089, 0x3808C, 0x38091, 0x38094, 0x38099, 0x3809C, 0x380A1, 0x380A4, + 0x380A9, 0x380AC, 0x38100, 0x38102, 0x38104, 0x38107, 0x38114, 0x38119, + 0x38124, 0x3812E, 0x38180, 0x38197, 0x38340, 0x38341, 0x38344, 0x38347, + 0x3834C, 0x3834F, 0x38351, 0x38354, 0x38356, 0x38359, 0x3835B, 0x3835E, + 0x38360, 0x38363, 0x38365, 0x38368, 0x3836A, 0x3836D, 0x3836F, 0x38372, + 0x383EC, 0x383EF, 0x383F4, 0x383F7, 0x383F9, 0x383FC, 0x383FE, 0x38401, + 0x38403, 0x38406, 0x38408, 0x3840B, 0x3840D, 0x38410, 0x38412, 0x38415, + 0x38417, 0x3841A, 0x38494, 0x38497, 0x3849C, 0x3849F, 0x384A1, 0x384A4, + 0x384A6, 0x384A9, 0x384AB, 0x384AE, 0x384B0, 0x384B3, 0x384B5, 0x384B8, + 0x384BA, 0x384BD, 0x384BF, 0x384C2, 0x3853C, 0x3853F, 0x38544, 0x38547, + 0x38549, 0x3854C, 0x3854E, 0x38551, 0x38553, 0x38556, 0x38558, 0x3855B, + 0x3855D, 0x38560, 0x38562, 0x38565, 0x38567, 0x3856A, 0x385E4, 0x385E7, + 0x385EC, 0x385EF, 0x385F1, 0x385F4, 0x385F6, 0x385F9, 0x385FB, 0x385FE, + 0x38600, 0x38603, 0x38605, 0x38608, 0x3860A, 0x3860D, 0x3860F, 0x38612, + 0x3868C, 0x3868F, 0x38694, 0x38697, 0x38699, 0x3869C, 0x3869E, 0x386A1, + 0x386A3, 0x386A6, 0x386A8, 0x386AB, 0x386AD, 0x386B0, 0x386B2, 0x386B5, + 0x386B7, 0x386BA, 0x38734, 0x38737, 0x3873C, 0x3873F, 0x38741, 0x38744, + 0x38746, 0x38749, 0x3874B, 0x3874E, 0x38750, 0x38753, 0x38755, 0x38758, + 0x3875A, 0x3875D, 0x3875F, 0x38762, 0x387DC, 0x387DF, 0x387E4, 0x387E7, + 0x387E9, 0x387EC, 0x387EE, 0x387F1, 0x387F3, 0x387F6, 0x387F8, 0x387FB, + 0x387FD, 0x38800, 0x38802, 0x38805, 0x38807, 0x3880A, 0x38884, 0x38887, + 0x3888C, 0x3888F, 0x38891, 0x38894, 0x38896, 0x38899, 0x3889B, 0x3889E, + 0x388A0, 0x388A3, 0x388A5, 0x388A8, 0x388AA, 0x388AD, 0x388AF, 0x388B2, + 0x3892C, 0x3892F, 0x38934, 0x38937, 0x38939, 0x3893C, 0x3893E, 0x38941, + 0x38943, 0x38946, 0x38948, 0x3894B, 0x3894D, 0x38950, 0x38952, 0x38955, + 0x38957, 0x3895A, 0x38B50, 0x38B51, 0x38B53, 0x38B55, 0x38B5A, 0x38B5A, + 0x38B5F, 0x38B5F, 0x38B64, 0x38B64, 0x38B69, 0x38B69, 0x38B6E, 0x38B6E, + 0x38B73, 0x38B73, 0x38BF8, 0x38BF8, 0x38BFD, 0x38BFD, 0x38C02, 0x38C02, + 0x38C07, 0x38C07, 0x38C0C, 0x38C0C, 0x38C11, 0x38C11, 0x38C16, 0x38C16, + 0x38C1B, 0x38C1B, 0x38CA0, 0x38CA0, 0x38CA5, 0x38CA5, 0x38CAA, 0x38CAA, + 0x38CAF, 0x38CAF, 0x38CB4, 0x38CB4, 0x38CB9, 0x38CB9, 0x38CBE, 0x38CBE, + 0x38CC3, 0x38CC3, 0x38D48, 0x38D48, 0x38D4D, 0x38D4D, 0x38D52, 0x38D52, + 0x38D57, 0x38D57, 0x38D5C, 0x38D5C, 0x38D61, 0x38D61, 0x38D66, 0x38D66, + 0x38D6B, 0x38D6B, 0x38DF0, 0x38DF0, 0x38DF5, 0x38DF5, 0x38DFA, 0x38DFA, + 0x38DFF, 0x38DFF, 0x38E04, 0x38E04, 0x38E09, 0x38E09, 0x38E0E, 0x38E0E, + 0x38E13, 0x38E13, 0x38E98, 0x38E98, 0x38E9D, 0x38E9D, 0x38EA2, 0x38EA2, + 0x38EA7, 0x38EA7, 0x38EAC, 0x38EAC, 0x38EB1, 0x38EB1, 0x38EB6, 0x38EB6, + 0x38EBB, 0x38EBB, 0x38F40, 0x38F40, 0x38F45, 0x38F45, 0x38F4A, 0x38F4A, + 0x38F4F, 0x38F4F, 0x38F54, 0x38F54, 0x38F59, 0x38F59, 0x38F5E, 0x38F5E, + 0x38F63, 0x38F63, 0x38FE8, 0x38FE8, 0x38FED, 0x38FED, 0x38FF2, 0x38FF2, + 0x38FF7, 0x38FF7, 0x38FFC, 0x38FFC, 0x39001, 0x39001, 0x39006, 0x39006, + 0x3900B, 0x3900B, 0x39090, 0x39090, 0x39095, 0x39095, 0x3909A, 0x3909A, + 0x3909F, 0x3909F, 0x390A4, 0x390A4, 0x390A9, 0x390A9, 0x390AE, 0x390AE, + 0x390B3, 0x390B3, 0x39138, 0x39138, 0x3913D, 0x3913D, 0x39142, 0x39142, + 0x39147, 0x39147, 0x3914C, 0x3914C, 0x39151, 0x39151, 0x39156, 0x39156, + 0x3915B, 0x3915B, +}; + +struct gmu_mem_type_desc { + struct gmu_memdesc *memdesc; + uint32_t type; +}; + +static size_t a6xx_snapshot_gmu_mem(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_gmu_mem *mem_hdr = + (struct kgsl_snapshot_gmu_mem *)buf; + unsigned int *data = (unsigned int *) + (buf + sizeof(*mem_hdr)); + struct gmu_mem_type_desc *desc = priv; + + if (priv == NULL) + return 0; + + if (remain < desc->memdesc->size + sizeof(*mem_hdr)) { + dev_err(device->dev, + "snapshot: Not enough memory for the gmu section %d\n", + desc->type); + return 0; + } + + memset(mem_hdr, 0, sizeof(*mem_hdr)); + mem_hdr->type = desc->type; + mem_hdr->hostaddr = (uintptr_t)desc->memdesc->hostptr; + mem_hdr->gmuaddr = desc->memdesc->gmuaddr; + mem_hdr->gpuaddr = 0; + + /* Just copy the ringbuffer, there are no active IBs */ + memcpy(data, desc->memdesc->hostptr, desc->memdesc->size); + + return desc->memdesc->size + sizeof(*mem_hdr); +} + +static size_t a6xx_gmu_snapshot_dtcm(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_gmu_mem *mem_hdr = + (struct kgsl_snapshot_gmu_mem *)buf; + struct a6xx_gmu_device *gmu = (struct a6xx_gmu_device *)priv; + u32 *data = (u32 *)(buf + sizeof(*mem_hdr)); + u32 i; + + if (remain < gmu->vma[GMU_DTCM].size + sizeof(*mem_hdr)) { + SNAPSHOT_ERR_NOMEM(device, "GMU DTCM Memory"); + return 0; + } + + mem_hdr->type = SNAPSHOT_GMU_MEM_BIN_BLOCK; + mem_hdr->hostaddr = 0; + mem_hdr->gmuaddr = gmu->vma[GMU_DTCM].start; + mem_hdr->gpuaddr = 0; + + for (i = 0; i < (gmu->vma[GMU_DTCM].size >> 2); i++) + gmu_core_regread(device, A6XX_GMU_CM3_DTCM_START + i, data++); + + return gmu->vma[GMU_DTCM].size + sizeof(*mem_hdr); +} + +static size_t a6xx_gmu_snapshot_itcm(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_gmu_mem *mem_hdr = + (struct kgsl_snapshot_gmu_mem *)buf; + void *dest = buf + sizeof(*mem_hdr); + struct a6xx_gmu_device *gmu = (struct a6xx_gmu_device *)priv; + + if (!gmu->itcm_shadow) { + dev_err(&gmu->pdev->dev, "ITCM not captured\n"); + return 0; + } + + if (remain < gmu->vma[GMU_ITCM].size + sizeof(*mem_hdr)) { + SNAPSHOT_ERR_NOMEM(device, "GMU ITCM Memory"); + return 0; + } + + mem_hdr->type = SNAPSHOT_GMU_MEM_BIN_BLOCK; + mem_hdr->hostaddr = 0; + mem_hdr->gmuaddr = gmu->vma[GMU_ITCM].start; + mem_hdr->gpuaddr = 0; + + memcpy(dest, gmu->itcm_shadow, gmu->vma[GMU_ITCM].size); + + return gmu->vma[GMU_ITCM].size + sizeof(*mem_hdr); +} + +static void a6xx_gmu_snapshot_memories(struct kgsl_device *device, + struct a6xx_gmu_device *gmu, struct kgsl_snapshot *snapshot) +{ + struct gmu_mem_type_desc desc; + struct gmu_memdesc *md; + int i; + + for (i = 0; i < ARRAY_SIZE(gmu->gmu_globals); i++) { + + md = &gmu->gmu_globals[i]; + if (!md->size) + continue; + + desc.memdesc = md; + if (md == gmu->hfi.hfi_mem) + desc.type = SNAPSHOT_GMU_MEM_HFI; + else if (md == gmu->gmu_log) + desc.type = SNAPSHOT_GMU_MEM_LOG; + else if (md == gmu->dump_mem) + desc.type = SNAPSHOT_GMU_MEM_DEBUG; + else + desc.type = SNAPSHOT_GMU_MEM_BIN_BLOCK; + + kgsl_snapshot_add_section(device, + KGSL_SNAPSHOT_SECTION_GMU_MEMORY, + snapshot, a6xx_snapshot_gmu_mem, &desc); + } +} + +struct kgsl_snapshot_gmu_version { + uint32_t type; + uint32_t value; +}; + +static size_t a6xx_snapshot_gmu_version(struct kgsl_device *device, + u8 *buf, size_t remain, void *priv) +{ + struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf; + uint32_t *data = (uint32_t *) (buf + sizeof(*header)); + struct kgsl_snapshot_gmu_version *ver = priv; + + if (remain < DEBUG_SECTION_SZ(1)) { + SNAPSHOT_ERR_NOMEM(device, "GMU Version"); + return 0; + } + + header->type = ver->type; + header->size = 1; + + *data = ver->value; + + return DEBUG_SECTION_SZ(1); +} + +static void a6xx_gmu_snapshot_versions(struct kgsl_device *device, + struct a6xx_gmu_device *gmu, + struct kgsl_snapshot *snapshot) +{ + int i; + + struct kgsl_snapshot_gmu_version gmu_vers[] = { + { .type = SNAPSHOT_DEBUG_GMU_CORE_VERSION, + .value = gmu->ver.core, }, + { .type = SNAPSHOT_DEBUG_GMU_CORE_DEV_VERSION, + .value = gmu->ver.core_dev, }, + { .type = SNAPSHOT_DEBUG_GMU_PWR_VERSION, + .value = gmu->ver.pwr, }, + { .type = SNAPSHOT_DEBUG_GMU_PWR_DEV_VERSION, + .value = gmu->ver.pwr_dev, }, + { .type = SNAPSHOT_DEBUG_GMU_HFI_VERSION, + .value = gmu->ver.hfi, }, + }; + + for (i = 0; i < ARRAY_SIZE(gmu_vers); i++) + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, + snapshot, a6xx_snapshot_gmu_version, + &gmu_vers[i]); +} + +#define RSCC_OFFSET_DWORDS 0x38000 + +static size_t a6xx_snapshot_rscc_registers(struct kgsl_device *device, u8 *buf, + size_t remain, void *priv) +{ + struct kgsl_snapshot_regs *header = (struct kgsl_snapshot_regs *)buf; + struct kgsl_snapshot_registers *regs = priv; + unsigned int *data = (unsigned int *)(buf + sizeof(*header)); + int count = 0, j, k; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + + /* Figure out how many registers we are going to dump */ + for (j = 0; j < regs->count; j++) { + int start = regs->regs[j * 2]; + int end = regs->regs[j * 2 + 1]; + + count += (end - start + 1); + } + + if (remain < (count * 8) + sizeof(*header)) { + SNAPSHOT_ERR_NOMEM(device, "RSCC REGISTERS"); + return 0; + } + + for (j = 0; j < regs->count; j++) { + unsigned int start = regs->regs[j * 2]; + unsigned int end = regs->regs[j * 2 + 1]; + + for (k = start; k <= end; k++) { + unsigned int val; + + val = __raw_readl(gmu->rscc_virt + + ((k - RSCC_OFFSET_DWORDS) << 2)); + *data++ = k; + *data++ = val; + } + } + + header->count = count; + + /* Return the size of the section */ + return (count * 8) + sizeof(*header); +} + +static void snapshot_rscc_registers(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* RSCC registers are on cx */ + if (adreno_is_a650_family(adreno_dev)) { + struct kgsl_snapshot_registers r; + + r.regs = a650_rscc_registers; + r.count = ARRAY_SIZE(a650_rscc_registers) / 2; + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, + snapshot, a6xx_snapshot_rscc_registers, &r); + } else if (adreno_is_a615_family(adreno_dev) || + adreno_is_a630(adreno_dev)) { + adreno_snapshot_registers(device, snapshot, + a630_rscc_snapshot_registers, + ARRAY_SIZE(a630_rscc_snapshot_registers) / 2); + } else if (adreno_is_a640(adreno_dev) || adreno_is_a680(adreno_dev)) { + adreno_snapshot_registers(device, snapshot, + a6xx_rscc_snapshot_registers, + ARRAY_SIZE(a6xx_rscc_snapshot_registers) / 2); + } +} + +/* + * a6xx_gmu_device_snapshot() - A6XX GMU snapshot function + * @device: Device being snapshotted + * @snapshot: Pointer to the snapshot instance + * + * This is where all of the A6XX GMU specific bits and pieces are grabbed + * into the snapshot memory + */ +void a6xx_gmu_device_snapshot(struct kgsl_device *device, + struct kgsl_snapshot *snapshot) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + unsigned int val; + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GMU_MEMORY, + snapshot, a6xx_gmu_snapshot_itcm, gmu); + + kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_GMU_MEMORY, + snapshot, a6xx_gmu_snapshot_dtcm, gmu); + + a6xx_gmu_snapshot_versions(device, gmu, snapshot); + + a6xx_gmu_snapshot_memories(device, gmu, snapshot); + + /* Snapshot tcms as registers for legacy targets */ + if (adreno_is_a630(adreno_dev) || + adreno_is_a615_family(adreno_dev)) + adreno_snapshot_registers(device, snapshot, + a6xx_gmu_tcm_registers, + ARRAY_SIZE(a6xx_gmu_tcm_registers) / 2); + + adreno_snapshot_registers(device, snapshot, a6xx_gmu_registers, + ARRAY_SIZE(a6xx_gmu_registers) / 2); + + /* Snapshot A660 specific GMU registers */ + if (adreno_is_a660(adreno_dev)) + adreno_snapshot_registers(device, snapshot, a660_gmu_registers, + ARRAY_SIZE(a660_gmu_registers) / 2); + + snapshot_rscc_registers(adreno_dev, snapshot); + + if (!a6xx_gmu_gx_is_on(device)) + return; + + /* Set fence to ALLOW mode so registers can be read */ + kgsl_regwrite(device, A6XX_GMU_AO_AHB_FENCE_CTRL, 0); + /* Make sure the previous write posted before reading */ + wmb(); + kgsl_regread(device, A6XX_GMU_AO_AHB_FENCE_CTRL, &val); + + adreno_snapshot_registers(device, snapshot, + a6xx_gmu_gx_registers, + ARRAY_SIZE(a6xx_gmu_gx_registers) / 2); +} diff --git a/drivers/gpu/msm/kgsl_hfi.c b/drivers/gpu/msm/adreno_a6xx_hfi.c similarity index 62% rename from drivers/gpu/msm/kgsl_hfi.c rename to drivers/gpu/msm/adreno_a6xx_hfi.c index ff67032061eb..06bbe3561e04 100644 --- a/drivers/gpu/msm/kgsl_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hfi.c @@ -9,10 +9,22 @@ #include "adreno.h" #include "adreno_a6xx.h" #include "kgsl_device.h" -#include "kgsl_gmu.h" -#include "kgsl_hfi.h" #include "kgsl_trace.h" +/* Below section is for all structures related to HFI queues */ +#define HFI_QUEUE_DEFAULT_CNT 3 +#define HFI_QUEUE_DISPATCH_CNT 1 +#define HFI_QUEUE_MAX (HFI_QUEUE_DEFAULT_CNT + HFI_QUEUE_DISPATCH_CNT) + +struct hfi_queue_table { + struct hfi_queue_table_header qtbl_hdr; + struct hfi_queue_header qhdr[HFI_QUEUE_MAX]; +}; + +/* Total header sizes + queue sizes + 16 for alignment */ +#define HFIMEM_SIZE (sizeof(struct hfi_queue_table) + 16 + \ + (HFI_QUEUE_SIZE * HFI_QUEUE_MAX)) + #define HFI_QUEUE_OFFSET(i) \ (ALIGN(sizeof(struct hfi_queue_table), SZ_16) + \ ((i) * HFI_QUEUE_SIZE)) @@ -28,20 +40,14 @@ #define MSG_HDR_GET_TYPE(hdr) (((hdr) >> 16) & 0xF) #define MSG_HDR_GET_SEQNUM(hdr) (((hdr) >> 20) & 0xFFF) -/* Size is converted from Bytes to DWords */ -#define CREATE_MSG_HDR(id, size, type) \ - (((type) << 16) | ((((size) >> 2) & 0xFF) << 8) | ((id) & 0xFF)) -#define CMD_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_CMD) -#define ACK_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_ACK) - -static void hfi_process_queue(struct gmu_device *gmu, uint32_t queue_idx, - struct pending_cmd *ret_cmd); +static int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, + uint32_t queue_idx, struct pending_cmd *ret_cmd); /* Size in below functions are in unit of dwords */ -static int hfi_queue_read(struct gmu_device *gmu, uint32_t queue_idx, +static int a6xx_hfi_queue_read(struct a6xx_gmu_device *gmu, uint32_t queue_idx, unsigned int *output, unsigned int max_size) { - struct gmu_memdesc *mem_addr = gmu->hfi_mem; + struct gmu_memdesc *mem_addr = gmu->hfi.hfi_mem; struct hfi_queue_table *tbl = mem_addr->hostptr; struct hfi_queue_header *hdr = &tbl->qhdr[queue_idx]; uint32_t *queue; @@ -97,15 +103,16 @@ done: } /* Size in below functions are in unit of dwords */ -static int hfi_queue_write(struct gmu_device *gmu, uint32_t queue_idx, - uint32_t *msg) +static int a6xx_hfi_queue_write(struct adreno_device *adreno_dev, + uint32_t queue_idx, uint32_t *msg) { - struct hfi_queue_table *tbl = gmu->hfi_mem->hostptr; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct hfi_queue_table *tbl = gmu->hfi.hfi_mem->hostptr; struct hfi_queue_header *hdr = &tbl->qhdr[queue_idx]; uint32_t *queue; - struct kgsl_hfi *hfi = &gmu->hfi; uint32_t i, write, empty_space; uint32_t size = MSG_HDR_GET_SIZE(*msg); + u32 align_size = ALIGN(size, SZ_4); uint32_t id = MSG_HDR_GET_ID(*msg); if (hdr->status == HFI_QUEUE_STATUS_DISABLED) @@ -118,24 +125,16 @@ static int hfi_queue_write(struct gmu_device *gmu, uint32_t queue_idx, return -EINVAL; } - queue = HOST_QUEUE_START_ADDR(gmu->hfi_mem, queue_idx); + queue = HOST_QUEUE_START_ADDR(gmu->hfi.hfi_mem, queue_idx); trace_kgsl_hfi_send(id, size, MSG_HDR_GET_SEQNUM(*msg)); - mutex_lock(&hfi->cmdq_mutex); - empty_space = (hdr->write_index >= hdr->read_index) ? (hdr->queue_size - (hdr->write_index - hdr->read_index)) : (hdr->read_index - hdr->write_index); - if (empty_space < size) { - dev_err(&gmu->pdev->dev, - "Insufficient bufsize %d for msg id=%d of size %d\n", - empty_space, id, size); - - mutex_unlock(&hfi->cmdq_mutex); + if (empty_space <= align_size) return -ENOSPC; - } write = hdr->write_index; @@ -146,14 +145,14 @@ static int hfi_queue_write(struct gmu_device *gmu, uint32_t queue_idx, /* Cookify any non used data at the end of the write buffer */ if (GMU_VER_MAJOR(gmu->ver.hfi) >= 2) { - for (; write % 4; write = (write + 1) % hdr->queue_size) + for (; i < align_size; i++) { queue[write] = 0xFAFAFAFA; + write = (write + 1) % hdr->queue_size; + } } hdr->write_index = write; - mutex_unlock(&hfi->cmdq_mutex); - /* * Memory barrier to make sure packet and write index are written before * an interrupt is raised @@ -161,8 +160,8 @@ static int hfi_queue_write(struct gmu_device *gmu, uint32_t queue_idx, wmb(); /* Send interrupt to GMU to receive the message */ - adreno_write_gmureg(ADRENO_DEVICE(hfi->kgsldev), - ADRENO_REG_GMU_HOST2GMU_INTR_SET, 0x1); + gmu_core_regwrite(KGSL_DEVICE(adreno_dev), A6XX_GMU_HOST2GMU_INTR_SET, + 0x1); return 0; } @@ -173,11 +172,10 @@ static int hfi_queue_write(struct gmu_device *gmu, uint32_t queue_idx, /* Sizes of the queue and message are in unit of dwords */ -void hfi_init(struct gmu_device *gmu) +static void init_queues(struct adreno_device *adreno_dev) { - struct kgsl_hfi *hfi = &gmu->hfi; - struct adreno_device *adreno_dev = ADRENO_DEVICE(hfi->kgsldev); - struct gmu_memdesc *mem_addr = gmu->hfi_mem; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct gmu_memdesc *mem_addr = gmu->hfi.hfi_mem; int i; struct hfi_queue_table *tbl; struct hfi_queue_header *hdr; @@ -222,23 +220,38 @@ void hfi_init(struct gmu_device *gmu) hdr->status = queue[i].status; hdr->queue_size = HFI_QUEUE_SIZE >> 2; /* convert to dwords */ } +} - mutex_init(&hfi->cmdq_mutex); +int a6xx_hfi_init(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hfi *hfi = &gmu->hfi; + + /* Allocates & maps memory for HFI */ + if (IS_ERR_OR_NULL(hfi->hfi_mem)) { + hfi->hfi_mem = reserve_gmu_kernel_block(gmu, 0, HFIMEM_SIZE, + GMU_NONCACHED_KERNEL); + if (!IS_ERR(hfi->hfi_mem)) + init_queues(adreno_dev); + } + + return PTR_ERR_OR_ZERO(hfi->hfi_mem); } #define HDR_CMP_SEQNUM(out_hdr, in_hdr) \ (MSG_HDR_GET_SEQNUM(out_hdr) == MSG_HDR_GET_SEQNUM(in_hdr)) -static void receive_ack_cmd(struct gmu_device *gmu, void *rcvd, +static int receive_ack_cmd(struct a6xx_gmu_device *gmu, void *rcvd, struct pending_cmd *ret_cmd) { + struct adreno_device *adreno_dev = a6xx_gmu_to_adreno(gmu); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); uint32_t *ack = rcvd; uint32_t hdr = ack[0]; uint32_t req_hdr = ack[1]; - struct kgsl_hfi *hfi = &gmu->hfi; if (ret_cmd == NULL) - return; + return -EINVAL; trace_kgsl_hfi_receive(MSG_HDR_GET_ID(req_hdr), MSG_HDR_GET_SIZE(req_hdr), @@ -246,7 +259,7 @@ static void receive_ack_cmd(struct gmu_device *gmu, void *rcvd, if (HDR_CMP_SEQNUM(ret_cmd->sent_hdr, req_hdr)) { memcpy(&ret_cmd->results, ack, MSG_HDR_GET_SIZE(hdr) << 2); - return; + return 0; } /* Didn't find the sender, list the waiter */ @@ -254,36 +267,37 @@ static void receive_ack_cmd(struct gmu_device *gmu, void *rcvd, "HFI ACK: Cannot find sender for 0x%8.8x Waiter: 0x%8.8x\n", req_hdr, ret_cmd->sent_hdr); - adreno_set_gpu_fault(ADRENO_DEVICE(hfi->kgsldev), ADRENO_GMU_FAULT); - adreno_dispatcher_schedule(hfi->kgsldev); + gmu_fault_snapshot(device); + + return -ENODEV; } #define MSG_HDR_SET_SEQNUM(hdr, num) \ (((hdr) & 0xFFFFF) | ((num) << 20)) -static int poll_adreno_gmu_reg(struct adreno_device *adreno_dev, - enum adreno_regs offset_name, unsigned int expected_val, +static int poll_gmu_reg(struct adreno_device *adreno_dev, + u32 offsetdwords, unsigned int expected_val, unsigned int mask, unsigned int timeout_ms) { unsigned int val; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); u64 ao_pre_poll, ao_post_poll; - ao_pre_poll = gmu_core_dev_read_alwayson(device); + ao_pre_poll = a6xx_read_alwayson(adreno_dev); while (time_is_after_jiffies(timeout)) { - adreno_read_gmureg(adreno_dev, offset_name, &val); + gmu_core_regread(device, offsetdwords, &val); if ((val & mask) == expected_val) return 0; usleep_range(10, 100); } - ao_post_poll = gmu_core_dev_read_alwayson(device); + ao_post_poll = a6xx_read_alwayson(adreno_dev); /* Check one last time */ - adreno_read_gmureg(adreno_dev, offset_name, &val); + gmu_core_regread(device, offsetdwords, &val); if ((val & mask) == expected_val) return 0; @@ -293,29 +307,31 @@ static int poll_adreno_gmu_reg(struct adreno_device *adreno_dev, return -ETIMEDOUT; } -static int hfi_send_cmd(struct gmu_device *gmu, uint32_t queue_idx, - void *data, struct pending_cmd *ret_cmd) +static int a6xx_hfi_send_cmd(struct adreno_device *adreno_dev, + uint32_t queue_idx, void *data, struct pending_cmd *ret_cmd) { + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); int rc; uint32_t *cmd = data; - struct kgsl_hfi *hfi = &gmu->hfi; + struct a6xx_hfi *hfi = &gmu->hfi; unsigned int seqnum = atomic_inc_return(&hfi->seqnum); - struct adreno_device *adreno_dev = ADRENO_DEVICE(hfi->kgsldev); *cmd = MSG_HDR_SET_SEQNUM(*cmd, seqnum); if (ret_cmd == NULL) - return hfi_queue_write(gmu, queue_idx, cmd); + return a6xx_hfi_queue_write(adreno_dev, queue_idx, cmd); ret_cmd->sent_hdr = cmd[0]; - rc = hfi_queue_write(gmu, queue_idx, cmd); + rc = a6xx_hfi_queue_write(adreno_dev, queue_idx, cmd); if (rc) return rc; - rc = poll_adreno_gmu_reg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_INFO, + rc = poll_gmu_reg(adreno_dev, A6XX_GMU_GMU2HOST_INTR_INFO, HFI_IRQ_MSGQ_MASK, HFI_IRQ_MSGQ_MASK, HFI_RSP_TIMEOUT); if (rc) { + gmu_fault_snapshot(device); dev_err(&gmu->pdev->dev, "Timed out waiting on ack for 0x%8.8x (id %d, sequence %d)\n", cmd[0], MSG_HDR_GET_ID(*cmd), MSG_HDR_GET_SEQNUM(*cmd)); @@ -323,27 +339,31 @@ static int hfi_send_cmd(struct gmu_device *gmu, uint32_t queue_idx, } /* Clear the interrupt */ - adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_CLR, + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, HFI_IRQ_MSGQ_MASK); - hfi_process_queue(gmu, HFI_MSG_ID, ret_cmd); + rc = a6xx_hfi_process_queue(gmu, HFI_MSG_ID, ret_cmd); return rc; } #define HFI_ACK_ERROR 0xffffffff -static int hfi_send_generic_req(struct gmu_device *gmu, uint32_t queue, - void *cmd) +static int a6xx_hfi_send_generic_req(struct adreno_device *adreno_dev, + uint32_t queue, void *cmd) { struct pending_cmd ret_cmd; int rc; memset(&ret_cmd, 0, sizeof(ret_cmd)); - rc = hfi_send_cmd(gmu, queue, cmd, &ret_cmd); + rc = a6xx_hfi_send_cmd(adreno_dev, queue, cmd, &ret_cmd); if (!rc && ret_cmd.results[2] == HFI_ACK_ERROR) { + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + gmu_fault_snapshot(device); dev_err(&gmu->pdev->dev, "HFI ACK failure: Req 0x%8.8X\n", ret_cmd.results[1]); return -EINVAL; @@ -352,22 +372,24 @@ static int hfi_send_generic_req(struct gmu_device *gmu, uint32_t queue, return rc; } -static int hfi_send_gmu_init(struct gmu_device *gmu, uint32_t boot_state) +static int a6xx_hfi_send_gmu_init(struct adreno_device *adreno_dev) { + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_gmu_init_cmd cmd = { .hdr = CMD_MSG_HDR(H2F_MSG_INIT, sizeof(cmd)), .seg_id = 0, .dbg_buffer_addr = (unsigned int) gmu->dump_mem->gmuaddr, .dbg_buffer_size = (unsigned int) gmu->dump_mem->size, - .boot_state = boot_state, + .boot_state = 0x1, }; - return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); } -static int hfi_get_fw_version(struct gmu_device *gmu, +static int a6xx_hfi_get_fw_version(struct adreno_device *adreno_dev, uint32_t expected_ver, uint32_t *ver) { + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_fw_version_cmd cmd = { .hdr = CMD_MSG_HDR(H2F_MSG_FW_VER, sizeof(cmd)), .supported_ver = expected_ver, @@ -377,7 +399,7 @@ static int hfi_get_fw_version(struct gmu_device *gmu, memset(&ret_cmd, 0, sizeof(ret_cmd)); - rc = hfi_send_cmd(gmu, HFI_CMD_ID, &cmd, &ret_cmd); + rc = a6xx_hfi_send_cmd(adreno_dev, HFI_CMD_ID, &cmd, &ret_cmd); if (rc) return rc; @@ -391,33 +413,34 @@ static int hfi_get_fw_version(struct gmu_device *gmu, return rc; } -static int hfi_send_core_fw_start(struct gmu_device *gmu) +static int a6xx_hfi_send_core_fw_start(struct adreno_device *adreno_dev) { struct hfi_core_fw_start_cmd cmd = { .hdr = CMD_MSG_HDR(H2F_MSG_CORE_FW_START, sizeof(cmd)), .handle = 0x0, }; - return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); } -static const char * const hfi_features[] = { - [HFI_FEATURE_ECP] = "ECP", +static const char * const a6xx_hfi_features[] = { [HFI_FEATURE_ACD] = "ACD", [HFI_FEATURE_LM] = "LM", }; static const char *feature_to_string(uint32_t feature) { - if (feature < ARRAY_SIZE(hfi_features) && hfi_features[feature]) - return hfi_features[feature]; + if (feature < ARRAY_SIZE(a6xx_hfi_features) && + a6xx_hfi_features[feature]) + return a6xx_hfi_features[feature]; return "unknown"; } -static int hfi_send_feature_ctrl(struct gmu_device *gmu, - uint32_t feature, uint32_t enable, uint32_t data) +static int a6xx_hfi_send_feature_ctrl(struct adreno_device *adreno_dev, + uint32_t feature, uint32_t enable, uint32_t data) { + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); struct hfi_feature_ctrl_cmd cmd = { .hdr = CMD_MSG_HDR(H2F_MSG_FEATURE_CTRL, sizeof(cmd)), .feature = feature, @@ -426,7 +449,7 @@ static int hfi_send_feature_ctrl(struct gmu_device *gmu, }; int ret; - ret = hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd); + ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); if (ret) dev_err(&gmu->pdev->dev, "Unable to %s feature %s (%d)\n", @@ -436,30 +459,31 @@ static int hfi_send_feature_ctrl(struct gmu_device *gmu, return ret; } -static int hfi_send_dcvstbl_v1(struct gmu_device *gmu) +static int a6xx_hfi_send_dcvstbl_v1(struct adreno_device *adreno_dev) { + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct hfi_dcvstable_cmd *table = &gmu->hfi.dcvs_table; struct hfi_dcvstable_v1_cmd cmd = { .hdr = CMD_MSG_HDR(H2F_MSG_PERF_TBL, sizeof(cmd)), - .gpu_level_num = gmu->num_gpupwrlevels, - .gmu_level_num = GMU_PWR_LEVELS, + .gpu_level_num = table->gpu_level_num, + .gmu_level_num = table->gmu_level_num, }; int i; - for (i = 0; i < gmu->num_gpupwrlevels; i++) { - cmd.gx_votes[i].vote = gmu->rpmh_votes.gx_votes[i]; - /* Divide by 1000 to convert to kHz */ - cmd.gx_votes[i].freq = gmu->pwrlevels[i].freq / 1000; + for (i = 0; i < table->gpu_level_num; i++) { + cmd.gx_votes[i].vote = table->gx_votes[i].vote; + cmd.gx_votes[i].freq = table->gx_votes[i].freq; } - cmd.cx_votes[0].vote = gmu->rpmh_votes.cx_votes[0]; - cmd.cx_votes[0].freq = 0; - cmd.cx_votes[1].vote = gmu->rpmh_votes.cx_votes[1]; - cmd.cx_votes[1].freq = GMU_FREQUENCY / 1000; + cmd.cx_votes[0].vote = table->cx_votes[0].vote; + cmd.cx_votes[0].freq = table->cx_votes[0].freq; + cmd.cx_votes[1].vote = table->cx_votes[1].vote; + cmd.cx_votes[1].freq = table->cx_votes[1].freq; - return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); } -static int hfi_send_get_value(struct gmu_device *gmu, +static int a6xx_hfi_send_get_value(struct adreno_device *adreno_dev, struct hfi_get_value_req *req) { struct hfi_get_value_cmd *cmd = &req->cmd; @@ -470,7 +494,7 @@ static int hfi_send_get_value(struct gmu_device *gmu, cmd->hdr = CMD_MSG_HDR(H2F_MSG_GET_VALUE, sizeof(*cmd)); - rc = hfi_send_cmd(gmu, HFI_CMD_ID, cmd, &ret_cmd); + rc = a6xx_hfi_send_cmd(adreno_dev, HFI_CMD_ID, cmd, &ret_cmd); if (rc) return rc; @@ -480,59 +504,16 @@ static int hfi_send_get_value(struct gmu_device *gmu, return 0; } -static int hfi_send_dcvstbl(struct gmu_device *gmu) -{ - struct hfi_dcvstable_cmd cmd = { - .hdr = CMD_MSG_HDR(H2F_MSG_PERF_TBL, sizeof(cmd)), - .gpu_level_num = gmu->num_gpupwrlevels, - .gmu_level_num = GMU_PWR_LEVELS, - }; - int i; - - for (i = 0; i < gmu->num_gpupwrlevels; i++) { - cmd.gx_votes[i].vote = gmu->rpmh_votes.gx_votes[i]; - /* Hardcode this to the max threshold since it is not used */ - cmd.gx_votes[i].acd = 0xFFFFFFFF; - /* Divide by 1000 to convert to kHz */ - cmd.gx_votes[i].freq = gmu->pwrlevels[i].freq / 1000; - } - - cmd.cx_votes[0].vote = gmu->rpmh_votes.cx_votes[0]; - cmd.cx_votes[0].freq = 0; - cmd.cx_votes[1].vote = gmu->rpmh_votes.cx_votes[1]; - cmd.cx_votes[1].freq = GMU_FREQUENCY / 1000; - - return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd); -} - -static int hfi_send_bwtbl(struct gmu_device *gmu) -{ - struct hfi_bwtable_cmd *cmd = &gmu->hfi.bwtbl_cmd; - - cmd->hdr = CMD_MSG_HDR(H2F_MSG_BW_VOTE_TBL, sizeof(*cmd)); - - return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd); -} - -static int hfi_send_acd_tbl(struct gmu_device *gmu) -{ - struct hfi_acd_table_cmd *cmd = &gmu->hfi.acd_tbl_cmd; - - cmd->hdr = CMD_MSG_HDR(H2F_MSG_ACD_TBL, sizeof(*cmd)); - - return hfi_send_generic_req(gmu, HFI_CMD_IDX, cmd); -} - -static int hfi_send_test(struct gmu_device *gmu) +static int a6xx_hfi_send_test(struct adreno_device *adreno_dev) { struct hfi_test_cmd cmd = { .hdr = CMD_MSG_HDR(H2F_MSG_TEST, sizeof(cmd)), }; - return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, &cmd); } -static void receive_err_req(struct gmu_device *gmu, void *rcvd) +static void receive_err_req(struct a6xx_gmu_device *gmu, void *rcvd) { struct hfi_err_cmd *cmd = rcvd; @@ -542,7 +523,7 @@ static void receive_err_req(struct gmu_device *gmu, void *rcvd) (char *) cmd->data); } -static void receive_debug_req(struct gmu_device *gmu, void *rcvd) +static void receive_debug_req(struct a6xx_gmu_device *gmu, void *rcvd) { struct hfi_debug_cmd *cmd = rcvd; @@ -550,7 +531,7 @@ static void receive_debug_req(struct gmu_device *gmu, void *rcvd) cmd->type, cmd->timestamp, cmd->data); } -static void hfi_v1_receiver(struct gmu_device *gmu, uint32_t *rcvd, +static void a6xx_hfi_v1_receiver(struct a6xx_gmu_device *gmu, uint32_t *rcvd, struct pending_cmd *ret_cmd) { /* V1 ACK Handler */ @@ -575,21 +556,24 @@ static void hfi_v1_receiver(struct gmu_device *gmu, uint32_t *rcvd, } } -static void hfi_process_queue(struct gmu_device *gmu, uint32_t queue_idx, - struct pending_cmd *ret_cmd) +static int a6xx_hfi_process_queue(struct a6xx_gmu_device *gmu, + uint32_t queue_idx, struct pending_cmd *ret_cmd) { uint32_t rcvd[MAX_RCVD_SIZE]; - while (hfi_queue_read(gmu, queue_idx, rcvd, sizeof(rcvd)) > 0) { + while (a6xx_hfi_queue_read(gmu, queue_idx, rcvd, sizeof(rcvd)) > 0) { /* Special case if we're v1 */ if (GMU_VER_MAJOR(gmu->ver.hfi) < 2) { - hfi_v1_receiver(gmu, rcvd, ret_cmd); + a6xx_hfi_v1_receiver(gmu, rcvd, ret_cmd); continue; } /* V2 ACK Handler */ if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) { - receive_ack_cmd(gmu, rcvd, ret_cmd); + int ret = receive_ack_cmd(gmu, rcvd, ret_cmd); + + if (ret) + return ret; continue; } @@ -608,18 +592,13 @@ static void hfi_process_queue(struct gmu_device *gmu, uint32_t queue_idx, break; } } + + return 0; } -void hfi_receiver(unsigned long data) +static int a6xx_hfi_verify_fw_version(struct adreno_device *adreno_dev) { - /* Process all asynchronous read (firmware to host) queues */ - hfi_process_queue((struct gmu_device *) data, HFI_DBG_ID, NULL); -} - -static int hfi_verify_fw_version(struct kgsl_device *device, - struct gmu_device *gmu) -{ - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); int result; unsigned int ver, major, minor; @@ -631,7 +610,8 @@ static int hfi_verify_fw_version(struct kgsl_device *device, major = a6xx_core->gmu_major; minor = a6xx_core->gmu_minor; - result = hfi_get_fw_version(gmu, GMU_VERSION(major, minor), &ver); + result = a6xx_hfi_get_fw_version(adreno_dev, GMU_VERSION(major, minor), + &ver); if (result) { dev_err_once(&gmu->pdev->dev, "Failed to get FW version via HFI\n"); @@ -655,8 +635,7 @@ static int hfi_verify_fw_version(struct kgsl_device *device, return 0; } -static int hfi_send_lm_feature_ctrl(struct gmu_device *gmu, - struct adreno_device *adreno_dev) +static int a6xx_hfi_send_lm_feature_ctrl(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct hfi_set_value_cmd req; @@ -674,41 +653,40 @@ static int hfi_send_lm_feature_ctrl(struct gmu_device *gmu, req.subtype = 0; req.data = slope; - ret = hfi_send_feature_ctrl(gmu, HFI_FEATURE_LM, 1, + ret = a6xx_hfi_send_feature_ctrl(adreno_dev, HFI_FEATURE_LM, 1, device->pwrctrl.throttle_mask); if (!ret) - ret = hfi_send_req(gmu, H2F_MSG_SET_VALUE, &req); + ret = a6xx_hfi_send_req(adreno_dev, H2F_MSG_SET_VALUE, &req); return ret; } -static int hfi_send_acd_feature_ctrl(struct gmu_device *gmu, - struct adreno_device *adreno_dev) +static int a6xx_hfi_send_acd_feature_ctrl(struct adreno_device *adreno_dev) { + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); int ret = 0; if (adreno_dev->acd_enabled) { - ret = hfi_send_acd_tbl(gmu); + ret = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, + &gmu->hfi.acd_table); if (!ret) - ret = hfi_send_feature_ctrl(gmu, HFI_FEATURE_ACD, 1, 0); + ret = a6xx_hfi_send_feature_ctrl(adreno_dev, + HFI_FEATURE_ACD, 1, 0); } return ret; } -int hfi_start(struct kgsl_device *device, - struct gmu_device *gmu, uint32_t boot_state) +int a6xx_hfi_start(struct adreno_device *adreno_dev) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_memdesc *mem_addr = gmu->hfi_mem; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct gmu_memdesc *mem_addr = gmu->hfi.hfi_mem; struct hfi_queue_table *tbl = mem_addr->hostptr; struct hfi_queue_header *hdr; int result, i; - if (test_bit(GMU_HFI_ON, &device->gmu_core.flags)) - return 0; - /* Force read_index to the write_index no matter what */ for (i = 0; i < HFI_QUEUE_MAX; i++) { hdr = &tbl->qhdr[i]; @@ -725,25 +703,27 @@ int hfi_start(struct kgsl_device *device, /* This is legacy HFI message for A630 and A615 family firmware */ if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) { - result = hfi_send_gmu_init(gmu, boot_state); + result = a6xx_hfi_send_gmu_init(adreno_dev); if (result) - return result; + goto err; } - result = hfi_verify_fw_version(device, gmu); + result = a6xx_hfi_verify_fw_version(adreno_dev); if (result) - return result; + goto err; if (GMU_VER_MAJOR(gmu->ver.hfi) < 2) - result = hfi_send_dcvstbl_v1(gmu); + result = a6xx_hfi_send_dcvstbl_v1(adreno_dev); else - result = hfi_send_dcvstbl(gmu); + result = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, + &gmu->hfi.dcvs_table); if (result) - return result; + goto err; - result = hfi_send_bwtbl(gmu); + result = a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, + &gmu->hfi.bw_table); if (result) - return result; + goto err; /* * If quirk is enabled send H2F_MSG_TEST and tell the GMU @@ -751,48 +731,52 @@ int hfi_start(struct kgsl_device *device, * send H2F_MSG_CORE_FW_START and features for A640 devices */ if (GMU_VER_MAJOR(gmu->ver.hfi) >= 2) { - if (ADRENO_FEATURE(adreno_dev, ADRENO_ECP)) { - result = hfi_send_feature_ctrl(gmu, - HFI_FEATURE_ECP, 1, 0); - if (result) - return result; - } - - result = hfi_send_acd_feature_ctrl(gmu, adreno_dev); + result = a6xx_hfi_send_acd_feature_ctrl(adreno_dev); if (result) - return result; + goto err; - result = hfi_send_lm_feature_ctrl(gmu, adreno_dev); + result = a6xx_hfi_send_lm_feature_ctrl(adreno_dev); if (result) - return result; + goto err; - result = hfi_send_core_fw_start(gmu); + result = a6xx_hfi_send_core_fw_start(adreno_dev); if (result) - return result; + goto err; } else { if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) { - result = hfi_send_test(gmu); + result = a6xx_hfi_send_test(adreno_dev); if (result) - return result; + goto err; } } - set_bit(GMU_HFI_ON, &device->gmu_core.flags); - return 0; + + set_bit(GMU_PRIV_HFI_STARTED, &gmu->flags); + + /* Request default DCVS level */ + result = kgsl_pwrctrl_set_default_gpu_pwrlevel(device); + if (result) + goto err; + + /* Request default BW vote */ + result = kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_ON); + +err: + if (result) + a6xx_hfi_stop(adreno_dev); + + return result; + } -void hfi_stop(struct gmu_device *gmu) +void a6xx_hfi_stop(struct adreno_device *adreno_dev) { - struct gmu_memdesc *mem_addr = gmu->hfi_mem; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct gmu_memdesc *mem_addr = gmu->hfi.hfi_mem; struct hfi_queue_table *tbl = mem_addr->hostptr; struct hfi_queue_header *hdr; - struct kgsl_hfi *hfi = &gmu->hfi; - struct kgsl_device *device = hfi->kgsldev; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); unsigned int i; - - if (!test_bit(GMU_HFI_ON, &device->gmu_core.flags)) - return; - /* Flush HFI queues */ for (i = 0; i < HFI_QUEUE_MAX; i++) { hdr = &tbl->qhdr[i]; @@ -805,11 +789,14 @@ void hfi_stop(struct gmu_device *gmu) i, hdr->read_index, hdr->write_index); } - clear_bit(GMU_HFI_ON, &device->gmu_core.flags); + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_OFF); + + clear_bit(GMU_PRIV_HFI_STARTED, &gmu->flags); + } -/* Entry point for external HFI requests */ -int hfi_send_req(struct gmu_device *gmu, unsigned int id, void *data) +int a6xx_hfi_send_req(struct adreno_device *adreno_dev, unsigned int id, + void *data) { switch (id) { case H2F_MSG_GX_BW_PERF_VOTE: { @@ -817,7 +804,7 @@ int hfi_send_req(struct gmu_device *gmu, unsigned int id, void *data) cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); } case H2F_MSG_PREPARE_SLUMBER: { struct hfi_prep_slumber_cmd *cmd = data; @@ -827,24 +814,24 @@ int hfi_send_req(struct gmu_device *gmu, unsigned int id, void *data) cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); } case H2F_MSG_START: { struct hfi_start_cmd *cmd = data; cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); } case H2F_MSG_GET_VALUE: { - return hfi_send_get_value(gmu, data); + return a6xx_hfi_send_get_value(adreno_dev, data); } case H2F_MSG_SET_VALUE: { struct hfi_set_value_cmd *cmd = data; cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd)); - return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd); + return a6xx_hfi_send_generic_req(adreno_dev, HFI_CMD_ID, cmd); } default: break; @@ -854,20 +841,17 @@ int hfi_send_req(struct gmu_device *gmu, unsigned int id, void *data) } /* HFI interrupt handler */ -irqreturn_t hfi_irq_handler(int irq, void *data) +irqreturn_t a6xx_hfi_irq_handler(int irq, void *data) { struct kgsl_device *device = data; - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - struct kgsl_hfi *hfi = &gmu->hfi; + struct a6xx_gmu_device *gmu = to_a6xx_gmu(ADRENO_DEVICE(device)); unsigned int status = 0; - adreno_read_gmureg(ADRENO_DEVICE(device), - ADRENO_REG_GMU_GMU2HOST_INTR_INFO, &status); - adreno_write_gmureg(ADRENO_DEVICE(device), - ADRENO_REG_GMU_GMU2HOST_INTR_CLR, HFI_IRQ_MASK); + gmu_core_regread(device, A6XX_GMU_GMU2HOST_INTR_INFO, &status); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, HFI_IRQ_MASK); if (status & HFI_IRQ_DBGQ_MASK) - tasklet_hi_schedule(&hfi->tasklet); + a6xx_hfi_process_queue(gmu, HFI_DBG_ID, NULL); if (status & HFI_IRQ_CM3_FAULT_MASK) { dev_err_ratelimited(&gmu->pdev->dev, "GMU CM3 fault interrupt received\n"); diff --git a/drivers/gpu/msm/kgsl_hfi.h b/drivers/gpu/msm/adreno_a6xx_hfi.h similarity index 65% rename from drivers/gpu/msm/kgsl_hfi.h rename to drivers/gpu/msm/adreno_a6xx_hfi.h index a6d054be7e4c..b56efb70a9dc 100644 --- a/drivers/gpu/msm/kgsl_hfi.h +++ b/drivers/gpu/msm/adreno_a6xx_hfi.h @@ -1,26 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. */ -#ifndef __KGSL_HFI_H -#define __KGSL_HFI_H +#ifndef __ADRENO_A6XX_HFI_H +#define __ADRENO_A6XX_HFI_H #define HFI_QUEUE_SIZE SZ_4K /* bytes, must be base 4dw */ #define MAX_RCVD_PAYLOAD_SIZE 16 /* dwords */ #define MAX_RCVD_SIZE (MAX_RCVD_PAYLOAD_SIZE + 3) /* dwords */ #define HFI_MAX_MSG_SIZE (SZ_1K>>2) /* dwords */ -/* Below section is for all structures related to HFI queues */ -#define HFI_QUEUE_DEFAULT_CNT 3 -#define HFI_QUEUE_DISPATCH_CNT 1 -#define HFI_QUEUE_MAX (HFI_QUEUE_DEFAULT_CNT + HFI_QUEUE_DISPATCH_CNT) - -struct hfi_queue_table; - -/* Total header sizes + queue sizes + 16 for alignment */ -#define HFIMEM_SIZE (sizeof(struct hfi_queue_table) + 16 + \ - (HFI_QUEUE_SIZE * HFI_QUEUE_MAX)) - #define HFI_CMD_ID 0 #define HFI_MSG_ID 1 #define HFI_DBG_ID 2 @@ -47,7 +36,6 @@ struct hfi_queue_table; #define HFI_DSP_PRI_0 20 #define HFI_RSP_TIMEOUT 100 /* msec */ -#define HFI_H2F_CMD_IRQ_MASK BIT(0) #define HFI_IRQ_MSGQ_MASK BIT(0) #define HFI_IRQ_SIDEMSGQ_MASK BIT(1) @@ -58,16 +46,11 @@ struct hfi_queue_table; HFI_IRQ_DBGQ_MASK |\ HFI_IRQ_CM3_FAULT_MASK) -#define CLKSET_OPTION_DEFAULT 0 -#define CLKSET_OPTION_CLOSEST 1 -#define CLKSET_OPTION_ATMOST 2 -#define CLKSET_OPTION_ATLEAST 3 - #define DCVS_ACK_NONBLOCK 0 #define DCVS_ACK_BLOCK 1 #define HFI_FEATURE_DCVS 0 -#define HFI_FEATURE_ECP 1 +#define HFI_FEATURE_HWSCHED 1 #define HFI_FEATURE_PREEMPTION 2 #define HFI_FEATURE_CLOCKS_ON 3 #define HFI_FEATURE_BUS_ON 4 @@ -144,17 +127,16 @@ struct hfi_queue_header { uint32_t write_index; }; -struct hfi_queue_table { - struct hfi_queue_table_header qtbl_hdr; - struct hfi_queue_header qhdr[HFI_QUEUE_MAX]; -}; +#define HFI_MSG_CMD 0 /* V1 and V2 */ +#define HFI_MSG_ACK 1 /* V2 only */ +#define HFI_V1_MSG_POST 1 /* V1 only */ +#define HFI_V1_MSG_ACK 2/* V1 only */ -enum hfi_msg_type { - HFI_MSG_CMD = 0, /* V1 and V2 */ - HFI_MSG_ACK = 1, /* V2 only */ - HFI_V1_MSG_POST = 1, /* V1 only */ - HFI_V1_MSG_ACK = 2, /* V1 only */ -}; +/* Size is converted from Bytes to DWords */ +#define CREATE_MSG_HDR(id, size, type) \ + (((type) << 16) | ((((size) >> 2) & 0xFF) << 8) | ((id) & 0xFF)) +#define CMD_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_CMD) +#define ACK_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_ACK) #define H2F_MSG_INIT 0 #define H2F_MSG_FW_VER 1 @@ -174,6 +156,7 @@ enum hfi_msg_type { #define H2F_MSG_PREPARE_SLUMBER 33 #define F2H_MSG_ERR 100 #define F2H_MSG_DEBUG 101 +#define F2H_MSG_LOG_BLOCK 102 #define F2H_MSG_GMU_CNTR_REGISTER 110 #define F2H_MSG_GMU_CNTR_RELEASE 111 #define F2H_MSG_ACK 126 /* Deprecated for v2.0*/ @@ -195,20 +178,13 @@ struct hfi_gmu_init_cmd { uint32_t dbg_buffer_addr; uint32_t dbg_buffer_size; uint32_t boot_state; -}; +} __packed; /* H2F */ struct hfi_fw_version_cmd { uint32_t hdr; uint32_t supported_ver; -}; - -#define ARC_VOTE_GET_PRI(_v) ((_v) & 0xFF) -#define ARC_VOTE_GET_SEC(_v) (((_v) >> 8) & 0xFF) -#define ARC_VOTE_GET_VLVL(_v) (((_v) >> 16) & 0xFFFF) - -#define ARC_VOTE_SET(pri, sec, vlvl) \ - ((((vlvl) & 0xFFFF) << 16) | (((sec) & 0xFF) << 8) | ((pri) & 0xFF)) +} __packed; /* H2F */ struct hfi_bwtable_cmd { @@ -222,7 +198,7 @@ struct hfi_bwtable_cmd { uint32_t cnoc_cmd_data[MAX_CNOC_LEVELS][MAX_CNOC_CMDS]; uint32_t ddr_cmd_addrs[MAX_BW_CMDS]; uint32_t ddr_cmd_data[MAX_GX_LEVELS][MAX_BW_CMDS]; -}; +} __packed; struct opp_gx_desc { uint32_t vote; @@ -242,7 +218,7 @@ struct hfi_dcvstable_v1_cmd { uint32_t gmu_level_num; struct opp_desc gx_votes[MAX_GX_LEVELS]; struct opp_desc cx_votes[MAX_CX_LEVELS]; -}; +} __packed; /* H2F */ struct hfi_dcvstable_cmd { @@ -251,9 +227,8 @@ struct hfi_dcvstable_cmd { uint32_t gmu_level_num; struct opp_gx_desc gx_votes[MAX_GX_LEVELS]; struct opp_desc cx_votes[MAX_CX_LEVELS]; -}; +} __packed; -#define HFI_ACD_INIT_VERSION 1 #define MAX_ACD_STRIDE 2 #define MAX_ACD_NUM_LEVELS 6 @@ -265,18 +240,18 @@ struct hfi_acd_table_cmd { uint32_t stride; uint32_t num_levels; uint32_t data[MAX_ACD_NUM_LEVELS * MAX_ACD_STRIDE]; -}; +} __packed; /* H2F */ struct hfi_test_cmd { uint32_t hdr; uint32_t data; -}; +} __packed; /* H2F */ struct hfi_start_cmd { uint32_t hdr; -}; +} __packed; /* H2F */ struct hfi_feature_ctrl_cmd { @@ -284,27 +259,27 @@ struct hfi_feature_ctrl_cmd { uint32_t feature; uint32_t enable; uint32_t data; -}; +} __packed; /* H2F */ struct hfi_get_value_cmd { uint32_t hdr; uint32_t type; uint32_t subtype; -}; +} __packed; /* Internal */ struct hfi_get_value_req { struct hfi_get_value_cmd cmd; uint32_t data[16]; -}; +} __packed; /* F2H */ struct hfi_get_value_reply_cmd { uint32_t hdr; uint32_t req_hdr; uint32_t data[16]; -}; +} __packed; /* H2F */ struct hfi_set_value_cmd { @@ -312,53 +287,13 @@ struct hfi_set_value_cmd { uint32_t type; uint32_t subtype; uint32_t data; -}; +} __packed; /* H2F */ struct hfi_core_fw_start_cmd { uint32_t hdr; uint32_t handle; -}; - -/* CP/GFX pipeline can access, The mem_kind may imply restrictions for non-CP */ -#define MEMFLAG_GFX_ACC BIT(0) -/* Buffer has APRIV protection in GFX PTEs */ -#define MEMFLAG_GFX_PRIV BIT(1) -/* Buffer is read-write for GFX PTEs. A 0 indicates read-only */ -#define MEMFLAG_GFX_WRITEABLE BIT(2) -/* GMU can access */ -#define MEMFLAG_GMU_ACC BIT(3) -/* Buffer has APRIV protection in GMU PTEs */ -#define MEMFLAG_GMU_PRIV BIT(4) -/* Buffer is read-write for GMU PTEs. A 0 indicates read-only */ -#define MEMFLAG_GMU_WRITEABLE BIT(5) -/* Buffer is located in GMU's non-cached bufferable VA range */ -#define MEMFLAG_GMU_BUFFERABLE BIT(6) -/* Buffer is located in GMU's cacheable VA range */ -#define MEMFLAG_GMU_CACHEABLE BIT(7) -/* Host can access */ -#define MEMFLAG_HOST_ACC BIT(8) -/* - * Request that Host initialize the buffer. - * Implies zero-init, unless Memkind implies otherwise - */ -#define MEMFLAG_HOST_INIT BIT(9) - -#define HFI_MEMKIND_GENERIC 0 -#define HFI_MEMKIND_RB 1 -#define HFI_MEMKIND_MEMSTORE 2 -#define HFI_MEMKIND_CSW_SMMU_INFO 3 -#define HFI_MEMKIND_CSW_PRIV_NON_SECURE 4 -#define HFI_MEMKIND_CSW_PRIV_SECURE 5 -#define HFI_MEMKIND_CSW_NON_PRIV 6 -#define HFI_MEMKIND_CSW_COUNTER 7 -#define HFI_MEMKIND_CTXTREC_PERF_CNTR_SAVE_RESTORE 8 -#define HFI_MEMKIND_CTXTREC_PREEMPT_CNTR 9 -#define HFI_MEMKIND_SYS_LOG 10 -#define HFI_MEMKIND_CRASH_DUMP 11 -#define HFI_MEMKIND_MMIO_DPU 12 -#define HFI_MEMKIND_MMIO_TCSR 13 -#define HFI_MEMKIND_MMIO_QDSS_STM 14 +} __packed; struct hfi_mem_alloc_desc { uint64_t gpu_addr; @@ -368,21 +303,21 @@ struct hfi_mem_alloc_desc { uint32_t gmu_mem_handle; uint32_t gmu_addr; uint32_t size; /* Bytes */ -}; +} __packed; /* F2H */ struct hfi_mem_alloc_cmd { uint32_t hdr; uint32_t reserved; /* Padding to ensure alignment of 'desc' below */ struct hfi_mem_alloc_desc desc; -}; +} __packed; /* H2F */ struct hfi_mem_alloc_reply_cmd { uint32_t hdr; uint32_t req_hdr; struct hfi_mem_alloc_desc desc; -}; +} __packed; /* H2F */ struct hfi_gx_bw_perf_vote_cmd { @@ -390,27 +325,27 @@ struct hfi_gx_bw_perf_vote_cmd { uint32_t ack_type; uint32_t freq; uint32_t bw; -}; +} __packed; /* H2F */ struct hfi_fw_halt_cmd { uint32_t hdr; uint32_t en_halt; -}; +} __packed; /* H2F */ struct hfi_prep_slumber_cmd { uint32_t hdr; uint32_t bw; uint32_t freq; -}; +} __packed; /* F2H */ struct hfi_err_cmd { uint32_t hdr; uint32_t error_code; uint32_t data[16]; -}; +} __packed; /* F2H */ struct hfi_debug_cmd { @@ -418,14 +353,14 @@ struct hfi_debug_cmd { uint32_t type; uint32_t timestamp; uint32_t data; -}; +} __packed; /* F2H */ struct hfi_gmu_cntr_register_cmd { uint32_t hdr; uint32_t group_id; uint32_t countable; -}; +} __packed; /* H2F */ struct hfi_gmu_cntr_register_reply_cmd { @@ -434,37 +369,14 @@ struct hfi_gmu_cntr_register_reply_cmd { uint32_t group_id; uint32_t countable; uint64_t counter_addr; -}; +} __packed; /* F2H */ struct hfi_gmu_cntr_release_cmd { uint32_t hdr; uint32_t group_id; uint32_t countable; -}; - -#define CTXT_FLAG_PMODE 0x00000001 -#define CTXT_FLAG_SWITCH_INTERNAL 0x00000002 -#define CTXT_FLAG_SWITCH 0x00000008 -#define CTXT_FLAG_NOTIFY 0x00000020 -#define CTXT_FLAG_NO_FAULT_TOLERANCE 0x00000200 -#define CTXT_FLAG_PWR_RULE 0x00000800 -#define CTXT_FLAG_PRIORITY_MASK 0x0000F000 -#define CTXT_FLAG_IFH_NOP 0x00010000 -#define CTXT_FLAG_SECURE 0x00020000 -#define CTXT_FLAG_TYPE_MASK 0x01F00000 -#define CTXT_FLAG_TYPE_SHIFT 20 -#define CTXT_FLAG_TYPE_ANY 0 -#define CTXT_FLAG_TYPE_GL 1 -#define CTXT_FLAG_TYPE_CL 2 -#define CTXT_FLAG_TYPE_C2D 3 -#define CTXT_FLAG_TYPE_RS 4 -#define CTXT_FLAG_TYPE_UNKNOWN 0x1E -#define CTXT_FLAG_PREEMPT_STYLE_MASK 0x0E000000 -#define CTXT_FLAG_PREEMPT_STYLE_SHIFT 25 -#define CTXT_FLAG_PREEMPT_STYLE_ANY 0 -#define CTXT_FLAG_PREEMPT_STYLE_RB 1 -#define CTXT_FLAG_PREEMPT_STYLE_FG 2 +} __packed; /* H2F */ struct hfi_register_ctxt_cmd { @@ -474,25 +386,19 @@ struct hfi_register_ctxt_cmd { uint64_t pt_addr; uint32_t ctxt_idr; uint32_t ctxt_bank; -}; +} __packed; /* H2F */ struct hfi_unregister_ctxt_cmd { uint32_t hdr; uint32_t ctxt_id; uint32_t ts; -}; - -#define CMDBATCH_SWITCH CTXT_FLAG_SWITCH -#define CMDBATCH_NOTIFY CTXT_FLAG_NOTIFY -#define CMDBATCH_PROFILING 0x00000010 -#define CMDBATCH_EOF 0x00000100 -#define CMDBATCH_PWR_STRICT CTXT_FLAG_PWR_RULE +} __packed; struct hfi_issue_ib { uint64_t addr; uint32_t size; -}; +} __packed; /* H2F */ struct hfi_issue_cmd_cmd { @@ -502,21 +408,21 @@ struct hfi_issue_cmd_cmd { uint32_t ts; uint32_t count; struct hfi_issue_ib *ibs[]; -}; +} __packed; /* Internal */ struct hfi_issue_cmd_req { uint32_t queue; uint32_t ctxt_id; struct hfi_issue_cmd_cmd cmd; -}; +} __packed; /* H2F */ /* The length of *buf will be embedded in the hdr */ struct hfi_issue_cmd_raw_cmd { uint32_t hdr; uint32_t *buf; -}; +} __packed; /* Internal */ struct hfi_issue_cmd_raw_req { @@ -524,27 +430,27 @@ struct hfi_issue_cmd_raw_req { uint32_t ctxt_id; uint32_t len; uint32_t *buf; -}; +} __packed; /* H2F */ struct hfi_ts_notify_cmd { uint32_t hdr; uint32_t ctxt_id; uint32_t ts; -}; +} __packed; -#define TS_RETIRE_FLUSH 1 -#define TS_RETIRE_ERROR 2 -#define TS_RETIRE_PAST 3 -#define TS_RETIRE_DONE 4 +#define CMDBATCH_SUCCESS 0 +#define CMDBATCH_RETIRED 1 +#define CMDBATCH_ERROR 2 +#define CMDBATCH_SKIP 3 /* F2H */ struct hfi_ts_retire_cmd { uint32_t hdr; uint32_t ctxt_id; uint32_t ts; - uint32_t type; -}; + uint32_t ret; +} __packed; /* H2F */ struct hfi_context_pointers_cmd { @@ -552,7 +458,7 @@ struct hfi_context_pointers_cmd { uint32_t ctxt_id; uint64_t sop_addr; uint64_t eop_addr; -}; +} __packed; /* H2F */ struct hfi_context_rule_cmd { @@ -560,7 +466,7 @@ struct hfi_context_rule_cmd { uint32_t ctxt_id; uint32_t type; uint32_t status; -}; +} __packed; /* F2H */ struct hfi_context_bad_cmd { @@ -568,13 +474,13 @@ struct hfi_context_bad_cmd { uint32_t ctxt_id; uint32_t status; uint32_t error; -}; +} __packed; /* H2F */ struct hfi_context_bad_reply_cmd { uint32_t hdr; uint32_t req_hdr; -}; +} __packed; /** * struct pending_cmd - data structure to track outstanding HFI @@ -588,36 +494,64 @@ struct pending_cmd { }; /** - * struct kgsl_hfi - HFI control structure - * @kgsldev: Point to the kgsl device - * @hfi_interrupt_num: number of GMU asserted HFI interrupt - * @cmdq_mutex: mutex to protect command queue access from multiple senders - * @tasklet: the thread handling received messages from GMU + * struct a6xx_hfi - HFI control structure * @seqnum: atomic counter that is incremented for each message sent. The * value of the counter is used as sequence number for HFI message - * @bwtbl_cmd: HFI BW table buffer - * @acd_tbl_cmd: HFI table for ACD data + * @bw_table: HFI BW table buffer + * @acd_table: HFI table for ACD data */ -struct kgsl_hfi { - struct kgsl_device *kgsldev; - int hfi_interrupt_num; - struct mutex cmdq_mutex; - struct tasklet_struct tasklet; +struct a6xx_hfi { + /** @irq: HFI interrupt line */ + int irq; atomic_t seqnum; - struct hfi_bwtable_cmd bwtbl_cmd; - struct hfi_acd_table_cmd acd_tbl_cmd; + /** @hfi_mem: Memory descriptor for the hfi memory */ + struct gmu_memdesc *hfi_mem; + struct hfi_bwtable_cmd bw_table; + struct hfi_acd_table_cmd acd_table; + /** @dcvs_table: HFI table for gpu dcvs levels */ + struct hfi_dcvstable_cmd dcvs_table; }; -struct gmu_device; -struct gmu_memdesc; +struct a6xx_gmu_device; -irqreturn_t hfi_irq_handler(int irq, void *data); -int hfi_start(struct kgsl_device *device, struct gmu_device *gmu, - uint32_t boot_state); -void hfi_stop(struct gmu_device *gmu); -void hfi_receiver(unsigned long data); -void hfi_init(struct gmu_device *gmu); +/* a6xx_hfi_irq_handler - IRQ handler for HFI interripts */ +irqreturn_t a6xx_hfi_irq_handler(int irq, void *data); -/* hfi_send_req is only for external (to HFI) requests */ -int hfi_send_req(struct gmu_device *gmu, unsigned int id, void *data); -#endif /* __KGSL_HFI_H */ +/** + * a6xx_hfi_start - Send the various HFIs during device boot up + * @adreno_dev: Pointer to the adreno device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_start(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_start - Send the various HFIs during device boot up + * @adreno_dev: Pointer to the adreno device + * + * Return: 0 on success or negative error on failure + */ +void a6xx_hfi_stop(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_init - Initialize hfi resources + * @adreno_dev: Pointer to the adreno device + * + * This function allocates and sets up hfi queues + * when a process creates the very first kgsl instance + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_init(struct adreno_device *adreno_dev); + +/** + * a6xx_hfi_send_req - Send an HFI packet to GMU + * @adreno_dev: Pointer to the adreno device + * @id: Packet id to be sent + * @data: Container for the data sent as part of this pcket + * + * Return: 0 on success or negative error on failure + */ +int a6xx_hfi_send_req(struct adreno_device *adreno_dev, + unsigned int id, void *data); +#endif diff --git a/drivers/gpu/msm/adreno_a6xx_preempt.c b/drivers/gpu/msm/adreno_a6xx_preempt.c index d83c095ab1c9..da8851edb4a2 100644 --- a/drivers/gpu/msm/adreno_a6xx_preempt.c +++ b/drivers/gpu/msm/adreno_a6xx_preempt.c @@ -375,7 +375,7 @@ void a6xx_preemption_trigger(struct adreno_device *adreno_dev) err: /* If fenced write fails, take inline snapshot and trigger recovery */ if (!in_interrupt()) { - gmu_core_snapshot(device); + gmu_fault_snapshot(device); adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT_SKIP_SNAPSHOT); } else { diff --git a/drivers/gpu/msm/adreno_a6xx_rgmu.c b/drivers/gpu/msm/adreno_a6xx_rgmu.c index af229d91f81d..e1f0dde2f9d8 100644 --- a/drivers/gpu/msm/adreno_a6xx_rgmu.c +++ b/drivers/gpu/msm/adreno_a6xx_rgmu.c @@ -3,14 +3,25 @@ * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. */ +#include +#include +#include #include +#include +#include +#include #include +#include #include "adreno.h" #include "adreno_a6xx.h" +#include "adreno_a6xx_rgmu.h" #include "adreno_snapshot.h" -#include "kgsl_rgmu.h" +#include "kgsl_bus.h" #include "kgsl_trace.h" +#include "kgsl_util.h" + +#define RGMU_CLK_FREQ 200000000 /* RGMU timeouts */ #define RGMU_IDLE_TIMEOUT 100 /* ms */ @@ -31,10 +42,42 @@ static const unsigned int a6xx_rgmu_registers[] = { 0x26000, 0x26002, }; -irqreturn_t rgmu_irq_handler(int irq, void *data) +static struct a6xx_rgmu_device *to_a6xx_rgmu(struct adreno_device *adreno_dev) +{ + struct a6xx_device *a6xx_dev = container_of(adreno_dev, + struct a6xx_device, adreno_dev); + + return &a6xx_dev->rgmu; +} + +static void a6xx_rgmu_active_count_put(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return; + + if (WARN(atomic_read(&device->active_cnt) == 0, + "Unbalanced get/put calls to KGSL active count\n")) + return; + + if (atomic_dec_and_test(&device->active_cnt)) { + kgsl_pwrscale_update_stats(device); + kgsl_pwrscale_update(device); + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + } + + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + + wake_up(&device->active_cnt_wq); +} + +static irqreturn_t a6xx_rgmu_irq_handler(int irq, void *data) { struct kgsl_device *device = data; - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); unsigned int status = 0; gmu_core_regread(device, A6XX_GMU_AO_HOST_INTERRUPT_STATUS, &status); @@ -59,11 +102,11 @@ irqreturn_t rgmu_irq_handler(int irq, void *data) return IRQ_HANDLED; } -irqreturn_t oob_irq_handler(int irq, void *data) +static irqreturn_t a6xx_oob_irq_handler(int irq, void *data) { struct kgsl_device *device = data; - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int status = 0; gmu_core_regread(device, A6XX_GMU_GMU2HOST_INTR_INFO, &status); @@ -101,7 +144,7 @@ static const char *oob_to_str(enum oob_request req) static int a6xx_rgmu_oob_set(struct kgsl_device *device, enum oob_request req) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); int ret, set, check; set = BIT(req + 16); @@ -122,6 +165,7 @@ static int a6xx_rgmu_oob_set(struct kgsl_device *device, dev_err(&rgmu->pdev->dev, "Timed out while setting OOB req:%s status:0x%x\n", oob_to_str(req), status); + gmu_fault_snapshot(device); return ret; } @@ -144,7 +188,7 @@ static void a6xx_rgmu_oob_clear(struct kgsl_device *device, static void a6xx_rgmu_bcl_config(struct kgsl_device *device, bool on) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); if (on) { /* Enable BCL CRC HW i/f */ @@ -169,35 +213,47 @@ static void a6xx_rgmu_bcl_config(struct kgsl_device *device, bool on) } } -static void a6xx_rgmu_irq_enable(struct kgsl_device *device) +static void a6xx_rgmu_irq_enable(struct adreno_device *adreno_dev) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); /* Clear pending IRQs and Unmask needed IRQs */ - adreno_gmu_clear_and_unmask_irqs(ADRENO_DEVICE(device)); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, 0xffffffff); + + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, + ~((unsigned int)RGMU_OOB_IRQ_MASK)); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK, + (unsigned int)~RGMU_AO_IRQ_MASK); /* Enable all IRQs on host */ enable_irq(rgmu->oob_interrupt_num); enable_irq(rgmu->rgmu_interrupt_num); } -static void a6xx_rgmu_irq_disable(struct kgsl_device *device) +static void a6xx_rgmu_irq_disable(struct adreno_device *adreno_dev) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); /* Disable all IRQs on host */ disable_irq(rgmu->rgmu_interrupt_num); disable_irq(rgmu->oob_interrupt_num); /* Mask all IRQs and clear pending IRQs */ - adreno_gmu_mask_and_clear_irqs(ADRENO_DEVICE(device)); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_MASK, 0xffffffff); + + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_AO_HOST_INTERRUPT_CLR, 0xffffffff); } static int a6xx_rgmu_ifpc_store(struct kgsl_device *device, unsigned int val) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int requested_idle_level; if (!ADRENO_FEATURE(adreno_dev, ADRENO_IFPC)) @@ -211,21 +267,14 @@ static int a6xx_rgmu_ifpc_store(struct kgsl_device *device, if (requested_idle_level == rgmu->idle_level) return 0; - mutex_lock(&device->mutex); - - /* Power down the GPU before changing the idle level */ - kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); - rgmu->idle_level = requested_idle_level; - kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); - - mutex_unlock(&device->mutex); - - return 0; + /* Power cycle the GPU for changes to take effect */ + return adreno_power_cycle_u32(adreno_dev, &rgmu->idle_level, + requested_idle_level); } static unsigned int a6xx_rgmu_ifpc_show(struct kgsl_device *device) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); return rgmu->idle_level == GPU_HW_IFPC; } @@ -233,10 +282,8 @@ static unsigned int a6xx_rgmu_ifpc_show(struct kgsl_device *device) static void a6xx_rgmu_prepare_stop(struct kgsl_device *device) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); /* Turn off GX_MEM retention */ - kgsl_regwrite(KGSL_DEVICE(adreno_dev), - A6XX_RBBM_BLOCK_GX_RETENTION_CNTL, 0); + kgsl_regwrite(device, A6XX_RBBM_BLOCK_GX_RETENTION_CNTL, 0); } #define GX_GDSC_POWER_OFF BIT(6) @@ -254,9 +301,10 @@ static bool a6xx_rgmu_gx_is_on(struct kgsl_device *device) return !(val & GX_GDSC_POWER_OFF); } -static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) +static int a6xx_rgmu_wait_for_lowest_idle(struct adreno_device *adreno_dev) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int reg[10] = {0}; unsigned long t; uint64_t ts1, ts2, ts3; @@ -264,7 +312,7 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) if (rgmu->idle_level != GPU_HW_IFPC) return 0; - ts1 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts1 = a6xx_read_alwayson(adreno_dev); t = jiffies + msecs_to_jiffies(RGMU_IDLE_TIMEOUT); do { @@ -278,7 +326,7 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) usleep_range(10, 100); } while (!time_after(jiffies, t)); - ts2 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts2 = a6xx_read_alwayson(adreno_dev); /* Do one last read incase it succeeds */ gmu_core_regread(device, @@ -287,7 +335,7 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) if (reg[0] & GX_GDSC_POWER_OFF) return 0; - ts3 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts3 = a6xx_read_alwayson(adreno_dev); /* Collect abort data to help with debugging */ gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, ®[1]); @@ -318,6 +366,7 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) reg[7], reg[8], reg[9]); WARN_ON(1); + gmu_fault_snapshot(device); return -ETIMEDOUT; } @@ -336,15 +385,11 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) /* RGMU FENCE RANGE MASK */ #define RGMU_FENCE_RANGE_MASK ((0x1 << 31) | ((0xA << 2) << 18) | (0x8A0)) -/* - * a6xx_rgmu_fw_start() - set up GMU and start FW - * @device: Pointer to KGSL device - * @boot_state: State of the rgmu being started - */ -static int a6xx_rgmu_fw_start(struct kgsl_device *device, +static int a6xx_rgmu_fw_start(struct adreno_device *adreno_dev, unsigned int boot_state) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int status; int i; @@ -358,12 +403,6 @@ static int a6xx_rgmu_fw_start(struct kgsl_device *device, for (i = 0; i < rgmu->fw_size; i++) gmu_core_regwrite(device, A6XX_GMU_CM3_ITCM_START + i, rgmu->fw_hostptr[i]); - /* - * Enable power counter because it was disabled before - * slumber. - */ - gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, - 1); break; } @@ -411,6 +450,7 @@ static int a6xx_rgmu_fw_start(struct kgsl_device *device, gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, &status); dev_err(&rgmu->pdev->dev, "rgmu boot Failed. status:%08x\n", status); + gmu_fault_snapshot(device); return -ETIMEDOUT; } @@ -420,82 +460,158 @@ static int a6xx_rgmu_fw_start(struct kgsl_device *device, return 0; } -static int a6xx_rgmu_suspend(struct kgsl_device *device) +static void a6xx_rgmu_notify_slumber(struct adreno_device *adreno_dev) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - int ret = 0; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* Disable the power counter so that the RGMU is not busy */ + gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0); + + /* BCL OFF Sequence */ + a6xx_rgmu_bcl_config(device, false); +} + +static void a6xx_rgmu_disable_clks(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; /* Check GX GDSC is status */ if (a6xx_rgmu_gx_is_on(device)) { - /* Switch gx gdsc control from RGMU to CPU - * force non-zero reference count in clk driver - * so next disable call will turn - * off the GDSC + if (IS_ERR_OR_NULL(rgmu->gx_gdsc)) + return; + + /* + * Switch gx gdsc control from RGMU to CPU. Force non-zero + * reference count in clk driver so next disable call will + * turn off the GDSC. */ ret = regulator_enable(rgmu->gx_gdsc); if (ret) dev_err(&rgmu->pdev->dev, - "Fail to enable gx gdsc, error:%d\n", ret); + "Fail to enable gx gdsc:%d\n", ret); ret = regulator_disable(rgmu->gx_gdsc); if (ret) dev_err(&rgmu->pdev->dev, - "Fail to disable gx gdsc, error:%d\n", ret); + "Fail to disable gx gdsc:%d\n", ret); if (a6xx_rgmu_gx_is_on(device)) dev_err(&rgmu->pdev->dev, "gx is stuck on\n"); } - return ret; + clk_bulk_disable_unprepare(rgmu->num_clks, rgmu->clks); } -/* - * a6xx_rgmu_gpu_pwrctrl() - GPU power control via rgmu interface - * @adreno_dev: Pointer to adreno device - * @mode: requested power mode - * @arg1: first argument for mode control - * @arg2: second argument for mode control - */ -static int a6xx_rgmu_gpu_pwrctrl(struct kgsl_device *device, - unsigned int mode, unsigned int arg1, unsigned int arg2) +static int a6xx_rgmu_disable_gdsc(struct adreno_device *adreno_dev) { - int ret = 0; + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - switch (mode) { - case GMU_FW_START: - ret = a6xx_rgmu_fw_start(device, arg1); - break; - case GMU_SUSPEND: - ret = a6xx_rgmu_suspend(device); - break; - case GMU_NOTIFY_SLUMBER: + /* Wait up to 5 seconds for the regulator to go off */ + if (kgsl_regulator_disable_wait(rgmu->cx_gdsc, 5000)) + return 0; - /* Disable the power counter so that the RGMU is not busy */ - gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, - 0); + dev_err(&rgmu->pdev->dev, "RGMU CX gdsc off timeout\n"); - /* BCL OFF Sequence */ - a6xx_rgmu_bcl_config(device, false); + device->state = KGSL_STATE_NONE; - break; - default: - ret = -EINVAL; - break; + return -ETIMEDOUT; +} + + +static void a6xx_rgmu_halt_execution(struct kgsl_device *device); + +void a6xx_rgmu_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + + /* + * Halt RGMU execution so that GX will not + * be collapsed while dumping snapshot. + */ + a6xx_rgmu_halt_execution(device); + + adreno_snapshot_registers(device, snapshot, a6xx_rgmu_registers, + ARRAY_SIZE(a6xx_rgmu_registers) / 2); + + a6xx_snapshot(adreno_dev, snapshot); + + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, 0xffffffff); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, + RGMU_OOB_IRQ_MASK); + + if (device->gmu_fault) + rgmu->fault_count++; +} + +static void a6xx_rgmu_suspend(struct adreno_device *adreno_dev) +{ + a6xx_rgmu_irq_disable(adreno_dev); + + a6xx_rgmu_disable_clks(adreno_dev); + a6xx_rgmu_disable_gdsc(adreno_dev); +} + +static int a6xx_rgmu_enable_clks(struct adreno_device *adreno_dev) +{ + int ret; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + + ret = clk_set_rate(rgmu->rgmu_clk, RGMU_CLK_FREQ); + if (ret) { + dev_err(&rgmu->pdev->dev, "Couldn't set the RGMU clock\n"); + return ret; } + ret = clk_set_rate(rgmu->gpu_clk, + pwr->pwrlevels[pwr->default_pwrlevel].gpu_freq); + if (ret) { + dev_err(&rgmu->pdev->dev, "Couldn't set the GPU clock\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(rgmu->num_clks, rgmu->clks); + if (ret) { + dev_err(&rgmu->pdev->dev, "Failed to enable RGMU clocks\n"); + return ret; + } + + device->state = KGSL_STATE_AWARE; + + return 0; +} + +static int a6xx_rgmu_enable_gdsc(struct adreno_device *adreno_dev) +{ + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + if (IS_ERR_OR_NULL(rgmu->cx_gdsc)) + return 0; + + ret = regulator_enable(rgmu->cx_gdsc); + if (ret) + dev_err(&rgmu->pdev->dev, + "Fail to enable CX gdsc:%d\n", ret); + return ret; } /* * a6xx_rgmu_load_firmware() - Load the ucode into the RGMU TCM - * @device: Pointer to KGSL device + * @adreno_dev: Pointer to adreno device */ -static int a6xx_rgmu_load_firmware(struct kgsl_device *device) +static int a6xx_rgmu_load_firmware(struct adreno_device *adreno_dev) { const struct firmware *fw = NULL; - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); int ret; @@ -523,9 +639,18 @@ static int a6xx_rgmu_load_firmware(struct kgsl_device *device) /* Halt RGMU execution */ static void a6xx_rgmu_halt_execution(struct kgsl_device *device) { - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); unsigned int index, status, fence; + if (!device->gmu_fault) + return; + + /* Mask so there's no interrupt caused by NMI */ + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF); + + /* Make sure the interrupt is masked */ + wmb(); + gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, &index); gmu_core_regread(device, A6XX_RGMU_CX_PCC_STATUS, &status); gmu_core_regread(device, A6XX_GMU_AO_AHB_FENCE_CTRL, &fence); @@ -548,42 +673,714 @@ static void a6xx_rgmu_halt_execution(struct kgsl_device *device) } -/* - * a6xx_rgmu_snapshot() - A6XX GMU snapshot function - * @adreno_dev: Device being snapshotted - * @snapshot: Pointer to the snapshot instance - * - * This is where all of the A6XX GMU specific bits and pieces are grabbed - * into the snapshot memory - */ -static void a6xx_rgmu_snapshot(struct kgsl_device *device, - struct kgsl_snapshot *snapshot) +static void halt_gbif_arb(struct adreno_device *adreno_dev) { + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - adreno_snapshot_registers(device, snapshot, a6xx_rgmu_registers, - ARRAY_SIZE(a6xx_rgmu_registers) / 2); + /* Halt all AXI requests */ + kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_ARB_HALT_MASK); + adreno_wait_for_halt_ack(device, ADRENO_REG_GBIF_HALT_ACK, + A6XX_GBIF_ARB_HALT_MASK); + + /* De-assert the halts */ + kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0); } -static u64 a6xx_rgmu_read_alwayson(struct kgsl_device *device) + +/* Caller shall ensure GPU is ready for SLUMBER */ +static void a6xx_rgmu_power_off(struct adreno_device *adreno_dev) { - return a6xx_read_alwayson(ADRENO_DEVICE(device)); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_OFF); + + if (device->gmu_fault) + return a6xx_rgmu_suspend(adreno_dev); + + /* Wait for the lowest idle level we requested */ + ret = a6xx_rgmu_wait_for_lowest_idle(adreno_dev); + if (ret) + return a6xx_rgmu_suspend(adreno_dev); + + a6xx_rgmu_notify_slumber(adreno_dev); + + /* Halt CX traffic and de-assert halt */ + halt_gbif_arb(adreno_dev); + + a6xx_rgmu_irq_disable(adreno_dev); + a6xx_rgmu_disable_clks(adreno_dev); + a6xx_rgmu_disable_gdsc(adreno_dev); + } -struct gmu_dev_ops adreno_a6xx_rgmudev = { - .load_firmware = a6xx_rgmu_load_firmware, +static int a6xx_rgmu_clock_set(struct adreno_device *adreno_dev, + u32 pwrlevel) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + unsigned long rate; + + if (pwrlevel == INVALID_DCVS_IDX) + return -EINVAL; + + rate = device->pwrctrl.pwrlevels[pwrlevel].gpu_freq; + + ret = clk_set_rate(rgmu->gpu_clk, rate); + if (ret) + dev_err(&rgmu->pdev->dev, "Couldn't set the GPU clock\n"); + + return ret; +} + +static int a6xx_gpu_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* Clear any GPU faults that might have been left over */ + adreno_clear_gpu_fault(adreno_dev); + + adreno_set_active_ctxs_null(adreno_dev); + + adreno_ringbuffer_set_global(adreno_dev, 0); + + ret = kgsl_mmu_start(device); + if (ret) + goto err; + + ret = a6xx_rgmu_oob_set(device, oob_gpu); + if (ret) + goto err_oob_clear; + + adreno_clear_dcvs_counters(adreno_dev); + + /* Restore performance counter registers with saved values */ + adreno_perfcounter_restore(adreno_dev); + + a6xx_start(adreno_dev); + + /* Re-initialize the coresight registers if applicable */ + adreno_coresight_start(adreno_dev); + + adreno_perfcounter_start(adreno_dev); + + /* Clear FSR here in case it is set from a previous pagefault */ + kgsl_mmu_clear_fsr(&device->mmu); + + a6xx_enable_gpu_irq(adreno_dev); + + ret = a6xx_rb_start(adreno_dev); + if (ret) { + a6xx_disable_gpu_irq(adreno_dev); + goto err_oob_clear; + } + + /* Start the dispatcher */ + adreno_dispatcher_start(device); + + device->reset_counter++; + + a6xx_rgmu_oob_clear(device, oob_gpu); + + return 0; + +err_oob_clear: + a6xx_rgmu_oob_clear(device, oob_gpu); + +err: + a6xx_rgmu_power_off(adreno_dev); + + return ret; +} + +static int a6xx_rgmu_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = a6xx_rgmu_enable_gdsc(adreno_dev); + if (ret) + return ret; + + ret = a6xx_rgmu_enable_clks(adreno_dev); + if (ret) { + a6xx_rgmu_disable_gdsc(adreno_dev); + return ret; + } + + a6xx_rgmu_irq_enable(adreno_dev); + + ret = a6xx_rgmu_fw_start(adreno_dev, GMU_COLD_BOOT); + if (ret) + goto err; + + /* Request default DCVS level */ + ret = kgsl_pwrctrl_set_default_gpu_pwrlevel(device); + if (ret) + goto err; + + ret = kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_ON); + if (ret) + goto err; + + device->gmu_fault = false; + + return 0; + +err: + a6xx_rgmu_power_off(adreno_dev); + + return ret; +} + +static int a6xx_power_off(struct adreno_device *adreno_dev); + +static void rgmu_idle_check(struct work_struct *work) +{ + struct kgsl_device *device = container_of(work, + struct kgsl_device, idle_check_ws); + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + mutex_lock(&device->mutex); + + if (!atomic_read(&device->active_cnt)) + a6xx_power_off(adreno_dev); + else + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + + mutex_unlock(&device->mutex); +} + +static void rgmu_idle_timer(struct timer_list *t) +{ + struct kgsl_device *device = container_of(t, struct kgsl_device, + idle_timer); + + kgsl_schedule_work(&device->idle_check_ws); +} + +static int a6xx_boot(struct adreno_device *adreno_dev) +{ + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + if (test_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags)) + return 0; + + ret = a6xx_rgmu_boot(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return ret; + + mod_timer(&device->idle_timer, jiffies + + device->pwrctrl.interval_timeout); + + kgsl_pwrscale_wake(device); + + set_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + device->state = KGSL_STATE_ACTIVE; + + return 0; +} + +static void a6xx_rgmu_touch_wakeup(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + /* Do not wake up a suspended device through touch event */ + if (test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) + return; + + if (test_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags)) + goto done; + + ret = a6xx_rgmu_boot(adreno_dev); + if (ret) + return; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return; + + kgsl_pwrscale_wake(device); + + set_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + device->state = KGSL_STATE_ACTIVE; + +done: + /* + * When waking up from a touch event we want to stay active long enough + * for the user to send a draw command. The default idle timer timeout + * is shorter than we want so go ahead and push the idle timer out + * further for this special case + */ + mod_timer(&device->idle_timer, jiffies + + msecs_to_jiffies(adreno_wake_timeout)); + +} + +static int a6xx_first_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + if (test_bit(RGMU_PRIV_FIRST_BOOT_DONE, &rgmu->flags)) + return a6xx_boot(adreno_dev); + + ret = adreno_dispatcher_init(adreno_dev); + if (ret) + return ret; + + ret = adreno_ringbuffer_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_microcode_read(adreno_dev); + if (ret) + return ret; + + ret = a6xx_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_rgmu_load_firmware(adreno_dev); + if (ret) + return ret; + + ret = a6xx_rgmu_boot(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return ret; + + adreno_get_bus_counters(adreno_dev); + + adreno_dev->profile_buffer = kgsl_allocate_global(device, PAGE_SIZE, 0, + 0, "alwayson"); + + adreno_dev->profile_index = 0; + + if (!IS_ERR(adreno_dev->profile_buffer)) + set_bit(ADRENO_DEVICE_DRAWOBJ_PROFILE, &adreno_dev->priv); + + set_bit(RGMU_PRIV_FIRST_BOOT_DONE, &rgmu->flags); + set_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + device->state = KGSL_STATE_ACTIVE; + + return 0; +} + +static int a6xx_rgmu_first_open(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = a6xx_first_boot(adreno_dev); + if (ret) + return ret; + + /* + * A client that does a first_open but never closes the device + * may prevent us from going back to SLUMBER. So trigger the idle + * check by incrementing the active count and immediately releasing it. + */ + atomic_inc(&device->active_cnt); + a6xx_rgmu_active_count_put(adreno_dev); + + return 0; +} + +static int a6xx_power_off(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + if (!test_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags)) + return 0; + + ret = a6xx_rgmu_oob_set(device, oob_gpu); + if (ret) { + a6xx_rgmu_oob_clear(device, oob_gpu); + goto no_gx_power; + } + + kgsl_pwrscale_update_stats(device); + + /* Save active coresight registers if applicable */ + adreno_coresight_stop(adreno_dev); + + /* Save physical performance counter values before GPU power down*/ + adreno_perfcounter_save(adreno_dev); + + a6xx_rgmu_prepare_stop(device); + + a6xx_rgmu_oob_clear(device, oob_gpu); + +no_gx_power: + /* Halt all gx traffic */ + kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_CLIENT_HALT_MASK); + + adreno_wait_for_halt_ack(device, ADRENO_REG_GBIF_HALT_ACK, + A6XX_GBIF_CLIENT_HALT_MASK); + + a6xx_disable_gpu_irq(adreno_dev); + + a6xx_rgmu_power_off(adreno_dev); + + adreno_set_active_ctxs_null(adreno_dev); + + adreno_dispatcher_stop(adreno_dev); + + adreno_ringbuffer_stop(adreno_dev); + + if (!IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpu_llc_slice); + + if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpuhtw_llc_slice); + + clear_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + del_timer_sync(&device->idle_timer); + + kgsl_pwrscale_sleep(device); + + return ret; +} + +int a6xx_rgmu_restart(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + + a6xx_disable_gpu_irq(adreno_dev); + + /* Hard reset the rgmu and gpu */ + a6xx_rgmu_suspend(adreno_dev); + + clear_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + /* Attempt rebooting the rgmu and gpu */ + return a6xx_boot(adreno_dev); +} + +static int a6xx_rgmu_active_count_get(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret = 0; + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return -EINVAL; + + if (test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) + return -EINVAL; + + if (atomic_read(&device->active_cnt) == 0) + ret = a6xx_boot(adreno_dev); + + if (ret == 0) { + atomic_inc(&device->active_cnt); + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + } + + return ret; +} + +static int a6xx_rgmu_pm_suspend(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + if (test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) + return 0; + + /* Halt any new submissions */ + reinit_completion(&device->halt_gate); + + /* wait for active count so device can be put in slumber */ + ret = kgsl_active_count_wait(device, 0); + if (ret) { + dev_err(device->dev, + "Timed out waiting for the active count\n"); + goto err; + } + + ret = adreno_idle(device); + if (ret) + goto err; + + a6xx_power_off(adreno_dev); + + set_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags); + + adreno_dispatcher_halt(device); + + return 0; +err: + adreno_dispatcher_start(device); + return ret; +} + +static void a6xx_rgmu_pm_resume(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + + if (!test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) { + dev_err(device->dev, "resume invoked without a suspend\n"); + return; + } + + adreno_dispatcher_unhalt(device); + + clear_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags); + + adreno_dispatcher_start(device); +} + +static struct gmu_dev_ops a6xx_rgmudev = { .oob_set = a6xx_rgmu_oob_set, .oob_clear = a6xx_rgmu_oob_clear, - .irq_enable = a6xx_rgmu_irq_enable, - .irq_disable = a6xx_rgmu_irq_disable, - .rpmh_gpu_pwrctrl = a6xx_rgmu_gpu_pwrctrl, .gx_is_on = a6xx_rgmu_gx_is_on, - .prepare_stop = a6xx_rgmu_prepare_stop, - .wait_for_lowest_idle = a6xx_rgmu_wait_for_lowest_idle, .ifpc_store = a6xx_rgmu_ifpc_store, .ifpc_show = a6xx_rgmu_ifpc_show, - .snapshot = a6xx_rgmu_snapshot, - .halt_execution = a6xx_rgmu_halt_execution, - .read_alwayson = a6xx_rgmu_read_alwayson, - .gmu2host_intr_mask = RGMU_OOB_IRQ_MASK, - .gmu_ao_intr_mask = RGMU_AO_IRQ_MASK, +}; + +static int a6xx_rgmu_irq_probe(struct kgsl_device *device) +{ + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); + int ret; + + ret = kgsl_request_irq(rgmu->pdev, "kgsl_oob", + a6xx_oob_irq_handler, device); + if (ret < 0) + return ret; + + rgmu->oob_interrupt_num = ret; + + ret = kgsl_request_irq(rgmu->pdev, + "kgsl_rgmu", a6xx_rgmu_irq_handler, device); + if (ret < 0) + return ret; + + rgmu->rgmu_interrupt_num = ret; + return 0; +} + +static int a6xx_rgmu_regulators_probe(struct a6xx_rgmu_device *rgmu) +{ + int ret = 0; + + rgmu->cx_gdsc = devm_regulator_get(&rgmu->pdev->dev, "vddcx"); + if (IS_ERR(rgmu->cx_gdsc)) { + ret = PTR_ERR(rgmu->cx_gdsc); + if (ret != -EPROBE_DEFER) + dev_err(&rgmu->pdev->dev, + "Couldn't get CX gdsc error:%d\n", ret); + return ret; + } + + rgmu->gx_gdsc = devm_regulator_get(&rgmu->pdev->dev, "vdd"); + if (IS_ERR(rgmu->gx_gdsc)) { + ret = PTR_ERR(rgmu->gx_gdsc); + if (ret != -EPROBE_DEFER) + dev_err(&rgmu->pdev->dev, + "Couldn't get GX gdsc error:%d\n", ret); + } + + return ret; +} + +static int a6xx_rgmu_clocks_probe(struct a6xx_rgmu_device *rgmu, + struct device_node *node) +{ + int ret; + + ret = devm_clk_bulk_get_all(&rgmu->pdev->dev, &rgmu->clks); + if (ret < 0) + return ret; + + rgmu->num_clks = ret; + + rgmu->gpu_clk = kgsl_of_clk_by_name(rgmu->clks, ret, "core"); + if (!rgmu->gpu_clk) { + dev_err(&rgmu->pdev->dev, "The GPU clock isn't defined\n"); + return -ENODEV; + } + + rgmu->rgmu_clk = kgsl_of_clk_by_name(rgmu->clks, ret, "gmu"); + if (!rgmu->rgmu_clk) { + dev_err(&rgmu->pdev->dev, "The RGMU clock isn't defined\n"); + return -ENODEV; + } + + return 0; +} + +const struct adreno_power_ops a6xx_rgmu_power_ops = { + .first_open = a6xx_rgmu_first_open, + .last_close = a6xx_power_off, + .active_count_get = a6xx_rgmu_active_count_get, + .active_count_put = a6xx_rgmu_active_count_put, + .pm_suspend = a6xx_rgmu_pm_suspend, + .pm_resume = a6xx_rgmu_pm_resume, + .touch_wakeup = a6xx_rgmu_touch_wakeup, + .gpu_clock_set = a6xx_rgmu_clock_set, +}; + +int a6xx_rgmu_device_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + struct a6xx_device *a6xx_dev; + int ret; + + a6xx_dev = devm_kzalloc(&pdev->dev, sizeof(*a6xx_dev), + GFP_KERNEL); + if (!a6xx_dev) + return -ENOMEM; + + adreno_dev = &a6xx_dev->adreno_dev; + + ret = a6xx_probe_common(pdev, adreno_dev, chipid, gpucore); + if (ret) + return ret; + + device = KGSL_DEVICE(adreno_dev); + + INIT_WORK(&device->idle_check_ws, rgmu_idle_check); + + timer_setup(&device->idle_timer, rgmu_idle_timer, 0); + + return 0; +} + +/* Do not access any RGMU registers in RGMU probe function */ +static int a6xx_rgmu_probe(struct kgsl_device *device, + struct platform_device *pdev) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct resource *res; + int ret; + + rgmu->pdev = pdev; + + /* Set up RGMU regulators */ + ret = a6xx_rgmu_regulators_probe(rgmu); + if (ret) + return ret; + + /* Set up RGMU clocks */ + ret = a6xx_rgmu_clocks_probe(rgmu, pdev->dev.of_node); + if (ret) + return ret; + + /* Map and reserve RGMU CSRs registers */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "kgsl_rgmu"); + if (!res) { + dev_err(&pdev->dev, "The RGMU register region isn't defined\n"); + return -ENODEV; + } + + device->gmu_core.gmu2gpu_offset = (res->start - device->reg_phys) >> 2; + device->gmu_core.reg_len = resource_size(res); + device->gmu_core.reg_virt = devm_ioremap_resource(&pdev->dev, res); + + if (IS_ERR(device->gmu_core.reg_virt)) { + dev_err(&pdev->dev, "Unable to map the RGMU registers\n"); + return PTR_ERR(device->gmu_core.reg_virt); + } + + /* Initialize OOB and RGMU interrupts */ + ret = a6xx_rgmu_irq_probe(device); + if (ret) + return ret; + + /* Don't enable RGMU interrupts until RGMU started */ + /* We cannot use rgmu_irq_disable because it writes registers */ + disable_irq(rgmu->rgmu_interrupt_num); + disable_irq(rgmu->oob_interrupt_num); + + /* Set up RGMU idle states */ + if (ADRENO_FEATURE(ADRENO_DEVICE(device), ADRENO_IFPC)) + rgmu->idle_level = GPU_HW_IFPC; + else + rgmu->idle_level = GPU_HW_ACTIVE; + + set_bit(GMU_ENABLED, &device->gmu_core.flags); + device->gmu_core.dev_ops = &a6xx_rgmudev; + + return 0; +} + +static void a6xx_rgmu_remove(struct kgsl_device *device) +{ + + memset(&device->gmu_core, 0, sizeof(device->gmu_core)); +} + +static int a6xx_rgmu_bind(struct device *dev, struct device *master, void *data) +{ + struct kgsl_device *device = dev_get_drvdata(master); + + return a6xx_rgmu_probe(device, to_platform_device(dev)); +} + +static void a6xx_rgmu_unbind(struct device *dev, struct device *master, + void *data) +{ + struct kgsl_device *device = dev_get_drvdata(master); + + a6xx_rgmu_remove(device); +} + +static const struct component_ops a6xx_rgmu_component_ops = { + .bind = a6xx_rgmu_bind, + .unbind = a6xx_rgmu_unbind, +}; + +static int a6xx_rgmu_probe_dev(struct platform_device *pdev) +{ + return component_add(&pdev->dev, &a6xx_rgmu_component_ops); +} + +static int a6xx_rgmu_remove_dev(struct platform_device *pdev) +{ + component_del(&pdev->dev, &a6xx_rgmu_component_ops); + return 0; +} + +static const struct of_device_id a6xx_rgmu_match_table[] = { + { .compatible = "qcom,gpu-rgmu" }, + { }, +}; + +struct platform_driver a6xx_rgmu_driver = { + .probe = a6xx_rgmu_probe_dev, + .remove = a6xx_rgmu_remove_dev, + .driver = { + .name = "kgsl-rgmu", + .of_match_table = a6xx_rgmu_match_table, + }, }; diff --git a/drivers/gpu/msm/adreno_a6xx_rgmu.h b/drivers/gpu/msm/adreno_a6xx_rgmu.h new file mode 100644 index 000000000000..6defcbc5ac46 --- /dev/null +++ b/drivers/gpu/msm/adreno_a6xx_rgmu.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + */ +#ifndef __ADRENO_A6XX_RGMU_H +#define __ADRENO_A6XX_RGMU_H + +#define RGMU_AO_IRQ_FENCE_ERR BIT(3) +#define RGMU_AO_IRQ_MASK RGMU_AO_IRQ_FENCE_ERR + +#define RGMU_OOB_IRQ_ERR_MSG BIT(24) +#define RGMU_OOB_IRQ_ACK_MASK GENMASK(23, 16) +#define RGMU_OOB_IRQ_ERR_MSG_MASK GENMASK(31, 24) +#define RGMU_OOB_IRQ_MASK RGMU_OOB_IRQ_ERR_MSG_MASK + +#define MAX_RGMU_CLKS 8 + +enum { + /* @RGMU_PRIV_FIRST_BOOT_DONE: The very first ggpu boot is done */ + RGMU_PRIV_FIRST_BOOT_DONE, + /* @RGMU_PRIV_GPU_STARTED: GPU has been started */ + RGMU_PRIV_GPU_STARTED, + /* @RGMU_PRIV_PM_SUSPEND: The rgmu driver is suspended */ + RGMU_PRIV_PM_SUSPEND, +}; + +/** + * struct a6xx_rgmu_device - rGMU device structure + * @ver: RGMU firmware version + * @rgmu_interrupt_num: RGMU interrupt number + * @oob_interrupt_num: number of RGMU asserted OOB interrupt + * @fw_hostptr: Buffer which holds the RGMU firmware + * @fw_size: Size of RGMU firmware buffer + * @cx_gdsc: CX headswitch that controls power of RGMU and + subsystem peripherals + * @clks: RGMU clocks including the GPU + * @gpu_clk: Pointer to GPU core clock + * @rgmu_clk: Pointer to rgmu clock + * @flags: RGMU flags + * @idle_level: Minimal GPU idle power level + * @fault_count: RGMU fault count + */ +struct a6xx_rgmu_device { + u32 ver; + struct platform_device *pdev; + unsigned int rgmu_interrupt_num; + unsigned int oob_interrupt_num; + unsigned int *fw_hostptr; + uint32_t fw_size; + struct regulator *cx_gdsc; + struct regulator *gx_gdsc; + struct clk_bulk_data *clks; + /** @num_clks: Number of clocks in @clks */ + int num_clks; + struct clk *gpu_clk; + struct clk *rgmu_clk; + unsigned int idle_level; + unsigned int fault_count; + /** @flags: rgmu internal flags */ + unsigned long flags; +}; + +/** + * a6xx_rgmu_device_probe - Probe a6xx rgmu resources + * @pdev: Pointer to the platform device + * @chipid: Chipid of the target + * @gpucore: Pointer to the gpucore + * + * The target specific probe function for rgmu based a6xx targets. + */ +int a6xx_rgmu_device_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore); + +/** + * a6xx_rgmu_restart - Reset and restart the rgmu + * @device: Pointer to the kgsl device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_rgmu_restart(struct kgsl_device *device); + +/** + * a6xx_rgmu_snapshot - Take snapshot for rgmu based targets + * @adreno_dev: Pointer to the adreno device + * @snapshot: Pointer to the snapshot + * + * This function halts rgmu execution if we hit a rgmu + * fault. And then, it takes rgmu and gpu snapshot. + */ +void a6xx_rgmu_snapshot(struct adreno_device *adreno_dev, + struct kgsl_snapshot *snapshot); +#endif diff --git a/drivers/gpu/msm/adreno_a6xx_rpmh.c b/drivers/gpu/msm/adreno_a6xx_rpmh.c new file mode 100644 index 000000000000..fe03df7239f6 --- /dev/null +++ b/drivers/gpu/msm/adreno_a6xx_rpmh.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include + +#include "adreno.h" +#include "adreno_a6xx.h" +#include "kgsl_bus.h" +#include "kgsl_device.h" + +struct rpmh_arc_vals { + u32 num; + const u16 *val; +}; + +struct bcm { + const char *name; + u32 buswidth; + u32 channels; + u32 unit; + u16 width; + u8 vcd; + bool fixed; +}; + +struct bcm_data { + __le32 unit; + __le16 width; + u8 vcd; + u8 reserved; +}; + +struct rpmh_bw_votes { + u32 wait_bitmask; + u32 num_cmds; + u32 *addrs; + u32 num_levels; + u32 **cmds; +}; + +#define ARC_VOTE_SET(pri, sec, vlvl) \ + ((((vlvl) & 0xFFFF) << 16) | (((sec) & 0xFF) << 8) | ((pri) & 0xFF)) + +static int rpmh_arc_cmds(struct rpmh_arc_vals *arc, const char *res_id) +{ + size_t len = 0; + + arc->val = cmd_db_read_aux_data(res_id, &len); + + /* + * cmd_db_read_aux_data() gives us a zero-padded table of + * size len that contains the arc values. To determine the + * number of arc values, we loop through the table and count + * them until we get to the end of the buffer or hit the + * zero padding. + */ + for (arc->num = 1; arc->num < (len >> 1); arc->num++) { + if (arc->val[arc->num - 1] != 0 && arc->val[arc->num] == 0) + break; + } + + return 0; +} + +static int setup_volt_dependency_tbl(uint32_t *votes, + struct rpmh_arc_vals *pri_rail, struct rpmh_arc_vals *sec_rail, + u16 *vlvl, unsigned int num_entries) +{ + int i, j, k; + uint16_t cur_vlvl; + bool found_match; + + /* i tracks current KGSL GPU frequency table entry + * j tracks secondary rail voltage table entry + * k tracks primary rail voltage table entry + */ + for (i = 0; i < num_entries; i++) { + found_match = false; + + /* Look for a primary rail voltage that matches a VLVL level */ + for (k = 0; k < pri_rail->num; k++) { + if (pri_rail->val[k] >= vlvl[i]) { + cur_vlvl = pri_rail->val[k]; + found_match = true; + break; + } + } + + /* If we did not find a matching VLVL level then abort */ + if (!found_match) + return -EINVAL; + + /* + * Look for a secondary rail index whose VLVL value + * is greater than or equal to the VLVL value of the + * corresponding index of the primary rail + */ + for (j = 0; j < sec_rail->num; j++) { + if (sec_rail->val[j] >= cur_vlvl || + j + 1 == sec_rail->num) + break; + } + + if (j == sec_rail->num) + j = 0; + + votes[i] = ARC_VOTE_SET(k, j, cur_vlvl); + } + + return 0; +} + +/* Generate a set of bandwidth votes for the list of BCMs */ +static void tcs_cmd_data(struct bcm *bcms, int count, u32 ab, u32 ib, + u32 *data) +{ + int i; + + for (i = 0; i < count; i++) { + bool valid = true; + bool commit = false; + u64 avg, peak, x, y; + + if (i == count - 1 || bcms[i].vcd != bcms[i + 1].vcd) + commit = true; + + /* + * On a660, the "ACV" y vote should be 0x08 if there is a valid + * vote and 0x00 if not. This is kind of hacky and a660 specific + * but we can clean it up when we add a new target + */ + if (bcms[i].fixed) { + if (!ab && !ib) + data[i] = BCM_TCS_CMD(commit, false, 0x0, 0x0); + else + data[i] = BCM_TCS_CMD(commit, true, 0x0, 0x8); + continue; + } + + /* Multiple the bandwidth by the width of the connection */ + avg = ((u64) ab) * bcms[i].width; + + /* And then divide by the total width across channels */ + do_div(avg, bcms[i].buswidth * bcms[i].channels); + + peak = ((u64) ib) * bcms[i].width; + do_div(peak, bcms[i].buswidth); + + /* Input bandwidth value is in KBps */ + x = avg * 1000ULL; + do_div(x, bcms[i].unit); + + /* Input bandwidth value is in KBps */ + y = peak * 1000ULL; + do_div(y, bcms[i].unit); + + /* + * If a bandwidth value was specified but the calculation ends + * rounding down to zero, set a minimum level + */ + if (ab && x == 0) + x = 1; + + if (ib && y == 0) + y = 1; + + x = min_t(u64, x, BCM_TCS_CMD_VOTE_MASK); + y = min_t(u64, y, BCM_TCS_CMD_VOTE_MASK); + + if (!x && !y) + valid = false; + + data[i] = BCM_TCS_CMD(commit, valid, x, y); + } +} + +static void free_rpmh_bw_votes(struct rpmh_bw_votes *votes) +{ + int i; + + if (!votes) + return; + + for (i = 0; votes->cmds && i < votes->num_levels; i++) + kfree(votes->cmds[i]); + + kfree(votes->cmds); + kfree(votes->addrs); + kfree(votes); +} + +/* Build the votes table from the specified bandwidth levels */ +static struct rpmh_bw_votes *build_rpmh_bw_votes(struct bcm *bcms, + int bcm_count, u32 *levels, int levels_count) +{ + struct rpmh_bw_votes *votes; + int i; + + votes = kzalloc(sizeof(*votes), GFP_KERNEL); + if (!votes) + return ERR_PTR(-ENOMEM); + + votes->addrs = kcalloc(bcm_count, sizeof(*votes->cmds), GFP_KERNEL); + if (!votes->addrs) { + free_rpmh_bw_votes(votes); + return ERR_PTR(-ENOMEM); + } + + votes->cmds = kcalloc(levels_count, sizeof(*votes->cmds), GFP_KERNEL); + if (!votes->cmds) { + free_rpmh_bw_votes(votes); + return ERR_PTR(-ENOMEM); + } + + votes->num_cmds = bcm_count; + votes->num_levels = levels_count; + + /* Get the cmd-db information for each BCM */ + for (i = 0; i < bcm_count; i++) { + size_t l; + const struct bcm_data *data; + + data = cmd_db_read_aux_data(bcms[i].name, &l); + + votes->addrs[i] = cmd_db_read_addr(bcms[i].name); + + bcms[i].unit = le32_to_cpu(data->unit); + bcms[i].width = le16_to_cpu(data->width); + bcms[i].vcd = data->vcd; + } + + for (i = 0; i < bcm_count; i++) { + if (i == (bcm_count - 1) || bcms[i].vcd != bcms[i + 1].vcd) + votes->wait_bitmask |= (1 << i); + } + + for (i = 0; i < levels_count; i++) { + votes->cmds[i] = kcalloc(bcm_count, sizeof(u32), GFP_KERNEL); + if (!votes->cmds[i]) { + free_rpmh_bw_votes(votes); + return ERR_PTR(-ENOMEM); + } + + tcs_cmd_data(bcms, bcm_count, 0, levels[i], votes->cmds[i]); + } + + return votes; +} + +/* + * setup_gmu_arc_votes - Build the gmu voting table + * @hfi: Pointer to hfi device + * @pri_rail: Pointer to primary power rail vlvl table + * @sec_rail: Pointer to second/dependent power rail vlvl table + * + * This function initializes the cx votes for all gmu frequencies + * for gmu dcvs + */ +static int setup_cx_arc_votes(struct a6xx_hfi *hfi, + struct rpmh_arc_vals *pri_rail, struct rpmh_arc_vals *sec_rail) +{ + /* Hardcoded values of GMU CX voltage levels */ + u16 gmu_cx_vlvl[] = { 0, RPMH_REGULATOR_LEVEL_MIN_SVS }; + u32 cx_votes[MAX_CX_LEVELS]; + struct hfi_dcvstable_cmd *table = &hfi->dcvs_table; + int ret, i; + + table->gmu_level_num = 2; + + table->cx_votes[0].freq = 0; + table->cx_votes[1].freq = GMU_FREQUENCY / 1000; + + ret = setup_volt_dependency_tbl(cx_votes, pri_rail, + sec_rail, gmu_cx_vlvl, table->gmu_level_num); + if (!ret) { + for (i = 0; i < table->gmu_level_num; i++) + table->cx_votes[i].vote = cx_votes[i]; + } + + return ret; +} + +/* + * setup_gx_arc_votes - Build the gpu dcvs voting table + * @hfi: Pointer to hfi device + * @pri_rail: Pointer to primary power rail vlvl table + * @sec_rail: Pointer to second/dependent power rail vlvl table + * + * This function initializes the gx votes for all gpu frequencies + * for gpu dcvs + */ +static int setup_gx_arc_votes(struct adreno_device *adreno_dev, + struct rpmh_arc_vals *pri_rail, struct rpmh_arc_vals *sec_rail) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct hfi_dcvstable_cmd *table = &gmu->hfi.dcvs_table; + u32 index; + u16 vlvl_tbl[MAX_GX_LEVELS]; + u32 gx_votes[MAX_GX_LEVELS]; + int ret, i; + + /* Add the zero powerlevel for the perf table */ + table->gpu_level_num = device->pwrctrl.num_pwrlevels + 1; + + if (table->gpu_level_num > pri_rail->num || + table->gpu_level_num > ARRAY_SIZE(vlvl_tbl)) { + dev_err(&gmu->pdev->dev, + "Defined more GPU DCVS levels than RPMh can support\n"); + return -ERANGE; + } + + memset(vlvl_tbl, 0, sizeof(vlvl_tbl)); + + table->gx_votes[0].freq = 0; + + /* GMU power levels are in ascending order */ + for (index = 1, i = pwr->num_pwrlevels - 1; i >= 0; i--, index++) { + vlvl_tbl[index] = pwr->pwrlevels[i].voltage_level; + table->gx_votes[index].freq = pwr->pwrlevels[i].gpu_freq / 1000; + } + + ret = setup_volt_dependency_tbl(gx_votes, pri_rail, + sec_rail, vlvl_tbl, table->gpu_level_num); + if (!ret) { + for (i = 0; i < table->gpu_level_num; i++) { + table->gx_votes[i].vote = gx_votes[i]; + table->gx_votes[i].acd = 0xffffffff; + } + } + + return ret; + +} + +static int build_dcvs_table(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct a6xx_hfi *hfi = &gmu->hfi; + struct rpmh_arc_vals gx_arc, cx_arc, mx_arc; + int ret; + + hfi->dcvs_table.hdr = CMD_MSG_HDR(H2F_MSG_PERF_TBL, + sizeof(hfi->dcvs_table)); + + ret = rpmh_arc_cmds(&gx_arc, "gfx.lvl"); + if (ret) + return ret; + + ret = rpmh_arc_cmds(&cx_arc, "cx.lvl"); + if (ret) + return ret; + + ret = rpmh_arc_cmds(&mx_arc, "mx.lvl"); + if (ret) + return ret; + + ret = setup_cx_arc_votes(hfi, &cx_arc, &mx_arc); + if (ret) + return ret; + + return setup_gx_arc_votes(adreno_dev, &gx_arc, &mx_arc); +} + +/* + * List of Bus Control Modules (BCMs) that need to be configured for the GPU + * to access DDR. For each bus level we will generate a vote each BC + */ +static struct bcm a660_ddr_bcms[] = { + { .name = "SH0", .buswidth = 16 }, + { .name = "MC0", .buswidth = 4 }, + { .name = "ACV", .fixed = true }, +}; + +/* Same as above, but for the CNOC BCMs */ +static struct bcm a660_cnoc_bcms[] = { + { .name = "CN0", .buswidth = 4 }, +}; + +static void build_bw_table_cmd(struct hfi_bwtable_cmd *cmd, + struct rpmh_bw_votes *ddr, struct rpmh_bw_votes *cnoc) +{ + u32 i, j; + + cmd->hdr = CMD_MSG_HDR(H2F_MSG_BW_VOTE_TBL, sizeof(*cmd)); + cmd->bw_level_num = ddr->num_levels; + cmd->ddr_cmds_num = ddr->num_cmds; + cmd->ddr_wait_bitmask = ddr->wait_bitmask; + + for (i = 0; i < ddr->num_cmds; i++) + cmd->ddr_cmd_addrs[i] = ddr->addrs[i]; + + for (i = 0; i < ddr->num_levels; i++) + for (j = 0; j < ddr->num_cmds; j++) + cmd->ddr_cmd_data[i][j] = (u32) ddr->cmds[i][j]; + + if (!cnoc) + return; + + cmd->cnoc_cmds_num = cnoc->num_cmds; + cmd->cnoc_wait_bitmask = cnoc->wait_bitmask; + + for (i = 0; i < cnoc->num_cmds; i++) + cmd->cnoc_cmd_addrs[i] = cnoc->addrs[i]; + + for (i = 0; i < cnoc->num_levels; i++) + for (j = 0; j < cnoc->num_cmds; j++) + cmd->cnoc_cmd_data[i][j] = (u32) cnoc->cmds[i][j]; +} + +static int build_bw_table(struct adreno_device *adreno_dev) +{ + struct a6xx_gmu_device *gmu = to_a6xx_gmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct kgsl_pwrctrl *pwr = &device->pwrctrl; + struct rpmh_bw_votes *ddr, *cnoc = NULL; + u32 *cnoc_table; + u32 count; + + ddr = build_rpmh_bw_votes(a660_ddr_bcms, ARRAY_SIZE(a660_ddr_bcms), + pwr->ddr_table, pwr->ddr_table_count); + if (IS_ERR(ddr)) + return PTR_ERR(ddr); + + cnoc_table = kgsl_bus_get_table(device->pdev, "qcom,bus-table-cnoc", + &count); + + if (count > 0) + cnoc = build_rpmh_bw_votes(a660_cnoc_bcms, + ARRAY_SIZE(a660_cnoc_bcms), cnoc_table, count); + + kfree(cnoc_table); + + if (IS_ERR(cnoc)) { + free_rpmh_bw_votes(ddr); + return PTR_ERR(cnoc); + } + + build_bw_table_cmd(&gmu->hfi.bw_table, ddr, cnoc); + + free_rpmh_bw_votes(ddr); + free_rpmh_bw_votes(cnoc); + + return 0; +} + +int a6xx_build_rpmh_tables(struct adreno_device *adreno_dev) +{ + int ret; + + ret = build_dcvs_table(adreno_dev); + if (ret) + return ret; + + return build_bw_table(adreno_dev); +} diff --git a/drivers/gpu/msm/adreno_a6xx_snapshot.c b/drivers/gpu/msm/adreno_a6xx_snapshot.c index 8d7c9cdb8540..23f20f2802a2 100644 --- a/drivers/gpu/msm/adreno_a6xx_snapshot.c +++ b/drivers/gpu/msm/adreno_a6xx_snapshot.c @@ -46,74 +46,6 @@ static const unsigned int a6xx_pc_vs_cluster[] = { 0x9100, 0x9108, 0x9300, 0x9306, 0x9980, 0x9981, 0x9B00, 0x9B07, }; -static const unsigned int a630_rscc_snapshot_registers[] = { - 0x23400, 0x23434, 0x23436, 0x23436, 0x23480, 0x23484, 0x23489, 0x2348C, - 0x23491, 0x23494, 0x23499, 0x2349C, 0x234A1, 0x234A4, 0x234A9, 0x234AC, - 0x23500, 0x23502, 0x23504, 0x23507, 0x23514, 0x23519, 0x23524, 0x2352B, - 0x23580, 0x23597, 0x23740, 0x23741, 0x23744, 0x23747, 0x2374C, 0x23787, - 0x237EC, 0x237EF, 0x237F4, 0x2382F, 0x23894, 0x23897, 0x2389C, 0x238D7, - 0x2393C, 0x2393F, 0x23944, 0x2397F, -}; - -static const unsigned int a6xx_rscc_snapshot_registers[] = { - 0x23400, 0x23434, 0x23436, 0x23436, 0x23440, 0x23440, 0x23480, 0x23484, - 0x23489, 0x2348C, 0x23491, 0x23494, 0x23499, 0x2349C, 0x234A1, 0x234A4, - 0x234A9, 0x234AC, 0x23500, 0x23502, 0x23504, 0x23507, 0x23514, 0x23519, - 0x23524, 0x2352B, 0x23580, 0x23597, 0x23740, 0x23741, 0x23744, 0x23747, - 0x2374C, 0x23787, 0x237EC, 0x237EF, 0x237F4, 0x2382F, 0x23894, 0x23897, - 0x2389C, 0x238D7, 0x2393C, 0x2393F, 0x23944, 0x2397F, -}; - -static const unsigned int a650_rscc_registers[] = { - 0x38000, 0x38034, 0x38036, 0x38036, 0x38040, 0x38042, 0x38080, 0x38084, - 0x38089, 0x3808C, 0x38091, 0x38094, 0x38099, 0x3809C, 0x380A1, 0x380A4, - 0x380A9, 0x380AC, 0x38100, 0x38102, 0x38104, 0x38107, 0x38114, 0x38119, - 0x38124, 0x3812E, 0x38180, 0x38197, 0x38340, 0x38341, 0x38344, 0x38347, - 0x3834C, 0x3834F, 0x38351, 0x38354, 0x38356, 0x38359, 0x3835B, 0x3835E, - 0x38360, 0x38363, 0x38365, 0x38368, 0x3836A, 0x3836D, 0x3836F, 0x38372, - 0x383EC, 0x383EF, 0x383F4, 0x383F7, 0x383F9, 0x383FC, 0x383FE, 0x38401, - 0x38403, 0x38406, 0x38408, 0x3840B, 0x3840D, 0x38410, 0x38412, 0x38415, - 0x38417, 0x3841A, 0x38494, 0x38497, 0x3849C, 0x3849F, 0x384A1, 0x384A4, - 0x384A6, 0x384A9, 0x384AB, 0x384AE, 0x384B0, 0x384B3, 0x384B5, 0x384B8, - 0x384BA, 0x384BD, 0x384BF, 0x384C2, 0x3853C, 0x3853F, 0x38544, 0x38547, - 0x38549, 0x3854C, 0x3854E, 0x38551, 0x38553, 0x38556, 0x38558, 0x3855B, - 0x3855D, 0x38560, 0x38562, 0x38565, 0x38567, 0x3856A, 0x385E4, 0x385E7, - 0x385EC, 0x385EF, 0x385F1, 0x385F4, 0x385F6, 0x385F9, 0x385FB, 0x385FE, - 0x38600, 0x38603, 0x38605, 0x38608, 0x3860A, 0x3860D, 0x3860F, 0x38612, - 0x3868C, 0x3868F, 0x38694, 0x38697, 0x38699, 0x3869C, 0x3869E, 0x386A1, - 0x386A3, 0x386A6, 0x386A8, 0x386AB, 0x386AD, 0x386B0, 0x386B2, 0x386B5, - 0x386B7, 0x386BA, 0x38734, 0x38737, 0x3873C, 0x3873F, 0x38741, 0x38744, - 0x38746, 0x38749, 0x3874B, 0x3874E, 0x38750, 0x38753, 0x38755, 0x38758, - 0x3875A, 0x3875D, 0x3875F, 0x38762, 0x387DC, 0x387DF, 0x387E4, 0x387E7, - 0x387E9, 0x387EC, 0x387EE, 0x387F1, 0x387F3, 0x387F6, 0x387F8, 0x387FB, - 0x387FD, 0x38800, 0x38802, 0x38805, 0x38807, 0x3880A, 0x38884, 0x38887, - 0x3888C, 0x3888F, 0x38891, 0x38894, 0x38896, 0x38899, 0x3889B, 0x3889E, - 0x388A0, 0x388A3, 0x388A5, 0x388A8, 0x388AA, 0x388AD, 0x388AF, 0x388B2, - 0x3892C, 0x3892F, 0x38934, 0x38937, 0x38939, 0x3893C, 0x3893E, 0x38941, - 0x38943, 0x38946, 0x38948, 0x3894B, 0x3894D, 0x38950, 0x38952, 0x38955, - 0x38957, 0x3895A, 0x38B50, 0x38B51, 0x38B53, 0x38B55, 0x38B5A, 0x38B5A, - 0x38B5F, 0x38B5F, 0x38B64, 0x38B64, 0x38B69, 0x38B69, 0x38B6E, 0x38B6E, - 0x38B73, 0x38B73, 0x38BF8, 0x38BF8, 0x38BFD, 0x38BFD, 0x38C02, 0x38C02, - 0x38C07, 0x38C07, 0x38C0C, 0x38C0C, 0x38C11, 0x38C11, 0x38C16, 0x38C16, - 0x38C1B, 0x38C1B, 0x38CA0, 0x38CA0, 0x38CA5, 0x38CA5, 0x38CAA, 0x38CAA, - 0x38CAF, 0x38CAF, 0x38CB4, 0x38CB4, 0x38CB9, 0x38CB9, 0x38CBE, 0x38CBE, - 0x38CC3, 0x38CC3, 0x38D48, 0x38D48, 0x38D4D, 0x38D4D, 0x38D52, 0x38D52, - 0x38D57, 0x38D57, 0x38D5C, 0x38D5C, 0x38D61, 0x38D61, 0x38D66, 0x38D66, - 0x38D6B, 0x38D6B, 0x38DF0, 0x38DF0, 0x38DF5, 0x38DF5, 0x38DFA, 0x38DFA, - 0x38DFF, 0x38DFF, 0x38E04, 0x38E04, 0x38E09, 0x38E09, 0x38E0E, 0x38E0E, - 0x38E13, 0x38E13, 0x38E98, 0x38E98, 0x38E9D, 0x38E9D, 0x38EA2, 0x38EA2, - 0x38EA7, 0x38EA7, 0x38EAC, 0x38EAC, 0x38EB1, 0x38EB1, 0x38EB6, 0x38EB6, - 0x38EBB, 0x38EBB, 0x38F40, 0x38F40, 0x38F45, 0x38F45, 0x38F4A, 0x38F4A, - 0x38F4F, 0x38F4F, 0x38F54, 0x38F54, 0x38F59, 0x38F59, 0x38F5E, 0x38F5E, - 0x38F63, 0x38F63, 0x38FE8, 0x38FE8, 0x38FED, 0x38FED, 0x38FF2, 0x38FF2, - 0x38FF7, 0x38FF7, 0x38FFC, 0x38FFC, 0x39001, 0x39001, 0x39006, 0x39006, - 0x3900B, 0x3900B, 0x39090, 0x39090, 0x39095, 0x39095, 0x3909A, 0x3909A, - 0x3909F, 0x3909F, 0x390A4, 0x390A4, 0x390A9, 0x390A9, 0x390AE, 0x390AE, - 0x390B3, 0x390B3, 0x39138, 0x39138, 0x3913D, 0x3913D, 0x39142, 0x39142, - 0x39147, 0x39147, 0x3914C, 0x3914C, 0x39151, 0x39151, 0x39156, 0x39156, - 0x3915B, 0x3915B, -}; - static const unsigned int a650_isense_registers[] = { 0x22C00, 0x22C19, 0x22C26, 0x22C2D, 0x22C2F, 0x22C36, 0x22C40, 0x22C44, 0x22C50, 0x22C57, 0x22C60, 0x22C67, 0x22C80, 0x22C87, 0x22D25, 0x22D2A, @@ -1655,48 +1587,6 @@ static void _a6xx_do_crashdump(struct kgsl_device *device) crash_dump_valid = true; } -static size_t a6xx_snapshot_rscc_registers(struct kgsl_device *device, u8 *buf, - size_t remain, void *priv) -{ - struct kgsl_snapshot_regs *header = (struct kgsl_snapshot_regs *)buf; - struct kgsl_snapshot_registers *regs = priv; - unsigned int *data = (unsigned int *)(buf + sizeof(*header)); - int count = 0, j, k; - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - - /* Figure out how many registers we are going to dump */ - for (j = 0; j < regs->count; j++) { - int start = regs->regs[j * 2]; - int end = regs->regs[j * 2 + 1]; - - count += (end - start + 1); - } - - if (remain < (count * 8) + sizeof(*header)) { - SNAPSHOT_ERR_NOMEM(device, "RSCC REGISTERS"); - return 0; - } - - for (j = 0; j < regs->count; j++) { - unsigned int start = regs->regs[j * 2]; - unsigned int end = regs->regs[j * 2 + 1]; - - for (k = start; k <= end; k++) { - unsigned int val; - - adreno_rscc_regread(adreno_dev, - k - (adreno_dev->rscc_base >> 2), &val); - *data++ = k; - *data++ = val; - } - } - - header->count = count; - - /* Return the size of the section */ - return (count * 8) + sizeof(*header); -} - static size_t a6xx_snapshot_isense_registers(struct kgsl_device *device, u8 *buf, size_t remain, void *priv) { @@ -1784,9 +1674,6 @@ void a6xx_snapshot(struct adreno_device *adreno_dev, unsigned int i, roq_size; u32 hi, lo; - /* GMU TCM data dumped through AHB */ - gmu_core_dev_snapshot(device, snapshot); - /* * Dump debugbus data here to capture it for both * GMU and GPU snapshot. Debugbus data can be accessed @@ -1800,26 +1687,11 @@ void a6xx_snapshot(struct adreno_device *adreno_dev, if (adreno_is_a650_family(adreno_dev)) { struct kgsl_snapshot_registers r; - r.regs = a650_rscc_registers; - r.count = ARRAY_SIZE(a650_rscc_registers) / 2; - - kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, - snapshot, a6xx_snapshot_rscc_registers, &r); - r.regs = a650_isense_registers; r.count = ARRAY_SIZE(a650_isense_registers) / 2; kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_REGS, snapshot, a6xx_snapshot_isense_registers, &r); - } else if (adreno_is_a615_family(adreno_dev) || - adreno_is_a630(adreno_dev)) { - adreno_snapshot_registers(device, snapshot, - a630_rscc_snapshot_registers, - ARRAY_SIZE(a630_rscc_snapshot_registers) / 2); - } else if (adreno_is_a640(adreno_dev) || adreno_is_a680(adreno_dev)) { - adreno_snapshot_registers(device, snapshot, - a6xx_rscc_snapshot_registers, - ARRAY_SIZE(a6xx_rscc_snapshot_registers) / 2); } else if (adreno_is_a610(adreno_dev)) { adreno_snapshot_registers(device, snapshot, a6xx_gmu_wrapper_registers, diff --git a/drivers/gpu/msm/adreno_coresight.c b/drivers/gpu/msm/adreno_coresight.c index 67b35b4660cf..faa4006c2a93 100644 --- a/drivers/gpu/msm/adreno_coresight.c +++ b/drivers/gpu/msm/adreno_coresight.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. */ #include @@ -61,7 +61,7 @@ ssize_t adreno_coresight_show_register(struct device *dev, if (device->state == KGSL_STATE_ACTIVE || device->state == KGSL_STATE_NAP) { - if (!kgsl_active_count_get(device)) { + if (!adreno_active_count_get(adreno_dev)) { if (!is_cx) kgsl_regread(device, cattr->reg->offset, &cattr->reg->value); @@ -69,7 +69,7 @@ ssize_t adreno_coresight_show_register(struct device *dev, adreno_cx_dbgc_regread(device, cattr->reg->offset, &cattr->reg->value); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); } } @@ -116,7 +116,7 @@ ssize_t adreno_coresight_store_register(struct device *dev, /* Program the hardware if it is not power collapsed */ if (device->state == KGSL_STATE_ACTIVE || device->state == KGSL_STATE_NAP) { - if (!kgsl_active_count_get(device)) { + if (!adreno_active_count_get(adreno_dev)) { if (!is_cx) kgsl_regwrite(device, cattr->reg->offset, cattr->reg->value); @@ -125,7 +125,7 @@ ssize_t adreno_coresight_store_register(struct device *dev, cattr->reg->offset, cattr->reg->value); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); } } @@ -166,7 +166,7 @@ static void adreno_coresight_disable(struct coresight_device *csdev, mutex_lock(&device->mutex); - if (!kgsl_active_count_get(device)) { + if (!adreno_active_count_get(adreno_dev)) { if (cs_id == GPU_CORESIGHT_GX) for (i = 0; i < coresight->count; i++) kgsl_regwrite(device, @@ -176,7 +176,7 @@ static void adreno_coresight_disable(struct coresight_device *csdev, adreno_cx_dbgc_regwrite(device, coresight->registers[i].offset, 0); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); } if (cs_id == GPU_CORESIGHT_GX) @@ -289,10 +289,10 @@ static int adreno_coresight_enable(struct coresight_device *csdev, coresight->registers[i].initial; if (kgsl_state_is_awake(device)) { - ret = kgsl_active_count_get(device); + ret = adreno_active_count_get(adreno_dev); if (!ret) { ret = _adreno_coresight_set(adreno_dev, cs_id); - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); } } } diff --git a/drivers/gpu/msm/adreno_debugfs.c b/drivers/gpu/msm/adreno_debugfs.c index 2d2a6473089d..11a1df147bd8 100644 --- a/drivers/gpu/msm/adreno_debugfs.c +++ b/drivers/gpu/msm/adreno_debugfs.c @@ -8,6 +8,11 @@ #include "adreno.h" extern struct dentry *kgsl_debugfs_dir; +static void set_isdb(struct adreno_device *adreno_dev, void *priv) +{ + set_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv); +} + static int _isdb_set(void *data, u64 val) { struct kgsl_device *device = data; @@ -17,19 +22,11 @@ static int _isdb_set(void *data, u64 val) if (test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv)) return 0; - mutex_lock(&device->mutex); - /* * Bring down the GPU so we can bring it back up with the correct power * and clock settings */ - kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); - set_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv); - kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); - - mutex_unlock(&device->mutex); - - return 0; + return adreno_power_cycle(adreno_dev, set_isdb, NULL); } static int _isdb_get(void *data, u64 *val) @@ -57,14 +54,9 @@ static int _lm_limit_set(void *data, u64 val) else if (val < 3000) val = 3000; - adreno_dev->lm_limit = val; - - if (adreno_dev->lm_enabled) { - mutex_lock(&device->mutex); - kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); - kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); - mutex_unlock(&device->mutex); - } + if (adreno_dev->lm_enabled) + return adreno_power_cycle_u32(adreno_dev, + &adreno_dev->lm_limit, val); return 0; } diff --git a/drivers/gpu/msm/adreno_dispatch.c b/drivers/gpu/msm/adreno_dispatch.c index 9e2ec67844ae..8710fca99199 100644 --- a/drivers/gpu/msm/adreno_dispatch.c +++ b/drivers/gpu/msm/adreno_dispatch.c @@ -557,7 +557,7 @@ static int sendcmd(struct adreno_device *adreno_dev, if (dispatcher->inflight == 1 && !test_bit(ADRENO_DISPATCHER_POWER, &dispatcher->priv)) { /* Time to make the donuts. Turn on the GPU */ - ret = kgsl_active_count_get(device); + ret = adreno_active_count_get(adreno_dev); if (ret) { dispatcher->inflight--; dispatch_q->inflight--; @@ -608,7 +608,7 @@ static int sendcmd(struct adreno_device *adreno_dev, kgsl_pwrscale_midframe_timer_restart(device); } else { - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); clear_bit(ADRENO_DISPATCHER_POWER, &dispatcher->priv); } } @@ -2501,7 +2501,7 @@ static void _dispatcher_power_down(struct adreno_device *adreno_dev) del_timer_sync(&dispatcher->fault_timer); if (test_bit(ADRENO_DISPATCHER_POWER, &dispatcher->priv)) { - kgsl_active_count_put(device); + adreno_active_count_put(adreno_dev); clear_bit(ADRENO_DISPATCHER_POWER, &dispatcher->priv); } @@ -2909,7 +2909,7 @@ int adreno_dispatcher_idle(struct adreno_device *adreno_dev) struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher; int ret; - if (!test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv)) + if (device->state != KGSL_STATE_ACTIVE) return 0; /* diff --git a/drivers/gpu/msm/adreno_ioctl.c b/drivers/gpu/msm/adreno_ioctl.c index 11f3827801e4..4e2b5254d46c 100644 --- a/drivers/gpu/msm/adreno_ioctl.c +++ b/drivers/gpu/msm/adreno_ioctl.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved. */ #include @@ -68,7 +68,7 @@ long adreno_ioctl_perfcounter_get(struct kgsl_device_private *dev_priv, * active count inside that function. */ - result = adreno_perfcntr_active_oob_get(device); + result = adreno_perfcntr_active_oob_get(adreno_dev); if (result) { mutex_unlock(&device->mutex); return (long)result; @@ -87,7 +87,7 @@ long adreno_ioctl_perfcounter_get(struct kgsl_device_private *dev_priv, get->countable, PERFCOUNTER_FLAG_NONE); } - adreno_perfcntr_active_oob_put(device); + adreno_perfcntr_active_oob_put(adreno_dev); mutex_unlock(&device->mutex); diff --git a/drivers/gpu/msm/adreno_perfcounter.c b/drivers/gpu/msm/adreno_perfcounter.c index 19c961e97cb2..7b2966896400 100644 --- a/drivers/gpu/msm/adreno_perfcounter.c +++ b/drivers/gpu/msm/adreno_perfcounter.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved. */ #include @@ -248,7 +248,7 @@ int adreno_perfcounter_read_group(struct adreno_device *adreno_dev, mutex_lock(&device->mutex); - ret = adreno_perfcntr_active_oob_get(device); + ret = adreno_perfcntr_active_oob_get(adreno_dev); if (ret) { mutex_unlock(&device->mutex); goto done; @@ -277,7 +277,7 @@ int adreno_perfcounter_read_group(struct adreno_device *adreno_dev, } } - adreno_perfcntr_active_oob_put(device); + adreno_perfcntr_active_oob_put(adreno_dev); mutex_unlock(&device->mutex); diff --git a/drivers/gpu/msm/adreno_profile.c b/drivers/gpu/msm/adreno_profile.c index 6c52f75a1b12..fcd7b0b3d4e9 100644 --- a/drivers/gpu/msm/adreno_profile.c +++ b/drivers/gpu/msm/adreno_profile.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. */ #include @@ -691,7 +691,7 @@ static ssize_t profile_assignments_write(struct file *filep, goto error_unlock; } - ret = adreno_perfcntr_active_oob_get(device); + ret = adreno_perfcntr_active_oob_get(adreno_dev); if (ret) { size = ret; goto error_unlock; @@ -738,7 +738,7 @@ static ssize_t profile_assignments_write(struct file *filep, size = len; error_put: - adreno_perfcntr_active_oob_put(device); + adreno_perfcntr_active_oob_put(adreno_dev); error_unlock: mutex_unlock(&device->mutex); error_free: diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index 00857d97adeb..b7d134929625 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -104,7 +104,7 @@ static void adreno_ringbuffer_wptr(struct adreno_device *adreno_dev, * If WPTR update fails, take inline snapshot and trigger * recovery. */ - gmu_core_snapshot(device); + gmu_fault_snapshot(device); adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT_SKIP_SNAPSHOT); adreno_dispatcher_schedule(device); diff --git a/drivers/gpu/msm/adreno_sysfs.c b/drivers/gpu/msm/adreno_sysfs.c index f5fb8641fd5c..72f314257605 100644 --- a/drivers/gpu/msm/adreno_sysfs.c +++ b/drivers/gpu/msm/adreno_sysfs.c @@ -139,43 +139,13 @@ static bool _ft_hang_intr_status_show(struct adreno_device *adreno_dev) return true; } -static int pwrflag_store(struct adreno_device *adreno_dev, - unsigned int val, bool *flag) -{ - struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - - if (*flag == val) - return 0; - - mutex_lock(&device->mutex); - - /* Power down the GPU before changing the state */ - kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); - *flag = val; - kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); - - mutex_unlock(&device->mutex); - - return 0; -} - -static int _preemption_store(struct adreno_device *adreno_dev, bool val) +static void change_preemption(struct adreno_device *adreno_dev, void *priv) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct kgsl_context *context; struct adreno_context *drawctxt; int id; - mutex_lock(&device->mutex); - - if (!(ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION)) || - (test_bit(ADRENO_DEVICE_PREEMPTION, - &adreno_dev->priv) == val)) { - mutex_unlock(&device->mutex); - return 0; - } - - kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND); change_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv); adreno_dev->cur_rb = &(adreno_dev->ringbuffers[0]); @@ -186,12 +156,16 @@ static int _preemption_store(struct adreno_device *adreno_dev, bool val) drawctxt->rb = adreno_ctx_get_rb(adreno_dev, drawctxt); } write_unlock(&device->context_lock); +} - kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER); +static int _preemption_store(struct adreno_device *adreno_dev, bool val) +{ + if (!(ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION)) || + (test_bit(ADRENO_DEVICE_PREEMPTION, + &adreno_dev->priv) == val)) + return 0; - mutex_unlock(&device->mutex); - - return 0; + return adreno_power_cycle(adreno_dev, change_preemption, NULL); } static bool _preemption_show(struct adreno_device *adreno_dev) @@ -201,7 +175,11 @@ static bool _preemption_show(struct adreno_device *adreno_dev) static int _hwcg_store(struct adreno_device *adreno_dev, bool val) { - return pwrflag_store(adreno_dev, val, &adreno_dev->hwcg_enabled); + if (adreno_dev->hwcg_enabled == val) + return 0; + + return adreno_power_cycle_bool(adreno_dev, &adreno_dev->hwcg_enabled, + val); } static bool _hwcg_show(struct adreno_device *adreno_dev) @@ -211,10 +189,12 @@ static bool _hwcg_show(struct adreno_device *adreno_dev) static int _throttling_store(struct adreno_device *adreno_dev, bool val) { - if (!adreno_is_a540(adreno_dev)) + if (!adreno_is_a540(adreno_dev) || + adreno_dev->throttling_enabled == val) return 0; - return pwrflag_store(adreno_dev, val, &adreno_dev->throttling_enabled); + return adreno_power_cycle_bool(adreno_dev, + &adreno_dev->throttling_enabled, val); } static bool _throttling_show(struct adreno_device *adreno_dev) @@ -224,10 +204,12 @@ static bool _throttling_show(struct adreno_device *adreno_dev) static int _sptp_pc_store(struct adreno_device *adreno_dev, bool val) { - if (!ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC)) + if (!ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC) || + adreno_dev->sptp_pc_enabled == val) return 0; - return pwrflag_store(adreno_dev, val, &adreno_dev->sptp_pc_enabled); + return adreno_power_cycle_bool(adreno_dev, &adreno_dev->sptp_pc_enabled, + val); } static bool _sptp_pc_show(struct adreno_device *adreno_dev) @@ -237,10 +219,12 @@ static bool _sptp_pc_show(struct adreno_device *adreno_dev) static int _lm_store(struct adreno_device *adreno_dev, bool val) { - if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM)) + if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM) || + adreno_dev->lm_enabled == val) return 0; - return pwrflag_store(adreno_dev, val, &adreno_dev->lm_enabled); + return adreno_power_cycle_bool(adreno_dev, &adreno_dev->lm_enabled, + val); } static bool _lm_show(struct adreno_device *adreno_dev) @@ -277,7 +261,7 @@ static bool _acd_show(struct adreno_device *adreno_dev) static int _acd_store(struct adreno_device *adreno_dev, bool val) { - return gmu_core_acd_set(KGSL_DEVICE(adreno_dev), val); + return gmu_core_dev_acd_set(KGSL_DEVICE(adreno_dev), val); } static ssize_t _sysfs_store_u32(struct device *dev, diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h index 43b5a4c5c54a..8ffbf0b07b8b 100644 --- a/drivers/gpu/msm/kgsl.h +++ b/drivers/gpu/msm/kgsl.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2008-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2008-2020, The Linux Foundation. All rights reserved. */ #ifndef __KGSL_H #define __KGSL_H diff --git a/drivers/gpu/msm/kgsl_bus.c b/drivers/gpu/msm/kgsl_bus.c index 84d71dfa30ad..648c81152d0b 100644 --- a/drivers/gpu/msm/kgsl_bus.c +++ b/drivers/gpu/msm/kgsl_bus.c @@ -10,24 +10,6 @@ #include "kgsl_device.h" #include "kgsl_trace.h" -static int interconnect_bus_set(struct kgsl_device *device, int level, - u32 ab) -{ - struct kgsl_pwrctrl *pwr = &device->pwrctrl; - - if ((level == pwr->cur_buslevel) && (ab == pwr->cur_ab)) - return 0; - - pwr->cur_buslevel = level; - pwr->cur_ab = ab; - - icc_set_bw(pwr->icc_path, MBps_to_icc(ab), - kBps_to_icc(pwr->ddr_table[level])); - - trace_kgsl_buslevel(device, pwr->active_pwrlevel, level); - - return 0; -} static u32 _ab_buslevel_update(struct kgsl_pwrctrl *pwr, u32 ib) @@ -49,7 +31,7 @@ static u32 _ab_buslevel_update(struct kgsl_pwrctrl *pwr, } -void kgsl_bus_update(struct kgsl_device *device, bool on) +int kgsl_bus_update(struct kgsl_device *device, bool on) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; /* FIXME: this might be wrong? */ @@ -59,7 +41,7 @@ void kgsl_bus_update(struct kgsl_device *device, bool on) /* the bus should be ON to update the active frequency */ if (on && !(test_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->power_flags))) - return; + return 0; /* * If the bus should remain on calculate our request and submit it, * otherwise request bus level 0, off. @@ -78,7 +60,7 @@ void kgsl_bus_update(struct kgsl_device *device, bool on) /* buslevel is the IB vote, update the AB */ ab = _ab_buslevel_update(pwr, pwr->ddr_table[buslevel]); - pwr->bus_set(device, buslevel, ab); + return device->ftbl->gpu_bus_set(device, buslevel, ab); } static void validate_pwrlevels(struct kgsl_device *device, u32 *ibs, @@ -127,7 +109,7 @@ u32 *kgsl_bus_get_table(struct platform_device *pdev, if (num <= 0) return ERR_PTR(-EINVAL); - levels = devm_kcalloc(&pdev->dev, num, sizeof(*levels), GFP_KERNEL); + levels = kcalloc(num, sizeof(*levels), GFP_KERNEL); if (!levels) return ERR_PTR(-ENOMEM); @@ -171,15 +153,16 @@ done: pwr->icc_path = of_icc_get(&pdev->dev, NULL); if (IS_ERR(pwr->icc_path) && !gmu_core_scales_bandwidth(device)) { WARN(1, "The CPU has no way to set the GPU bus levels\n"); + + kfree(pwr->ddr_table); return PTR_ERR(pwr->icc_path); } - pwr->bus_set = interconnect_bus_set; - return 0; } void kgsl_bus_close(struct kgsl_device *device) { + kfree(device->pwrctrl.ddr_table); icc_put(device->pwrctrl.icc_path); } diff --git a/drivers/gpu/msm/kgsl_bus.h b/drivers/gpu/msm/kgsl_bus.h index 2329cce8000b..8fd6d6d8ef08 100644 --- a/drivers/gpu/msm/kgsl_bus.h +++ b/drivers/gpu/msm/kgsl_bus.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. */ #ifndef _KGSL_BUS_H @@ -11,7 +11,7 @@ struct platform_device; int kgsl_bus_init(struct kgsl_device *device, struct platform_device *pdev); void kgsl_bus_close(struct kgsl_device *device); -void kgsl_bus_update(struct kgsl_device *device, bool on); +int kgsl_bus_update(struct kgsl_device *device, bool on); u32 *kgsl_bus_get_table(struct platform_device *pdev, const char *name, int *count); diff --git a/drivers/gpu/msm/kgsl_device.h b/drivers/gpu/msm/kgsl_device.h index 861c7ca59989..eafb9b342a5d 100644 --- a/drivers/gpu/msm/kgsl_device.h +++ b/drivers/gpu/msm/kgsl_device.h @@ -35,7 +35,6 @@ #define KGSL_STATE_SUSPEND 0x00000010 #define KGSL_STATE_AWARE 0x00000020 #define KGSL_STATE_SLUMBER 0x00000080 -#define KGSL_STATE_RESET 0x00000100 /** * enum kgsl_event_results - result codes passed to an event callback when the @@ -168,6 +167,10 @@ struct kgsl_functable { int (*query_property_list)(struct kgsl_device *device, u32 *list, u32 count); bool (*is_hwcg_on)(struct kgsl_device *device); + /** @gpu_clock_set: Target specific function to set gpu frequency */ + int (*gpu_clock_set)(struct kgsl_device *device, u32 pwrlevel); + /** @gpu_bus_set: Target specific function to set gpu bandwidth */ + int (*gpu_bus_set)(struct kgsl_device *device, int bus_level, u32 ab); }; struct kgsl_ioctl { @@ -316,6 +319,8 @@ struct kgsl_device { rwlock_t event_groups_lock; /** @speed_bin: Speed bin for the GPU device if applicable */ u32 speed_bin; + /** @gmu_fault: Set when a gmu or rgmu fault is encountered */ + bool gmu_fault; }; #define KGSL_MMU_DEVICE(_mmu) \ @@ -481,7 +486,6 @@ struct kgsl_device_private { * @process: the process that caused the hang, if known. * @sysfs_read: Count of current reads via sysfs * @first_read: True until the snapshot read is started - * @gmu_fault: Snapshot collected when GMU fault happened * @recovered: True if GPU was recovered after previous snapshot */ struct kgsl_snapshot { @@ -505,7 +509,6 @@ struct kgsl_snapshot { struct kgsl_process_private *process; unsigned int sysfs_read; bool first_read; - bool gmu_fault; bool recovered; struct kgsl_device *device; }; @@ -589,9 +592,6 @@ static inline int kgsl_state_is_awake(struct kgsl_device *device) if (device->state == KGSL_STATE_ACTIVE || device->state == KGSL_STATE_AWARE) return true; - else if (gmu_core_isenabled(device) && - test_bit(GMU_CLK_ON, &device->gmu_core.flags)) - return true; else return false; } diff --git a/drivers/gpu/msm/kgsl_gmu.c b/drivers/gpu/msm/kgsl_gmu.c index 9a833f301da9..d6316cf3b9fd 100644 --- a/drivers/gpu/msm/kgsl_gmu.c +++ b/drivers/gpu/msm/kgsl_gmu.c @@ -1739,20 +1739,12 @@ static void gmu_remove(struct kgsl_device *device) memset(&device->gmu_core, 0, sizeof(device->gmu_core)); } -static bool gmu_regulator_isenabled(struct kgsl_device *device) -{ - struct gmu_device *gmu = KGSL_GMU_DEVICE(device); - - return (gmu->gx_gdsc && regulator_is_enabled(gmu->gx_gdsc)); -} - static struct gmu_core_ops gmu_ops = { .init = gmu_init, .start = gmu_start, .stop = gmu_stop, .dcvs_set = gmu_dcvs_set, .snapshot = gmu_snapshot, - .regulator_isenabled = gmu_regulator_isenabled, .suspend = gmu_suspend, .acd_set = gmu_acd_set, }; diff --git a/drivers/gpu/msm/kgsl_gmu_core.c b/drivers/gpu/msm/kgsl_gmu_core.c index 87df810bd63a..4654507ee743 100644 --- a/drivers/gpu/msm/kgsl_gmu_core.c +++ b/drivers/gpu/msm/kgsl_gmu_core.c @@ -11,8 +11,8 @@ #include "kgsl_trace.h" static const struct of_device_id gmu_match_table[] = { - { .compatible = "qcom,gpu-gmu", .data = &kgsl_gmu_driver }, - { .compatible = "qcom,gpu-rgmu", .data = &kgsl_rgmu_driver }, + { .compatible = "qcom,gpu-gmu", .data = &a6xx_gmu_driver }, + { .compatible = "qcom,gpu-rgmu", .data = &a6xx_rgmu_driver }, {}, }; @@ -51,7 +51,7 @@ bool gmu_core_isenabled(struct kgsl_device *device) bool gmu_core_gpmu_isenabled(struct kgsl_device *device) { - return (device->gmu_core.core_ops != NULL); + return (device->gmu_core.dev_ops != NULL); } bool gmu_core_scales_bandwidth(struct kgsl_device *device) @@ -64,83 +64,16 @@ bool gmu_core_scales_bandwidth(struct kgsl_device *device) return false; } -int gmu_core_init(struct kgsl_device *device) +int gmu_core_dev_acd_set(struct kgsl_device *device, bool val) { - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); + struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - if (gmu_core_ops && gmu_core_ops->init) - return gmu_core_ops->init(device); - - return 0; -} - -int gmu_core_start(struct kgsl_device *device) -{ - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); - - if (gmu_core_ops && gmu_core_ops->start) - return gmu_core_ops->start(device); + if (ops && ops->acd_set) + return ops->acd_set(device, val); return -EINVAL; } -void gmu_core_stop(struct kgsl_device *device) -{ - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); - - if (gmu_core_ops && gmu_core_ops->stop) - gmu_core_ops->stop(device); -} - -int gmu_core_suspend(struct kgsl_device *device) -{ - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); - - if (gmu_core_ops && gmu_core_ops->suspend) - return gmu_core_ops->suspend(device); - - return -EINVAL; -} - -void gmu_core_snapshot(struct kgsl_device *device) -{ - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); - - if (gmu_core_ops && gmu_core_ops->snapshot) - gmu_core_ops->snapshot(device); -} - -int gmu_core_dcvs_set(struct kgsl_device *device, int gpu_pwrlevel, - int bus_level) -{ - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); - - if (gmu_core_ops && gmu_core_ops->dcvs_set) - return gmu_core_ops->dcvs_set(device, gpu_pwrlevel, bus_level); - - return -EINVAL; -} - -int gmu_core_acd_set(struct kgsl_device *device, bool val) -{ - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); - - if (gmu_core_ops && gmu_core_ops->acd_set) - return gmu_core_ops->acd_set(device, val); - - return -EINVAL; -} - -bool gmu_core_regulator_isenabled(struct kgsl_device *device) -{ - struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device); - - if (gmu_core_ops && gmu_core_ops->regulator_isenabled) - return gmu_core_ops->regulator_isenabled(device); - - return false; -} - bool gmu_core_is_register_offset(struct kgsl_device *device, unsigned int offsetwords) { @@ -241,43 +174,6 @@ void gmu_core_dev_oob_clear(struct kgsl_device *device, enum oob_request req) ops->oob_clear(device, req); } -int gmu_core_dev_hfi_start_msg(struct kgsl_device *device) -{ - struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - - if (ops && ops->hfi_start_msg) - return ops->hfi_start_msg(device); - - return 0; -} - -int gmu_core_dev_wait_for_lowest_idle(struct kgsl_device *device) -{ - struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - - if (ops && ops->wait_for_lowest_idle) - return ops->wait_for_lowest_idle(device); - - return 0; -} - -void gmu_core_dev_enable_lm(struct kgsl_device *device) -{ - struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - - if (ops && ops->enable_lm) - ops->enable_lm(device); -} - -void gmu_core_dev_snapshot(struct kgsl_device *device, - struct kgsl_snapshot *snapshot) -{ - struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - - if (ops && ops->snapshot) - ops->snapshot(device, snapshot); -} - void gmu_core_dev_cooperative_reset(struct kgsl_device *device) { @@ -317,14 +213,6 @@ int gmu_core_dev_ifpc_store(struct kgsl_device *device, unsigned int val) return -EINVAL; } -void gmu_core_dev_prepare_stop(struct kgsl_device *device) -{ - struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - - if (ops && ops->prepare_stop) - ops->prepare_stop(device); -} - int gmu_core_dev_wait_for_active_transition(struct kgsl_device *device) { struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); @@ -334,13 +222,3 @@ int gmu_core_dev_wait_for_active_transition(struct kgsl_device *device) return 0; } - -u64 gmu_core_dev_read_alwayson(struct kgsl_device *device) -{ - struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - - if (ops && ops->read_alwayson) - return ops->read_alwayson(device); - - return 0; -} diff --git a/drivers/gpu/msm/kgsl_gmu_core.h b/drivers/gpu/msm/kgsl_gmu_core.h index b35e573c9be2..dd2685c6f1a5 100644 --- a/drivers/gpu/msm/kgsl_gmu_core.h +++ b/drivers/gpu/msm/kgsl_gmu_core.h @@ -7,7 +7,6 @@ /* GMU_DEVICE - Given an KGSL device return the GMU specific struct */ #define GMU_DEVICE_OPS(_a) ((_a)->gmu_core.dev_ops) -#define GMU_CORE_OPS(_a) ((_a)->gmu_core.core_ops) #define NUM_BW_LEVELS 100 #define MAX_GX_LEVELS 16 @@ -35,7 +34,6 @@ enum gmu_core_boot { /* Bits for the flags field in the gmu structure */ enum gmu_core_flags { GMU_BOOT_INIT_DONE = 0, - GMU_CLK_ON, GMU_HFI_ON, GMU_FAULT, GMU_DCVS_REPLAY, @@ -100,44 +98,17 @@ struct device_node; struct kgsl_device; struct kgsl_snapshot; -struct gmu_core_ops { - int (*dcvs_set)(struct kgsl_device *device, - int gpu_pwrlevel, int bus_level); - int (*init)(struct kgsl_device *device); - int (*start)(struct kgsl_device *device); - void (*stop)(struct kgsl_device *device); - void (*snapshot)(struct kgsl_device *device); - bool (*regulator_isenabled)(struct kgsl_device *device); - int (*suspend)(struct kgsl_device *device); - int (*acd_set)(struct kgsl_device *device, bool val); -}; - struct gmu_dev_ops { - int (*load_firmware)(struct kgsl_device *device); int (*oob_set)(struct kgsl_device *device, enum oob_request req); void (*oob_clear)(struct kgsl_device *device, enum oob_request req); - void (*bcl_config)(struct kgsl_device *device, bool on); - void (*irq_enable)(struct kgsl_device *device); - void (*irq_disable)(struct kgsl_device *device); - int (*hfi_start_msg)(struct kgsl_device *device); - void (*enable_lm)(struct kgsl_device *device); - int (*rpmh_gpu_pwrctrl)(struct kgsl_device *device, unsigned int ops, - unsigned int arg1, unsigned int arg2); - int (*wait_for_lowest_idle)(struct kgsl_device *device); - int (*wait_for_gmu_idle)(struct kgsl_device *device); bool (*gx_is_on)(struct kgsl_device *device); - void (*prepare_stop)(struct kgsl_device *device); int (*ifpc_store)(struct kgsl_device *device, unsigned int val); unsigned int (*ifpc_show)(struct kgsl_device *device); - void (*snapshot)(struct kgsl_device *device, - struct kgsl_snapshot *shapshot); void (*cooperative_reset)(struct kgsl_device *device); void (*halt_execution)(struct kgsl_device *device); int (*wait_for_active_transition)(struct kgsl_device *device); bool (*scales_bandwidth)(struct kgsl_device *device); - u64 (*read_alwayson)(struct kgsl_device *device); - const unsigned int gmu2host_intr_mask; - const unsigned int gmu_ao_intr_mask; + int (*acd_set)(struct kgsl_device *device, bool val); }; /** @@ -148,7 +119,6 @@ struct gmu_dev_ops { * gmu registers using offset defined in GPU register space. * @reg_len: GMU registers length * @reg_virt: GMU CSR virtual address - * @core_ops: Pointer to gmu core operations * @dev_ops: Pointer to gmu device operations * @flags: GMU flags */ @@ -157,31 +127,22 @@ struct gmu_core_device { unsigned int gmu2gpu_offset; unsigned int reg_len; void __iomem *reg_virt; - struct gmu_core_ops *core_ops; struct gmu_dev_ops *dev_ops; unsigned long flags; }; -extern struct platform_driver kgsl_gmu_driver; -extern struct platform_driver kgsl_rgmu_driver; +extern struct platform_driver a6xx_gmu_driver; +extern struct platform_driver a6xx_rgmu_driver; /* GMU core functions */ void __init gmu_core_register(void); void __exit gmu_core_unregister(void); -int gmu_core_init(struct kgsl_device *device); -int gmu_core_start(struct kgsl_device *device); -void gmu_core_stop(struct kgsl_device *device); -int gmu_core_suspend(struct kgsl_device *device); -void gmu_core_snapshot(struct kgsl_device *device); bool gmu_core_gpmu_isenabled(struct kgsl_device *device); bool gmu_core_scales_bandwidth(struct kgsl_device *device); bool gmu_core_isenabled(struct kgsl_device *device); -int gmu_core_dcvs_set(struct kgsl_device *device, int gpu_pwrlevel, - int bus_level); -int gmu_core_acd_set(struct kgsl_device *device, bool val); -bool gmu_core_regulator_isenabled(struct kgsl_device *device); +int gmu_core_dev_acd_set(struct kgsl_device *device, bool val); bool gmu_core_is_register_offset(struct kgsl_device *device, unsigned int offsetwords); void gmu_core_regread(struct kgsl_device *device, unsigned int offsetwords, @@ -207,17 +168,10 @@ void gmu_core_regrmw(struct kgsl_device *device, unsigned int offsetwords, unsigned int mask, unsigned int bits); int gmu_core_dev_oob_set(struct kgsl_device *device, enum oob_request req); void gmu_core_dev_oob_clear(struct kgsl_device *device, enum oob_request req); -int gmu_core_dev_hfi_start_msg(struct kgsl_device *device); -int gmu_core_dev_wait_for_lowest_idle(struct kgsl_device *device); -void gmu_core_dev_enable_lm(struct kgsl_device *device); -void gmu_core_dev_snapshot(struct kgsl_device *device, - struct kgsl_snapshot *snapshot); bool gmu_core_dev_gx_is_on(struct kgsl_device *device); int gmu_core_dev_ifpc_show(struct kgsl_device *device); int gmu_core_dev_ifpc_store(struct kgsl_device *device, unsigned int val); -void gmu_core_dev_prepare_stop(struct kgsl_device *device); int gmu_core_dev_wait_for_active_transition(struct kgsl_device *device); void gmu_core_dev_cooperative_reset(struct kgsl_device *device); -u64 gmu_core_dev_read_alwayson(struct kgsl_device *device); #endif /* __KGSL_GMU_CORE_H */ diff --git a/drivers/gpu/msm/kgsl_pwrctrl.c b/drivers/gpu/msm/kgsl_pwrctrl.c index 692451e863b3..2a7dec418778 100644 --- a/drivers/gpu/msm/kgsl_pwrctrl.c +++ b/drivers/gpu/msm/kgsl_pwrctrl.c @@ -42,12 +42,9 @@ static const char * const clocks[] = { static void kgsl_pwrctrl_clk(struct kgsl_device *device, int state, int requested_state); -static void kgsl_pwrctrl_axi(struct kgsl_device *device, int state); static int kgsl_pwrctrl_pwrrail(struct kgsl_device *device, int state); static void kgsl_pwrctrl_set_state(struct kgsl_device *device, unsigned int state); -static void kgsl_pwrctrl_request_state(struct kgsl_device *device, - unsigned int state); static int _isense_clk_set_rate(struct kgsl_pwrctrl *pwr, int level); static int kgsl_pwrctrl_clk_set_rate(struct clk *grp_clk, unsigned int freq, const char *name); @@ -103,33 +100,6 @@ static unsigned int _adjust_pwrlevel(struct kgsl_pwrctrl *pwr, int level, return level; } -/** - * kgsl_clk_set_rate() - set GPU clock rate - * @device: Pointer to the kgsl_device struct - * @pwrlevel: power level in pwrlevels[] table - */ -int kgsl_clk_set_rate(struct kgsl_device *device, - unsigned int pwrlevel) -{ - struct kgsl_pwrctrl *pwr = &device->pwrctrl; - struct kgsl_pwrlevel *pl = &pwr->pwrlevels[pwrlevel]; - int ret = 0; - - /* GMU scales GPU freq */ - if (gmu_core_gpmu_isenabled(device)) - ret = gmu_core_dcvs_set(device, pwrlevel, INVALID_DCVS_IDX); - else - /* Linux clock driver scales GPU freq */ - ret = kgsl_pwrctrl_clk_set_rate(pwr->grp_clks[0], - pl->gpu_freq, clocks[0]); - - if (ret) - dev_err(device->dev, "GPU clk freq set failure: %d\n", - ret); - - return ret; -} - /** * kgsl_pwrctrl_pwrlevel_change_settings() - Program h/w during powerlevel * transitions @@ -204,10 +174,6 @@ void kgsl_pwrctrl_pwrlevel_change(struct kgsl_device *device, new_level = kgsl_pwrctrl_adjust_pwrlevel(device, new_level); - if (new_level == old_level && - !test_bit(GMU_DCVS_REPLAY, &device->gmu_core.flags)) - return; - kgsl_pwrscale_update_stats(device); /* @@ -235,7 +201,7 @@ void kgsl_pwrctrl_pwrlevel_change(struct kgsl_device *device, pwrlevel = &pwr->pwrlevels[pwr->active_pwrlevel]; /* Change register settings if any BEFORE pwrlevel change*/ kgsl_pwrctrl_pwrlevel_change_settings(device, 0); - kgsl_clk_set_rate(device, pwr->active_pwrlevel); + device->ftbl->gpu_clock_set(device, pwr->active_pwrlevel); _isense_clk_set_rate(pwr, pwr->active_pwrlevel); trace_kgsl_pwrlevel(device, @@ -1218,7 +1184,7 @@ static void kgsl_pwrctrl_clk(struct kgsl_device *device, int state, (requested_state != KGSL_STATE_NAP)) { for (i = KGSL_MAX_CLKS - 1; i > 0; i--) clk_unprepare(pwr->grp_clks[i]); - kgsl_clk_set_rate(device, + device->ftbl->gpu_clock_set(device, pwr->num_pwrlevels - 1); _isense_clk_set_rate(pwr, pwr->num_pwrlevels - 1); @@ -1231,7 +1197,7 @@ static void kgsl_pwrctrl_clk(struct kgsl_device *device, int state, for (i = KGSL_MAX_CLKS - 1; i > 0; i--) clk_unprepare(pwr->grp_clks[i]); if ((pwr->pwrlevels[0].gpu_freq > 0)) { - kgsl_clk_set_rate(device, + device->ftbl->gpu_clock_set(device, pwr->num_pwrlevels - 1); _isense_clk_set_rate(pwr, pwr->num_pwrlevels - 1); @@ -1245,7 +1211,7 @@ static void kgsl_pwrctrl_clk(struct kgsl_device *device, int state, /* High latency clock maintenance. */ if (device->state != KGSL_STATE_NAP) { if (pwr->pwrlevels[0].gpu_freq > 0) { - kgsl_clk_set_rate(device, + device->ftbl->gpu_clock_set(device, pwr->active_pwrlevel); _isense_clk_set_rate(pwr, pwr->active_pwrlevel); @@ -1278,26 +1244,28 @@ static void kgsl_pwrctrl_clk(struct kgsl_device *device, int state, } } -static void kgsl_pwrctrl_axi(struct kgsl_device *device, int state) +int kgsl_pwrctrl_axi(struct kgsl_device *device, int state) { struct kgsl_pwrctrl *pwr = &device->pwrctrl; if (test_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->ctrl_flags)) - return; + return 0; if (state == KGSL_PWRFLAGS_OFF) { if (test_and_clear_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->power_flags)) { trace_kgsl_bus(device, state); - kgsl_bus_update(device, false); + return kgsl_bus_update(device, false); } } else if (state == KGSL_PWRFLAGS_ON) { if (!test_and_set_bit(KGSL_PWRFLAGS_AXI_ON, &pwr->power_flags)) { trace_kgsl_bus(device, state); - kgsl_bus_update(device, true); + return kgsl_bus_update(device, true); } } + + return 0; } static int enable_regulator(struct device *dev, struct regulator *regulator, @@ -1485,8 +1453,6 @@ static int kgsl_pwrctrl_clk_set_rate(struct clk *grp_clk, unsigned int freq, return ret; } -static void kgsl_idle_check(struct work_struct *work); - int kgsl_pwrctrl_init(struct kgsl_device *device) { int i, result, freq; @@ -1516,7 +1482,6 @@ int kgsl_pwrctrl_init(struct kgsl_device *device) return -EINVAL; } - INIT_WORK(&device->idle_check_ws, kgsl_idle_check); init_waitqueue_head(&device->active_cnt_wq); /* Initialize the user and thermal clock constraints */ @@ -1538,7 +1503,8 @@ int kgsl_pwrctrl_init(struct kgsl_device *device) pwr->pwrlevels[i].gpu_freq = freq; } - kgsl_clk_set_rate(device, pwr->num_pwrlevels - 1); + clk_set_rate(pwr->grp_clks[0], + pwr->pwrlevels[pwr->num_pwrlevels - 1].gpu_freq); freq = clk_round_rate(pwr->grp_clks[6], KGSL_RBBMTIMER_CLK_FREQ); if (freq > 0) @@ -1575,12 +1541,7 @@ void kgsl_pwrctrl_close(struct kgsl_device *device) pm_runtime_disable(&device->pdev->dev); } -/* - * This function is called for work that is queued by the interrupt - * handler or the idle timer. It attempts to transition to a clocks - * off state if the active_cnt is 0 and the hardware is idle. - */ -static void kgsl_idle_check(struct work_struct *work) +void kgsl_idle_check(struct work_struct *work) { struct kgsl_device *device = container_of(work, struct kgsl_device, idle_check_ws); @@ -1692,14 +1653,6 @@ static int kgsl_pwrctrl_enable(struct kgsl_device *device) kgsl_pwrctrl_pwrlevel_change(device, level); - if (gmu_core_gpmu_isenabled(device)) { - int ret = gmu_core_start(device); - - if (!ret) - kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_ON); - return ret; - } - /* Order pwrrail/clk sequence based upon platform */ status = kgsl_pwrctrl_pwrrail(device, KGSL_PWRFLAGS_ON); if (status) @@ -1720,11 +1673,6 @@ static void kgsl_pwrctrl_disable(struct kgsl_device *device) dev_err(device->dev, "Could not clear l3_vote: %d\n", status); - if (gmu_core_gpmu_isenabled(device)) { - kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_OFF); - return gmu_core_stop(device); - } - /* Order pwrrail/clk sequence based upon platform */ device->ftbl->regulator_disable(device); kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_OFF); @@ -1757,18 +1705,6 @@ static int _init(struct kgsl_device *device) int status = 0; switch (device->state) { - case KGSL_STATE_RESET: - if (gmu_core_isenabled(device)) { - /* - * If we fail a INIT -> AWARE transition, we will - * transition back to INIT. However, we must hard reset - * the GMU as we go back to INIT. This is done by - * forcing a RESET -> INIT transition. - */ - gmu_core_suspend(device); - kgsl_pwrctrl_set_state(device, KGSL_STATE_INIT); - } - break; case KGSL_STATE_NAP: /* Force power on to do the stop */ status = kgsl_pwrctrl_enable(device); @@ -1878,12 +1814,6 @@ _aware(struct kgsl_device *device) int status = 0; switch (device->state) { - case KGSL_STATE_RESET: - if (!gmu_core_gpmu_isenabled(device)) - break; - kgsl_pwrctrl_irq(device, KGSL_PWRFLAGS_OFF); - status = gmu_core_start(device); - break; case KGSL_STATE_INIT: status = kgsl_pwrctrl_enable(device); break; @@ -1903,18 +1833,7 @@ _aware(struct kgsl_device *device) status = -EINVAL; } - if (status && gmu_core_isenabled(device)) - /* - * If a SLUMBER/INIT -> AWARE fails, we transition back to - * SLUMBER/INIT state. We must hard reset the GMU while - * transitioning back to SLUMBER/INIT. A RESET -> AWARE - * transition is different. It happens when dispatcher is - * attempting reset/recovery as part of fault handling. If it - * fails, we should still transition back to RESET in case - * we want to attempt another reset/recovery. - */ - kgsl_pwrctrl_set_state(device, KGSL_STATE_RESET); - else + if (!status) kgsl_pwrctrl_set_state(device, KGSL_STATE_AWARE); return status; @@ -1946,8 +1865,6 @@ _nap(struct kgsl_device *device) kgsl_pwrctrl_set_state(device, KGSL_STATE_NAP); /* fallthrough */ case KGSL_STATE_SLUMBER: - case KGSL_STATE_RESET: - break; case KGSL_STATE_AWARE: dev_warn(device->dev, "transition AWARE -> NAP is not permitted\n"); @@ -1993,13 +1910,6 @@ _slumber(struct kgsl_device *device) kgsl_pwrctrl_disable(device); kgsl_pwrctrl_set_state(device, KGSL_STATE_SLUMBER); break; - case KGSL_STATE_RESET: - if (gmu_core_isenabled(device)) { - /* Reset the GMU if we failed to boot the GMU */ - gmu_core_suspend(device); - kgsl_pwrctrl_set_state(device, KGSL_STATE_SLUMBER); - } - break; default: kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); break; @@ -2088,9 +1998,6 @@ int kgsl_pwrctrl_change_state(struct kgsl_device *device, int state) case KGSL_STATE_SUSPEND: status = _suspend(device); break; - case KGSL_STATE_RESET: - kgsl_pwrctrl_set_state(device, KGSL_STATE_RESET); - break; default: dev_err(device->dev, "bad state request 0x%x\n", state); kgsl_pwrctrl_request_state(device, KGSL_STATE_NONE); @@ -2116,7 +2023,7 @@ static void kgsl_pwrctrl_set_state(struct kgsl_device *device, spin_unlock(&device->submit_lock); } -static void kgsl_pwrctrl_request_state(struct kgsl_device *device, +void kgsl_pwrctrl_request_state(struct kgsl_device *device, unsigned int state) { if (state != KGSL_STATE_NONE && state != device->requested_state) @@ -2141,90 +2048,12 @@ const char *kgsl_pwrstate_to_str(unsigned int state) return "SUSPEND"; case KGSL_STATE_SLUMBER: return "SLUMBER"; - case KGSL_STATE_RESET: - return "RESET"; default: break; } return "UNKNOWN"; } - -/** - * kgsl_active_count_get() - Increase the device active count - * @device: Pointer to a KGSL device - * - * Increase the active count for the KGSL device and turn on - * clocks if this is the first reference. Code paths that need - * to touch the hardware or wait for the hardware to complete - * an operation must hold an active count reference until they - * are finished. An error code will be returned if waking the - * device fails. The device mutex must be held while *calling - * this function. - */ -int kgsl_active_count_get(struct kgsl_device *device) -{ - int ret = 0; - - if (WARN_ON(!mutex_is_locked(&device->mutex))) - return -EINVAL; - - if ((atomic_read(&device->active_cnt) == 0) && - (device->state != KGSL_STATE_ACTIVE)) { - mutex_unlock(&device->mutex); - wait_for_completion(&device->hwaccess_gate); - mutex_lock(&device->mutex); - device->pwrctrl.superfast = true; - ret = kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE); - } - if (ret == 0) - atomic_inc(&device->active_cnt); - trace_kgsl_active_count(device, - (unsigned long) __builtin_return_address(0)); - return ret; -} - -/** - * kgsl_active_count_put() - Decrease the device active count - * @device: Pointer to a KGSL device - * - * Decrease the active count for the KGSL device and turn off - * clocks if there are no remaining references. This function will - * transition the device to NAP if there are no other pending state - * changes. It also completes the suspend gate. The device mutex must - * be held while calling this function. - */ -void kgsl_active_count_put(struct kgsl_device *device) -{ - if (WARN_ON(!mutex_is_locked(&device->mutex))) - return; - - if (WARN(atomic_read(&device->active_cnt) == 0, - "Unbalanced get/put calls to KGSL active count\n")) - return; - - if (atomic_dec_and_test(&device->active_cnt)) { - bool nap_on = !(device->pwrctrl.ctrl_flags & - BIT(KGSL_PWRFLAGS_NAP_OFF)); - if (nap_on && device->state == KGSL_STATE_ACTIVE && - device->requested_state == KGSL_STATE_NONE) { - kgsl_pwrctrl_request_state(device, KGSL_STATE_NAP); - kgsl_schedule_work(&device->idle_check_ws); - } else if (!nap_on) { - kgsl_pwrscale_update_stats(device); - kgsl_pwrscale_update(device); - } - - mod_timer(&device->idle_timer, - jiffies + device->pwrctrl.interval_timeout); - } - - trace_kgsl_active_count(device, - (unsigned long) __builtin_return_address(0)); - - wake_up(&device->active_cnt_wq); -} - static int _check_active_count(struct kgsl_device *device, int count) { /* Return 0 if the active count is greater than the desired value */ @@ -2283,5 +2112,5 @@ int kgsl_pwrctrl_set_default_gpu_pwrlevel(struct kgsl_device *device) pwr->previous_pwrlevel = old_level; /* Request adjusted DCVS level */ - return kgsl_clk_set_rate(device, pwr->active_pwrlevel); + return device->ftbl->gpu_clock_set(device, pwr->active_pwrlevel); } diff --git a/drivers/gpu/msm/kgsl_pwrctrl.h b/drivers/gpu/msm/kgsl_pwrctrl.h index ebc3d208472f..483ab3464576 100644 --- a/drivers/gpu/msm/kgsl_pwrctrl.h +++ b/drivers/gpu/msm/kgsl_pwrctrl.h @@ -154,8 +154,6 @@ struct kgsl_pwrctrl { int cur_buslevel; /** @bus_max: The maximum bandwidth available to the device */ unsigned long bus_max; - /** @bus_set: Function for setting the bus constraints */ - int (*bus_set)(struct kgsl_device *device, int buslevel, u32 ab); struct kgsl_pwr_constraint constraint; bool superfast; unsigned int gpu_bimc_int_clk_freq; @@ -175,8 +173,7 @@ void kgsl_pwrctrl_pwrlevel_change(struct kgsl_device *device, unsigned int level); int kgsl_pwrctrl_init_sysfs(struct kgsl_device *device); int kgsl_pwrctrl_change_state(struct kgsl_device *device, int state); -int kgsl_clk_set_rate(struct kgsl_device *device, - unsigned int pwrlevel); + unsigned int kgsl_pwrctrl_adjust_pwrlevel(struct kgsl_device *device, unsigned int new_level); @@ -197,12 +194,38 @@ kgsl_pwrctrl_active_freq(struct kgsl_pwrctrl *pwr) return pwr->pwrlevels[pwr->active_pwrlevel].gpu_freq; } -int __must_check kgsl_active_count_get(struct kgsl_device *device); -void kgsl_active_count_put(struct kgsl_device *device); int kgsl_active_count_wait(struct kgsl_device *device, int count); void kgsl_pwrctrl_busy_time(struct kgsl_device *device, u64 time, u64 busy); void kgsl_pwrctrl_set_constraint(struct kgsl_device *device, struct kgsl_pwr_constraint *pwrc, uint32_t id); int kgsl_pwrctrl_set_default_gpu_pwrlevel(struct kgsl_device *device); +/** + * kgsl_pwrctrl_request_state - Request a specific power state + * @device: Pointer to the kgsl device + * @state: Power state requested + */ +void kgsl_pwrctrl_request_state(struct kgsl_device *device, u32 state); + +/** + * kgsl_pwrctrl_axi - Propagate bus votes during slumber entry and exit + * @device: Pointer to the kgsl device + * @state: Whether we are going to slumber or coming out of slumber + * + * This function will propagate the default bus vote when coming out of + * slumber and set bus bandwidth to 0 when going into slumber + * + * Return: 0 on success or negative error on failure + */ +int kgsl_pwrctrl_axi(struct kgsl_device *device, int state); + +/** + * kgsl_idle_check - kgsl idle function + * @work: work item being run by the function + * + * This function is called for work that is queued by the interrupt + * handler or the idle timer. It attempts to transition to a clocks + * off state if the active_cnt is 0 and the hardware is idle. + */ +void kgsl_idle_check(struct work_struct *work); #endif /* __KGSL_PWRCTRL_H */ diff --git a/drivers/gpu/msm/kgsl_rgmu.c b/drivers/gpu/msm/kgsl_rgmu.c deleted file mode 100644 index dcd4f679769c..000000000000 --- a/drivers/gpu/msm/kgsl_rgmu.c +++ /dev/null @@ -1,469 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "adreno.h" -#include "kgsl_device.h" -#include "kgsl_rgmu.h" -#include "kgsl_util.h" - -#define RGMU_CLK_FREQ 200000000 - -static int rgmu_irq_probe(struct kgsl_device *device) -{ - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - int ret; - - ret = kgsl_request_irq(rgmu->pdev, "kgsl_oob", oob_irq_handler, device); - if (ret < 0) - return ret; - - rgmu->oob_interrupt_num = ret; - - ret = kgsl_request_irq(rgmu->pdev, - "kgsl_rgmu", rgmu_irq_handler, device); - if (ret < 0) - return ret; - - rgmu->rgmu_interrupt_num = ret; - return 0; -} - -static int rgmu_regulators_probe(struct rgmu_device *rgmu) -{ - int ret = 0; - - rgmu->cx_gdsc = devm_regulator_get(&rgmu->pdev->dev, "vddcx"); - if (IS_ERR(rgmu->cx_gdsc)) { - ret = PTR_ERR(rgmu->cx_gdsc); - if (ret != -EPROBE_DEFER) - dev_err(&rgmu->pdev->dev, - "Couldn't get CX gdsc error:%d\n", ret); - return ret; - } - - rgmu->gx_gdsc = devm_regulator_get(&rgmu->pdev->dev, "vdd"); - if (IS_ERR(rgmu->gx_gdsc)) { - ret = PTR_ERR(rgmu->gx_gdsc); - if (ret != -EPROBE_DEFER) - dev_err(&rgmu->pdev->dev, - "Couldn't get GX gdsc error:%d\n", ret); - } - - return ret; -} - -static int rgmu_clocks_probe(struct rgmu_device *rgmu, struct device_node *node) -{ - int ret; - - ret = devm_clk_bulk_get_all(&rgmu->pdev->dev, &rgmu->clks); - if (ret < 0) - return ret; - - rgmu->num_clks = ret; - - rgmu->gpu_clk = kgsl_of_clk_by_name(rgmu->clks, ret, "core"); - if (!rgmu->gpu_clk) { - dev_err(&rgmu->pdev->dev, "The GPU clock isn't defined\n"); - return -ENODEV; - } - - rgmu->rgmu_clk = kgsl_of_clk_by_name(rgmu->clks, ret, "gmu"); - if (!rgmu->rgmu_clk) { - dev_err(&rgmu->pdev->dev, "The RGMU clock isn't defined\n"); - return -ENODEV; - } - - return 0; -} - -static void rgmu_disable_clks(struct kgsl_device *device) -{ - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device); - int ret; - - /* Check GX GDSC is status */ - if (gmu_dev_ops->gx_is_on(device)) { - - if (IS_ERR_OR_NULL(rgmu->gx_gdsc)) - return; - - /* - * Switch gx gdsc control from RGMU to CPU. Force non-zero - * reference count in clk driver so next disable call will - * turn off the GDSC. - */ - ret = regulator_enable(rgmu->gx_gdsc); - if (ret) - dev_err(&rgmu->pdev->dev, - "Fail to enable gx gdsc:%d\n", ret); - - ret = regulator_disable(rgmu->gx_gdsc); - if (ret) - dev_err(&rgmu->pdev->dev, - "Fail to disable gx gdsc:%d\n", ret); - - if (gmu_dev_ops->gx_is_on(device)) - dev_err(&rgmu->pdev->dev, "gx is stuck on\n"); - } - - clk_bulk_disable_unprepare(rgmu->num_clks, rgmu->clks); - - clear_bit(GMU_CLK_ON, &device->gmu_core.flags); -} - -static int rgmu_enable_clks(struct kgsl_device *device) -{ - int ret; - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - struct kgsl_pwrctrl *pwr = &device->pwrctrl; - - ret = clk_set_rate(rgmu->rgmu_clk, RGMU_CLK_FREQ); - if (ret) { - dev_err(&rgmu->pdev->dev, "Couldn't set the RGMU clock\n"); - return ret; - } - - ret = clk_set_rate(rgmu->gpu_clk, - pwr->pwrlevels[pwr->default_pwrlevel].gpu_freq); - if (ret) { - dev_err(&rgmu->pdev->dev, "Couldn't set the GPU clock\n"); - return ret; - } - - ret = clk_bulk_prepare_enable(rgmu->num_clks, rgmu->clks); - if (ret) { - dev_err(&rgmu->pdev->dev, "Failed to enable RGMU clocks\n"); - return ret; - } - - set_bit(GMU_CLK_ON, &device->gmu_core.flags); - return 0; -} - -static int rgmu_disable_gdsc(struct kgsl_device *device) -{ - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - - /* Wait up to 5 seconds for the regulator to go off */ - if (kgsl_regulator_disable_wait(rgmu->cx_gdsc, 5000)) - return 0; - - dev_err(&rgmu->pdev->dev, "RGMU CX gdsc off timeout\n"); - return -ETIMEDOUT; -} - -static int rgmu_enable_gdsc(struct rgmu_device *rgmu) -{ - int ret; - - if (IS_ERR_OR_NULL(rgmu->cx_gdsc)) - return 0; - - ret = regulator_enable(rgmu->cx_gdsc); - if (ret) - dev_err(&rgmu->pdev->dev, - "Fail to enable CX gdsc:%d\n", ret); - - return ret; -} - -static void rgmu_snapshot(struct kgsl_device *device) -{ - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device); - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - - /* Mask so there's no interrupt caused by NMI */ - adreno_write_gmureg(adreno_dev, - ADRENO_REG_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF); - - /* Make sure the interrupt is masked */ - wmb(); - - /* - * Halt RGMU execution so that GX will not - * be collapsed while dumping snapshot. - */ - gmu_dev_ops->halt_execution(device); - - kgsl_device_snapshot(device, NULL, true); - - adreno_write_gmureg(adreno_dev, - ADRENO_REG_GMU_GMU2HOST_INTR_CLR, 0xFFFFFFFF); - adreno_write_gmureg(adreno_dev, - ADRENO_REG_GMU_GMU2HOST_INTR_MASK, - ~(gmu_dev_ops->gmu2host_intr_mask)); - - rgmu->fault_count++; -} - -/* Caller shall ensure GPU is ready for SLUMBER */ -static void rgmu_stop(struct kgsl_device *device) -{ - struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device); - - if (!test_bit(GMU_CLK_ON, &device->gmu_core.flags)) - return; - - /* Wait for the lowest idle level we requested */ - if (gmu_dev_ops->wait_for_lowest_idle(device)) - goto error; - - gmu_dev_ops->rpmh_gpu_pwrctrl(device, - GMU_NOTIFY_SLUMBER, 0, 0); - - gmu_dev_ops->irq_disable(device); - rgmu_disable_clks(device); - rgmu_disable_gdsc(device); - return; - -error: - - /* - * The power controller will change state to SLUMBER anyway - * Set GMU_FAULT flag to indicate to power contrller - * that hang recovery is needed to power on GPU - */ - set_bit(GMU_FAULT, &device->gmu_core.flags); - rgmu_snapshot(device); -} - -static struct gmu_core_ops rgmu_ops; - -/* Do not access any RGMU registers in RGMU probe function */ -static int rgmu_probe(struct kgsl_device *device, struct platform_device *pdev) -{ - struct rgmu_device *rgmu; - struct resource *res; - int ret = -ENXIO; - - rgmu = devm_kzalloc(&pdev->dev, sizeof(*rgmu), GFP_KERNEL); - - if (rgmu == NULL) - return -ENOMEM; - - rgmu->pdev = pdev; - - /* Set up RGMU regulators */ - ret = rgmu_regulators_probe(rgmu); - if (ret) - return ret; - - /* Set up RGMU clocks */ - ret = rgmu_clocks_probe(rgmu, pdev->dev.of_node); - if (ret) - return ret; - - /* Map and reserve RGMU CSRs registers */ - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "kgsl_rgmu"); - if (!res) { - dev_err(&pdev->dev, "The RGMU register region isn't defined\n"); - return -ENODEV; - } - - device->gmu_core.gmu2gpu_offset = (res->start - device->reg_phys) >> 2; - device->gmu_core.reg_len = resource_size(res); - device->gmu_core.reg_virt = devm_ioremap_resource(&pdev->dev, res); - - if (IS_ERR(device->gmu_core.reg_virt)) { - dev_err(&pdev->dev, "Unable to map the RGMU registers\n"); - return PTR_ERR(device->gmu_core.reg_virt); - } - - device->gmu_core.ptr = (void *)rgmu; - - /* Initialize OOB and RGMU interrupts */ - ret = rgmu_irq_probe(device); - if (ret) - return ret; - - /* Don't enable RGMU interrupts until RGMU started */ - /* We cannot use rgmu_irq_disable because it writes registers */ - disable_irq(rgmu->rgmu_interrupt_num); - disable_irq(rgmu->oob_interrupt_num); - - /* Set up RGMU idle states */ - if (ADRENO_FEATURE(ADRENO_DEVICE(device), ADRENO_IFPC)) - rgmu->idle_level = GPU_HW_IFPC; - else - rgmu->idle_level = GPU_HW_ACTIVE; - - set_bit(GMU_ENABLED, &device->gmu_core.flags); - device->gmu_core.core_ops = &rgmu_ops; - device->gmu_core.dev_ops = &adreno_a6xx_rgmudev; - - return 0; -} - -static int rgmu_suspend(struct kgsl_device *device) -{ - struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device); - - if (!test_bit(GMU_CLK_ON, &device->gmu_core.flags)) - return 0; - - gmu_dev_ops->irq_disable(device); - - if (gmu_dev_ops->rpmh_gpu_pwrctrl(device, GMU_SUSPEND, 0, 0)) - return -EINVAL; - - rgmu_disable_clks(device); - rgmu_disable_gdsc(device); - return 0; -} - -static int rgmu_init(struct kgsl_device *device) -{ - struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device); - int ret; - - ret = ops->load_firmware(device); - if (ret) - return ret; - - return 0; -} - -/* To be called to power on both GPU and RGMU */ -static int rgmu_start(struct kgsl_device *device) -{ - int ret = 0; - struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device); - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - - switch (device->state) { - case KGSL_STATE_RESET: - ret = rgmu_suspend(device); - if (ret) - goto error_rgmu; - /* Fall-thru */ - case KGSL_STATE_INIT: - case KGSL_STATE_SUSPEND: - case KGSL_STATE_SLUMBER: - rgmu_enable_gdsc(rgmu); - rgmu_enable_clks(device); - gmu_dev_ops->irq_enable(device); - ret = gmu_dev_ops->rpmh_gpu_pwrctrl(device, - GMU_FW_START, GMU_COLD_BOOT, 0); - if (ret) - goto error_rgmu; - break; - } - /* Request default DCVS level */ - kgsl_pwrctrl_set_default_gpu_pwrlevel(device); - return 0; - -error_rgmu: - set_bit(GMU_FAULT, &device->gmu_core.flags); - rgmu_snapshot(device); - return ret; -} - -/* - * rgmu_dcvs_set() - Change GPU frequency and/or bandwidth. - * @rgmu: Pointer to RGMU device - * @pwrlevel: index to GPU DCVS table used by KGSL - * @bus_level: index to GPU bus table used by KGSL - * - * The function converts GPU power level and bus level index used by KGSL - * to index being used by GMU/RPMh. - */ -static int rgmu_dcvs_set(struct kgsl_device *device, - int pwrlevel, int bus_level) -{ - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - int ret; - unsigned long rate; - - if (pwrlevel == INVALID_DCVS_IDX) - return -EINVAL; - - rate = device->pwrctrl.pwrlevels[pwrlevel].gpu_freq; - - ret = clk_set_rate(rgmu->gpu_clk, rate); - if (ret) - dev_err(&rgmu->pdev->dev, "Couldn't set the GPU clock\n"); - - return ret; -} - -static bool rgmu_regulator_isenabled(struct kgsl_device *device) -{ - struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device); - - return (rgmu->gx_gdsc && regulator_is_enabled(rgmu->gx_gdsc)); -} - -static void rgmu_remove(struct kgsl_device *device) -{ - rgmu_stop(device); - - memset(&device->gmu_core, 0, sizeof(device->gmu_core)); -} - -static struct gmu_core_ops rgmu_ops = { - .init = rgmu_init, - .start = rgmu_start, - .stop = rgmu_stop, - .dcvs_set = rgmu_dcvs_set, - .snapshot = rgmu_snapshot, - .regulator_isenabled = rgmu_regulator_isenabled, - .suspend = rgmu_suspend, -}; - -static int kgsl_rgmu_bind(struct device *dev, struct device *master, void *data) -{ - struct kgsl_device *device = dev_get_drvdata(master); - - return rgmu_probe(device, to_platform_device(dev)); -} - -static void kgsl_rgmu_unbind(struct device *dev, struct device *master, - void *data) -{ - struct kgsl_device *device = dev_get_drvdata(master); - - rgmu_remove(device); -} - -static const struct component_ops kgsl_rgmu_ops = { - .bind = kgsl_rgmu_bind, - .unbind = kgsl_rgmu_unbind, -}; - -static int kgsl_rgmu_probe(struct platform_device *pdev) -{ - return component_add(&pdev->dev, &kgsl_rgmu_ops); -} - -static int kgsl_rgmu_remove(struct platform_device *pdev) -{ - component_del(&pdev->dev, &kgsl_rgmu_ops); - return 0; -} - -static const struct of_device_id kgsl_rgmu_match_table[] = { - { .compatible = "qcom,gpu-rgmu" }, - { }, -}; - -struct platform_driver kgsl_rgmu_driver = { - .probe = kgsl_rgmu_probe, - .remove = kgsl_rgmu_remove, - .driver = { - .name = "kgsl-rgmu", - .of_match_table = kgsl_rgmu_match_table, - }, -}; diff --git a/drivers/gpu/msm/kgsl_rgmu.h b/drivers/gpu/msm/kgsl_rgmu.h deleted file mode 100644 index 112f6e52c6ef..000000000000 --- a/drivers/gpu/msm/kgsl_rgmu.h +++ /dev/null @@ -1,57 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. - */ -#ifndef __KGSL_RGMU_H -#define __KGSL_RGMU_H - -#define RGMU_AO_IRQ_FENCE_ERR BIT(3) -#define RGMU_AO_IRQ_MASK RGMU_AO_IRQ_FENCE_ERR - -#define RGMU_OOB_IRQ_ERR_MSG BIT(24) -#define RGMU_OOB_IRQ_ACK_MASK GENMASK(23, 16) -#define RGMU_OOB_IRQ_ERR_MSG_MASK GENMASK(31, 24) -#define RGMU_OOB_IRQ_MASK RGMU_OOB_IRQ_ERR_MSG_MASK - -#define MAX_RGMU_CLKS 8 - -/** - * struct rgmu_device - rGMU device structure - * @ver: RGMU firmware version - * @rgmu_interrupt_num: RGMU interrupt number - * @oob_interrupt_num: number of RGMU asserted OOB interrupt - * @fw_hostptr: Buffer which holds the RGMU firmware - * @fw_size: Size of RGMU firmware buffer - * @cx_gdsc: CX headswitch that controls power of RGMU and - subsystem peripherals - * @clks: RGMU clocks including the GPU - * @gpu_clk: Pointer to GPU core clock - * @rgmu_clk: Pointer to rgmu clock - * @flags: RGMU flags - * @idle_level: Minimal GPU idle power level - * @fault_count: RGMU fault count - */ -struct rgmu_device { - u32 ver; - struct platform_device *pdev; - unsigned int rgmu_interrupt_num; - unsigned int oob_interrupt_num; - unsigned int *fw_hostptr; - uint32_t fw_size; - struct regulator *cx_gdsc; - struct regulator *gx_gdsc; - struct clk_bulk_data *clks; - /** @num_clks: Number of clocks in @clks */ - int num_clks; - struct clk *gpu_clk; - struct clk *rgmu_clk; - unsigned int idle_level; - unsigned int fault_count; -}; - -extern struct gmu_dev_ops adreno_a6xx_rgmudev; -#define KGSL_RGMU_DEVICE(_a) ((struct rgmu_device *)((_a)->gmu_core.ptr)) - -irqreturn_t rgmu_irq_handler(int irq, void *data); -irqreturn_t oob_irq_handler(int irq, void *data); -#endif /* __KGSL_RGMU_H */ diff --git a/drivers/gpu/msm/kgsl_snapshot.c b/drivers/gpu/msm/kgsl_snapshot.c index d12564a9ec76..06f63a85561e 100644 --- a/drivers/gpu/msm/kgsl_snapshot.c +++ b/drivers/gpu/msm/kgsl_snapshot.c @@ -728,7 +728,6 @@ void kgsl_device_snapshot(struct kgsl_device *device, snapshot->start = device->snapshot_memory.ptr; snapshot->ptr = device->snapshot_memory.ptr; snapshot->remain = device->snapshot_memory.size; - snapshot->gmu_fault = gmu_fault; snapshot->recovered = false; snapshot->first_read = true; snapshot->sysfs_read = 0; @@ -1269,7 +1268,7 @@ static void kgsl_snapshot_save_frozen_objs(struct work_struct *work) size_t size = 0; void *ptr; - if (snapshot->gmu_fault) + if (snapshot->device->gmu_fault) goto gmu_only; kgsl_snapshot_process_ib_obj_list(snapshot);