diff --git a/drivers/gpu/msm/adreno-gpulist.h b/drivers/gpu/msm/adreno-gpulist.h index e7760b2ab6dd..9a0c401a8934 100644 --- a/drivers/gpu/msm/adreno-gpulist.h +++ b/drivers/gpu/msm/adreno-gpulist.h @@ -1382,7 +1382,7 @@ static const struct adreno_a6xx_core adreno_gpu_core_a612 = { .features = ADRENO_CONTENT_PROTECTION | ADRENO_IOCOHERENT | ADRENO_PREEMPTION | ADRENO_GPMU | ADRENO_IFPC, - .gpudev = &adreno_a6xx_gpudev, + .gpudev = &adreno_a6xx_rgmu_gpudev, .gmem_base = 0x100000, .gmem_size = (SZ_128K + SZ_4K), .bus_width = 32, diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index 994efb868905..7b1bf2781dde 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -906,6 +906,7 @@ extern struct adreno_gpudev adreno_a3xx_gpudev; extern struct adreno_gpudev adreno_a5xx_gpudev; extern struct adreno_gpudev adreno_a6xx_gpudev; extern struct adreno_gpudev adreno_a6xx_gmu_gpudev; +extern struct adreno_gpudev adreno_a6xx_rgmu_gpudev; extern int adreno_wake_nice; extern unsigned int adreno_wake_timeout; diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c index 91087ae772b6..58ed94ab6eef 100644 --- a/drivers/gpu/msm/adreno_a6xx.c +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -2712,3 +2712,38 @@ struct adreno_gpudev adreno_a6xx_gmu_gpudev = { .read_alwayson = a6xx_read_alwayson, .power_ops = &a6xx_gmu_power_ops, }; + +struct adreno_gpudev adreno_a6xx_rgmu_gpudev = { + .reg_offsets = a6xx_register_offsets, + .probe = a6xx_rgmu_device_probe, + .start = a6xx_start, + .snapshot = a6xx_snapshot, + .init = a6xx_init, + .irq_handler = a6xx_irq_handler, + .rb_start = a6xx_rb_start, + .regulator_enable = a6xx_sptprac_enable, + .regulator_disable = a6xx_sptprac_disable, + .perfcounters = &a6xx_perfcounters, + .read_throttling_counters = a6xx_read_throttling_counters, + .microcode_read = a6xx_microcode_read, + .gpu_keepalive = a6xx_gpu_keepalive, + .hw_isidle = a6xx_hw_isidle, + .iommu_fault_block = a6xx_iommu_fault_block, + .reset = a6xx_rgmu_restart, + .preemption_pre_ibsubmit = a6xx_preemption_pre_ibsubmit, + .preemption_post_ibsubmit = a6xx_preemption_post_ibsubmit, + .preemption_init = a6xx_preemption_init, + .preemption_schedule = a6xx_preemption_schedule, + .set_marker = a6xx_set_marker, + .preemption_context_init = a6xx_preemption_context_init, + .preemption_context_destroy = a6xx_preemption_context_destroy, + .sptprac_is_on = a6xx_sptprac_is_on, + .ccu_invalidate = a6xx_ccu_invalidate, + .perfcounter_update = a6xx_perfcounter_update, +#ifdef CONFIG_QCOM_KGSL_CORESIGHT + .coresight = {&a6xx_coresight, &a6xx_coresight_cx}, +#endif + .clk_set_options = a6xx_clk_set_options, + .read_alwayson = a6xx_read_alwayson, + .power_ops = &a6xx_rgmu_power_ops, +}; diff --git a/drivers/gpu/msm/adreno_a6xx.h b/drivers/gpu/msm/adreno_a6xx.h index 301a3320a537..3de5b699ce9c 100644 --- a/drivers/gpu/msm/adreno_a6xx.h +++ b/drivers/gpu/msm/adreno_a6xx.h @@ -11,8 +11,10 @@ #include "a6xx_reg.h" #include "adreno_a6xx_gmu.h" +#include "adreno_a6xx_rgmu.h" extern const struct adreno_power_ops a6xx_gmu_power_ops; +extern const struct adreno_power_ops a6xx_rgmu_power_ops; /** * struct a6xx_device - Container for the a6xx_device @@ -20,6 +22,8 @@ extern const struct adreno_power_ops a6xx_gmu_power_ops; struct a6xx_device { /** @gmu: Container for the a6xx GMU device */ struct a6xx_gmu_device gmu; + /** @rgmu: Container for the a6xx rGMU device */ + struct a6xx_rgmu_device rgmu; /** @adreno_dev: Container for the generic adreno device */ struct adreno_device adreno_dev; }; diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.c b/drivers/gpu/msm/adreno_a6xx_gmu.c index 6e71a5287e5b..1754a8580881 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.c +++ b/drivers/gpu/msm/adreno_a6xx_gmu.c @@ -2706,7 +2706,7 @@ error: return ret; } -static void enable_gpu_irq(struct adreno_device *adreno_dev) +void a6xx_enable_gpu_irq(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; @@ -2717,7 +2717,7 @@ static void enable_gpu_irq(struct adreno_device *adreno_dev) trace_kgsl_irq(device, 1); } -static void disable_gpu_irq(struct adreno_device *adreno_dev) +void a6xx_disable_gpu_irq(struct adreno_device *adreno_dev) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; @@ -2769,11 +2769,11 @@ static int a6xx_gpu_boot(struct adreno_device *adreno_dev) /* Clear FSR here in case it is set from a previous pagefault */ kgsl_mmu_clear_fsr(&device->mmu); - enable_gpu_irq(adreno_dev); + a6xx_enable_gpu_irq(adreno_dev); ret = a6xx_rb_start(adreno_dev); if (ret) { - disable_gpu_irq(adreno_dev); + a6xx_disable_gpu_irq(adreno_dev); goto oob_clear; } @@ -2953,7 +2953,7 @@ static int a6xx_power_off(struct adreno_device *adreno_dev) a6xx_gmu_oob_clear(device, oob_gpu); no_gx_power: - disable_gpu_irq(adreno_dev); + a6xx_disable_gpu_irq(adreno_dev); a6xx_gmu_power_off(adreno_dev); @@ -3157,7 +3157,7 @@ int a6xx_gmu_restart(struct kgsl_device *device) a6xx_hfi_stop(adreno_dev); - disable_gpu_irq(adreno_dev); + a6xx_disable_gpu_irq(adreno_dev); /* Hard reset the gmu and gpu */ a6xx_gmu_suspend(adreno_dev); diff --git a/drivers/gpu/msm/adreno_a6xx_gmu.h b/drivers/gpu/msm/adreno_a6xx_gmu.h index a0caf28c25fe..2e7ca929b1d2 100644 --- a/drivers/gpu/msm/adreno_a6xx_gmu.h +++ b/drivers/gpu/msm/adreno_a6xx_gmu.h @@ -274,4 +274,18 @@ int a6xx_gmu_device_probe(struct platform_device *pdev, * Return: 0 on success or negative error on failure */ int a6xx_gmu_restart(struct kgsl_device *device); + +/** + * a6xx_enable_gpu_irq - Enable gpu interrupt + * @adreno_dev: Pointer to the adreno device + */ +void a6xx_enable_gpu_irq(struct adreno_device *adreno_dev); + +/** + * a6xx_disable_gpu_irq - Disable gpu interrupt + * @adreno_dev: Pointer to the adreno device + */ +void a6xx_disable_gpu_irq(struct adreno_device *adreno_dev); + + #endif diff --git a/drivers/gpu/msm/adreno_a6xx_rgmu.c b/drivers/gpu/msm/adreno_a6xx_rgmu.c index 124e6eaf6291..2b6b3add062c 100644 --- a/drivers/gpu/msm/adreno_a6xx_rgmu.c +++ b/drivers/gpu/msm/adreno_a6xx_rgmu.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "adreno.h" #include "adreno_a6xx.h" @@ -40,10 +41,42 @@ static const unsigned int a6xx_rgmu_registers[] = { 0x26000, 0x26002, }; +static struct a6xx_rgmu_device *to_a6xx_rgmu(struct adreno_device *adreno_dev) +{ + struct a6xx_device *a6xx_dev = container_of(adreno_dev, + struct a6xx_device, adreno_dev); + + return &a6xx_dev->rgmu; +} + +static void a6xx_rgmu_active_count_put(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return; + + if (WARN(atomic_read(&device->active_cnt) == 0, + "Unbalanced get/put calls to KGSL active count\n")) + return; + + if (atomic_dec_and_test(&device->active_cnt)) { + kgsl_pwrscale_update_stats(device); + kgsl_pwrscale_update(device); + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + } + + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + + wake_up(&device->active_cnt_wq); +} + static irqreturn_t a6xx_rgmu_irq_handler(int irq, void *data) { struct kgsl_device *device = data; - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); unsigned int status = 0; gmu_core_regread(device, A6XX_GMU_AO_HOST_INTERRUPT_STATUS, &status); @@ -71,8 +104,8 @@ static irqreturn_t a6xx_rgmu_irq_handler(int irq, void *data) static irqreturn_t a6xx_oob_irq_handler(int irq, void *data) { struct kgsl_device *device = data; - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int status = 0; gmu_core_regread(device, A6XX_GMU_GMU2HOST_INTR_INFO, &status); @@ -110,7 +143,7 @@ static const char *oob_to_str(enum oob_request req) static int a6xx_rgmu_oob_set(struct kgsl_device *device, enum oob_request req) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); int ret, set, check; set = BIT(req + 16); @@ -153,7 +186,7 @@ static void a6xx_rgmu_oob_clear(struct kgsl_device *device, static void a6xx_rgmu_bcl_config(struct kgsl_device *device, bool on) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); if (on) { /* Enable BCL CRC HW i/f */ @@ -178,35 +211,35 @@ static void a6xx_rgmu_bcl_config(struct kgsl_device *device, bool on) } } -static void a6xx_rgmu_irq_enable(struct kgsl_device *device) +static void a6xx_rgmu_irq_enable(struct adreno_device *adreno_dev) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); /* Clear pending IRQs and Unmask needed IRQs */ - adreno_gmu_clear_and_unmask_irqs(ADRENO_DEVICE(device)); + adreno_gmu_clear_and_unmask_irqs(adreno_dev); /* Enable all IRQs on host */ enable_irq(rgmu->oob_interrupt_num); enable_irq(rgmu->rgmu_interrupt_num); } -static void a6xx_rgmu_irq_disable(struct kgsl_device *device) +static void a6xx_rgmu_irq_disable(struct adreno_device *adreno_dev) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); /* Disable all IRQs on host */ disable_irq(rgmu->rgmu_interrupt_num); disable_irq(rgmu->oob_interrupt_num); /* Mask all IRQs and clear pending IRQs */ - adreno_gmu_mask_and_clear_irqs(ADRENO_DEVICE(device)); + adreno_gmu_mask_and_clear_irqs(adreno_dev); } static int a6xx_rgmu_ifpc_store(struct kgsl_device *device, unsigned int val) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int requested_idle_level; if (!ADRENO_FEATURE(adreno_dev, ADRENO_IFPC)) @@ -227,7 +260,7 @@ static int a6xx_rgmu_ifpc_store(struct kgsl_device *device, static unsigned int a6xx_rgmu_ifpc_show(struct kgsl_device *device) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); return rgmu->idle_level == GPU_HW_IFPC; } @@ -235,10 +268,8 @@ static unsigned int a6xx_rgmu_ifpc_show(struct kgsl_device *device) static void a6xx_rgmu_prepare_stop(struct kgsl_device *device) { - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); /* Turn off GX_MEM retention */ - kgsl_regwrite(KGSL_DEVICE(adreno_dev), - A6XX_RBBM_BLOCK_GX_RETENTION_CNTL, 0); + kgsl_regwrite(device, A6XX_RBBM_BLOCK_GX_RETENTION_CNTL, 0); } #define GX_GDSC_POWER_OFF BIT(6) @@ -256,9 +287,10 @@ static bool a6xx_rgmu_gx_is_on(struct kgsl_device *device) return !(val & GX_GDSC_POWER_OFF); } -static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) +static int a6xx_rgmu_wait_for_lowest_idle(struct adreno_device *adreno_dev) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int reg[10] = {0}; unsigned long t; uint64_t ts1, ts2, ts3; @@ -266,7 +298,7 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) if (rgmu->idle_level != GPU_HW_IFPC) return 0; - ts1 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts1 = a6xx_read_alwayson(adreno_dev); t = jiffies + msecs_to_jiffies(RGMU_IDLE_TIMEOUT); do { @@ -280,7 +312,7 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) usleep_range(10, 100); } while (!time_after(jiffies, t)); - ts2 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts2 = a6xx_read_alwayson(adreno_dev); /* Do one last read incase it succeeds */ gmu_core_regread(device, @@ -289,7 +321,7 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) if (reg[0] & GX_GDSC_POWER_OFF) return 0; - ts3 = a6xx_read_alwayson(ADRENO_DEVICE(device)); + ts3 = a6xx_read_alwayson(adreno_dev); /* Collect abort data to help with debugging */ gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, ®[1]); @@ -338,15 +370,11 @@ static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device) /* RGMU FENCE RANGE MASK */ #define RGMU_FENCE_RANGE_MASK ((0x1 << 31) | ((0xA << 2) << 18) | (0x8A0)) -/* - * a6xx_rgmu_fw_start() - set up GMU and start FW - * @device: Pointer to KGSL device - * @boot_state: State of the rgmu being started - */ -static int a6xx_rgmu_fw_start(struct kgsl_device *device, +static int a6xx_rgmu_fw_start(struct adreno_device *adreno_dev, unsigned int boot_state) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); unsigned int status; int i; @@ -416,76 +444,21 @@ static int a6xx_rgmu_fw_start(struct kgsl_device *device, return 0; } -static int a6xx_rgmu_suspend_device(struct kgsl_device *device) +static void a6xx_rgmu_notify_slumber(struct adreno_device *adreno_dev) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); - int ret = 0; + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - /* Check GX GDSC is status */ - if (a6xx_rgmu_gx_is_on(device)) { + /* Disable the power counter so that the RGMU is not busy */ + gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0); - /* Switch gx gdsc control from RGMU to CPU - * force non-zero reference count in clk driver - * so next disable call will turn - * off the GDSC - */ - ret = regulator_enable(rgmu->gx_gdsc); - if (ret) - dev_err(&rgmu->pdev->dev, - "Fail to enable gx gdsc, error:%d\n", ret); - - ret = regulator_disable(rgmu->gx_gdsc); - if (ret) - dev_err(&rgmu->pdev->dev, - "Fail to disable gx gdsc, error:%d\n", ret); - - if (a6xx_rgmu_gx_is_on(device)) - dev_err(&rgmu->pdev->dev, "gx is stuck on\n"); - } - - return ret; + /* BCL OFF Sequence */ + a6xx_rgmu_bcl_config(device, false); } -/* - * a6xx_rgmu_gpu_pwrctrl() - GPU power control via rgmu interface - * @adreno_dev: Pointer to adreno device - * @mode: requested power mode - * @arg1: first argument for mode control - * @arg2: second argument for mode control - */ -static int a6xx_rgmu_gpu_pwrctrl(struct kgsl_device *device, - unsigned int mode, unsigned int arg1, unsigned int arg2) +static void a6xx_rgmu_disable_clks(struct adreno_device *adreno_dev) { - int ret = 0; - - switch (mode) { - case GMU_FW_START: - ret = a6xx_rgmu_fw_start(device, arg1); - break; - case GMU_SUSPEND: - ret = a6xx_rgmu_suspend_device(device); - break; - case GMU_NOTIFY_SLUMBER: - - /* Disable the power counter so that the RGMU is not busy */ - gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, - 0); - - /* BCL OFF Sequence */ - a6xx_rgmu_bcl_config(device, false); - - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - -static void a6xx_rgmu_disable_clks(struct kgsl_device *device) -{ - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); int ret; /* Check GX GDSC is status */ @@ -514,13 +487,11 @@ static void a6xx_rgmu_disable_clks(struct kgsl_device *device) } clk_bulk_disable_unprepare(rgmu->num_clks, rgmu->clks); - - clear_bit(GMU_CLK_ON, &device->gmu_core.flags); } -static int a6xx_rgmu_disable_gdsc(struct kgsl_device *device) +static int a6xx_rgmu_disable_gdsc(struct adreno_device *adreno_dev) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); /* Wait up to 5 seconds for the regulator to go off */ if (kgsl_regulator_disable_wait(rgmu->cx_gdsc, 5000)) @@ -536,11 +507,13 @@ static void a6xx_rgmu_halt_execution(struct kgsl_device *device); static void a6xx_rgmu_snapshot(struct kgsl_device *device) { struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + + if (rgmu->fault) + return; /* Mask so there's no interrupt caused by NMI */ - adreno_write_gmureg(adreno_dev, - ADRENO_REG_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF); + gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF); /* Make sure the interrupt is masked */ wmb(); @@ -558,27 +531,23 @@ static void a6xx_rgmu_snapshot(struct kgsl_device *device) RGMU_OOB_IRQ_MASK); rgmu->fault_count++; + + rgmu->fault = true; } -static int a6xx_rgmu_suspend(struct kgsl_device *device) +static void a6xx_rgmu_suspend(struct adreno_device *adreno_dev) { - if (!test_bit(GMU_CLK_ON, &device->gmu_core.flags)) - return 0; + a6xx_rgmu_irq_disable(adreno_dev); - a6xx_rgmu_irq_disable(device); - - if (a6xx_rgmu_gpu_pwrctrl(device, GMU_SUSPEND, 0, 0)) - return -EINVAL; - - a6xx_rgmu_disable_clks(device); - a6xx_rgmu_disable_gdsc(device); - return 0; + a6xx_rgmu_disable_clks(adreno_dev); + a6xx_rgmu_disable_gdsc(adreno_dev); } -static int a6xx_rgmu_enable_clks(struct kgsl_device *device) +static int a6xx_rgmu_enable_clks(struct adreno_device *adreno_dev) { int ret; - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); struct kgsl_pwrctrl *pwr = &device->pwrctrl; ret = clk_set_rate(rgmu->rgmu_clk, RGMU_CLK_FREQ); @@ -600,12 +569,12 @@ static int a6xx_rgmu_enable_clks(struct kgsl_device *device) return ret; } - set_bit(GMU_CLK_ON, &device->gmu_core.flags); return 0; } -static int a6xx_rgmu_enable_gdsc(struct a6xx_rgmu_device *rgmu) +static int a6xx_rgmu_enable_gdsc(struct adreno_device *adreno_dev) { + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); int ret; if (IS_ERR_OR_NULL(rgmu->cx_gdsc)) @@ -619,51 +588,14 @@ static int a6xx_rgmu_enable_gdsc(struct a6xx_rgmu_device *rgmu) return ret; } -/* To be called to power on both GPU and RGMU */ -static int a6xx_rgmu_start(struct kgsl_device *device) -{ - int ret = 0; - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); - - switch (device->state) { - case KGSL_STATE_RESET: - ret = a6xx_rgmu_suspend(device); - if (ret) - goto error_rgmu; - /* Fall-through */ - case KGSL_STATE_INIT: - case KGSL_STATE_SUSPEND: - case KGSL_STATE_SLUMBER: - a6xx_rgmu_enable_gdsc(rgmu); - a6xx_rgmu_enable_clks(device); - a6xx_rgmu_irq_enable(device); - ret = a6xx_rgmu_gpu_pwrctrl(device, - GMU_FW_START, GMU_COLD_BOOT, 0); - if (ret) - goto error_rgmu; - break; - } - /* Request default DCVS level */ - kgsl_pwrctrl_set_default_gpu_pwrlevel(device); - return 0; - -error_rgmu: - set_bit(GMU_FAULT, &device->gmu_core.flags); - a6xx_rgmu_snapshot(device); - return ret; -} - - - /* * a6xx_rgmu_load_firmware() - Load the ucode into the RGMU TCM - * @device: Pointer to KGSL device + * @adreno_dev: Pointer to adreno device */ -static int a6xx_rgmu_load_firmware(struct kgsl_device *device) +static int a6xx_rgmu_load_firmware(struct adreno_device *adreno_dev) { const struct firmware *fw = NULL; - struct adreno_device *adreno_dev = ADRENO_DEVICE(device); - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); int ret; @@ -688,15 +620,10 @@ static int a6xx_rgmu_load_firmware(struct kgsl_device *device) return rgmu->fw_hostptr ? 0 : -ENOMEM; } -static int a6xx_rgmu_init(struct kgsl_device *device) -{ - return a6xx_rgmu_load_firmware(device); -} - /* Halt RGMU execution */ static void a6xx_rgmu_halt_execution(struct kgsl_device *device) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); unsigned int index, status, fence; gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, &index); @@ -737,53 +664,52 @@ static void a6xx_rgmu_device_snapshot(struct kgsl_device *device, ARRAY_SIZE(a6xx_rgmu_registers) / 2); } -static u64 a6xx_rgmu_read_alwayson(struct kgsl_device *device) +static void halt_gbif_arb(struct adreno_device *adreno_dev) { - return a6xx_read_alwayson(ADRENO_DEVICE(device)); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + + /* Halt all AXI requests */ + kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_ARB_HALT_MASK); + adreno_wait_for_halt_ack(device, ADRENO_REG_GBIF_HALT_ACK, + A6XX_GBIF_ARB_HALT_MASK); + + /* De-assert the halts */ + kgsl_regwrite(device, A6XX_GBIF_HALT, 0x0); } + /* Caller shall ensure GPU is ready for SLUMBER */ -static void a6xx_rgmu_stop(struct kgsl_device *device) +static void a6xx_rgmu_power_off(struct adreno_device *adreno_dev) { - if (!test_bit(GMU_CLK_ON, &device->gmu_core.flags)) - return; + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_OFF); + + if (rgmu->fault) + return a6xx_rgmu_suspend(adreno_dev); /* Wait for the lowest idle level we requested */ - if (a6xx_rgmu_wait_for_lowest_idle(device)) - goto error; + ret = a6xx_rgmu_wait_for_lowest_idle(adreno_dev); + if (ret) + return a6xx_rgmu_suspend(adreno_dev); - a6xx_rgmu_gpu_pwrctrl(device, - GMU_NOTIFY_SLUMBER, 0, 0); + a6xx_rgmu_notify_slumber(adreno_dev); - a6xx_rgmu_irq_disable(device); - a6xx_rgmu_disable_clks(device); - a6xx_rgmu_disable_gdsc(device); - return; + /* Halt CX traffic and de-assert halt */ + halt_gbif_arb(adreno_dev); -error: + a6xx_rgmu_irq_disable(adreno_dev); + a6xx_rgmu_disable_clks(adreno_dev); + a6xx_rgmu_disable_gdsc(adreno_dev); - /* - * The power controller will change state to SLUMBER anyway - * Set GMU_FAULT flag to indicate to power contrller - * that hang recovery is needed to power on GPU - */ - set_bit(GMU_FAULT, &device->gmu_core.flags); - a6xx_rgmu_snapshot(device); } -/* - * a6xx_rgmu_dcvs_set() - Change GPU frequency and/or bandwidth. - * @rgmu: Pointer to RGMU device - * @pwrlevel: index to GPU DCVS table used by KGSL - * @bus_level: index to GPU bus table used by KGSL - * - * The function converts GPU power level and bus level index used by KGSL - * to index being used by GMU/RPMh. - */ static int a6xx_rgmu_dcvs_set(struct kgsl_device *device, int pwrlevel, int bus_level) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); int ret; unsigned long rate; @@ -799,37 +725,410 @@ static int a6xx_rgmu_dcvs_set(struct kgsl_device *device, return ret; } +static int a6xx_gpu_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + /* Clear any GPU faults that might have been left over */ + adreno_clear_gpu_fault(adreno_dev); + + adreno_set_active_ctxs_null(adreno_dev); + + adreno_ringbuffer_set_global(adreno_dev, 0); + + ret = kgsl_mmu_start(device); + if (ret) + goto err; + + ret = a6xx_rgmu_oob_set(device, oob_gpu); + if (ret) + goto err_oob_clear; + + adreno_clear_dcvs_counters(adreno_dev); + + /* Restore performance counter registers with saved values */ + adreno_perfcounter_restore(adreno_dev); + + a6xx_start(adreno_dev); + + /* Re-initialize the coresight registers if applicable */ + adreno_coresight_start(adreno_dev); + + adreno_perfcounter_start(adreno_dev); + + /* Clear FSR here in case it is set from a previous pagefault */ + kgsl_mmu_clear_fsr(&device->mmu); + + a6xx_enable_gpu_irq(adreno_dev); + + ret = a6xx_rb_start(adreno_dev); + if (ret) { + a6xx_disable_gpu_irq(adreno_dev); + goto err_oob_clear; + } + + /* Start the dispatcher */ + adreno_dispatcher_start(device); + + device->reset_counter++; + + a6xx_rgmu_oob_clear(device, oob_gpu); + + return 0; + +err_oob_clear: + a6xx_rgmu_oob_clear(device, oob_gpu); + +err: + a6xx_rgmu_power_off(adreno_dev); + + return ret; +} + +static int a6xx_rgmu_boot(struct adreno_device *adreno_dev) +{ + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = a6xx_rgmu_enable_gdsc(adreno_dev); + if (ret) + return ret; + + ret = a6xx_rgmu_enable_clks(adreno_dev); + if (ret) { + a6xx_rgmu_disable_gdsc(adreno_dev); + return ret; + } + + a6xx_rgmu_irq_enable(adreno_dev); + + ret = a6xx_rgmu_fw_start(adreno_dev, GMU_COLD_BOOT); + if (ret) + goto err; + + /* Request default DCVS level */ + ret = kgsl_pwrctrl_set_default_gpu_pwrlevel(device); + if (ret) + goto err; + + kgsl_pwrctrl_axi(device, KGSL_PWRFLAGS_ON); + + rgmu->fault = false; + + return 0; + +err: + a6xx_rgmu_power_off(adreno_dev); + + return ret; +} + +static int a6xx_power_off(struct adreno_device *adreno_dev); + +static void rgmu_idle_check(struct work_struct *work) +{ + struct kgsl_device *device = container_of(work, + struct kgsl_device, idle_check_ws); + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + + mutex_lock(&device->mutex); + + if (!atomic_read(&device->active_cnt)) + a6xx_power_off(adreno_dev); + else + mod_timer(&device->idle_timer, + jiffies + device->pwrctrl.interval_timeout); + + mutex_unlock(&device->mutex); +} + +static void rgmu_idle_timer(struct timer_list *t) +{ + struct kgsl_device *device = container_of(t, struct kgsl_device, + idle_timer); + + kgsl_schedule_work(&device->idle_check_ws); +} + +static int a6xx_boot(struct adreno_device *adreno_dev) +{ + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + if (test_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags)) + return 0; + + ret = a6xx_rgmu_boot(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return ret; + + mod_timer(&device->idle_timer, jiffies + + device->pwrctrl.interval_timeout); + + kgsl_pwrscale_wake(device); + + set_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + device->state = KGSL_STATE_ACTIVE; + + return 0; +} + +static int a6xx_first_boot(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + if (test_bit(RGMU_PRIV_FIRST_BOOT_DONE, &rgmu->flags)) + return a6xx_boot(adreno_dev); + + ret = adreno_dispatcher_init(adreno_dev); + if (ret) + return ret; + + ret = adreno_ringbuffer_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_microcode_read(adreno_dev); + if (ret) + return ret; + + ret = a6xx_init(adreno_dev); + if (ret) + return ret; + + ret = a6xx_rgmu_load_firmware(adreno_dev); + if (ret) + return ret; + + ret = a6xx_rgmu_boot(adreno_dev); + if (ret) + return ret; + + ret = a6xx_gpu_boot(adreno_dev); + if (ret) + return ret; + + adreno_get_bus_counters(adreno_dev); + + adreno_dev->profile_buffer = kgsl_allocate_global(device, PAGE_SIZE, 0, + 0, "alwayson"); + + adreno_dev->profile_index = 0; + + if (!IS_ERR(adreno_dev->profile_buffer)) + set_bit(ADRENO_DEVICE_DRAWOBJ_PROFILE, &adreno_dev->priv); + + set_bit(RGMU_PRIV_FIRST_BOOT_DONE, &rgmu->flags); + set_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + device->state = KGSL_STATE_ACTIVE; + + return 0; +} + +static int a6xx_rgmu_first_open(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + int ret; + + ret = a6xx_first_boot(adreno_dev); + if (ret) + return ret; + + /* + * A client that does a first_open but never closes the device + * may prevent us from going back to SLUMBER. So trigger the idle + * check by incrementing the active count and immediately releasing it. + */ + atomic_inc(&device->active_cnt); + a6xx_rgmu_active_count_put(adreno_dev); + + return 0; +} + +static int a6xx_power_off(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + if (!test_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags)) + return 0; + + ret = a6xx_rgmu_oob_set(device, oob_gpu); + if (ret) { + a6xx_rgmu_oob_clear(device, oob_gpu); + goto no_gx_power; + } + + kgsl_pwrscale_update_stats(device); + + /* Save active coresight registers if applicable */ + adreno_coresight_stop(adreno_dev); + + /* Save physical performance counter values before GPU power down*/ + adreno_perfcounter_save(adreno_dev); + + a6xx_rgmu_prepare_stop(device); + + a6xx_rgmu_oob_clear(device, oob_gpu); + +no_gx_power: + /* Halt all gx traffic */ + kgsl_regwrite(device, A6XX_GBIF_HALT, A6XX_GBIF_CLIENT_HALT_MASK); + + adreno_wait_for_halt_ack(device, ADRENO_REG_GBIF_HALT_ACK, + A6XX_GBIF_CLIENT_HALT_MASK); + + a6xx_disable_gpu_irq(adreno_dev); + + a6xx_rgmu_power_off(adreno_dev); + + adreno_set_active_ctxs_null(adreno_dev); + + adreno_dispatcher_stop(adreno_dev); + + adreno_ringbuffer_stop(adreno_dev); + + if (!IS_ERR_OR_NULL(adreno_dev->gpu_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpu_llc_slice); + + if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice)) + llcc_slice_deactivate(adreno_dev->gpuhtw_llc_slice); + + clear_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + device->state = KGSL_STATE_NONE; + + del_timer_sync(&device->idle_timer); + + kgsl_pwrscale_sleep(device); + + return ret; +} + +int a6xx_rgmu_restart(struct kgsl_device *device) +{ + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + + a6xx_disable_gpu_irq(adreno_dev); + + /* Hard reset the rgmu and gpu */ + a6xx_rgmu_suspend(adreno_dev); + + clear_bit(RGMU_PRIV_GPU_STARTED, &rgmu->flags); + + /* Attempt rebooting the rgmu and gpu */ + return a6xx_boot(adreno_dev); +} + +static int a6xx_rgmu_active_count_get(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret = 0; + + if (WARN_ON(!mutex_is_locked(&device->mutex))) + return -EINVAL; + + if (test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) + return -EINVAL; + + if (atomic_read(&device->active_cnt) == 0) + ret = a6xx_boot(adreno_dev); + + if (ret == 0) { + atomic_inc(&device->active_cnt); + trace_kgsl_active_count(device, + (unsigned long) __builtin_return_address(0)); + } + + return ret; +} + +static int a6xx_rgmu_pm_suspend(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + int ret; + + if (test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) + return 0; + + /* Halt any new submissions */ + reinit_completion(&device->halt_gate); + + /* wait for active count so device can be put in slumber */ + ret = kgsl_active_count_wait(device, 0); + if (ret) { + dev_err(device->dev, + "Timed out waiting for the active count\n"); + goto err; + } + + ret = adreno_idle(device); + if (ret) + goto err; + + a6xx_power_off(adreno_dev); + + set_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags); + + adreno_dispatcher_halt(device); + + return 0; +err: + adreno_dispatcher_start(device); + return ret; +} + +static void a6xx_rgmu_pm_resume(struct adreno_device *adreno_dev) +{ + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); + + if (!test_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags)) { + dev_err(device->dev, "resume invoked without a suspend\n"); + return; + } + + adreno_dispatcher_unhalt(device); + + clear_bit(RGMU_PRIV_PM_SUSPEND, &rgmu->flags); + + adreno_dispatcher_start(device); +} + static struct gmu_dev_ops a6xx_rgmudev = { - .load_firmware = a6xx_rgmu_load_firmware, .oob_set = a6xx_rgmu_oob_set, .oob_clear = a6xx_rgmu_oob_clear, - .irq_enable = a6xx_rgmu_irq_enable, - .irq_disable = a6xx_rgmu_irq_disable, - .rpmh_gpu_pwrctrl = a6xx_rgmu_gpu_pwrctrl, .gx_is_on = a6xx_rgmu_gx_is_on, - .prepare_stop = a6xx_rgmu_prepare_stop, - .wait_for_lowest_idle = a6xx_rgmu_wait_for_lowest_idle, .ifpc_store = a6xx_rgmu_ifpc_store, .ifpc_show = a6xx_rgmu_ifpc_show, .snapshot = a6xx_rgmu_device_snapshot, - .halt_execution = a6xx_rgmu_halt_execution, - .read_alwayson = a6xx_rgmu_read_alwayson, .gmu2host_intr_mask = RGMU_OOB_IRQ_MASK, .gmu_ao_intr_mask = RGMU_AO_IRQ_MASK, }; static struct gmu_core_ops a6xx_rgmu_ops = { - .init = a6xx_rgmu_init, - .start = a6xx_rgmu_start, - .stop = a6xx_rgmu_stop, .dcvs_set = a6xx_rgmu_dcvs_set, .snapshot = a6xx_rgmu_snapshot, - .suspend = a6xx_rgmu_suspend, }; static int a6xx_rgmu_irq_probe(struct kgsl_device *device) { - struct a6xx_rgmu_device *rgmu = A6XX_RGMU_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(ADRENO_DEVICE(device)); int ret; ret = kgsl_request_irq(rgmu->pdev, "kgsl_oob", @@ -898,18 +1197,51 @@ static int a6xx_rgmu_clocks_probe(struct a6xx_rgmu_device *rgmu, return 0; } +const struct adreno_power_ops a6xx_rgmu_power_ops = { + .first_open = a6xx_rgmu_first_open, + .last_close = a6xx_power_off, + .active_count_get = a6xx_rgmu_active_count_get, + .active_count_put = a6xx_rgmu_active_count_put, + .pm_suspend = a6xx_rgmu_pm_suspend, + .pm_resume = a6xx_rgmu_pm_resume, +}; + +int a6xx_rgmu_device_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore) +{ + struct adreno_device *adreno_dev; + struct kgsl_device *device; + struct a6xx_device *a6xx_dev; + int ret; + + a6xx_dev = devm_kzalloc(&pdev->dev, sizeof(*a6xx_dev), + GFP_KERNEL); + if (!a6xx_dev) + return -ENOMEM; + + adreno_dev = &a6xx_dev->adreno_dev; + + ret = a6xx_probe_common(pdev, adreno_dev, chipid, gpucore); + if (ret) + return ret; + + device = KGSL_DEVICE(adreno_dev); + + INIT_WORK(&device->idle_check_ws, rgmu_idle_check); + + timer_setup(&device->idle_timer, rgmu_idle_timer, 0); + + return 0; +} + /* Do not access any RGMU registers in RGMU probe function */ static int a6xx_rgmu_probe(struct kgsl_device *device, struct platform_device *pdev) { - struct a6xx_rgmu_device *rgmu; + struct adreno_device *adreno_dev = ADRENO_DEVICE(device); + struct a6xx_rgmu_device *rgmu = to_a6xx_rgmu(adreno_dev); struct resource *res; - int ret = -ENXIO; - - rgmu = devm_kzalloc(&pdev->dev, sizeof(*rgmu), GFP_KERNEL); - - if (rgmu == NULL) - return -ENOMEM; + int ret; rgmu->pdev = pdev; @@ -939,8 +1271,6 @@ static int a6xx_rgmu_probe(struct kgsl_device *device, return PTR_ERR(device->gmu_core.reg_virt); } - device->gmu_core.ptr = (void *)rgmu; - /* Initialize OOB and RGMU interrupts */ ret = a6xx_rgmu_irq_probe(device); if (ret) @@ -966,7 +1296,6 @@ static int a6xx_rgmu_probe(struct kgsl_device *device, static void a6xx_rgmu_remove(struct kgsl_device *device) { - a6xx_rgmu_stop(device); memset(&device->gmu_core, 0, sizeof(device->gmu_core)); } diff --git a/drivers/gpu/msm/adreno_a6xx_rgmu.h b/drivers/gpu/msm/adreno_a6xx_rgmu.h index 02839b16001e..f59c4de09dfd 100644 --- a/drivers/gpu/msm/adreno_a6xx_rgmu.h +++ b/drivers/gpu/msm/adreno_a6xx_rgmu.h @@ -15,6 +15,15 @@ #define MAX_RGMU_CLKS 8 +enum { + /* @RGMU_PRIV_FIRST_BOOT_DONE: The very first ggpu boot is done */ + RGMU_PRIV_FIRST_BOOT_DONE, + /* @RGMU_PRIV_GPU_STARTED: GPU has been started */ + RGMU_PRIV_GPU_STARTED, + /* @RGMU_PRIV_PM_SUSPEND: The rgmu driver is suspended */ + RGMU_PRIV_PM_SUSPEND, +}; + /** * struct a6xx_rgmu_device - rGMU device structure * @ver: RGMU firmware version @@ -47,8 +56,28 @@ struct a6xx_rgmu_device { struct clk *rgmu_clk; unsigned int idle_level; unsigned int fault_count; + /** @fault: to track whether we encountered a rgmu fault */ + bool fault; + /** @flags: rgmu internal flags */ + unsigned long flags; }; -#define A6XX_RGMU_DEVICE(_a) ((struct a6xx_rgmu_device *)((_a)->gmu_core.ptr)) +/** + * a6xx_rgmu_device_probe - Probe a6xx rgmu resources + * @pdev: Pointer to the platform device + * @chipid: Chipid of the target + * @gpucore: Pointer to the gpucore + * + * The target specific probe function for rgmu based a6xx targets. + */ +int a6xx_rgmu_device_probe(struct platform_device *pdev, + u32 chipid, const struct adreno_gpu_core *gpucore); -#endif /* __ADRENO_A6XX_RGMU_H */ +/** + * a6xx_rgmu_restart - Reset and restart the rgmu + * @device: Pointer to the kgsl device + * + * Return: 0 on success or negative error on failure + */ +int a6xx_rgmu_restart(struct kgsl_device *device); +#endif