From f919cbe3df1bfbc63b8a671d1e510a86f44d7fb6 Mon Sep 17 00:00:00 2001 From: Jordan Crouse Date: Wed, 15 Jul 2020 09:31:29 -0600 Subject: [PATCH 1/2] msm: kgsl: Use per-target scratch memory for preemption a5xx and a6xx both have a need for a bit of scratch memory for preemption. Rename the existing "counters" member in struct adreno_preemption from a5xx and use it for a6xx to store the context restore address. Change-Id: Ic0dedbad6493e99b35d8c771673e8f9f686f927c Signed-off-by: Jordan Crouse --- drivers/gpu/msm/adreno.h | 6 ++---- drivers/gpu/msm/adreno_a5xx_preempt.c | 8 ++++---- drivers/gpu/msm/adreno_a6xx_preempt.c | 22 +++++++++++++--------- drivers/gpu/msm/adreno_ioctl.c | 2 +- drivers/gpu/msm/kgsl.h | 8 -------- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index c5ad9d41038d..2afa04df690f 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -249,8 +249,7 @@ enum adreno_preempt_states { /** * struct adreno_preemption * @state: The current state of preemption - * @counters: Memory descriptor for the memory where the GPU writes the - * preemption counters on switch + * @scratch: Per-target scratch memory for implementation specific functionality * @timer: A timer to make sure preemption doesn't stall * @work: A work struct for the preemption worker (for 5XX) * preempt_level: The level of preemption (for 6XX) @@ -260,7 +259,7 @@ enum adreno_preempt_states { */ struct adreno_preemption { atomic_t state; - struct kgsl_memdesc *counters; + struct kgsl_memdesc *scratch; struct timer_list timer; struct work_struct work; unsigned int preempt_level; @@ -269,7 +268,6 @@ struct adreno_preemption { unsigned int count; }; - struct adreno_busy_data { unsigned int gpu_busy; unsigned int bif_ram_cycles; diff --git a/drivers/gpu/msm/adreno_a5xx_preempt.c b/drivers/gpu/msm/adreno_a5xx_preempt.c index 6155933da4d6..b48c2d2b9c4b 100644 --- a/drivers/gpu/msm/adreno_a5xx_preempt.c +++ b/drivers/gpu/msm/adreno_a5xx_preempt.c @@ -530,17 +530,17 @@ int a5xx_preemption_init(struct adreno_device *adreno_dev) INIT_WORK(&preempt->work, _a5xx_preemption_worker); /* Allocate mem for storing preemption counters */ - if (IS_ERR_OR_NULL(preempt->counters)) - preempt->counters = kgsl_allocate_global(device, + if (IS_ERR_OR_NULL(preempt->scratch)) + preempt->scratch = kgsl_allocate_global(device, adreno_dev->num_ringbuffers * A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE, 0, 0, "preemption_counters"); - ret = PTR_ERR_OR_ZERO(preempt->counters); + ret = PTR_ERR_OR_ZERO(preempt->scratch); if (ret) return ret; - addr = preempt->counters->gpuaddr; + addr = preempt->scratch->gpuaddr; /* Allocate mem for storing preemption switch record */ FOR_EACH_RINGBUFFER(adreno_dev, rb, i) { diff --git a/drivers/gpu/msm/adreno_a6xx_preempt.c b/drivers/gpu/msm/adreno_a6xx_preempt.c index 7a52c786174e..d8be38d767ec 100644 --- a/drivers/gpu/msm/adreno_a6xx_preempt.c +++ b/drivers/gpu/msm/adreno_a6xx_preempt.c @@ -295,8 +295,8 @@ void a6xx_preemption_trigger(struct adreno_device *adreno_dev) kgsl_sharedmem_writel(iommu->smmu_info, PREEMPT_SMMU_RECORD(context_idr), contextidr); - kgsl_sharedmem_readq(device->scratch, &gpuaddr, - SCRATCH_PREEMPTION_CTXT_RESTORE_ADDR_OFFSET(next->id)); + kgsl_sharedmem_readq(preempt->scratch, &gpuaddr, + next->id * sizeof(u64)); /* * Set a keepalive bit before the first preemption register write. @@ -530,12 +530,10 @@ unsigned int a6xx_preemption_pre_ibsubmit( rb->perfcounter_save_restore_desc->gpuaddr); if (context) { - struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct adreno_context *drawctxt = ADRENO_CONTEXT(context); struct adreno_ringbuffer *rb = drawctxt->rb; - uint64_t dest = - SCRATCH_PREEMPTION_CTXT_RESTORE_GPU_ADDR(device, - rb->id); + uint64_t dest = adreno_dev->preempt.scratch->gpuaddr + + (rb->id * sizeof(u64)); *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2); cmds += cp_gpuaddr(adreno_dev, cmds, dest); @@ -553,9 +551,8 @@ unsigned int a6xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev, struct adreno_ringbuffer *rb = adreno_dev->cur_rb; if (rb) { - struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - uint64_t dest = SCRATCH_PREEMPTION_CTXT_RESTORE_GPU_ADDR(device, - rb->id); + uint64_t dest = adreno_dev->preempt.scratch->gpuaddr + + (rb->id * sizeof(u64)); *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2); cmds += cp_gpuaddr(adreno_dev, cmds, dest); @@ -689,6 +686,13 @@ int a6xx_preemption_init(struct adreno_device *adreno_dev) return ret; } + if (IS_ERR_OR_NULL(preempt->scratch)) { + preempt->scratch = kgsl_allocate_global(device, PAGE_SIZE, + 0, 0, "preempt_scratch"); + if (IS_ERR(preempt->scratch)) + return PTR_ERR(preempt->scratch); + } + /* Allocate mem for storing preemption smmu record */ if (IS_ERR_OR_NULL(iommu->smmu_info)) iommu->smmu_info = kgsl_allocate_global(device, PAGE_SIZE, diff --git a/drivers/gpu/msm/adreno_ioctl.c b/drivers/gpu/msm/adreno_ioctl.c index 4e2b5254d46c..4894d4ede4dc 100644 --- a/drivers/gpu/msm/adreno_ioctl.c +++ b/drivers/gpu/msm/adreno_ioctl.c @@ -160,7 +160,7 @@ static long adreno_ioctl_preemption_counters_query( ARRAY_SIZE(adreno_dev->ringbuffers)); if (copy_to_user(u64_to_user_ptr(read->counters), - adreno_dev->preempt.counters->hostptr, + adreno_dev->preempt.scratch->hostptr, levels_to_copy * size_level)) return -EFAULT; diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h index 8ffbf0b07b8b..1be47571ddb0 100644 --- a/drivers/gpu/msm/kgsl.h +++ b/drivers/gpu/msm/kgsl.h @@ -65,7 +65,6 @@ * Used Data: * Offset: Length(bytes): What * 0x0: 4 * KGSL_PRIORITY_MAX_RB_LEVELS: RB0 RPTR - * 0x10: 8 * KGSL_PRIORITY_MAX_RB_LEVELS: RB0 CTXT RESTORE ADDR */ /* Shadow global helpers */ @@ -73,13 +72,6 @@ #define SCRATCH_RPTR_GPU_ADDR(dev, id) \ ((dev)->scratch->gpuaddr + SCRATCH_RPTR_OFFSET(id)) -#define SCRATCH_PREEMPTION_CTXT_RESTORE_ADDR_OFFSET(id) \ - (SCRATCH_RPTR_OFFSET(KGSL_PRIORITY_MAX_RB_LEVELS) + \ - ((id) * sizeof(uint64_t))) -#define SCRATCH_PREEMPTION_CTXT_RESTORE_GPU_ADDR(dev, id) \ - ((dev)->scratch->gpuaddr + \ - SCRATCH_PREEMPTION_CTXT_RESTORE_ADDR_OFFSET(id)) - /* Timestamp window used to detect rollovers (half of integer range) */ #define KGSL_TIMESTAMP_WINDOW 0x80000000 From bea2e01a92bc43d536d2e1633a639dca0cc2fb89 Mon Sep 17 00:00:00 2001 From: Akhil P Oommen Date: Thu, 16 Jul 2020 00:49:09 +0530 Subject: [PATCH 2/2] msm: kgsl: Mark the scratch buffer as privileged Mark the scratch buffer as privileged so that it can only be accessed by GPU through the ringbuffer. To accomplish this, we need to: 1. Disable the shadow rptr feature. 2. Trigger RPTR update from GPU using a WHERE_AM_I packet. 3. Add support for the new ucode. Change-Id: I9b388f55f53b69028b9bbb2306cb43fd1297c52f Signed-off-by: Akhil P Oommen --- drivers/gpu/msm/adreno.c | 15 +++++++- drivers/gpu/msm/adreno_a5xx.c | 11 +++--- drivers/gpu/msm/adreno_a5xx.h | 2 +- drivers/gpu/msm/adreno_a6xx.c | 8 ++--- drivers/gpu/msm/adreno_a6xx_preempt.c | 4 ++- drivers/gpu/msm/adreno_pm4types.h | 4 ++- drivers/gpu/msm/adreno_ringbuffer.c | 52 +++++++++++++++++++++++---- drivers/gpu/msm/adreno_ringbuffer.h | 3 ++ 8 files changed, 81 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index 3a5f8a08cb64..f61c83bdab9b 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -60,6 +60,19 @@ int adreno_wake_nice = -7; /* Number of milliseconds to stay active active after a wake on touch */ unsigned int adreno_wake_timeout = 100; +static u32 get_ucode_version(const u32 *data) +{ + u32 version; + + version = data[1]; + + if ((version & 0xf) != 0xa) + return version; + + version &= ~0xfff; + return version | ((data[3] & 0xfff000) >> 12); +} + int adreno_get_firmware(struct adreno_device *adreno_dev, const char *fwfile, struct adreno_firmware *firmware) { @@ -86,7 +99,7 @@ int adreno_get_firmware(struct adreno_device *adreno_dev, if (!ret) { memcpy(firmware->memdesc->hostptr, &fw->data[4], fw->size - 4); firmware->size = (fw->size - 4) / sizeof(u32); - firmware->version = *((u32 *)&fw->data[4]); + firmware->version = get_ucode_version((u32 *)fw->data); } release_firmware(fw); diff --git a/drivers/gpu/msm/adreno_a5xx.c b/drivers/gpu/msm/adreno_a5xx.c index 9801f912ce63..0962eb9c5c11 100644 --- a/drivers/gpu/msm/adreno_a5xx.c +++ b/drivers/gpu/msm/adreno_a5xx.c @@ -1621,12 +1621,15 @@ static int a5xx_post_start(struct adreno_device *adreno_dev) *cmds++ = 0xF; } - if (adreno_is_preemption_enabled(adreno_dev)) + if (adreno_is_preemption_enabled(adreno_dev)) { cmds += _preemption_init(adreno_dev, rb, cmds, NULL); + rb->_wptr = rb->_wptr - (42 - (cmds - start)); + ret = adreno_ringbuffer_submit_spin_nosync(rb, NULL, 2000); + } else { + rb->_wptr = rb->_wptr - (42 - (cmds - start)); + ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + } - rb->_wptr = rb->_wptr - (42 - (cmds - start)); - - ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); if (ret) adreno_spin_idle_debug(adreno_dev, "hw initialization failed to idle\n"); diff --git a/drivers/gpu/msm/adreno_a5xx.h b/drivers/gpu/msm/adreno_a5xx.h index 90baffa9fc52..1495a56036f1 100644 --- a/drivers/gpu/msm/adreno_a5xx.h +++ b/drivers/gpu/msm/adreno_a5xx.h @@ -102,7 +102,7 @@ void a5xx_crashdump_init(struct adreno_device *adreno_dev); void a5xx_hwcg_set(struct adreno_device *adreno_dev, bool on); -#define A5XX_CP_RB_CNTL_DEFAULT (((ilog2(4) << 8) & 0x1F00) | \ +#define A5XX_CP_RB_CNTL_DEFAULT ((1 << 27) | ((ilog2(4) << 8) & 0x1F00) | \ (ilog2(KGSL_RB_DWORDS >> 1) & 0x3F)) /* GPMU interrupt multiplexor */ #define FW_INTR_INFO (0) diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c index acacc69320d2..f274b7fca891 100644 --- a/drivers/gpu/msm/adreno_a6xx.c +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -870,7 +870,7 @@ static int a6xx_post_start(struct adreno_device *adreno_dev) rb->_wptr = rb->_wptr - (42 - (cmds - start)); - ret = adreno_ringbuffer_submit_spin(rb, NULL, 2000); + ret = adreno_ringbuffer_submit_spin_nosync(rb, NULL, 2000); if (ret) adreno_spin_idle_debug(adreno_dev, "hw preemption initialization failed to idle\n"); @@ -883,6 +883,8 @@ int a6xx_rb_start(struct adreno_device *adreno_dev) const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 cp_rb_cntl = A6XX_CP_RB_CNTL_DEFAULT | + (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV) ? 0 : (1 << 27)); struct adreno_ringbuffer *rb; uint64_t addr; int ret, i; @@ -911,8 +913,7 @@ int a6xx_rb_start(struct adreno_device *adreno_dev) * The size of the ringbuffer in the hardware is the log2 * representation of the size in quadwords (sizedwords / 2). */ - kgsl_regwrite(device, A6XX_CP_RB_CNTL, - A6XX_CP_RB_CNTL_DEFAULT); + kgsl_regwrite(device, A6XX_CP_RB_CNTL, cp_rb_cntl); kgsl_regwrite(device, A6XX_CP_RB_BASE, lower_32_bits(rb->buffer_desc->gpuaddr)); @@ -2438,7 +2439,6 @@ int a6xx_probe_common(struct platform_device *pdev, adreno_dev->perfctr_ifpc_lo = A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L; - return adreno_device_probe(pdev, adreno_dev); } diff --git a/drivers/gpu/msm/adreno_a6xx_preempt.c b/drivers/gpu/msm/adreno_a6xx_preempt.c index d8be38d767ec..21cfcf7811de 100644 --- a/drivers/gpu/msm/adreno_a6xx_preempt.c +++ b/drivers/gpu/msm/adreno_a6xx_preempt.c @@ -615,6 +615,8 @@ static int a6xx_preemption_ringbuffer_init(struct adreno_device *adreno_dev, struct adreno_ringbuffer *rb) { struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + u32 cp_rb_cntl = A6XX_CP_RB_CNTL_DEFAULT | + (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV) ? 0 : (1 << 27)); if (IS_ERR_OR_NULL(rb->preemption_desc)) rb->preemption_desc = kgsl_allocate_global(device, @@ -648,7 +650,7 @@ static int a6xx_preemption_ringbuffer_init(struct adreno_device *adreno_dev, kgsl_sharedmem_writel(rb->preemption_desc, PREEMPT_RECORD(data), 0); kgsl_sharedmem_writel(rb->preemption_desc, - PREEMPT_RECORD(cntl), A6XX_CP_RB_CNTL_DEFAULT); + PREEMPT_RECORD(cntl), cp_rb_cntl); kgsl_sharedmem_writel(rb->preemption_desc, PREEMPT_RECORD(rptr), 0); kgsl_sharedmem_writel(rb->preemption_desc, diff --git a/drivers/gpu/msm/adreno_pm4types.h b/drivers/gpu/msm/adreno_pm4types.h index 55c586860f8e..8be6637b088a 100644 --- a/drivers/gpu/msm/adreno_pm4types.h +++ b/drivers/gpu/msm/adreno_pm4types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved. */ #ifndef __ADRENO_PM4TYPES_H #define __ADRENO_PM4TYPES_H @@ -95,6 +95,8 @@ /* A5XX Enable yield in RB only */ #define CP_YIELD_ENABLE 0x1C +#define CP_WHERE_AM_I 0x62 + /* Enable/Disable/Defer A5x global preemption model */ #define CP_PREEMPT_ENABLE_GLOBAL 0x69 diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index 1b2a13e5e901..fe158328fba7 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -169,7 +169,7 @@ void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb, adreno_ringbuffer_wptr(adreno_dev, rb); } -int adreno_ringbuffer_submit_spin(struct adreno_ringbuffer *rb, +int adreno_ringbuffer_submit_spin_nosync(struct adreno_ringbuffer *rb, struct adreno_submit_time *time, unsigned int timeout) { struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); @@ -178,6 +178,40 @@ int adreno_ringbuffer_submit_spin(struct adreno_ringbuffer *rb, return adreno_spin_idle(adreno_dev, timeout); } +/* + * adreno_ringbuffer_submit_spin() - Submit the cmds and wait until GPU is idle + * @rb: Pointer to ringbuffer + * @time: Pointer to adreno_submit_time + * @timeout: timeout value in ms + * + * Add commands to the ringbuffer and wait until GPU goes to idle. This routine + * inserts a WHERE_AM_I packet to trigger a shadow rptr update. So, use + * adreno_ringbuffer_submit_spin_nosync() if the previous cmd in the RB is a + * CSY packet because CSY followed by WHERE_AM_I is not legal. + */ +int adreno_ringbuffer_submit_spin(struct adreno_ringbuffer *rb, + struct adreno_submit_time *time, unsigned int timeout) +{ + struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb); + struct kgsl_device *device = KGSL_DEVICE(adreno_dev); + unsigned int *cmds; + + /* GPUs which support APRIV feature doesn't require a WHERE_AM_I */ + if (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV) || + adreno_is_a3xx(adreno_dev)) + return adreno_ringbuffer_submit_spin_nosync(rb, time, timeout); + + cmds = adreno_ringbuffer_allocspace(rb, 3); + if (IS_ERR(cmds)) + return PTR_ERR(cmds); + + *cmds++ = cp_packet(adreno_dev, CP_WHERE_AM_I, 2); + cmds += cp_gpuaddr(adreno_dev, cmds, + SCRATCH_RPTR_GPU_ADDR(device, rb->id)); + + return adreno_ringbuffer_submit_spin_nosync(rb, time, timeout); +} + unsigned int *adreno_ringbuffer_allocspace(struct adreno_ringbuffer *rb, unsigned int dwords) { @@ -304,11 +338,8 @@ int adreno_ringbuffer_init(struct adreno_device *adreno_dev) int status = -ENOMEM; if (!adreno_is_a3xx(adreno_dev)) { - unsigned int priv = KGSL_MEMDESC_RANDOM; - - /* For targets that support it, make the scratch privileged */ - if (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) - priv |= KGSL_MEMDESC_PRIVILEGED; + unsigned int priv = + KGSL_MEMDESC_RANDOM | KGSL_MEMDESC_PRIVILEGED; if (IS_ERR_OR_NULL(device->scratch)) { device->scratch = kgsl_allocate_global(device, @@ -495,6 +526,9 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, if (gpudev->preemption_post_ibsubmit && adreno_is_preemption_enabled(adreno_dev)) total_sizedwords += 10; + else if (!adreno_is_a3xx(adreno_dev) && + !ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) + total_sizedwords += 3; /* * a5xx uses 64 bit memory address. pm4 commands that involve read/write @@ -701,6 +735,12 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, adreno_is_preemption_enabled(adreno_dev)) ringcmds += gpudev->preemption_post_ibsubmit(adreno_dev, ringcmds); + else if (!adreno_is_a3xx(adreno_dev) && + !ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) { + *ringcmds++ = cp_packet(adreno_dev, CP_WHERE_AM_I, 2); + ringcmds += cp_gpuaddr(adreno_dev, ringcmds, + SCRATCH_RPTR_GPU_ADDR(device, rb->id)); + } /* * If we have more ringbuffer commands than space reserved diff --git a/drivers/gpu/msm/adreno_ringbuffer.h b/drivers/gpu/msm/adreno_ringbuffer.h index f68521abccfc..9a804d45c63b 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.h +++ b/drivers/gpu/msm/adreno_ringbuffer.h @@ -163,6 +163,9 @@ int adreno_ringbuffer_issue_internal_cmds(struct adreno_ringbuffer *rb, void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb, struct adreno_submit_time *time); +int adreno_ringbuffer_submit_spin_nosync(struct adreno_ringbuffer *rb, + struct adreno_submit_time *time, unsigned int timeout); + int adreno_ringbuffer_submit_spin(struct adreno_ringbuffer *rb, struct adreno_submit_time *time, unsigned int timeout);