diff --git a/drivers/gpu/msm/a6xx_reg.h b/drivers/gpu/msm/a6xx_reg.h index 9d16d09cb3d2..df52db8ac6d1 100644 --- a/drivers/gpu/msm/a6xx_reg.h +++ b/drivers/gpu/msm/a6xx_reg.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2017-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _A6XX_REG_H @@ -406,6 +407,8 @@ #define A6XX_RBBM_PERFCTR_RBBM_SEL_2 0x509 #define A6XX_RBBM_PERFCTR_RBBM_SEL_3 0x50A #define A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED 0x50B +#define A6XX_RBBM_PERFCTR_SRAM_INIT_CMD 0x50e +#define A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS 0x50f #define A6XX_RBBM_ISDB_CNT 0x533 #define A6XX_RBBM_NC_MODE_CNTL 0X534 diff --git a/drivers/gpu/msm/adreno.h b/drivers/gpu/msm/adreno.h index e7dd6c18a6ad..ac685b24144b 100644 --- a/drivers/gpu/msm/adreno.h +++ b/drivers/gpu/msm/adreno.h @@ -16,6 +16,9 @@ #include "adreno_ringbuffer.h" #include "kgsl_sharedmem.h" +/* Index to preemption scratch buffer to store KMD postamble */ +#define KMD_POSTAMBLE_IDX 100 + /* ADRENO_DEVICE - Given a kgsl_device return the adreno device struct */ #define ADRENO_DEVICE(device) \ container_of(device, struct adreno_device, dev) @@ -232,6 +235,9 @@ struct adreno_gpudev; /* Time to allow preemption to complete (in ms) */ #define ADRENO_PREEMPT_TIMEOUT 10000 +#define PREEMPT_SCRATCH_ADDR(dev, id) \ + ((dev)->preempt.scratch->gpuaddr + (id * sizeof(u64))) + /** * enum adreno_preempt_states * ADRENO_PREEMPT_NONE: No preemption is scheduled @@ -261,6 +267,7 @@ enum adreno_preempt_states { * skipsaverestore: To skip saverestore during L1 preemption (for 6XX) * usesgmem: enable GMEM save/restore across preemption (for 6XX) * count: Track the number of preemptions triggered + * @postamble_len: Number of dwords in KMD postamble pm4 packet */ struct adreno_preemption { atomic_t state; @@ -271,6 +278,7 @@ struct adreno_preemption { bool skipsaverestore; bool usesgmem; unsigned int count; + u32 postamble_len; }; struct adreno_busy_data { diff --git a/drivers/gpu/msm/adreno_a6xx_preempt.c b/drivers/gpu/msm/adreno_a6xx_preempt.c index 49b082dd0e7c..f0c5cf5a4869 100644 --- a/drivers/gpu/msm/adreno_a6xx_preempt.c +++ b/drivers/gpu/msm/adreno_a6xx_preempt.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "adreno.h" @@ -543,13 +544,24 @@ unsigned int a6xx_preemption_pre_ibsubmit( if (context) { struct adreno_context *drawctxt = ADRENO_CONTEXT(context); struct adreno_ringbuffer *rb = drawctxt->rb; - uint64_t dest = adreno_dev->preempt.scratch->gpuaddr - + (rb->id * sizeof(u64)); + uint64_t dest = PREEMPT_SCRATCH_ADDR(adreno_dev, rb->id); *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2); cmds += cp_gpuaddr(adreno_dev, cmds, dest); *cmds++ = lower_32_bits(gpuaddr); *cmds++ = upper_32_bits(gpuaddr); + + /* Add a KMD post amble to clear the perf counters during preemption */ + if (!adreno_dev->perfcounter) { + u64 kmd_postamble_addr = + PREEMPT_SCRATCH_ADDR(adreno_dev, KMD_POSTAMBLE_IDX); + + *cmds++ = cp_type7_packet(CP_SET_AMBLE, 3); + *cmds++ = lower_32_bits(kmd_postamble_addr); + *cmds++ = upper_32_bits(kmd_postamble_addr); + *cmds++ = FIELD_PREP(GENMASK(22, 20), CP_KMD_AMBLE_TYPE) + | (FIELD_PREP(GENMASK(19, 0), adreno_dev->preempt.postamble_len)); + } } return (unsigned int) (cmds - cmds_orig); @@ -562,8 +574,7 @@ unsigned int a6xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev, struct adreno_ringbuffer *rb = adreno_dev->cur_rb; if (rb) { - uint64_t dest = adreno_dev->preempt.scratch->gpuaddr - + (rb->id * sizeof(u64)); + uint64_t dest = PREEMPT_SCRATCH_ADDR(adreno_dev, adreno_dev->cur_rb->id); *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2); cmds += cp_gpuaddr(adreno_dev, cmds, dest); @@ -721,6 +732,31 @@ int a6xx_preemption_init(struct adreno_device *adreno_dev) if (ret) return ret; + /* + * First 8 dwords of the preemption scratch buffer is used to store the address for CP + * to save/restore VPC data. Reserve 11 dwords in the preemption scratch buffer from + * index KMD_POSTAMBLE_IDX for KMD postamble pm4 packets + */ + if (!adreno_dev->perfcounter) { + u32 *postamble = preempt->scratch->hostptr + (KMD_POSTAMBLE_IDX * sizeof(u64)); + u32 count = 0; + + postamble[count++] = cp_type7_packet(CP_REG_RMW, 3); + postamble[count++] = A6XX_RBBM_PERFCTR_SRAM_INIT_CMD; + postamble[count++] = 0x0; + postamble[count++] = 0x1; + + postamble[count++] = cp_type7_packet(CP_WAIT_REG_MEM, 6); + postamble[count++] = 0x3; + postamble[count++] = A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS; + postamble[count++] = 0x0; + postamble[count++] = 0x1; + postamble[count++] = 0x1; + postamble[count++] = 0x0; + + preempt->postamble_len = count; + } + set_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv); return 0; } diff --git a/drivers/gpu/msm/adreno_iommu.c b/drivers/gpu/msm/adreno_iommu.c index 73e6016bf1d9..be88bafb12da 100644 --- a/drivers/gpu/msm/adreno_iommu.c +++ b/drivers/gpu/msm/adreno_iommu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -217,6 +218,12 @@ static unsigned int _adreno_iommu_set_pt_v2_a6xx(struct kgsl_device *device, struct adreno_device *adreno_dev = ADRENO_DEVICE(device); unsigned int *cmds = cmds_orig; + /* Clear performance counters during contect switches */ + if (!adreno_dev->perfcounter) { + *cmds++ = cp_type4_packet(A6XX_RBBM_PERFCTR_SRAM_INIT_CMD, 1); + *cmds++ = 0x1; + } + /* CP switches the pagetable and flushes the Caches */ *cmds++ = cp_packet(adreno_dev, CP_SMMU_TABLE_UPDATE, 4); *cmds++ = lower_32_bits(ttbr0); @@ -245,6 +252,17 @@ static unsigned int _adreno_iommu_set_pt_v2_a6xx(struct kgsl_device *device, *cmds++ = 0; } + /* Wait for performance counter clear to finish */ + if (!adreno_dev->perfcounter) { + *cmds++ = cp_type7_packet(CP_WAIT_REG_MEM, 6); + *cmds++ = 0x3; + *cmds++ = A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS; + *cmds++ = 0x0; + *cmds++ = 0x1; + *cmds++ = 0x1; + *cmds++ = 0x0; + } + return cmds - cmds_orig; } diff --git a/drivers/gpu/msm/adreno_perfcounter.c b/drivers/gpu/msm/adreno_perfcounter.c index d0b0bd3a2846..49ff0356a065 100644 --- a/drivers/gpu/msm/adreno_perfcounter.c +++ b/drivers/gpu/msm/adreno_perfcounter.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -72,7 +73,8 @@ void adreno_perfcounter_restore(struct adreno_device *adreno_dev) const struct adreno_perfcount_group *group; unsigned int counter, groupid; - if (counters == NULL) + /* Do not save/restore if not requested */ + if (counters == NULL || !adreno_dev->perfcounter) return; for (groupid = 0; groupid < counters->group_count; groupid++) { @@ -106,7 +108,8 @@ inline void adreno_perfcounter_save(struct adreno_device *adreno_dev) const struct adreno_perfcount_group *group; unsigned int counter, groupid; - if (counters == NULL) + /* Do not save/restore if not requested */ + if (counters == NULL || !adreno_dev->perfcounter) return; for (groupid = 0; groupid < counters->group_count; groupid++) { diff --git a/drivers/gpu/msm/adreno_pm4types.h b/drivers/gpu/msm/adreno_pm4types.h index b64492044400..deb54dcc6ccc 100644 --- a/drivers/gpu/msm/adreno_pm4types.h +++ b/drivers/gpu/msm/adreno_pm4types.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef __ADRENO_PM4TYPES_H #define __ADRENO_PM4TYPES_H @@ -47,6 +48,9 @@ /* switches SMMU pagetable, used on a5xx only */ #define CP_SMMU_TABLE_UPDATE 0x53 +/* Designate command streams to be executed before/after CP does state restore during preemption */ +#define CP_SET_AMBLE 0x55 + /* Set internal CP registers, used to indicate context save data addresses */ #define CP_SET_PSEUDO_REGISTER 0x56 @@ -157,6 +161,9 @@ #define CP_LOADSTATE_STATETYPE_SHIFT 0x00000000 #define CP_LOADSTATE_EXTSRCADDR_SHIFT 0x00000002 +/* Used to define amble type in SET_AMBLE packet to execute during preemption */ +#define CP_KMD_AMBLE_TYPE 3 + static inline uint pm4_calc_odd_parity_bit(uint val) { return (0x9669 >> (0xf & ((val) ^ diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index 33e67cd0390a..0e25e7bfcf81 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2002,2007-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -482,7 +483,7 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb, if (gpudev->preemption_pre_ibsubmit && adreno_is_preemption_enabled(adreno_dev)) - total_sizedwords += 27; + total_sizedwords += 31; if (gpudev->preemption_post_ibsubmit && adreno_is_preemption_enabled(adreno_dev))