msm: kgsl: Zap performance counters across context switches

Performance counter values need not be retained across contexts unless
specifically requested for debug. Zap the counters by initialising
perfcounter SRAM with 0's using GPU_RBBM_PERFCTR_SRAM_INIT_CMD.

Add pm4 packets during context switches and add a KMD postamble packet to
clear the counters during preemption. Do not enable perfcounter save and
restore unless requested.

Change-Id: I371779ce659c07a1cc664327f5ecdcf0374201d8
Signed-off-by: Mohammed Mirza Mandayappurath Manzoor <quic_mmandaya@quicinc.com>
Signed-off-by: Harshitha Sai Neelati <quic_hsaineel@quicinc.com>

drivers/gpu/msm/a6xx_reg.h

@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2017-2021, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef _A6XX_REG_H
@@ -406,6 +407,8 @@
 #define A6XX_RBBM_PERFCTR_RBBM_SEL_2             0x509
 #define A6XX_RBBM_PERFCTR_RBBM_SEL_3             0x50A
 #define A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED        0x50B
+#define A6XX_RBBM_PERFCTR_SRAM_INIT_CMD          0x50e
+#define A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS       0x50f
 
 #define A6XX_RBBM_ISDB_CNT                       0x533
 #define A6XX_RBBM_NC_MODE_CNTL                   0X534

drivers/gpu/msm/adreno.h

@@ -16,6 +16,9 @@
 #include "adreno_ringbuffer.h"
 #include "kgsl_sharedmem.h"
 
+/* Index to preemption scratch buffer to store KMD postamble */
+#define KMD_POSTAMBLE_IDX 100
+
 /* ADRENO_DEVICE - Given a kgsl_device return the adreno device struct */
 #define ADRENO_DEVICE(device) \
 		container_of(device, struct adreno_device, dev)
@@ -232,6 +235,9 @@ struct adreno_gpudev;
 /* Time to allow preemption to complete (in ms) */
 #define ADRENO_PREEMPT_TIMEOUT 10000
 
+#define PREEMPT_SCRATCH_ADDR(dev, id) \
+	((dev)->preempt.scratch->gpuaddr + (id * sizeof(u64)))
+
 /**
  * enum adreno_preempt_states
  * ADRENO_PREEMPT_NONE: No preemption is scheduled
@@ -261,6 +267,7 @@ enum adreno_preempt_states {
  * skipsaverestore: To skip saverestore during L1 preemption (for 6XX)
  * usesgmem: enable GMEM save/restore across preemption (for 6XX)
  * count: Track the number of preemptions triggered
+ * @postamble_len: Number of dwords in KMD postamble pm4 packet
  */
 struct adreno_preemption {
 	atomic_t state;
@@ -271,6 +278,7 @@ struct adreno_preemption {
 	bool skipsaverestore;
 	bool usesgmem;
 	unsigned int count;
+	u32 postamble_len;
 };
 
 struct adreno_busy_data {

drivers/gpu/msm/adreno_a6xx_preempt.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include "adreno.h"
@@ -543,13 +544,24 @@ unsigned int a6xx_preemption_pre_ibsubmit(
 	if (context) {
 		struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
 		struct adreno_ringbuffer *rb = drawctxt->rb;
-		uint64_t dest = adreno_dev->preempt.scratch->gpuaddr
-			+ (rb->id * sizeof(u64));
+		uint64_t dest = PREEMPT_SCRATCH_ADDR(adreno_dev, rb->id);
 
 		*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2);
 		cmds += cp_gpuaddr(adreno_dev, cmds, dest);
 		*cmds++ = lower_32_bits(gpuaddr);
 		*cmds++ = upper_32_bits(gpuaddr);
+
+		/* Add a KMD post amble to clear the perf counters during preemption */
+		if (!adreno_dev->perfcounter) {
+			u64 kmd_postamble_addr =
+					PREEMPT_SCRATCH_ADDR(adreno_dev, KMD_POSTAMBLE_IDX);
+
+			*cmds++ = cp_type7_packet(CP_SET_AMBLE, 3);
+			*cmds++ = lower_32_bits(kmd_postamble_addr);
+			*cmds++ = upper_32_bits(kmd_postamble_addr);
+			*cmds++ = FIELD_PREP(GENMASK(22, 20), CP_KMD_AMBLE_TYPE)
+				| (FIELD_PREP(GENMASK(19, 0), adreno_dev->preempt.postamble_len));
+		}
 	}
 
 	return (unsigned int) (cmds - cmds_orig);
@@ -562,8 +574,7 @@ unsigned int a6xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev,
 	struct adreno_ringbuffer *rb = adreno_dev->cur_rb;
 
 	if (rb) {
-		uint64_t dest = adreno_dev->preempt.scratch->gpuaddr
-			+ (rb->id * sizeof(u64));
+		uint64_t dest = PREEMPT_SCRATCH_ADDR(adreno_dev, adreno_dev->cur_rb->id);
 
 		*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2);
 		cmds += cp_gpuaddr(adreno_dev, cmds, dest);
@@ -721,6 +732,31 @@ int a6xx_preemption_init(struct adreno_device *adreno_dev)
 	if (ret)
 		return ret;
 
+	/*
+	 * First 8 dwords of the preemption scratch buffer is used to store the address for CP
+	 * to save/restore VPC data. Reserve 11 dwords in the preemption scratch buffer from
+	 * index KMD_POSTAMBLE_IDX for KMD postamble pm4 packets
+	 */
+	if (!adreno_dev->perfcounter) {
+		u32 *postamble = preempt->scratch->hostptr + (KMD_POSTAMBLE_IDX * sizeof(u64));
+		u32 count = 0;
+
+		postamble[count++] = cp_type7_packet(CP_REG_RMW, 3);
+		postamble[count++] = A6XX_RBBM_PERFCTR_SRAM_INIT_CMD;
+		postamble[count++] = 0x0;
+		postamble[count++] = 0x1;
+
+		postamble[count++] = cp_type7_packet(CP_WAIT_REG_MEM, 6);
+		postamble[count++] = 0x3;
+		postamble[count++] = A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS;
+		postamble[count++] = 0x0;
+		postamble[count++] = 0x1;
+		postamble[count++] = 0x1;
+		postamble[count++] = 0x0;
+
+		preempt->postamble_len = count;
+	}
+
 	set_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv);
 	return 0;
 }

drivers/gpu/msm/adreno_iommu.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/slab.h>
@@ -217,6 +218,12 @@ static unsigned int _adreno_iommu_set_pt_v2_a6xx(struct kgsl_device *device,
 	struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
 	unsigned int *cmds = cmds_orig;
 
+	/* Clear performance counters during contect switches */
+	if (!adreno_dev->perfcounter) {
+		*cmds++ = cp_type4_packet(A6XX_RBBM_PERFCTR_SRAM_INIT_CMD, 1);
+		*cmds++ = 0x1;
+	}
+
 	/* CP switches the pagetable and flushes the Caches */
 	*cmds++ = cp_packet(adreno_dev, CP_SMMU_TABLE_UPDATE, 4);
 	*cmds++ = lower_32_bits(ttbr0);
@@ -245,6 +252,17 @@ static unsigned int _adreno_iommu_set_pt_v2_a6xx(struct kgsl_device *device,
 		*cmds++ = 0;
 	}
 
+	/* Wait for performance counter clear to finish */
+	if (!adreno_dev->perfcounter) {
+		*cmds++ = cp_type7_packet(CP_WAIT_REG_MEM, 6);
+		*cmds++ = 0x3;
+		*cmds++ = A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS;
+		*cmds++ = 0x0;
+		*cmds++ = 0x1;
+		*cmds++ = 0x1;
+		*cmds++ = 0x0;
+	}
+
 	return cmds - cmds_orig;
 }
 

drivers/gpu/msm/adreno_perfcounter.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/slab.h>
@@ -72,7 +73,8 @@ void adreno_perfcounter_restore(struct adreno_device *adreno_dev)
 	const struct adreno_perfcount_group *group;
 	unsigned int counter, groupid;
 
-	if (counters == NULL)
+	/* Do not save/restore if not requested */
+	if (counters == NULL || !adreno_dev->perfcounter)
 		return;
 
 	for (groupid = 0; groupid < counters->group_count; groupid++) {
@@ -106,7 +108,8 @@ inline void adreno_perfcounter_save(struct adreno_device *adreno_dev)
 	const struct adreno_perfcount_group *group;
 	unsigned int counter, groupid;
 
-	if (counters == NULL)
+	/* Do not save/restore if not requested */
+	if (counters == NULL || !adreno_dev->perfcounter)
 		return;
 
 	for (groupid = 0; groupid < counters->group_count; groupid++) {

drivers/gpu/msm/adreno_pm4types.h

@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2002,2007-2020, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 #ifndef __ADRENO_PM4TYPES_H
 #define __ADRENO_PM4TYPES_H
@@ -47,6 +48,9 @@
 /* switches SMMU pagetable, used on a5xx only */
 #define CP_SMMU_TABLE_UPDATE 0x53
 
+/* Designate command streams to be executed before/after CP does state restore during preemption */
+#define CP_SET_AMBLE		0x55
+
 /*  Set internal CP registers, used to indicate context save data addresses */
 #define CP_SET_PSEUDO_REGISTER      0x56
 
@@ -157,6 +161,9 @@
 #define CP_LOADSTATE_STATETYPE_SHIFT 0x00000000
 #define CP_LOADSTATE_EXTSRCADDR_SHIFT 0x00000002
 
+/* Used to define amble type in SET_AMBLE packet to execute during preemption */
+#define CP_KMD_AMBLE_TYPE 3
+
 static inline uint pm4_calc_odd_parity_bit(uint val)
 {
 	return (0x9669 >> (0xf & ((val) ^

drivers/gpu/msm/adreno_ringbuffer.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2002,2007-2021, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/interconnect.h>
@@ -482,7 +483,7 @@ adreno_ringbuffer_addcmds(struct adreno_ringbuffer *rb,
 
 	if (gpudev->preemption_pre_ibsubmit &&
 			adreno_is_preemption_enabled(adreno_dev))
-		total_sizedwords += 27;
+		total_sizedwords += 31;
 
 	if (gpudev->preemption_post_ibsubmit &&
 			adreno_is_preemption_enabled(adreno_dev))