msm: kgsl: Add a6xx_hwsched_snapshot function

Since GMU tells us the exact submission that faulted via
the context bad hfi, look up the command object corresponding
to the submission and mark it for snapshot purposes. Create the
top level gpudev snapshot function for hw scheduling which first
snapshots the faulted command object and then calls into the
generic gmu snapshot function.

Change-Id: I5c5300b7b147dfcf0c009b44376ba0c6e1f8e25b
Signed-off-by: Harshdeep Dhatt <hdhatt@codeaurora.org>

drivers/gpu/msm/adreno.h

@@ -1773,6 +1773,13 @@ static inline void adreno_reg_offset_init(u32 *reg_offsets)
 	}
 }
 
+static inline u32 adreno_get_level(u32 priority)
+{
+	u32 level = priority / KGSL_PRIORITY_MAX_RB_LEVELS;
+
+	return min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1);
+}
+
 int adreno_gmu_fenced_write(struct adreno_device *adreno_dev,
 	enum adreno_regs offset, unsigned int val,
 	unsigned int fence_mask);

drivers/gpu/msm/adreno_a6xx.c

@@ -2801,7 +2801,7 @@ struct adreno_gpudev adreno_a6xx_gpudev = {
 struct adreno_gpudev adreno_a6xx_hwsched_gpudev = {
 	.reg_offsets = a6xx_register_offsets,
 	.probe = a6xx_hwsched_probe,
-	.snapshot = a6xx_gmu_snapshot,
+	.snapshot = a6xx_hwsched_snapshot,
 	.irq_handler = a6xx_irq_handler,
 	.perfcounters = &a6xx_perfcounters,
 	.read_throttling_counters = a6xx_read_throttling_counters,

drivers/gpu/msm/adreno_a6xx_hwsched.c

@@ -15,6 +15,14 @@
 #include "kgsl_trace.h"
 #include "kgsl_util.h"
 
+void a6xx_hwsched_snapshot(struct adreno_device *adreno_dev,
+	struct kgsl_snapshot *snapshot)
+{
+	adreno_hwsched_parse_fault_cmdobj(adreno_dev, snapshot);
+
+	a6xx_gmu_snapshot(adreno_dev, snapshot);
+}
+
 static int a6xx_hwsched_gmu_first_boot(struct adreno_device *adreno_dev)
 {
 	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

drivers/gpu/msm/adreno_a6xx_hwsched.h

@@ -39,4 +39,14 @@ int a6xx_hwsched_probe(struct platform_device *pdev,
  * @adreno_dev: Pointer to the adreno device
  */
 void a6xx_hwsched_restart(struct adreno_device *adreno_dev);
+
+/**
+ * a6xx_hwsched_snapshot - take a6xx hwsched snapshot
+ * @adreno_dev: Pointer to the adreno device
+ * @snapshot: Pointer to the snapshot instance
+ *
+ * Snapshot the faulty ib and then snapshot rest of a6xx gmu things
+ */
+void a6xx_hwsched_snapshot(struct adreno_device *adreno_dev,
+	struct kgsl_snapshot *snapshot);
 #endif

drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c

@@ -151,13 +151,6 @@ static void a6xx_receive_ack_async(struct adreno_device *adreno_dev, void *rcvd)
 			MSG_HDR_GET_SEQNUM(waiters[i]));
 }
 
-static u32 get_level(u32 priority)
-{
-	u32 level = priority / KGSL_PRIORITY_MAX_RB_LEVELS;
-
-	return min_t(u32, level, KGSL_PRIORITY_MAX_RB_LEVELS - 1);
-}
-
 static void log_profiling_info(struct adreno_device *adreno_dev, u32 *rcvd)
 {
 	struct hfi_ts_retire_cmd *cmd = (struct hfi_ts_retire_cmd *)rcvd;
@@ -169,7 +162,7 @@ static void log_profiling_info(struct adreno_device *adreno_dev, u32 *rcvd)
 		return;
 
 	info.timestamp = cmd->ts;
-	info.rb_id = get_level(context->priority);
+	info.rb_id = adreno_get_level(context->priority);
 	info.gmu_dispatch_queue = context->gmu_dispatch_queue;
 	info.submitted_to_rb = cmd->submitted_to_rb;
 	info.sop = cmd->sop;
@@ -832,10 +825,12 @@ static void process_ts_retire(struct adreno_device *adreno_dev, u32 *rcvd)
 
 static void process_ctx_bad(struct adreno_device *adreno_dev, void *rcvd)
 {
+	struct hfi_context_bad_cmd *cmd = rcvd;
+
 	/* Block dispatcher to submit more commands */
 	adreno_get_gpu_halt(adreno_dev);
 
-	adreno_hwsched_set_fault(adreno_dev);
+	adreno_hwsched_mark_drawobj(adreno_dev, cmd->ctxt_id, cmd->ts);
 }
 
 static int hfi_f2h_main(void *arg)
@@ -942,7 +937,7 @@ static void add_profile_events(struct adreno_device *adreno_dev,
 	time_in_ns = do_div(time_in_s, 1000000000);
 
 	info.inflight = -1;
-	info.rb_id = get_level(context->priority);
+	info.rb_id = adreno_get_level(context->priority);
 	info.gmu_dispatch_queue = context->gmu_dispatch_queue;
 
 	trace_adreno_cmdbatch_submitted(drawobj, &info, time->ticks,
@@ -985,7 +980,7 @@ static u32 get_next_dq(u32 priority)
 
 static u32 get_dq_id(u32 priority)
 {
-	u32 level = get_level(priority);
+	u32 level = adreno_get_level(priority);
 
 	return get_next_dq(level);
 }

drivers/gpu/msm/adreno_hwsched.c

@@ -6,6 +6,7 @@
 #include "adreno.h"
 #include "adreno_a6xx.h"
 #include "adreno_a6xx_hwsched.h"
+#include "adreno_snapshot.h"
 #include "adreno_trace.h"
 
 /* This structure represents inflight command object */
@@ -1005,6 +1006,18 @@ static void adreno_hwsched_init_replay(struct adreno_hwsched *hwsched)
 	}
 }
 
+static void force_retire_timestamp(struct kgsl_device *device,
+	struct kgsl_drawobj *drawobj)
+{
+	kgsl_sharedmem_writel(device->memstore,
+		KGSL_MEMSTORE_OFFSET(drawobj->context->id, soptimestamp),
+		drawobj->timestamp);
+
+	kgsl_sharedmem_writel(device->memstore,
+		KGSL_MEMSTORE_OFFSET(drawobj->context->id, eoptimestamp),
+		drawobj->timestamp);
+}
+
 static void adreno_hwsched_complete_replay(struct adreno_device *adreno_dev)
 {
 	struct adreno_hwsched *hwsched = to_hwsched(adreno_dev);
@@ -1054,6 +1067,93 @@ static void adreno_hwsched_complete_replay(struct adreno_device *adreno_dev)
 	}
 }
 
+static void do_fault_header(struct adreno_device *adreno_dev,
+	struct kgsl_drawobj *drawobj)
+{
+	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
+	struct adreno_context *drawctxt = ADRENO_CONTEXT(drawobj->context);
+	u32 status, rptr, wptr, ib1sz, ib2sz;
+	u64 ib1base, ib2base;
+
+	adreno_readreg(adreno_dev, ADRENO_REG_RBBM_STATUS, &status);
+	adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_RPTR, &rptr);
+	adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr);
+	adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB1_BASE,
+			ADRENO_REG_CP_IB1_BASE_HI, &ib1base);
+	adreno_readreg(adreno_dev, ADRENO_REG_CP_IB1_BUFSZ, &ib1sz);
+	adreno_readreg64(adreno_dev, ADRENO_REG_CP_IB2_BASE,
+			ADRENO_REG_CP_IB2_BASE_HI, &ib2base);
+	adreno_readreg(adreno_dev, ADRENO_REG_CP_IB2_BUFSZ, &ib2sz);
+
+	drawobj->context->last_faulted_cmd_ts = drawobj->timestamp;
+	drawobj->context->total_fault_count++;
+
+	pr_context(device, drawobj->context,
+		"ctx %d ctx_type %s ts %d status %8.8X dispatch_queue=%d rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
+		drawobj->context->id, kgsl_context_type(drawctxt->type),
+		drawobj->timestamp, status,
+		drawobj->context->gmu_dispatch_queue, rptr, wptr,
+		ib1base, ib1sz,	ib2base, ib2sz);
+
+	trace_adreno_gpu_fault(drawobj->context->id, drawobj->timestamp, status,
+		rptr, wptr, ib1base, ib1sz, ib2base, ib2sz,
+		adreno_get_level(drawobj->context->priority));
+}
+
+static struct cmd_list_obj *get_fault_cmdobj(struct adreno_device *adreno_dev)
+{
+	struct adreno_hwsched *hwsched = to_hwsched(adreno_dev);
+	struct cmd_list_obj *obj, *tmp;
+
+	list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
+
+		if (test_bit(CMDOBJ_FAULT, &obj->cmdobj->priv))
+			return obj;
+	}
+
+	return NULL;
+}
+
+static void reset_and_snapshot(struct adreno_device *adreno_dev)
+{
+	struct kgsl_drawobj *drawobj = NULL;
+	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
+	struct kgsl_context *context = NULL;
+	struct cmd_list_obj *obj = get_fault_cmdobj(adreno_dev);
+	struct adreno_hwsched *hwsched = to_hwsched(adreno_dev);
+
+	if (!obj) {
+		kgsl_device_snapshot(device, NULL, false);
+		goto done;
+	}
+
+	drawobj = DRAWOBJ(obj->cmdobj);
+
+	context = drawobj->context;
+
+	do_fault_header(adreno_dev, drawobj);
+
+	kgsl_device_snapshot(device, context, false);
+
+	force_retire_timestamp(device, drawobj);
+
+	if (context->flags & KGSL_CONTEXT_INVALIDATE_ON_FAULT) {
+		adreno_mark_guilty_context(device, context->id);
+		adreno_drawctxt_invalidate(device, context);
+	}
+
+	/* This must always happen after taking snapshot */
+	clear_bit(CMDOBJ_FAULT, &obj->cmdobj->priv);
+
+	/*
+	 * Put back the reference which we incremented when marking the faulty
+	 *  cmdobj
+	 */
+	kgsl_drawobj_put(drawobj);
+done:
+	a6xx_hwsched_restart(adreno_dev);
+	adreno_hwsched_init_replay(hwsched);
+}
 
 static void adreno_hwsched_recovery(struct adreno_device *adreno_dev)
 {
@@ -1063,10 +1163,8 @@ static void adreno_hwsched_recovery(struct adreno_device *adreno_dev)
 	mutex_lock(&device->mutex);
 
 	if ((atomic_cmpxchg(&hwsched->fault, ADRENO_HWSCHED_FAULT_RESTART,
-		ADRENO_HWSCHED_FAULT_REPLAY) == ADRENO_HWSCHED_FAULT_RESTART)) {
-		a6xx_hwsched_restart(adreno_dev);
-		adreno_hwsched_init_replay(hwsched);
-	}
+		ADRENO_HWSCHED_FAULT_REPLAY) == ADRENO_HWSCHED_FAULT_RESTART))
+		reset_and_snapshot(adreno_dev);
 
 	adreno_hwsched_complete_replay(adreno_dev);
 
@@ -1135,3 +1233,51 @@ void adreno_hwsched_init(struct adreno_device *adreno_dev)
 		init_llist_head(&hwsched->requeue[i]);
 	}
 }
+
+void adreno_hwsched_mark_drawobj(struct adreno_device *adreno_dev,
+	u32 ctxt_id, u32 ts)
+{
+	struct adreno_hwsched *hwsched = to_hwsched(adreno_dev);
+	struct cmd_list_obj *obj, *tmp;
+	struct kgsl_drawobj *drawobj = NULL;
+
+	mutex_lock(&hwsched->mutex);
+
+	list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
+
+		drawobj = DRAWOBJ(obj->cmdobj);
+
+		if (ctxt_id == drawobj->context->id &&
+			ts == drawobj->timestamp) {
+
+			if (kref_get_unless_zero(&drawobj->refcount))
+				set_bit(CMDOBJ_FAULT, &obj->cmdobj->priv);
+			break;
+		}
+	}
+
+	adreno_hwsched_set_fault(adreno_dev);
+
+	mutex_unlock(&hwsched->mutex);
+}
+
+void adreno_hwsched_parse_fault_cmdobj(struct adreno_device *adreno_dev,
+	struct kgsl_snapshot *snapshot)
+{
+	struct adreno_hwsched *hwsched = to_hwsched(adreno_dev);
+	struct cmd_list_obj *obj, *tmp;
+
+	list_for_each_entry_safe(obj, tmp, &hwsched->cmd_list, node) {
+		struct kgsl_drawobj_cmd *cmdobj = obj->cmdobj;
+
+		if (test_bit(CMDOBJ_FAULT, &cmdobj->priv)) {
+			struct kgsl_memobj_node *ib;
+
+			list_for_each_entry(ib, &cmdobj->cmdlist, node) {
+				adreno_parse_ib(KGSL_DEVICE(adreno_dev),
+					snapshot, snapshot->process,
+					ib->gpuaddr, ib->size >> 2);
+			}
+		}
+	}
+}

drivers/gpu/msm/adreno_hwsched.h

@@ -97,4 +97,27 @@ void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev);
  * @adreno_dev: A handle to adreno device
  */
 void adreno_hwsched_set_fault(struct adreno_device *adreno_dev);
+
+/**
+ * adreno_hwsched_mark_drawobj() - Get the drawobj that faulted
+ * @adreno_dev: pointer to the adreno device
+ * @ctxt_id: context id of the faulty submission
+ * @ts: timestamp of the faulty submission
+ *
+ * When we get a context bad hfi, use this function to get to the
+ * faulty submission and mark the submission for snapshot purposes
+ */
+void adreno_hwsched_mark_drawobj(struct adreno_device *adreno_dev, u32 ctxt_id,
+	u32 ts);
+
+/**
+ * adreno_hwsched_parse_fault_ib - Parse the faulty submission
+ * @adreno_dev: pointer to the adreno device
+ * @snapshot: Pointer to the snapshot structure
+ *
+ * Walk the list of active submissions to find the one that faulted and
+ * parse it so that relevant command buffers can be added to the snapshot
+ */
+void adreno_hwsched_parse_fault_cmdobj(struct adreno_device *adreno_dev,
+	struct kgsl_snapshot *snapshot);
 #endif

drivers/gpu/msm/adreno_snapshot.c

@@ -171,13 +171,7 @@ static int snapshot_freeze_obj_list(struct kgsl_snapshot *snapshot,
 	return ret;
 }
 
-/*
- * We want to store the last executed IB1 and IB2 in the static region to ensure
- * that we get at least some information out of the snapshot even if we can't
- * access the dynamic data from the sysfs file.  Push all other IBs on the
- * dynamic list
- */
-static inline void parse_ib(struct kgsl_device *device,
+void adreno_parse_ib(struct kgsl_device *device,
 		struct kgsl_snapshot *snapshot,
 		struct kgsl_process_private *process,
 		uint64_t gpuaddr, uint64_t dwords)
@@ -245,8 +239,8 @@ static void dump_all_ibs(struct kgsl_device *device,
 				ibaddr, ibsize))
 				continue;
 
-			parse_ib(device, snapshot, snapshot->process, ibaddr,
-				ibsize);
+			adreno_parse_ib(device, snapshot, snapshot->process,
+				ibaddr, ibsize);
 		} else
 			index = index + 1;
 	}
@@ -398,7 +392,7 @@ static void snapshot_rb_ibs(struct kgsl_device *device,
 				ibaddr, ibsize))
 				continue;
 
-			parse_ib(device, snapshot, snapshot->process,
+			adreno_parse_ib(device, snapshot, snapshot->process,
 				ibaddr, ibsize);
 		} else
 			index = (index + 1) % KGSL_RB_DWORDS;
@@ -561,7 +555,7 @@ static void kgsl_snapshot_add_active_ib_obj_list(struct kgsl_device *device,
 		index = find_object(snapshot->ib2base, snapshot->process);
 
 		if (index != -ENOENT)
-			parse_ib(device, snapshot, snapshot->process,
+			adreno_parse_ib(device, snapshot, snapshot->process,
 				snapshot->ib2base, objbuf[index].size >> 2);
 	}
 }
@@ -830,13 +824,13 @@ void adreno_snapshot(struct kgsl_device *device, struct kgsl_snapshot *snapshot,
 
 	snapshot_frozen_objsize = 0;
 
+	setup_fault_process(device, snapshot,
+			context ? context->proc_priv : NULL);
+
 	/* Add GPU specific sections - registers mainly, but other stuff too */
 	if (gpudev->snapshot)
 		gpudev->snapshot(adreno_dev, snapshot);
 
-	setup_fault_process(device, snapshot,
-			context ? context->proc_priv : NULL);
-
 	snapshot->ib1dumped = false;
 	snapshot->ib2dumped = false;
 

drivers/gpu/msm/adreno_snapshot.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2013-2015,2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2013-2015,2020, The Linux Foundation. All rights reserved.
  */
 #ifndef __ADRENO_SNAPSHOT_H
 #define __ADRENO_SNAPSHOT_H
@@ -39,4 +39,22 @@ void adreno_snapshot_vbif_registers(struct kgsl_device *device,
 		const struct adreno_vbif_snapshot_registers *list,
 		unsigned int count);
 
+/**
+ * adreno_parse_ib - Parse the given IB
+ * @device: Pointer to the kgsl device
+ * @snapshot: Pointer to the snapshot structure
+ * @process: Process to which this IB belongs
+ * @gpuaddr: Gpu address of the IB
+ * @dwords: Size in dwords of the IB
+ *
+ * We want to store the last executed IB1 and IB2 in the static region to ensure
+ * that we get at least some information out of the snapshot even if we can't
+ * access the dynamic data from the sysfs file.  Push all other IBs on the
+ * dynamic list
+ */
+void adreno_parse_ib(struct kgsl_device *device,
+	struct kgsl_snapshot *snapshot,
+	struct kgsl_process_private *process,
+	u64 gpuaddr, u64 dwords);
+
 #endif /*__ADRENO_SNAPSHOT_H */

drivers/gpu/msm/kgsl_drawobj.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved.
  */
 
 #ifndef __KGSL_DRAWOBJ_H
@@ -165,6 +165,7 @@ struct kgsl_drawobj_sparse {
  *           command obj
  * @CMDOBJ_WFI - Force wait-for-idle for the submission
  * @CMDOBJ_PROFILE - store the start / retire ticks for
+ * @CMDOBJ_FAULT - Mark the command object as faulted
  * the command obj in the profiling buffer
  */
 enum kgsl_drawobj_cmd_priv {
@@ -172,6 +173,7 @@ enum kgsl_drawobj_cmd_priv {
 	CMDOBJ_FORCE_PREAMBLE,
 	CMDOBJ_WFI,
 	CMDOBJ_PROFILE,
+	CMDOBJ_FAULT,
 };
 
 struct kgsl_ibdesc;