msm: kgsl: Add the hwsched dispatcher

We need a dispatcher which can keep track of inflight submissions
so that in case of a fault, the remaining submissions can be re-
submitted after reset. Also, we need a dispatcher to hold queued
submissions until the input fences are signaled.

Change-Id: I59f5cf0fed6d443cf8a55fd0f40e3f16600d9a6f
Signed-off-by: Harshdeep Dhatt <hdhatt@codeaurora.org>

drivers/gpu/msm/Makefile

@@ -44,6 +44,7 @@ msm_kgsl-y += \
 	adreno_cp_parser.o \
 	adreno_dispatch.o \
 	adreno_drawctxt.o \
+	adreno_hwsched.o \
 	adreno_ioctl.o \
 	adreno_perfcounter.o \
 	adreno_ringbuffer.o \

drivers/gpu/msm/adreno.c

@@ -22,6 +22,7 @@
 #include "adreno_a5xx.h"
 #include "adreno_a6xx.h"
 #include "adreno_compat.h"
+#include "adreno_hwsched.h"
 #include "adreno_iommu.h"
 #include "adreno_trace.h"
 #include "kgsl_bus.h"
@@ -1522,10 +1523,15 @@ static void adreno_unbind(struct device *dev)
 
 	kgsl_pwrscale_close(device);
 
-	adreno_dispatcher_close(adreno_dev);
-	adreno_ringbuffer_close(adreno_dev);
+	if (test_bit(GMU_DISPATCH, &device->gmu_core.flags))
+		adreno_hwsched_dispatcher_close(adreno_dev);
+	else {
+		adreno_dispatcher_close(adreno_dev);
 
-	adreno_fault_detect_stop(adreno_dev);
+		adreno_ringbuffer_close(adreno_dev);
+
+		adreno_fault_detect_stop(adreno_dev);
+	}
 
 	kfree(adreno_ft_regs);
 	adreno_ft_regs = NULL;
@@ -3607,6 +3613,12 @@ static int adreno_queue_cmds(struct kgsl_device_private *dev_priv,
 	struct kgsl_context *context, struct kgsl_drawobj *drawobj[],
 	u32 count, u32 *timestamp)
 {
+	struct kgsl_device *device = dev_priv->device;
+
+	if (test_bit(GMU_DISPATCH, &device->gmu_core.flags))
+		return adreno_hwsched_queue_cmds(dev_priv, context, drawobj,
+				count, timestamp);
+
 	return adreno_dispatcher_queue_cmds(dev_priv, context, drawobj, count,
 			timestamp);
 }
@@ -3614,6 +3626,10 @@ static int adreno_queue_cmds(struct kgsl_device_private *dev_priv,
 static void adreno_drawctxt_sched(struct kgsl_device *device,
 		struct kgsl_context *context)
 {
+	if (test_bit(GMU_DISPATCH, &device->gmu_core.flags))
+		return adreno_hwsched_queue_context(device,
+			ADRENO_CONTEXT(context));
+
 	adreno_dispatcher_queue_context(device, ADRENO_CONTEXT(context));
 }
 

drivers/gpu/msm/adreno_a6xx_hwsched.c

@@ -393,6 +393,8 @@ static int a6xx_hwsched_boot(struct adreno_device *adreno_dev)
 	if (ret)
 		return ret;
 
+	adreno_hwsched_start(adreno_dev);
+
 	mod_timer(&device->idle_timer, jiffies +
 			device->pwrctrl.interval_timeout);
 
@@ -437,6 +439,10 @@ static int a6xx_hwsched_first_boot(struct adreno_device *adreno_dev)
 	if (ret)
 		return ret;
 
+	adreno_hwsched_init(adreno_dev);
+
+	adreno_hwsched_start(adreno_dev);
+
 	adreno_get_bus_counters(adreno_dev);
 
 	adreno_dev->cooperative_reset = ADRENO_FEATURE(adreno_dev,

drivers/gpu/msm/adreno_a6xx_hwsched.h

@@ -7,6 +7,7 @@
 #define _ADRENO_A6XX_HWSCHED_H_
 
 #include "adreno_a6xx_hwsched_hfi.h"
+#include "adreno_hwsched.h"
 
 /**
  * struct a6xx_hwsched_device - Container for the a6xx hwscheduling device
@@ -16,6 +17,8 @@ struct a6xx_hwsched_device {
 	struct a6xx_device a6xx_dev;
 	/** @hwsched_hfi: Container for hwscheduling specific hfi resources */
 	struct a6xx_hwsched_hfi hwsched_hfi;
+	/** @hwsched: Container for the hardware dispatcher */
+	struct adreno_hwsched hwsched;
 };
 
 /**

drivers/gpu/msm/adreno_a6xx_hwsched_hfi.c

@@ -849,7 +849,7 @@ static int hfi_context_register(struct adreno_device *adreno_dev,
 
 #define DISPQ_IRQ_BIT(_idx) BIT((_idx) + HFI_DSP_IRQ_BASE)
 
-int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags,
+int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev,
 	struct kgsl_drawobj_cmd *cmdobj)
 {
 	struct a6xx_hfi *hfi = to_a6xx_hfi(adreno_dev);
@@ -868,6 +868,10 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags,
 	list_for_each_entry(ib, &cmdobj->cmdlist, node)
 		numibs++;
 
+	/* We need to dispatch a marker object but not execute it on the GPU */
+	if (test_bit(CMDOBJ_SKIP, &cmdobj->priv))
+		numibs = 0;
+
 	/* Add a *issue_ib struct for each IB */
 	cmd_sizebytes = sizeof(*cmd) + (sizeof(*issue_ib) * numibs);
 
@@ -880,16 +884,18 @@ int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags,
 			atomic_inc_return(&hfi->seqnum));
 
 	cmd->ctxt_id = drawobj->context->id;
-	cmd->flags = flags;
+	cmd->flags = CTXT_FLAG_NOTIFY;
 	cmd->ts = drawobj->timestamp;
 	cmd->numibs = numibs;
 
-	issue_ib = (struct hfi_issue_ib *)&cmd[1];
+	if (numibs) {
+		issue_ib = (struct hfi_issue_ib *)&cmd[1];
 
-	list_for_each_entry(ib, &cmdobj->cmdlist, node) {
-		issue_ib->addr = ib->gpuaddr;
-		issue_ib->size = ib->size;
-		issue_ib++;
+		list_for_each_entry(ib, &cmdobj->cmdlist, node) {
+			issue_ib->addr = ib->gpuaddr;
+			issue_ib->size = ib->size;
+			issue_ib++;
+		}
 	}
 
 	ret = a6xx_hfi_queue_write(adreno_dev, HFI_DSP_ID_0, (u32 *)cmd);

drivers/gpu/msm/adreno_a6xx_hwsched_hfi.h

@@ -181,7 +181,6 @@ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data);
 /**
  * a6xx_hwsched_submit_cmdobj - Dispatch IBs to dispatch queues
  * @adreno_dev: Pointer to adreno device structure
- * @flags: Flags associated with the submission
  * @cmdobj: The command object which needs to be submitted
  *
  * This function is used to register the context if needed and submit
@@ -189,7 +188,7 @@ int a6xx_hfi_send_cmd_async(struct adreno_device *adreno_dev, void *data);
 
  * Return: 0 on success and negative error on failure
  */
-int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev, u32 flags,
+int a6xx_hwsched_submit_cmdobj(struct adreno_device *adreno_dev,
 	struct kgsl_drawobj_cmd *cmdobj);
 
 /**

drivers/gpu/msm/adreno_hwsched.h

@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _ADRENO_HWSCHED_H_
+#define _ADRENO_HWSCHED_H_
+
+/**
+ * struct adreno_hwsched - Container for the hardware scheduler
+ */
+struct adreno_hwsched {
+	 /** @mutex: Mutex needed to run dispatcher function */
+	struct mutex mutex;
+	/** @flags: Container for the dispatcher internal flags */
+	unsigned long flags;
+	/** @inflight: Number of active submissions to the dispatch queues */
+	u32 inflight;
+	/** @jobs - Array of dispatch job lists for each priority level */
+	struct llist_head jobs[16];
+	/** @requeue - Array of lists for dispatch jobs that got requeued */
+	struct llist_head requeue[16];
+	/** @work: The work structure to execute dispatcher function */
+	struct kthread_work work;
+	/** @cmd_list: List of objects submitted to dispatch queues */
+	struct list_head cmd_list;
+};
+
+enum adreno_hwsched_flags {
+	ADRENO_HWSCHED_POWER = 0,
+};
+
+/**
+ * adreno_hwsched_queue_cmds() - Queue a new draw object in the context
+ * @dev_priv: Pointer to the device private struct
+ * @context: Pointer to the kgsl draw context
+ * @drawobj: Pointer to the array of drawobj's being submitted
+ * @count: Number of drawobj's being submitted
+ * @timestamp: Pointer to the requested timestamp
+ *
+ * Queue a command in the context - if there isn't any room in the queue, then
+ * block until there is
+ *
+ * Return: 0 on success and negative error on failure to queue
+ */
+int adreno_hwsched_queue_cmds(struct kgsl_device_private *dev_priv,
+	struct kgsl_context *context, struct kgsl_drawobj *drawobj[],
+	u32 count, u32 *timestamp);
+/**
+ * adreno_hwsched_queue_context() - schedule a drawctxt in the hw dispatcher
+ * @device: pointer to the KGSL device
+ * @drawctxt: pointer to the drawctxt to schedule
+ *
+ * Put a draw context on the dispatcher job listse and schedule the
+ * dispatcher. This is used to reschedule changes that might have been blocked
+ * for sync points or other concerns
+ */
+void adreno_hwsched_queue_context(struct kgsl_device *device,
+	struct adreno_context *drawctxt);
+/**
+ * adreno_hwsched_start() - activate the hwsched dispatcher
+ * @adreno_dev: pointer to the adreno device
+ *
+ * Enable dispatcher thread to execute
+ */
+void adreno_hwsched_start(struct adreno_device *adreno_dev);
+/**
+ * adreno_hwsched_dispatcher_init() - Initialize the hwsched dispatcher
+ * @adreno_dev: pointer to the adreno device
+ *
+ * Set up the dispatcher resources
+ */
+void adreno_hwsched_init(struct adreno_device *adreno_dev);
+
+/**
+ * adreno_hwsched_dispatcher_close() - close the hwsched dispatcher
+ * @adreno_dev: pointer to the adreno device structure
+ *
+ * Free the dispatcher resources
+ */
+void adreno_hwsched_dispatcher_close(struct adreno_device *adreno_dev);
+#endif