msm: kgsl: Add a requeue list for unprocessed dispatcher jobs

Requeue dispatcher jobs that could not be completed because inflight is
full and handle them before processing new requests from the main queue.

Change-Id: Ic0dedbad70f844372127e707c188c9f4a103624e
Signed-off-by: Jordan Crouse <jcrouse@codeaurora.org>

drivers/gpu/msm/adreno_dispatch.c

@@ -869,12 +869,18 @@ static bool adreno_gpu_stopped(struct adreno_device *adreno_dev)
 	return (adreno_gpu_fault(adreno_dev) || adreno_gpu_halt(adreno_dev));
 }
 
-static void _adreno_dispatcher_handle_jobs(struct adreno_device *adreno_dev,
-		struct llist_head *head)
+static void dispatcher_handle_jobs_list(struct adreno_device *adreno_dev,
+		int id, unsigned long *map, struct llist_node *list)
 {
-	struct llist_node *list = llist_del_all(head);
+	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;
 	struct adreno_dispatch_job *job, *next;
 
+	if (!list)
+		return;
+
+	/* Reverse the order so the oldest context is considered first */
+	list = llist_reverse_order(list);
+
 	llist_for_each_entry_safe(job, next, list, node) {
 		int ret;
 
@@ -885,19 +891,60 @@ static void _adreno_dispatcher_handle_jobs(struct adreno_device *adreno_dev,
 			continue;
 		}
 
+		/*
+		 * Due to the nature of the lockless queue the same context
+		 * might have multiple jobs on the list. We allow this so we
+		 * don't have to query the list on the producer side but on the
+		 * consumer side we only want each context to be considered
+		 * once. Use a bitmap to remember which contexts we've already
+		 * seen and quietly discard duplicate jobs
+		 */
+		if (test_and_set_bit(job->drawctxt->base.id, map)) {
+			kgsl_context_put(&job->drawctxt->base);
+			kmem_cache_free(jobs_cache, job);
+			continue;
+		}
+
 		ret = dispatcher_context_sendcmds(adreno_dev, job->drawctxt);
 
+		/*
+		 * If the context had nothing queued or the context has been
+		 * destroyed then drop the job
+		 */
 		if (!ret || ret == -ENOENT) {
 			kgsl_context_put(&job->drawctxt->base);
 			kmem_cache_free(jobs_cache, job);
 			continue;
 		}
 
-		/* The job didn't finish, put it back on the job queue */
-		llist_add(&job->node, head);
+		/*
+		 * If the ringbuffer is full then requeue the job to be
+		 * considered first next time. Otherwise the context
+		 * either successfully submmitted to the GPU or another error
+		 * happened and it should go back on the regular queue
+		 */
+		if (ret == -EBUSY)
+			llist_add(&job->node, &dispatcher->requeue[id]);
+		else
+			llist_add(&job->node, &dispatcher->jobs[id]);
 	}
 }
 
+static void dispatcher_handle_jobs(struct adreno_device *adreno_dev, int id)
+{
+	struct adreno_dispatcher *dispatcher = &adreno_dev->dispatcher;
+	unsigned long map[BITS_TO_LONGS(KGSL_MEMSTORE_MAX)];
+	struct llist_node *requeue, *jobs;
+
+	memset(map, 0, sizeof(map));
+
+	requeue = llist_del_all(&dispatcher->requeue[id]);
+	jobs = llist_del_all(&dispatcher->jobs[id]);
+
+	dispatcher_handle_jobs_list(adreno_dev, id, map, requeue);
+	dispatcher_handle_jobs_list(adreno_dev, id, map, jobs);
+}
+
 /**
  * _adreno_dispatcher_issuecmds() - Issue commmands from pending contexts
  * @adreno_dev: Pointer to the adreno device struct
@@ -915,8 +962,7 @@ static void _adreno_dispatcher_issuecmds(struct adreno_device *adreno_dev)
 		return;
 
 	for (i = 0; i < ARRAY_SIZE(dispatcher->jobs); i++)
-		_adreno_dispatcher_handle_jobs(adreno_dev,
-			&dispatcher->jobs[i]);
+		dispatcher_handle_jobs(adreno_dev, i);
 }
 
 static inline void _decrement_submit_now(struct kgsl_device *device)
@@ -2827,8 +2873,10 @@ int adreno_dispatcher_init(struct adreno_device *adreno_dev)
 
 	jobs_cache = KMEM_CACHE(adreno_dispatch_job, 0);
 
-	for (i = 0; i < ARRAY_SIZE(dispatcher->jobs); i++)
+	for (i = 0; i < ARRAY_SIZE(dispatcher->jobs); i++) {
 		init_llist_head(&dispatcher->jobs[i]);
+		init_llist_head(&dispatcher->requeue[i]);
+	}
 
 	set_bit(ADRENO_DISPATCHER_INIT, &dispatcher->priv);
 

drivers/gpu/msm/adreno_dispatch.h

@@ -76,6 +76,8 @@ struct adreno_dispatcher {
 	atomic_t fault;
 	/** @jobs - Array of dispatch job lists for each priority level */
 	struct llist_head jobs[16];
+	/** @requeue - Array of lists for dispatch jobs that got requeued */
+	struct llist_head requeue[16];
 	struct kthread_work work;
 	struct kobject kobj;
 	struct completion idle_gate;