diff --git a/drivers/gpu/msm/Makefile b/drivers/gpu/msm/Makefile index 4940cd5878ce..0d0e5513c52b 100644 --- a/drivers/gpu/msm/Makefile +++ b/drivers/gpu/msm/Makefile @@ -16,6 +16,7 @@ msm_kgsl-y = \ kgsl_pwrscale.o \ kgsl_sharedmem.o \ kgsl_snapshot.o \ + kgsl_timeline.o \ kgsl_trace.o \ kgsl_util.o diff --git a/drivers/gpu/msm/adreno_debugfs.c b/drivers/gpu/msm/adreno_debugfs.c index 0a1dae436700..509857ecd8f6 100644 --- a/drivers/gpu/msm/adreno_debugfs.c +++ b/drivers/gpu/msm/adreno_debugfs.c @@ -139,10 +139,20 @@ static void sync_event_print(struct seq_file *s, } case KGSL_CMD_SYNCPOINT_TYPE_FENCE: { int i; + struct event_fence_info *info = sync_event->priv; - for (i = 0; i < sync_event->info.num_fences; i++) + for (i = 0; info && i < info->num_fences; i++) seq_printf(s, "sync: %s", - sync_event->info.fences[i].name); + info->fences[i].name); + break; + } + case KGSL_CMD_SYNCPOINT_TYPE_TIMELINE: { + int j; + struct event_timeline_info *info = sync_event->priv; + + for (j = 0; info && info[j].timeline; j++) + seq_printf(s, "timeline: %d seqno: %d", + info[j].timeline, info[j].seqno); break; } default: diff --git a/drivers/gpu/msm/adreno_dispatch.c b/drivers/gpu/msm/adreno_dispatch.c index 4d1b1bf759b3..f6c3853e4ca2 100644 --- a/drivers/gpu/msm/adreno_dispatch.c +++ b/drivers/gpu/msm/adreno_dispatch.c @@ -8,6 +8,7 @@ #include "adreno.h" #include "adreno_trace.h" #include "kgsl_gmu_core.h" +#include "kgsl_timeline.h" #define DRAWQUEUE_NEXT(_i, _s) (((_i) + 1) % (_s)) @@ -286,6 +287,8 @@ static void _retire_timestamp(struct kgsl_drawobj *drawobj) KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp), drawobj->timestamp); + drawctxt->submitted_timestamp = drawobj->timestamp; + /* Retire pending GPU events for the object */ kgsl_process_event_group(device, &context->events); @@ -357,12 +360,14 @@ static inline void _pop_drawobj(struct adreno_context *drawctxt) drawctxt->queued--; } -static int _retire_markerobj(struct kgsl_drawobj_cmd *cmdobj, +static int dispatch_retire_markerobj(struct kgsl_drawobj *drawobj, struct adreno_context *drawctxt) { + struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj); + if (_marker_expired(cmdobj)) { _pop_drawobj(drawctxt); - _retire_timestamp(DRAWOBJ(cmdobj)); + _retire_timestamp(drawobj); return 0; } @@ -378,12 +383,14 @@ static int _retire_markerobj(struct kgsl_drawobj_cmd *cmdobj, return test_bit(CMDOBJ_SKIP, &cmdobj->priv) ? 1 : -EAGAIN; } -static int _retire_syncobj(struct kgsl_drawobj_sync *syncobj, +static int dispatch_retire_syncobj(struct kgsl_drawobj *drawobj, struct adreno_context *drawctxt) { + struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj); + if (!kgsl_drawobj_events_pending(syncobj)) { _pop_drawobj(drawctxt); - kgsl_drawobj_destroy(DRAWOBJ(syncobj)); + kgsl_drawobj_destroy(drawobj); return 0; } @@ -399,6 +406,22 @@ static int _retire_syncobj(struct kgsl_drawobj_sync *syncobj, return -EAGAIN; } +static int drawqueue_retire_timelineobj(struct kgsl_drawobj *drawobj, + struct adreno_context *drawctxt) +{ + struct kgsl_drawobj_timeline *timelineobj = TIMELINEOBJ(drawobj); + int i; + + for (i = 0; i < timelineobj->count; i++) + kgsl_timeline_signal(timelineobj->timelines[i].timeline, + timelineobj->timelines[i].seqno); + + _pop_drawobj(drawctxt); + _retire_timestamp(drawobj); + + return 0; +} + /* * Retires all expired marker and sync objs from the context * queue and returns one of the below @@ -412,35 +435,40 @@ static struct kgsl_drawobj *_process_drawqueue_get_next_drawobj( { struct kgsl_drawobj *drawobj; unsigned int i = drawctxt->drawqueue_head; - int ret = 0; if (drawctxt->drawqueue_head == drawctxt->drawqueue_tail) return NULL; for (i = drawctxt->drawqueue_head; i != drawctxt->drawqueue_tail; i = DRAWQUEUE_NEXT(i, ADRENO_CONTEXT_DRAWQUEUE_SIZE)) { + int ret = 0; drawobj = drawctxt->drawqueue[i]; - - if (drawobj == NULL) + if (!drawobj) return NULL; - if (drawobj->type == CMDOBJ_TYPE) + switch (drawobj->type) { + case CMDOBJ_TYPE: return drawobj; - else if (drawobj->type == MARKEROBJ_TYPE) { - ret = _retire_markerobj(CMDOBJ(drawobj), drawctxt); + case MARKEROBJ_TYPE: + ret = dispatch_retire_markerobj(drawobj, drawctxt); /* Special case where marker needs to be sent to GPU */ if (ret == 1) return drawobj; - } else if (drawobj->type == SYNCOBJ_TYPE) - ret = _retire_syncobj(SYNCOBJ(drawobj), drawctxt); - else - return ERR_PTR(-EINVAL); + break; + case SYNCOBJ_TYPE: + ret = dispatch_retire_syncobj(drawobj, drawctxt); + break; + case TIMELINEOBJ_TYPE: + ret = drawqueue_retire_timelineobj(drawobj, drawctxt); + break; + default: + ret = -EINVAL; + break; + } - if (ret == -EAGAIN) - return ERR_PTR(-EAGAIN); - - continue; + if (ret) + return ERR_PTR(ret); } return NULL; @@ -1184,11 +1212,27 @@ static void _queue_drawobj(struct adreno_context *drawctxt, trace_adreno_cmdbatch_queued(drawobj, drawctxt->queued); } -static int _queue_markerobj(struct adreno_device *adreno_dev, - struct adreno_context *drawctxt, struct kgsl_drawobj_cmd *markerobj, +static int drawctxt_queue_auxobj(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, struct kgsl_drawobj *drawobj, + u32 *timestamp, u32 user_ts) +{ + int ret; + + ret = get_timestamp(drawctxt, drawobj, timestamp, user_ts); + if (ret) + return ret; + + drawctxt->queued_timestamp = *timestamp; + _queue_drawobj(drawctxt, drawobj); + + return 0; +} + +static int drawctxt_queue_markerobj(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, struct kgsl_drawobj *drawobj, uint32_t *timestamp, unsigned int user_ts) { - struct kgsl_drawobj *drawobj = DRAWOBJ(markerobj); + struct kgsl_drawobj_cmd *markerobj = CMDOBJ(drawobj); int ret; ret = get_timestamp(drawctxt, drawobj, timestamp, user_ts); @@ -1221,11 +1265,11 @@ static int _queue_markerobj(struct adreno_device *adreno_dev, return 0; } -static int _queue_cmdobj(struct adreno_device *adreno_dev, - struct adreno_context *drawctxt, struct kgsl_drawobj_cmd *cmdobj, +static int drawctxt_queue_cmdobj(struct adreno_device *adreno_dev, + struct adreno_context *drawctxt, struct kgsl_drawobj *drawobj, uint32_t *timestamp, unsigned int user_ts) { - struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj); + struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj); unsigned int j; int ret; @@ -1259,11 +1303,9 @@ static int _queue_cmdobj(struct adreno_device *adreno_dev, return 0; } -static void _queue_syncobj(struct adreno_context *drawctxt, - struct kgsl_drawobj_sync *syncobj, uint32_t *timestamp) +static void drawctxt_queue_syncobj(struct adreno_context *drawctxt, + struct kgsl_drawobj *drawobj, uint32_t *timestamp) { - struct kgsl_drawobj *drawobj = DRAWOBJ(syncobj); - *timestamp = 0; drawobj->timestamp = 0; @@ -1337,29 +1379,34 @@ int adreno_dispatcher_queue_cmds(struct kgsl_device_private *dev_priv, switch (drawobj[i]->type) { case MARKEROBJ_TYPE: - ret = _queue_markerobj(adreno_dev, drawctxt, - CMDOBJ(drawobj[i]), - timestamp, user_ts); - if (ret == 1) { + ret = drawctxt_queue_markerobj(adreno_dev, drawctxt, + drawobj[i], timestamp, user_ts); + if (ret) spin_unlock(&drawctxt->lock); + + if (ret == 1) goto done; - } else if (ret) { - spin_unlock(&drawctxt->lock); + else if (ret) return ret; - } break; case CMDOBJ_TYPE: - ret = _queue_cmdobj(adreno_dev, drawctxt, - CMDOBJ(drawobj[i]), - timestamp, user_ts); + ret = drawctxt_queue_cmdobj(adreno_dev, drawctxt, + drawobj[i], timestamp, user_ts); if (ret) { spin_unlock(&drawctxt->lock); return ret; } break; case SYNCOBJ_TYPE: - _queue_syncobj(drawctxt, SYNCOBJ(drawobj[i]), - timestamp); + drawctxt_queue_syncobj(drawctxt, drawobj[i], timestamp); + break; + case TIMELINEOBJ_TYPE: + ret = drawctxt_queue_auxobj(adreno_dev, + drawctxt, drawobj[i], timestamp, user_ts); + if (ret) { + spin_unlock(&drawctxt->lock); + return ret; + } break; default: spin_unlock(&drawctxt->lock); diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index e529c6143e88..f5783d9db171 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -1858,6 +1858,138 @@ done: return result; } +long kgsl_ioctl_gpu_aux_command(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_gpu_aux_command *param = data; + struct kgsl_device *device = dev_priv->device; + struct kgsl_context *context; + struct kgsl_drawobj **drawobjs; + struct kgsl_drawobj_sync *tsobj; + void __user *cmdlist; + u32 queued, count; + int i, index = 0; + long ret; + + /* Aux commands don't make sense without commands */ + if (!param->numcmds) + return -EINVAL; + + if (!(param->flags & + (KGSL_GPU_AUX_COMMAND_TIMELINE))) + return -EINVAL; + + /* Make sure we don't overflow count */ + if (param->numcmds == UINT_MAX) + return -EINVAL; + + context = kgsl_context_get_owner(dev_priv, param->context_id); + if (!context) + return -EINVAL; + + /* + * We have one drawobj for the timestamp sync plus one for all of the + * commands + */ + count = param->numcmds + 1; + + if (param->flags & KGSL_GPU_AUX_COMMAND_SYNC) + count++; + + drawobjs = kvcalloc(count, sizeof(*drawobjs), + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + + if (!drawobjs) { + kgsl_context_put(context); + return -ENOMEM; + } + + if (param->flags & KGSL_GPU_AUX_COMMAND_SYNC) { + struct kgsl_drawobj_sync *syncobj = + kgsl_drawobj_sync_create(device, context); + + if (IS_ERR(syncobj)) { + ret = PTR_ERR(syncobj); + goto err; + } + + drawobjs[index++] = DRAWOBJ(syncobj); + + ret = kgsl_drawobj_sync_add_synclist(device, syncobj, + u64_to_user_ptr(param->synclist), + param->syncsize, param->numsyncs); + if (ret) + goto err; + } + + kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_QUEUED, &queued); + + /* + * Make an implicit sync object for the last queued timestamp on this + * context + */ + tsobj = kgsl_drawobj_create_timestamp_syncobj(device, + context, queued); + + if (IS_ERR(tsobj)) { + ret = PTR_ERR(tsobj); + goto err; + } + + drawobjs[index++] = DRAWOBJ(tsobj); + + cmdlist = u64_to_user_ptr(param->cmdlist); + + /* Create a draw object for each command */ + for (i = 0; i < param->numcmds; i++) { + struct kgsl_gpu_aux_command_generic generic; + + if (copy_struct_from_user(&generic, sizeof(generic), + cmdlist, param->cmdsize)) { + ret = -EFAULT; + goto err; + } + + if (generic.type == KGSL_GPU_AUX_COMMAND_TIMELINE) { + struct kgsl_drawobj_timeline *timelineobj; + + timelineobj = kgsl_drawobj_timeline_create(device, + context); + + if (IS_ERR(timelineobj)) { + ret = PTR_ERR(timelineobj); + goto err; + } + + ret = kgsl_drawobj_add_timeline(dev_priv, timelineobj, + cmdlist, param->cmdsize); + if (ret) + goto err; + + drawobjs[index++] = DRAWOBJ(timelineobj); + } else { + ret = -EINVAL; + goto err; + } + + cmdlist += param->cmdsize; + } + + ret = device->ftbl->queue_cmds(dev_priv, context, + drawobjs, index, ¶m->timestamp); + +err: + kgsl_context_put(context); + + if (ret && ret != -EPROTO) { + for (i = 0; i < count; i++) + kgsl_drawobj_destroy(drawobjs[i]); + } + + kvfree(drawobjs); + return ret; +} + long kgsl_ioctl_cmdstream_readtimestamp_ctxtid(struct kgsl_device_private *dev_priv, unsigned int cmd, void *data) @@ -4124,6 +4256,9 @@ int kgsl_device_platform_probe(struct kgsl_device *device) rwlock_init(&device->context_lock); spin_lock_init(&device->submit_lock); + /* Use XA_FLAGS_ALLOC1 to disallow 0 as an option */ + xa_init_flags(&device->timelines, XA_FLAGS_ALLOC1); + kgsl_device_debugfs_init(device); dma_set_coherent_mask(&pdev->dev, KGSL_DMA_BIT_MASK); diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h index 4fb03d06b0e6..7116fedc5716 100644 --- a/drivers/gpu/msm/kgsl.h +++ b/drivers/gpu/msm/kgsl.h @@ -425,6 +425,20 @@ long kgsl_ioctl_gpu_command(struct kgsl_device_private *dev_priv, unsigned int cmd, void *data); long kgsl_ioctl_gpuobj_set_info(struct kgsl_device_private *dev_priv, unsigned int cmd, void *data); +long kgsl_ioctl_gpu_aux_command(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); +long kgsl_ioctl_timeline_create(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); +long kgsl_ioctl_timeline_wait(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); +long kgsl_ioctl_timeline_query(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); +long kgsl_ioctl_timeline_fence_get(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); +long kgsl_ioctl_timeline_signal(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); +long kgsl_ioctl_timeline_destroy(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data); void kgsl_mem_entry_destroy(struct kref *kref); diff --git a/drivers/gpu/msm/kgsl_compat.c b/drivers/gpu/msm/kgsl_compat.c index 0a5cd74ddcb6..b69e921833dd 100644 --- a/drivers/gpu/msm/kgsl_compat.c +++ b/drivers/gpu/msm/kgsl_compat.c @@ -352,6 +352,20 @@ static const struct kgsl_ioctl kgsl_compat_ioctl_funcs[] = { NULL), KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_SPARSE_COMMAND, NULL), + KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_AUX_COMMAND, + kgsl_ioctl_gpu_aux_command), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_CREATE, + kgsl_ioctl_timeline_create), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_WAIT, + kgsl_ioctl_timeline_wait), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_FENCE_GET, + kgsl_ioctl_timeline_fence_get), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_QUERY, + kgsl_ioctl_timeline_query), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_SIGNAL, + kgsl_ioctl_timeline_signal), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_DESTROY, + kgsl_ioctl_timeline_destroy), }; long kgsl_compat_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) diff --git a/drivers/gpu/msm/kgsl_device.h b/drivers/gpu/msm/kgsl_device.h index 626a3d0b9726..6140fc2b8055 100644 --- a/drivers/gpu/msm/kgsl_device.h +++ b/drivers/gpu/msm/kgsl_device.h @@ -309,6 +309,8 @@ struct kgsl_device { u32 speed_bin; /** @gmu_fault: Set when a gmu or rgmu fault is encountered */ bool gmu_fault; + /** @timelines: xarray for the timelines */ + struct xarray timelines; }; #define KGSL_MMU_DEVICE(_mmu) \ diff --git a/drivers/gpu/msm/kgsl_drawobj.c b/drivers/gpu/msm/kgsl_drawobj.c index 19c601324fdd..3c1140362588 100644 --- a/drivers/gpu/msm/kgsl_drawobj.c +++ b/drivers/gpu/msm/kgsl_drawobj.c @@ -19,12 +19,14 @@ */ #include +#include #include "adreno_drawctxt.h" #include "kgsl_compat.h" #include "kgsl_device.h" #include "kgsl_drawobj.h" #include "kgsl_sync.h" +#include "kgsl_timeline.h" #include "kgsl_trace.h" /* @@ -33,38 +35,47 @@ */ static struct kmem_cache *memobjs_cache; -static void free_fence_names(struct kgsl_drawobj_sync *syncobj) +static void syncobj_destroy_object(struct kgsl_drawobj *drawobj) { - unsigned int i; + struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj); + int i; for (i = 0; i < syncobj->numsyncs; i++) { struct kgsl_drawobj_sync_event *event = &syncobj->synclist[i]; - if (event->type == KGSL_CMD_SYNCPOINT_TYPE_FENCE) - kfree(event->info.fences); + if (event->type == KGSL_CMD_SYNCPOINT_TYPE_FENCE) { + struct event_fence_info *priv = event->priv; + + if (priv) { + kfree(priv->fences); + kfree(priv); + } + } else if (event->type == KGSL_CMD_SYNCPOINT_TYPE_TIMELINE) { + kfree(event->priv); + } } + + kfree(syncobj->synclist); + kfree(syncobj); +} + +static void cmdobj_destroy_object(struct kgsl_drawobj *drawobj) +{ + kfree(CMDOBJ(drawobj)); +} + +static void timelineobj_destroy_object(struct kgsl_drawobj *drawobj) +{ + kfree(TIMELINEOBJ(drawobj)); } void kgsl_drawobj_destroy_object(struct kref *kref) { struct kgsl_drawobj *drawobj = container_of(kref, struct kgsl_drawobj, refcount); - struct kgsl_drawobj_sync *syncobj; kgsl_context_put(drawobj->context); - - switch (drawobj->type) { - case SYNCOBJ_TYPE: - syncobj = SYNCOBJ(drawobj); - free_fence_names(syncobj); - kfree(syncobj->synclist); - kfree(syncobj); - break; - case CMDOBJ_TYPE: - case MARKEROBJ_TYPE: - kfree(CMDOBJ(drawobj)); - break; - } + drawobj->destroy_object(drawobj); } void kgsl_dump_syncpoints(struct kgsl_device *device, @@ -95,13 +106,22 @@ void kgsl_dump_syncpoints(struct kgsl_device *device, } case KGSL_CMD_SYNCPOINT_TYPE_FENCE: { int j; - struct event_fence_info *info = &event->info; + struct event_fence_info *info = event->priv; - for (j = 0; j < info->num_fences; j++) + for (j = 0; info && j < info->num_fences; j++) dev_err(device->dev, "[%d] fence: %s\n", i, info->fences[j].name); break; } + case KGSL_CMD_SYNCPOINT_TYPE_TIMELINE: { + int j; + struct event_timeline_info *info = event->priv; + + for (j = 0; info && info[j].timeline; j++) + dev_err(device->dev, "[%d] timeline: %d seqno %lld\n", + i, info[j].timeline, info[j].seqno); + break; + } } } } @@ -152,13 +172,26 @@ static void syncobj_timer(struct timer_list *t) break; case KGSL_CMD_SYNCPOINT_TYPE_FENCE: { int j; - struct event_fence_info *info = &event->info; + struct event_fence_info *info = event->priv; - for (j = 0; j < info->num_fences; j++) + for (j = 0; info && j < info->num_fences; j++) dev_err(device->dev, " [%u] FENCE %s\n", i, info->fences[j].name); break; } + case KGSL_CMD_SYNCPOINT_TYPE_TIMELINE: { + int j; + struct event_timeline_info *info = event->priv; + + dev_err(device->dev, " [%u] FENCE %s\n", + i, dma_fence_is_signaled(event->fence) ? + "signaled" : "not signaled"); + + for (j = 0; info && info[j].timeline; j++) + dev_err(device->dev, " TIMELINE %d SEQNO %lld\n", + info[j].timeline, info[j].seqno); + break; + } } } @@ -220,7 +253,30 @@ static void drawobj_sync_func(struct kgsl_device *device, kgsl_drawobj_put(&event->syncobj->base); } -static void drawobj_destroy_sync(struct kgsl_drawobj *drawobj) +static void drawobj_sync_timeline_fence_work(struct irq_work *work) +{ + struct kgsl_drawobj_sync_event *event = container_of(work, + struct kgsl_drawobj_sync_event, work); + + dma_fence_put(event->fence); + kgsl_drawobj_put(&event->syncobj->base); +} + +static void drawobj_sync_timeline_fence_callback(struct dma_fence *f, + struct dma_fence_cb *cb) +{ + struct kgsl_drawobj_sync_event *event = container_of(cb, + struct kgsl_drawobj_sync_event, cb); + + /* + * Mark the event as synced and then fire off a worker to handle + * removing the fence + */ + if (drawobj_sync_expire(event->device, event)) + irq_work_queue(&event->work); +} + +static void syncobj_destroy(struct kgsl_drawobj *drawobj) { struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj); unsigned int i; @@ -258,6 +314,11 @@ static void drawobj_destroy_sync(struct kgsl_drawobj *drawobj) kgsl_sync_fence_async_cancel(event->handle); kgsl_drawobj_put(drawobj); break; + case KGSL_CMD_SYNCPOINT_TYPE_TIMELINE: + dma_fence_remove_callback(event->fence, &event->cb); + dma_fence_put(event->fence); + kgsl_drawobj_put(drawobj); + break; } } @@ -272,7 +333,20 @@ static void drawobj_destroy_sync(struct kgsl_drawobj *drawobj) } -static void drawobj_destroy_cmd(struct kgsl_drawobj *drawobj) +static void timelineobj_destroy(struct kgsl_drawobj *drawobj) +{ + struct kgsl_drawobj_timeline *timelineobj = TIMELINEOBJ(drawobj); + int i; + + for (i = 0; i < timelineobj->count; i++) + kgsl_timeline_put(timelineobj->timelines[i].timeline); + + kvfree(timelineobj->timelines); + timelineobj->timelines = NULL; + timelineobj->count = 0; +} + +static void cmdobj_destroy(struct kgsl_drawobj *drawobj) { struct kgsl_drawobj_cmd *cmdobj = CMDOBJ(drawobj); struct kgsl_memobj_node *mem, *tmpmem; @@ -307,15 +381,10 @@ static void drawobj_destroy_cmd(struct kgsl_drawobj *drawobj) */ void kgsl_drawobj_destroy(struct kgsl_drawobj *drawobj) { - if (!drawobj) + if (IS_ERR_OR_NULL(drawobj)) return; - if (drawobj->type & SYNCOBJ_TYPE) - drawobj_destroy_sync(drawobj); - else if (drawobj->type & (CMDOBJ_TYPE | MARKEROBJ_TYPE)) - drawobj_destroy_cmd(drawobj); - else - return; + drawobj->destroy(drawobj); kgsl_drawobj_put(drawobj); } @@ -323,11 +392,12 @@ void kgsl_drawobj_destroy(struct kgsl_drawobj *drawobj) static bool drawobj_sync_fence_func(void *priv) { struct kgsl_drawobj_sync_event *event = priv; + struct event_fence_info *info = event->priv; int i; - for (i = 0; i < event->info.num_fences; i++) + for (i = 0; info && i < info->num_fences; i++) trace_syncpoint_fence_expire(event->syncobj, - event->info.fences[i].name); + info->fences[i].name); /* * Only call kgsl_drawobj_put() if it's not marked for cancellation @@ -340,21 +410,107 @@ static bool drawobj_sync_fence_func(void *priv) return false; } -/* drawobj_add_sync_fence() - Add a new sync fence syncpoint - * @device: KGSL device - * @syncobj: KGSL sync obj to add the sync point to - * @priv: Private structure passed by the user - * - * Add a new fence sync syncpoint to the sync obj. - */ -static int drawobj_add_sync_fence(struct kgsl_device *device, - struct kgsl_drawobj_sync *syncobj, void *priv) +static struct event_timeline_info * +drawobj_get_sync_timeline_priv(void __user *uptr, u64 usize, u32 count) { - struct kgsl_cmd_syncpoint_fence *sync = priv; + int i; + struct event_timeline_info *priv; + + /* Make sure we don't accidently overflow count */ + if (count == UINT_MAX) + return NULL; + + priv = kcalloc(count + 1, sizeof(*priv), GFP_KERNEL); + if (!priv) + return NULL; + + for (i = 0; i < count; i++, uptr += usize) { + struct kgsl_timeline_val val; + + if (copy_struct_from_user(&val, sizeof(val), uptr, usize)) + continue; + + priv[i].timeline = val.timeline; + priv[i].seqno = val.seqno; + } + + priv[i].timeline = 0; + return priv; +} + +static int drawobj_add_sync_timeline(struct kgsl_device *device, + + struct kgsl_drawobj_sync *syncobj, void __user *uptr, + u64 usize) +{ + struct kgsl_drawobj *drawobj = DRAWOBJ(syncobj); + struct kgsl_cmd_syncpoint_timeline sync; + struct kgsl_drawobj_sync_event *event; + struct dma_fence *fence; + unsigned int id; + int ret; + + if (copy_struct_from_user(&sync, sizeof(sync), uptr, usize)) + return -EFAULT; + + fence = kgsl_timelines_to_fence_array(device, sync.timelines, + sync.count, sync.timelines_size, false); + if (IS_ERR(fence)) + return PTR_ERR(fence); + + kref_get(&drawobj->refcount); + + id = syncobj->numsyncs++; + + event = &syncobj->synclist[id]; + + event->id = id; + event->type = KGSL_CMD_SYNCPOINT_TYPE_TIMELINE; + event->syncobj = syncobj; + event->device = device; + event->context = NULL; + event->fence = fence; + init_irq_work(&event->work, drawobj_sync_timeline_fence_work); + + INIT_LIST_HEAD(&event->cb.node); + + event->priv = + drawobj_get_sync_timeline_priv(u64_to_user_ptr(sync.timelines), + sync.timelines_size, sync.count); + + ret = dma_fence_add_callback(event->fence, + &event->cb, drawobj_sync_timeline_fence_callback); + + set_bit(event->id, &syncobj->pending); + + if (ret) { + clear_bit(event->id, &syncobj->pending); + + if (dma_fence_is_signaled(event->fence)) { + trace_syncpoint_fence_expire(syncobj, "signaled"); + dma_fence_put(event->fence); + ret = 0; + } + + kgsl_drawobj_put(drawobj); + } + + return ret; +} + +static int drawobj_add_sync_fence(struct kgsl_device *device, + struct kgsl_drawobj_sync *syncobj, void __user *data, + u64 datasize) +{ + struct kgsl_cmd_syncpoint_fence sync; struct kgsl_drawobj *drawobj = DRAWOBJ(syncobj); struct kgsl_drawobj_sync_event *event; + struct event_fence_info *priv; unsigned int id, i; + if (copy_struct_from_user(&sync, sizeof(sync), data, datasize)) + return -EFAULT; + kref_get(&drawobj->refcount); id = syncobj->numsyncs++; @@ -367,11 +523,14 @@ static int drawobj_add_sync_fence(struct kgsl_device *device, event->device = device; event->context = NULL; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + set_bit(event->id, &syncobj->pending); - event->handle = kgsl_sync_fence_async_wait(sync->fd, - drawobj_sync_fence_func, event, - &event->info); + event->handle = kgsl_sync_fence_async_wait(sync.fd, + drawobj_sync_fence_func, event, priv); + + event->priv = priv; if (IS_ERR_OR_NULL(event->handle)) { int ret = PTR_ERR(event->handle); @@ -391,8 +550,8 @@ static int drawobj_add_sync_fence(struct kgsl_device *device, return ret; } - for (i = 0; i < event->info.num_fences; i++) - trace_syncpoint_fence(syncobj, event->info.fences[i].name); + for (i = 0; priv && i < priv->num_fences; i++) + trace_syncpoint_fence(syncobj, priv->fences[i].name); return 0; } @@ -405,12 +564,13 @@ static int drawobj_add_sync_fence(struct kgsl_device *device, * Add a new sync point timestamp event to the sync obj. */ static int drawobj_add_sync_timestamp(struct kgsl_device *device, - struct kgsl_drawobj_sync *syncobj, void *priv) + struct kgsl_drawobj_sync *syncobj, + struct kgsl_cmd_syncpoint_timestamp *timestamp) + { - struct kgsl_cmd_syncpoint_timestamp *sync = priv; struct kgsl_drawobj *drawobj = DRAWOBJ(syncobj); struct kgsl_context *context = kgsl_context_get(device, - sync->context_id); + timestamp->context_id); struct kgsl_drawobj_sync_event *event; int ret = -EINVAL; unsigned int id; @@ -431,10 +591,10 @@ static int drawobj_add_sync_timestamp(struct kgsl_device *device, kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_QUEUED, &queued); - if (timestamp_cmp(sync->timestamp, queued) > 0) { + if (timestamp_cmp(timestamp->timestamp, queued) > 0) { dev_err(device->dev, "Cannot create syncpoint for future timestamp %d (current %d)\n", - sync->timestamp, queued); + timestamp->timestamp, queued); goto done; } } @@ -449,19 +609,20 @@ static int drawobj_add_sync_timestamp(struct kgsl_device *device, event->type = KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP; event->syncobj = syncobj; event->context = context; - event->timestamp = sync->timestamp; + event->timestamp = timestamp->timestamp; event->device = device; set_bit(event->id, &syncobj->pending); - ret = kgsl_add_event(device, &context->events, sync->timestamp, + ret = kgsl_add_event(device, &context->events, timestamp->timestamp, drawobj_sync_func, event); if (ret) { clear_bit(event->id, &syncobj->pending); kgsl_drawobj_put(drawobj); } else { - trace_syncpoint_timestamp(syncobj, context, sync->timestamp); + trace_syncpoint_timestamp(syncobj, context, + timestamp->timestamp); } done: @@ -471,6 +632,19 @@ done: return ret; } +static int drawobj_add_sync_timestamp_from_user(struct kgsl_device *device, + struct kgsl_drawobj_sync *syncobj, void __user *data, + u64 datasize) +{ + struct kgsl_cmd_syncpoint_timestamp timestamp; + + if (copy_struct_from_user(×tamp, sizeof(timestamp), + data, datasize)) + return -EFAULT; + + return drawobj_add_sync_timestamp(device, syncobj, ×tamp); +} + /** * kgsl_drawobj_sync_add_sync() - Add a sync point to a command * batch @@ -485,44 +659,22 @@ int kgsl_drawobj_sync_add_sync(struct kgsl_device *device, struct kgsl_drawobj_sync *syncobj, struct kgsl_cmd_syncpoint *sync) { - void *priv; - int ret, psize; struct kgsl_drawobj *drawobj = DRAWOBJ(syncobj); - int (*func)(struct kgsl_device *device, - struct kgsl_drawobj_sync *syncobj, - void *priv); - switch (sync->type) { - case KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP: - psize = sizeof(struct kgsl_cmd_syncpoint_timestamp); - func = drawobj_add_sync_timestamp; - break; - case KGSL_CMD_SYNCPOINT_TYPE_FENCE: - psize = sizeof(struct kgsl_cmd_syncpoint_fence); - func = drawobj_add_sync_fence; - break; - default: - dev_err(device->dev, - "bad syncpoint type ctxt %d type 0x%x size %zu\n", - drawobj->context->id, sync->type, sync->size); - return -EINVAL; - } + if (sync->type == KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP) + return drawobj_add_sync_timestamp_from_user(device, + syncobj, sync->priv, sync->size); + else if (sync->type == KGSL_CMD_SYNCPOINT_TYPE_FENCE) + return drawobj_add_sync_fence(device, + syncobj, sync->priv, sync->size); + else if (sync->type == KGSL_CMD_SYNCPOINT_TYPE_TIMELINE) + return drawobj_add_sync_timeline(device, + syncobj, sync->priv, sync->size); - if (sync->size != psize) { - dev_err(device->dev, - "bad syncpoint size ctxt %d type 0x%x size %zu\n", - drawobj->context->id, sync->type, sync->size); - return -EINVAL; - } + dev_err(device->dev, "bad syncpoint type %d for ctxt %d\n", + sync->type, drawobj->context->id); - priv = memdup_user(sync->priv, sync->size); - if (IS_ERR(priv)) - return PTR_ERR(priv); - - ret = func(device, syncobj, priv); - kfree(priv); - - return ret; + return -EINVAL; } static void add_profiling_buffer(struct kgsl_device *device, @@ -647,26 +799,16 @@ int kgsl_drawobj_cmd_add_ibdesc(struct kgsl_device *device, return 0; } -static void *_drawobj_create(struct kgsl_device *device, - struct kgsl_context *context, unsigned int size, - unsigned int type) +static int drawobj_init(struct kgsl_device *device, + struct kgsl_context *context, struct kgsl_drawobj *drawobj, + int type) { - void *obj = kzalloc(size, GFP_KERNEL); - struct kgsl_drawobj *drawobj; - - if (obj == NULL) - return ERR_PTR(-ENOMEM); - /* * Increase the reference count on the context so it doesn't disappear * during the lifetime of this object */ - if (!_kgsl_context_get(context)) { - kfree(obj); - return ERR_PTR(-ENOENT); - } - - drawobj = obj; + if (!_kgsl_context_get(context)) + return -ENOENT; kref_init(&drawobj->refcount); @@ -674,7 +816,111 @@ static void *_drawobj_create(struct kgsl_device *device, drawobj->context = context; drawobj->type = type; - return obj; + return 0; +} + +static int get_aux_command(void __user *ptr, u64 generic_size, + int type, void *auxcmd, size_t auxcmd_size) +{ + struct kgsl_gpu_aux_command_generic generic; + u64 size; + + if (copy_struct_from_user(&generic, sizeof(generic), ptr, generic_size)) + return -EFAULT; + + if (generic.type != type) + return -EINVAL; + + size = min_t(u64, auxcmd_size, generic.size); + if (copy_from_user(auxcmd, u64_to_user_ptr(generic.priv), size)) + return -EFAULT; + + return 0; +} + +struct kgsl_drawobj_timeline * +kgsl_drawobj_timeline_create(struct kgsl_device *device, + struct kgsl_context *context) +{ + int ret; + struct kgsl_drawobj_timeline *timelineobj = + kzalloc(sizeof(*timelineobj), GFP_KERNEL); + + if (!timelineobj) + return ERR_PTR(-ENOMEM); + + ret = drawobj_init(device, context, &timelineobj->base, + TIMELINEOBJ_TYPE); + if (ret) { + kfree(timelineobj); + return ERR_PTR(ret); + } + + timelineobj->base.destroy = timelineobj_destroy; + timelineobj->base.destroy_object = timelineobj_destroy_object; + + return timelineobj; +} + +int kgsl_drawobj_add_timeline(struct kgsl_device_private *dev_priv, + struct kgsl_drawobj_timeline *timelineobj, + void __user *src, u64 cmdsize) +{ + struct kgsl_gpu_aux_command_timeline cmd; + int i, ret; + + memset(&cmd, 0, sizeof(cmd)); + + ret = get_aux_command(src, cmdsize, + KGSL_GPU_AUX_COMMAND_TIMELINE, &cmd, sizeof(cmd)); + if (ret) + return ret; + + if (!cmd.count) + return -EINVAL; + + timelineobj->timelines = kvcalloc(cmd.count, + sizeof(*timelineobj->timelines), + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + if (!timelineobj->timelines) + return -ENOMEM; + + src = u64_to_user_ptr(cmd.timelines); + + for (i = 0; i < cmd.count; i++) { + struct kgsl_timeline_val val; + + if (copy_struct_from_user(&val, sizeof(val), src, + cmd.timelines_size)) { + ret = -EFAULT; + goto err; + } + + if (val.padding) { + ret = -EINVAL; + goto err; + } + + timelineobj->timelines[i].timeline = + kgsl_timeline_by_id(dev_priv->device, + val.timeline); + + if (!timelineobj->timelines[i].timeline) { + ret = -ENODEV; + goto err; + } + + timelineobj->timelines[i].seqno = val.seqno; + } + + timelineobj->count = cmd.count; + return 0; +err: + for (i = 0; i < cmd.count; i++) + kgsl_timeline_put(timelineobj->timelines[i].timeline); + + kvfree(timelineobj->timelines); + return ret; } /** @@ -688,12 +934,23 @@ static void *_drawobj_create(struct kgsl_device *device, struct kgsl_drawobj_sync *kgsl_drawobj_sync_create(struct kgsl_device *device, struct kgsl_context *context) { - struct kgsl_drawobj_sync *syncobj = _drawobj_create(device, - context, sizeof(*syncobj), SYNCOBJ_TYPE); + struct kgsl_drawobj_sync *syncobj = + kzalloc(sizeof(*syncobj), GFP_KERNEL); + int ret; - /* Add a timer to help debug sync deadlocks */ - if (!IS_ERR(syncobj)) - timer_setup(&syncobj->timer, syncobj_timer, 0); + if (!syncobj) + return ERR_PTR(-ENOMEM); + + ret = drawobj_init(device, context, &syncobj->base, SYNCOBJ_TYPE); + if (ret) { + kfree(syncobj); + return ERR_PTR(ret); + } + + syncobj->base.destroy = syncobj_destroy; + syncobj->base.destroy_object = syncobj_destroy_object; + + timer_setup(&syncobj->timer, syncobj_timer, 0); return syncobj; } @@ -712,24 +969,34 @@ struct kgsl_drawobj_cmd *kgsl_drawobj_cmd_create(struct kgsl_device *device, struct kgsl_context *context, unsigned int flags, unsigned int type) { - struct kgsl_drawobj_cmd *cmdobj = _drawobj_create(device, - context, sizeof(*cmdobj), + struct kgsl_drawobj_cmd *cmdobj = kzalloc(sizeof(*cmdobj), GFP_KERNEL); + int ret; + + if (!cmdobj) + return ERR_PTR(-ENOMEM); + + ret = drawobj_init(device, context, &cmdobj->base, (type & (CMDOBJ_TYPE | MARKEROBJ_TYPE))); - - if (!IS_ERR(cmdobj)) { - /* sanitize our flags for drawobj's */ - cmdobj->base.flags = flags & (KGSL_DRAWOBJ_CTX_SWITCH - | KGSL_DRAWOBJ_MARKER - | KGSL_DRAWOBJ_END_OF_FRAME - | KGSL_DRAWOBJ_PWR_CONSTRAINT - | KGSL_DRAWOBJ_MEMLIST - | KGSL_DRAWOBJ_PROFILING - | KGSL_DRAWOBJ_PROFILING_KTIME); - - INIT_LIST_HEAD(&cmdobj->cmdlist); - INIT_LIST_HEAD(&cmdobj->memlist); + if (ret) { + kfree(cmdobj); + return ERR_PTR(ret); } + cmdobj->base.destroy = cmdobj_destroy; + cmdobj->base.destroy_object = cmdobj_destroy_object; + + /* sanitize our flags for drawobjs */ + cmdobj->base.flags = flags & (KGSL_DRAWOBJ_CTX_SWITCH + | KGSL_DRAWOBJ_MARKER + | KGSL_DRAWOBJ_END_OF_FRAME + | KGSL_DRAWOBJ_PWR_CONSTRAINT + | KGSL_DRAWOBJ_MEMLIST + | KGSL_DRAWOBJ_PROFILING + | KGSL_DRAWOBJ_PROFILING_KTIME); + + INIT_LIST_HEAD(&cmdobj->cmdlist); + INIT_LIST_HEAD(&cmdobj->memlist); + return cmdobj; } @@ -1008,6 +1275,36 @@ int kgsl_drawobj_cmd_add_memlist(struct kgsl_device *device, return 0; } +struct kgsl_drawobj_sync * +kgsl_drawobj_create_timestamp_syncobj(struct kgsl_device *device, + struct kgsl_context *context, unsigned int timestamp) +{ + struct kgsl_drawobj_sync *syncobj; + struct kgsl_cmd_syncpoint_timestamp priv; + int ret; + + syncobj = kgsl_drawobj_sync_create(device, context); + if (IS_ERR(syncobj)) + return syncobj; + + syncobj->synclist = kzalloc(sizeof(*syncobj->synclist), GFP_KERNEL); + if (!syncobj->synclist) { + kgsl_drawobj_destroy(DRAWOBJ(syncobj)); + return ERR_PTR(-ENOMEM); + } + + priv.timestamp = timestamp; + priv.context_id = context->id; + + ret = drawobj_add_sync_timestamp(device, syncobj, &priv); + if (ret) { + kgsl_drawobj_destroy(DRAWOBJ(syncobj)); + return ERR_PTR(ret); + } + + return syncobj; +} + int kgsl_drawobj_sync_add_synclist(struct kgsl_device *device, struct kgsl_drawobj_sync *syncobj, void __user *ptr, unsigned int size, unsigned int count) diff --git a/drivers/gpu/msm/kgsl_drawobj.h b/drivers/gpu/msm/kgsl_drawobj.h index 3c14cda8057e..3719dc171e1d 100644 --- a/drivers/gpu/msm/kgsl_drawobj.h +++ b/drivers/gpu/msm/kgsl_drawobj.h @@ -6,6 +6,8 @@ #ifndef __KGSL_DRAWOBJ_H #define __KGSL_DRAWOBJ_H +#include +#include #include #define DRAWOBJ(obj) (&obj->base) @@ -17,6 +19,7 @@ #define CMDOBJ_TYPE BIT(0) #define MARKEROBJ_TYPE BIT(1) #define SYNCOBJ_TYPE BIT(2) +#define TIMELINEOBJ_TYPE BIT(4) /** * struct kgsl_drawobj - KGSL drawobj descriptor @@ -34,6 +37,10 @@ struct kgsl_drawobj { uint32_t timestamp; unsigned long flags; struct kref refcount; + /** @destroy: Callbak function to take down the object */ + void (*destroy)(struct kgsl_drawobj *drawobj); + /** @destroy_object: Callback function to free the object memory */ + void (*destroy_object)(struct kgsl_drawobj *drawobj); }; /** @@ -97,6 +104,28 @@ struct kgsl_drawobj_sync { unsigned long timeout_jiffies; }; +/** + * struct kgsl_drawobj_timeline - KGSL timeline signal operation + */ +struct kgsl_drawobj_timeline { + /** @base: &struct kgsl_drawobj container */ + struct kgsl_drawobj base; + struct { + /** @timeline: Pointer to a &struct kgsl_timeline */ + struct kgsl_timeline *timeline; + /** @seqno: Sequence number to signal */ + u64 seqno; + } *timelines; + /** @count: Number of items in timelines */ + int count; +}; + +static inline struct kgsl_drawobj_timeline * +TIMELINEOBJ(struct kgsl_drawobj *obj) +{ + return container_of(obj, struct kgsl_drawobj_timeline, base); +} + #define KGSL_FENCE_NAME_LEN 74 struct fence_info { @@ -108,6 +137,11 @@ struct event_fence_info { int num_fences; }; +struct event_timeline_info { + u64 seqno; + u32 timeline; +}; + /** * struct kgsl_drawobj_sync_event * @id: identifer (positiion within the pending bitmap) @@ -118,7 +152,6 @@ struct event_fence_info { * @timestamp: Pending timestamp for the event * @handle: Pointer to a sync fence handle * @device: Pointer to the KGSL device - * @info: structure to hold info about the fence */ struct kgsl_drawobj_sync_event { unsigned int id; @@ -128,7 +161,17 @@ struct kgsl_drawobj_sync_event { unsigned int timestamp; struct kgsl_sync_fence_cb *handle; struct kgsl_device *device; - struct event_fence_info info; + /** @priv: Type specific private information */ + void *priv; + /** + * @fence: Pointer to a dma fence for KGSL_CMD_SYNCPOINT_TYPE_TIMELINE + * events + */ + struct dma_fence *fence; + /** @cb: Callback struct for KGSL_CMD_SYNCPOINT_TYPE_TIMELINE */ + struct dma_fence_cb cb; + /** @work : irq worker for KGSL_CMD_SYNCPOINT_TYPE_TIMELINE */ + struct irq_work work; }; #define KGSL_DRAWOBJ_FLAGS \ @@ -217,4 +260,44 @@ static inline void kgsl_drawobj_put(struct kgsl_drawobj *drawobj) kref_put(&drawobj->refcount, kgsl_drawobj_destroy_object); } +/** + * kgsl_drawobj_create_timestamp_syncobj - Create a syncobj for a timestamp + * @device: A GPU device handle + * @context: Draw context for the syncobj + * @timestamp: Timestamp to sync on + * + * Create a sync object for @timestamp on @context. + * Return: A pointer to the sync object + */ +struct kgsl_drawobj_sync * +kgsl_drawobj_create_timestamp_syncobj(struct kgsl_device *device, + struct kgsl_context *context, unsigned int timestamp); + + +/** + * kgsl_drawobj_timeline_create - Create a timeline draw object + * @device: A GPU device handle + * @context: Draw context for the drawobj + * + * Create a timeline draw object on @context. + * Return: A pointer to the draw object + */ +struct kgsl_drawobj_timeline * +kgsl_drawobj_timeline_create(struct kgsl_device *device, + struct kgsl_context *context); + +/** + * kgsl_drwobj_add_timeline - Add a timeline to a timeline drawobj + * @dev_priv: Pointer to the process private data + * @timelineobj: Pointer to a timeline drawobject + * @src: Ponter to the &struct kgsl_timeline_val from userspace + * @cmdsize: size of the object in @src + * + * Add a timeline to an draw object. + * Return: 0 on success or negative on failure + */ +int kgsl_drawobj_add_timeline(struct kgsl_device_private *dev_priv, + struct kgsl_drawobj_timeline *timelineobj, + void __user *src, u64 cmdsize); + #endif /* __KGSL_DRAWOBJ_H */ diff --git a/drivers/gpu/msm/kgsl_ioctl.c b/drivers/gpu/msm/kgsl_ioctl.c index 3126e3baf7f5..746acb4a8161 100644 --- a/drivers/gpu/msm/kgsl_ioctl.c +++ b/drivers/gpu/msm/kgsl_ioctl.c @@ -84,6 +84,20 @@ static const struct kgsl_ioctl kgsl_ioctl_funcs[] = { NULL), KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_SPARSE_COMMAND, NULL), + KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_AUX_COMMAND, + kgsl_ioctl_gpu_aux_command), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_CREATE, + kgsl_ioctl_timeline_create), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_WAIT, + kgsl_ioctl_timeline_wait), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_FENCE_GET, + kgsl_ioctl_timeline_fence_get), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_QUERY, + kgsl_ioctl_timeline_query), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_SIGNAL, + kgsl_ioctl_timeline_signal), + KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMELINE_DESTROY, + kgsl_ioctl_timeline_destroy), }; long kgsl_ioctl_copy_in(unsigned int kernel_cmd, unsigned int user_cmd, diff --git a/drivers/gpu/msm/kgsl_timeline.c b/drivers/gpu/msm/kgsl_timeline.c new file mode 100644 index 000000000000..8a24e74ce08d --- /dev/null +++ b/drivers/gpu/msm/kgsl_timeline.c @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include "kgsl_device.h" +#include "kgsl_sharedmem.h" +#include "kgsl_timeline.h" + +static DEFINE_SPINLOCK(fence_lock); + +struct kgsl_timeline_fence { + struct dma_fence base; + struct kgsl_timeline *timeline; + struct list_head node; +}; + +struct dma_fence *kgsl_timelines_to_fence_array(struct kgsl_device *device, + u64 timelines, u64 count, u64 usize, bool any) +{ + void __user *uptr = u64_to_user_ptr(timelines); + struct dma_fence_array *array; + struct dma_fence **fences; + int i, ret = 0; + + if (!count) + return ERR_PTR(-EINVAL); + + fences = kcalloc(count, sizeof(*fences), + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); + + if (!fences) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < count; i++) { + struct kgsl_timeline_val val; + struct kgsl_timeline *timeline; + + if (copy_struct_from_user(&val, sizeof(val), uptr, usize)) { + ret = -EFAULT; + goto err; + } + + if (val.padding) { + ret = -EINVAL; + goto err; + } + + timeline = kgsl_timeline_by_id(device, val.timeline); + if (!timeline) { + ret = -ENOENT; + goto err; + } + + fences[i] = kgsl_timeline_fence_alloc(timeline, val.seqno); + kgsl_timeline_put(timeline); + + if (IS_ERR(fences[i])) { + ret = PTR_ERR(fences[i]); + goto err; + } + + uptr += usize; + } + + /* No need for a fence array for only one fence */ + if (count == 1) { + struct dma_fence *fence = fences[0]; + + kfree(fences); + return fence; + } + + array = dma_fence_array_create(count, fences, + dma_fence_context_alloc(1), 0, any); + + if (array) + return &array->base; + + ret = -ENOMEM; +err: + for (i = 0; i < count; i++) { + if (!IS_ERR_OR_NULL(fences[i])) + dma_fence_put(fences[i]); + } + + kfree(fences); + return ERR_PTR(ret); +} + +void kgsl_timeline_destroy(struct kref *kref) +{ + struct kgsl_timeline *timeline = container_of(kref, + struct kgsl_timeline, ref); + + WARN_ON(!list_empty(&timeline->fences)); + + kfree(timeline); +} + +struct kgsl_timeline *kgsl_timeline_get(struct kgsl_timeline *timeline) +{ + if (timeline) { + if (!kref_get_unless_zero(&timeline->ref)) + return NULL; + } + + return timeline; +} + +static struct kgsl_timeline *kgsl_timeline_alloc(struct kgsl_device *device, + u64 initial) +{ + struct kgsl_timeline *timeline; + u32 id; + int ret; + + timeline = kzalloc(sizeof(*timeline), GFP_KERNEL); + if (!timeline) + return ERR_PTR(-ENOMEM); + + ret = xa_alloc(&device->timelines, &id, + timeline, xa_limit_32b, GFP_KERNEL); + + if (ret) { + kfree(timeline); + return ERR_PTR(ret); + } + + timeline->context = dma_fence_context_alloc(1); + timeline->id = id; + INIT_LIST_HEAD(&timeline->fences); + timeline->value = initial; + + snprintf((char *) timeline->name, sizeof(timeline->name), + "kgsl-sw-timeline-%d", id); + + spin_lock_init(&timeline->lock); + + kref_init(&timeline->ref); + + return timeline; +} + +static struct kgsl_timeline_fence *to_timeline_fence(struct dma_fence *fence) +{ + return container_of(fence, struct kgsl_timeline_fence, base); +} + +static void timeline_fence_release(struct dma_fence *fence) +{ + struct kgsl_timeline_fence *f = to_timeline_fence(fence); + struct kgsl_timeline *timeline = f->timeline; + struct kgsl_timeline_fence *cur, *temp; + unsigned long flags; + + spin_lock_irqsave(&fence_lock, flags); + + /* If the fence is still on the active list, remove it */ + list_for_each_entry_safe(cur, temp, &timeline->fences, node) { + if (f != cur) + continue; + + list_del_init(&f->node); + break; + } + spin_unlock_irqrestore(&fence_lock, flags); + + kgsl_timeline_put(f->timeline); + dma_fence_free(fence); +} + +static bool timeline_fence_signaled(struct dma_fence *fence) +{ + struct kgsl_timeline_fence *f = to_timeline_fence(fence); + + return !__dma_fence_is_later(fence->seqno, f->timeline->value, + fence->ops); +} + +static const char *timeline_get_driver_name(struct dma_fence *fence) +{ + return "kgsl-sw-timeline"; +} + +static const char *timeline_get_timeline_name(struct dma_fence *fence) +{ + struct kgsl_timeline_fence *f = to_timeline_fence(fence); + + return f->timeline->name; +} + +static void timeline_get_value_str(struct dma_fence *fence, + char *str, int size) +{ + struct kgsl_timeline_fence *f = to_timeline_fence(fence); + + snprintf(str, size, "%lld", f->timeline->value); +} + +static const struct dma_fence_ops timeline_fence_ops = { + .get_driver_name = timeline_get_driver_name, + .get_timeline_name = timeline_get_timeline_name, + .signaled = timeline_fence_signaled, + .release = timeline_fence_release, + .timeline_value_str = timeline_get_value_str, + .use_64bit_seqno = true, +}; + +static void kgsl_timeline_add_fence(struct kgsl_timeline *timeline, + struct kgsl_timeline_fence *fence) +{ + struct kgsl_timeline_fence *entry; + unsigned long flags; + + spin_lock_irqsave(&fence_lock, flags); + list_for_each_entry(entry, &timeline->fences, node) { + if (fence->base.seqno < entry->base.seqno) { + list_add_tail(&fence->node, &entry->node); + spin_unlock_irqrestore(&fence_lock, flags); + return; + } + } + + list_add_tail(&fence->node, &timeline->fences); + spin_unlock_irqrestore(&fence_lock, flags); +} + +void kgsl_timeline_signal(struct kgsl_timeline *timeline, u64 seqno) +{ + struct kgsl_timeline_fence *fence, *tmp; + struct list_head temp; + + INIT_LIST_HEAD(&temp); + + spin_lock_irq(&timeline->lock); + + if (seqno < timeline->value) + goto unlock; + + timeline->value = seqno; + + /* Copy the list out so we can walk it without holding the lock */ + spin_lock(&fence_lock); + list_replace_init(&timeline->fences, &temp); + spin_unlock(&fence_lock); + + list_for_each_entry_safe(fence, tmp, &temp, node) { + if (timeline_fence_signaled(&fence->base)) { + list_del_init(&fence->node); + dma_fence_signal_locked(&fence->base); + } + } + + /* Put the active fences back in the timeline list */ + list_for_each_entry_safe(fence, tmp, &temp, node) { + list_del_init(&fence->node); + kgsl_timeline_add_fence(timeline, fence); + } + +unlock: + spin_unlock_irq(&timeline->lock); +} + +struct dma_fence *kgsl_timeline_fence_alloc(struct kgsl_timeline *timeline, + u64 seqno) +{ + struct kgsl_timeline_fence *fence; + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return ERR_PTR(-ENOMEM); + + fence->timeline = kgsl_timeline_get(timeline); + + dma_fence_init(&fence->base, &timeline_fence_ops, + &timeline->lock, timeline->context, seqno); + + INIT_LIST_HEAD(&fence->node); + + if (!dma_fence_is_signaled(&fence->base)) + kgsl_timeline_add_fence(timeline, fence); + + return &fence->base; +} + +long kgsl_ioctl_timeline_create(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_timeline_create *param = data; + struct kgsl_timeline *timeline; + + timeline = kgsl_timeline_alloc(device, param->seqno); + if (IS_ERR(timeline)) + return PTR_ERR(timeline); + + param->id = timeline->id; + return 0; +} + +struct kgsl_timeline *kgsl_timeline_by_id(struct kgsl_device *device, + u32 id) +{ + struct kgsl_timeline *timeline = xa_load(&device->timelines, id); + + return kgsl_timeline_get(timeline); +} + +long kgsl_ioctl_timeline_wait(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_timeline_wait *param = data; + struct dma_fence *fence; + unsigned long timeout; + signed long ret; + + if (param->flags != KGSL_TIMELINE_WAIT_ANY && + param->flags != KGSL_TIMELINE_WAIT_ALL) + return -EINVAL; + + if (param->padding) + return -EINVAL; + + fence = kgsl_timelines_to_fence_array(device, param->timelines, + param->count, param->timelines_size, + (param->flags == KGSL_TIMELINE_WAIT_ANY)); + + if (IS_ERR(fence)) + return PTR_ERR(fence); + + if (param->tv_sec >= KTIME_SEC_MAX) + timeout = MAX_SCHEDULE_TIMEOUT; + else { + ktime_t time = ktime_set(param->tv_sec, param->tv_nsec); + + timeout = msecs_to_jiffies(ktime_to_ms(time)); + } + + /* secs.nsecs to jiffies */ + if (!timeout) + ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; + else { + ret = dma_fence_wait_timeout(fence, true, timeout); + + if (!ret) + ret = -ETIMEDOUT; + else if (ret > 0) + ret = 0; + } + + dma_fence_put(fence); + + return ret; +} + +long kgsl_ioctl_timeline_query(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_timeline_val *param = data; + struct kgsl_timeline *timeline; + + if (param->padding) + return -EINVAL; + + timeline = kgsl_timeline_by_id(dev_priv->device, param->timeline); + if (!timeline) + return -ENODEV; + + param->seqno = timeline->value; + kgsl_timeline_put(timeline); + + return 0; +} + +long kgsl_ioctl_timeline_fence_get(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_timeline_fence_get *param = data; + struct kgsl_timeline *timeline; + struct sync_file *sync_file; + struct dma_fence *fence; + int ret = 0, fd; + + timeline = kgsl_timeline_by_id(device, param->timeline); + if (!timeline) + return -ENODEV; + + fence = kgsl_timeline_fence_alloc(timeline, param->seqno); + + if (IS_ERR(fence)) { + kgsl_timeline_put(timeline); + return PTR_ERR(fence); + } + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + ret = fd; + goto out; + } + + sync_file = sync_file_create(fence); + if (sync_file) { + fd_install(fd, sync_file->file); + param->handle = fd; + } else { + put_unused_fd(fd); + ret = -ENOMEM; + } + +out: + dma_fence_put(fence); + kgsl_timeline_put(timeline); + + return ret; +} + +long kgsl_ioctl_timeline_signal(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_timeline_signal *param = data; + u64 timelines; + int i; + + if (!param->timelines_size) { + param->timelines_size = sizeof(struct kgsl_timeline_val); + return -EAGAIN; + } + + if (!param->count) + return -EINVAL; + + timelines = param->timelines; + + for (i = 0; i < param->count; i++) { + struct kgsl_timeline *timeline; + struct kgsl_timeline_val val; + + if (copy_struct_from_user(&val, sizeof(val), + u64_to_user_ptr(timelines), param->timelines_size)) + return -EFAULT; + + if (val.padding) + return -EINVAL; + + timeline = kgsl_timeline_by_id(device, val.timeline); + if (!timeline) + return -ENODEV; + + kgsl_timeline_signal(timeline, val.seqno); + + kgsl_timeline_put(timeline); + + timelines += param->timelines_size; + } + + return 0; +} + +long kgsl_ioctl_timeline_destroy(struct kgsl_device_private *dev_priv, + unsigned int cmd, void *data) +{ + struct kgsl_device *device = dev_priv->device; + struct kgsl_timeline_fence *fence, *tmp; + struct kgsl_timeline *timeline; + struct list_head temp; + u32 *param = data; + + if (*param == 0) + return -ENODEV; + + timeline = xa_erase(&device->timelines, *param); + if (!timeline) + return -ENODEV; + + INIT_LIST_HEAD(&temp); + + spin_lock_irq(&timeline->lock); + + /* Copy any still pending fences to a temporary list */ + list_replace_init(&timeline->fences, &temp); + + spin_unlock_irq(&timeline->lock); + + /* + * Set an error on each still pending fence and signal them outside of + * the lock so we don't get a spinlock recursion if the fences get + * destroyed inside of the signal callback + */ + list_for_each_entry_safe(fence, tmp, &temp, node) { + list_del_init(&fence->node); + dma_fence_set_error(&fence->base, -ENOENT); + dma_fence_signal(&fence->base); + } + + kgsl_timeline_put(timeline); + + return timeline ? 0 : -ENODEV; +} diff --git a/drivers/gpu/msm/kgsl_timeline.h b/drivers/gpu/msm/kgsl_timeline.h new file mode 100644 index 000000000000..0e7d6537b626 --- /dev/null +++ b/drivers/gpu/msm/kgsl_timeline.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#ifndef __KGSL_TIMELINE_H +#define __KGSL_TIMELINE_H + +/** + * struct kgsl_timeline - Container for a timeline object + */ +struct kgsl_timeline { + /** @context: dma-fence timeline context */ + u64 context; + /** @id: Timeline identifier */ + int id; + /** @value: Current value of the timeline */ + u64 value; + /** @lock: Lock to protect @fences */ + spinlock_t lock; + /** @ref: Reference count for the struct */ + struct kref ref; + /** @fences: sorted list of active fences */ + struct list_head fences; + /** @name: Name of the timeline for debugging */ + const char name[32]; +}; + +/** + * kgsl_timeline_signal - Signal the timeline + * @timeline: Pointer to a timeline container + * @seqno: Seqeuence number to signal + * + * Advance @timeline to sequence number @seqno and signal any fences that might + * have expired. + */ +void kgsl_timeline_signal(struct kgsl_timeline *timeline, u64 seqno); + +/** + * kgsl_timeline_destroy - Timeline destroy callback + * @kref: Refcount pointer for the timeline + * + * Reference count callback for the timeline called when the all the object + * references have been released. + */ +void kgsl_timeline_destroy(struct kref *kref); + +/** + * kgsl_timeline_fence_alloc - Allocate a new fence on a timeline + * @timeline: Pointer to a timeline container + * @seqno: Sequence number for the new fence to wait for + * + * Create and return a new fence on the timeline that will expire when the + * timeline value is greater or equal to @seqno. + * Return: A pointer to the newly created fence + */ +struct dma_fence *kgsl_timeline_fence_alloc(struct kgsl_timeline *timeline, + u64 seqno); + +/** + * kgsl_timeline_by_id - Look up a timeline by an id + * @device: A KGSL device handle + * @id: Lookup identifier + * + * Find and return the timeline associated with identifer @id. + * Return: A pointer to a timeline or PTR_ERR() encoded error on failure. + */ +struct kgsl_timeline *kgsl_timeline_by_id(struct kgsl_device *device, + u32 id); + +/** + * kgsl_timeline_get - Get a reference to an existing timeline + * @timeline: Pointer to a timeline container + * + * Get a new reference to the timeline and return the pointer back to the user. + * Return: The pointer to the timeline or PTR_ERR encoded error on failure + */ +struct kgsl_timeline *kgsl_timeline_get(struct kgsl_timeline *timeline); + +/** + * kgsl_timeline_put - Release a reference to a timeline + * @timeline: Pointer to a timeline container + * + * Release a reference to a timeline and destroy it if there are no other + * references + */ +static inline void kgsl_timeline_put(struct kgsl_timeline *timeline) +{ + if (!IS_ERR_OR_NULL(timeline)) + kref_put(&timeline->ref, kgsl_timeline_destroy); +} + +/** + * kgsl_timelines_to_fence_array - Return a dma-fence array of timeline fences + * @device: A KGSL device handle + * @timelines: Userspace pointer to an array of &struct kgsl_timeline_val + * @count: Number of entries in @timelines + * @usize: Size of each entry in @timelines + * @any: True if the fence should expire on any timeline expiring or false if it + * should wait until all timelines have expired + * + * Give a list of &struct kgsl_timeline_val entries, create a dma-fence-array + * containing fences for each timeline/seqno pair. If @any is set the + * dma-fence-array will be set to expire if any of the encapsulated timeline + * fences expire. If @any is false, then the fence will wait for ALL of the + * encapsulated timeline fences to expire. + */ +struct dma_fence *kgsl_timelines_to_fence_array(struct kgsl_device *device, + u64 timelines, u64 count, u64 usize, bool any); + +#endif