From b79b527a45905e9b6a9bc0dde0b8c8d5a00a4362 Mon Sep 17 00:00:00 2001 From: Anuj Garg Date: Mon, 22 Feb 2021 22:42:11 +0530 Subject: [PATCH] char: virtio_fastrpc: Add snapshot of virtio fastrpc driver This snapshot is taken as of dev-sa8155 branch commit 18dcf6bbb4a9 ("char: virtio_fastrpc: add dma handle support"). Change-Id: I3a918907a319ee397f99162e57644d07504a5dd6 Signed-off-by: Anuj Garg --- drivers/char/Kconfig | 10 + drivers/char/Makefile | 5 + drivers/char/virtio_fastrpc.c | 2383 +++++++++++++++++++++++++++++++++ 3 files changed, 2398 insertions(+) create mode 100644 drivers/char/virtio_fastrpc.c diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 0278f4b33a31..e225efc05355 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -581,6 +581,16 @@ config VIRTIO_EAVB interface for userspace. Say Y if you want to support virtual eavb. +config VIRTIO_FASTRPC + tristate "Virtio fastrpc driver" + depends on VIRTIO + help + This is the virtual fastrpc frontend driver based on virtio. It + provides a communication mechanism that allows for clients to + make remote method invocations across processor boundary to + applications DSP processor. Say Y if you want to support virtual + fastrpc. + config MSM_RDBG tristate "QTI Remote debug driver" help diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 4e52abdb72ac..184236195cd6 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -59,6 +59,11 @@ ifdef CONFIG_COMPAT frpc-adsprpc-objs += adsprpc_compat.o endif obj-$(CONFIG_MSM_ADSPRPC) += frpc-adsprpc.o +vfastrpc-objs += virtio_fastrpc.o +ifdef CONFIG_COMPAT + vfastrpc-objs += adsprpc_compat.o +endif +obj-$(CONFIG_VIRTIO_FASTRPC) += vfastrpc.o obj-$(CONFIG_ADI) += adi.o obj-$(CONFIG_MSM_RDBG) += rdbg.o obj-$(CONFIG_MSM_FASTCVPD) += fastcvpd.o diff --git a/drivers/char/virtio_fastrpc.c b/drivers/char/virtio_fastrpc.c new file mode 100644 index 000000000000..69e9bf904f17 --- /dev/null +++ b/drivers/char/virtio_fastrpc.c @@ -0,0 +1,2383 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adsprpc_compat.h" +#include "adsprpc_shared.h" + +#define VIRTIO_ID_FASTRPC 34 +/* indicates remote invoke with buffer attributes is supported */ +#define VIRTIO_FASTRPC_F_INVOKE_ATTR 1 +/* indicates remote invoke with CRC is supported */ +#define VIRTIO_FASTRPC_F_INVOKE_CRC 2 +/* indicates remote mmap/munmap is supported */ +#define VIRTIO_FASTRPC_F_MMAP 3 +/* indicates QOS setting is supported */ +#define VIRTIO_FASTRPC_F_CONTROL 4 +/* indicates smmu passthrough is supported */ +#define VIRTIO_FASTRPC_F_SMMU_PASSTHROUGH 5 + +#define NUM_CHANNELS 4 /* adsp, mdsp, slpi, cdsp*/ +#define NUM_DEVICES 2 /* adsprpc-smd, adsprpc-smd-secure */ +#define M_FDLIST 16 +#define MINOR_NUM_DEV 0 +#define MINOR_NUM_SECURE_DEV 1 +#define ADSP_MMAP_HEAP_ADDR 4 +#define ADSP_MMAP_REMOTE_HEAP_ADDR 8 +#define FASTRPC_DMAHANDLE_NOMAP 16 +#define ADSP_MMAP_ADD_PAGES 0x1000 + +#define INIT_FILELEN_MAX (2*1024*1024) +#define INIT_MEMLEN_MAX (8*1024*1024) +#define MAX_CACHE_BUF_SIZE (8*1024*1024) + +#define FASTRPC_MSG_MAX 256 +#define MAX_FASTRPC_BUF_SIZE (128*1024) +#define DEBUGFS_SIZE 3072 +#define PID_SIZE 10 +#define UL_SIZE 25 + +#define VIRTIO_FASTRPC_CMD_OPEN 1 +#define VIRTIO_FASTRPC_CMD_CLOSE 2 +#define VIRTIO_FASTRPC_CMD_INVOKE 3 +#define VIRTIO_FASTRPC_CMD_MMAP 4 +#define VIRTIO_FASTRPC_CMD_MUNMAP 5 +#define VIRTIO_FASTRPC_CMD_CONTROL 6 + +#define ADSP_DOMAIN_ID 0 +#define MDSP_DOMAIN_ID 1 +#define SDSP_DOMAIN_ID 1 +#define CDSP_DOMAIN_ID 3 + +#define STATIC_PD 0 +#define DYNAMIC_PD 1 +#define GUEST_OS 2 + +#define K_COPY_FROM_USER(err, kernel, dst, src, size) \ + do {\ + if (!(kernel))\ + VERIFY(err, 0 == copy_from_user((dst),\ + (void const __user *)(src),\ + (size)));\ + else\ + memmove((dst), (src), (size));\ + } while (0) + +#define K_COPY_TO_USER(err, kernel, dst, src, size) \ + do {\ + if (!(kernel))\ + VERIFY(err, 0 == copy_to_user((void __user *)(dst),\ + (src), (size)));\ + else\ + memmove((dst), (src), (size));\ + } while (0) + +#define FASTRPC_STATIC_HANDLE_KERNEL 1 +#define FASTRPC_STATIC_HANDLE_LISTENER 3 +#define FASTRPC_STATIC_HANDLE_MAX 20 + +struct virt_msg_hdr { + u32 pid; /* GVM pid */ + u32 tid; /* GVM tid */ + s32 cid; /* channel id connected to DSP */ + u32 cmd; /* command type */ + u16 len; /* command length */ + u16 msgid; /* unique message id */ + u32 result; /* message return value */ +} __packed; + +struct virt_fastrpc_msg { + struct completion work; + u16 msgid; + void *txbuf; + void *rxbuf; +}; + +struct virt_open_msg { + struct virt_msg_hdr hdr; /* virtio fastrpc message header */ + u32 domain; /* DSP domain id */ + u32 pd; /* DSP PD */ +} __packed; + +struct virt_control_msg { + struct virt_msg_hdr hdr; /* virtio fastrpc message header */ + u32 enable; /* latency control enable */ + u32 latency; /* latency value */ +} __packed; + +struct virt_fastrpc_buf { + u64 pv; /* buffer physical address, 0 for non-ION buffer */ + u64 len; /* buffer length */ +}; + +struct virt_fastrpc_dmahandle { + u32 fd; + u32 offset; +}; + +struct virt_invoke_msg { + struct virt_msg_hdr hdr; /* virtio fastrpc message header */ + u32 handle; /* remote handle */ + u32 sc; /* scalars describing the data */ + struct virt_fastrpc_buf pra[0]; /* remote arguments list */ +} __packed; + +struct virt_mmap_msg { + struct virt_msg_hdr hdr; /* virtio fastrpc message header */ + u32 nents; /* number of map entries */ + u32 flags; /* mmap flags */ + u64 size; /* mmap length */ + u64 vapp; /* application virtual address */ + u64 vdsp; /* dsp address */ + struct virt_fastrpc_buf sgl[0]; /* sg list */ +} __packed; + +struct virt_munmap_msg { + struct virt_msg_hdr hdr; /* virtio fastrpc message header */ + u64 vdsp; /* dsp address */ + u64 size; /* mmap length */ +} __packed; + + +struct virt_fastrpc_vq { + /* protects vq */ + spinlock_t vq_lock; + struct virtqueue *vq; +}; + +struct fastrpc_apps { + struct virtio_device *vdev; + struct virt_fastrpc_vq rvq; + struct virt_fastrpc_vq svq; + void *rbufs; + void *sbufs; + unsigned int order; + unsigned int num_bufs; + unsigned int buf_size; + int last_sbuf; + + bool has_invoke_attr; + bool has_invoke_crc; + bool has_mmap; + bool has_control; + + struct device *dev; + struct cdev cdev; + struct class *class; + dev_t dev_no; + + spinlock_t msglock; + struct virt_fastrpc_msg *msgtable[FASTRPC_MSG_MAX]; +}; + +struct fastrpc_file { + spinlock_t hlock; + struct hlist_head maps; + struct hlist_head cached_bufs; + struct hlist_head remote_bufs; + uint32_t mode; + uint32_t profile; + int tgid; + int cid; + int domain; + int pd; + int file_close; + int dsp_proc_init; + struct fastrpc_apps *apps; + struct dentry *debugfs_file; + struct mutex map_mutex; + /* Identifies the device (MINOR_NUM_DEV / MINOR_NUM_SECURE_DEV) */ + int dev_minor; + char *debug_buf; +}; + +struct fastrpc_mmap { + struct hlist_node hn; + struct fastrpc_file *fl; + int fd; + uint32_t flags; + struct dma_buf *buf; + struct sg_table *table; + struct dma_buf_attachment *attach; + uint64_t phys; + size_t size; + uintptr_t va; + size_t len; + uintptr_t raddr; + int refs; +}; + +struct fastrpc_buf { + struct hlist_node hn; + struct hlist_node hn_rem; + struct fastrpc_file *fl; + size_t size; + struct sg_table sgt; + struct page **pages; + void *va; + unsigned long dma_attr; + uintptr_t raddr; + uint32_t flags; + int remote; +}; + +enum fastrpc_buf_type { + FASTRPC_BUF_TYPE_NORMAL, + FASTRPC_BUF_TYPE_ION, + FASTRPC_BUF_TYPE_INTERNAL, +}; + +struct fastrpc_buf_desc { + enum fastrpc_buf_type type; + struct fastrpc_buf *buf; +}; + +static struct fastrpc_apps gfa; +static struct dentry *debugfs_root; +static int virt_fastrpc_close(struct fastrpc_file *fl); +static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, + unsigned long dma_attr, uint32_t rflags, + int remote, struct fastrpc_buf **obuf); +static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache); + +static inline int64_t getnstimediff(struct timespec64 *start) +{ + int64_t ns; + struct timespec64 ts, b; + + ktime_get_real_ts64(&ts); + b = timespec64_sub(ts, *start); + ns = timespec64_to_ns(&b); + return ns; +} +static void virt_init_vq(struct virt_fastrpc_vq *fastrpc_vq, + struct virtqueue *vq) +{ + spin_lock_init(&fastrpc_vq->vq_lock); + fastrpc_vq->vq = vq; +} + +static void *get_a_tx_buf(void) +{ + struct fastrpc_apps *me = &gfa; + unsigned int len; + void *ret; + unsigned long flags; + + /* support multiple concurrent senders */ + spin_lock_irqsave(&me->svq.vq_lock, flags); + /* + * either pick the next unused tx buffer + * (half of our buffers are used for sending messages) + */ + if (me->last_sbuf < me->num_bufs / 2) + ret = me->sbufs + me->buf_size * me->last_sbuf++; + /* or recycle a used one */ + else + ret = virtqueue_get_buf(me->svq.vq, &len); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + return ret; +} + +static struct virt_fastrpc_msg *virt_alloc_msg(int size) +{ + struct fastrpc_apps *me = &gfa; + struct virt_fastrpc_msg *msg; + void *buf; + unsigned long flags; + int i; + + if (size > me->buf_size) { + dev_err(me->dev, "message is too big (%d)\n", size); + return NULL; + } + + msg = kzalloc(sizeof(*msg), GFP_KERNEL); + if (!msg) + return NULL; + + buf = get_a_tx_buf(); + if (!buf) { + dev_err(me->dev, "can't get tx buffer\n"); + kfree(msg); + return NULL; + } + + msg->txbuf = buf; + init_completion(&msg->work); + spin_lock_irqsave(&me->msglock, flags); + for (i = 0; i < FASTRPC_MSG_MAX; i++) { + if (!me->msgtable[i]) { + me->msgtable[i] = msg; + msg->msgid = i; + break; + } + } + spin_unlock_irqrestore(&me->msglock, flags); + + if (i == FASTRPC_MSG_MAX) { + dev_err(me->dev, "message queue is full\n"); + kfree(msg); + return NULL; + } + return msg; +} + +static void virt_free_msg(struct virt_fastrpc_msg *msg) +{ + struct fastrpc_apps *me = &gfa; + unsigned long flags; + + spin_lock_irqsave(&me->msglock, flags); + if (me->msgtable[msg->msgid] == msg) + me->msgtable[msg->msgid] = NULL; + else + dev_err(me->dev, "can't find msg %d in table\n", msg->msgid); + spin_unlock_irqrestore(&me->msglock, flags); + + kfree(msg); +} + +static void fastrpc_mmap_add(struct fastrpc_mmap *map) +{ + if (map->flags == ADSP_MMAP_HEAP_ADDR || + map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) { + struct fastrpc_apps *me = &gfa; + + dev_err(me->dev, "%s ADSP_MMAP_HEAP_ADDR is not supported\n", + __func__); + } else { + struct fastrpc_file *fl = map->fl; + + hlist_add_head(&map->hn, &fl->maps); + } +} + +static void fastrpc_mmap_free(struct fastrpc_mmap *map, uint32_t flags) +{ + struct fastrpc_apps *me = &gfa; + + if (!map) + return; + + if (map->flags == ADSP_MMAP_HEAP_ADDR || + map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) { + dev_err(me->dev, "%s ADSP_MMAP_HEAP_ADDR is not supported\n", + __func__); + } else { + map->refs--; + if (!map->refs) + hlist_del_init(&map->hn); + if (map->refs > 0 && !flags) + return; + } + if (!IS_ERR_OR_NULL(map->table)) + dma_buf_unmap_attachment(map->attach, map->table, + DMA_BIDIRECTIONAL); + if (!IS_ERR_OR_NULL(map->attach)) + dma_buf_detach(map->buf, map->attach); + if (!IS_ERR_OR_NULL(map->buf)) + dma_buf_put(map->buf); + + kfree(map); +} + +static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd, + uintptr_t va, size_t len, int mflags, int refs, + struct fastrpc_mmap **ppmap) +{ + struct fastrpc_apps *me = &gfa; + struct fastrpc_mmap *match = NULL, *map = NULL; + struct hlist_node *n; + + if ((va + len) < va) + return -EOVERFLOW; + if (mflags == ADSP_MMAP_HEAP_ADDR || + mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) { + dev_err(me->dev, "%s ADSP_MMAP_HEAP_ADDR is not supported\n", + __func__); + } else { + hlist_for_each_entry_safe(map, n, &fl->maps, hn) { + if (va >= map->va && + va + len <= map->va + map->len && + map->fd == fd) { + if (refs) { + if (map->refs + 1 == INT_MAX) + return -ETOOMANYREFS; + map->refs++; + } + match = map; + break; + } + } + } + if (match) { + *ppmap = match; + return 0; + } + return -ENOTTY; +} + +static int fastrpc_mmap_remove(struct fastrpc_file *fl, uintptr_t va, + size_t len, struct fastrpc_mmap **ppmap) +{ + struct fastrpc_mmap *match = NULL, *map; + struct hlist_node *n; + + hlist_for_each_entry_safe(map, n, &fl->maps, hn) { + if (map->raddr == va && + map->raddr + map->len == va + len) { + match = map; + hlist_del_init(&map->hn); + break; + } + } + if (match) { + *ppmap = match; + return 0; + } + return -ENOTTY; +} + +static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, + uintptr_t va, size_t len, int mflags, struct fastrpc_mmap **ppmap) +{ + struct fastrpc_apps *me = &gfa; + struct fastrpc_mmap *map = NULL; + int err = 0, sgl_index = 0; + unsigned long flags; + struct scatterlist *sgl = NULL; + + if (!fastrpc_mmap_find(fl, fd, va, len, mflags, 1, ppmap)) + return 0; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + VERIFY(err, !IS_ERR_OR_NULL(map)); + if (err) + goto bail; + + INIT_HLIST_NODE(&map->hn); + map->flags = mflags; + map->refs = 1; + map->fl = fl; + map->fd = fd; + if (mflags == ADSP_MMAP_HEAP_ADDR || + mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) { + dev_err(me->dev, "%s ADSP_MMAP_HEAP_ADDR is not supported\n", + __func__); + err = -EINVAL; + goto bail; + } else { + VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd))); + if (err) { + dev_err(me->dev, "can't get dma buf fd %d\n", fd); + goto bail; + } + VERIFY(err, !dma_buf_get_flags(map->buf, &flags)); + if (err) { + dev_err(me->dev, "can't get dma buf flags %d\n", fd); + goto bail; + } + + VERIFY(err, !IS_ERR_OR_NULL(map->attach = + dma_buf_attach(map->buf, me->dev))); + if (err) { + dev_err(me->dev, "can't attach dma buf\n"); + goto bail; + } + + if (!(flags & ION_FLAG_CACHED)) + map->attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC; + VERIFY(err, !IS_ERR_OR_NULL(map->table = + dma_buf_map_attachment(map->attach, + DMA_BIDIRECTIONAL))); + if (err) { + dev_err(me->dev, "can't get sg table of dma buf\n"); + goto bail; + } + map->phys = sg_dma_address(map->table->sgl); + for_each_sg(map->table->sgl, sgl, map->table->nents, sgl_index) + map->size += sg_dma_len(sgl); + map->va = va; + } + + map->len = len; + fastrpc_mmap_add(map); + *ppmap = map; +bail: + if (err && map) + fastrpc_mmap_free(map, 0); + return err; +} + +static int virt_fastrpc_invoke(struct fastrpc_file *fl, uint32_t kernel, + struct fastrpc_ioctl_invoke_crc *inv) +{ + struct fastrpc_apps *me = fl->apps; + struct virt_invoke_msg *vmsg, *rsp = NULL; + struct virt_fastrpc_msg *msg = NULL; + struct fastrpc_ioctl_invoke *invoke = &inv->inv; + struct scatterlist sg[1]; + int inbufs = REMOTE_SCALARS_INBUFS(invoke->sc); + int outbufs = REMOTE_SCALARS_OUTBUFS(invoke->sc); + int i, err = 0, bufs, handles, total; + remote_arg_t *lpra = NULL; + struct virt_fastrpc_buf *rpra; + struct virt_fastrpc_dmahandle *handle; + uint64_t *fdlist; + int *fds, outbufs_offset = 0; + unsigned int *attrs; + struct fastrpc_mmap **maps, *mmap = NULL; + size_t copylen = 0, size = 0, handle_len = 0, metalen; + char *payload; + struct fastrpc_buf_desc *desc = NULL; + unsigned long flags; + + bufs = inbufs + outbufs; + handles = REMOTE_SCALARS_INHANDLES(invoke->sc) + + REMOTE_SCALARS_OUTHANDLES(invoke->sc); + total = REMOTE_SCALARS_LENGTH(invoke->sc); + size = total * sizeof(*lpra) + total * sizeof(*fds) + + total * sizeof(*attrs) + total * sizeof(*maps); + lpra = kzalloc(size, GFP_KERNEL); + if (!lpra) + return -ENOMEM; + fds = (int *)&lpra[total]; + attrs = (unsigned int *)&fds[total]; + maps = (struct fastrpc_mmap **)&attrs[total]; + K_COPY_FROM_USER(err, kernel, (void *)lpra, invoke->pra, + total * sizeof(*lpra)); + if (err) + goto bail; + if (inv->fds) { + K_COPY_FROM_USER(err, kernel, fds, inv->fds, + total * sizeof(*fds)); + if (err) + goto bail; + } else { + fds = NULL; + } + + if (inv->attrs) { + K_COPY_FROM_USER(err, kernel, attrs, inv->attrs, + total * sizeof(*attrs)); + if (err) + goto bail; + } + + /* calculate len required for copying */ + for (i = 0; i < bufs; i++) { + size_t len = lpra[i].buf.len; + + if (!len) + continue; + if (fds && (fds[i] != -1)) { + /* map ion buffers */ + mutex_lock(&fl->map_mutex); + err = fastrpc_mmap_create(fl, fds[i], + (uintptr_t)lpra[i].buf.pv, + len, 0, &maps[i]); + mutex_unlock(&fl->map_mutex); + if (err) + goto bail; + len = maps[i]->table->nents * + sizeof(struct virt_fastrpc_buf); + } + copylen += len; + if (i < inbufs) + outbufs_offset += len; + } + + mutex_lock(&fl->map_mutex); + for (i = bufs; i < total; i++) { + int dmaflags = 0; + + if (attrs && (attrs[i] & FASTRPC_ATTR_NOMAP)) + dmaflags = FASTRPC_DMAHANDLE_NOMAP; + if (fds && (fds[i] != -1)) { + err = fastrpc_mmap_create(fl, fds[i], + 0, 0, dmaflags, &maps[i]); + if (err) { + mutex_unlock(&fl->map_mutex); + goto bail; + } + handle_len += maps[i]->table->nents * + sizeof(struct virt_fastrpc_buf); + } + } + mutex_unlock(&fl->map_mutex); + + metalen = sizeof(*vmsg) + total * sizeof(*rpra) + + handles * sizeof(struct virt_fastrpc_dmahandle) + + sizeof(uint64_t) * M_FDLIST; + size = metalen + copylen + handle_len; + if (size > me->buf_size) { + /* if user buffer contents exceed virtio buffer limits, + * try to alloc an internal buffer to copy + */ + copylen = 0; + outbufs_offset = 0; + desc = kcalloc(bufs, sizeof(*desc), GFP_KERNEL); + if (!desc) { + err = -ENOMEM; + goto bail; + } + for (i = 0; i < bufs; i++) { + size_t len = lpra[i].buf.len; + + if (maps[i]) { + len = maps[i]->table->nents * + sizeof(struct virt_fastrpc_buf); + desc[i].type = FASTRPC_BUF_TYPE_ION; + } else if (len < PAGE_SIZE) { + desc[i].type = FASTRPC_BUF_TYPE_NORMAL; + } else { + desc[i].type = FASTRPC_BUF_TYPE_INTERNAL; + len = PAGE_ALIGN(len); + err = fastrpc_buf_alloc(fl, len, 0, + 0, 0, &desc[i].buf); + if (err) + goto bail; + len = desc[i].buf->sgt.nents * + sizeof(struct virt_fastrpc_buf); + } + copylen += len; + if (i < inbufs) + outbufs_offset += len; + } + size = metalen + copylen + handle_len; + } + + msg = virt_alloc_msg(size); + if (!msg) { + err = -ENOMEM; + goto bail; + } + + vmsg = (struct virt_invoke_msg *)msg->txbuf; + if (kernel) + vmsg->hdr.pid = 0; + else + vmsg->hdr.pid = fl->tgid; + vmsg->hdr.tid = current->pid; + vmsg->hdr.cid = fl->cid; + vmsg->hdr.cmd = VIRTIO_FASTRPC_CMD_INVOKE; + vmsg->hdr.len = size; + vmsg->hdr.msgid = msg->msgid; + vmsg->hdr.result = 0xffffffff; + vmsg->handle = invoke->handle; + vmsg->sc = invoke->sc; + rpra = (struct virt_fastrpc_buf *)vmsg->pra; + handle = (struct virt_fastrpc_dmahandle *)&rpra[total]; + fdlist = (uint64_t *)&handle[handles]; + payload = (char *)&fdlist[M_FDLIST]; + + for (i = 0; i < M_FDLIST; i++) + fdlist[i] = 0; + + for (i = 0; i < bufs; i++) { + size_t len = lpra[i].buf.len; + struct sg_table *table; + struct virt_fastrpc_buf *sgbuf; + struct scatterlist *sgl = NULL; + int index = 0; + + if (maps[i]) { + table = maps[i]->table; + rpra[i].pv = len; + rpra[i].len = table->nents * + sizeof(struct virt_fastrpc_buf); + sgbuf = (struct virt_fastrpc_buf *)payload; + for_each_sg(table->sgl, sgl, table->nents, index) { + sgbuf[index].pv = sg_dma_address(sgl); + sgbuf[index].len = sg_dma_len(sgl); + } + payload += rpra[i].len; + } else if (desc && desc[i].type == FASTRPC_BUF_TYPE_INTERNAL) { + table = &desc[i].buf->sgt; + rpra[i].pv = len; + rpra[i].len = table->nents * + sizeof(struct virt_fastrpc_buf); + sgbuf = (struct virt_fastrpc_buf *)payload; + for_each_sg(table->sgl, sgl, table->nents, index) { + sgbuf[index].pv = page_to_phys(sg_page(sgl)); + sgbuf[index].len = sgl->length; + } + if (i < inbufs && len) { + K_COPY_FROM_USER(err, 0, desc[i].buf->va, + lpra[i].buf.pv, len); + if (err) + goto bail; + + } + payload += rpra[i].len; + } else { + /* copy non ion buffers */ + rpra[i].pv = 0; + rpra[i].len = len; + if (i < inbufs && len) { + K_COPY_FROM_USER(err, kernel, payload, + lpra[i].buf.pv, len); + if (err) + goto bail; + } + payload += len; + } + } + + for (i = bufs; i < total; i++) { + struct sg_table *table; + struct virt_fastrpc_buf *sgbuf; + struct scatterlist *sgl = NULL; + int index = 0, hlist; + + if (maps[i]) { + /* fill in dma handle list */ + hlist = i - bufs; + handle[hlist].fd = fds[i]; + handle[hlist].offset = (u32)lpra[i].buf.pv; + /* copy dma handle sglist to data area */ + table = maps[i]->table; + rpra[i].pv = lpra[i].buf.len; + rpra[i].len = table->nents * + sizeof(struct virt_fastrpc_buf); + sgbuf = (struct virt_fastrpc_buf *)payload; + for_each_sg(table->sgl, sgl, table->nents, index) { + sgbuf[index].pv = sg_dma_address(sgl); + sgbuf[index].len = sg_dma_len(sgl); + } + payload += rpra[i].len; + } + } + + sg_init_one(sg, vmsg, size); + + spin_lock_irqsave(&me->svq.vq_lock, flags); + err = virtqueue_add_outbuf(me->svq.vq, sg, 1, vmsg, GFP_KERNEL); + if (err) { + dev_err(me->dev, "%s: fail to add output buffer\n", __func__); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + goto bail; + } + + virtqueue_kick(me->svq.vq); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + wait_for_completion(&msg->work); + + rsp = msg->rxbuf; + err = rsp->hdr.result; + if (err) + goto bail; + + rpra = (struct virt_fastrpc_buf *)rsp->pra; + handle = (struct virt_fastrpc_dmahandle *)&rpra[total]; + fdlist = (uint64_t *)&handle[handles]; + payload = (char *)&fdlist[M_FDLIST] + outbufs_offset; + + for (i = inbufs; i < bufs; i++) { + if (maps[i]) { + mutex_lock(&fl->map_mutex); + fastrpc_mmap_free(maps[i], 0); + mutex_unlock(&fl->map_mutex); + maps[i] = NULL; + } else if (desc && desc[i].type == FASTRPC_BUF_TYPE_INTERNAL) { + K_COPY_TO_USER(err, 0, lpra[i].buf.pv, + desc[i].buf->va, lpra[i].buf.len); + if (err) + goto bail; + } else { + K_COPY_TO_USER(err, kernel, lpra[i].buf.pv, + payload, rpra[i].len); + if (err) + goto bail; + } + payload += rpra[i].len; + } +bail: + if (rsp) { + sg_init_one(sg, rsp, me->buf_size); + + spin_lock_irqsave(&me->rvq.vq_lock, flags); + /* add the buffer back to the remote processor's virtqueue */ + if (virtqueue_add_inbuf(me->rvq.vq, sg, 1, rsp, GFP_KERNEL)) + dev_err(me->dev, + "%s: fail to add input buffer\n", __func__); + else + virtqueue_kick(me->rvq.vq); + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + } + + mutex_lock(&fl->map_mutex); + for (i = 0; i < bufs; i++) + fastrpc_mmap_free(maps[i], 0); + + if (total) { + for (i = 0; i < M_FDLIST; i++) { + if (!fdlist[i]) + break; + if (!fastrpc_mmap_find(fl, (int)fdlist[i], 0, 0, + 0, 0, &mmap)) + fastrpc_mmap_free(mmap, 0); + } + } + mutex_unlock(&fl->map_mutex); + + if (msg) + virt_free_msg(msg); + if (desc) { + for (i = 0; i < bufs; i++) { + if (desc[i].buf) + fastrpc_buf_free(desc[i].buf, 1); + } + kfree(desc); + } + kfree(lpra); + + return err; +} + +static int fastrpc_internal_invoke(struct fastrpc_file *fl, + uint32_t mode, uint32_t kernel, + struct fastrpc_ioctl_invoke_crc *inv) +{ + struct fastrpc_ioctl_invoke *invoke = &inv->inv; + struct fastrpc_apps *me = fl->apps; + int domain = fl->domain; + int err = 0; + + if (!kernel) { + VERIFY(err, invoke->handle != FASTRPC_STATIC_HANDLE_KERNEL); + if (err) { + dev_err(me->dev, "user application %s trying to send a kernel RPC message to channel %d\n", + current->comm, domain); + goto bail; + } + } + + VERIFY(err, fl->domain >= 0 && fl->domain < NUM_CHANNELS); + if (err) { + dev_err(me->dev, "user application %s domain is not set\n", + current->comm); + err = -EBADR; + goto bail; + } + + err = virt_fastrpc_invoke(fl, kernel, inv); +bail: + return err; +} + +static ssize_t fastrpc_debugfs_read(struct file *filp, char __user *buffer, + size_t count, loff_t *position) +{ + struct fastrpc_file *fl = filp->private_data; + struct fastrpc_buf *buf = NULL; + struct hlist_node *n; + char *fileinfo = NULL; + unsigned int len = 0; + int err = 0; + char single_line[UL_SIZE] = "----------------"; + char title[UL_SIZE] = "========================="; + + fileinfo = kzalloc(DEBUGFS_SIZE, GFP_KERNEL); + if (!fileinfo) + goto bail; + if (fl) { + len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len, + "\n%s %d\n", "CHANNEL =", fl->domain); + len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len, + "%s %9s %d\n", "profile", ":", fl->profile); + len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len, + "\n======%s %s %s======\n", title, + " LIST OF BUFS ", title); + spin_lock(&fl->hlock); + len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len, + "%-19s|%-19s|%-19s\n", + "virt", "phys", "size"); + len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len, + "%s%s%s%s%s\n", single_line, single_line, + single_line, single_line, single_line); + hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) { + len += scnprintf(fileinfo + len, + DEBUGFS_SIZE - len, + "0x%-17lX|0x%-17llX|%-19zu\n", + (unsigned long)buf->va, + (uint64_t)page_to_phys(buf->pages[0]), + buf->size); + } + spin_unlock(&fl->hlock); + } + + if (len > DEBUGFS_SIZE) + len = DEBUGFS_SIZE; + err = simple_read_from_buffer(buffer, count, position, fileinfo, len); + kfree(fileinfo); +bail: + return err; +} + +static const struct file_operations debugfs_fops = { + .open = simple_open, + .read = fastrpc_debugfs_read, +}; + +static inline void fastprc_free_pages(struct page **pages, int count) +{ + while (count--) + __free_page(pages[count]); + kvfree(pages); +} + +static struct page **fastrpc_alloc_pages(unsigned int count, gfp_t gfp) +{ + struct page **pages; + unsigned long order_mask = (2U << MAX_ORDER) - 1; + unsigned int i = 0, array_size = count * sizeof(*pages); + + if (array_size <= PAGE_SIZE) + pages = kzalloc(array_size, GFP_KERNEL); + else + pages = vzalloc(array_size); + if (!pages) + return NULL; + + /* IOMMU can map any pages, so himem can also be used here */ + gfp |= __GFP_NOWARN | __GFP_HIGHMEM; + + while (count) { + struct page *page = NULL; + unsigned int order_size; + + /* + * Higher-order allocations are a convenience rather + * than a necessity, hence using __GFP_NORETRY until + * falling back to minimum-order allocations. + */ + for (order_mask &= (2U << __fls(count)) - 1; + order_mask; order_mask &= ~order_size) { + unsigned int order = __fls(order_mask); + + order_size = 1U << order; + page = alloc_pages(order ? + (gfp | __GFP_NORETRY) & + ~__GFP_RECLAIM : gfp, order); + if (!page) + continue; + if (!order) + break; + if (!PageCompound(page)) { + split_page(page, order); + break; + } else if (!split_huge_page(page)) { + break; + } + __free_pages(page, order); + } + if (!page) { + fastprc_free_pages(pages, i); + return NULL; + } + count -= order_size; + while (order_size--) + pages[i++] = page++; + } + return pages; +} + +static struct page **fastrpc_alloc_buffer(struct fastrpc_buf *buf, gfp_t gfp) +{ + struct page **pages; + unsigned int count = PAGE_ALIGN(buf->size) >> PAGE_SHIFT; + + pages = fastrpc_alloc_pages(count, gfp); + if (!pages) + return NULL; + + if (sg_alloc_table_from_pages(&buf->sgt, pages, count, 0, + buf->size, GFP_KERNEL)) + goto out_free_pages; + + if (!(buf->dma_attr & DMA_ATTR_NO_KERNEL_MAPPING)) { + buf->va = vmap(pages, count, VM_USERMAP, + pgprot_noncached(PAGE_KERNEL)); + if (!buf->va) + goto out_free_sg; + } + return pages; + +out_free_sg: + sg_free_table(&buf->sgt); +out_free_pages: + fastprc_free_pages(pages, count); + return NULL; +} + +static inline void fastrpc_free_buffer(struct fastrpc_buf *buf) +{ + unsigned int count = PAGE_ALIGN(buf->size) >> PAGE_SHIFT; + + vunmap(buf->va); + sg_free_table(&buf->sgt); + fastprc_free_pages(buf->pages, count); +} + +static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache) +{ + struct fastrpc_file *fl = buf == NULL ? NULL : buf->fl; + + if (!fl) + return; + + if (cache && buf->size < MAX_CACHE_BUF_SIZE) { + spin_lock(&fl->hlock); + hlist_add_head(&buf->hn, &fl->cached_bufs); + spin_unlock(&fl->hlock); + return; + } + + if (buf->remote) { + spin_lock(&fl->hlock); + hlist_del_init(&buf->hn_rem); + spin_unlock(&fl->hlock); + buf->remote = 0; + buf->raddr = 0; + } + + if (!IS_ERR_OR_NULL(buf->pages)) + fastrpc_free_buffer(buf); + kfree(buf); +} + +static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size, + unsigned long dma_attr, uint32_t rflags, + int remote, struct fastrpc_buf **obuf) +{ + struct fastrpc_apps *me = &gfa; + struct fastrpc_buf *buf = NULL, *fr = NULL; + struct hlist_node *n; + int err = 0; + + VERIFY(err, size > 0); + if (err) + goto bail; + + if (!remote) { + /* find the smallest buffer that fits in the cache */ + spin_lock(&fl->hlock); + hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) { + if (buf->size >= size && (!fr || fr->size > buf->size)) + fr = buf; + } + if (fr) + hlist_del_init(&fr->hn); + spin_unlock(&fl->hlock); + if (fr) { + *obuf = fr; + return 0; + } + } + + VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL))); + if (err) + goto bail; + buf->fl = fl; + buf->size = size; + buf->va = NULL; + buf->dma_attr = dma_attr; + buf->flags = rflags; + buf->raddr = 0; + buf->remote = 0; + buf->pages = fastrpc_alloc_buffer(buf, GFP_KERNEL); + if (IS_ERR_OR_NULL(buf->pages)) { + err = -ENOMEM; + dev_err(me->dev, + "%s: %s: fastrpc_alloc_buffer failed for size 0x%zx, returned %ld\n", + current->comm, __func__, size, PTR_ERR(buf->pages)); + goto bail; + } + + if (remote) { + INIT_HLIST_NODE(&buf->hn_rem); + spin_lock(&fl->hlock); + hlist_add_head(&buf->hn_rem, &fl->remote_bufs); + spin_unlock(&fl->hlock); + buf->remote = remote; + } + + *obuf = buf; + bail: + if (err && buf) + fastrpc_buf_free(buf, 0); + return err; +} + +static void fastrpc_cached_buf_list_free(struct fastrpc_file *fl) +{ + struct fastrpc_buf *buf, *free; + + do { + struct hlist_node *n; + + free = NULL; + spin_lock(&fl->hlock); + hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) { + hlist_del_init(&buf->hn); + free = buf; + break; + } + spin_unlock(&fl->hlock); + if (free) + fastrpc_buf_free(free, 0); + } while (free); +} + + +static void fastrpc_remote_buf_list_free(struct fastrpc_file *fl) +{ + struct fastrpc_buf *buf, *free; + + do { + struct hlist_node *n; + + free = NULL; + spin_lock(&fl->hlock); + hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) { + free = buf; + break; + } + spin_unlock(&fl->hlock); + if (free) + fastrpc_buf_free(free, 0); + } while (free); +} + +static int fastrpc_open(struct inode *inode, struct file *filp) +{ + int err = 0; + struct dentry *debugfs_file; + struct fastrpc_file *fl = NULL; + struct fastrpc_apps *me = &gfa; + char strpid[PID_SIZE]; + int buf_size = 0; + + /* + * Indicates the device node opened + * MINOR_NUM_DEV or MINOR_NUM_SECURE_DEV + */ + int dev_minor = MINOR(inode->i_rdev); + + VERIFY(err, ((dev_minor == MINOR_NUM_DEV) || + (dev_minor == MINOR_NUM_SECURE_DEV))); + if (err) { + dev_err(me->dev, "Invalid dev minor num %d\n", dev_minor); + return err; + } + + VERIFY(err, NULL != (fl = kzalloc(sizeof(*fl), GFP_KERNEL))); + if (err) + return err; + + snprintf(strpid, PID_SIZE, "%d", current->pid); + buf_size = strlen(current->comm) + strlen("_") + strlen(strpid) + 1; + VERIFY(err, NULL != (fl->debug_buf = kzalloc(buf_size, GFP_KERNEL))); + if (err) { + kfree(fl); + return err; + } + snprintf(fl->debug_buf, UL_SIZE, "%.10s%s%d", + current->comm, "_", current->pid); + debugfs_file = debugfs_create_file(fl->debug_buf, 0644, debugfs_root, + fl, &debugfs_fops); + + spin_lock_init(&fl->hlock); + INIT_HLIST_HEAD(&fl->maps); + INIT_HLIST_HEAD(&fl->cached_bufs); + INIT_HLIST_HEAD(&fl->remote_bufs); + fl->tgid = current->tgid; + fl->apps = me; + fl->mode = FASTRPC_MODE_SERIAL; + fl->domain = -1; + fl->cid = -1; + fl->dev_minor = dev_minor; + if (debugfs_file != NULL) + fl->debugfs_file = debugfs_file; + fl->dsp_proc_init = 0; + filp->private_data = fl; + mutex_init(&fl->map_mutex); + return 0; +} + +static int fastrpc_file_free(struct fastrpc_file *fl) +{ + struct fastrpc_mmap *map = NULL, *lmap = NULL; + + if (!fl) + return 0; + + virt_fastrpc_close(fl); + + kfree(fl->debug_buf); + + spin_lock(&fl->hlock); + fl->file_close = 1; + spin_unlock(&fl->hlock); + + mutex_lock(&fl->map_mutex); + do { + struct hlist_node *n = NULL; + + lmap = NULL; + hlist_for_each_entry_safe(map, n, &fl->maps, hn) { + hlist_del_init(&map->hn); + lmap = map; + break; + } + fastrpc_mmap_free(lmap, 1); + } while (lmap); + mutex_unlock(&fl->map_mutex); + + fastrpc_cached_buf_list_free(fl); + fastrpc_remote_buf_list_free(fl); + mutex_destroy(&fl->map_mutex); + kfree(fl); + return 0; +} + +static int fastrpc_release(struct inode *inode, struct file *file) +{ + struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data; + + if (fl) { + debugfs_remove(fl->debugfs_file); + fastrpc_file_free(fl); + file->private_data = NULL; + } + return 0; +} + + +static inline void get_fastrpc_ioctl_mmap_64( + struct fastrpc_ioctl_mmap_64 *mmap64, + struct fastrpc_ioctl_mmap *immap) +{ + immap->fd = mmap64->fd; + immap->flags = mmap64->flags; + immap->vaddrin = (uintptr_t)mmap64->vaddrin; + immap->size = mmap64->size; +} + +static inline void put_fastrpc_ioctl_mmap_64( + struct fastrpc_ioctl_mmap_64 *mmap64, + struct fastrpc_ioctl_mmap *immap) +{ + mmap64->vaddrout = (uint64_t)immap->vaddrout; +} + +static inline void get_fastrpc_ioctl_munmap_64( + struct fastrpc_ioctl_munmap_64 *munmap64, + struct fastrpc_ioctl_munmap *imunmap) +{ + imunmap->vaddrout = (uintptr_t)munmap64->vaddrout; + imunmap->size = munmap64->size; +} + +static int virt_fastrpc_munmap(struct fastrpc_file *fl, uintptr_t raddr, + size_t size) +{ + struct fastrpc_apps *me = fl->apps; + struct virt_munmap_msg *vmsg, *rsp = NULL; + struct virt_fastrpc_msg *msg; + struct scatterlist sg[1]; + int err; + unsigned long flags; + + msg = virt_alloc_msg(sizeof(*vmsg)); + if (!msg) + return -ENOMEM; + + vmsg = (struct virt_munmap_msg *)msg->txbuf; + vmsg->hdr.pid = fl->tgid; + vmsg->hdr.tid = current->pid; + vmsg->hdr.cid = fl->cid; + vmsg->hdr.cmd = VIRTIO_FASTRPC_CMD_MUNMAP; + vmsg->hdr.len = sizeof(*vmsg); + vmsg->hdr.msgid = msg->msgid; + vmsg->hdr.result = 0xffffffff; + vmsg->vdsp = raddr; + vmsg->size = size; + sg_init_one(sg, vmsg, sizeof(*vmsg)); + + spin_lock_irqsave(&me->svq.vq_lock, flags); + err = virtqueue_add_outbuf(me->svq.vq, sg, 1, vmsg, GFP_KERNEL); + if (err) { + dev_err(me->dev, "%s: fail to add output buffer\n", __func__); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + goto bail; + } + + virtqueue_kick(me->svq.vq); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + + wait_for_completion(&msg->work); + + rsp = msg->rxbuf; + err = rsp->hdr.result; +bail: + if (rsp) { + sg_init_one(sg, rsp, me->buf_size); + + spin_lock_irqsave(&me->rvq.vq_lock, flags); + /* add the buffer back to the remote processor's virtqueue */ + if (virtqueue_add_inbuf(me->rvq.vq, sg, 1, rsp, GFP_KERNEL)) + dev_err(me->dev, + "%s: fail to add input buffer\n", __func__); + else + virtqueue_kick(me->rvq.vq); + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + } + virt_free_msg(msg); + + return err; +} + +static int fastrpc_internal_munmap(struct fastrpc_file *fl, + struct fastrpc_ioctl_munmap *ud) +{ + int err = 0; + struct fastrpc_apps *me = fl->apps; + struct fastrpc_mmap *map = NULL; + struct fastrpc_buf *rbuf = NULL, *free = NULL; + struct hlist_node *n; + + VERIFY(err, fl->dsp_proc_init == 1); + if (err) { + dev_err(me->dev, "%s: user application %s trying to unmap without initialization\n", + __func__, current->comm); + err = -EBADR; + goto bail; + } + + spin_lock(&fl->hlock); + hlist_for_each_entry_safe(rbuf, n, &fl->remote_bufs, hn_rem) { + if (rbuf->raddr && (rbuf->flags == ADSP_MMAP_ADD_PAGES)) { + if ((rbuf->raddr == ud->vaddrout) && + (rbuf->size == ud->size)) { + free = rbuf; + break; + } + } + } + spin_unlock(&fl->hlock); + + if (free) { + VERIFY(err, !virt_fastrpc_munmap(fl, free->raddr, free->size)); + if (err) + goto bail; + fastrpc_buf_free(rbuf, 0); + return err; + } + + mutex_lock(&fl->map_mutex); + VERIFY(err, !fastrpc_mmap_remove(fl, ud->vaddrout, ud->size, &map)); + mutex_unlock(&fl->map_mutex); + if (err) { + dev_err(me->dev, "mapping not found to unmap va 0x%lx, len 0x%x\n", + ud->vaddrout, (unsigned int)ud->size); + goto bail; + } + VERIFY(err, !virt_fastrpc_munmap(fl, map->raddr, map->size)); + if (err) + goto bail; + mutex_lock(&fl->map_mutex); + fastrpc_mmap_free(map, 0); + mutex_unlock(&fl->map_mutex); +bail: + if (err && map) { + mutex_lock(&fl->map_mutex); + fastrpc_mmap_add(map); + mutex_unlock(&fl->map_mutex); + } + return err; +} + +static int fastrpc_internal_munmap_fd(struct fastrpc_file *fl, + struct fastrpc_ioctl_munmap_fd *ud) +{ + int err = 0; + struct fastrpc_apps *me = fl->apps; + struct fastrpc_mmap *map = NULL; + + VERIFY(err, (fl && ud)); + if (err) + goto bail; + VERIFY(err, fl->dsp_proc_init == 1); + if (err) { + dev_err(me->dev, "%s: user application %s trying to unmap without initialization\n", + __func__, current->comm); + err = -EBADR; + goto bail; + } + mutex_lock(&fl->map_mutex); + if (fastrpc_mmap_find(fl, ud->fd, ud->va, ud->len, 0, 0, &map)) { + dev_err(me->dev, "mapping not found to unmap fd 0x%x, va 0x%lx, len 0x%x\n", + ud->fd, ud->va, (unsigned int)ud->len); + err = -1; + mutex_unlock(&fl->map_mutex); + goto bail; + } + if (map) + fastrpc_mmap_free(map, 0); + mutex_unlock(&fl->map_mutex); +bail: + return err; +} + +static int virt_fastrpc_mmap(struct fastrpc_file *fl, uint32_t flags, + uintptr_t va, struct scatterlist *table, + unsigned int nents, size_t size, uintptr_t *raddr) +{ + struct fastrpc_apps *me = fl->apps; + struct virt_mmap_msg *vmsg, *rsp = NULL; + struct virt_fastrpc_msg *msg; + struct virt_fastrpc_buf *sgbuf; + struct scatterlist sg[1]; + int err, sgbuf_size, total_size; + struct scatterlist *sgl = NULL; + int sgl_index = 0; + unsigned long int_flags; + + sgbuf_size = nents * sizeof(*sgbuf); + total_size = sizeof(*vmsg) + sgbuf_size; + + msg = virt_alloc_msg(total_size); + if (!msg) + return -ENOMEM; + + vmsg = (struct virt_mmap_msg *)msg->txbuf; + vmsg->hdr.pid = fl->tgid; + vmsg->hdr.tid = current->pid; + vmsg->hdr.cid = fl->cid; + vmsg->hdr.cmd = VIRTIO_FASTRPC_CMD_MMAP; + vmsg->hdr.len = total_size; + vmsg->hdr.msgid = msg->msgid; + vmsg->hdr.result = 0xffffffff; + vmsg->flags = flags; + vmsg->size = size; + vmsg->vapp = va; + vmsg->vdsp = 0; + vmsg->nents = nents; + sgbuf = vmsg->sgl; + + for_each_sg(table, sgl, nents, sgl_index) { + if (sg_dma_len(sgl)) { + sgbuf[sgl_index].pv = sg_dma_address(sgl); + sgbuf[sgl_index].len = sg_dma_len(sgl); + } else { + sgbuf[sgl_index].pv = page_to_phys(sg_page(sgl)); + sgbuf[sgl_index].len = sgl->length; + } + } + + sg_init_one(sg, vmsg, total_size); + + spin_lock_irqsave(&me->svq.vq_lock, int_flags); + err = virtqueue_add_outbuf(me->svq.vq, sg, 1, vmsg, GFP_KERNEL); + if (err) { + dev_err(me->dev, "%s: fail to add output buffer\n", __func__); + spin_unlock_irqrestore(&me->svq.vq_lock, int_flags); + goto bail; + } + + virtqueue_kick(me->svq.vq); + spin_unlock_irqrestore(&me->svq.vq_lock, int_flags); + + wait_for_completion(&msg->work); + + rsp = msg->rxbuf; + err = rsp->hdr.result; + if (err) + goto bail; + *raddr = (uintptr_t)rsp->vdsp; +bail: + if (rsp) { + sg_init_one(sg, rsp, me->buf_size); + + spin_lock_irqsave(&me->rvq.vq_lock, int_flags); + /* add the buffer back to the remote processor's virtqueue */ + if (virtqueue_add_inbuf(me->rvq.vq, sg, 1, rsp, GFP_KERNEL)) + dev_err(me->dev, + "%s: fail to add input buffer\n", __func__); + else + virtqueue_kick(me->rvq.vq); + spin_unlock_irqrestore(&me->rvq.vq_lock, int_flags); + } + virt_free_msg(msg); + + return err; +} + +static int fastrpc_internal_mmap(struct fastrpc_file *fl, + struct fastrpc_ioctl_mmap *ud) +{ + struct fastrpc_apps *me = fl->apps; + struct fastrpc_mmap *map = NULL; + struct fastrpc_buf *rbuf = NULL; + unsigned long dma_attr = 0; + uintptr_t raddr = 0; + int err = 0; + + VERIFY(err, fl->dsp_proc_init == 1); + if (err) { + dev_err(me->dev, "%s: user application %s trying to map without initialization\n", + __func__, current->comm); + err = -EBADR; + goto bail; + } + if (ud->flags == ADSP_MMAP_ADD_PAGES) { + if (ud->vaddrin) { + err = -EINVAL; + dev_err(me->dev, "%s: %s: ERROR: adding user allocated pages is not supported\n", + current->comm, __func__); + goto bail; + } + dma_attr = DMA_ATTR_NO_KERNEL_MAPPING; + err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags, + 1, &rbuf); + if (err) + goto bail; + err = virt_fastrpc_mmap(fl, ud->flags, 0, rbuf->sgt.sgl, + rbuf->sgt.nents, rbuf->size, &raddr); + if (err) + goto bail; + rbuf->raddr = raddr; + } else { + uintptr_t va_to_dsp; + + mutex_lock(&fl->map_mutex); + VERIFY(err, !fastrpc_mmap_create(fl, ud->fd, + (uintptr_t)ud->vaddrin, ud->size, + ud->flags, &map)); + mutex_unlock(&fl->map_mutex); + if (err) + goto bail; + + if (ud->flags == ADSP_MMAP_HEAP_ADDR || + ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) + va_to_dsp = 0; + else + va_to_dsp = (uintptr_t)map->va; + + VERIFY(err, 0 == virt_fastrpc_mmap(fl, ud->flags, va_to_dsp, + map->table->sgl, map->table->nents, + map->size, &raddr)); + if (err) + goto bail; + map->raddr = raddr; + } + ud->vaddrout = raddr; + bail: + if (err && map) { + mutex_lock(&fl->map_mutex); + fastrpc_mmap_free(map, 0); + mutex_unlock(&fl->map_mutex); + } + return err; +} + +static int virt_fastrpc_control(struct fastrpc_file *fl, + struct fastrpc_ctrl_latency *lp) +{ + struct fastrpc_apps *me = fl->apps; + struct virt_control_msg *vmsg, *rsp = NULL; + struct virt_fastrpc_msg *msg; + struct scatterlist sg[1]; + int err; + unsigned long flags; + + msg = virt_alloc_msg(sizeof(*vmsg)); + if (!msg) + return -ENOMEM; + + vmsg = (struct virt_control_msg *)msg->txbuf; + vmsg->hdr.pid = fl->tgid; + vmsg->hdr.tid = current->pid; + vmsg->hdr.cid = fl->cid; + vmsg->hdr.cmd = VIRTIO_FASTRPC_CMD_CONTROL; + vmsg->hdr.len = sizeof(*vmsg); + vmsg->hdr.msgid = msg->msgid; + vmsg->hdr.result = 0xffffffff; + vmsg->enable = lp->enable; + vmsg->latency = lp->latency; + sg_init_one(sg, vmsg, sizeof(*vmsg)); + + spin_lock_irqsave(&me->svq.vq_lock, flags); + err = virtqueue_add_outbuf(me->svq.vq, sg, 1, vmsg, GFP_KERNEL); + if (err) { + dev_err(me->dev, "%s: fail to add output buffer\n", __func__); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + goto bail; + } + + virtqueue_kick(me->svq.vq); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + + wait_for_completion(&msg->work); + + rsp = msg->rxbuf; + err = rsp->hdr.result; +bail: + if (rsp) { + sg_init_one(sg, rsp, me->buf_size); + + spin_lock_irqsave(&me->rvq.vq_lock, flags); + /* add the buffer back to the remote processor's virtqueue */ + if (virtqueue_add_inbuf(me->rvq.vq, sg, 1, rsp, GFP_KERNEL)) + dev_err(me->dev, + "%s: fail to add input buffer\n", __func__); + else + virtqueue_kick(me->rvq.vq); + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + } + virt_free_msg(msg); + + return err; +} + +static int fastrpc_internal_control(struct fastrpc_file *fl, + struct fastrpc_ioctl_control *cp) +{ + struct fastrpc_apps *me = fl->apps; + int err = 0; + + VERIFY(err, !IS_ERR_OR_NULL(fl) && !IS_ERR_OR_NULL(fl->apps)); + if (err) + goto bail; + VERIFY(err, !IS_ERR_OR_NULL(cp)); + if (err) + goto bail; + + switch (cp->req) { + case FASTRPC_CONTROL_LATENCY: + if (!(me->has_control)) { + dev_err(me->dev, "qos setting is not supported\n"); + err = -ENOTTY; + goto bail; + } + virt_fastrpc_control(fl, &cp->lp); + break; + case FASTRPC_CONTROL_KALLOC: + cp->kalloc.kalloc_support = 1; + break; + default: + err = -ENOTTY; + break; + } +bail: + return err; +} + +static int fastrpc_ioctl_get_info(struct fastrpc_file *fl, + uint32_t *info) +{ + int err = 0; + uint32_t domain; + + VERIFY(err, fl != NULL); + if (err) + goto bail; + if (fl->domain == -1) { + domain = *info; + VERIFY(err, domain < NUM_CHANNELS); + if (err) + goto bail; + fl->domain = domain; + } + *info = 1; +bail: + return err; +} + +static int virt_fastrpc_open(struct fastrpc_file *fl) +{ + struct fastrpc_apps *me = fl->apps; + struct virt_open_msg *vmsg, *rsp = NULL; + struct virt_fastrpc_msg *msg; + struct scatterlist sg[1]; + int err; + unsigned long flags; + + msg = virt_alloc_msg(sizeof(*vmsg)); + if (!msg) { + dev_err(me->dev, "%s: no memory\n", __func__); + return -ENOMEM; + } + + vmsg = (struct virt_open_msg *)msg->txbuf; + vmsg->hdr.pid = fl->tgid; + vmsg->hdr.tid = current->pid; + vmsg->hdr.cid = -1; + vmsg->hdr.cmd = VIRTIO_FASTRPC_CMD_OPEN; + vmsg->hdr.len = sizeof(*vmsg); + vmsg->hdr.msgid = msg->msgid; + vmsg->hdr.result = 0xffffffff; + vmsg->domain = fl->domain; + vmsg->pd = fl->pd; + sg_init_one(sg, vmsg, sizeof(*vmsg)); + + spin_lock_irqsave(&me->svq.vq_lock, flags); + err = virtqueue_add_outbuf(me->svq.vq, sg, 1, vmsg, GFP_KERNEL); + if (err) { + dev_err(me->dev, "%s: fail to add output buffer\n", __func__); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + goto bail; + } + + virtqueue_kick(me->svq.vq); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + + wait_for_completion(&msg->work); + + rsp = msg->rxbuf; + err = rsp->hdr.result; + if (err) + goto bail; + if (rsp->hdr.cid < 0) { + dev_err(me->dev, "channel id %d is invalid\n", rsp->hdr.cid); + err = -EINVAL; + goto bail; + } + fl->cid = rsp->hdr.cid; +bail: + if (rsp) { + sg_init_one(sg, rsp, me->buf_size); + + spin_lock_irqsave(&me->rvq.vq_lock, flags); + /* add the buffer back to the remote processor's virtqueue */ + if (virtqueue_add_inbuf(me->rvq.vq, sg, 1, rsp, GFP_KERNEL)) + dev_err(me->dev, + "%s: fail to add input buffer\n", __func__); + else + virtqueue_kick(me->rvq.vq); + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + } + virt_free_msg(msg); + + return err; +} + +static int virt_fastrpc_close(struct fastrpc_file *fl) +{ + struct fastrpc_apps *me = fl->apps; + struct virt_msg_hdr *vmsg, *rsp = NULL; + struct virt_fastrpc_msg *msg; + struct scatterlist sg[1]; + int err; + unsigned long flags; + + if (fl->cid < 0) { + dev_err(me->dev, "channel id %d is invalid\n", fl->cid); + return -EINVAL; + } + + msg = virt_alloc_msg(sizeof(*vmsg)); + if (!msg) { + dev_err(me->dev, "%s: no memory\n", __func__); + return -ENOMEM; + } + + vmsg = (struct virt_msg_hdr *)msg->txbuf; + vmsg->pid = fl->tgid; + vmsg->tid = current->pid; + vmsg->cid = fl->cid; + vmsg->cmd = VIRTIO_FASTRPC_CMD_CLOSE; + vmsg->len = sizeof(*vmsg); + vmsg->msgid = msg->msgid; + vmsg->result = 0xffffffff; + sg_init_one(sg, vmsg, sizeof(*vmsg)); + + spin_lock_irqsave(&me->svq.vq_lock, flags); + err = virtqueue_add_outbuf(me->svq.vq, sg, 1, vmsg, GFP_KERNEL); + if (err) { + dev_err(me->dev, "%s: fail to add output buffer\n", __func__); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + goto bail; + } + + virtqueue_kick(me->svq.vq); + spin_unlock_irqrestore(&me->svq.vq_lock, flags); + + wait_for_completion(&msg->work); + + rsp = msg->rxbuf; + err = rsp->result; +bail: + if (rsp) { + sg_init_one(sg, rsp, me->buf_size); + + spin_lock_irqsave(&me->rvq.vq_lock, flags); + /* add the buffer back to the remote processor's virtqueue */ + if (virtqueue_add_inbuf(me->rvq.vq, sg, 1, rsp, GFP_KERNEL)) + dev_err(me->dev, + "%s: fail to add input buffer\n", __func__); + else + virtqueue_kick(me->rvq.vq); + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + } + virt_free_msg(msg); + + return err; +} + +static int fastrpc_init_process(struct fastrpc_file *fl, + struct fastrpc_ioctl_init_attrs *uproc) +{ + int err = 0; + struct fastrpc_ioctl_init *init = &uproc->init; + + if (init->flags == FASTRPC_INIT_ATTACH || + init->flags == FASTRPC_INIT_ATTACH_SENSORS) { + fl->pd = GUEST_OS; + err = virt_fastrpc_open(fl); + if (err) + goto bail; + } else if (init->flags == FASTRPC_INIT_CREATE) { + fl->pd = DYNAMIC_PD; + err = virt_fastrpc_open(fl); + if (err) + goto bail; + } else if (init->flags == FASTRPC_INIT_CREATE_STATIC) { + fl->pd = STATIC_PD; + err = virt_fastrpc_open(fl); + if (err) + goto bail; + } else { + err = -ENOTTY; + goto bail; + } + fl->dsp_proc_init = 1; +bail: + return err; +} + +static long fastrpc_ioctl(struct file *file, unsigned int ioctl_num, + unsigned long ioctl_param) +{ + union { + struct fastrpc_ioctl_invoke_crc inv; + struct fastrpc_ioctl_mmap mmap; + struct fastrpc_ioctl_mmap_64 mmap64; + struct fastrpc_ioctl_munmap munmap; + struct fastrpc_ioctl_munmap_64 munmap64; + struct fastrpc_ioctl_munmap_fd munmap_fd; + struct fastrpc_ioctl_init_attrs init; + struct fastrpc_ioctl_control cp; + } p; + union { + struct fastrpc_ioctl_mmap mmap; + struct fastrpc_ioctl_munmap munmap; + } i; + void *param = (char *)ioctl_param; + struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data; + struct fastrpc_apps *me = &gfa; + int size = 0, err = 0; + uint32_t info; + + p.inv.fds = NULL; + p.inv.attrs = NULL; + p.inv.crc = NULL; + + spin_lock(&fl->hlock); + if (fl->file_close == 1) { + err = -EBADF; + dev_warn(me->dev, "fastrpc_device_release is happening, So not sending any new requests to DSP\n"); + spin_unlock(&fl->hlock); + goto bail; + } + spin_unlock(&fl->hlock); + + switch (ioctl_num) { + case FASTRPC_IOCTL_INVOKE: + size = sizeof(struct fastrpc_ioctl_invoke); + fallthrough; + case FASTRPC_IOCTL_INVOKE_FD: + if (!size) + size = sizeof(struct fastrpc_ioctl_invoke_fd); + fallthrough; + case FASTRPC_IOCTL_INVOKE_ATTRS: + if (!size) + size = sizeof(struct fastrpc_ioctl_invoke_attrs); + fallthrough; + case FASTRPC_IOCTL_INVOKE_CRC: + if (!size) + size = sizeof(struct fastrpc_ioctl_invoke_crc); + + K_COPY_FROM_USER(err, 0, &p.inv, param, size); + if (err) + goto bail; + + if (p.inv.attrs && !(me->has_invoke_attr)) { + dev_err(me->dev, "invoke attr is not supported\n"); + err = -ENOTTY; + goto bail; + } + if (p.inv.crc && !(me->has_invoke_crc)) { + dev_err(me->dev, "invoke crc is not supported\n"); + err = -ENOTTY; + goto bail; + } + + VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl, fl->mode, + 0, &p.inv))); + if (err) + goto bail; + break; + case FASTRPC_IOCTL_MMAP: + if (!me->has_mmap) { + dev_err(me->dev, "mmap is not supported\n"); + err = -ENOTTY; + goto bail; + } + + K_COPY_FROM_USER(err, 0, &p.mmap, param, + sizeof(p.mmap)); + if (err) + goto bail; + VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &p.mmap))); + if (err) + goto bail; + K_COPY_TO_USER(err, 0, param, &p.mmap, sizeof(p.mmap)); + if (err) + goto bail; + break; + case FASTRPC_IOCTL_MUNMAP: + if (!(me->has_mmap)) { + dev_err(me->dev, "munmap is not supported\n"); + err = -ENOTTY; + goto bail; + } + + K_COPY_FROM_USER(err, 0, &p.munmap, param, + sizeof(p.munmap)); + if (err) + goto bail; + VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl, + &p.munmap))); + if (err) + goto bail; + break; + case FASTRPC_IOCTL_MMAP_64: + if (!(me->has_mmap)) { + dev_err(me->dev, "mmap is not supported\n"); + err = -ENOTTY; + goto bail; + } + + K_COPY_FROM_USER(err, 0, &p.mmap64, param, + sizeof(p.mmap64)); + if (err) + goto bail; + get_fastrpc_ioctl_mmap_64(&p.mmap64, &i.mmap); + VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &i.mmap))); + if (err) + goto bail; + put_fastrpc_ioctl_mmap_64(&p.mmap64, &i.mmap); + K_COPY_TO_USER(err, 0, param, &p.mmap64, sizeof(p.mmap64)); + if (err) + goto bail; + break; + case FASTRPC_IOCTL_MUNMAP_64: + if (!(me->has_mmap)) { + dev_err(me->dev, "munmap is not supported\n"); + err = -ENOTTY; + goto bail; + } + + K_COPY_FROM_USER(err, 0, &p.munmap64, param, + sizeof(p.munmap64)); + if (err) + goto bail; + get_fastrpc_ioctl_munmap_64(&p.munmap64, &i.munmap); + VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl, + &i.munmap))); + if (err) + goto bail; + break; + case FASTRPC_IOCTL_MUNMAP_FD: + K_COPY_FROM_USER(err, 0, &p.munmap_fd, param, + sizeof(p.munmap_fd)); + if (err) + goto bail; + VERIFY(err, 0 == (err = fastrpc_internal_munmap_fd(fl, + &p.munmap_fd))); + if (err) + goto bail; + break; + case FASTRPC_IOCTL_SETMODE: + switch ((uint32_t)ioctl_param) { + case FASTRPC_MODE_PARALLEL: + case FASTRPC_MODE_SERIAL: + fl->mode = (uint32_t)ioctl_param; + break; + case FASTRPC_MODE_PROFILE: + fl->profile = (uint32_t)ioctl_param; + break; + case FASTRPC_MODE_SESSION: + err = -ENOTTY; + dev_err(me->dev, "session mode is not supported\n"); + break; + default: + err = -ENOTTY; + break; + } + break; + case FASTRPC_IOCTL_CONTROL: + K_COPY_FROM_USER(err, 0, &p.cp, param, + sizeof(p.cp)); + if (err) + goto bail; + VERIFY(err, 0 == (err = fastrpc_internal_control(fl, &p.cp))); + if (err) + goto bail; + if (p.cp.req == FASTRPC_CONTROL_KALLOC) { + K_COPY_TO_USER(err, 0, param, &p.cp, sizeof(p.cp)); + if (err) + goto bail; + } + break; + case FASTRPC_IOCTL_GETINFO: + K_COPY_FROM_USER(err, 0, &info, param, sizeof(info)); + if (err) + goto bail; + VERIFY(err, 0 == (err = fastrpc_ioctl_get_info(fl, &info))); + if (err) + goto bail; + K_COPY_TO_USER(err, 0, param, &info, sizeof(info)); + if (err) + goto bail; + break; + case FASTRPC_IOCTL_INIT: + p.init.attrs = 0; + p.init.siglen = 0; + size = sizeof(struct fastrpc_ioctl_init); + fallthrough; + case FASTRPC_IOCTL_INIT_ATTRS: + if (!size) + size = sizeof(struct fastrpc_ioctl_init_attrs); + K_COPY_FROM_USER(err, 0, &p.init, param, size); + if (err) + goto bail; + VERIFY(err, p.init.init.filelen >= 0 && + p.init.init.filelen < INIT_FILELEN_MAX); + if (err) + goto bail; + VERIFY(err, p.init.init.memlen >= 0 && + p.init.init.memlen < INIT_MEMLEN_MAX); + if (err) + goto bail; + VERIFY(err, 0 == (err = fastrpc_init_process(fl, &p.init))); + if (err) + goto bail; + break; + + default: + err = -ENOTTY; + dev_info(me->dev, "bad ioctl: %d\n", ioctl_num); + break; + } + bail: + return err; +} + +static const struct file_operations fops = { + .open = fastrpc_open, + .release = fastrpc_release, + .unlocked_ioctl = fastrpc_ioctl, + .compat_ioctl = compat_fastrpc_device_ioctl, +}; + +static void fastrpc_init(struct fastrpc_apps *me) +{ + spin_lock_init(&me->msglock); +} + +static int recv_single(struct virt_msg_hdr *rsp, unsigned int len) +{ + struct fastrpc_apps *me = &gfa; + struct virt_fastrpc_msg *msg; + + if (len != rsp->len) { + dev_err(me->dev, "msg %u len mismatch,expected %u but %d found\n", + rsp->cmd, rsp->len, len); + return -EINVAL; + } + spin_lock(&me->msglock); + msg = me->msgtable[rsp->msgid]; + spin_unlock(&me->msglock); + + if (!msg) { + dev_err(me->dev, "msg %u already free in table[%u]\n", + rsp->cmd, rsp->msgid); + return -EINVAL; + } + msg->rxbuf = (void *)rsp; + + complete(&msg->work); + return 0; +} + +static void recv_done(struct virtqueue *rvq) +{ + + struct fastrpc_apps *me = &gfa; + struct virt_msg_hdr *rsp; + unsigned int len, msgs_received = 0; + int err; + unsigned long flags; + + spin_lock_irqsave(&me->rvq.vq_lock, flags); + rsp = virtqueue_get_buf(rvq, &len); + if (!rsp) { + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + dev_err(me->dev, "incoming signal, but no used buffer\n"); + return; + } + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + + while (rsp) { + err = recv_single(rsp, len); + if (err) + break; + + msgs_received++; + + spin_lock_irqsave(&me->rvq.vq_lock, flags); + rsp = virtqueue_get_buf(rvq, &len); + spin_unlock_irqrestore(&me->rvq.vq_lock, flags); + } +} + +static int init_vqs(struct fastrpc_apps *me) +{ + struct virtqueue *vqs[2]; + static const char * const names[] = { "output", "input" }; + vq_callback_t *cbs[] = { NULL, recv_done }; + size_t total_buf_space; + void *bufs; + int err; + + err = virtio_find_vqs(me->vdev, 2, vqs, cbs, names, NULL); + if (err) + return err; + + virt_init_vq(&me->svq, vqs[0]); + virt_init_vq(&me->rvq, vqs[1]); + + /* we expect symmetric tx/rx vrings */ + WARN_ON(virtqueue_get_vring_size(me->rvq.vq) != + virtqueue_get_vring_size(me->svq.vq)); + me->num_bufs = virtqueue_get_vring_size(me->rvq.vq) * 2; + + me->buf_size = MAX_FASTRPC_BUF_SIZE; + total_buf_space = me->num_bufs * me->buf_size; + me->order = get_order(total_buf_space); + bufs = (void *)__get_free_pages(GFP_KERNEL, + me->order); + if (!bufs) { + err = -ENOMEM; + goto vqs_del; + } + + /* half of the buffers is dedicated for RX */ + me->rbufs = bufs; + + /* and half is dedicated for TX */ + me->sbufs = bufs + total_buf_space / 2; + return 0; + +vqs_del: + me->vdev->config->del_vqs(me->vdev); + return err; +} + +static int virt_fastrpc_probe(struct virtio_device *vdev) +{ + struct fastrpc_apps *me = &gfa; + struct device *dev = NULL; + struct device *secure_dev = NULL; + int err, i; + + if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) + return -ENODEV; + + memset(me, 0, sizeof(*me)); + fastrpc_init(me); + + if (virtio_has_feature(vdev, VIRTIO_FASTRPC_F_INVOKE_ATTR)) + me->has_invoke_attr = true; + + if (virtio_has_feature(vdev, VIRTIO_FASTRPC_F_INVOKE_CRC)) + me->has_invoke_crc = true; + + if (virtio_has_feature(vdev, VIRTIO_FASTRPC_F_MMAP)) + me->has_mmap = true; + + if (virtio_has_feature(vdev, VIRTIO_FASTRPC_F_CONTROL)) + me->has_control = true; + + vdev->priv = me; + me->vdev = vdev; + me->dev = vdev->dev.parent; + + err = init_vqs(me); + if (err) { + dev_err(&vdev->dev, "failed to initialized virtqueue\n"); + return err; + } + + debugfs_root = debugfs_create_dir("adsprpc", NULL); + err = alloc_chrdev_region(&me->dev_no, 0, NUM_CHANNELS, DEVICE_NAME); + if (err) + goto alloc_chrdev_bail; + + cdev_init(&me->cdev, &fops); + me->cdev.owner = THIS_MODULE; + err = cdev_add(&me->cdev, MKDEV(MAJOR(me->dev_no), 0), NUM_DEVICES); + if (err) + goto cdev_init_bail; + + me->class = class_create(THIS_MODULE, "fastrpc"); + if (IS_ERR(me->class)) + goto class_create_bail; + + /* + * Create devices and register with sysfs + * Create first device with minor number 0 + */ + dev = device_create(me->class, NULL, + MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV), + NULL, DEVICE_NAME); + if (IS_ERR_OR_NULL(dev)) + goto device_create_bail; + + /* Create secure device with minor number for secure device */ + secure_dev = device_create(me->class, NULL, + MKDEV(MAJOR(me->dev_no), MINOR_NUM_SECURE_DEV), + NULL, DEVICE_NAME_SECURE); + if (IS_ERR_OR_NULL(secure_dev)) + goto device_create_bail; + + virtio_device_ready(vdev); + + /* set up the receive buffers */ + for (i = 0; i < me->num_bufs / 2; i++) { + struct scatterlist sg; + void *cpu_addr = me->rbufs + i * me->buf_size; + + sg_init_one(&sg, cpu_addr, me->buf_size); + err = virtqueue_add_inbuf(me->rvq.vq, &sg, 1, cpu_addr, + GFP_KERNEL); + WARN_ON(err); /* sanity check; this can't really happen */ + } + + /* suppress "tx-complete" interrupts */ + virtqueue_disable_cb(me->svq.vq); + + virtqueue_enable_cb(me->rvq.vq); + virtqueue_kick(me->rvq.vq); + + dev_info(&vdev->dev, "Registered virtio fastrpc device\n"); + + return 0; +device_create_bail: + if (!IS_ERR_OR_NULL(dev)) + device_destroy(me->class, MKDEV(MAJOR(me->dev_no), + MINOR_NUM_DEV)); + if (!IS_ERR_OR_NULL(secure_dev)) + device_destroy(me->class, MKDEV(MAJOR(me->dev_no), + MINOR_NUM_SECURE_DEV)); + class_destroy(me->class); +class_create_bail: + cdev_del(&me->cdev); +cdev_init_bail: + unregister_chrdev_region(me->dev_no, NUM_CHANNELS); +alloc_chrdev_bail: + vdev->config->del_vqs(vdev); + return err; +} + +static void virt_fastrpc_remove(struct virtio_device *vdev) +{ + struct fastrpc_apps *me = &gfa; + + device_destroy(me->class, MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV)); + device_destroy(me->class, MKDEV(MAJOR(me->dev_no), + MINOR_NUM_SECURE_DEV)); + class_destroy(me->class); + cdev_del(&me->cdev); + unregister_chrdev_region(me->dev_no, NUM_CHANNELS); + debugfs_remove_recursive(debugfs_root); + + vdev->config->reset(vdev); + vdev->config->del_vqs(vdev); + free_pages((unsigned long)me->rbufs, me->order); +} + +const struct virtio_device_id id_table[] = { + { VIRTIO_ID_FASTRPC, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static unsigned int features[] = { + VIRTIO_FASTRPC_F_INVOKE_ATTR, + VIRTIO_FASTRPC_F_INVOKE_CRC, + VIRTIO_FASTRPC_F_MMAP, + VIRTIO_FASTRPC_F_CONTROL, +}; + +static struct virtio_driver virtio_fastrpc_driver = { + .feature_table = features, + .feature_table_size = ARRAY_SIZE(features), + .driver.name = KBUILD_MODNAME, + .driver.owner = THIS_MODULE, + .id_table = id_table, + .probe = virt_fastrpc_probe, + .remove = virt_fastrpc_remove, +}; + +static int __init virtio_fastrpc_init(void) +{ + return register_virtio_driver(&virtio_fastrpc_driver); +} + +static void __exit virtio_fastrpc_exit(void) +{ + unregister_virtio_driver(&virtio_fastrpc_driver); +} +module_init(virtio_fastrpc_init); +module_exit(virtio_fastrpc_exit); + +MODULE_DEVICE_TABLE(virtio, id_table); +MODULE_DESCRIPTION("Virtio fastrpc driver"); +MODULE_LICENSE("GPL v2");