diff --git a/arch/arm64/configs/vendor/quinvmauto_QGKI.config b/arch/arm64/configs/vendor/quinvmauto_QGKI.config index 64c6568ad41b..731a983eb5d7 100644 --- a/arch/arm64/configs/vendor/quinvmauto_QGKI.config +++ b/arch/arm64/configs/vendor/quinvmauto_QGKI.config @@ -31,3 +31,5 @@ CONFIG_RENAME_BLOCK_DEVICE=y CONFIG_DM_INIT=y CONFIG_BUILD_ARM64_UNCOMPRESSED_KERNEL=y # CONFIG_BUILD_ARM64_KERNEL_COMPRESSION_GZIP is not set +CONFIG_QCOM_HGSL=y +CONFIG_QCOM_HGSL_TCSR_SIGNAL=y diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index 713614f86c7b..1695228503b5 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -658,6 +658,7 @@ config QCOM_GLINK_PKT some glink packets channel. source "drivers/soc/qcom/hab/Kconfig" +source "drivers/soc/qcom/hgsl/Kconfig" config MSM_PERFORMANCE tristate "msm performance driver to support userspace fmin/fmax request" diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index 51b5fa9221fa..ee0f55a7a79b 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -96,6 +96,7 @@ obj-$(CONFIG_QTI_HW_KEY_MANAGER) += hwkm.o crypto-qti-hwkm.o obj-$(CONFIG_QCOM_WDT_CORE) += qcom_wdt_core.o obj-$(CONFIG_QCOM_SOC_WATCHDOG) += qcom_soc_wdt.o obj-$(CONFIG_MSM_HAB) += hab/ +obj-$(CONFIG_QCOM_HGSL) += hgsl/ ifdef CONFIG_DEBUG_FS obj-$(CONFIG_MSM_RPM_SMD) += rpm-smd-debug.o endif diff --git a/drivers/soc/qcom/hgsl/Kconfig b/drivers/soc/qcom/hgsl/Kconfig new file mode 100644 index 000000000000..149be0b20765 --- /dev/null +++ b/drivers/soc/qcom/hgsl/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Hypervisor graphics system layer configuration +# +config QCOM_HGSL + bool "Graphics driver for Hypervisor" + depends on QTI_QUIN_GVM + help + This driver could help commands submmitting functions for + hypervisor Linux. With HFI feature provided by A6x, it + could submit commands directly to hardware without passing + them to host system. + +config QCOM_HGSL_TCSR_SIGNAL + bool "TCSR signal for Hypervisor GSL" + depends on MFD_SYSCON && QCOM_HGSL + help + The TCSR compute signal module provides hgsl driver + in hypervisor Linux a way to send/receive signals + to/from A6x GPU hardware directly, without going + through host system. diff --git a/drivers/soc/qcom/hgsl/Makefile b/drivers/soc/qcom/hgsl/Makefile new file mode 100644 index 000000000000..4968d1004de0 --- /dev/null +++ b/drivers/soc/qcom/hgsl/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_QCOM_HGSL) += hgsl.o +obj-$(CONFIG_QCOM_HGSL_TCSR_SIGNAL) += hgsl_tcsr.o diff --git a/drivers/soc/qcom/hgsl/hgsl.c b/drivers/soc/qcom/hgsl/hgsl.c new file mode 100644 index 000000000000..c0159812e18e --- /dev/null +++ b/drivers/soc/qcom/hgsl/hgsl.c @@ -0,0 +1,1510 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hgsl_tcsr.h" + +#define HGSL_DEVICE_NAME "hgsl" +#define HGSL_DEV_NUM 1 + +/* Support upto 3 GVMs: 3 DBQs(Low/Medium/High priority) per GVM */ +#define MAX_DB_QUEUE 9 + +#define IORESOURCE_HWINF "hgsl_reg_hwinf" +#define IORESOURCE_GMUCX "hgsl_reg_gmucx" + + +/* Set-up profiling packets as needed by scope */ +#define CMDBATCH_PROFILING 0x00000010 + +/* Ping the user of HFI when this command is done */ +#define CMDBATCH_NOTIFY 0x00000020 + +#define CMDBATCH_EOF 0x00000100 + +/* timeout of waiting for free space of doorbell queue. */ +#define HGSL_IDLE_TIMEOUT msecs_to_jiffies(1000) +#define GLB_DB_SRC_ISSUEIB_IRQ_ID_0 TCSR_SRC_IRQ_ID_0 +#define GLB_DB_SRC_ISSUEIB_IRQ_ID_1 TCSR_SRC_IRQ_ID_1 +#define GLB_DB_SRC_ISSUEIB_IRQ_ID_2 TCSR_SRC_IRQ_ID_2 +#define GLB_DB_DEST_TS_RETIRE_IRQ_ID TCSR_DEST_IRQ_ID_0 +#define GLB_DB_DEST_TS_RETIRE_IRQ_MASK TCSR_DEST_IRQ_MASK_0 + +#define HGSL_TCSR_NUM 2 + +struct hw_version { + unsigned int version; + unsigned int release; +}; + +struct reg { + unsigned long paddr; + unsigned long size; + void __iomem *vaddr; +}; + +struct db_buffer { + int32_t dwords; + void *vaddr; +}; + +#define QHDR_STATUS_INACTIVE 0x00 +#define QHDR_STATUS_ACTIVE 0x01 + +/* Updated by HGSL */ +#define DBQ_WRITE_INDEX_IN_DWORD 0 + +/* Updated by GMU */ +#define DBQ_READ_INDEX_IN_DWORD 1 + +#define HGSL_DBQ_HFI_Q_INDEX_BASE_OFFSET_IN_DWORD (1536 >> 2) +#define HGSL_DBQ_CONTEXT_INFO_BASE_OFFSET_IN_DWORD (2048 >> 2) + +#define HGSL_CONTEXT_NUM 128 + +/* Meta info for context */ +enum HGSL_DBQ_METADATA_INFO { + HGSL_DBQ_METADATA_CTXT_ID_IN_DWORD = 0x0, + HGSL_DBQ_METADATA_TIMESTAMP_IN_DWORD = 0x1, + HGSL_DBQ_METADATA_CTXT_DESTROY_IN_DWORD = 0x2, + HGSL_DBQ_METADATA_TOTAL_ENTITY_NUM, + /* The context metadata size is 2K in DBQ, + * so we can have maximum 4 dwords for each context + */ + HGSL_DBQ_METADATA_TOTAL_ENTITY_MAX = 0x4 +}; + +/* DBQ structure + * | IBs storage | reserved | w.idx/r.idx | ctxt.info| + * --------------------------------------------------------------------------- + * | [0] | [1K] | [1.5K] | [2K] | + * --------------------------------------------------------------------------- + */ +static inline void dbq_set_qindex(uint32_t *va_base, + uint32_t offset, + uint32_t value) +{ + uint32_t *dest; + + dest = va_base + HGSL_DBQ_HFI_Q_INDEX_BASE_OFFSET_IN_DWORD + offset; + *dest = value; +} + +static inline uint32_t dbq_get_qindex(uint32_t *va_base, + uint32_t offset) +{ + uint32_t *dest; + uint32_t val; + + dest = va_base + HGSL_DBQ_HFI_Q_INDEX_BASE_OFFSET_IN_DWORD + offset; + + val = *dest; + + /* ensure read is done before return */ + rmb(); + return val; +} + +static inline void dbq_update_ctxt_info(uint32_t *va_base, + uint32_t ctxt_id, + uint32_t offset, + uint32_t value) +{ + uint32_t *dest = (uint32_t *)(va_base + + HGSL_DBQ_CONTEXT_INFO_BASE_OFFSET_IN_DWORD + + (HGSL_DBQ_METADATA_TOTAL_ENTITY_NUM * ctxt_id) + + offset); + + if (WARN_ON(ctxt_id >= HGSL_CONTEXT_NUM)) + return; + + *dest = value; +} + +#define RGS_HFI_DEFAULT_TIMEOUT_MS 5000 + +#define HFI_MSG_TYPE_CMD 0 +#define HFI_MSG_TYPE_RET 1 + +/* HFI command define. */ +#define HTOF_MSG_ISSUE_CMD 130 + +#define MSG_ISSUE_INF_SZ() (sizeof(struct hgsl_db_cmds) >> 2) +#define MSG_ISSUE_IBS_SZ(numIB) \ + ((numIB) * (sizeof(struct hgsl_fw_ib_desc) >> 2)) + +#define MSG_SEQ_NO_MASK 0xFFF00000 +#define MSG_SEQ_NO_SHIFT 20 +#define MSG_SEQ_NO_GET(x) (((x) & MSG_SEQ_NO_MASK) >> MSG_SEQ_NO_SHIFT) +#define MSG_TYPE_MASK 0x000F0000 +#define MSG_TYPE_SHIFT 16 +#define MSG_TYPE_GET(x) (((x) & MSG_TYPE_MASK) >> MSG_TYPE_SHIFT) +#define MSG_SZ_MASK 0x0000FF00 +#define MSG_SZ_SHIFT 8 +#define MSG_SZ_GET(x) (((x) & MSG_SZ_MASK) >> MSG_SZ_SHIFT) +#define MSG_ID_MASK 0x000000FF +#define MSG_ID_GET(x) ((x) & MSG_ID_MASK) + +#define MAKE_HFI_MSG_HEADER(msgID, msgType, msgSize, msgSeqnum) \ + ((msgID) | ((msgSize) << MSG_SZ_SHIFT) | \ + ((msgType) << MSG_TYPE_SHIFT) | \ + ((msgSeqnum) << MSG_SEQ_NO_SHIFT)) + +#define HFI_ISSUE_IB_HEADER(numIB, sz, msgSeqnum) \ + MAKE_HFI_MSG_HEADER( \ + HTOF_MSG_ISSUE_CMD, \ + HFI_MSG_TYPE_CMD, \ + sz,\ + msgSeqnum) + +/* + * GMU HFI memory allocation options: + * RGS_GMU_HFI_BUFFER_DTCM: Allocated from GMU CM3 DTCM. + * RGS_GMU_HFI_BUFFER_NON_CACHEMEM: POR mode. Allocated from non cached memory. + */ +enum db_buffer_mode_t { + RGS_GMU_HFI_BUFFER_DTCM = 0, + RGS_GMU_HFI_BUFFER_NON_CACHEMEM = 1, + RGS_GMU_HFI_BUFFER_DEFAULT = 1 +}; + +struct db_msg_request { + int msg_has_response; + int msg_has_ret_packet; + int ignore_ret_packet; + void *ptr_data; + unsigned int msg_dwords; +} __packed; + +struct db_msg_response { + void *ptr_data; + unsigned int size_dword; +} __packed; + +/* + * IB start address + * IB size + */ +struct hgsl_fw_ib_desc { + uint64_t addr; + uint32_t sz; +} __packed; + +/* + * Context ID + * cmd_flags + * Per-context user space gsl timestamp. It has to be + * greater than last retired timestamp. + * Number of IB descriptors + * An array of IB descriptors + */ +struct hgsl_db_cmds { + uint32_t header; + uint32_t ctx_id; + uint32_t cmd_flags; + uint32_t timestamp; + uint64_t user_profile_gpuaddr; + uint32_t num_ibs; + uint32_t ib_desc_gmuaddr; + struct hgsl_fw_ib_desc ib_descs[]; +} __packed; + +struct hgsl_db_msg_ret { + uint32_t header; + uint32_t ack; + uint32_t err; +} __packed; + +struct db_msg_id { + uint32_t seq_no; + uint32_t msg_id; +} __packed; + +struct db_wait_retpacket { + size_t event_signal; + int in_use; + struct db_msg_id db_msg_id; + struct db_msg_response response; +} __packed; + +struct db_ignore_retpacket { + int in_use; + struct db_msg_id db_msg_id; +} __packed; + +struct doorbell_queue { + struct dma_buf *dma; + void *vbase; + struct db_buffer data; + uint32_t state; + struct mutex lock; +}; + +struct hgsl_context { + uint32_t context_id; + struct dma_buf *shadow_dma; + void *shadow_vbase; + uint32_t shadow_sop_off; + uint32_t shadow_eop_off; + wait_queue_head_t wait_q; + pid_t pid; + bool dbq_assigned; + + bool in_destroy; + struct kref kref; +}; + +struct hgsl_active_wait { + struct list_head head; + struct hgsl_context *ctxt; + unsigned int timestamp; +}; + +struct qcom_hgsl { + struct device *dev; + struct mutex lock; + + /* character device info */ + struct cdev cdev; + dev_t device_no; + struct class *driver_class; + struct device *class_dev; + + /* registers mapping */ + struct reg reg_ver; + struct reg reg_dbidx; + + atomic_t seq_num; + + struct doorbell_queue dbq[MAX_DB_QUEUE]; + + /* global doorbell tcsr */ + struct hgsl_tcsr *tcsr[HGSL_TCSR_NUM][HGSL_TCSR_ROLE_MAX]; + int tcsr_idx; + struct hgsl_context **contexts; + rwlock_t ctxt_lock; + + struct list_head active_wait_list; + spinlock_t active_wait_lock; + + struct workqueue_struct *wq; + struct work_struct ts_retire_work; + + struct hw_version *ver; +}; + +struct hgsl_priv { + struct qcom_hgsl *dev; + uint32_t dbq_idx; + pid_t pid; +}; + +static int hgsl_reg_map(struct platform_device *pdev, + char *res_name, struct reg *reg); + +static void hgsl_reg_read(struct reg *reg, unsigned int off, + unsigned int *value) +{ + if (reg == NULL) + return; + + if (WARN(off > reg->size, + "Invalid reg read:0x%x, reg size:0x%x\n", + off, reg->size)) + return; + *value = __raw_readl(reg->vaddr + off); + + /* ensure this read finishes before the next one.*/ + rmb(); +} + +static void hgsl_reg_write(struct reg *reg, unsigned int off, + unsigned int value) +{ + if (reg == NULL) + return; + + if (WARN(off > reg->size, + "Invalid reg write:0x%x, reg size:0x%x\n", + off, reg->size)) + return; + + /* + * ensure previous writes post before this one, + * i.e. act like normal writel() + */ + wmb(); + __raw_writel(value, (reg->vaddr + off)); +} + +static inline bool is_global_db(int tcsr_idx) +{ + return (tcsr_idx >= 0); +} + +static void gmu_ring_local_db(struct qcom_hgsl *hgsl, unsigned int value) +{ + hgsl_reg_write(&hgsl->reg_dbidx, 0, value); +} + +static void tcsr_ring_global_db(struct qcom_hgsl *hgsl, uint32_t tcsr_idx, + uint32_t dbq_idx) +{ + hgsl_tcsr_irq_trigger(hgsl->tcsr[tcsr_idx][HGSL_TCSR_ROLE_SENDER], + GLB_DB_SRC_ISSUEIB_IRQ_ID_0 + dbq_idx); +} + +static bool hgsl_ctx_dbq_ready(struct hgsl_priv *priv) +{ + struct qcom_hgsl *hgsl; + struct doorbell_queue *dbq; + + hgsl = priv->dev; + dbq = &hgsl->dbq[priv->dbq_idx]; + + if ((dbq->state & DB_STATE_Q_MASK) == DB_STATE_Q_INIT_DONE) + return true; + + return false; +} + +static uint32_t db_queue_freedwords(struct doorbell_queue *dbq) +{ + uint32_t queue_size; + uint32_t queue_used; + uint32_t wptr; + uint32_t rptr; + + if (dbq == NULL) + return 0; + + wptr = dbq_get_qindex(dbq->vbase, DBQ_WRITE_INDEX_IN_DWORD); + rptr = dbq_get_qindex(dbq->vbase, DBQ_READ_INDEX_IN_DWORD); + + queue_size = dbq->data.dwords; + queue_used = (wptr + queue_size - rptr) % queue_size; + return (queue_size - queue_used - 1); +} + +static int db_queue_wait_freewords(struct doorbell_queue *dbq, + uint32_t size) +{ + unsigned long t = jiffies + HGSL_IDLE_TIMEOUT; + + if (size == 0) + return 0; + + if (dbq == NULL) + return -EINVAL; + + do { + if (db_queue_freedwords(dbq) >= size) + return 0; + } while (time_before(jiffies, t)); + + return -ETIMEDOUT; +} + +static int db_send_msg(struct hgsl_priv *priv, + struct db_msg_id *db_msg_id, + struct db_msg_request *msg_req, + struct db_msg_response *msg_resp) +{ + uint32_t msg_size_align; + int ret; + uint8_t *src, *dst; + uint32_t move_dwords, resid_move_dwords; + uint32_t queue_size_dword; + struct qcom_hgsl *hgsl; + struct doorbell_queue *dbq; + uint32_t wptr; + + hgsl = priv->dev; + dbq = &hgsl->dbq[priv->dbq_idx]; + + mutex_lock(&hgsl->lock); + queue_size_dword = dbq->data.dwords; + msg_size_align = ALIGN(msg_req->msg_dwords, 4); + + ret = db_queue_wait_freewords(dbq, msg_size_align); + if (ret < 0) { + dev_err(hgsl->dev, + "Timed out waiting for queue to free up\n"); + goto quit; + } + + wptr = dbq_get_qindex(dbq->vbase, DBQ_WRITE_INDEX_IN_DWORD); + move_dwords = msg_req->msg_dwords; + if ((msg_req->msg_dwords + wptr) >= queue_size_dword) { + move_dwords = queue_size_dword - wptr; + resid_move_dwords = msg_req->msg_dwords - move_dwords; + dst = (uint8_t *)dbq->data.vaddr; + src = msg_req->ptr_data + (move_dwords << 2); + memcpy(dst, src, (resid_move_dwords << 2)); + } + + dst = dbq->data.vaddr + (wptr << 2); + src = msg_req->ptr_data; + memcpy(dst, src, (move_dwords << 2)); + + wptr = (wptr + msg_size_align) % queue_size_dword; + dbq_set_qindex((uint32_t *)dbq->vbase, + DBQ_WRITE_INDEX_IN_DWORD, + wptr); + + dbq_update_ctxt_info((uint32_t *)dbq->vbase, + ((struct hgsl_db_cmds *)src)->ctx_id, + HGSL_DBQ_METADATA_TIMESTAMP_IN_DWORD, + ((struct hgsl_db_cmds *)src)->timestamp); + + /* confirm write to memory done before ring door bell. */ + wmb(); + + if (is_global_db(hgsl->tcsr_idx)) + /* trigger TCSR interrupt for global doorbell */ + tcsr_ring_global_db(hgsl, hgsl->tcsr_idx, priv->dbq_idx); + else + /* trigger GMU interrupt */ + gmu_ring_local_db(hgsl, priv->dbq_idx); + +quit: + mutex_unlock(&hgsl->lock); + return ret; +} + +static int hgsl_db_issue_cmd(struct hgsl_priv *priv, + uint32_t ctx_id, uint32_t num_ibs, + uint32_t gmu_cmd_flags, + uint32_t timestamp, + struct hgsl_fw_ib_desc ib_descs[]) +{ + int ret; + uint32_t msg_dwords; + uint32_t msg_buf_sz; + struct hgsl_db_cmds *cmds; + struct db_msg_request req; + struct db_msg_response resp; + struct db_msg_id db_msg_id; + struct doorbell_queue *dbq; + struct qcom_hgsl *hgsl = priv->dev; + uint32_t seq_num; + + db_msg_id.msg_id = HTOF_MSG_ISSUE_CMD; + seq_num = atomic_inc_return(&hgsl->seq_num); + db_msg_id.seq_no = seq_num; + + dbq = &hgsl->dbq[priv->dbq_idx]; + + msg_dwords = MSG_ISSUE_INF_SZ() + MSG_ISSUE_IBS_SZ(num_ibs); + msg_buf_sz = ALIGN(msg_dwords, 4) << 2; + + if (msg_buf_sz > dbq->data.dwords) { + dev_err(hgsl->dev, "number of IBs exceed\n"); + return -EINVAL; + } + + cmds = kmalloc(msg_buf_sz, GFP_KERNEL); + if (cmds == NULL) + return -ENOMEM; + + cmds->header = HFI_ISSUE_IB_HEADER(num_ibs, + msg_dwords, + db_msg_id.seq_no); + cmds->ctx_id = ctx_id; + cmds->num_ibs = num_ibs; + cmds->cmd_flags = gmu_cmd_flags; + cmds->timestamp = timestamp; + memcpy(cmds->ib_descs, ib_descs, sizeof(ib_descs[0]) * num_ibs); + + req.msg_has_response = 0; + req.msg_has_ret_packet = 0; + req.ignore_ret_packet = 1; + req.msg_dwords = msg_dwords; + req.ptr_data = cmds; + + ret = db_send_msg(priv, &db_msg_id, &req, &resp); + + kfree(cmds); + return ret; +} + +#define USRPTR(a) u64_to_user_ptr((uint64_t)(a)) + +static int hgsl_cmdstream_db_issueib(struct file *filep, + unsigned long arg) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct hgsl_ibdesc *ibs; + struct hgsl_mem_object *bos; + struct hgsl_fw_ib_desc *fw_ib_list = NULL; + struct hgsl_fhi_issud_cmds submit_info; + struct hgsl_fhi_issud_cmds *submit; + uint64_t submit_size; + uint32_t gmu_flags = CMDBATCH_NOTIFY; + int idx; + int ret = 0; + + copy_from_user(&submit_info, USRPTR(arg), sizeof(submit_info)); + + if (!hgsl_ctx_dbq_ready(priv)) { + dev_err(hgsl->dev, "Doorbell invalid\n"); + return -EINVAL; + } + + submit_size = sizeof(*submit) + + submit_info.num_ibs * sizeof(ibs[0]) + + submit_info.num_bos * sizeof(bos[0]); + + if (submit_size == 0 || submit_size > U32_MAX) + return -EINVAL; + + submit = kmalloc(submit_size, GFP_KERNEL); + if (submit == NULL) + return -ENOMEM; + + submit->context_id = submit_info.context_id; + submit->timestamp = submit_info.timestamp; + submit->flags = submit_info.flags; + submit->num_ibs = submit_info.num_ibs; + submit->num_bos = submit_info.num_bos; + bos = (void *)(submit) + sizeof(*submit); + ibs = (void *)&(bos[submit->num_bos]); + + if (submit->num_ibs) { + ret = copy_from_user(ibs, USRPTR(submit_info.ibs), + (sizeof(ibs[0]) * submit->num_ibs)); + if (ret) { + ret = -EFAULT; + goto exit; + } + + fw_ib_list = kmalloc((sizeof(*fw_ib_list) * submit->num_ibs), + GFP_KERNEL); + if (fw_ib_list == NULL) { + ret = -ENOMEM; + goto exit; + } + } + + if (submit->num_bos) { + ret = copy_from_user(bos, USRPTR(submit_info.bos), + (sizeof(bos[0]) * submit->num_bos)); + if (ret) { + ret = -EFAULT; + goto exit; + } + } + + for (idx = 0; idx < submit->num_ibs; ++idx) { + fw_ib_list[idx].addr = ibs[idx].gpuaddr; + fw_ib_list[idx].sz = ibs[idx].sizedwords << 2; + } + + if (submit->num_ibs) + ret = hgsl_db_issue_cmd(priv, submit->context_id, + submit->num_ibs, + gmu_flags, + submit->timestamp, + fw_ib_list); + +exit: + if (ret) { + dev_err(hgsl->dev, + "issueib with ts (%d) from ctxt (%d) failed\n", + submit->timestamp, submit->context_id); + } + + kfree(submit); + kfree(fw_ib_list); + + return ret; +} + +static int hgsl_dbq_get_state(struct file *filep, + unsigned long arg) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct doorbell_queue *dbq = &hgsl->dbq[priv->dbq_idx]; + uint32_t state; + + state = dbq->state & DB_STATE_Q_MASK; + + if (copy_to_user(USRPTR(arg), &state, sizeof(state))) + return -EFAULT; + + return 0; +} + +static void hgsl_reset_dbq(struct doorbell_queue *dbq) +{ + if (dbq->dma) { + dma_buf_end_cpu_access(dbq->dma, + DMA_BIDIRECTIONAL); + if (dbq->vbase) { + dma_buf_vunmap(dbq->dma, dbq->vbase); + dbq->vbase = NULL; + } + dma_buf_put(dbq->dma); + dbq->dma = NULL; + } + + dbq->state = DB_STATE_Q_UNINIT; +} + +static int hgsl_dbq_assign(struct file *filep, unsigned long arg) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct doorbell_queue *dbq; + unsigned int dbq_idx; + + if (copy_from_user(&dbq_idx, USRPTR(arg), sizeof(dbq_idx))) + return -EFAULT; + + if (dbq_idx > MAX_DB_QUEUE) + return -EINVAL; + + priv->dbq_idx = dbq_idx; + dbq = &hgsl->dbq[priv->dbq_idx]; + + return dbq->state; +} + +static inline bool _timestamp_retired(struct hgsl_context *ctxt, + unsigned int timestamp) +{ + unsigned int ts = *(unsigned int *)(ctxt->shadow_vbase + + ctxt->shadow_eop_off); + + /* ensure read is done before comparison */ + rmb(); + return (ts >= timestamp); +} + +static irqreturn_t hgsl_tcsr_isr(struct device *dev, uint32_t status) +{ + struct platform_device *pdev = to_platform_device(dev); + struct qcom_hgsl *hgsl = platform_get_drvdata(pdev); + + if ((status & GLB_DB_DEST_TS_RETIRE_IRQ_MASK) == 0) + return IRQ_NONE; + + queue_work(hgsl->wq, &hgsl->ts_retire_work); + + return IRQ_HANDLED; +} + +static void ts_retire_worker(struct work_struct *work) +{ + struct qcom_hgsl *hgsl = + container_of(work, struct qcom_hgsl, ts_retire_work); + struct hgsl_active_wait *wait, *w; + + spin_lock(&hgsl->active_wait_lock); + list_for_each_entry_safe(wait, w, &hgsl->active_wait_list, head) { + if (_timestamp_retired(wait->ctxt, wait->timestamp)) + wake_up_all(&wait->ctxt->wait_q); + } + spin_unlock(&hgsl->active_wait_lock); +} + +static int hgsl_init_global_db(struct qcom_hgsl *hgsl, + enum hgsl_tcsr_role role, int idx) +{ + struct device *dev = hgsl->dev; + struct device_node *np = dev->of_node; + bool is_sender = (role == HGSL_TCSR_ROLE_SENDER); + const char *node_name = is_sender ? "qcom,glb-db-senders" : + "qcom,glb-db-receivers"; + struct device_node *tcsr_np; + struct platform_device *tcsr_pdev; + struct hgsl_tcsr *tcsr; + int ret; + + if (hgsl->tcsr[idx][role] != NULL) + return 0; + + tcsr_np = of_parse_phandle(np, node_name, idx); + if (IS_ERR_OR_NULL(tcsr_np)) { + dev_err(dev, "failed to find %s node\n", node_name); + ret = -ENODEV; + goto fail; + } + + tcsr_pdev = of_find_device_by_node(tcsr_np); + if (IS_ERR_OR_NULL(tcsr_pdev)) { + dev_err(dev, + "failed to find %s tcsr dev from node\n", + is_sender ? "sender" : "receiver"); + ret = -ENODEV; + goto fail; + } + + tcsr = hgsl_tcsr_request(tcsr_pdev, role, dev, + is_sender ? NULL : hgsl_tcsr_isr); + if (IS_ERR_OR_NULL(tcsr)) { + dev_err(dev, + "failed to request %s tcsr, ret %lx\n", + is_sender ? "sender" : "receiver", PTR_ERR(tcsr)); + ret = tcsr ? PTR_ERR(tcsr) : -ENODEV; + goto fail; + } + + ret = hgsl_tcsr_enable(tcsr); + if (ret) { + dev_err(dev, + "failed to enable %s tcsr, ret %d\n", + is_sender ? "sender" : "receiver", ret); + goto free_tcsr; + } + + if (!is_sender) { + hgsl->contexts = kzalloc(sizeof(struct hgsl_context *) * + HGSL_CONTEXT_NUM, GFP_KERNEL); + if (!hgsl->contexts) { + ret = -ENOMEM; + goto disable_tcsr; + } + + hgsl->wq = create_workqueue("hgsl-wq"); + if (IS_ERR_OR_NULL(hgsl->wq)) { + dev_err(dev, "failed to create workqueue\n"); + ret = PTR_ERR(hgsl->wq); + goto free_contexts; + } + INIT_WORK(&hgsl->ts_retire_work, ts_retire_worker); + + INIT_LIST_HEAD(&hgsl->active_wait_list); + spin_lock_init(&hgsl->active_wait_lock); + rwlock_init(&hgsl->ctxt_lock); + + hgsl_tcsr_irq_enable(tcsr, GLB_DB_DEST_TS_RETIRE_IRQ_MASK, + true); + } + + hgsl->tcsr[idx][role] = tcsr; + return 0; + +free_contexts: + kfree(hgsl->contexts); + hgsl->contexts = NULL; +disable_tcsr: + hgsl_tcsr_disable(tcsr); +free_tcsr: + hgsl_tcsr_free(tcsr); +fail: + return ret; +} + +static int hgsl_init_local_db(struct qcom_hgsl *hgsl) +{ + struct platform_device *pdev = to_platform_device(hgsl->dev); + + if (hgsl->reg_dbidx.vaddr != NULL) + return 0; + else + return hgsl_reg_map(pdev, IORESOURCE_GMUCX, &hgsl->reg_dbidx); +} + +static int hgsl_init_db_signal(struct qcom_hgsl *hgsl, int tcsr_idx) +{ + int ret; + + if (is_global_db(tcsr_idx)) { + ret = hgsl_init_global_db(hgsl, HGSL_TCSR_ROLE_SENDER, + tcsr_idx); + ret |= hgsl_init_global_db(hgsl, HGSL_TCSR_ROLE_RECEIVER, + tcsr_idx); + } else { + ret = hgsl_init_local_db(hgsl); + } + + return ret; +} + +static int hgsl_dbq_init(struct file *filep, unsigned long arg) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct dma_buf *dmabuf; + struct doorbell_queue *dbq; + struct hgsl_db_queue_inf param; + int tcsr_idx; + int ret; + + copy_from_user(¶m, USRPTR(arg), sizeof(param)); + if (param.fd < 0) { + dev_err(hgsl->dev, "Invalid dbq fd\n"); + return -EINVAL; + } + + if (param.head_off_dwords > INT_MAX || + param.queue_off_dwords > INT_MAX) { + dev_err(hgsl->dev, "Invalid dbq offset\n"); + return -EINVAL; + } + + if ((param.db_signal <= DB_SIGNAL_INVALID) || + (param.db_signal > DB_SIGNAL_MAX)) { + dev_err(hgsl->dev, "Invalid db signal %d\n", param.db_signal); + return -EINVAL; + } + + dbq = &hgsl->dbq[priv->dbq_idx]; + mutex_lock(&dbq->lock); + if (dbq->state == DB_STATE_Q_INIT_DONE) { + mutex_unlock(&dbq->lock); + return 0; + } + + dbq->state = DB_STATE_Q_FAULT; + dmabuf = dma_buf_get(param.fd); + if (IS_ERR_OR_NULL(dmabuf)) { + dev_err(hgsl->dev, "Import DBQ buffer fail\n"); + ret = -EFAULT; + goto err; + } + dbq->dma = dmabuf; + + dma_buf_begin_cpu_access(dbq->dma, DMA_BIDIRECTIONAL); + dbq->vbase = dma_buf_vmap(dbq->dma); + if (dbq->vbase == NULL) { + dma_buf_put(dmabuf); + ret = -EFAULT; + goto err; + } + WARN_ON(param.head_dwords < 2); + + dbq_set_qindex((uint32_t *)dbq->vbase, + DBQ_WRITE_INDEX_IN_DWORD, 0); + + dbq_set_qindex((uint32_t *)dbq->vbase, + DBQ_READ_INDEX_IN_DWORD, 0); + + dbq->data.vaddr = dbq->vbase + (param.queue_off_dwords << 2); + dbq->data.dwords = param.queue_dwords; + + tcsr_idx = (param.db_signal != DB_SIGNAL_LOCAL) ? + param.db_signal - DB_SIGNAL_GLOBAL_0 : -1; + ret = hgsl_init_db_signal(hgsl, tcsr_idx); + if (ret != 0) { + dev_err(hgsl->dev, "failed to init dbq signal %d, idx %d\n", + param.db_signal, priv->dbq_idx); + goto err; + } + + hgsl->tcsr_idx = tcsr_idx; + dbq->state = DB_STATE_Q_INIT_DONE; + + mutex_unlock(&dbq->lock); + return 0; +err: + hgsl_reset_dbq(dbq); + mutex_unlock(&dbq->lock); + + return ret; +} + +static inline void _destroy_context(struct kref *kref) +{ + struct hgsl_context *ctxt = + container_of(kref, struct hgsl_context, kref); + + dma_buf_vunmap(ctxt->shadow_dma, ctxt->shadow_vbase); + dma_buf_end_cpu_access(ctxt->shadow_dma, DMA_FROM_DEVICE); + dma_buf_put(ctxt->shadow_dma); + kfree(ctxt); +} + +static int hgsl_context_create(struct file *filep, unsigned long arg) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct hgsl_ctxt_create_info param; + struct dma_buf *dmabuf; + void *vbase; + struct hgsl_context *ctxt; + int ret; + + if (!is_global_db(hgsl->tcsr_idx)) { + dev_err(hgsl->dev, "Global doorbell not supported for this process\n"); + return -ENODEV; + } + + copy_from_user(¶m, USRPTR(arg), sizeof(param)); + if (param.shadow_fd < 0) { + dev_err(hgsl->dev, "Invalid shadow fd %d\n", + param.shadow_fd); + return -EBADF; + } + + if (param.context_id >= HGSL_CONTEXT_NUM) { + dev_err(hgsl->dev, "Invalid context id %d\n", + param.context_id); + return -EINVAL; + } + + dmabuf = dma_buf_get(param.shadow_fd); + if (IS_ERR_OR_NULL(dmabuf)) { + dev_err(hgsl->dev, "Import shadow buffer fail\n"); + return -EFAULT; + } + + dma_buf_begin_cpu_access(dmabuf, DMA_FROM_DEVICE); + vbase = dma_buf_vmap(dmabuf); + if (vbase == NULL) { + ret = -EFAULT; + goto err_dma_put; + } + + ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); + if (ctxt == NULL) { + ret = -ENOMEM; + goto err_dma_unmap; + } + + ctxt->context_id = param.context_id; + ctxt->shadow_dma = dmabuf; + ctxt->shadow_vbase = vbase; + ctxt->shadow_sop_off = param.shadow_sop_offset; + ctxt->shadow_eop_off = param.shadow_eop_offset; + init_waitqueue_head(&ctxt->wait_q); + + write_lock(&hgsl->ctxt_lock); + if (hgsl->contexts[param.context_id] != NULL) { + dev_err(hgsl->dev, + "context id %d already created\n", + param.context_id); + ret = -EBUSY; + goto err_unlock; + } + + hgsl->contexts[param.context_id] = ctxt; + + priv->pid = task_pid_nr(current); + + hgsl->contexts[param.context_id]->pid = priv->pid; + hgsl->contexts[param.context_id]->dbq_assigned = true; + + kref_init(&ctxt->kref); + write_unlock(&hgsl->ctxt_lock); + + return 0; + +err_unlock: + write_unlock(&hgsl->ctxt_lock); + kfree(ctxt); +err_dma_unmap: + dma_buf_vunmap(dmabuf, vbase); +err_dma_put: + dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL); + dma_buf_put(dmabuf); + + return ret; +} + +static int hgsl_context_destroy(struct file *filep, unsigned long arg, + bool force_cleanup) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + uint32_t context_id; + struct hgsl_context *ctxt; + + if (!is_global_db(hgsl->tcsr_idx)) { + dev_err(hgsl->dev, "Global doorbell not supported for this process\n"); + return -ENODEV; + } + + if (!force_cleanup) + copy_from_user(&context_id, USRPTR(arg), + sizeof(context_id)); + else + context_id = *(uint32_t *)arg; + + if (context_id >= HGSL_CONTEXT_NUM) { + dev_err(hgsl->dev, "Invalid context id %d\n", context_id); + return -EINVAL; + } + + write_lock(&hgsl->ctxt_lock); + if (hgsl->contexts[context_id] == NULL) { + write_unlock(&hgsl->ctxt_lock); + dev_err(hgsl->dev, + "context id %d is not created\n", + context_id); + return -EINVAL; + } + + ctxt = hgsl->contexts[context_id]; + hgsl->contexts[context_id] = NULL; + + /* unblock all waiting threads on this context */ + ctxt->in_destroy = true; + wake_up_all(&ctxt->wait_q); + + ctxt->dbq_assigned = false; + + write_unlock(&hgsl->ctxt_lock); + + kref_put(&ctxt->kref, _destroy_context); + return 0; +} + +static int hgsl_wait_timestamp(struct file *filep, unsigned long arg) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct hgsl_wait_ts_info param; + struct hgsl_active_wait *wait; + struct hgsl_context *ctxt; + unsigned int timestamp; + int ret; + + if (!is_global_db(hgsl->tcsr_idx)) { + dev_err(hgsl->dev, "Global doorbell not supported for this process\n"); + return -ENODEV; + } + + copy_from_user(¶m, USRPTR(arg), sizeof(param)); + + if (param.context_id >= HGSL_CONTEXT_NUM) { + dev_err(hgsl->dev, "Invalid context id %d\n", + param.context_id); + return -EINVAL; + } + + timestamp = param.timestamp; + + read_lock(&hgsl->ctxt_lock); + ctxt = hgsl->contexts[param.context_id]; + if (ctxt == NULL) { + read_unlock(&hgsl->ctxt_lock); + dev_err(hgsl->dev, + "context id %d is not created\n", + param.context_id); + return -EINVAL; + } + read_unlock(&hgsl->ctxt_lock); + + if (_timestamp_retired(ctxt, timestamp)) + return 0; + + kref_get(&ctxt->kref); + + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + + wait->ctxt = ctxt; + wait->timestamp = timestamp; + + spin_lock(&hgsl->active_wait_lock); + list_add_tail(&wait->head, &hgsl->active_wait_list); + spin_unlock(&hgsl->active_wait_lock); + + ret = wait_event_interruptible_timeout(ctxt->wait_q, + _timestamp_retired(ctxt, timestamp) || + ctxt->in_destroy, + msecs_to_jiffies(param.timeout)); + if (ret == 0) + ret = -ETIMEDOUT; + else if (ret == -ERESTARTSYS) + /* Let user handle this */ + ret = -EINTR; + else + ret = 0; + + spin_lock(&hgsl->active_wait_lock); + list_del(&wait->head); + spin_unlock(&hgsl->active_wait_lock); + + kfree(wait); + + kref_put(&ctxt->kref, _destroy_context); + + return ret; +} + +static int hgsl_dbq_release(struct file *filep, unsigned long arg, + bool force_cleanup) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct doorbell_queue *dbq; + struct hgsl_dbq_release_info rel_info; + + if (!force_cleanup) + copy_from_user(&rel_info, USRPTR(arg), + sizeof(rel_info)); + else + rel_info = *(struct hgsl_dbq_release_info *)arg; + + dbq = &hgsl->dbq[priv->dbq_idx]; + + dbq_update_ctxt_info(dbq->vbase, + rel_info.ctxt_id, + HGSL_DBQ_METADATA_CTXT_ID_IN_DWORD, + rel_info.ctxt_id); + + dbq_update_ctxt_info(dbq->vbase, + rel_info.ctxt_id, + HGSL_DBQ_METADATA_CTXT_DESTROY_IN_DWORD, + 1); + + rel_info.ref_count = (dbq->state == DB_STATE_Q_INIT_DONE) ? 1 : 0; + + if (!force_cleanup) + copy_to_user(USRPTR(arg), &rel_info, sizeof(rel_info)); + + return priv->dbq_idx; +} + +static int hgsl_open(struct inode *inodep, struct file *filep) +{ + struct hgsl_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL); + struct qcom_hgsl *hgsl = container_of(inodep->i_cdev, + struct qcom_hgsl, cdev); + + if (!priv) + return -ENOMEM; + + priv->dev = hgsl; + filep->private_data = priv; + return 0; +} + +static int hgsl_release(struct inode *inodep, struct file *filep) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + struct hgsl_dbq_release_info rel_info; + int i; + + for (i = 0; i < HGSL_CONTEXT_NUM; i++) { + if ((hgsl->contexts != NULL) && + (hgsl->contexts[i] != NULL) && + (priv->pid == hgsl->contexts[i]->pid)) { + rel_info.ctxt_id = hgsl->contexts[i]->context_id; + if (hgsl->contexts[i]->dbq_assigned) + hgsl_dbq_release(filep, + (unsigned long)&rel_info, + true); + + hgsl_context_destroy(filep, + (unsigned long)&rel_info.ctxt_id, + true); + } + } + + kfree(priv); + return 0; +} + +static ssize_t hgsl_read(struct file *filep, char __user *buf, size_t count, + loff_t *pos) +{ + struct hgsl_priv *priv = filep->private_data; + struct qcom_hgsl *hgsl = priv->dev; + uint32_t version = 0; + uint32_t release = 0; + char buff[100]; + + hgsl_reg_read(&hgsl->reg_ver, 0, &version); + hgsl_reg_read(&hgsl->reg_ver, 4, &release); + snprintf(buff, 100, "gpu HW Version:%x HW Release:%x\n", + version, release); + + return simple_read_from_buffer(buf, count, pos, + buff, strlen(buff) + 1); +} + +static long hgsl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) +{ + int ret; + + switch (cmd) { + case HGSL_IOCTL_ISSUE_CMDS: + ret = hgsl_cmdstream_db_issueib(filep, arg); + break; + case HGSL_IOCTL_DBQ_GETSTATE: + ret = hgsl_dbq_get_state(filep, arg); + break; + case HGSL_IOCTL_DBQ_ASSIGN: + ret = hgsl_dbq_assign(filep, arg); + break; + case HGSL_IOCTL_DBQ_INIT: + ret = hgsl_dbq_init(filep, arg); + break; + case HGSL_IOCTL_DBQ_RELEASE: + ret = hgsl_dbq_release(filep, arg, false); + break; + case HGSL_IOCTL_CTXT_CREATE: + ret = hgsl_context_create(filep, arg); + break; + case HGSL_IOCTL_CTXT_DESTROY: + ret = hgsl_context_destroy(filep, arg, false); + break; + case HGSL_IOCTL_WAIT_TIMESTAMP: + ret = hgsl_wait_timestamp(filep, arg); + break; + default: + ret = -ENOIOCTLCMD; + } + + return ret; +} + +static long hgsl_compat_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + return hgsl_ioctl(filep, cmd, arg); +} + +static const struct file_operations hgsl_fops = { + .owner = THIS_MODULE, + .open = hgsl_open, + .release = hgsl_release, + .read = hgsl_read, + .unlocked_ioctl = hgsl_ioctl, + .compat_ioctl = hgsl_compat_ioctl +}; + +static int qcom_hgsl_register(struct platform_device *pdev, + struct qcom_hgsl *hgsl_dev) +{ + int ret; + + ret = alloc_chrdev_region(&hgsl_dev->device_no, 0, + HGSL_DEV_NUM, + HGSL_DEVICE_NAME); + if (ret < 0) { + dev_err(&pdev->dev, "alloc_chrdev_region failed %d\n", ret); + return ret; + } + + hgsl_dev->driver_class = class_create(THIS_MODULE, HGSL_DEVICE_NAME); + if (IS_ERR(hgsl_dev->driver_class)) { + ret = -ENOMEM; + dev_err(&pdev->dev, "class_create failed %d\n", ret); + goto exit_unreg_chrdev_region; + } + + hgsl_dev->class_dev = device_create(hgsl_dev->driver_class, + NULL, + hgsl_dev->device_no, + hgsl_dev, HGSL_DEVICE_NAME); + + if (IS_ERR(hgsl_dev->class_dev)) { + dev_err(&pdev->dev, "class_device_create failed %d\n", ret); + ret = -ENOMEM; + goto exit_destroy_class; + } + + cdev_init(&hgsl_dev->cdev, &hgsl_fops); + + hgsl_dev->cdev.owner = THIS_MODULE; + + ret = cdev_add(&hgsl_dev->cdev, + MKDEV(MAJOR(hgsl_dev->device_no), 0), + 1); + if (ret < 0) { + dev_err(&pdev->dev, "cdev_add failed %d\n", ret); + goto exit_destroy_device; + } + + return 0; + +exit_destroy_device: + device_destroy(hgsl_dev->driver_class, hgsl_dev->device_no); +exit_destroy_class: + class_destroy(hgsl_dev->driver_class); +exit_unreg_chrdev_region: + unregister_chrdev_region(hgsl_dev->device_no, 1); + return ret; +} + +static void qcom_hgsl_deregister(struct platform_device *pdev) +{ + struct qcom_hgsl *hgsl_dev = platform_get_drvdata(pdev); + + cdev_del(&hgsl_dev->cdev); + device_destroy(hgsl_dev->driver_class, hgsl_dev->device_no); + class_destroy(hgsl_dev->driver_class); + unregister_chrdev_region(hgsl_dev->device_no, HGSL_DEV_NUM); +} + +static int hgsl_reg_map(struct platform_device *pdev, + char *res_name, struct reg *reg) +{ + struct resource *res; + int ret = 0; + + if ((pdev == NULL) || (res_name == NULL) || (reg == NULL)) { + ret = -EINVAL; + goto exit; + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + res_name); + if (res == NULL) { + dev_err(&pdev->dev, "get resource :%s failed\n", + res_name); + ret = -EINVAL; + goto exit; + } + + if (res->start == 0 || resource_size(res) == 0) { + dev_err(&pdev->dev, "Register region %s is invalid\n", + res_name); + ret = -EINVAL; + goto exit; + } + + reg->paddr = res->start; + reg->size = resource_size(res); + if (devm_request_mem_region(&pdev->dev, + reg->paddr, reg->size, + res_name) == NULL) { + dev_err(&pdev->dev, "request_mem_region for %s failed\n", + res_name); + ret = -ENODEV; + goto exit; + } + + reg->vaddr = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (reg->vaddr == NULL) { + dev_err(&pdev->dev, "Unable to remap %s registers\n", + res_name); + ret = -ENODEV; + goto exit; + } + +exit: + return ret; +} + +static int qcom_hgsl_probe(struct platform_device *pdev) +{ + struct qcom_hgsl *hgsl_dev; + int ret; + int i; + + hgsl_dev = devm_kzalloc(&pdev->dev, sizeof(*hgsl_dev), GFP_KERNEL); + if (!hgsl_dev) + return -ENOMEM; + + hgsl_dev->dev = &pdev->dev; + mutex_init(&hgsl_dev->lock); + + ret = qcom_hgsl_register(pdev, hgsl_dev); + if (ret < 0) { + dev_err(&pdev->dev, "qcom_hgsl_register failed, ret %d\n", + ret); + return ret; + } + + ret = hgsl_reg_map(pdev, IORESOURCE_HWINF, &hgsl_dev->reg_ver); + if (ret < 0) { + dev_err(&pdev->dev, "Unable to map resource:%s\n", + IORESOURCE_HWINF); + goto exit_dereg; + } + + for (i = 0; i < MAX_DB_QUEUE; i++) { + mutex_init(&hgsl_dev->dbq[i].lock); + hgsl_dev->dbq[i].state = DB_STATE_Q_UNINIT; + } + atomic_set(&hgsl_dev->seq_num, 0); + + platform_set_drvdata(pdev, hgsl_dev); + + return 0; + +exit_dereg: + qcom_hgsl_deregister(pdev); + return ret; +} + +static int qcom_hgsl_remove(struct platform_device *pdev) +{ + struct qcom_hgsl *hgsl = platform_get_drvdata(pdev); + struct hgsl_tcsr *tcsr_sender, *tcsr_receiver; + int i; + + for (i = 0; i < HGSL_TCSR_NUM; i++) { + tcsr_sender = hgsl->tcsr[i][HGSL_TCSR_ROLE_SENDER]; + tcsr_receiver = hgsl->tcsr[i][HGSL_TCSR_ROLE_RECEIVER]; + + if (tcsr_sender) { + hgsl_tcsr_disable(tcsr_sender); + hgsl_tcsr_free(tcsr_sender); + } + + if (tcsr_receiver) { + hgsl_tcsr_disable(tcsr_receiver); + hgsl_tcsr_free(tcsr_receiver); + flush_workqueue(hgsl->wq); + destroy_workqueue(hgsl->wq); + kfree(hgsl->contexts); + hgsl->contexts = NULL; + } + } + + memset(hgsl->tcsr, 0, sizeof(hgsl->tcsr)); + + for (i = 0; i < MAX_DB_QUEUE; i++) + if (hgsl->dbq[i].state == DB_STATE_Q_INIT_DONE) + hgsl_reset_dbq(&hgsl->dbq[i]); + + qcom_hgsl_deregister(pdev); + return 0; +} + +static const struct of_device_id qcom_hgsl_of_match[] = { + { .compatible = "qcom,hgsl" }, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_hgsl_of_match); + +static struct platform_driver qcom_hgsl_driver = { + .probe = qcom_hgsl_probe, + .remove = qcom_hgsl_remove, + .driver = { + .name = "qcom-hgsl", + .of_match_table = qcom_hgsl_of_match, + }, +}; +module_platform_driver(qcom_hgsl_driver); + +MODULE_DESCRIPTION("QTI Hypervisor Graphics system driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/soc/qcom/hgsl/hgsl_tcsr.c b/drivers/soc/qcom/hgsl/hgsl_tcsr.c new file mode 100644 index 000000000000..02fb07f032a7 --- /dev/null +++ b/drivers/soc/qcom/hgsl/hgsl_tcsr.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "hgsl_tcsr.h" + +/* Sender registers */ +#define TCSR_GLB_CFG_COMPUTE_SIGNALING_REG 0x000 +#define TCSR_COMPUTE_SIGNALING_REG 0x000 + +/* Receiver registers */ +#define TCSR_COMPUTE_SIGNAL_STATUS_REG 0x000 +#define TCSR_COMPUTE_SIGNAL_CLEAR_REG 0x400 +#define TCSR_COMPUTE_SIGNAL_MASK_REG 0x800 + +struct hgsl_tcsr { + struct platform_device *pdev; + struct device *client_dev; + + struct regmap *regmap; + struct regmap *glb_regmap; + + enum hgsl_tcsr_role role; + unsigned int enable_count; + struct mutex dev_mutex; + + unsigned int irq_num; + irqreturn_t (*isr)(struct device *dev, u32 status); +}; + +static irqreturn_t hgsl_tcsr_isr(int irq, void *ptr) +{ + struct hgsl_tcsr *tcsr = ptr; + u32 status; + + regmap_read(tcsr->regmap, TCSR_COMPUTE_SIGNAL_STATUS_REG, &status); + regmap_write(tcsr->regmap, TCSR_COMPUTE_SIGNAL_CLEAR_REG, status); + + return tcsr->isr(tcsr->client_dev, status); +} + +static int hgsl_tcsr_init_sender(struct hgsl_tcsr *tcsr) +{ + struct device *dev = &tcsr->pdev->dev; + struct device_node *np = dev->of_node; + + if (tcsr->glb_regmap != NULL) + return 0; + + tcsr->regmap = syscon_regmap_lookup_by_phandle(np, "syscon"); + if (IS_ERR_OR_NULL(tcsr->regmap)) { + dev_err(dev, "failed to map sender register\n"); + return -ENODEV; + } + + tcsr->glb_regmap = syscon_regmap_lookup_by_phandle(np, "syscon-glb"); + if (IS_ERR_OR_NULL(tcsr->glb_regmap)) { + dev_err(dev, "failed to map sender global register\n"); + tcsr->glb_regmap = NULL; + return -ENODEV; + } + + return 0; +} + +static int hgsl_tcsr_init_receiver(struct hgsl_tcsr *tcsr) +{ + struct device *dev = &tcsr->pdev->dev; + struct device_node *np = dev->of_node; + int ret; + + if (tcsr->irq_num != 0) + return 0; + + tcsr->regmap = syscon_regmap_lookup_by_phandle(np, "syscon"); + if (IS_ERR_OR_NULL(tcsr->regmap)) { + dev_err(dev, "failed to map receiver register\n"); + return -ENODEV; + } + + tcsr->irq_num = irq_of_parse_and_map(np, 0); + if (tcsr->irq_num == 0) { + dev_err(dev, "failed to get irq\n"); + return -ENODEV; + } + + ret = devm_request_irq(dev, tcsr->irq_num, hgsl_tcsr_isr, + IRQF_TRIGGER_HIGH, "hgsl-tcsr", tcsr); + if (ret < 0) { + dev_err(dev, "failed to request IRQ%u: %d\n", + tcsr->irq_num, ret); + tcsr->irq_num = 0; + return ret; + } + + disable_irq(tcsr->irq_num); + + return 0; +} + +struct hgsl_tcsr *hgsl_tcsr_request(struct platform_device *pdev, + enum hgsl_tcsr_role role, + struct device *client, + irqreturn_t (*isr)(struct device *, u32)) +{ + struct hgsl_tcsr *tcsr = platform_get_drvdata(pdev); + int ret; + + if (!tcsr) + return ERR_PTR(-ENODEV); + else if (tcsr->role != role) + return ERR_PTR(-EINVAL); + else if (tcsr->client_dev) + return ERR_PTR(-EBUSY); + + if (role == HGSL_TCSR_ROLE_RECEIVER) { + if (tcsr->isr) + return ERR_PTR(-EBUSY); + else if (!isr) + return ERR_PTR(-EINVAL); + + ret = hgsl_tcsr_init_receiver(tcsr); + } else { /* HGSL_TCSR_ROLE_SENDER */ + if (isr) + return ERR_PTR(-EINVAL); + + ret = hgsl_tcsr_init_sender(tcsr); + } + + if (ret == 0) { + tcsr->client_dev = client; + tcsr->isr = isr; + } + + return tcsr; +} + +void hgsl_tcsr_free(struct hgsl_tcsr *tcsr) +{ + tcsr->client_dev = NULL; + tcsr->isr = NULL; +} + +int hgsl_tcsr_enable(struct hgsl_tcsr *tcsr) +{ + mutex_lock(&tcsr->dev_mutex); + if (tcsr->enable_count > 0) + goto done; + + if (tcsr->irq_num) + enable_irq(tcsr->irq_num); + +done: + tcsr->enable_count++; + mutex_unlock(&tcsr->dev_mutex); + return 0; +} + +void hgsl_tcsr_disable(struct hgsl_tcsr *tcsr) +{ + mutex_lock(&tcsr->dev_mutex); + if (--tcsr->enable_count > 0) + goto done; + + if (tcsr->irq_num) + disable_irq(tcsr->irq_num); + +done: + mutex_unlock(&tcsr->dev_mutex); +} + +bool hgsl_tcsr_is_enabled(struct hgsl_tcsr *tcsr) +{ + return (tcsr->enable_count > 0); +} + +void hgsl_tcsr_irq_trigger(struct hgsl_tcsr *tcsr, int irq_id) +{ + u32 reg; + + /* + * Read back this global config register in case + * it has been modified by others. + */ + regmap_read(tcsr->glb_regmap, + TCSR_GLB_CFG_COMPUTE_SIGNALING_REG, ®); + reg = irq_id << reg; + regmap_write(tcsr->regmap, TCSR_COMPUTE_SIGNALING_REG, reg); +} + +void hgsl_tcsr_irq_enable(struct hgsl_tcsr *tcsr, u32 mask, bool enable) +{ + u32 reg; + + regmap_read(tcsr->regmap, TCSR_COMPUTE_SIGNAL_MASK_REG, ®); + reg = enable ? (reg | mask) : (reg & ~mask); + regmap_write(tcsr->regmap, TCSR_COMPUTE_SIGNAL_MASK_REG, reg); +} + +static const struct of_device_id hgsl_tcsr_match_table[] = { + { .compatible = "qcom,hgsl-tcsr-sender" }, + { .compatible = "qcom,hgsl-tcsr-receiver" }, + {} +}; + +static int hgsl_tcsr_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct hgsl_tcsr *tcsr = devm_kzalloc(dev, sizeof(*tcsr), + GFP_KERNEL); + + if (!tcsr) + return -ENOMEM; + + if (of_device_is_compatible(np, "qcom,hgsl-tcsr-receiver")) { + tcsr->role = HGSL_TCSR_ROLE_RECEIVER; + } else if (of_device_is_compatible(np, "qcom,hgsl-tcsr-sender")) { + tcsr->role = HGSL_TCSR_ROLE_SENDER; + } else { + dev_err(dev, "Not compatible device\n"); + return -ENODEV; + } + + mutex_init(&tcsr->dev_mutex); + tcsr->pdev = pdev; + platform_set_drvdata(pdev, tcsr); + + return 0; +} + +static int hgsl_tcsr_remove(struct platform_device *pdev) +{ + struct hgsl_tcsr *tcsr = platform_get_drvdata(pdev); + + mutex_destroy(&tcsr->dev_mutex); + + return 0; +} + +static struct platform_driver hgsl_tcsr_driver = { + .probe = hgsl_tcsr_probe, + .remove = hgsl_tcsr_remove, + .driver = { + .name = "hgsl-tcsr", + .of_match_table = hgsl_tcsr_match_table, + } +}; + +static int __init hgsl_tcsr_init(void) +{ + return platform_driver_register(&hgsl_tcsr_driver); +} + +static void __exit hgsl_tcsr_exit(void) +{ + platform_driver_unregister(&hgsl_tcsr_driver); +} + +subsys_initcall(hgsl_tcsr_init); +module_exit(hgsl_tcsr_exit); diff --git a/drivers/soc/qcom/hgsl/hgsl_tcsr.h b/drivers/soc/qcom/hgsl/hgsl_tcsr.h new file mode 100644 index 000000000000..1fd550796275 --- /dev/null +++ b/drivers/soc/qcom/hgsl/hgsl_tcsr.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +#ifndef __HGSL_TCSR_H +#define __HGSL_TCSR_H + +#include +#include +#include + +/* + * We need to separate bit slots between KMD and GMU, + * to avoid one processor signaling back to itself. + * GMU should have the same definitions. + * + * Allocated 4 bits for KMD and 2 for GMU. 3 KMD bits + * can be used by submission and 1 GMU bits for TS + * retire. The other 1 bit is reserved for furture use. + */ +#define TCSR_KMD_TRIGGER_IRQ_ID_0 0 +#define TCSR_KMD_TRIGGER_IRQ_ID_1 1 +#define TCSR_KMD_TRIGGER_IRQ_ID_2 2 +#define TCSR_KMD_TRIGGER_IRQ_ID_3 3 +#define TCSR_GMU_TRIGGER_IRQ_ID_0 8 +#define TCSR_GMU_TRIGGER_IRQ_ID_1 9 + +/* Define Source and Destination IRQ for KMD */ +#define TCSR_SRC_IRQ_ID_0 TCSR_KMD_TRIGGER_IRQ_ID_0 +#define TCSR_SRC_IRQ_ID_1 TCSR_KMD_TRIGGER_IRQ_ID_1 +#define TCSR_SRC_IRQ_ID_2 TCSR_KMD_TRIGGER_IRQ_ID_2 +#define TCSR_SRC_IRQ_ID_3 TCSR_KMD_TRIGGER_IRQ_ID_3 +#define TCSR_DEST_IRQ_ID_0 TCSR_GMU_TRIGGER_IRQ_ID_0 +#define TCSR_DEST_IRQ_ID_1 TCSR_GMU_TRIGGER_IRQ_ID_1 + +#define TCSR_DEST_IRQ_MASK_0 (1 << TCSR_DEST_IRQ_ID_0) +#define TCSR_DEST_IRQ_MASK_1 (1 << TCSR_DEST_IRQ_ID_1) + +#define TCSR_DEST_IRQ_MASK (TCSR_DEST_IRQ_MASK_0 | TCSR_DEST_IRQ_MASK_1) + +enum hgsl_tcsr_role { + HGSL_TCSR_ROLE_SENDER = 0, + HGSL_TCSR_ROLE_RECEIVER = 1, + HGSL_TCSR_ROLE_MAX, +}; + +struct hgsl_tcsr; + +#ifdef CONFIG_QCOM_HGSL_TCSR_SIGNAL +struct hgsl_tcsr *hgsl_tcsr_request(struct platform_device *pdev, + enum hgsl_tcsr_role role, + struct device *client, + irqreturn_t (*isr)(struct device *, u32)); +void hgsl_tcsr_free(struct hgsl_tcsr *tcsr); +int hgsl_tcsr_enable(struct hgsl_tcsr *tcsr); +void hgsl_tcsr_disable(struct hgsl_tcsr *tcsr); +bool hgsl_tcsr_is_enabled(struct hgsl_tcsr *tcsr); +void hgsl_tcsr_irq_trigger(struct hgsl_tcsr *tcsr, int irq_id); +void hgsl_tcsr_irq_enable(struct hgsl_tcsr *tcsr, u32 mask, bool enable); +#else +static inline struct hgsl_tcsr *hgsl_tcsr_request(struct platform_device *pdev, + enum hgsl_tcsr_role role, + struct device *client, + irqreturn_t (*isr)(struct device *, u32)) +{ + return NULL; +} + +static inline void hgsl_tcsr_free(struct hgsl_tcsr *tcsr) +{ +} + +static inline int hgsl_tcsr_enable(struct hgsl_tcsr *tcsr) +{ + return -ENODEV; +} + +static inline void hgsl_tcsr_disable(struct hgsl_tcsr *tcsr) +{ +} + +static inline bool hgsl_tcsr_is_enabled(struct hgsl_tcsr *tcsr) +{ + return false; +} + +static inline void hgsl_tcsr_irq_trigger(struct hgsl_tcsr *tcsr, int irq_id) +{ +} + +static inline void hgsl_tcsr_irq_enable(struct hgsl_tcsr *tcsr, u32 mask, + bool enable) +{ +} +#endif + +#endif /* __HGSL_TCSR_H */ + diff --git a/gen_headers_arm.bp b/gen_headers_arm.bp index 82187648f21d..ab81a004cab3 100644 --- a/gen_headers_arm.bp +++ b/gen_headers_arm.bp @@ -644,6 +644,7 @@ gen_headers_out_arm = [ "linux/zorro_ids.h", "linux/hab_ioctl.h", "linux/habmmid.h", + "linux/hgsl.h", "media/msm_cvp_private.h", "media/radio-iris-commands.h", "media/radio-iris.h", diff --git a/gen_headers_arm64.bp b/gen_headers_arm64.bp index 64a93431c11f..6d30f1697e52 100644 --- a/gen_headers_arm64.bp +++ b/gen_headers_arm64.bp @@ -639,6 +639,7 @@ gen_headers_out_arm64 = [ "linux/zorro_ids.h", "linux/hab_ioctl.h", "linux/habmmid.h", + "linux/hgsl.h", "media/msm_cvp_private.h", "media/radio-iris-commands.h", "media/radio-iris.h", diff --git a/gen_headers_arm64_auto.bp b/gen_headers_arm64_auto.bp index 827d6c66fc93..5d22ae5b01e7 100644 --- a/gen_headers_arm64_auto.bp +++ b/gen_headers_arm64_auto.bp @@ -639,6 +639,7 @@ gen_headers_out_arm64 = [ "linux/zorro_ids.h", "linux/hab_ioctl.h", "linux/habmmid.h", + "linux/hgsl.h", "media/msm_cvp_private.h", "media/radio-iris-commands.h", "media/radio-iris.h", diff --git a/gen_headers_arm_auto.bp b/gen_headers_arm_auto.bp index 02ac67e06b8f..e685187745fc 100644 --- a/gen_headers_arm_auto.bp +++ b/gen_headers_arm_auto.bp @@ -644,6 +644,7 @@ gen_headers_out_arm = [ "linux/zorro_ids.h", "linux/hab_ioctl.h", "linux/habmmid.h", + "linux/hgsl.h", "media/msm_cvp_private.h", "media/radio-iris-commands.h", "media/radio-iris.h", diff --git a/include/uapi/linux/hgsl.h b/include/uapi/linux/hgsl.h new file mode 100644 index 000000000000..fda6b1aa6dd0 --- /dev/null +++ b/include/uapi/linux/hgsl.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _UAPI_MSM_HGSL_H +#define _UAPI_MSM_HGSL_H + +#include +#include + +struct hgsl_ibdesc { + __u64 gpuaddr; + __u64 sizedwords; +}; + +struct hgsl_mem_object { + __u64 gpuaddr; + __u64 sizedwords; +}; + +#define HGSL_IOCTL_BASE 'h' +#define HGSL_IORW(n, t) _IOWR(HGSL_IOCTL_BASE, n, t) +#define HGSL_IOW(n, t) _IOW(HGSL_IOCTL_BASE, n, t) + +/** + * return current status of Doorbell system + */ +#define DB_STATE_Q_MASK 0xffff +#define DB_STATE_Q_UNINIT 1 +#define DB_STATE_Q_INIT_DONE 2 +#define DB_STATE_Q_FAULT 3 + +#define HGSL_IOCTL_DBQ_GETSTATE HGSL_IORW(0x01, __s32) + +/** + * struct hgsl_db_queue_inf - initialize Doorbell + * @fd: File descriptor of DB queue + * @head_dwords: Size of DB queue header + * @head_off_dwords: offset of DB queue header + * @queue_dwords: size of DB queue + * @queue_off_dwords: offset of DB queue + * @db_signal: DB signal type, One of DB_SIGNAL_* values + * + **/ +struct hgsl_db_queue_inf { + __s32 fd; + __u32 head_dwords; + __s32 head_off_dwords; + __u32 queue_dwords; + __s32 queue_off_dwords; + __u32 db_signal; +}; + +#define DB_SIGNAL_INVALID 0 +#define DB_SIGNAL_GLOBAL_0 1 +#define DB_SIGNAL_GLOBAL_1 2 +#define DB_SIGNAL_LOCAL 3 +#define DB_SIGNAL_MAX DB_SIGNAL_LOCAL + +#define HGSL_IOCTL_DBQ_INIT \ + HGSL_IORW(0x02, struct hgsl_db_queue_inf) + +/** + * Assign a DB to current instance, input is DB index. + */ +#define HGSL_IOCTL_DBQ_ASSIGN HGSL_IORW(0x03, __u32) + +/** + * struct hgsl_dbq_release_info - release DB queue + * @ref_count; userspace could inform PVM to do cleanup + * if this param is zero + * @ctxt_id: Current context + */ +struct hgsl_dbq_release_info { + __u32 ref_count; + __u32 ctxt_id; +}; + +#define HGSL_IOCTL_DBQ_RELEASE \ + HGSL_IORW(0x04, struct hgsl_dbq_release_info) + + +/** + * struct hgsl_fhi_issud_cmds - submit cmds to DB queue + * @ibs: db commands list + * @num_ibs: Number of db commands + * @bos: buffer objects list + * @num_bos: Number of buffer objects + * @context_id: Current context for these cmds + * @flags: reserved + * @timestamp: Userspace time stamp + */ +struct hgsl_fhi_issud_cmds { + __u64 ibs; + __u32 num_ibs; + __u64 bos; + __u32 num_bos; + __u32 context_id; + __u32 flags; + __u32 timestamp; + __u32 padding; +}; + +#define HGSL_IOCTL_ISSUE_CMDS \ + HGSL_IORW(0x05, struct hgsl_fhi_issud_cmds) + +/** + * struct hgsl_ctxt_create_info - create a DB context + * @context_id: Current context for these cmds + * @shadow_fd: File descriptor of time stamp buffer + * @shadow_sop_offset: Offset for start timestamp written + * by GPU before cmds + * @shadow_eop_offset: Offset for end timestamp written + * by GPU after cmds + */ +struct hgsl_ctxt_create_info { + __u32 context_id; + __s32 shadow_fd; + __u32 shadow_sop_offset; + __u32 shadow_eop_offset; +}; + +#define HGSL_IOCTL_CTXT_CREATE HGSL_IOW(0x10, \ + struct hgsl_ctxt_create_info) + +/** + * Destroy a DB context - param is id of DB context + */ +#define HGSL_IOCTL_CTXT_DESTROY HGSL_IOW(0x11, __u32) + +/** + * struct hgsl_wait_ts_info - wait a time step to be retired + * @timestamp: The user timestamp to wait + * @timeout: Expiry timeout + */ +struct hgsl_wait_ts_info { + __u32 context_id; + __u32 timestamp; + __u32 timeout; + __u32 padding; +}; + +#define HGSL_IOCTL_WAIT_TIMESTAMP \ + HGSL_IOW(0x12, struct hgsl_wait_ts_info) + +#endif /* _UAPI_MSM_HGSL_H */