diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index b2fe72a8f019..f9fef7375f10 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -105,7 +105,6 @@ static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smm #define ARM_MMU500_ACTLR_CPRE (1 << 1) -#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10) #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8) @@ -162,6 +161,8 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) break; case CAVIUM_SMMUV2: return cavium_smmu_impl_init(smmu); + case QCOM_SMMUV500: + return qsmmuv500_impl_init(smmu); default: break; } diff --git a/drivers/iommu/arm-smmu-qcom.c b/drivers/iommu/arm-smmu-qcom.c index 06e5799dcb87..fd23a1fc5072 100644 --- a/drivers/iommu/arm-smmu-qcom.c +++ b/drivers/iommu/arm-smmu-qcom.c @@ -1,10 +1,17 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. */ #include +#include +#include +#include +#include +#include #include +#include +#include #include "arm-smmu.h" @@ -69,6 +76,840 @@ static const struct arm_smmu_impl qcom_smmu_impl = { .reset = qcom_sdm845_smmu500_reset, }; +#define TCU_HW_VERSION_HLOS1 (0x18) + +#define DEBUG_SID_HALT_REG 0x0 +#define DEBUG_SID_HALT_REQ BIT(16) +#define DEBUG_SID_HALT_SID GENMASK(9, 0) + +#define DEBUG_VA_ADDR_REG 0x8 + +#define DEBUG_TXN_TRIGG_REG 0x18 +#define DEBUG_TXN_AXPROT GENMASK(8, 6) +#define DEBUG_TXN_AXCACHE GENMASK(5, 2) +#define DEBUG_TXN_WRITE BIT(1) +#define DEBUG_TXN_AXPROT_PRIV 0x1 +#define DEBUG_TXN_AXPROT_UNPRIV 0x0 +#define DEBUG_TXN_AXPROT_NSEC 0x2 +#define DEBUG_TXN_AXPROT_SEC 0x0 +#define DEBUG_TXN_AXPROT_INST 0x4 +#define DEBUG_TXN_AXPROT_DATA 0x0 +#define DEBUG_TXN_READ (0x0 << 1) +#define DEBUG_TXN_TRIGGER BIT(0) + +#define DEBUG_SR_HALT_ACK_REG 0x20 +#define DEBUG_SR_HALT_ACK_VAL (0x1 << 1) +#define DEBUG_SR_ECATS_RUNNING_VAL (0x1 << 0) + +#define DEBUG_PAR_REG 0x28 +#define DEBUG_PAR_PA GENMASK_ULL(47, 12) +#define DEBUG_PAR_FAULT_VAL BIT(0) + +#define DEBUG_AXUSER_REG 0x30 +#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) +#define DEBUG_AXUSER_CDMID_VAL 255 + +#define TBU_DBG_TIMEOUT_US 100 + +#define TNX_TCR_CNTL 0x130 +#define TNX_TCR_CNTL_TBU_OT_CAPTURE_EN BIT(18) +#define TNX_TCR_CNTL_ALWAYS_CAPTURE BIT(15) +#define TNX_TCR_CNTL_MATCH_MASK_UPD BIT(7) +#define TNX_TCR_CNTL_MATCH_MASK_VALID BIT(6) + +#define CAPTURE1_SNAPSHOT_1 0x138 + +#define TNX_TCR_CNTL_2 0x178 +#define TNX_TCR_CNTL_2_CAP1_VALID BIT(0) + +struct actlr_setting { + struct arm_smmu_smr smr; + u32 actlr; +}; + +struct qsmmuv500_archdata { + struct list_head tbus; + void __iomem *tcu_base; + u32 version; + struct actlr_setting *actlrs; + u32 actlr_tbl_size; + struct work_struct outstanding_tnx_work; + spinlock_t atos_lock; + struct arm_smmu_device smmu; +}; +#define to_qsmmuv500_archdata(smmu) \ + container_of(smmu, struct qsmmuv500_archdata, smmu) + +struct qsmmuv500_group_iommudata { + bool has_actlr; + u32 actlr; +}; +#define to_qsmmuv500_group_iommudata(group) \ + ((struct qsmmuv500_group_iommudata *) \ + (iommu_group_get_iommudata(group))) + +static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( + struct arm_smmu_device *smmu, u32 sid); + +/* + * Provides mutually exclusive access to the registers used by the + * outstanding transaction snapshot feature and the transaction + * snapshot capture feature. + */ +static DEFINE_MUTEX(capture_reg_lock); + +static void qsmmuv500_log_outstanding_transactions(struct work_struct *work) +{ + struct qsmmuv500_tbu_device *tbu = NULL; + u64 outstanding_tnxs; + u64 tcr_cntl_val, res; + struct qsmmuv500_archdata *data = container_of(work, + struct qsmmuv500_archdata, + outstanding_tnx_work); + struct arm_smmu_device *smmu = &data->smmu; + void __iomem *base; + + if (!mutex_trylock(&capture_reg_lock)) { + dev_warn_ratelimited(smmu->dev, + "Tnx snapshot regs in use, not dumping OT tnxs.\n"); + goto bug; + } + + if (arm_smmu_power_on(smmu->pwr)) { + dev_err_ratelimited(smmu->dev, + "%s: Failed to power on SMMU.\n", + __func__); + goto unlock; + } + + list_for_each_entry(tbu, &data->tbus, list) { + if (arm_smmu_power_on(tbu->pwr)) { + dev_err_ratelimited(tbu->dev, + "%s: Failed to power on TBU.\n", + __func__); + continue; + } + base = tbu->base; + + tcr_cntl_val = readq_relaxed(base + TNX_TCR_CNTL); + + /* Write 1 into MATCH_MASK_UPD of TNX_TCR_CNTL */ + writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_MATCH_MASK_UPD, + base + TNX_TCR_CNTL); + + /* + * Simultaneously write 0 into MATCH_MASK_UPD, 0 into + * ALWAYS_CAPTURE, 0 into MATCH_MASK_VALID, and 1 into + * TBU_OT_CAPTURE_EN of TNX_TCR_CNTL + */ + tcr_cntl_val &= ~(TNX_TCR_CNTL_MATCH_MASK_UPD | + TNX_TCR_CNTL_ALWAYS_CAPTURE | + TNX_TCR_CNTL_MATCH_MASK_VALID); + writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, + base + TNX_TCR_CNTL); + + /* Poll for CAPTURE1_VALID to become 1 on TNX_TCR_CNTL_2 */ + if (readq_poll_timeout_atomic(base + TNX_TCR_CNTL_2, res, + res & TNX_TCR_CNTL_2_CAP1_VALID, + 0, TBU_DBG_TIMEOUT_US)) { + dev_err_ratelimited(tbu->dev, + "Timeout on TNX snapshot poll\n"); + goto poll_timeout; + } + + /* Read Register CAPTURE1_SNAPSHOT_1 */ + outstanding_tnxs = readq_relaxed(base + CAPTURE1_SNAPSHOT_1); + dev_err_ratelimited(tbu->dev, + "Outstanding Transaction Bitmap: 0x%llx\n", + outstanding_tnxs); +poll_timeout: + /* Write TBU_OT_CAPTURE_EN to 0 of TNX_TCR_CNTL */ + writeq_relaxed(tcr_cntl_val & ~TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, + tbu->base + TNX_TCR_CNTL); + + arm_smmu_power_off(smmu, tbu->pwr); + } + + arm_smmu_power_off(smmu, smmu->pwr); +unlock: + mutex_unlock(&capture_reg_lock); +bug: + BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); +} + +static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu) +{ + u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress; + u32 tbu_inv_pending = 0, tbu_sync_pending = 0; + u32 tbu_inv_acked = 0, tbu_sync_acked = 0; + u32 tcu_inv_pending = 0, tcu_sync_pending = 0; + u32 tbu_ids = 0; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + int ret; + + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + dev_err_ratelimited(smmu->dev, + "TLB sync timed out -- SMMU may be deadlocked\n"); + + sync_inv_ack = arm_smmu_readl(smmu, + ARM_SMMU_IMPL_DEF0, + ARM_SMMU_STATS_SYNC_INV_TBU_ACK); + ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + + ARM_SMMU_TBU_PWR_STATUS), &tbu_pwr_status); + if (ret) { + dev_err_ratelimited(smmu->dev, + "SCM read of TBU power status fails: %d\n", + ret); + goto out; + } + + ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + + ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR), + &sync_inv_progress); + if (ret) { + dev_err_ratelimited(smmu->dev, + "SCM read of TBU sync/inv prog fails: %d\n", + ret); + goto out; + } + + if (sync_inv_ack) { + tbu_inv_pending = FIELD_GET(TBU_INV_REQ, sync_inv_ack); + tbu_inv_acked = FIELD_GET(TBU_INV_ACK, sync_inv_ack); + tbu_sync_pending = FIELD_GET(TBU_SYNC_REQ, sync_inv_ack); + tbu_sync_acked = FIELD_GET(TBU_SYNC_ACK, sync_inv_ack); + } + + if (tbu_pwr_status) { + if (tbu_sync_pending) + tbu_ids = tbu_pwr_status & ~tbu_sync_acked; + else if (tbu_inv_pending) + tbu_ids = tbu_pwr_status & ~tbu_inv_acked; + } + + tcu_inv_pending = FIELD_GET(TCU_INV_IN_PRGSS, sync_inv_progress); + tcu_sync_pending = FIELD_GET(TCU_SYNC_IN_PRGSS, sync_inv_progress); + + if (__ratelimit(&_rs)) { + unsigned long tbu_id; + + dev_err(smmu->dev, + "TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n", + sync_inv_ack, tbu_pwr_status, sync_inv_progress); + dev_err(smmu->dev, + "TCU invalidation %s, TCU sync %s\n", + tcu_inv_pending?"pending":"completed", + tcu_sync_pending?"pending":"completed"); + + for_each_set_bit(tbu_id, (unsigned long *) &tbu_ids, + sizeof(tbu_ids) * BITS_PER_BYTE) { + + struct qsmmuv500_tbu_device *tbu; + + tbu = qsmmuv500_find_tbu(smmu, + (u16)(tbu_id << TBUID_SHIFT)); + if (tbu) { + dev_err(smmu->dev, + "TBU %s ack pending for TBU %s, %s\n", + tbu_sync_pending?"sync" : "inv", + dev_name(tbu->dev), + tbu_sync_pending ? + "check pending transactions on TBU" + : "check for TBU power status"); + } + } + } + + if (tcu_sync_pending) { + schedule_work(&data->outstanding_tnx_work); + return; + } +out: + BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); +} + +static void qsmmuv500_device_remove(struct arm_smmu_device *smmu) +{ + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + + cancel_work_sync(&data->outstanding_tnx_work); +} + +static bool arm_smmu_fwspec_match_smr(struct iommu_fwspec *fwspec, + struct arm_smmu_smr *smr) +{ + struct arm_smmu_smr *smr2; + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + int i, idx; + + for_each_cfg_sme(fwspec, i, idx) { + smr2 = &smmu->smrs[idx]; + /* Continue if table entry does not match */ + if ((smr->id ^ smr2->id) & ~(smr->mask | smr2->mask)) + continue; + return true; + } + return false; +} + +static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu, + struct arm_smmu_domain *smmu_domain) +{ + unsigned long flags; + struct arm_smmu_device *smmu = smmu_domain->smmu; + int idx = smmu_domain->cfg.cbndx; + u32 halt, fsr, status; + void __iomem *tbu_base; + + if (of_property_read_bool(tbu->dev->of_node, + "qcom,opt-out-tbu-halting")) { + dev_notice(tbu->dev, "TBU opted-out for halting!\n"); + return -EBUSY; + } + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (tbu->halt_count) { + tbu->halt_count++; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return 0; + } + + tbu_base = tbu->base; + halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); + halt |= DEBUG_SID_HALT_REQ; + writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if ((fsr & FSR_FAULT) && (fsr & FSR_SS)) { + u32 sctlr_orig, sctlr; + /* + * We are in a fault; Our request to halt the bus will not + * complete until transactions in front of us (such as the fault + * itself) have completed. Disable iommu faults and terminate + * any existing transactions. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation + */ + wmb(); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + RESUME_TERMINATE); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + } + + if (readl_poll_timeout_atomic(tbu_base + DEBUG_SR_HALT_ACK_REG, status, + (status & DEBUG_SR_HALT_ACK_VAL), + 0, TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "Couldn't halt TBU!\n"); + + halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); + halt &= ~DEBUG_SID_HALT_REQ; + writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); + + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return -ETIMEDOUT; + } + + tbu->halt_count = 1; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return 0; +} + +static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu) +{ + unsigned long flags; + u32 val; + void __iomem *base; + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (!tbu->halt_count) { + WARN(1, "%s: bad tbu->halt_count", dev_name(tbu->dev)); + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return; + + } else if (tbu->halt_count > 1) { + tbu->halt_count--; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return; + } + + base = tbu->base; + val = readl_relaxed(base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_REQ; + writel_relaxed(val, base + DEBUG_SID_HALT_REG); + + tbu->halt_count = 0; + spin_unlock_irqrestore(&tbu->halt_lock, flags); +} + +static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( + struct arm_smmu_device *smmu, u32 sid) +{ + struct qsmmuv500_tbu_device *tbu = NULL; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + + list_for_each_entry(tbu, &data->tbus, list) { + if (tbu->sid_start <= sid && + sid < tbu->sid_start + tbu->num_sids) + return tbu; + } + return NULL; +} + +static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain, + struct qsmmuv500_tbu_device *tbu, + unsigned long *flags) + __acquires(&smmu->atos_lock) +{ + struct arm_smmu_device *smmu = tbu->smmu; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + u32 val; + + spin_lock_irqsave(&data->atos_lock, *flags); + /* The status register is not accessible on version 1.0 */ + if (data->version == 0x01000000) + return 0; + + if (readl_poll_timeout_atomic(tbu->status_reg, + val, (val == 0x1), 0, + TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "ECATS hw busy!\n"); + spin_unlock_irqrestore(&data->atos_lock, *flags); + return -ETIMEDOUT; + } + + return 0; +} + +static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, + struct qsmmuv500_tbu_device *tbu, + unsigned long *flags) + __releases(&smmu->atos_lock) +{ + struct arm_smmu_device *smmu = tbu->smmu; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + + /* The status register is not accessible on version 1.0 */ + if (data->version != 0x01000000) + writel_relaxed(0, tbu->status_reg); + spin_unlock_irqrestore(&data->atos_lock, *flags); +} + +/* + * Zero means failure. + */ +static phys_addr_t qsmmuv500_iova_to_phys( + struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 sid, + unsigned long trans_flags) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct qsmmuv500_tbu_device *tbu; + int ret; + phys_addr_t phys = 0; + u64 val, fsr; + unsigned long flags; + int idx = cfg->cbndx; + u32 sctlr_orig, sctlr; + int needs_redo = 0; + ktime_t timeout; + + /* only 36 bit iova is supported */ + if (iova >= (1ULL << 36)) { + dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n", + &iova); + return 0; + } + + tbu = qsmmuv500_find_tbu(smmu, sid); + if (!tbu) + return 0; + + ret = arm_smmu_power_on(tbu->pwr); + if (ret) + return 0; + + ret = qsmmuv500_tbu_halt(tbu, smmu_domain); + if (ret) + goto out_power_off; + + /* + * ECATS can trigger the fault interrupt, so disable it temporarily + * and check for an interrupt manually. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation. + */ + wmb(); + + /* + * TBU halt takes care of resuming any stalled transcation. + * Kept it here for completeness sake. + */ + if (fsr & FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + RESUME_TERMINATE); + } + + /* Only one concurrent atos operation */ + ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags); + if (ret) + goto out_resume; + +redo: + /* Set address and stream-id */ + val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); + writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG); + val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); + writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); + + /* Write-back Read and Write-Allocate */ + val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF); + + /* Non-secure Access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); + + /* Write or Read Access */ + if (flags & IOMMU_TRANS_WRITE) + val |= DEBUG_TXN_WRITE; + + /* Priviledged or Unpriviledged Access */ + if (flags & IOMMU_TRANS_PRIV) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); + + /* Data or Instruction Access */ + if (flags & IOMMU_TRANS_INST) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST); + + val |= DEBUG_TXN_TRIGGER; + writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); + + ret = 0; + timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); + for (;;) { + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) + break; + val = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (val & FSR_FAULT) + break; + if (ktime_compare(ktime_get(), timeout) > 0) { + dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); + ret = -ETIMEDOUT; + break; + } + } + + val = readq_relaxed(tbu->base + DEBUG_PAR_REG); + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (val & DEBUG_PAR_FAULT_VAL) { + dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx, SID=0x%x\n", + fsr, sid); + + dev_err(tbu->dev, "ECATS translation failed! PAR = %llx\n", + val); + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation. + */ + wmb(); + + if (fsr & FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + RESUME_TERMINATE); + + ret = -EINVAL; + } + + phys = FIELD_GET(DEBUG_PAR_PA, val); + if (ret < 0) + phys = 0; + + /* Reset hardware */ + writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG); + writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG); + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + /* + * After a failed translation, the next successful translation will + * incorrectly be reported as a failure. + */ + if (!phys && needs_redo++ < 2) + goto redo; + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + qsmmuv500_ecats_unlock(smmu_domain, tbu, &flags); + +out_resume: + qsmmuv500_tbu_resume(tbu); + +out_power_off: + /* Read to complete prior write transcations */ + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + + /* Wait for read to complete before off */ + rmb(); + + arm_smmu_power_off(tbu->smmu, tbu->pwr); + + return phys; +} + +static phys_addr_t qsmmuv500_iova_to_phys_hard( + struct arm_smmu_domain *smmu_domain, + dma_addr_t iova, + unsigned long trans_flags) +{ + u16 sid; + struct msm_iommu_domain *msm_domain = &smmu_domain->domain; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct iommu_fwspec *fwspec; + u32 frsynra; + + /* Check to see if the domain is associated with the test + * device. If the domain belongs to the test device, then + * pick the SID from fwspec. + */ + if (msm_domain->is_debug_domain) { + fwspec = dev_iommu_fwspec_get(smmu_domain->dev); + sid = (u16)fwspec->ids[0]; + } else { + + /* If the domain belongs to an actual device, read + * SID from the corresponding frsynra register + */ + frsynra = arm_smmu_gr1_read(smmu, + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); + frsynra &= CBFRSYNRA_SID_MASK; + sid = frsynra; + } + return qsmmuv500_iova_to_phys(smmu_domain, iova, sid, trans_flags); +} + +static void qsmmuv500_release_group_iommudata(void *data) +{ + kfree(data); +} + +/* If a device has a valid actlr, it must match */ +static int qsmmuv500_device_group(struct device *dev, + struct iommu_group *group) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + struct qsmmuv500_group_iommudata *iommudata; + u32 actlr, i; + struct arm_smmu_smr *smr; + + iommudata = to_qsmmuv500_group_iommudata(group); + if (!iommudata) { + iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); + if (!iommudata) + return -ENOMEM; + + iommu_group_set_iommudata(group, iommudata, + qsmmuv500_release_group_iommudata); + } + + for (i = 0; i < data->actlr_tbl_size; i++) { + smr = &data->actlrs[i].smr; + actlr = data->actlrs[i].actlr; + + if (!arm_smmu_fwspec_match_smr(fwspec, smr)) + continue; + + if (!iommudata->has_actlr) { + iommudata->actlr = actlr; + iommudata->has_actlr = true; + } else if (iommudata->actlr != actlr) { + return -EINVAL; + } + } + + return 0; +} + +static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, + struct device *dev) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct qsmmuv500_group_iommudata *iommudata = + to_qsmmuv500_group_iommudata(dev->iommu_group); + int idx = smmu_domain->cfg.cbndx; + const struct iommu_flush_ops *tlb; + + if (!iommudata->has_actlr) + return; + + tlb = smmu_domain->pgtbl_info.pgtbl_cfg.tlb; + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ACTLR, iommudata->actlr); + + /* + * Flush the context bank after modifying ACTLR to ensure there + * are no cache entries with stale state + */ + tlb->tlb_flush_all(smmu_domain); +} + +static int qsmmuv500_tbu_register(struct device *dev, void *cookie) +{ + struct qsmmuv500_tbu_device *tbu; + struct qsmmuv500_archdata *data = cookie; + + if (!dev->driver) { + dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); + return -EINVAL; + } + + tbu = dev_get_drvdata(dev); + + INIT_LIST_HEAD(&tbu->list); + tbu->smmu = &data->smmu; + list_add(&tbu->list, &data->tbus); + return 0; +} + +static int qsmmuv500_read_actlr_tbl(struct qsmmuv500_archdata *data) +{ + int len, i; + struct device *dev = data->smmu.dev; + struct actlr_setting *actlrs; + const __be32 *cell; + + cell = of_get_property(dev->of_node, "qcom,actlr", NULL); + if (!cell) + return 0; + + len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr", + sizeof(u32) * 3); + if (len < 0) + return 0; + + actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL); + if (!actlrs) + return -ENOMEM; + + for (i = 0; i < len; i++) { + actlrs[i].smr.id = of_read_number(cell++, 1); + actlrs[i].smr.mask = of_read_number(cell++, 1); + actlrs[i].actlr = of_read_number(cell++, 1); + } + + data->actlrs = actlrs; + data->actlr_tbl_size = len; + return 0; +} + +static int qsmmuv500_cfg_probe(struct arm_smmu_device *smmu) +{ + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + u32 val; + + data->version = readl_relaxed(data->tcu_base + TCU_HW_VERSION_HLOS1); + + val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); + val &= ~ARM_MMU500_ACR_CACHE_LOCK; + arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, val); + val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); + /* + * Modifiying the nonsecure copy of the sACR register is only + * allowed if permission is given in the secure sACR register. + * Attempt to detect if we were able to update the value. + */ + WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); + + return 0; +} + +struct arm_smmu_arch_ops qsmmuv500_arch_ops = { + .device_group = qsmmuv500_device_group, + .device_remove = qsmmuv500_device_remove, +}; + +static const struct arm_smmu_impl qsmmuv500_impl = { + .cfg_probe = qsmmuv500_cfg_probe, + .init_context_bank = qsmmuv500_init_cb, + .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, + .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, +}; + +struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu) +{ + struct resource *res; + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data; + struct platform_device *pdev; + int ret; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&data->tbus); + + pdev = to_platform_device(dev); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base"); + if (!res) { + dev_err(dev, "Unable to get the tcu-base\n"); + return ERR_PTR(-EINVAL); + } + data->tcu_base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->tcu_base)) + return ERR_CAST(data->tcu_base); + + spin_lock_init(&data->atos_lock); + data->smmu = *smmu; + data->smmu.impl = &qsmmuv500_impl; + + ret = qsmmuv500_read_actlr_tbl(data); + if (ret) + return ERR_PTR(ret); + + INIT_WORK(&data->outstanding_tnx_work, + qsmmuv500_log_outstanding_transactions); + + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + return ERR_PTR(ret); + + /* Attempt to register child devices */ + ret = device_for_each_child(dev, data, qsmmuv500_tbu_register); + if (ret) + return ERR_PTR(-EPROBE_DEFER); + + devm_kfree(smmu->dev, smmu); + return &data->smmu; +} + struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { struct qcom_smmu *qsmmu; diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 664bdfe1a99c..a274ba0c27f7 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -44,12 +44,10 @@ #include #include #include -#include #include #include -#include #include "arm-smmu.h" #include "iommu-logger.h" @@ -103,18 +101,6 @@ struct arm_smmu_cb { struct arm_smmu_cfg *cfg; }; -struct arm_smmu_master_cfg { - struct arm_smmu_device *smmu; - s16 smendx[]; -}; -#define INVALID_SMENDX -1 -#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) -#define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) -#define fwspec_smendx(fw, i) \ - (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) -#define for_each_cfg_sme(fw, i, idx) \ - for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) - #define INVALID_CBNDX 0xff #define INVALID_ASID 0xffff /* @@ -364,74 +350,6 @@ static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) } #endif -/* - * init() - * Hook for additional device tree parsing at probe time. - * - * device_reset() - * Hook for one-time architecture-specific register settings. - * - * iova_to_phys_hard() - * Provides debug information. May be called from the context fault irq handler. - * - * init_context_bank() - * Hook for architecture-specific settings which require knowledge of the - * dynamically allocated context bank number. - * - * device_group() - * Hook for checking whether a device is compatible with a said group. - * - * tlb_sync_timeout() - * Hook for performing architecture-specific procedures to collect additional - * debugging information on a TLB sync timeout. - * - * device_remove() - * Hook for performing architecture-specific procedures prior to powering off - * the SMMU. - */ -struct arm_smmu_arch_ops { - int (*init)(struct arm_smmu_device *smmu); - void (*device_reset)(struct arm_smmu_device *smmu); - phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain, - dma_addr_t iova, - unsigned long trans_flags); - void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, - struct device *dev); - int (*device_group)(struct device *dev, struct iommu_group *group); - void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); - void (*device_remove)(struct arm_smmu_device *smmu); -}; - -static int arm_smmu_arch_init(struct arm_smmu_device *smmu) -{ - if (!smmu->arch_ops) - return 0; - if (!smmu->arch_ops->init) - return 0; - return smmu->arch_ops->init(smmu); -} - -static void arm_smmu_arch_device_reset(struct arm_smmu_device *smmu) -{ - if (!smmu->arch_ops) - return; - if (!smmu->arch_ops->device_reset) - return; - return smmu->arch_ops->device_reset(smmu); -} - -static void arm_smmu_arch_init_context_bank( - struct arm_smmu_domain *smmu_domain, struct device *dev) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - - if (!smmu->arch_ops) - return; - if (!smmu->arch_ops->init_context_bank) - return; - return smmu->arch_ops->init_context_bank(smmu_domain, dev); -} - static int arm_smmu_arch_device_group(struct device *dev, struct iommu_group *group) { @@ -445,15 +363,6 @@ static int arm_smmu_arch_device_group(struct device *dev, return smmu->arch_ops->device_group(dev, group); } -static void arm_smmu_arch_tlb_sync_timeout(struct arm_smmu_device *smmu) -{ - if (!smmu->arch_ops || !smmu->arch_ops->tlb_sync_timeout) - return; - - smmu->arch_ops->tlb_sync_timeout(smmu); -} - - static void arm_smmu_arch_device_remove(struct arm_smmu_device *smmu) { if (!smmu->arch_ops) @@ -839,7 +748,7 @@ static void arm_smmu_power_off_slow(struct arm_smmu_power_resources *pwr) mutex_unlock(&pwr->power_lock); } -static int arm_smmu_power_on(struct arm_smmu_power_resources *pwr) +int arm_smmu_power_on(struct arm_smmu_power_resources *pwr) { int ret; @@ -858,8 +767,8 @@ out_disable: return ret; } -static void arm_smmu_power_off(struct arm_smmu_device *smmu, - struct arm_smmu_power_resources *pwr) +void arm_smmu_power_off(struct arm_smmu_device *smmu, + struct arm_smmu_power_resources *pwr) { arm_smmu_power_off_atomic(smmu, pwr); arm_smmu_power_off_slow(pwr); @@ -936,8 +845,8 @@ static int __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, goto out; trace_tlbsync_timeout(smmu->dev, 0); - arm_smmu_arch_tlb_sync_timeout(smmu); - + if (smmu->impl && smmu->impl->tlb_sync_timeout) + smmu->impl->tlb_sync_timeout(smmu); out: return -EINVAL; } @@ -1888,7 +1797,8 @@ static int arm_smmu_setup_context_bank(struct arm_smmu_domain *smmu_domain, arm_smmu_write_context_bank(smmu, cfg->cbndx, smmu_domain->attributes); - arm_smmu_arch_init_context_bank(smmu_domain, dev); + if (smmu->impl && smmu->impl->init_context_bank) + smmu->impl->init_context_bank(smmu_domain, dev); if (smmu->version < ARM_SMMU_V2) { cfg->irptndx = atomic_inc_return(&smmu->irptndx); @@ -3231,10 +3141,9 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, if (arm_smmu_power_on(smmu_domain->smmu->pwr)) return 0; - if (smmu_domain->smmu->arch_ops && - smmu_domain->smmu->arch_ops->iova_to_phys_hard) { - ret = smmu_domain->smmu->arch_ops->iova_to_phys_hard( - smmu_domain, iova, trans_flags); + if (smmu->impl && smmu->impl->iova_to_phys_hard) { + ret = smmu->impl->iova_to_phys_hard(smmu_domain, iova, + trans_flags); goto out; } @@ -4054,9 +3963,6 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) arm_smmu_tlb_sync_global(smmu); wmb(); arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); - - /* Manage any implementation defined features */ - arm_smmu_arch_device_reset(smmu); } static int arm_smmu_id_size_to_bits(int size) @@ -4574,8 +4480,6 @@ static struct arm_smmu_match_data name = { \ .arch_ops = ops, \ } \ -static struct arm_smmu_arch_ops qsmmuv500_arch_ops; - ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU, NULL); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU, NULL); ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU, NULL); @@ -4738,9 +4642,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) return -ENOMEM; smmu->dev = dev; - spin_lock_init(&smmu->atos_lock); - idr_init(&smmu->asid_idr); - mutex_init(&smmu->idr_mutex); data = of_device_get_match_data(dev); smmu->version = data->version; @@ -4754,6 +4655,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) if (IS_ERR(smmu)) return PTR_ERR(smmu); + idr_init(&smmu->asid_idr); + mutex_init(&smmu->idr_mutex); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (res == NULL) { dev_err(dev, "no MEM resource info\n"); @@ -4850,10 +4754,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } } - err = arm_smmu_arch_init(smmu); - if (err) - goto out_power_off; - iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops.iommu_ops); iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); @@ -4978,882 +4878,6 @@ static const struct dev_pm_ops arm_smmu_pm_ops = { arm_smmu_runtime_resume, NULL) }; -static struct platform_driver arm_smmu_driver = { - .driver = { - .name = "arm-smmu", - .of_match_table = of_match_ptr(arm_smmu_of_match), - .pm = &arm_smmu_pm_ops, - .suppress_bind_attrs = true, - }, - .probe = arm_smmu_device_dt_probe, - .remove = arm_smmu_device_remove, -}; - -static struct platform_driver qsmmuv500_tbu_driver; -static int __init arm_smmu_init(void) -{ - int ret; - ktime_t cur; - - cur = ktime_get(); - ret = platform_driver_register(&qsmmuv500_tbu_driver); - if (ret) - return ret; - - ret = platform_driver_register(&arm_smmu_driver); - if (ret) { - platform_driver_unregister(&qsmmuv500_tbu_driver); - return ret; - } - - trace_smmu_init(ktime_us_delta(ktime_get(), cur)); - return ret; -} - -static void __exit arm_smmu_exit(void) -{ - platform_driver_unregister(&arm_smmu_driver); - platform_driver_unregister(&qsmmuv500_tbu_driver); -} - -subsys_initcall(arm_smmu_init); -module_exit(arm_smmu_exit); - -#define TCU_HW_VERSION_HLOS1 (0x18) - -#define DEBUG_SID_HALT_REG 0x0 -#define DEBUG_SID_HALT_REQ BIT(16) -#define DEBUG_SID_HALT_SID GENMASK(9, 0) - -#define DEBUG_VA_ADDR_REG 0x8 - -#define DEBUG_TXN_TRIGG_REG 0x18 -#define DEBUG_TXN_AXPROT GENMASK(8, 6) -#define DEBUG_TXN_AXCACHE GENMASK(5, 2) -#define DEBUG_TXN_WRITE BIT(1) -#define DEBUG_TXN_AXPROT_PRIV 0x1 -#define DEBUG_TXN_AXPROT_UNPRIV 0x0 -#define DEBUG_TXN_AXPROT_NSEC 0x2 -#define DEBUG_TXN_AXPROT_SEC 0x0 -#define DEBUG_TXN_AXPROT_INST 0x4 -#define DEBUG_TXN_AXPROT_DATA 0x0 -#define DEBUG_TXN_READ (0x0 << 1) -#define DEBUG_TXN_TRIGGER BIT(0) - -#define DEBUG_SR_HALT_ACK_REG 0x20 -#define DEBUG_SR_HALT_ACK_VAL (0x1 << 1) -#define DEBUG_SR_ECATS_RUNNING_VAL (0x1 << 0) - -#define DEBUG_PAR_REG 0x28 -#define DEBUG_PAR_PA GENMASK_ULL(47, 12) -#define DEBUG_PAR_FAULT_VAL BIT(0) - -#define DEBUG_AXUSER_REG 0x30 -#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) -#define DEBUG_AXUSER_CDMID_VAL 255 - -#define TBU_DBG_TIMEOUT_US 100 - -#define TNX_TCR_CNTL 0x130 -#define TNX_TCR_CNTL_TBU_OT_CAPTURE_EN BIT(18) -#define TNX_TCR_CNTL_ALWAYS_CAPTURE BIT(15) -#define TNX_TCR_CNTL_MATCH_MASK_UPD BIT(7) -#define TNX_TCR_CNTL_MATCH_MASK_VALID BIT(6) - -#define CAPTURE1_SNAPSHOT_1 0x138 - -#define TNX_TCR_CNTL_2 0x178 -#define TNX_TCR_CNTL_2_CAP1_VALID BIT(0) - -struct actlr_setting { - struct arm_smmu_smr smr; - u32 actlr; -}; - -struct qsmmuv500_archdata { - struct list_head tbus; - void __iomem *tcu_base; - u32 version; - struct actlr_setting *actlrs; - u32 actlr_tbl_size; - struct work_struct outstanding_tnx_work; - struct arm_smmu_device *smmu; -}; -#define get_qsmmuv500_archdata(smmu) \ - ((struct qsmmuv500_archdata *)(smmu->archdata)) - -struct qsmmuv500_tbu_device { - struct list_head list; - struct device *dev; - struct arm_smmu_device *smmu; - void __iomem *base; - void __iomem *status_reg; - - struct arm_smmu_power_resources *pwr; - u32 sid_start; - u32 num_sids; - - /* Protects halt count */ - spinlock_t halt_lock; - u32 halt_count; -}; - -struct qsmmuv500_group_iommudata { - bool has_actlr; - u32 actlr; -}; -#define to_qsmmuv500_group_iommudata(group) \ - ((struct qsmmuv500_group_iommudata *) \ - (iommu_group_get_iommudata(group))) - -static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( - struct arm_smmu_device *smmu, u32 sid); - -/* - * Provides mutually exclusive access to the registers used by the - * outstanding transaction snapshot feature and the transaction - * snapshot capture feature. - */ -static DEFINE_MUTEX(capture_reg_lock); - -static void qsmmuv500_log_outstanding_transactions(struct work_struct *work) -{ - struct qsmmuv500_tbu_device *tbu = NULL; - u64 outstanding_tnxs; - u64 tcr_cntl_val, res; - struct qsmmuv500_archdata *data = container_of(work, - struct qsmmuv500_archdata, - outstanding_tnx_work); - struct arm_smmu_device *smmu = data->smmu; - void __iomem *base; - - if (!mutex_trylock(&capture_reg_lock)) { - dev_warn_ratelimited(smmu->dev, - "Tnx snapshot regs in use, not dumping OT tnxs.\n"); - goto bug; - } - - if (arm_smmu_power_on(smmu->pwr)) { - dev_err_ratelimited(smmu->dev, - "%s: Failed to power on SMMU.\n", - __func__); - goto unlock; - } - - list_for_each_entry(tbu, &data->tbus, list) { - if (arm_smmu_power_on(tbu->pwr)) { - dev_err_ratelimited(tbu->dev, - "%s: Failed to power on TBU.\n", - __func__); - continue; - } - base = tbu->base; - - tcr_cntl_val = readq_relaxed(base + TNX_TCR_CNTL); - - /* Write 1 into MATCH_MASK_UPD of TNX_TCR_CNTL */ - writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_MATCH_MASK_UPD, - base + TNX_TCR_CNTL); - - /* - * Simultaneously write 0 into MATCH_MASK_UPD, 0 into - * ALWAYS_CAPTURE, 0 into MATCH_MASK_VALID, and 1 into - * TBU_OT_CAPTURE_EN of TNX_TCR_CNTL - */ - tcr_cntl_val &= ~(TNX_TCR_CNTL_MATCH_MASK_UPD | - TNX_TCR_CNTL_ALWAYS_CAPTURE | - TNX_TCR_CNTL_MATCH_MASK_VALID); - writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, - base + TNX_TCR_CNTL); - - /* Poll for CAPTURE1_VALID to become 1 on TNX_TCR_CNTL_2 */ - if (readq_poll_timeout_atomic(base + TNX_TCR_CNTL_2, res, - res & TNX_TCR_CNTL_2_CAP1_VALID, - 0, TBU_DBG_TIMEOUT_US)) { - dev_err_ratelimited(tbu->dev, - "Timeout on TNX snapshot poll\n"); - goto poll_timeout; - } - - /* Read Register CAPTURE1_SNAPSHOT_1 */ - outstanding_tnxs = readq_relaxed(base + CAPTURE1_SNAPSHOT_1); - dev_err_ratelimited(tbu->dev, - "Outstanding Transaction Bitmap: 0x%llx\n", - outstanding_tnxs); -poll_timeout: - /* Write TBU_OT_CAPTURE_EN to 0 of TNX_TCR_CNTL */ - writeq_relaxed(tcr_cntl_val & ~TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, - tbu->base + TNX_TCR_CNTL); - - arm_smmu_power_off(smmu, tbu->pwr); - } - - arm_smmu_power_off(smmu, smmu->pwr); -unlock: - mutex_unlock(&capture_reg_lock); -bug: - BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); -} - -static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu) -{ - u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress; - u32 tbu_inv_pending = 0, tbu_sync_pending = 0; - u32 tbu_inv_acked = 0, tbu_sync_acked = 0; - u32 tcu_inv_pending = 0, tcu_sync_pending = 0; - u32 tbu_ids = 0; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - int ret; - - static DEFINE_RATELIMIT_STATE(_rs, - DEFAULT_RATELIMIT_INTERVAL, - DEFAULT_RATELIMIT_BURST); - - dev_err_ratelimited(smmu->dev, - "TLB sync timed out -- SMMU may be deadlocked\n"); - - sync_inv_ack = arm_smmu_readl(smmu, - ARM_SMMU_IMPL_DEF0, - ARM_SMMU_STATS_SYNC_INV_TBU_ACK); - ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + - ARM_SMMU_TBU_PWR_STATUS), &tbu_pwr_status); - if (ret) { - dev_err_ratelimited(smmu->dev, - "SCM read of TBU power status fails: %d\n", - ret); - goto out; - } - - ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + - ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR), - &sync_inv_progress); - if (ret) { - dev_err_ratelimited(smmu->dev, - "SCM read of TBU sync/inv prog fails: %d\n", - ret); - goto out; - } - - if (sync_inv_ack) { - tbu_inv_pending = FIELD_GET(TBU_INV_REQ, sync_inv_ack); - tbu_inv_acked = FIELD_GET(TBU_INV_ACK, sync_inv_ack); - tbu_sync_pending = FIELD_GET(TBU_SYNC_REQ, sync_inv_ack); - tbu_sync_acked = FIELD_GET(TBU_SYNC_ACK, sync_inv_ack); - } - - if (tbu_pwr_status) { - if (tbu_sync_pending) - tbu_ids = tbu_pwr_status & ~tbu_sync_acked; - else if (tbu_inv_pending) - tbu_ids = tbu_pwr_status & ~tbu_inv_acked; - } - - tcu_inv_pending = FIELD_GET(TCU_INV_IN_PRGSS, sync_inv_progress); - tcu_sync_pending = FIELD_GET(TCU_SYNC_IN_PRGSS, sync_inv_progress); - - if (__ratelimit(&_rs)) { - unsigned long tbu_id; - - dev_err(smmu->dev, - "TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n", - sync_inv_ack, tbu_pwr_status, sync_inv_progress); - dev_err(smmu->dev, - "TCU invalidation %s, TCU sync %s\n", - tcu_inv_pending?"pending":"completed", - tcu_sync_pending?"pending":"completed"); - - for_each_set_bit(tbu_id, (unsigned long *) &tbu_ids, - sizeof(tbu_ids) * BITS_PER_BYTE) { - - struct qsmmuv500_tbu_device *tbu; - - tbu = qsmmuv500_find_tbu(smmu, - (u16)(tbu_id << TBUID_SHIFT)); - if (tbu) { - dev_err(smmu->dev, - "TBU %s ack pending for TBU %s, %s\n", - tbu_sync_pending?"sync" : "inv", - dev_name(tbu->dev), - tbu_sync_pending ? - "check pending transactions on TBU" - : "check for TBU power status"); - } - } - } - - if (tcu_sync_pending) - schedule_work(&data->outstanding_tnx_work); - else -out: - BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); -} - -static void qsmmuv500_device_remove(struct arm_smmu_device *smmu) -{ - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - cancel_work_sync(&data->outstanding_tnx_work); -} - -static bool arm_smmu_fwspec_match_smr(struct iommu_fwspec *fwspec, - struct arm_smmu_smr *smr) -{ - struct arm_smmu_smr *smr2; - struct arm_smmu_device *smmu = fwspec_smmu(fwspec); - int i, idx; - - for_each_cfg_sme(fwspec, i, idx) { - smr2 = &smmu->smrs[idx]; - /* Continue if table entry does not match */ - if ((smr->id ^ smr2->id) & ~(smr->mask | smr2->mask)) - continue; - return true; - } - return false; -} - -static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu, - struct arm_smmu_domain *smmu_domain) -{ - unsigned long flags; - struct arm_smmu_device *smmu = smmu_domain->smmu; - int idx = smmu_domain->cfg.cbndx; - u32 halt, fsr, status; - void __iomem *tbu_base; - - if (of_property_read_bool(tbu->dev->of_node, - "qcom,opt-out-tbu-halting")) { - dev_notice(tbu->dev, "TBU opted-out for halting!\n"); - return -EBUSY; - } - - spin_lock_irqsave(&tbu->halt_lock, flags); - if (tbu->halt_count) { - tbu->halt_count++; - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return 0; - } - - tbu_base = tbu->base; - halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); - halt |= DEBUG_SID_HALT_REQ; - writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if ((fsr & FSR_FAULT) && (fsr & FSR_SS)) { - u32 sctlr_orig, sctlr; - /* - * We are in a fault; Our request to halt the bus will not - * complete until transactions in front of us (such as the fault - * itself) have completed. Disable iommu faults and terminate - * any existing transactions. - */ - sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); - sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation - */ - wmb(); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - RESUME_TERMINATE); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); - } - - if (readl_poll_timeout_atomic(tbu_base + DEBUG_SR_HALT_ACK_REG, status, - (status & DEBUG_SR_HALT_ACK_VAL), - 0, TBU_DBG_TIMEOUT_US)) { - dev_err(tbu->dev, "Couldn't halt TBU!\n"); - - halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); - halt &= ~DEBUG_SID_HALT_REQ; - writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); - - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return -ETIMEDOUT; - } - - tbu->halt_count = 1; - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return 0; -} - -static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu) -{ - unsigned long flags; - u32 val; - void __iomem *base; - - spin_lock_irqsave(&tbu->halt_lock, flags); - if (!tbu->halt_count) { - WARN(1, "%s: bad tbu->halt_count", dev_name(tbu->dev)); - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return; - - } else if (tbu->halt_count > 1) { - tbu->halt_count--; - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return; - } - - base = tbu->base; - val = readl_relaxed(base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_REQ; - writel_relaxed(val, base + DEBUG_SID_HALT_REG); - - tbu->halt_count = 0; - spin_unlock_irqrestore(&tbu->halt_lock, flags); -} - -static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( - struct arm_smmu_device *smmu, u32 sid) -{ - struct qsmmuv500_tbu_device *tbu = NULL; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - list_for_each_entry(tbu, &data->tbus, list) { - if (tbu->sid_start <= sid && - sid < tbu->sid_start + tbu->num_sids) - return tbu; - } - return NULL; -} - -static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain, - struct qsmmuv500_tbu_device *tbu, - unsigned long *flags) -{ - struct arm_smmu_device *smmu = tbu->smmu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - u32 val; - - spin_lock_irqsave(&smmu->atos_lock, *flags); - /* The status register is not accessible on version 1.0 */ - if (data->version == 0x01000000) - return 0; - - if (readl_poll_timeout_atomic(tbu->status_reg, - val, (val == 0x1), 0, - TBU_DBG_TIMEOUT_US)) { - dev_err(tbu->dev, "ECATS hw busy!\n"); - spin_unlock_irqrestore(&smmu->atos_lock, *flags); - return -ETIMEDOUT; - } - - return 0; -} - -static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, - struct qsmmuv500_tbu_device *tbu, - unsigned long *flags) -{ - struct arm_smmu_device *smmu = tbu->smmu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - /* The status register is not accessible on version 1.0 */ - if (data->version != 0x01000000) - writel_relaxed(0, tbu->status_reg); - spin_unlock_irqrestore(&smmu->atos_lock, *flags); -} - -/* - * Zero means failure. - */ -static phys_addr_t qsmmuv500_iova_to_phys( - struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 sid, - unsigned long trans_flags) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct qsmmuv500_tbu_device *tbu; - int ret; - phys_addr_t phys = 0; - u64 val, fsr; - unsigned long flags; - int idx = cfg->cbndx; - u32 sctlr_orig, sctlr; - int needs_redo = 0; - ktime_t timeout; - - /* only 36 bit iova is supported */ - if (iova >= (1ULL << 36)) { - dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n", - &iova); - return 0; - } - - tbu = qsmmuv500_find_tbu(smmu, sid); - if (!tbu) - return 0; - - ret = arm_smmu_power_on(tbu->pwr); - if (ret) - return 0; - - ret = qsmmuv500_tbu_halt(tbu, smmu_domain); - if (ret) - goto out_power_off; - - /* - * ECATS can trigger the fault interrupt, so disable it temporarily - * and check for an interrupt manually. - */ - sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); - sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (fsr & FSR_FAULT) { - /* Clear pending interrupts */ - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation. - */ - wmb(); - - /* - * TBU halt takes care of resuming any stalled transcation. - * Kept it here for completeness sake. - */ - if (fsr & FSR_SS) - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - RESUME_TERMINATE); - } - - /* Only one concurrent atos operation */ - ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags); - if (ret) - goto out_resume; - -redo: - /* Set address and stream-id */ - val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_SID; - val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); - writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); - writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG); - val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); - writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); - - /* Write-back Read and Write-Allocate */ - val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF); - - /* Non-secure Access */ - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); - - /* Write or Read Access */ - if (flags & IOMMU_TRANS_WRITE) - val |= DEBUG_TXN_WRITE; - - /* Priviledged or Unpriviledged Access */ - if (flags & IOMMU_TRANS_PRIV) - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); - - /* Data or Instruction Access */ - if (flags & IOMMU_TRANS_INST) - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST); - - val |= DEBUG_TXN_TRIGGER; - writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); - - ret = 0; - timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); - for (;;) { - val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); - if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) - break; - val = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (val & FSR_FAULT) - break; - if (ktime_compare(ktime_get(), timeout) > 0) { - dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); - ret = -ETIMEDOUT; - break; - } - } - - val = readq_relaxed(tbu->base + DEBUG_PAR_REG); - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (val & DEBUG_PAR_FAULT_VAL) { - dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx, SID=0x%x\n", - fsr, sid); - - dev_err(tbu->dev, "ECATS translation failed! PAR = %llx\n", - val); - /* Clear pending interrupts */ - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation. - */ - wmb(); - - if (fsr & FSR_SS) - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - RESUME_TERMINATE); - - ret = -EINVAL; - } - - phys = FIELD_GET(DEBUG_PAR_PA, val); - if (ret < 0) - phys = 0; - - /* Reset hardware */ - writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG); - writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG); - val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_SID; - writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); - - /* - * After a failed translation, the next successful translation will - * incorrectly be reported as a failure. - */ - if (!phys && needs_redo++ < 2) - goto redo; - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); - qsmmuv500_ecats_unlock(smmu_domain, tbu, &flags); - -out_resume: - qsmmuv500_tbu_resume(tbu); - -out_power_off: - /* Read to complete prior write transcations */ - val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); - - /* Wait for read to complete before off */ - rmb(); - - arm_smmu_power_off(tbu->smmu, tbu->pwr); - - return phys; -} - -static phys_addr_t qsmmuv500_iova_to_phys_hard( - struct arm_smmu_domain *smmu_domain, - dma_addr_t iova, - unsigned long trans_flags) -{ - u16 sid; - struct msm_iommu_domain *msm_domain = &smmu_domain->domain; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct iommu_fwspec *fwspec; - u32 frsynra; - - /* Check to see if the domain is associated with the test - * device. If the domain belongs to the test device, then - * pick the SID from fwspec. - */ - if (msm_domain->is_debug_domain) { - fwspec = dev_iommu_fwspec_get(smmu_domain->dev); - sid = (u16)fwspec->ids[0]; - } else { - - /* If the domain belongs to an actual device, read - * SID from the corresponding frsynra register - */ - frsynra = arm_smmu_gr1_read(smmu, - ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); - frsynra &= CBFRSYNRA_SID_MASK; - sid = frsynra; - } - return qsmmuv500_iova_to_phys(smmu_domain, iova, sid, trans_flags); -} - -static void qsmmuv500_release_group_iommudata(void *data) -{ - kfree(data); -} - -/* If a device has a valid actlr, it must match */ -static int qsmmuv500_device_group(struct device *dev, - struct iommu_group *group) -{ - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_device *smmu = fwspec_smmu(fwspec); - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - struct qsmmuv500_group_iommudata *iommudata; - u32 actlr, i; - struct arm_smmu_smr *smr; - - iommudata = to_qsmmuv500_group_iommudata(group); - if (!iommudata) { - iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); - if (!iommudata) - return -ENOMEM; - - iommu_group_set_iommudata(group, iommudata, - qsmmuv500_release_group_iommudata); - } - - for (i = 0; i < data->actlr_tbl_size; i++) { - smr = &data->actlrs[i].smr; - actlr = data->actlrs[i].actlr; - - if (!arm_smmu_fwspec_match_smr(fwspec, smr)) - continue; - - if (!iommudata->has_actlr) { - iommudata->actlr = actlr; - iommudata->has_actlr = true; - } else if (iommudata->actlr != actlr) { - return -EINVAL; - } - } - - return 0; -} - -static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, - struct device *dev) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct qsmmuv500_group_iommudata *iommudata = - to_qsmmuv500_group_iommudata(dev->iommu_group); - int idx = smmu_domain->cfg.cbndx; - const struct iommu_flush_ops *tlb; - - if (!iommudata->has_actlr) - return; - - tlb = smmu_domain->pgtbl_info.pgtbl_cfg.tlb; - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ACTLR, iommudata->actlr); - - /* - * Flush the context bank after modifying ACTLR to ensure there - * are no cache entries with stale state - */ - tlb->tlb_flush_all(smmu_domain); -} - -static int qsmmuv500_tbu_register(struct device *dev, void *cookie) -{ - struct arm_smmu_device *smmu = cookie; - struct qsmmuv500_tbu_device *tbu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - if (!dev->driver) { - dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); - return -EINVAL; - } - - tbu = dev_get_drvdata(dev); - - INIT_LIST_HEAD(&tbu->list); - tbu->smmu = smmu; - list_add(&tbu->list, &data->tbus); - return 0; -} - -static int qsmmuv500_read_actlr_tbl(struct arm_smmu_device *smmu) -{ - int len, i; - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - struct actlr_setting *actlrs; - const __be32 *cell; - - cell = of_get_property(dev->of_node, "qcom,actlr", NULL); - if (!cell) - return 0; - - len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr", - sizeof(u32) * 3); - if (len < 0) - return 0; - - actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL); - if (!actlrs) - return -ENOMEM; - - for (i = 0; i < len; i++) { - actlrs[i].smr.id = of_read_number(cell++, 1); - actlrs[i].smr.mask = of_read_number(cell++, 1); - actlrs[i].actlr = of_read_number(cell++, 1); - } - - data->actlrs = actlrs; - data->actlr_tbl_size = len; - return 0; -} - -static int qsmmuv500_arch_init(struct arm_smmu_device *smmu) -{ - struct resource *res; - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data; - struct platform_device *pdev; - int ret; - u32 val; - - data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - INIT_LIST_HEAD(&data->tbus); - - pdev = container_of(dev, struct platform_device, dev); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base"); - if (!res) { - dev_err(dev, "Unable to get the tcu-base\n"); - return -EINVAL; - } - data->tcu_base = devm_ioremap(dev, res->start, resource_size(res)); - if (IS_ERR(data->tcu_base)) - return PTR_ERR(data->tcu_base); - - data->version = readl_relaxed(data->tcu_base + TCU_HW_VERSION_HLOS1); - smmu->archdata = data; - data->smmu = smmu; - - ret = qsmmuv500_read_actlr_tbl(smmu); - if (ret) - return ret; - - val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); - val &= ~ARM_MMU500_ACR_CACHE_LOCK; - arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, val); - val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); - /* - * Modifiying the nonsecure copy of the sACR register is only - * allowed if permission is given in the secure sACR register. - * Attempt to detect if we were able to update the value. - */ - WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); - - ret = of_platform_populate(dev->of_node, NULL, NULL, dev); - if (ret) - return ret; - - INIT_WORK(&data->outstanding_tnx_work, - qsmmuv500_log_outstanding_transactions); - - /* Attempt to register child devices */ - ret = device_for_each_child(dev, smmu, qsmmuv500_tbu_register); - if (ret) - return -EPROBE_DEFER; - - return 0; -} - -static struct arm_smmu_arch_ops qsmmuv500_arch_ops = { - .init = qsmmuv500_arch_init, - .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, - .init_context_bank = qsmmuv500_init_cb, - .device_group = qsmmuv500_device_group, - .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, - .device_remove = qsmmuv500_device_remove, -}; - static const struct of_device_id qsmmuv500_tbu_of_match[] = { {.compatible = "qcom,qsmmuv500-tbu"}, {} @@ -5917,6 +4941,45 @@ static struct platform_driver qsmmuv500_tbu_driver = { .remove = qsmmuv500_tbu_remove, }; +static struct platform_driver arm_smmu_driver = { + .driver = { + .name = "arm-smmu", + .of_match_table = of_match_ptr(arm_smmu_of_match), + .pm = &arm_smmu_pm_ops, + .suppress_bind_attrs = true, + }, + .probe = arm_smmu_device_dt_probe, + .remove = arm_smmu_device_remove, +}; + +static int __init arm_smmu_init(void) +{ + int ret; + ktime_t cur; + + cur = ktime_get(); + ret = platform_driver_register(&qsmmuv500_tbu_driver); + if (ret) + return ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) { + platform_driver_unregister(&qsmmuv500_tbu_driver); + return ret; + } + + trace_smmu_init(ktime_us_delta(ktime_get(), cur)); + return ret; +} +subsys_initcall(arm_smmu_init); + +static void __exit arm_smmu_exit(void) +{ + platform_driver_unregister(&arm_smmu_driver); + platform_driver_unregister(&qsmmuv500_tbu_driver); +} +module_exit(arm_smmu_exit); + MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index 3dfe70bb5b9e..35f7d2dbcbcb 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -394,17 +394,43 @@ struct arm_smmu_device { struct arm_smmu_power_resources *pwr; - spinlock_t atos_lock; - /* protects idr */ struct mutex idr_mutex; struct idr asid_idr; struct arm_smmu_arch_ops *arch_ops; - void *archdata; unsigned long sync_timed_out; }; +struct qsmmuv500_tbu_device { + struct list_head list; + struct device *dev; + struct arm_smmu_device *smmu; + void __iomem *base; + void __iomem *status_reg; + + struct arm_smmu_power_resources *pwr; + u32 sid_start; + u32 num_sids; + + /* Protects halt count */ + spinlock_t halt_lock; + u32 halt_count; +}; + +struct arm_smmu_master_cfg { + struct arm_smmu_device *smmu; + s16 smendx[]; +}; + +#define INVALID_SMENDX -1 +#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) +#define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) +#define fwspec_smendx(fw, i) \ + (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) +#define for_each_cfg_sme(fw, i, idx) \ + for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) + enum arm_smmu_context_fmt { ARM_SMMU_CTX_FMT_NONE, ARM_SMMU_CTX_FMT_AARCH64, @@ -465,6 +491,7 @@ struct arm_smmu_domain { /* Implementation details, yay! */ + struct arm_smmu_impl { u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset); void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset, @@ -475,6 +502,12 @@ struct arm_smmu_impl { int (*cfg_probe)(struct arm_smmu_device *smmu); int (*reset)(struct arm_smmu_device *smmu); int (*init_context)(struct arm_smmu_domain *smmu_domain); + void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, + struct device *dev); + phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain, + dma_addr_t iova, + unsigned long trans_flags); + void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); }; static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n) @@ -538,12 +571,33 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, #define arm_smmu_cb_writeq(s, n, o, v) \ arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v)) +/* + * device_group() + * Hook for checking whether a device is compatible with a said group. + * + * device_remove() + * Hook for performing architecture-specific procedures prior to powering off + * the SMMU. + */ +struct arm_smmu_arch_ops { + int (*device_group)(struct device *dev, struct iommu_group *group); + void (*device_remove)(struct arm_smmu_device *smmu); +}; + struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); +struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu); int arm_mmu500_reset(struct arm_smmu_device *smmu); +int arm_smmu_power_on(struct arm_smmu_power_resources *pwr); +void arm_smmu_power_off(struct arm_smmu_device *smmu, + struct arm_smmu_power_resources *pwr); + +extern struct arm_smmu_arch_ops qsmmuv500_arch_ops; + /* Misc. constants */ #define TBUID_SHIFT 10 +#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) #endif /* _ARM_SMMU_H */