From 3729473791e9fa5ddb626fe9addc21bfd820d52a Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Fri, 21 Feb 2020 11:55:16 -0800 Subject: [PATCH 1/4] iommu/arm-smmu: Prepare to migrate QSMMUV500 implementation details All of the implementation details related to the QSMMUV500 model of the SMMU-V2 architecture currently reside in arm-smmu.c. This clashes with the current ARM SMMU SW architecture, as arm-smmu.c is meant to house architecture specific details, and arm-smmu-[$VENDOR].c is meant to house vendor specific implementation details about those architectures. To prepare to move to comply with this architecture, migrate the QSMMUV500 related logic to arm-smmu-qcom.c, as well as export any shared data structures. This patch should not introduce any functional changes. Change-Id: I711e41e38cd35b6727faf5a399eefcf5db453612 Signed-off-by: Isaac J. Manjarres --- drivers/iommu/arm-smmu-impl.c | 1 - drivers/iommu/arm-smmu-qcom.c | 831 ++++++++++++++++++++++++++++- drivers/iommu/arm-smmu.c | 975 ++-------------------------------- drivers/iommu/arm-smmu.h | 75 +++ 4 files changed, 947 insertions(+), 935 deletions(-) diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index b2fe72a8f019..c25ad938e2e6 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -105,7 +105,6 @@ static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smm #define ARM_MMU500_ACTLR_CPRE (1 << 1) -#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10) #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8) diff --git a/drivers/iommu/arm-smmu-qcom.c b/drivers/iommu/arm-smmu-qcom.c index 06e5799dcb87..969fe28ae640 100644 --- a/drivers/iommu/arm-smmu-qcom.c +++ b/drivers/iommu/arm-smmu-qcom.c @@ -1,10 +1,17 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. */ #include +#include +#include +#include +#include +#include #include +#include +#include #include "arm-smmu.h" @@ -69,6 +76,828 @@ static const struct arm_smmu_impl qcom_smmu_impl = { .reset = qcom_sdm845_smmu500_reset, }; +#define TCU_HW_VERSION_HLOS1 (0x18) + +#define DEBUG_SID_HALT_REG 0x0 +#define DEBUG_SID_HALT_REQ BIT(16) +#define DEBUG_SID_HALT_SID GENMASK(9, 0) + +#define DEBUG_VA_ADDR_REG 0x8 + +#define DEBUG_TXN_TRIGG_REG 0x18 +#define DEBUG_TXN_AXPROT GENMASK(8, 6) +#define DEBUG_TXN_AXCACHE GENMASK(5, 2) +#define DEBUG_TXN_WRITE BIT(1) +#define DEBUG_TXN_AXPROT_PRIV 0x1 +#define DEBUG_TXN_AXPROT_UNPRIV 0x0 +#define DEBUG_TXN_AXPROT_NSEC 0x2 +#define DEBUG_TXN_AXPROT_SEC 0x0 +#define DEBUG_TXN_AXPROT_INST 0x4 +#define DEBUG_TXN_AXPROT_DATA 0x0 +#define DEBUG_TXN_READ (0x0 << 1) +#define DEBUG_TXN_TRIGGER BIT(0) + +#define DEBUG_SR_HALT_ACK_REG 0x20 +#define DEBUG_SR_HALT_ACK_VAL (0x1 << 1) +#define DEBUG_SR_ECATS_RUNNING_VAL (0x1 << 0) + +#define DEBUG_PAR_REG 0x28 +#define DEBUG_PAR_PA GENMASK_ULL(47, 12) +#define DEBUG_PAR_FAULT_VAL BIT(0) + +#define DEBUG_AXUSER_REG 0x30 +#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) +#define DEBUG_AXUSER_CDMID_VAL 255 + +#define TBU_DBG_TIMEOUT_US 100 + +#define TNX_TCR_CNTL 0x130 +#define TNX_TCR_CNTL_TBU_OT_CAPTURE_EN BIT(18) +#define TNX_TCR_CNTL_ALWAYS_CAPTURE BIT(15) +#define TNX_TCR_CNTL_MATCH_MASK_UPD BIT(7) +#define TNX_TCR_CNTL_MATCH_MASK_VALID BIT(6) + +#define CAPTURE1_SNAPSHOT_1 0x138 + +#define TNX_TCR_CNTL_2 0x178 +#define TNX_TCR_CNTL_2_CAP1_VALID BIT(0) + +struct actlr_setting { + struct arm_smmu_smr smr; + u32 actlr; +}; + +struct qsmmuv500_archdata { + struct list_head tbus; + void __iomem *tcu_base; + u32 version; + struct actlr_setting *actlrs; + u32 actlr_tbl_size; + struct work_struct outstanding_tnx_work; + struct arm_smmu_device *smmu; +}; +#define get_qsmmuv500_archdata(smmu) \ + ((struct qsmmuv500_archdata *)(smmu->archdata)) + +struct qsmmuv500_group_iommudata { + bool has_actlr; + u32 actlr; +}; +#define to_qsmmuv500_group_iommudata(group) \ + ((struct qsmmuv500_group_iommudata *) \ + (iommu_group_get_iommudata(group))) + +static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( + struct arm_smmu_device *smmu, u32 sid); + +/* + * Provides mutually exclusive access to the registers used by the + * outstanding transaction snapshot feature and the transaction + * snapshot capture feature. + */ +static DEFINE_MUTEX(capture_reg_lock); + +static void qsmmuv500_log_outstanding_transactions(struct work_struct *work) +{ + struct qsmmuv500_tbu_device *tbu = NULL; + u64 outstanding_tnxs; + u64 tcr_cntl_val, res; + struct qsmmuv500_archdata *data = container_of(work, + struct qsmmuv500_archdata, + outstanding_tnx_work); + struct arm_smmu_device *smmu = data->smmu; + void __iomem *base; + + if (!mutex_trylock(&capture_reg_lock)) { + dev_warn_ratelimited(smmu->dev, + "Tnx snapshot regs in use, not dumping OT tnxs.\n"); + goto bug; + } + + if (arm_smmu_power_on(smmu->pwr)) { + dev_err_ratelimited(smmu->dev, + "%s: Failed to power on SMMU.\n", + __func__); + goto unlock; + } + + list_for_each_entry(tbu, &data->tbus, list) { + if (arm_smmu_power_on(tbu->pwr)) { + dev_err_ratelimited(tbu->dev, + "%s: Failed to power on TBU.\n", + __func__); + continue; + } + base = tbu->base; + + tcr_cntl_val = readq_relaxed(base + TNX_TCR_CNTL); + + /* Write 1 into MATCH_MASK_UPD of TNX_TCR_CNTL */ + writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_MATCH_MASK_UPD, + base + TNX_TCR_CNTL); + + /* + * Simultaneously write 0 into MATCH_MASK_UPD, 0 into + * ALWAYS_CAPTURE, 0 into MATCH_MASK_VALID, and 1 into + * TBU_OT_CAPTURE_EN of TNX_TCR_CNTL + */ + tcr_cntl_val &= ~(TNX_TCR_CNTL_MATCH_MASK_UPD | + TNX_TCR_CNTL_ALWAYS_CAPTURE | + TNX_TCR_CNTL_MATCH_MASK_VALID); + writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, + base + TNX_TCR_CNTL); + + /* Poll for CAPTURE1_VALID to become 1 on TNX_TCR_CNTL_2 */ + if (readq_poll_timeout_atomic(base + TNX_TCR_CNTL_2, res, + res & TNX_TCR_CNTL_2_CAP1_VALID, + 0, TBU_DBG_TIMEOUT_US)) { + dev_err_ratelimited(tbu->dev, + "Timeout on TNX snapshot poll\n"); + goto poll_timeout; + } + + /* Read Register CAPTURE1_SNAPSHOT_1 */ + outstanding_tnxs = readq_relaxed(base + CAPTURE1_SNAPSHOT_1); + dev_err_ratelimited(tbu->dev, + "Outstanding Transaction Bitmap: 0x%llx\n", + outstanding_tnxs); +poll_timeout: + /* Write TBU_OT_CAPTURE_EN to 0 of TNX_TCR_CNTL */ + writeq_relaxed(tcr_cntl_val & ~TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, + tbu->base + TNX_TCR_CNTL); + + arm_smmu_power_off(smmu, tbu->pwr); + } + + arm_smmu_power_off(smmu, smmu->pwr); +unlock: + mutex_unlock(&capture_reg_lock); +bug: + BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); +} + +static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu) +{ + u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress; + u32 tbu_inv_pending = 0, tbu_sync_pending = 0; + u32 tbu_inv_acked = 0, tbu_sync_acked = 0; + u32 tcu_inv_pending = 0, tcu_sync_pending = 0; + u32 tbu_ids = 0; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + int ret; + + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + dev_err_ratelimited(smmu->dev, + "TLB sync timed out -- SMMU may be deadlocked\n"); + + sync_inv_ack = arm_smmu_readl(smmu, + ARM_SMMU_IMPL_DEF0, + ARM_SMMU_STATS_SYNC_INV_TBU_ACK); + ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + + ARM_SMMU_TBU_PWR_STATUS), &tbu_pwr_status); + if (ret) { + dev_err_ratelimited(smmu->dev, + "SCM read of TBU power status fails: %d\n", + ret); + goto out; + } + + ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + + ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR), + &sync_inv_progress); + if (ret) { + dev_err_ratelimited(smmu->dev, + "SCM read of TBU sync/inv prog fails: %d\n", + ret); + goto out; + } + + if (sync_inv_ack) { + tbu_inv_pending = FIELD_GET(TBU_INV_REQ, sync_inv_ack); + tbu_inv_acked = FIELD_GET(TBU_INV_ACK, sync_inv_ack); + tbu_sync_pending = FIELD_GET(TBU_SYNC_REQ, sync_inv_ack); + tbu_sync_acked = FIELD_GET(TBU_SYNC_ACK, sync_inv_ack); + } + + if (tbu_pwr_status) { + if (tbu_sync_pending) + tbu_ids = tbu_pwr_status & ~tbu_sync_acked; + else if (tbu_inv_pending) + tbu_ids = tbu_pwr_status & ~tbu_inv_acked; + } + + tcu_inv_pending = FIELD_GET(TCU_INV_IN_PRGSS, sync_inv_progress); + tcu_sync_pending = FIELD_GET(TCU_SYNC_IN_PRGSS, sync_inv_progress); + + if (__ratelimit(&_rs)) { + unsigned long tbu_id; + + dev_err(smmu->dev, + "TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n", + sync_inv_ack, tbu_pwr_status, sync_inv_progress); + dev_err(smmu->dev, + "TCU invalidation %s, TCU sync %s\n", + tcu_inv_pending?"pending":"completed", + tcu_sync_pending?"pending":"completed"); + + for_each_set_bit(tbu_id, (unsigned long *) &tbu_ids, + sizeof(tbu_ids) * BITS_PER_BYTE) { + + struct qsmmuv500_tbu_device *tbu; + + tbu = qsmmuv500_find_tbu(smmu, + (u16)(tbu_id << TBUID_SHIFT)); + if (tbu) { + dev_err(smmu->dev, + "TBU %s ack pending for TBU %s, %s\n", + tbu_sync_pending?"sync" : "inv", + dev_name(tbu->dev), + tbu_sync_pending ? + "check pending transactions on TBU" + : "check for TBU power status"); + } + } + } + + if (tcu_sync_pending) { + schedule_work(&data->outstanding_tnx_work); + return; + } +out: + BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); +} + +static void qsmmuv500_device_remove(struct arm_smmu_device *smmu) +{ + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + + cancel_work_sync(&data->outstanding_tnx_work); +} + +static bool arm_smmu_fwspec_match_smr(struct iommu_fwspec *fwspec, + struct arm_smmu_smr *smr) +{ + struct arm_smmu_smr *smr2; + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + int i, idx; + + for_each_cfg_sme(fwspec, i, idx) { + smr2 = &smmu->smrs[idx]; + /* Continue if table entry does not match */ + if ((smr->id ^ smr2->id) & ~(smr->mask | smr2->mask)) + continue; + return true; + } + return false; +} + +static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu, + struct arm_smmu_domain *smmu_domain) +{ + unsigned long flags; + struct arm_smmu_device *smmu = smmu_domain->smmu; + int idx = smmu_domain->cfg.cbndx; + u32 halt, fsr, status; + void __iomem *tbu_base; + + if (of_property_read_bool(tbu->dev->of_node, + "qcom,opt-out-tbu-halting")) { + dev_notice(tbu->dev, "TBU opted-out for halting!\n"); + return -EBUSY; + } + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (tbu->halt_count) { + tbu->halt_count++; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return 0; + } + + tbu_base = tbu->base; + halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); + halt |= DEBUG_SID_HALT_REQ; + writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if ((fsr & FSR_FAULT) && (fsr & FSR_SS)) { + u32 sctlr_orig, sctlr; + /* + * We are in a fault; Our request to halt the bus will not + * complete until transactions in front of us (such as the fault + * itself) have completed. Disable iommu faults and terminate + * any existing transactions. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation + */ + wmb(); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + RESUME_TERMINATE); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + } + + if (readl_poll_timeout_atomic(tbu_base + DEBUG_SR_HALT_ACK_REG, status, + (status & DEBUG_SR_HALT_ACK_VAL), + 0, TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "Couldn't halt TBU!\n"); + + halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); + halt &= ~DEBUG_SID_HALT_REQ; + writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); + + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return -ETIMEDOUT; + } + + tbu->halt_count = 1; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return 0; +} + +static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu) +{ + unsigned long flags; + u32 val; + void __iomem *base; + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (!tbu->halt_count) { + WARN(1, "%s: bad tbu->halt_count", dev_name(tbu->dev)); + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return; + + } else if (tbu->halt_count > 1) { + tbu->halt_count--; + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return; + } + + base = tbu->base; + val = readl_relaxed(base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_REQ; + writel_relaxed(val, base + DEBUG_SID_HALT_REG); + + tbu->halt_count = 0; + spin_unlock_irqrestore(&tbu->halt_lock, flags); +} + +static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( + struct arm_smmu_device *smmu, u32 sid) +{ + struct qsmmuv500_tbu_device *tbu = NULL; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + + list_for_each_entry(tbu, &data->tbus, list) { + if (tbu->sid_start <= sid && + sid < tbu->sid_start + tbu->num_sids) + return tbu; + } + return NULL; +} + +static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain, + struct qsmmuv500_tbu_device *tbu, + unsigned long *flags) + __acquires(&smmu->atos_lock) +{ + struct arm_smmu_device *smmu = tbu->smmu; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + u32 val; + + spin_lock_irqsave(&smmu->atos_lock, *flags); + /* The status register is not accessible on version 1.0 */ + if (data->version == 0x01000000) + return 0; + + if (readl_poll_timeout_atomic(tbu->status_reg, + val, (val == 0x1), 0, + TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "ECATS hw busy!\n"); + spin_unlock_irqrestore(&smmu->atos_lock, *flags); + return -ETIMEDOUT; + } + + return 0; +} + +static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, + struct qsmmuv500_tbu_device *tbu, + unsigned long *flags) + __releases(&smmu->atos_lock) +{ + struct arm_smmu_device *smmu = tbu->smmu; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + + /* The status register is not accessible on version 1.0 */ + if (data->version != 0x01000000) + writel_relaxed(0, tbu->status_reg); + spin_unlock_irqrestore(&smmu->atos_lock, *flags); +} + +/* + * Zero means failure. + */ +static phys_addr_t qsmmuv500_iova_to_phys( + struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 sid, + unsigned long trans_flags) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct qsmmuv500_tbu_device *tbu; + int ret; + phys_addr_t phys = 0; + u64 val, fsr; + unsigned long flags; + int idx = cfg->cbndx; + u32 sctlr_orig, sctlr; + int needs_redo = 0; + ktime_t timeout; + + /* only 36 bit iova is supported */ + if (iova >= (1ULL << 36)) { + dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n", + &iova); + return 0; + } + + tbu = qsmmuv500_find_tbu(smmu, sid); + if (!tbu) + return 0; + + ret = arm_smmu_power_on(tbu->pwr); + if (ret) + return 0; + + ret = qsmmuv500_tbu_halt(tbu, smmu_domain); + if (ret) + goto out_power_off; + + /* + * ECATS can trigger the fault interrupt, so disable it temporarily + * and check for an interrupt manually. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation. + */ + wmb(); + + /* + * TBU halt takes care of resuming any stalled transcation. + * Kept it here for completeness sake. + */ + if (fsr & FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + RESUME_TERMINATE); + } + + /* Only one concurrent atos operation */ + ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags); + if (ret) + goto out_resume; + +redo: + /* Set address and stream-id */ + val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); + writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG); + val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); + writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); + + /* Write-back Read and Write-Allocate */ + val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF); + + /* Non-secure Access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); + + /* Write or Read Access */ + if (flags & IOMMU_TRANS_WRITE) + val |= DEBUG_TXN_WRITE; + + /* Priviledged or Unpriviledged Access */ + if (flags & IOMMU_TRANS_PRIV) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); + + /* Data or Instruction Access */ + if (flags & IOMMU_TRANS_INST) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST); + + val |= DEBUG_TXN_TRIGGER; + writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); + + ret = 0; + timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); + for (;;) { + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) + break; + val = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (val & FSR_FAULT) + break; + if (ktime_compare(ktime_get(), timeout) > 0) { + dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); + ret = -ETIMEDOUT; + break; + } + } + + val = readq_relaxed(tbu->base + DEBUG_PAR_REG); + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (val & DEBUG_PAR_FAULT_VAL) { + dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx, SID=0x%x\n", + fsr, sid); + + dev_err(tbu->dev, "ECATS translation failed! PAR = %llx\n", + val); + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation. + */ + wmb(); + + if (fsr & FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + RESUME_TERMINATE); + + ret = -EINVAL; + } + + phys = FIELD_GET(DEBUG_PAR_PA, val); + if (ret < 0) + phys = 0; + + /* Reset hardware */ + writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG); + writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG); + val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); + + /* + * After a failed translation, the next successful translation will + * incorrectly be reported as a failure. + */ + if (!phys && needs_redo++ < 2) + goto redo; + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + qsmmuv500_ecats_unlock(smmu_domain, tbu, &flags); + +out_resume: + qsmmuv500_tbu_resume(tbu); + +out_power_off: + /* Read to complete prior write transcations */ + val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); + + /* Wait for read to complete before off */ + rmb(); + + arm_smmu_power_off(tbu->smmu, tbu->pwr); + + return phys; +} + +static phys_addr_t qsmmuv500_iova_to_phys_hard( + struct arm_smmu_domain *smmu_domain, + dma_addr_t iova, + unsigned long trans_flags) +{ + u16 sid; + struct msm_iommu_domain *msm_domain = &smmu_domain->domain; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct iommu_fwspec *fwspec; + u32 frsynra; + + /* Check to see if the domain is associated with the test + * device. If the domain belongs to the test device, then + * pick the SID from fwspec. + */ + if (msm_domain->is_debug_domain) { + fwspec = dev_iommu_fwspec_get(smmu_domain->dev); + sid = (u16)fwspec->ids[0]; + } else { + + /* If the domain belongs to an actual device, read + * SID from the corresponding frsynra register + */ + frsynra = arm_smmu_gr1_read(smmu, + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); + frsynra &= CBFRSYNRA_SID_MASK; + sid = frsynra; + } + return qsmmuv500_iova_to_phys(smmu_domain, iova, sid, trans_flags); +} + +static void qsmmuv500_release_group_iommudata(void *data) +{ + kfree(data); +} + +/* If a device has a valid actlr, it must match */ +static int qsmmuv500_device_group(struct device *dev, + struct iommu_group *group) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct arm_smmu_device *smmu = fwspec_smmu(fwspec); + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_group_iommudata *iommudata; + u32 actlr, i; + struct arm_smmu_smr *smr; + + iommudata = to_qsmmuv500_group_iommudata(group); + if (!iommudata) { + iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); + if (!iommudata) + return -ENOMEM; + + iommu_group_set_iommudata(group, iommudata, + qsmmuv500_release_group_iommudata); + } + + for (i = 0; i < data->actlr_tbl_size; i++) { + smr = &data->actlrs[i].smr; + actlr = data->actlrs[i].actlr; + + if (!arm_smmu_fwspec_match_smr(fwspec, smr)) + continue; + + if (!iommudata->has_actlr) { + iommudata->actlr = actlr; + iommudata->has_actlr = true; + } else if (iommudata->actlr != actlr) { + return -EINVAL; + } + } + + return 0; +} + +static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, + struct device *dev) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct qsmmuv500_group_iommudata *iommudata = + to_qsmmuv500_group_iommudata(dev->iommu_group); + int idx = smmu_domain->cfg.cbndx; + const struct iommu_flush_ops *tlb; + + if (!iommudata->has_actlr) + return; + + tlb = smmu_domain->pgtbl_info.pgtbl_cfg.tlb; + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ACTLR, iommudata->actlr); + + /* + * Flush the context bank after modifying ACTLR to ensure there + * are no cache entries with stale state + */ + tlb->tlb_flush_all(smmu_domain); +} + +static int qsmmuv500_tbu_register(struct device *dev, void *cookie) +{ + struct arm_smmu_device *smmu = cookie; + struct qsmmuv500_tbu_device *tbu; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + + if (!dev->driver) { + dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); + return -EINVAL; + } + + tbu = dev_get_drvdata(dev); + + INIT_LIST_HEAD(&tbu->list); + tbu->smmu = smmu; + list_add(&tbu->list, &data->tbus); + return 0; +} + +static int qsmmuv500_read_actlr_tbl(struct arm_smmu_device *smmu) +{ + int len, i; + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct actlr_setting *actlrs; + const __be32 *cell; + + cell = of_get_property(dev->of_node, "qcom,actlr", NULL); + if (!cell) + return 0; + + len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr", + sizeof(u32) * 3); + if (len < 0) + return 0; + + actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL); + if (!actlrs) + return -ENOMEM; + + for (i = 0; i < len; i++) { + actlrs[i].smr.id = of_read_number(cell++, 1); + actlrs[i].smr.mask = of_read_number(cell++, 1); + actlrs[i].actlr = of_read_number(cell++, 1); + } + + data->actlrs = actlrs; + data->actlr_tbl_size = len; + return 0; +} + +static int qsmmuv500_arch_init(struct arm_smmu_device *smmu) +{ + struct resource *res; + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data; + struct platform_device *pdev; + int ret; + u32 val; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + INIT_LIST_HEAD(&data->tbus); + + pdev = container_of(dev, struct platform_device, dev); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base"); + if (!res) { + dev_err(dev, "Unable to get the tcu-base\n"); + return -EINVAL; + } + data->tcu_base = devm_ioremap(dev, res->start, resource_size(res)); + if (IS_ERR(data->tcu_base)) + return PTR_ERR(data->tcu_base); + + data->version = readl_relaxed(data->tcu_base + TCU_HW_VERSION_HLOS1); + smmu->archdata = data; + data->smmu = smmu; + + ret = qsmmuv500_read_actlr_tbl(smmu); + if (ret) + return ret; + + val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); + val &= ~ARM_MMU500_ACR_CACHE_LOCK; + arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, val); + val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); + /* + * Modifiying the nonsecure copy of the sACR register is only + * allowed if permission is given in the secure sACR register. + * Attempt to detect if we were able to update the value. + */ + WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); + + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + return ret; + + INIT_WORK(&data->outstanding_tnx_work, + qsmmuv500_log_outstanding_transactions); + + /* Attempt to register child devices */ + ret = device_for_each_child(dev, smmu, qsmmuv500_tbu_register); + if (ret) + return -EPROBE_DEFER; + + return 0; +} + +struct arm_smmu_arch_ops qsmmuv500_arch_ops = { + .init = qsmmuv500_arch_init, + .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, + .init_context_bank = qsmmuv500_init_cb, + .device_group = qsmmuv500_device_group, + .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, + .device_remove = qsmmuv500_device_remove, +}; + struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { struct qcom_smmu *qsmmu; diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 664bdfe1a99c..a4eb32c6f252 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -44,12 +44,10 @@ #include #include #include -#include #include #include -#include #include "arm-smmu.h" #include "iommu-logger.h" @@ -103,18 +101,6 @@ struct arm_smmu_cb { struct arm_smmu_cfg *cfg; }; -struct arm_smmu_master_cfg { - struct arm_smmu_device *smmu; - s16 smendx[]; -}; -#define INVALID_SMENDX -1 -#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) -#define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) -#define fwspec_smendx(fw, i) \ - (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) -#define for_each_cfg_sme(fw, i, idx) \ - for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) - #define INVALID_CBNDX 0xff #define INVALID_ASID 0xffff /* @@ -364,44 +350,6 @@ static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) } #endif -/* - * init() - * Hook for additional device tree parsing at probe time. - * - * device_reset() - * Hook for one-time architecture-specific register settings. - * - * iova_to_phys_hard() - * Provides debug information. May be called from the context fault irq handler. - * - * init_context_bank() - * Hook for architecture-specific settings which require knowledge of the - * dynamically allocated context bank number. - * - * device_group() - * Hook for checking whether a device is compatible with a said group. - * - * tlb_sync_timeout() - * Hook for performing architecture-specific procedures to collect additional - * debugging information on a TLB sync timeout. - * - * device_remove() - * Hook for performing architecture-specific procedures prior to powering off - * the SMMU. - */ -struct arm_smmu_arch_ops { - int (*init)(struct arm_smmu_device *smmu); - void (*device_reset)(struct arm_smmu_device *smmu); - phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain, - dma_addr_t iova, - unsigned long trans_flags); - void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, - struct device *dev); - int (*device_group)(struct device *dev, struct iommu_group *group); - void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); - void (*device_remove)(struct arm_smmu_device *smmu); -}; - static int arm_smmu_arch_init(struct arm_smmu_device *smmu) { if (!smmu->arch_ops) @@ -839,7 +787,7 @@ static void arm_smmu_power_off_slow(struct arm_smmu_power_resources *pwr) mutex_unlock(&pwr->power_lock); } -static int arm_smmu_power_on(struct arm_smmu_power_resources *pwr) +int arm_smmu_power_on(struct arm_smmu_power_resources *pwr) { int ret; @@ -858,8 +806,8 @@ out_disable: return ret; } -static void arm_smmu_power_off(struct arm_smmu_device *smmu, - struct arm_smmu_power_resources *pwr) +void arm_smmu_power_off(struct arm_smmu_device *smmu, + struct arm_smmu_power_resources *pwr) { arm_smmu_power_off_atomic(smmu, pwr); arm_smmu_power_off_slow(pwr); @@ -4574,8 +4522,6 @@ static struct arm_smmu_match_data name = { \ .arch_ops = ops, \ } \ -static struct arm_smmu_arch_ops qsmmuv500_arch_ops; - ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU, NULL); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU, NULL); ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU, NULL); @@ -4978,882 +4924,6 @@ static const struct dev_pm_ops arm_smmu_pm_ops = { arm_smmu_runtime_resume, NULL) }; -static struct platform_driver arm_smmu_driver = { - .driver = { - .name = "arm-smmu", - .of_match_table = of_match_ptr(arm_smmu_of_match), - .pm = &arm_smmu_pm_ops, - .suppress_bind_attrs = true, - }, - .probe = arm_smmu_device_dt_probe, - .remove = arm_smmu_device_remove, -}; - -static struct platform_driver qsmmuv500_tbu_driver; -static int __init arm_smmu_init(void) -{ - int ret; - ktime_t cur; - - cur = ktime_get(); - ret = platform_driver_register(&qsmmuv500_tbu_driver); - if (ret) - return ret; - - ret = platform_driver_register(&arm_smmu_driver); - if (ret) { - platform_driver_unregister(&qsmmuv500_tbu_driver); - return ret; - } - - trace_smmu_init(ktime_us_delta(ktime_get(), cur)); - return ret; -} - -static void __exit arm_smmu_exit(void) -{ - platform_driver_unregister(&arm_smmu_driver); - platform_driver_unregister(&qsmmuv500_tbu_driver); -} - -subsys_initcall(arm_smmu_init); -module_exit(arm_smmu_exit); - -#define TCU_HW_VERSION_HLOS1 (0x18) - -#define DEBUG_SID_HALT_REG 0x0 -#define DEBUG_SID_HALT_REQ BIT(16) -#define DEBUG_SID_HALT_SID GENMASK(9, 0) - -#define DEBUG_VA_ADDR_REG 0x8 - -#define DEBUG_TXN_TRIGG_REG 0x18 -#define DEBUG_TXN_AXPROT GENMASK(8, 6) -#define DEBUG_TXN_AXCACHE GENMASK(5, 2) -#define DEBUG_TXN_WRITE BIT(1) -#define DEBUG_TXN_AXPROT_PRIV 0x1 -#define DEBUG_TXN_AXPROT_UNPRIV 0x0 -#define DEBUG_TXN_AXPROT_NSEC 0x2 -#define DEBUG_TXN_AXPROT_SEC 0x0 -#define DEBUG_TXN_AXPROT_INST 0x4 -#define DEBUG_TXN_AXPROT_DATA 0x0 -#define DEBUG_TXN_READ (0x0 << 1) -#define DEBUG_TXN_TRIGGER BIT(0) - -#define DEBUG_SR_HALT_ACK_REG 0x20 -#define DEBUG_SR_HALT_ACK_VAL (0x1 << 1) -#define DEBUG_SR_ECATS_RUNNING_VAL (0x1 << 0) - -#define DEBUG_PAR_REG 0x28 -#define DEBUG_PAR_PA GENMASK_ULL(47, 12) -#define DEBUG_PAR_FAULT_VAL BIT(0) - -#define DEBUG_AXUSER_REG 0x30 -#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) -#define DEBUG_AXUSER_CDMID_VAL 255 - -#define TBU_DBG_TIMEOUT_US 100 - -#define TNX_TCR_CNTL 0x130 -#define TNX_TCR_CNTL_TBU_OT_CAPTURE_EN BIT(18) -#define TNX_TCR_CNTL_ALWAYS_CAPTURE BIT(15) -#define TNX_TCR_CNTL_MATCH_MASK_UPD BIT(7) -#define TNX_TCR_CNTL_MATCH_MASK_VALID BIT(6) - -#define CAPTURE1_SNAPSHOT_1 0x138 - -#define TNX_TCR_CNTL_2 0x178 -#define TNX_TCR_CNTL_2_CAP1_VALID BIT(0) - -struct actlr_setting { - struct arm_smmu_smr smr; - u32 actlr; -}; - -struct qsmmuv500_archdata { - struct list_head tbus; - void __iomem *tcu_base; - u32 version; - struct actlr_setting *actlrs; - u32 actlr_tbl_size; - struct work_struct outstanding_tnx_work; - struct arm_smmu_device *smmu; -}; -#define get_qsmmuv500_archdata(smmu) \ - ((struct qsmmuv500_archdata *)(smmu->archdata)) - -struct qsmmuv500_tbu_device { - struct list_head list; - struct device *dev; - struct arm_smmu_device *smmu; - void __iomem *base; - void __iomem *status_reg; - - struct arm_smmu_power_resources *pwr; - u32 sid_start; - u32 num_sids; - - /* Protects halt count */ - spinlock_t halt_lock; - u32 halt_count; -}; - -struct qsmmuv500_group_iommudata { - bool has_actlr; - u32 actlr; -}; -#define to_qsmmuv500_group_iommudata(group) \ - ((struct qsmmuv500_group_iommudata *) \ - (iommu_group_get_iommudata(group))) - -static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( - struct arm_smmu_device *smmu, u32 sid); - -/* - * Provides mutually exclusive access to the registers used by the - * outstanding transaction snapshot feature and the transaction - * snapshot capture feature. - */ -static DEFINE_MUTEX(capture_reg_lock); - -static void qsmmuv500_log_outstanding_transactions(struct work_struct *work) -{ - struct qsmmuv500_tbu_device *tbu = NULL; - u64 outstanding_tnxs; - u64 tcr_cntl_val, res; - struct qsmmuv500_archdata *data = container_of(work, - struct qsmmuv500_archdata, - outstanding_tnx_work); - struct arm_smmu_device *smmu = data->smmu; - void __iomem *base; - - if (!mutex_trylock(&capture_reg_lock)) { - dev_warn_ratelimited(smmu->dev, - "Tnx snapshot regs in use, not dumping OT tnxs.\n"); - goto bug; - } - - if (arm_smmu_power_on(smmu->pwr)) { - dev_err_ratelimited(smmu->dev, - "%s: Failed to power on SMMU.\n", - __func__); - goto unlock; - } - - list_for_each_entry(tbu, &data->tbus, list) { - if (arm_smmu_power_on(tbu->pwr)) { - dev_err_ratelimited(tbu->dev, - "%s: Failed to power on TBU.\n", - __func__); - continue; - } - base = tbu->base; - - tcr_cntl_val = readq_relaxed(base + TNX_TCR_CNTL); - - /* Write 1 into MATCH_MASK_UPD of TNX_TCR_CNTL */ - writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_MATCH_MASK_UPD, - base + TNX_TCR_CNTL); - - /* - * Simultaneously write 0 into MATCH_MASK_UPD, 0 into - * ALWAYS_CAPTURE, 0 into MATCH_MASK_VALID, and 1 into - * TBU_OT_CAPTURE_EN of TNX_TCR_CNTL - */ - tcr_cntl_val &= ~(TNX_TCR_CNTL_MATCH_MASK_UPD | - TNX_TCR_CNTL_ALWAYS_CAPTURE | - TNX_TCR_CNTL_MATCH_MASK_VALID); - writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, - base + TNX_TCR_CNTL); - - /* Poll for CAPTURE1_VALID to become 1 on TNX_TCR_CNTL_2 */ - if (readq_poll_timeout_atomic(base + TNX_TCR_CNTL_2, res, - res & TNX_TCR_CNTL_2_CAP1_VALID, - 0, TBU_DBG_TIMEOUT_US)) { - dev_err_ratelimited(tbu->dev, - "Timeout on TNX snapshot poll\n"); - goto poll_timeout; - } - - /* Read Register CAPTURE1_SNAPSHOT_1 */ - outstanding_tnxs = readq_relaxed(base + CAPTURE1_SNAPSHOT_1); - dev_err_ratelimited(tbu->dev, - "Outstanding Transaction Bitmap: 0x%llx\n", - outstanding_tnxs); -poll_timeout: - /* Write TBU_OT_CAPTURE_EN to 0 of TNX_TCR_CNTL */ - writeq_relaxed(tcr_cntl_val & ~TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, - tbu->base + TNX_TCR_CNTL); - - arm_smmu_power_off(smmu, tbu->pwr); - } - - arm_smmu_power_off(smmu, smmu->pwr); -unlock: - mutex_unlock(&capture_reg_lock); -bug: - BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); -} - -static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu) -{ - u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress; - u32 tbu_inv_pending = 0, tbu_sync_pending = 0; - u32 tbu_inv_acked = 0, tbu_sync_acked = 0; - u32 tcu_inv_pending = 0, tcu_sync_pending = 0; - u32 tbu_ids = 0; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - int ret; - - static DEFINE_RATELIMIT_STATE(_rs, - DEFAULT_RATELIMIT_INTERVAL, - DEFAULT_RATELIMIT_BURST); - - dev_err_ratelimited(smmu->dev, - "TLB sync timed out -- SMMU may be deadlocked\n"); - - sync_inv_ack = arm_smmu_readl(smmu, - ARM_SMMU_IMPL_DEF0, - ARM_SMMU_STATS_SYNC_INV_TBU_ACK); - ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + - ARM_SMMU_TBU_PWR_STATUS), &tbu_pwr_status); - if (ret) { - dev_err_ratelimited(smmu->dev, - "SCM read of TBU power status fails: %d\n", - ret); - goto out; - } - - ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + - ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR), - &sync_inv_progress); - if (ret) { - dev_err_ratelimited(smmu->dev, - "SCM read of TBU sync/inv prog fails: %d\n", - ret); - goto out; - } - - if (sync_inv_ack) { - tbu_inv_pending = FIELD_GET(TBU_INV_REQ, sync_inv_ack); - tbu_inv_acked = FIELD_GET(TBU_INV_ACK, sync_inv_ack); - tbu_sync_pending = FIELD_GET(TBU_SYNC_REQ, sync_inv_ack); - tbu_sync_acked = FIELD_GET(TBU_SYNC_ACK, sync_inv_ack); - } - - if (tbu_pwr_status) { - if (tbu_sync_pending) - tbu_ids = tbu_pwr_status & ~tbu_sync_acked; - else if (tbu_inv_pending) - tbu_ids = tbu_pwr_status & ~tbu_inv_acked; - } - - tcu_inv_pending = FIELD_GET(TCU_INV_IN_PRGSS, sync_inv_progress); - tcu_sync_pending = FIELD_GET(TCU_SYNC_IN_PRGSS, sync_inv_progress); - - if (__ratelimit(&_rs)) { - unsigned long tbu_id; - - dev_err(smmu->dev, - "TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n", - sync_inv_ack, tbu_pwr_status, sync_inv_progress); - dev_err(smmu->dev, - "TCU invalidation %s, TCU sync %s\n", - tcu_inv_pending?"pending":"completed", - tcu_sync_pending?"pending":"completed"); - - for_each_set_bit(tbu_id, (unsigned long *) &tbu_ids, - sizeof(tbu_ids) * BITS_PER_BYTE) { - - struct qsmmuv500_tbu_device *tbu; - - tbu = qsmmuv500_find_tbu(smmu, - (u16)(tbu_id << TBUID_SHIFT)); - if (tbu) { - dev_err(smmu->dev, - "TBU %s ack pending for TBU %s, %s\n", - tbu_sync_pending?"sync" : "inv", - dev_name(tbu->dev), - tbu_sync_pending ? - "check pending transactions on TBU" - : "check for TBU power status"); - } - } - } - - if (tcu_sync_pending) - schedule_work(&data->outstanding_tnx_work); - else -out: - BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); -} - -static void qsmmuv500_device_remove(struct arm_smmu_device *smmu) -{ - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - cancel_work_sync(&data->outstanding_tnx_work); -} - -static bool arm_smmu_fwspec_match_smr(struct iommu_fwspec *fwspec, - struct arm_smmu_smr *smr) -{ - struct arm_smmu_smr *smr2; - struct arm_smmu_device *smmu = fwspec_smmu(fwspec); - int i, idx; - - for_each_cfg_sme(fwspec, i, idx) { - smr2 = &smmu->smrs[idx]; - /* Continue if table entry does not match */ - if ((smr->id ^ smr2->id) & ~(smr->mask | smr2->mask)) - continue; - return true; - } - return false; -} - -static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu, - struct arm_smmu_domain *smmu_domain) -{ - unsigned long flags; - struct arm_smmu_device *smmu = smmu_domain->smmu; - int idx = smmu_domain->cfg.cbndx; - u32 halt, fsr, status; - void __iomem *tbu_base; - - if (of_property_read_bool(tbu->dev->of_node, - "qcom,opt-out-tbu-halting")) { - dev_notice(tbu->dev, "TBU opted-out for halting!\n"); - return -EBUSY; - } - - spin_lock_irqsave(&tbu->halt_lock, flags); - if (tbu->halt_count) { - tbu->halt_count++; - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return 0; - } - - tbu_base = tbu->base; - halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); - halt |= DEBUG_SID_HALT_REQ; - writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if ((fsr & FSR_FAULT) && (fsr & FSR_SS)) { - u32 sctlr_orig, sctlr; - /* - * We are in a fault; Our request to halt the bus will not - * complete until transactions in front of us (such as the fault - * itself) have completed. Disable iommu faults and terminate - * any existing transactions. - */ - sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); - sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation - */ - wmb(); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - RESUME_TERMINATE); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); - } - - if (readl_poll_timeout_atomic(tbu_base + DEBUG_SR_HALT_ACK_REG, status, - (status & DEBUG_SR_HALT_ACK_VAL), - 0, TBU_DBG_TIMEOUT_US)) { - dev_err(tbu->dev, "Couldn't halt TBU!\n"); - - halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); - halt &= ~DEBUG_SID_HALT_REQ; - writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); - - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return -ETIMEDOUT; - } - - tbu->halt_count = 1; - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return 0; -} - -static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu) -{ - unsigned long flags; - u32 val; - void __iomem *base; - - spin_lock_irqsave(&tbu->halt_lock, flags); - if (!tbu->halt_count) { - WARN(1, "%s: bad tbu->halt_count", dev_name(tbu->dev)); - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return; - - } else if (tbu->halt_count > 1) { - tbu->halt_count--; - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return; - } - - base = tbu->base; - val = readl_relaxed(base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_REQ; - writel_relaxed(val, base + DEBUG_SID_HALT_REG); - - tbu->halt_count = 0; - spin_unlock_irqrestore(&tbu->halt_lock, flags); -} - -static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( - struct arm_smmu_device *smmu, u32 sid) -{ - struct qsmmuv500_tbu_device *tbu = NULL; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - list_for_each_entry(tbu, &data->tbus, list) { - if (tbu->sid_start <= sid && - sid < tbu->sid_start + tbu->num_sids) - return tbu; - } - return NULL; -} - -static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain, - struct qsmmuv500_tbu_device *tbu, - unsigned long *flags) -{ - struct arm_smmu_device *smmu = tbu->smmu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - u32 val; - - spin_lock_irqsave(&smmu->atos_lock, *flags); - /* The status register is not accessible on version 1.0 */ - if (data->version == 0x01000000) - return 0; - - if (readl_poll_timeout_atomic(tbu->status_reg, - val, (val == 0x1), 0, - TBU_DBG_TIMEOUT_US)) { - dev_err(tbu->dev, "ECATS hw busy!\n"); - spin_unlock_irqrestore(&smmu->atos_lock, *flags); - return -ETIMEDOUT; - } - - return 0; -} - -static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, - struct qsmmuv500_tbu_device *tbu, - unsigned long *flags) -{ - struct arm_smmu_device *smmu = tbu->smmu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - /* The status register is not accessible on version 1.0 */ - if (data->version != 0x01000000) - writel_relaxed(0, tbu->status_reg); - spin_unlock_irqrestore(&smmu->atos_lock, *flags); -} - -/* - * Zero means failure. - */ -static phys_addr_t qsmmuv500_iova_to_phys( - struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 sid, - unsigned long trans_flags) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct qsmmuv500_tbu_device *tbu; - int ret; - phys_addr_t phys = 0; - u64 val, fsr; - unsigned long flags; - int idx = cfg->cbndx; - u32 sctlr_orig, sctlr; - int needs_redo = 0; - ktime_t timeout; - - /* only 36 bit iova is supported */ - if (iova >= (1ULL << 36)) { - dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n", - &iova); - return 0; - } - - tbu = qsmmuv500_find_tbu(smmu, sid); - if (!tbu) - return 0; - - ret = arm_smmu_power_on(tbu->pwr); - if (ret) - return 0; - - ret = qsmmuv500_tbu_halt(tbu, smmu_domain); - if (ret) - goto out_power_off; - - /* - * ECATS can trigger the fault interrupt, so disable it temporarily - * and check for an interrupt manually. - */ - sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); - sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (fsr & FSR_FAULT) { - /* Clear pending interrupts */ - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation. - */ - wmb(); - - /* - * TBU halt takes care of resuming any stalled transcation. - * Kept it here for completeness sake. - */ - if (fsr & FSR_SS) - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - RESUME_TERMINATE); - } - - /* Only one concurrent atos operation */ - ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags); - if (ret) - goto out_resume; - -redo: - /* Set address and stream-id */ - val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_SID; - val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); - writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); - writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG); - val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); - writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); - - /* Write-back Read and Write-Allocate */ - val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF); - - /* Non-secure Access */ - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); - - /* Write or Read Access */ - if (flags & IOMMU_TRANS_WRITE) - val |= DEBUG_TXN_WRITE; - - /* Priviledged or Unpriviledged Access */ - if (flags & IOMMU_TRANS_PRIV) - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); - - /* Data or Instruction Access */ - if (flags & IOMMU_TRANS_INST) - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST); - - val |= DEBUG_TXN_TRIGGER; - writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); - - ret = 0; - timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); - for (;;) { - val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); - if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) - break; - val = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (val & FSR_FAULT) - break; - if (ktime_compare(ktime_get(), timeout) > 0) { - dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); - ret = -ETIMEDOUT; - break; - } - } - - val = readq_relaxed(tbu->base + DEBUG_PAR_REG); - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (val & DEBUG_PAR_FAULT_VAL) { - dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx, SID=0x%x\n", - fsr, sid); - - dev_err(tbu->dev, "ECATS translation failed! PAR = %llx\n", - val); - /* Clear pending interrupts */ - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation. - */ - wmb(); - - if (fsr & FSR_SS) - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - RESUME_TERMINATE); - - ret = -EINVAL; - } - - phys = FIELD_GET(DEBUG_PAR_PA, val); - if (ret < 0) - phys = 0; - - /* Reset hardware */ - writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG); - writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG); - val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_SID; - writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG); - - /* - * After a failed translation, the next successful translation will - * incorrectly be reported as a failure. - */ - if (!phys && needs_redo++ < 2) - goto redo; - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); - qsmmuv500_ecats_unlock(smmu_domain, tbu, &flags); - -out_resume: - qsmmuv500_tbu_resume(tbu); - -out_power_off: - /* Read to complete prior write transcations */ - val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); - - /* Wait for read to complete before off */ - rmb(); - - arm_smmu_power_off(tbu->smmu, tbu->pwr); - - return phys; -} - -static phys_addr_t qsmmuv500_iova_to_phys_hard( - struct arm_smmu_domain *smmu_domain, - dma_addr_t iova, - unsigned long trans_flags) -{ - u16 sid; - struct msm_iommu_domain *msm_domain = &smmu_domain->domain; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct iommu_fwspec *fwspec; - u32 frsynra; - - /* Check to see if the domain is associated with the test - * device. If the domain belongs to the test device, then - * pick the SID from fwspec. - */ - if (msm_domain->is_debug_domain) { - fwspec = dev_iommu_fwspec_get(smmu_domain->dev); - sid = (u16)fwspec->ids[0]; - } else { - - /* If the domain belongs to an actual device, read - * SID from the corresponding frsynra register - */ - frsynra = arm_smmu_gr1_read(smmu, - ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); - frsynra &= CBFRSYNRA_SID_MASK; - sid = frsynra; - } - return qsmmuv500_iova_to_phys(smmu_domain, iova, sid, trans_flags); -} - -static void qsmmuv500_release_group_iommudata(void *data) -{ - kfree(data); -} - -/* If a device has a valid actlr, it must match */ -static int qsmmuv500_device_group(struct device *dev, - struct iommu_group *group) -{ - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_device *smmu = fwspec_smmu(fwspec); - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - struct qsmmuv500_group_iommudata *iommudata; - u32 actlr, i; - struct arm_smmu_smr *smr; - - iommudata = to_qsmmuv500_group_iommudata(group); - if (!iommudata) { - iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); - if (!iommudata) - return -ENOMEM; - - iommu_group_set_iommudata(group, iommudata, - qsmmuv500_release_group_iommudata); - } - - for (i = 0; i < data->actlr_tbl_size; i++) { - smr = &data->actlrs[i].smr; - actlr = data->actlrs[i].actlr; - - if (!arm_smmu_fwspec_match_smr(fwspec, smr)) - continue; - - if (!iommudata->has_actlr) { - iommudata->actlr = actlr; - iommudata->has_actlr = true; - } else if (iommudata->actlr != actlr) { - return -EINVAL; - } - } - - return 0; -} - -static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, - struct device *dev) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct qsmmuv500_group_iommudata *iommudata = - to_qsmmuv500_group_iommudata(dev->iommu_group); - int idx = smmu_domain->cfg.cbndx; - const struct iommu_flush_ops *tlb; - - if (!iommudata->has_actlr) - return; - - tlb = smmu_domain->pgtbl_info.pgtbl_cfg.tlb; - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ACTLR, iommudata->actlr); - - /* - * Flush the context bank after modifying ACTLR to ensure there - * are no cache entries with stale state - */ - tlb->tlb_flush_all(smmu_domain); -} - -static int qsmmuv500_tbu_register(struct device *dev, void *cookie) -{ - struct arm_smmu_device *smmu = cookie; - struct qsmmuv500_tbu_device *tbu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - - if (!dev->driver) { - dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); - return -EINVAL; - } - - tbu = dev_get_drvdata(dev); - - INIT_LIST_HEAD(&tbu->list); - tbu->smmu = smmu; - list_add(&tbu->list, &data->tbus); - return 0; -} - -static int qsmmuv500_read_actlr_tbl(struct arm_smmu_device *smmu) -{ - int len, i; - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); - struct actlr_setting *actlrs; - const __be32 *cell; - - cell = of_get_property(dev->of_node, "qcom,actlr", NULL); - if (!cell) - return 0; - - len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr", - sizeof(u32) * 3); - if (len < 0) - return 0; - - actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL); - if (!actlrs) - return -ENOMEM; - - for (i = 0; i < len; i++) { - actlrs[i].smr.id = of_read_number(cell++, 1); - actlrs[i].smr.mask = of_read_number(cell++, 1); - actlrs[i].actlr = of_read_number(cell++, 1); - } - - data->actlrs = actlrs; - data->actlr_tbl_size = len; - return 0; -} - -static int qsmmuv500_arch_init(struct arm_smmu_device *smmu) -{ - struct resource *res; - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data; - struct platform_device *pdev; - int ret; - u32 val; - - data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - INIT_LIST_HEAD(&data->tbus); - - pdev = container_of(dev, struct platform_device, dev); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base"); - if (!res) { - dev_err(dev, "Unable to get the tcu-base\n"); - return -EINVAL; - } - data->tcu_base = devm_ioremap(dev, res->start, resource_size(res)); - if (IS_ERR(data->tcu_base)) - return PTR_ERR(data->tcu_base); - - data->version = readl_relaxed(data->tcu_base + TCU_HW_VERSION_HLOS1); - smmu->archdata = data; - data->smmu = smmu; - - ret = qsmmuv500_read_actlr_tbl(smmu); - if (ret) - return ret; - - val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); - val &= ~ARM_MMU500_ACR_CACHE_LOCK; - arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, val); - val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); - /* - * Modifiying the nonsecure copy of the sACR register is only - * allowed if permission is given in the secure sACR register. - * Attempt to detect if we were able to update the value. - */ - WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); - - ret = of_platform_populate(dev->of_node, NULL, NULL, dev); - if (ret) - return ret; - - INIT_WORK(&data->outstanding_tnx_work, - qsmmuv500_log_outstanding_transactions); - - /* Attempt to register child devices */ - ret = device_for_each_child(dev, smmu, qsmmuv500_tbu_register); - if (ret) - return -EPROBE_DEFER; - - return 0; -} - -static struct arm_smmu_arch_ops qsmmuv500_arch_ops = { - .init = qsmmuv500_arch_init, - .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, - .init_context_bank = qsmmuv500_init_cb, - .device_group = qsmmuv500_device_group, - .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, - .device_remove = qsmmuv500_device_remove, -}; - static const struct of_device_id qsmmuv500_tbu_of_match[] = { {.compatible = "qcom,qsmmuv500-tbu"}, {} @@ -5917,6 +4987,45 @@ static struct platform_driver qsmmuv500_tbu_driver = { .remove = qsmmuv500_tbu_remove, }; +static struct platform_driver arm_smmu_driver = { + .driver = { + .name = "arm-smmu", + .of_match_table = of_match_ptr(arm_smmu_of_match), + .pm = &arm_smmu_pm_ops, + .suppress_bind_attrs = true, + }, + .probe = arm_smmu_device_dt_probe, + .remove = arm_smmu_device_remove, +}; + +static int __init arm_smmu_init(void) +{ + int ret; + ktime_t cur; + + cur = ktime_get(); + ret = platform_driver_register(&qsmmuv500_tbu_driver); + if (ret) + return ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) { + platform_driver_unregister(&qsmmuv500_tbu_driver); + return ret; + } + + trace_smmu_init(ktime_us_delta(ktime_get(), cur)); + return ret; +} +subsys_initcall(arm_smmu_init); + +static void __exit arm_smmu_exit(void) +{ + platform_driver_unregister(&arm_smmu_driver); + platform_driver_unregister(&qsmmuv500_tbu_driver); +} +module_exit(arm_smmu_exit); + MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index 3dfe70bb5b9e..4b7447316121 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -405,6 +405,35 @@ struct arm_smmu_device { unsigned long sync_timed_out; }; +struct qsmmuv500_tbu_device { + struct list_head list; + struct device *dev; + struct arm_smmu_device *smmu; + void __iomem *base; + void __iomem *status_reg; + + struct arm_smmu_power_resources *pwr; + u32 sid_start; + u32 num_sids; + + /* Protects halt count */ + spinlock_t halt_lock; + u32 halt_count; +}; + +struct arm_smmu_master_cfg { + struct arm_smmu_device *smmu; + s16 smendx[]; +}; + +#define INVALID_SMENDX -1 +#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) +#define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) +#define fwspec_smendx(fw, i) \ + (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) +#define for_each_cfg_sme(fw, i, idx) \ + for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) + enum arm_smmu_context_fmt { ARM_SMMU_CTX_FMT_NONE, ARM_SMMU_CTX_FMT_AARCH64, @@ -465,6 +494,7 @@ struct arm_smmu_domain { /* Implementation details, yay! */ + struct arm_smmu_impl { u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset); void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset, @@ -538,12 +568,57 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, #define arm_smmu_cb_writeq(s, n, o, v) \ arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v)) +/* + * init() + * Hook for additional device tree parsing at probe time. + * + * device_reset() + * Hook for one-time architecture-specific register settings. + * + * iova_to_phys_hard() + * Provides debug information. May be called from the context fault irq handler. + * + * init_context_bank() + * Hook for architecture-specific settings which require knowledge of the + * dynamically allocated context bank number. + * + * device_group() + * Hook for checking whether a device is compatible with a said group. + * + * tlb_sync_timeout() + * Hook for performing architecture-specific procedures to collect additional + * debugging information on a TLB sync timeout. + * + * device_remove() + * Hook for performing architecture-specific procedures prior to powering off + * the SMMU. + */ +struct arm_smmu_arch_ops { + int (*init)(struct arm_smmu_device *smmu); + void (*device_reset)(struct arm_smmu_device *smmu); + phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *domain, + dma_addr_t iova, + unsigned long trans_flags); + void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, + struct device *dev); + int (*device_group)(struct device *dev, struct iommu_group *group); + void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); + void (*device_remove)(struct arm_smmu_device *smmu); +}; + struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); int arm_mmu500_reset(struct arm_smmu_device *smmu); +int arm_smmu_power_on(struct arm_smmu_power_resources *pwr); +void arm_smmu_power_off(struct arm_smmu_device *smmu, + struct arm_smmu_power_resources *pwr); + +extern struct arm_smmu_arch_ops qsmmuv500_arch_ops; + /* Misc. constants */ #define TBUID_SHIFT 10 +#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) #endif /* _ARM_SMMU_H */ From 5b663004407dfe362ad2cb4e6c15d6dfeb621890 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Fri, 21 Feb 2020 14:07:25 -0800 Subject: [PATCH 2/4] iommu/arm-smmu: Rework QSMMUV500 initialization Rework the QSMMUV500 initialization sequence, such that the QSMMUV500 implementation initialization uses the existing SMMU implementation architecture. This means ensuring that all of the implementation specific data structures are allocated as part of the initialization, as well as ensuring that all of the TBUs have probed. Also, move the logic for hardware version detection to the QSMMUV500 configuration probe function. Consequently, there are now no users of the arm_smmu_arch_init() and arm_smmu_arch_device_reset() hooks, so remove them. Change-Id: I8ca82ef8ea821b9e2171f7dca707f4da6ce2c0d7 Signed-off-by: Isaac J. Manjarres --- drivers/iommu/arm-smmu-impl.c | 2 + drivers/iommu/arm-smmu-qcom.c | 132 ++++++++++++++++++---------------- drivers/iommu/arm-smmu.c | 31 +------- drivers/iommu/arm-smmu.h | 10 +-- 4 files changed, 78 insertions(+), 97 deletions(-) diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c index c25ad938e2e6..f9fef7375f10 100644 --- a/drivers/iommu/arm-smmu-impl.c +++ b/drivers/iommu/arm-smmu-impl.c @@ -161,6 +161,8 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) break; case CAVIUM_SMMUV2: return cavium_smmu_impl_init(smmu); + case QCOM_SMMUV500: + return qsmmuv500_impl_init(smmu); default: break; } diff --git a/drivers/iommu/arm-smmu-qcom.c b/drivers/iommu/arm-smmu-qcom.c index 969fe28ae640..19a746184991 100644 --- a/drivers/iommu/arm-smmu-qcom.c +++ b/drivers/iommu/arm-smmu-qcom.c @@ -134,10 +134,11 @@ struct qsmmuv500_archdata { struct actlr_setting *actlrs; u32 actlr_tbl_size; struct work_struct outstanding_tnx_work; - struct arm_smmu_device *smmu; + spinlock_t atos_lock; + struct arm_smmu_device smmu; }; -#define get_qsmmuv500_archdata(smmu) \ - ((struct qsmmuv500_archdata *)(smmu->archdata)) +#define to_qsmmuv500_archdata(smmu) \ + container_of(smmu, struct qsmmuv500_archdata, smmu) struct qsmmuv500_group_iommudata { bool has_actlr; @@ -165,7 +166,7 @@ static void qsmmuv500_log_outstanding_transactions(struct work_struct *work) struct qsmmuv500_archdata *data = container_of(work, struct qsmmuv500_archdata, outstanding_tnx_work); - struct arm_smmu_device *smmu = data->smmu; + struct arm_smmu_device *smmu = &data->smmu; void __iomem *base; if (!mutex_trylock(&capture_reg_lock)) { @@ -243,7 +244,7 @@ static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu) u32 tbu_inv_acked = 0, tbu_sync_acked = 0; u32 tcu_inv_pending = 0, tcu_sync_pending = 0; u32 tbu_ids = 0; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); int ret; static DEFINE_RATELIMIT_STATE(_rs, @@ -332,7 +333,7 @@ out: static void qsmmuv500_device_remove(struct arm_smmu_device *smmu) { - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); cancel_work_sync(&data->outstanding_tnx_work); } @@ -455,7 +456,7 @@ static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu( struct arm_smmu_device *smmu, u32 sid) { struct qsmmuv500_tbu_device *tbu = NULL; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); list_for_each_entry(tbu, &data->tbus, list) { if (tbu->sid_start <= sid && @@ -471,10 +472,10 @@ static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain, __acquires(&smmu->atos_lock) { struct arm_smmu_device *smmu = tbu->smmu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); u32 val; - spin_lock_irqsave(&smmu->atos_lock, *flags); + spin_lock_irqsave(&data->atos_lock, *flags); /* The status register is not accessible on version 1.0 */ if (data->version == 0x01000000) return 0; @@ -483,7 +484,7 @@ static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain, val, (val == 0x1), 0, TBU_DBG_TIMEOUT_US)) { dev_err(tbu->dev, "ECATS hw busy!\n"); - spin_unlock_irqrestore(&smmu->atos_lock, *flags); + spin_unlock_irqrestore(&data->atos_lock, *flags); return -ETIMEDOUT; } @@ -496,12 +497,12 @@ static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, __releases(&smmu->atos_lock) { struct arm_smmu_device *smmu = tbu->smmu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); /* The status register is not accessible on version 1.0 */ if (data->version != 0x01000000) writel_relaxed(0, tbu->status_reg); - spin_unlock_irqrestore(&smmu->atos_lock, *flags); + spin_unlock_irqrestore(&data->atos_lock, *flags); } /* @@ -723,7 +724,7 @@ static int qsmmuv500_device_group(struct device *dev, { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_device *smmu = fwspec_smmu(fwspec); - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); struct qsmmuv500_group_iommudata *iommudata; u32 actlr, i; struct arm_smmu_smr *smr; @@ -781,9 +782,8 @@ static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, static int qsmmuv500_tbu_register(struct device *dev, void *cookie) { - struct arm_smmu_device *smmu = cookie; struct qsmmuv500_tbu_device *tbu; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct qsmmuv500_archdata *data = cookie; if (!dev->driver) { dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); @@ -793,16 +793,15 @@ static int qsmmuv500_tbu_register(struct device *dev, void *cookie) tbu = dev_get_drvdata(dev); INIT_LIST_HEAD(&tbu->list); - tbu->smmu = smmu; + tbu->smmu = &data->smmu; list_add(&tbu->list, &data->tbus); return 0; } -static int qsmmuv500_read_actlr_tbl(struct arm_smmu_device *smmu) +static int qsmmuv500_read_actlr_tbl(struct qsmmuv500_archdata *data) { int len, i; - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu); + struct device *dev = data->smmu.dev; struct actlr_setting *actlrs; const __be32 *cell; @@ -830,38 +829,12 @@ static int qsmmuv500_read_actlr_tbl(struct arm_smmu_device *smmu) return 0; } -static int qsmmuv500_arch_init(struct arm_smmu_device *smmu) +static int qsmmuv500_cfg_probe(struct arm_smmu_device *smmu) { - struct resource *res; - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data; - struct platform_device *pdev; - int ret; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); u32 val; - data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - INIT_LIST_HEAD(&data->tbus); - - pdev = container_of(dev, struct platform_device, dev); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base"); - if (!res) { - dev_err(dev, "Unable to get the tcu-base\n"); - return -EINVAL; - } - data->tcu_base = devm_ioremap(dev, res->start, resource_size(res)); - if (IS_ERR(data->tcu_base)) - return PTR_ERR(data->tcu_base); - data->version = readl_relaxed(data->tcu_base + TCU_HW_VERSION_HLOS1); - smmu->archdata = data; - data->smmu = smmu; - - ret = qsmmuv500_read_actlr_tbl(smmu); - if (ret) - return ret; val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); val &= ~ARM_MMU500_ACR_CACHE_LOCK; @@ -874,23 +847,10 @@ static int qsmmuv500_arch_init(struct arm_smmu_device *smmu) */ WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); - ret = of_platform_populate(dev->of_node, NULL, NULL, dev); - if (ret) - return ret; - - INIT_WORK(&data->outstanding_tnx_work, - qsmmuv500_log_outstanding_transactions); - - /* Attempt to register child devices */ - ret = device_for_each_child(dev, smmu, qsmmuv500_tbu_register); - if (ret) - return -EPROBE_DEFER; - return 0; } struct arm_smmu_arch_ops qsmmuv500_arch_ops = { - .init = qsmmuv500_arch_init, .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, .init_context_bank = qsmmuv500_init_cb, .device_group = qsmmuv500_device_group, @@ -898,6 +858,58 @@ struct arm_smmu_arch_ops qsmmuv500_arch_ops = { .device_remove = qsmmuv500_device_remove, }; +static const struct arm_smmu_impl qsmmuv500_impl = { + .cfg_probe = qsmmuv500_cfg_probe, +}; + +struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu) +{ + struct resource *res; + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data; + struct platform_device *pdev; + int ret; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&data->tbus); + + pdev = to_platform_device(dev); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base"); + if (!res) { + dev_err(dev, "Unable to get the tcu-base\n"); + return ERR_PTR(-EINVAL); + } + data->tcu_base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->tcu_base)) + return ERR_CAST(data->tcu_base); + + spin_lock_init(&data->atos_lock); + data->smmu = *smmu; + data->smmu.impl = &qsmmuv500_impl; + + ret = qsmmuv500_read_actlr_tbl(data); + if (ret) + return ERR_PTR(ret); + + INIT_WORK(&data->outstanding_tnx_work, + qsmmuv500_log_outstanding_transactions); + + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + return ERR_PTR(ret); + + /* Attempt to register child devices */ + ret = device_for_each_child(dev, data, qsmmuv500_tbu_register); + if (ret) + return ERR_PTR(-EPROBE_DEFER); + + devm_kfree(smmu->dev, smmu); + return &data->smmu; +} + struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { struct qcom_smmu *qsmmu; diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index a4eb32c6f252..f895b4afac0f 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -350,24 +350,6 @@ static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) } #endif -static int arm_smmu_arch_init(struct arm_smmu_device *smmu) -{ - if (!smmu->arch_ops) - return 0; - if (!smmu->arch_ops->init) - return 0; - return smmu->arch_ops->init(smmu); -} - -static void arm_smmu_arch_device_reset(struct arm_smmu_device *smmu) -{ - if (!smmu->arch_ops) - return; - if (!smmu->arch_ops->device_reset) - return; - return smmu->arch_ops->device_reset(smmu); -} - static void arm_smmu_arch_init_context_bank( struct arm_smmu_domain *smmu_domain, struct device *dev) { @@ -4002,9 +3984,6 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) arm_smmu_tlb_sync_global(smmu); wmb(); arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); - - /* Manage any implementation defined features */ - arm_smmu_arch_device_reset(smmu); } static int arm_smmu_id_size_to_bits(int size) @@ -4684,9 +4663,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) return -ENOMEM; smmu->dev = dev; - spin_lock_init(&smmu->atos_lock); - idr_init(&smmu->asid_idr); - mutex_init(&smmu->idr_mutex); data = of_device_get_match_data(dev); smmu->version = data->version; @@ -4700,6 +4676,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) if (IS_ERR(smmu)) return PTR_ERR(smmu); + idr_init(&smmu->asid_idr); + mutex_init(&smmu->idr_mutex); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (res == NULL) { dev_err(dev, "no MEM resource info\n"); @@ -4796,10 +4775,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } } - err = arm_smmu_arch_init(smmu); - if (err) - goto out_power_off; - iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops.iommu_ops); iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index 4b7447316121..fbf844fbfebb 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -394,14 +394,11 @@ struct arm_smmu_device { struct arm_smmu_power_resources *pwr; - spinlock_t atos_lock; - /* protects idr */ struct mutex idr_mutex; struct idr asid_idr; struct arm_smmu_arch_ops *arch_ops; - void *archdata; unsigned long sync_timed_out; }; @@ -569,12 +566,6 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v)) /* - * init() - * Hook for additional device tree parsing at probe time. - * - * device_reset() - * Hook for one-time architecture-specific register settings. - * * iova_to_phys_hard() * Provides debug information. May be called from the context fault irq handler. * @@ -608,6 +599,7 @@ struct arm_smmu_arch_ops { struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); +struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu); int arm_mmu500_reset(struct arm_smmu_device *smmu); From 1189ad27e687766951e5f610e7967158de89b7ee Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Fri, 21 Feb 2020 14:22:43 -0800 Subject: [PATCH 3/4] iommu/arm-smmu: Add implementation specific CB initialization hook Some SMMU implementations, such as the QSMMUV500 model, may need to program a specific context bank, in addition to the initial context bank programming. Thus, add support for implementation specific context bank initialization, and migrate the QSMMUV500 context bank initialization to use the new hook. Consequently, as there are no current users of the arm_smmu_arch_init_context_bank() hook, remove it. Change-Id: Ia89420cc8b669a142b37bf4250c09b6aa3646172 Signed-off-by: Isaac J. Manjarres --- drivers/iommu/arm-smmu-qcom.c | 2 +- drivers/iommu/arm-smmu.c | 15 ++------------- drivers/iommu/arm-smmu.h | 8 ++------ 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/arm-smmu-qcom.c b/drivers/iommu/arm-smmu-qcom.c index 19a746184991..c0fa310d047a 100644 --- a/drivers/iommu/arm-smmu-qcom.c +++ b/drivers/iommu/arm-smmu-qcom.c @@ -852,7 +852,6 @@ static int qsmmuv500_cfg_probe(struct arm_smmu_device *smmu) struct arm_smmu_arch_ops qsmmuv500_arch_ops = { .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, - .init_context_bank = qsmmuv500_init_cb, .device_group = qsmmuv500_device_group, .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, .device_remove = qsmmuv500_device_remove, @@ -860,6 +859,7 @@ struct arm_smmu_arch_ops qsmmuv500_arch_ops = { static const struct arm_smmu_impl qsmmuv500_impl = { .cfg_probe = qsmmuv500_cfg_probe, + .init_context_bank = qsmmuv500_init_cb, }; struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index f895b4afac0f..afb9dae150be 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -350,18 +350,6 @@ static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) } #endif -static void arm_smmu_arch_init_context_bank( - struct arm_smmu_domain *smmu_domain, struct device *dev) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - - if (!smmu->arch_ops) - return; - if (!smmu->arch_ops->init_context_bank) - return; - return smmu->arch_ops->init_context_bank(smmu_domain, dev); -} - static int arm_smmu_arch_device_group(struct device *dev, struct iommu_group *group) { @@ -1818,7 +1806,8 @@ static int arm_smmu_setup_context_bank(struct arm_smmu_domain *smmu_domain, arm_smmu_write_context_bank(smmu, cfg->cbndx, smmu_domain->attributes); - arm_smmu_arch_init_context_bank(smmu_domain, dev); + if (smmu->impl && smmu->impl->init_context_bank) + smmu->impl->init_context_bank(smmu_domain, dev); if (smmu->version < ARM_SMMU_V2) { cfg->irptndx = atomic_inc_return(&smmu->irptndx); diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index fbf844fbfebb..ef6d0d5274ed 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -502,6 +502,8 @@ struct arm_smmu_impl { int (*cfg_probe)(struct arm_smmu_device *smmu); int (*reset)(struct arm_smmu_device *smmu); int (*init_context)(struct arm_smmu_domain *smmu_domain); + void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, + struct device *dev); }; static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n) @@ -569,10 +571,6 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, * iova_to_phys_hard() * Provides debug information. May be called from the context fault irq handler. * - * init_context_bank() - * Hook for architecture-specific settings which require knowledge of the - * dynamically allocated context bank number. - * * device_group() * Hook for checking whether a device is compatible with a said group. * @@ -590,8 +588,6 @@ struct arm_smmu_arch_ops { phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *domain, dma_addr_t iova, unsigned long trans_flags); - void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, - struct device *dev); int (*device_group)(struct device *dev, struct iommu_group *group); void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); void (*device_remove)(struct arm_smmu_device *smmu); From f715a4c183cd0e5032037c32667114b10cadea49 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Fri, 21 Feb 2020 14:44:51 -0800 Subject: [PATCH 4/4] iommu/arm-smmu: Add support for implementation specific debugging Certain SMMU implementation, such as the QSMMUV500 model, have hardware that can be used to collect information that is useful for debugging adversarial scenarios, such as TLB sync timeouts. Add support for collecting information for debugging TLB sync timeouts, as well as triggering the SMMU HW to perform a translation for verification purpose, and wire up the QSMMUV500 implementations of these hooks. In doing so, there are no users of the iova_to_phys_hard() and tlb_sync_timeout() hooks in the arm_smmu_arch_ops structure, so remove these fields. Change-Id: I61df57e82f3058a7e8a09f4df1fe147a71ae2fc3 Signed-off-by: Isaac J. Manjarres --- drivers/iommu/arm-smmu-qcom.c | 4 ++-- drivers/iommu/arm-smmu.c | 20 +++++--------------- drivers/iommu/arm-smmu.h | 17 ++++------------- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/drivers/iommu/arm-smmu-qcom.c b/drivers/iommu/arm-smmu-qcom.c index c0fa310d047a..fd23a1fc5072 100644 --- a/drivers/iommu/arm-smmu-qcom.c +++ b/drivers/iommu/arm-smmu-qcom.c @@ -851,15 +851,15 @@ static int qsmmuv500_cfg_probe(struct arm_smmu_device *smmu) } struct arm_smmu_arch_ops qsmmuv500_arch_ops = { - .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, .device_group = qsmmuv500_device_group, - .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, .device_remove = qsmmuv500_device_remove, }; static const struct arm_smmu_impl qsmmuv500_impl = { .cfg_probe = qsmmuv500_cfg_probe, .init_context_bank = qsmmuv500_init_cb, + .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, + .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, }; struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index afb9dae150be..a274ba0c27f7 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -363,15 +363,6 @@ static int arm_smmu_arch_device_group(struct device *dev, return smmu->arch_ops->device_group(dev, group); } -static void arm_smmu_arch_tlb_sync_timeout(struct arm_smmu_device *smmu) -{ - if (!smmu->arch_ops || !smmu->arch_ops->tlb_sync_timeout) - return; - - smmu->arch_ops->tlb_sync_timeout(smmu); -} - - static void arm_smmu_arch_device_remove(struct arm_smmu_device *smmu) { if (!smmu->arch_ops) @@ -854,8 +845,8 @@ static int __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, goto out; trace_tlbsync_timeout(smmu->dev, 0); - arm_smmu_arch_tlb_sync_timeout(smmu); - + if (smmu->impl && smmu->impl->tlb_sync_timeout) + smmu->impl->tlb_sync_timeout(smmu); out: return -EINVAL; } @@ -3150,10 +3141,9 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, if (arm_smmu_power_on(smmu_domain->smmu->pwr)) return 0; - if (smmu_domain->smmu->arch_ops && - smmu_domain->smmu->arch_ops->iova_to_phys_hard) { - ret = smmu_domain->smmu->arch_ops->iova_to_phys_hard( - smmu_domain, iova, trans_flags); + if (smmu->impl && smmu->impl->iova_to_phys_hard) { + ret = smmu->impl->iova_to_phys_hard(smmu_domain, iova, + trans_flags); goto out; } diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index ef6d0d5274ed..35f7d2dbcbcb 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -504,6 +504,10 @@ struct arm_smmu_impl { int (*init_context)(struct arm_smmu_domain *smmu_domain); void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, struct device *dev); + phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain, + dma_addr_t iova, + unsigned long trans_flags); + void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); }; static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n) @@ -568,28 +572,15 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v)) /* - * iova_to_phys_hard() - * Provides debug information. May be called from the context fault irq handler. - * * device_group() * Hook for checking whether a device is compatible with a said group. * - * tlb_sync_timeout() - * Hook for performing architecture-specific procedures to collect additional - * debugging information on a TLB sync timeout. - * * device_remove() * Hook for performing architecture-specific procedures prior to powering off * the SMMU. */ struct arm_smmu_arch_ops { - int (*init)(struct arm_smmu_device *smmu); - void (*device_reset)(struct arm_smmu_device *smmu); - phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *domain, - dma_addr_t iova, - unsigned long trans_flags); int (*device_group)(struct device *dev, struct iommu_group *group); - void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); void (*device_remove)(struct arm_smmu_device *smmu); };