From d9a7beb3aad07cc94a8aa6337081b982073531dc Mon Sep 17 00:00:00 2001 From: Sudarshan Rajagopalan Date: Mon, 3 Feb 2020 17:15:39 -0800 Subject: [PATCH] iommu/arm-smmu: replicate faulty transaction With hardware having the capability, replicate the faulty transaction using ATOS to better reflect the fault transaction type with Read/Write, Privileged/Unprivileged and Data/Instruction access. This is done by extending the iova_to_phys_hard to account for the transaction type. Change-Id: Ifb3150ad78f159f04da9ba02b49851e9e119e172 Signed-off-by: Sudarshan Rajagopalan [isaacm@codeaurora.org: resolved merge conflicts] Signed-off-by: Isaac J. Manjarres --- drivers/iommu/arm-smmu.c | 105 +++++++++++++++++++++++++++--------- drivers/iommu/arm-smmu.h | 3 ++ drivers/iommu/iommu-debug.c | 17 +++--- drivers/iommu/iommu.c | 4 +- include/linux/iommu.h | 17 ++++-- 5 files changed, 108 insertions(+), 38 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 3b6086fc0746..192c3127805b 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -175,7 +175,7 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, - dma_addr_t iova); + dma_addr_t iova, unsigned long trans_flags); static void arm_smmu_destroy_domain_context(struct iommu_domain *domain); static int arm_smmu_prepare_pgtable(void *addr, void *cookie); @@ -416,7 +416,8 @@ struct arm_smmu_arch_ops { int (*init)(struct arm_smmu_device *smmu); void (*device_reset)(struct arm_smmu_device *smmu); phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain, - dma_addr_t iova); + dma_addr_t iova, + unsigned long trans_flags); void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, struct device *dev); int (*device_group)(struct device *dev, struct iommu_group *group); @@ -1332,25 +1333,57 @@ static void print_ctx_regs(struct arm_smmu_device *smmu, struct arm_smmu_cfg } static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain, - dma_addr_t iova, u32 fsr) + dma_addr_t iova, u32 fsr, u32 fsynr0) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct msm_io_pgtable_info *pgtbl_info = &smmu_domain->pgtbl_info; - phys_addr_t phys; - phys_addr_t phys_post_tlbiall; + phys_addr_t phys_hard_priv = 0; + phys_addr_t phys_stimu, phys_stimu_post_tlbiall; + unsigned long flags = 0; - phys = arm_smmu_iova_to_phys_hard(domain, iova); + /* Get the transaction type */ + if (fsynr0 & FSYNR0_WNR) + flags |= IOMMU_TRANS_WRITE; + if (fsynr0 & FSYNR0_PNU) + flags |= IOMMU_TRANS_PRIV; + if (fsynr0 & FSYNR0_IND) + flags |= IOMMU_TRANS_INST; + + /* Now replicate the faulty transaction */ + phys_stimu = arm_smmu_iova_to_phys_hard(domain, iova, flags); + + /* + * If the replicated transaction fails, it could be due to legitimate + * unmapped access (translation fault) or stale TLB with insufficient + * privileges (permission fault). Try ATOS operation with full access + * privileges to rule out stale entry with insufficient privileges case. + */ + if (!phys_stimu) + phys_hard_priv = arm_smmu_iova_to_phys_hard(domain, iova, + IOMMU_TRANS_DEFAULT | + IOMMU_TRANS_PRIV); + + /* Now replicate the faulty transaction post tlbiall */ pgtbl_info->pgtbl_cfg.tlb->tlb_flush_all(smmu_domain); - phys_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova); + phys_stimu_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova, + flags); - if (phys != phys_post_tlbiall) { + if (!phys_stimu && phys_hard_priv) { dev_err(smmu->dev, - "ATOS results differed across TLBIALL...\n" - "Before: %pa After: %pa\n", &phys, &phys_post_tlbiall); + "ATOS results differed across access privileges...\n" + "Before: %pa After: %pa\n", + &phys_stimu, &phys_hard_priv); } - return (phys == 0 ? phys_post_tlbiall : phys); + if (phys_stimu != phys_stimu_post_tlbiall) { + dev_err(smmu->dev, + "ATOS results differed across TLBIALL...\n" + "Before: %pa After: %pa\n", &phys_stimu, + &phys_stimu_post_tlbiall); + } + + return (phys_stimu == 0 ? phys_stimu_post_tlbiall : phys_stimu); } static irqreturn_t arm_smmu_context_fault(int irq, void *dev) @@ -1425,7 +1458,8 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) phys_addr_t phys_atos; print_ctx_regs(smmu, cfg, fsr); - phys_atos = arm_smmu_verify_fault(domain, iova, fsr); + phys_atos = arm_smmu_verify_fault(domain, iova, fsr, + fsynr0); dev_err(smmu->dev, "Unhandled context fault: iova=0x%08lx, cb=%d, fsr=0x%x, fsynr0=0x%x, fsynr1=0x%x\n", iova, cfg->cbndx, fsr, fsynr0, fsynr1); @@ -3173,7 +3207,7 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, * original iova_to_phys() op. */ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, - dma_addr_t iova) + dma_addr_t iova, unsigned long trans_flags) { phys_addr_t ret = 0; unsigned long flags; @@ -3189,7 +3223,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, if (smmu_domain->smmu->arch_ops && smmu_domain->smmu->arch_ops->iova_to_phys_hard) { ret = smmu_domain->smmu->arch_ops->iova_to_phys_hard( - smmu_domain, iova); + smmu_domain, iova, trans_flags); goto out; } @@ -4986,7 +5020,13 @@ module_exit(arm_smmu_exit); #define DEBUG_TXN_TRIGG_REG 0x18 #define DEBUG_TXN_AXPROT GENMASK(8, 6) #define DEBUG_TXN_AXCACHE GENMASK(5, 2) -#define DEBUG_TRX_WRITE (0x1 << 1) +#define DEBUG_TXN_WRITE BIT(1) +#define DEBUG_TXN_AXPROT_PRIV 0x1 +#define DEBUG_TXN_AXPROT_UNPRIV 0x0 +#define DEBUG_TXN_AXPROT_NSEC 0x2 +#define DEBUG_TXN_AXPROT_SEC 0x0 +#define DEBUG_TXN_AXPROT_INST 0x4 +#define DEBUG_TXN_AXPROT_DATA 0x0 #define DEBUG_TXN_READ (0x0 << 1) #define DEBUG_TXN_TRIGGER BIT(0) @@ -5414,7 +5454,8 @@ static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, * Zero means failure. */ static phys_addr_t qsmmuv500_iova_to_phys( - struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 sid) + struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 sid, + unsigned long trans_flags) { struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; @@ -5489,14 +5530,25 @@ redo: val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG); - /* - * Write-back Read and Write-Allocate - * Priviledged, nonsecure, data transaction - * Read operation. - */ - val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF) | - FIELD_PREP(DEBUG_TXN_AXPROT, 0x3) | - DEBUG_TXN_TRIGGER; + /* Write-back Read and Write-Allocate */ + val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF); + + /* Non-secure Access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); + + /* Write or Read Access */ + if (flags & IOMMU_TRANS_WRITE) + val |= DEBUG_TXN_WRITE; + + /* Priviledged or Unpriviledged Access */ + if (flags & IOMMU_TRANS_PRIV) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); + + /* Data or Instruction Access */ + if (flags & IOMMU_TRANS_INST) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST); + + val |= DEBUG_TXN_TRIGGER; writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG); ret = 0; @@ -5576,7 +5628,8 @@ out_power_off: static phys_addr_t qsmmuv500_iova_to_phys_hard( struct arm_smmu_domain *smmu_domain, - dma_addr_t iova) + dma_addr_t iova, + unsigned long trans_flags) { u16 sid; struct msm_iommu_domain *msm_domain = &smmu_domain->domain; @@ -5602,7 +5655,7 @@ static phys_addr_t qsmmuv500_iova_to_phys_hard( frsynra &= CBFRSYNRA_SID_MASK; sid = frsynra; } - return qsmmuv500_iova_to_phys(smmu_domain, iova, sid); + return qsmmuv500_iova_to_phys(smmu_domain, iova, sid, trans_flags); } static void qsmmuv500_release_group_iommudata(void *data) diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h index 691e908a04bc..5a03f078e432 100644 --- a/drivers/iommu/arm-smmu.h +++ b/drivers/iommu/arm-smmu.h @@ -209,6 +209,9 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_FSYNR0 0x68 #define FSYNR0_WNR BIT(4) +#define FSYNR0_PNU BIT(5) +#define FSYNR0_IND BIT(6) +#define FSYNR0_NSATTR BIT(8) #define ARM_SMMU_CB_FSYNR1 0x6c #define FSYNR1_BID GENMASK(15, 13) diff --git a/drivers/iommu/iommu-debug.c b/drivers/iommu/iommu-debug.c index 246e44e0b497..40641fc73a8d 100644 --- a/drivers/iommu/iommu-debug.c +++ b/drivers/iommu/iommu-debug.c @@ -902,7 +902,8 @@ out: static int __check_mapping(struct device *dev, struct iommu_domain *domain, dma_addr_t iova, phys_addr_t expected) { - phys_addr_t res = iommu_iova_to_phys_hard(domain, iova); + phys_addr_t res = iommu_iova_to_phys_hard(domain, iova, + IOMMU_TRANS_DEFAULT); phys_addr_t res2 = iommu_iova_to_phys(domain, iova); WARN(res != res2, "hard/soft iova_to_phys fns don't agree..."); @@ -1127,7 +1128,8 @@ static int __functional_dma_api_basic_test(struct device *dev, memset(data, 0xa5, size); iova = dma_map_single(dev, data, size, DMA_TO_DEVICE); pa = iommu_iova_to_phys(domain, iova); - pa2 = iommu_iova_to_phys_hard(domain, iova); + pa2 = iommu_iova_to_phys_hard(domain, iova, + IOMMU_TRANS_DEFAULT); if (pa != pa2) { dev_err_ratelimited(dev, "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n", @@ -1200,7 +1202,8 @@ static int __functional_dma_api_map_sg_test(struct device *dev, for_each_sg(table.sgl, sg, count, i) { iova = sg_dma_address(sg); pa = iommu_iova_to_phys(domain, iova); - pa2 = iommu_iova_to_phys_hard(domain, iova); + pa2 = iommu_iova_to_phys_hard(domain, iova, + IOMMU_TRANS_DEFAULT); if (pa != pa2) { dev_err_ratelimited(dev, "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n", @@ -1211,7 +1214,8 @@ static int __functional_dma_api_map_sg_test(struct device *dev, /* check mappings at end of buffer */ iova += sg_dma_len(sg) - 1; pa = iommu_iova_to_phys(domain, iova); - pa2 = iommu_iova_to_phys_hard(domain, iova); + pa2 = iommu_iova_to_phys_hard(domain, iova, + IOMMU_TRANS_DEFAULT); if (pa != pa2) { dev_err_ratelimited(dev, "iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n", @@ -1553,7 +1557,8 @@ static ssize_t iommu_debug_atos_read(struct file *file, char __user *ubuf, memset(buf, 0, 100); - phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova); + phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova, + IOMMU_TRANS_DEFAULT); if (!phys) { strlcpy(buf, "FAIL\n", 100); phys = iommu_iova_to_phys(ddev->domain, ddev->iova); @@ -1596,7 +1601,7 @@ static ssize_t iommu_debug_dma_atos_read(struct file *file, char __user *ubuf, memset(buf, 0, sizeof(buf)); phys = iommu_iova_to_phys_hard(ddev->domain, - ddev->iova); + ddev->iova, IOMMU_TRANS_DEFAULT); if (!phys) strlcpy(buf, "FAIL\n", sizeof(buf)); else diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index a51abbaf3beb..aa368ad03c19 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1838,14 +1838,14 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) EXPORT_SYMBOL_GPL(iommu_iova_to_phys); phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain, - dma_addr_t iova) + dma_addr_t iova, unsigned long trans_flags) { struct msm_iommu_ops *ops = to_msm_iommu_ops(domain->ops); if (unlikely(ops->iova_to_phys_hard == NULL)) return 0; - return ops->iova_to_phys_hard(domain, iova); + return ops->iova_to_phys_hard(domain, iova, trans_flags); } uint64_t iommu_iova_to_pte(struct iommu_domain *domain, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 38e470ea32bb..d11f0c03d2a9 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -71,6 +71,15 @@ struct iommu_domain_geometry { bool force_aperture; /* DMA only allowed in mappable range? */ }; +/* iommu transaction flags */ +#define IOMMU_TRANS_WRITE BIT(0) /* 1 Write, 0 Read */ +#define IOMMU_TRANS_PRIV BIT(1) /* 1 Privileged, 0 Unprivileged */ +#define IOMMU_TRANS_INST BIT(2) /* 1 Instruction fetch, 0 Data access */ +#define IOMMU_TRANS_SEC BIT(3) /* 1 Secure, 0 Non-secure access*/ + +/* Non secure unprivileged Data read operation */ +#define IOMMU_TRANS_DEFAULT (0U) + struct iommu_pgtbl_info { void *ops; }; @@ -384,7 +393,8 @@ struct msm_iommu_ops { size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova, struct scatterlist *sg, unsigned int nents, int prot); phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain, - dma_addr_t iova); + dma_addr_t iova, + unsigned long trans_flags); bool (*is_iova_coherent)(struct iommu_domain *domain, dma_addr_t iova); void (*trigger_fault)(struct iommu_domain *domain, unsigned long flags); void (*tlbi_domain)(struct iommu_domain *domain); @@ -519,7 +529,7 @@ extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned int nents, int prot); extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); extern phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain, - dma_addr_t iova); + dma_addr_t iova, unsigned long trans_flags); extern bool iommu_is_iova_coherent(struct iommu_domain *domain, dma_addr_t iova); extern void iommu_set_fault_handler(struct iommu_domain *domain, @@ -833,7 +843,7 @@ static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_ad } static inline phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain, - dma_addr_t iova) + dma_addr_t iova, unsigned long trans_flags) { return 0; } @@ -853,7 +863,6 @@ static inline void iommu_get_resv_regions(struct device *dev, struct list_head *list) { } - static inline void iommu_put_resv_regions(struct device *dev, struct list_head *list) {