From 0c294ef2ffff73618442c484c7eb1ec0e1adf01a Mon Sep 17 00:00:00 2001 From: Jordan Crouse Date: Tue, 7 Apr 2020 10:27:10 -0600 Subject: [PATCH] msm: kgsl: Add support for split pagetables Attempt to enable split pagetables on the global and LPAC domains. If successful we no longer need to map globals in all pagetables and we don't need to manually return to the default pagetable when coming out of slumber since all the globals will now live in the always on TTBR1 region. Change-Id: Ic0dedbaddd7d05299d7e85d7b0bbc96b46dad10e Signed-off-by: Jordan Crouse Signed-off-by: Harshdeep Dhatt --- drivers/gpu/msm/kgsl_iommu.c | 111 +++++++++++++++++++++++++++++++---- drivers/gpu/msm/kgsl_mmu.h | 6 ++ 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c index 56c29edc5565..97a9f028a182 100644 --- a/drivers/gpu/msm/kgsl_iommu.c +++ b/drivers/gpu/msm/kgsl_iommu.c @@ -25,10 +25,8 @@ #define _IOMMU_PRIV(_mmu) (&((_mmu)->priv.iommu)) -#define ADDR_IN_GLOBAL(_mmu, _a) \ - (((_a) >= KGSL_IOMMU_GLOBAL_MEM_BASE(_mmu)) && \ - ((_a) < (KGSL_IOMMU_GLOBAL_MEM_BASE(_mmu) + \ - KGSL_IOMMU_GLOBAL_MEM_SIZE))) + +#define KGSL_IOMMU_SPLIT_TABLE_BASE 0x0001ff8000000000ULL /* * Flag to set SMMU memory attributes required to @@ -59,6 +57,22 @@ struct kgsl_iommu_addr_entry { static struct kmem_cache *addr_entry_cache; +static bool kgsl_iommu_split_tables_enabled(struct kgsl_mmu *mmu) +{ + return (test_bit(KGSL_MMU_SPLIT_TABLES_GC, &mmu->features) && + test_bit(KGSL_MMU_SPLIT_TABLES_LPAC, &mmu->features)); +} + +static bool kgsl_iommu_addr_is_global(struct kgsl_mmu *mmu, u64 addr) +{ + if (kgsl_iommu_split_tables_enabled(mmu)) + return (addr >= KGSL_IOMMU_SPLIT_TABLE_BASE); + + return ((addr >= KGSL_IOMMU_GLOBAL_MEM_BASE(mmu)) && + (addr < KGSL_IOMMU_GLOBAL_MEM_BASE(mmu) + + KGSL_IOMMU_GLOBAL_MEM_SIZE)); +} + static void __iomem *kgsl_iommu_reg(struct kgsl_iommu_context *ctx, u32 offset) { @@ -116,12 +130,22 @@ static u32 KGSL_IOMMU_GET_CTX_REG(struct kgsl_iommu_context *ctx, u32 offset) return readl_relaxed(addr); } +static bool kgsl_iommu_is_global_pt(struct kgsl_pagetable *pt) +{ + return (pt->name == KGSL_MMU_GLOBAL_PT || + pt->name == KGSL_MMU_GLOBAL_LPAC_PT); +} + static void kgsl_iommu_unmap_globals(struct kgsl_mmu *mmu, struct kgsl_pagetable *pagetable) { struct kgsl_device *device = KGSL_MMU_DEVICE(mmu); struct kgsl_global_memdesc *md; + if (!kgsl_iommu_is_global_pt(pagetable) + && kgsl_iommu_split_tables_enabled(mmu)) + return; + list_for_each_entry(md, &device->globals, node) { if (md->memdesc.flags & KGSL_MEMFLAGS_SECURE) continue; @@ -136,6 +160,14 @@ static void kgsl_iommu_map_globals(struct kgsl_mmu *mmu, struct kgsl_device *device = KGSL_MMU_DEVICE(mmu); struct kgsl_global_memdesc *md; + if (IS_ERR_OR_NULL(pagetable)) + return; + + if (!kgsl_iommu_is_global_pt(pagetable) + && kgsl_iommu_split_tables_enabled(mmu)) { + return; + } + list_for_each_entry(md, &device->globals, node) { if (md->memdesc.flags & KGSL_MEMFLAGS_SECURE) continue; @@ -229,13 +261,21 @@ static void kgsl_iommu_map_global(struct kgsl_mmu *mmu, } if (!memdesc->gpuaddr) { - u64 offset = global_get_offset(device, memdesc->size, + u64 offset; + u64 base; + + offset = global_get_offset(device, memdesc->size, memdesc->priv); if (IS_ERR_VALUE(offset)) return; - memdesc->gpuaddr = KGSL_IOMMU_GLOBAL_MEM_BASE(mmu) + offset; + if (kgsl_iommu_split_tables_enabled(mmu)) + base = KGSL_IOMMU_SPLIT_TABLE_BASE; + else + base = KGSL_IOMMU_GLOBAL_MEM_BASE(mmu); + + memdesc->gpuaddr = base + offset; } /* @@ -243,7 +283,7 @@ static void kgsl_iommu_map_global(struct kgsl_mmu *mmu, * been created since we do not go back and retroactively add the * globals to existing pages */ - WARN_ON(iommu->ppt_active); + WARN_ON(!kgsl_iommu_split_tables_enabled(mmu) && iommu->ppt_active); kgsl_iommu_map_global_to_pt(mmu, memdesc, mmu->defaultpagetable); kgsl_iommu_map_global_to_pt(mmu, memdesc, mmu->lpac_pagetable); @@ -284,6 +324,10 @@ static int _iommu_map_single_page(struct kgsl_pagetable *pt, int i; int ret = 0; + /* Sign extend TTBR1 addresses all the way to avoid warning */ + if (gpuaddr & (1ULL << 48)) + gpuaddr |= 0xffff000000000000; + for (i = 0; i < times; i++) { ret = iommu_map(iommu_pt->domain, gpuaddr + mapped, physaddr, PAGE_SIZE, flags); @@ -311,6 +355,10 @@ static int _iommu_unmap(struct kgsl_pagetable *pt, struct kgsl_iommu_pt *iommu_pt = pt->priv; size_t unmapped = 0; + /* Sign extend TTBR1 addresses all the way to avoid warning */ + if (addr & (1ULL << 48)) + addr |= 0xffff000000000000; + unmapped = iommu_unmap(iommu_pt->domain, addr, size); if (unmapped != size) { @@ -336,6 +384,10 @@ static int _iommu_map_sg_offset(struct kgsl_pagetable *pt, phys_addr_t physaddr; int ret; + /* Sign extend TTBR1 addresses all the way to avoid warning */ + if (addr & (1ULL << 48)) + addr |= 0xffff000000000000; + for_each_sg(sg, s, nents, i) { /* Iterate until we find the offset */ if (offset_tmp >= s->length) { @@ -391,6 +443,10 @@ static int _iommu_map_sg(struct kgsl_pagetable *pt, struct kgsl_iommu_pt *iommu_pt = pt->priv; size_t mapped; + /* Sign extend TTBR1 addresses all the way to avoid warning */ + if (addr & (1ULL << 48)) + addr |= 0xffff000000000000; + mapped = iommu_map_sg(iommu_pt->domain, addr, sg, nents, flags); if (mapped == 0) { @@ -710,7 +766,7 @@ static int kgsl_iommu_fault_handler(struct iommu_domain *domain, * in the global region. These are rare and nobody needs * to know the addresses that are in here */ - if (ADDR_IN_GLOBAL(mmu, addr)) { + if (kgsl_iommu_addr_is_global(mmu, addr)) { dev_err(ctx->kgsldev->dev, "Fault in global memory\n"); } else { dev_err(ctx->kgsldev->dev, @@ -1074,10 +1130,20 @@ static int set_smmu_aperture(struct kgsl_device *device, int cb_num) return ret; } +static bool check_split_tables(struct kgsl_iommu_pt *iommu_pt) +{ + int val, ret; + + ret = iommu_domain_get_attr(iommu_pt->domain, + DOMAIN_ATTR_SPLIT_TABLES, &val); + + return (!ret && val == 1); +} + static int _init_global_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) { struct kgsl_device *device = KGSL_MMU_DEVICE(mmu); - int ret = 0; + int ret = 0, val = 1; struct kgsl_iommu_pt *iommu_pt = NULL; unsigned int cb_num; struct kgsl_iommu *iommu = _IOMMU_PRIV(mmu); @@ -1101,6 +1167,10 @@ static int _init_global_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) _enable_gpuhtw_llc(mmu, iommu_pt->domain); + if (test_bit(KGSL_MMU_64BIT, &mmu->features)) + iommu_domain_set_attr(iommu_pt->domain, + DOMAIN_ATTR_SPLIT_TABLES, &val); + ret = _attach_pt(iommu_pt, ctx); if (ret) goto done; @@ -1141,7 +1211,8 @@ static int _init_global_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) goto done; } - kgsl_iommu_map_globals(mmu, pt); + if (check_split_tables(iommu_pt)) + set_bit(KGSL_MMU_SPLIT_TABLES_GC, &mmu->features); done: if (ret) @@ -1153,7 +1224,7 @@ done: static int _init_global_lpac_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) { struct kgsl_device *device = KGSL_MMU_DEVICE(mmu); - int ret = 0; + int ret = 0, val = 1; struct kgsl_iommu_pt *iommu_pt = NULL; unsigned int cb_num; struct kgsl_iommu *iommu = _IOMMU_PRIV(mmu); @@ -1166,6 +1237,8 @@ static int _init_global_lpac_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) _enable_gpuhtw_llc(mmu, iommu_pt->domain); + iommu_domain_set_attr(iommu_pt->domain, DOMAIN_ATTR_SPLIT_TABLES, &val); + ret = _attach_pt(iommu_pt, ctx); if (ret) goto done; @@ -1194,7 +1267,8 @@ static int _init_global_lpac_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) ctx->cb_num = (int) cb_num; - kgsl_iommu_map_globals(mmu, pt); + if (check_split_tables(iommu_pt)) + set_bit(KGSL_MMU_SPLIT_TABLES_LPAC, &mmu->features); done: if (ret) @@ -1970,6 +2044,10 @@ static int kgsl_iommu_set_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) unsigned int contextidr; unsigned long wait_for_flush; + /* Not needed if split tables are enabled */ + if (kgsl_iommu_split_tables_enabled(mmu)) + return 0; + if ((pt != mmu->defaultpagetable) && !kgsl_mmu_is_perprocess(mmu)) return 0; @@ -2622,6 +2700,15 @@ int kgsl_iommu_probe(struct kgsl_device *device) iommu_probe_secure_context(device, node); of_node_put(node); + /* + * To preserve legacy behavior, make sure to map any globals that might + * have been allocated before the MMU probe. Do this after the probes + * so that we can be sure that split pagetable support is available + * (since both GC and LPAC need to enable it). + */ + kgsl_iommu_map_globals(mmu, mmu->defaultpagetable); + kgsl_iommu_map_globals(mmu, mmu->lpac_pagetable); + device->qdss_desc = kgsl_allocate_global_fixed(device, "qcom,gpu-qdss-stm", "gpu-qdss"); diff --git a/drivers/gpu/msm/kgsl_mmu.h b/drivers/gpu/msm/kgsl_mmu.h index 8ce9c1ac228d..0451310b37fe 100644 --- a/drivers/gpu/msm/kgsl_mmu.h +++ b/drivers/gpu/msm/kgsl_mmu.h @@ -128,6 +128,12 @@ enum kgsl_mmu_feature { KGSL_MMU_LLCC_ENABLE, /** @KGSL_MMU_SMMU_APERTURE: Set the SMMU aperture */ KGSL_MMU_SMMU_APERTURE, + /** @KGSL_MMU_SPLIT_TABLES_GC: Split pagetables are enabled for GC */ + KGSL_MMU_SPLIT_TABLES_GC, + /** + * @KGSL_MMU_SPLIT_TABLES_LPAC: Split pagetables are enabled for LPAC + */ + KGSL_MMU_SPLIT_TABLES_LPAC, }; /**