diff --git a/drivers/gpu/msm/adreno_a6xx.c b/drivers/gpu/msm/adreno_a6xx.c index 05d5dc527b2d..cc7df774a693 100644 --- a/drivers/gpu/msm/adreno_a6xx.c +++ b/drivers/gpu/msm/adreno_a6xx.c @@ -646,27 +646,6 @@ void a6xx_start(struct adreno_device *adreno_dev) } } -/* - * a6xx_microcode_load() - Load microcode - * @adreno_dev: Pointer to adreno device - */ -static int a6xx_microcode_load(struct adreno_device *adreno_dev) -{ - struct kgsl_device *device = KGSL_DEVICE(adreno_dev); - struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); - const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); - uint64_t gpuaddr; - - gpuaddr = fw->memdesc->gpuaddr; - kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_LO, - lower_32_bits(gpuaddr)); - kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_HI, - upper_32_bits(gpuaddr)); - - return adreno_zap_shader_load(adreno_dev, a6xx_core->zap_name); -} - - /* * CP_INIT_MAX_CONTEXT bit tells if the multiple hardware contexts can * be used at once of if they should be serialized @@ -862,6 +841,8 @@ static int a6xx_post_start(struct adreno_device *adreno_dev) int a6xx_rb_start(struct adreno_device *adreno_dev) { + const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev); + struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_SQE); struct kgsl_device *device = KGSL_DEVICE(adreno_dev); struct adreno_ringbuffer *rb; uint64_t addr; @@ -900,9 +881,12 @@ int a6xx_rb_start(struct adreno_device *adreno_dev) kgsl_regwrite(device, A6XX_CP_RB_BASE_HI, upper_32_bits(rb->buffer_desc->gpuaddr)); - ret = a6xx_microcode_load(adreno_dev); - if (ret) - return ret; + /* Program the ucode base for CP */ + kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_LO, + lower_32_bits(fw->memdesc->gpuaddr)); + + kgsl_regwrite(device, A6XX_CP_SQE_INSTR_BASE_HI, + upper_32_bits(fw->memdesc->gpuaddr)); /* Clear the SQE_HALT to start the CP engine */ kgsl_regwrite(device, A6XX_CP_SQE_CNTL, 1); @@ -911,17 +895,21 @@ int a6xx_rb_start(struct adreno_device *adreno_dev) if (ret) return ret; + ret = adreno_zap_shader_load(adreno_dev, a6xx_core->zap_name); + if (ret) + return ret; + /* * Take the GPU out of secure mode. Try the zap shader if it is loaded, * otherwise just try to write directly to the secure control register */ if (!adreno_dev->zap_loaded) kgsl_regwrite(device, A6XX_RBBM_SECVID_TRUST_CNTL, 0); - else + else { ret = adreno_switch_to_unsecure_mode(adreno_dev, rb); - - if (ret) - return ret; + if (ret) + return ret; + } return a6xx_post_start(adreno_dev); } diff --git a/drivers/gpu/msm/adreno_a6xx_hfi.c b/drivers/gpu/msm/adreno_a6xx_hfi.c index 06bbe3561e04..081eecff19a8 100644 --- a/drivers/gpu/msm/adreno_a6xx_hfi.c +++ b/drivers/gpu/msm/adreno_a6xx_hfi.c @@ -517,7 +517,7 @@ static void receive_err_req(struct a6xx_gmu_device *gmu, void *rcvd) { struct hfi_err_cmd *cmd = rcvd; - dev_err(&gmu->pdev->dev, "HFI Error Received: %d %d %s\n", + dev_err(&gmu->pdev->dev, "HFI Error Received: %d %d %.16s\n", ((cmd->error_code >> 16) & 0xFFFF), (cmd->error_code & 0xFFFF), (char *) cmd->data); diff --git a/drivers/gpu/msm/adreno_dispatch.c b/drivers/gpu/msm/adreno_dispatch.c index 8710fca99199..ed4708215e6a 100644 --- a/drivers/gpu/msm/adreno_dispatch.c +++ b/drivers/gpu/msm/adreno_dispatch.c @@ -1134,8 +1134,9 @@ static inline bool _verify_ib(struct kgsl_device_private *dev_priv, return false; } - /* Make sure that the address is mapped */ - if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, ib->gpuaddr)) { + /* Make sure that the address is in range and dword aligned */ + if (!kgsl_mmu_gpuaddr_in_range(private->pagetable, ib->gpuaddr) || + !IS_ALIGNED(ib->gpuaddr, 4)) { pr_context(device, context, "ctxt %d invalid ib gpuaddr %llX\n", context->id, ib->gpuaddr); return false; diff --git a/drivers/gpu/msm/adreno_iommu.c b/drivers/gpu/msm/adreno_iommu.c index 3cd93c56568f..e3bafea479de 100644 --- a/drivers/gpu/msm/adreno_iommu.c +++ b/drivers/gpu/msm/adreno_iommu.c @@ -6,6 +6,8 @@ #include #include "a3xx_reg.h" +#include "a5xx_reg.h" +#include "a6xx_reg.h" #include "adreno.h" #include "adreno_iommu.h" #include "adreno_pm4types.h" @@ -185,6 +187,10 @@ static unsigned int _adreno_iommu_set_pt_v2_a5xx(struct kgsl_device *device, *cmds++ = upper_32_bits(ttbr0); *cmds++ = contextidr; + + *cmds++ = cp_type4_packet(A5XX_CP_CNTL, 1); + *cmds++ = 1; + *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 4, 1); cmds += cp_gpuaddr(adreno_dev, cmds, (rb->pagetable_desc->gpuaddr + PT_INFO_OFFSET(ttbr0))); @@ -192,6 +198,9 @@ static unsigned int _adreno_iommu_set_pt_v2_a5xx(struct kgsl_device *device, *cmds++ = upper_32_bits(ttbr0); *cmds++ = contextidr; + *cmds++ = cp_type4_packet(A5XX_CP_CNTL, 1); + *cmds++ = 0; + return cmds - cmds_orig; } @@ -211,6 +220,11 @@ static unsigned int _adreno_iommu_set_pt_v2_a6xx(struct kgsl_device *device, *cmds++ = contextidr; *cmds++ = cb_num; + if (!ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) { + *cmds++ = cp_type4_packet(A6XX_CP_MISC_CNTL, 1); + *cmds++ = 1; + } + *cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 4, 1); cmds += cp_gpuaddr(adreno_dev, cmds, (rb->pagetable_desc->gpuaddr + PT_INFO_OFFSET(ttbr0))); @@ -218,6 +232,11 @@ static unsigned int _adreno_iommu_set_pt_v2_a6xx(struct kgsl_device *device, *cmds++ = upper_32_bits(ttbr0); *cmds++ = contextidr; + if (!ADRENO_FEATURE(adreno_dev, ADRENO_APRIV)) { + *cmds++ = cp_type4_packet(A6XX_CP_MISC_CNTL, 1); + *cmds++ = 0; + } + return cmds - cmds_orig; } diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index f0c00ed261e7..f31e6bf6fb84 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -2345,9 +2345,8 @@ static void _setup_cache_mode(struct kgsl_mem_entry *entry, uint64_t mode; pgprot_t pgprot = vma->vm_page_prot; - if (pgprot_val(pgprot) == pgprot_val(pgprot_noncached(pgprot))) - mode = KGSL_CACHEMODE_UNCACHED; - else if (pgprot_val(pgprot) == pgprot_val(pgprot_writecombine(pgprot))) + if ((pgprot_val(pgprot) == pgprot_val(pgprot_noncached(pgprot))) || + (pgprot_val(pgprot) == pgprot_val(pgprot_writecombine(pgprot)))) mode = KGSL_CACHEMODE_WRITECOMBINE; else mode = KGSL_CACHEMODE_WRITEBACK; @@ -2355,14 +2354,6 @@ static void _setup_cache_mode(struct kgsl_mem_entry *entry, entry->memdesc.flags |= (mode << KGSL_CACHEMODE_SHIFT); } -static bool is_cached(u64 flags) -{ - u32 mode = (flags & KGSL_CACHEMODE_MASK) >> KGSL_CACHEMODE_SHIFT; - - return (mode != KGSL_CACHEMODE_UNCACHED && - mode != KGSL_CACHEMODE_WRITECOMBINE); -} - static int kgsl_setup_dma_buf(struct kgsl_device *device, struct kgsl_pagetable *pagetable, struct kgsl_mem_entry *entry, @@ -2429,7 +2420,7 @@ static int kgsl_setup_dmabuf_useraddr(struct kgsl_device *device, _setup_cache_mode(entry, vma); if (IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT) && - is_cached(entry->memdesc.flags)) + kgsl_cachemode_is_cached(entry->memdesc.flags)) entry->memdesc.flags |= KGSL_MEMFLAGS_IOCOHERENT; up_read(¤t->mm->mmap_sem); @@ -2997,7 +2988,7 @@ static int _kgsl_gpumem_sync_cache(struct kgsl_mem_entry *entry, length = entry->memdesc.size; } - if (is_cached(entry->memdesc.flags)) { + if (kgsl_cachemode_is_cached(entry->memdesc.flags)) { trace_kgsl_mem_sync_cache(entry, offset, length, op); ret = kgsl_cache_range_op(&entry->memdesc, offset, length, cacheop); @@ -3306,7 +3297,7 @@ struct kgsl_mem_entry *gpumem_alloc_entry( return ERR_PTR(-ENOMEM); if (IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT) && - is_cached(flags)) + kgsl_cachemode_is_cached(flags)) flags |= KGSL_MEMFLAGS_IOCOHERENT; ret = kgsl_allocate_user(dev_priv->device, &entry->memdesc, @@ -3532,7 +3523,7 @@ long kgsl_ioctl_sparse_phys_alloc(struct kgsl_device_private *dev_priv, if (IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT) && - is_cached(flags)) + kgsl_cachemode_is_cached(flags)) flags |= KGSL_MEMFLAGS_IOCOHERENT; ret = kgsl_allocate_user(dev_priv->device, &entry->memdesc, @@ -4625,9 +4616,6 @@ static int kgsl_mmap(struct file *file, struct vm_area_struct *vma) cache = kgsl_memdesc_get_cachemode(&entry->memdesc); switch (cache) { - case KGSL_CACHEMODE_UNCACHED: - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - break; case KGSL_CACHEMODE_WRITETHROUGH: vma->vm_page_prot = pgprot_writethroughcache(vma->vm_page_prot); if (pgprot_val(vma->vm_page_prot) == @@ -4637,6 +4625,7 @@ static int kgsl_mmap(struct file *file, struct vm_area_struct *vma) case KGSL_CACHEMODE_WRITEBACK: vma->vm_page_prot = pgprot_writebackcache(vma->vm_page_prot); break; + case KGSL_CACHEMODE_UNCACHED: case KGSL_CACHEMODE_WRITECOMBINE: default: vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c index 74486c07a22a..aa12bafebc0a 100644 --- a/drivers/gpu/msm/kgsl_iommu.c +++ b/drivers/gpu/msm/kgsl_iommu.c @@ -438,50 +438,6 @@ struct _mem_entry { char name[32]; }; -static void _get_global_entries(struct kgsl_mmu *mmu, uint64_t faultaddr, - struct _mem_entry *prev, - struct _mem_entry *next) -{ - struct kgsl_device *device = KGSL_MMU_DEVICE(mmu); - struct kgsl_global_memdesc *p = NULL, *n = NULL, *md; - uint64_t prevaddr = 0; - uint64_t nextaddr = (uint64_t) -1; - - list_for_each_entry(md, &device->globals, node) { - struct kgsl_memdesc *memdesc = &md->memdesc; - u64 addr; - - addr = memdesc->gpuaddr; - if ((addr < faultaddr) && (addr > prevaddr)) { - prevaddr = addr; - p = md; - } - - if ((addr > faultaddr) && (addr < nextaddr)) { - nextaddr = addr; - n = md; - } - } - - if (p != NULL) { - prev->gpuaddr = p->memdesc.gpuaddr; - prev->size = p->memdesc.size; - prev->flags = p->memdesc.flags; - prev->priv = p->memdesc.priv; - prev->pid = 0; - strlcpy(prev->name, p->name, sizeof(prev->name)); - } - - if (n != NULL) { - next->gpuaddr = n->memdesc.gpuaddr; - next->size = n->memdesc.size; - next->flags = n->memdesc.flags; - next->priv = n->memdesc.priv; - next->pid = 0; - strlcpy(next->name, n->name, sizeof(next->name)); - } -} - static void _get_entries(struct kgsl_process_private *private, uint64_t faultaddr, struct _mem_entry *prev, struct _mem_entry *next) @@ -542,9 +498,7 @@ static void _find_mem_entries(struct kgsl_mmu *mmu, uint64_t faultaddr, /* Set the maximum possible size as an initial value */ nextentry->gpuaddr = (uint64_t) -1; - if (ADDR_IN_GLOBAL(mmu, faultaddr)) { - _get_global_entries(mmu, faultaddr, preventry, nextentry); - } else if (private) { + if (private) { spin_lock(&private->mem_lock); _get_entries(private, faultaddr, preventry, nextentry); spin_unlock(&private->mem_lock); @@ -737,27 +691,36 @@ static int kgsl_iommu_fault_handler(struct iommu_domain *domain, if (!(flags & IOMMU_FAULT_PERMISSION)) { _check_if_freed(ctx, addr, ptname); - dev_err(ctx->kgsldev->dev, + /* + * Don't print any debug information if the address is + * in the global region. These are rare and nobody needs + * to know the addresses that are in here + */ + if (ADDR_IN_GLOBAL(mmu, addr)) { + dev_err(ctx->kgsldev->dev, "Fault in global memory\n"); + } else { + dev_err(ctx->kgsldev->dev, "---- nearby memory ----\n"); - _find_mem_entries(mmu, addr, &prev, &next, private); - if (prev.gpuaddr) - _print_entry(ctx->kgsldev, &prev); - else - dev_err(ctx->kgsldev->dev, "*EMPTY*\n"); + _find_mem_entries(mmu, addr, &prev, &next, + private); + if (prev.gpuaddr) + _print_entry(ctx->kgsldev, &prev); + else + dev_err(ctx->kgsldev->dev, "*EMPTY*\n"); - dev_err(ctx->kgsldev->dev, + dev_err(ctx->kgsldev->dev, " <- fault @ %8.8lX\n", addr); - if (next.gpuaddr != (uint64_t) -1) - _print_entry(ctx->kgsldev, &next); - else - dev_err(ctx->kgsldev->dev, "*EMPTY*\n"); + if (next.gpuaddr != (uint64_t) -1) + _print_entry(ctx->kgsldev, &next); + else + dev_err(ctx->kgsldev->dev, "*EMPTY*\n"); + } } } - /* * We do not want the h/w to resume fetching data from an iommu * that has faulted, this is better for debugging as it will stall @@ -1059,6 +1022,7 @@ static void _free_pt(struct kgsl_iommu_context *ctx, struct kgsl_pagetable *pt) static void _enable_gpuhtw_llc(struct kgsl_mmu *mmu, struct iommu_domain *domain) { + struct kgsl_device *device = KGSL_MMU_DEVICE(mmu); int attr, ret; u32 val = 1; @@ -1072,12 +1036,10 @@ static void _enable_gpuhtw_llc(struct kgsl_mmu *mmu, ret = iommu_domain_set_attr(domain, attr, &val); - /* - * Warn once if the system cache will not be used for GPU - * pagetable walks. This is not a fatal error. - */ - WARN_ONCE(ret, "System cache not enabled for GPU pagetable walks: %d\n", - ret); + /* Print a one time error message if system cache isn't enabled */ + if (ret) + dev_err_once(device->dev, + "System cache no-write-alloc is disabled for GPU pagetables\n"); } static int set_smmu_aperture(struct kgsl_device *device, int cb_num) @@ -1302,9 +1264,15 @@ static int _init_per_process_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) ret = iommu_domain_set_attr(iommu_pt->domain, DOMAIN_ATTR_DYNAMIC, &dynamic); + + /* + * If -ENOTSUPP then dynamic pagetables aren't supported. Quietly + * return the error and the upper levels will handle it + */ if (ret) { - dev_err(device->dev, - "set DOMAIN_ATTR_DYNAMIC failed: %d\n", ret); + if (ret != -ENOTSUPP) + dev_err(device->dev, + "set DOMAIN_ATTR_DYNAMIC failed: %d\n", ret); goto done; } @@ -1383,6 +1351,7 @@ static int kgsl_iommu_init_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt) static struct kgsl_pagetable *kgsl_iommu_getpagetable(struct kgsl_mmu *mmu, unsigned long name) { + struct kgsl_device *device = KGSL_MMU_DEVICE(mmu); struct kgsl_pagetable *pt; if (!kgsl_mmu_is_perprocess(mmu) && (name != KGSL_MMU_SECURE_PT) && @@ -1393,9 +1362,23 @@ static struct kgsl_pagetable *kgsl_iommu_getpagetable(struct kgsl_mmu *mmu, } pt = kgsl_get_pagetable(name); - if (pt == NULL) + if (pt == NULL) { pt = kgsl_mmu_createpagetableobject(mmu, name); + /* + * Special fallback case -if we get ENOTSUPP that means that + * per-process pagetables are not supported by arm-smmu. This + * should happen on the first try so we safely set the global + * pagetable bit and avoid going down this path again + */ + if (PTR_ERR_OR_ZERO(pt) == -ENOTSUPP) { + dev_err_once(device->dev, + "Couldn't enable per-process pagetables. Default to global pagetables\n"); + set_bit(KGSL_MMU_GLOBAL_PAGETABLE, &mmu->features); + return mmu->defaultpagetable; + } + } + return pt; } diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c index 774452c1d4c6..c4d3ee7e6967 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.c +++ b/drivers/gpu/msm/kgsl_sharedmem.c @@ -8,7 +8,6 @@ #include #include #include -#include #include "kgsl_device.h" #include "kgsl_pool.h" @@ -564,7 +563,6 @@ void kgsl_memdesc_init(struct kgsl_device *device, { struct kgsl_mmu *mmu = &device->mmu; unsigned int align; - u32 cachemode; memset(memdesc, 0, sizeof(*memdesc)); /* Turn off SVM if the system doesn't support it */ @@ -590,9 +588,7 @@ void kgsl_memdesc_init(struct kgsl_device *device, * have stale data. This happens primarily due to the limitations * of dma caching APIs available on arm64 */ - cachemode = FIELD_GET(KGSL_CACHEMODE_MASK, flags); - if ((cachemode == KGSL_CACHEMODE_WRITECOMBINE || - cachemode == KGSL_CACHEMODE_UNCACHED)) + if (!kgsl_cachemode_is_cached(flags)) flags &= ~((u64) KGSL_MEMFLAGS_IOCOHERENT); if (kgsl_mmu_has_feature(device, KGSL_MMU_NEED_GUARD_PAGE) || diff --git a/drivers/gpu/msm/kgsl_sharedmem.h b/drivers/gpu/msm/kgsl_sharedmem.h index 390a010ecb47..20b176996aa2 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.h +++ b/drivers/gpu/msm/kgsl_sharedmem.h @@ -5,6 +5,7 @@ #ifndef __KGSL_SHAREDMEM_H #define __KGSL_SHAREDMEM_H +#include #include #include #include @@ -359,4 +360,18 @@ static inline void kgsl_free_sgt(struct sg_table *sgt) } } +/** + * kgsl_cachemode_is_cached - Return true if the passed flags indicate a cached + * buffer + * @flags: A bitmask of KGSL_MEMDESC_ flags + * + * Return: true if the flags indicate a cached buffer + */ +static inline bool kgsl_cachemode_is_cached(u64 flags) +{ + u64 mode = FIELD_GET(KGSL_CACHEMODE_MASK, flags); + + return (mode != KGSL_CACHEMODE_UNCACHED && + mode != KGSL_CACHEMODE_WRITECOMBINE); +} #endif /* __KGSL_SHAREDMEM_H */