A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-08 21:22:09 -04:00
Code Issues Projects Releases Packages Wiki Activity

msm: kgsl: Add support for the Adreno GPU

Browse source 
First take a snapshot of the Adreno GPU (KGSL) driver from msm-4.19
commit 963db0f98552 ("msm: ipa4: add generic header processing context").

Then copy the msm_adreno_devfreq.h header from its previous location in
include/linux and the user API header from include/uapi/linux into the
driver directory. Both these files no longer need to be in the "generic"
headers.

Next do cleanups, lots of cleanups. Remove a bunch of extra features that
are from long departed targets or that have otherwise been replaced and
improved.

And finally cleanup kernel APIs that are no longer used, remove the
exported symbols from the GPU driver since nobody depends on us and do a
few other minor bits to put us on a track to being friendly with a modern
kernel.

Change-Id: Ic0dedbad5200dd9359a7b2fc06de415d755934e9
Signed-off-by: Jordan Crouse <jcrouse@codeaurora.org>
This commit is contained in:
Jordan Crouse 2019-11-22 08:53:39 -07:00
commit 7a6368ee0b
88 changed files with 68495 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

1
drivers/gpu/Makefile
View file

@ -5,3 +5,4 @@
obj-$(CONFIG_TEGRA_HOST1X) += host1x/
obj-y += drm/ vga/
obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/
obj-$(CONFIG_QCOM_KGSL) += msm/

15
drivers/gpu/msm/Kconfig Normal file
View file

@ -0,0 +1,15 @@
# SPDX-License-Identifier: GPL-2.0-only
config QCOM_KGSL
tristate "Qualcomm Technologies, Inc. 3D Graphics driver"
depends on ARCH_QCOM
depends on QCOM_QFPROM
help
3D graphics driver for the Adreno family of GPUs from QTI.
Required to use hardware accelerated OpenGL, compute and Vulkan
on QTI targets. This includes power management, memory management,
and scheduling for the Adreno GPUs.
config QCOM_ADRENO_DEFAULT_GOVERNOR
string "devfreq governor for the adreno core"
default "simple_ondemand"
depends on QCOM_KGSL

53
drivers/gpu/msm/Makefile Normal file
View file

@ -0,0 +1,53 @@
# SPDX-License-Identifier: GPL-2.0-only
ccflags-y += -I$(src)
obj-$(CONFIG_QCOM_KGSL) += msm_kgsl.o
msm_kgsl-y = \
kgsl.o \
kgsl_drawobj.o \
kgsl_events.o \
kgsl_ioctl.o \
kgsl_gmu.o \
kgsl_gmu_core.o \
kgsl_hfi.o \
kgsl_mmu.o \
kgsl_pool.o \
kgsl_pwrctrl.o \
kgsl_pwrscale.o \
kgsl_rgmu.o \
kgsl_sharedmem.o \
kgsl_snapshot.o \
kgsl_trace.o
msm_kgsl-$(CONFIG_COMPAT) += kgsl_compat.o
msm_kgsl-$(CONFIG_DEBUG_FS) += kgsl_debugfs.o
msm_kgsl-$(CONFIG_ARM_SMMU) += kgsl_iommu.o
msm_kgsl-$(CONFIG_SYNC_FILE) += kgsl_sync.o
msm_kgsl-y += \
adreno.o \
adreno_a3xx.o \
adreno_a3xx_snapshot.o \
adreno_a5xx.o \
adreno_a5xx_preempt.o \
adreno_a5xx_snapshot.o \
adreno_a6xx.o \
adreno_a6xx_gmu.o \
adreno_a6xx_preempt.o \
adreno_a6xx_rgmu.o \
adreno_a6xx_snapshot.o \
adreno_coresight.o \
adreno_cp_parser.o \
adreno_dispatch.o \
adreno_drawctxt.o \
adreno_ioctl.o \
adreno_perfcounter.o \
adreno_ringbuffer.o \
adreno_snapshot.o \
adreno_sysfs.o \
adreno_trace.o
msm_kgsl-$(CONFIG_COMPAT) += adreno_compat.o
msm_kgsl-$(CONFIG_DEBUG_FS) += adreno_debugfs.o adreno_profile.o
msm_kgsl-$(CONFIG_ARM_SMMU) += adreno_iommu.o

567
drivers/gpu/msm/a3xx_reg.h Normal file
View file

@ -0,0 +1,567 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2012-2017,2019, The Linux Foundation. All rights reserved.
*/
#ifndef _A300_REG_H
#define _A300_REG_H
/* Interrupt bit positions within RBBM_INT_0 */
#define A3XX_INT_RBBM_GPU_IDLE 0
#define A3XX_INT_RBBM_AHB_ERROR 1
#define A3XX_INT_RBBM_REG_TIMEOUT 2
#define A3XX_INT_RBBM_ME_MS_TIMEOUT 3
#define A3XX_INT_RBBM_PFP_MS_TIMEOUT 4
#define A3XX_INT_RBBM_ATB_BUS_OVERFLOW 5
#define A3XX_INT_VFD_ERROR 6
#define A3XX_INT_CP_SW_INT 7
#define A3XX_INT_CP_T0_PACKET_IN_IB 8
#define A3XX_INT_CP_OPCODE_ERROR 9
#define A3XX_INT_CP_RESERVED_BIT_ERROR 10
#define A3XX_INT_CP_HW_FAULT 11
#define A3XX_INT_CP_DMA 12
#define A3XX_INT_CP_IB2_INT 13
#define A3XX_INT_CP_IB1_INT 14
#define A3XX_INT_CP_RB_INT 15
#define A3XX_INT_CP_REG_PROTECT_FAULT 16
#define A3XX_INT_CP_RB_DONE_TS 17
#define A3XX_INT_CP_VS_DONE_TS 18
#define A3XX_INT_CP_PS_DONE_TS 19
#define A3XX_INT_CACHE_FLUSH_TS 20
#define A3XX_INT_CP_AHB_ERROR_HALT 21
#define A3XX_INT_MISC_HANG_DETECT 24
#define A3XX_INT_UCHE_OOB_ACCESS 25
/* CP_EVENT_WRITE events */
#define CACHE_FLUSH_TS 4
/* Register definitions */
#define A3XX_RBBM_CLOCK_CTL 0x010
#define A3XX_RBBM_SP_HYST_CNT 0x012
#define A3XX_RBBM_SW_RESET_CMD 0x018
#define A3XX_RBBM_AHB_CTL0 0x020
#define A3XX_RBBM_AHB_CTL1 0x021
#define A3XX_RBBM_AHB_CMD 0x022
#define A3XX_RBBM_AHB_ERROR_STATUS 0x027
#define A3XX_RBBM_GPR0_CTL 0x02E
/* This the same register as on A2XX, just in a different place */
#define A3XX_RBBM_STATUS 0x030
#define A3XX_RBBM_WAIT_IDLE_CLOCKS_CTL 0x33
#define A3XX_RBBM_INTERFACE_HANG_INT_CTL 0x50
#define A3XX_RBBM_INT_CLEAR_CMD 0x061
#define A3XX_RBBM_INT_0_MASK 0x063
#define A3XX_RBBM_INT_0_STATUS 0x064
#define A3XX_RBBM_PERFCTR_CTL 0x80
#define A3XX_RBBM_PERFCTR_LOAD_CMD0 0x81
#define A3XX_RBBM_PERFCTR_LOAD_CMD1 0x82
#define A3XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x84
#define A3XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x85
#define A3XX_RBBM_PERFCOUNTER0_SELECT 0x86
#define A3XX_RBBM_PERFCOUNTER1_SELECT 0x87
#define A3XX_RBBM_GPU_BUSY_MASKED 0x88
#define A3XX_RBBM_PERFCTR_CP_0_LO 0x90
#define A3XX_RBBM_PERFCTR_CP_0_HI 0x91
#define A3XX_RBBM_PERFCTR_RBBM_0_LO 0x92
#define A3XX_RBBM_PERFCTR_RBBM_0_HI 0x93
#define A3XX_RBBM_PERFCTR_RBBM_1_LO 0x94
#define A3XX_RBBM_PERFCTR_RBBM_1_HI 0x95
#define A3XX_RBBM_PERFCTR_PC_0_LO 0x96
#define A3XX_RBBM_PERFCTR_PC_0_HI 0x97
#define A3XX_RBBM_PERFCTR_PC_1_LO 0x98
#define A3XX_RBBM_PERFCTR_PC_1_HI 0x99
#define A3XX_RBBM_PERFCTR_PC_2_LO 0x9A
#define A3XX_RBBM_PERFCTR_PC_2_HI 0x9B
#define A3XX_RBBM_PERFCTR_PC_3_LO 0x9C
#define A3XX_RBBM_PERFCTR_PC_3_HI 0x9D
#define A3XX_RBBM_PERFCTR_VFD_0_LO 0x9E
#define A3XX_RBBM_PERFCTR_VFD_0_HI 0x9F
#define A3XX_RBBM_PERFCTR_VFD_1_LO 0xA0
#define A3XX_RBBM_PERFCTR_VFD_1_HI 0xA1
#define A3XX_RBBM_PERFCTR_HLSQ_0_LO 0xA2
#define A3XX_RBBM_PERFCTR_HLSQ_0_HI 0xA3
#define A3XX_RBBM_PERFCTR_HLSQ_1_LO 0xA4
#define A3XX_RBBM_PERFCTR_HLSQ_1_HI 0xA5
#define A3XX_RBBM_PERFCTR_HLSQ_2_LO 0xA6
#define A3XX_RBBM_PERFCTR_HLSQ_2_HI 0xA7
#define A3XX_RBBM_PERFCTR_HLSQ_3_LO 0xA8
#define A3XX_RBBM_PERFCTR_HLSQ_3_HI 0xA9
#define A3XX_RBBM_PERFCTR_HLSQ_4_LO 0xAA
#define A3XX_RBBM_PERFCTR_HLSQ_4_HI 0xAB
#define A3XX_RBBM_PERFCTR_HLSQ_5_LO 0xAC
#define A3XX_RBBM_PERFCTR_HLSQ_5_HI 0xAD
#define A3XX_RBBM_PERFCTR_VPC_0_LO 0xAE
#define A3XX_RBBM_PERFCTR_VPC_0_HI 0xAF
#define A3XX_RBBM_PERFCTR_VPC_1_LO 0xB0
#define A3XX_RBBM_PERFCTR_VPC_1_HI 0xB1
#define A3XX_RBBM_PERFCTR_TSE_0_LO 0xB2
#define A3XX_RBBM_PERFCTR_TSE_0_HI 0xB3
#define A3XX_RBBM_PERFCTR_TSE_1_LO 0xB4
#define A3XX_RBBM_PERFCTR_TSE_1_HI 0xB5
#define A3XX_RBBM_PERFCTR_RAS_0_LO 0xB6
#define A3XX_RBBM_PERFCTR_RAS_0_HI 0xB7
#define A3XX_RBBM_PERFCTR_RAS_1_LO 0xB8
#define A3XX_RBBM_PERFCTR_RAS_1_HI 0xB9
#define A3XX_RBBM_PERFCTR_UCHE_0_LO 0xBA
#define A3XX_RBBM_PERFCTR_UCHE_0_HI 0xBB
#define A3XX_RBBM_PERFCTR_UCHE_1_LO 0xBC
#define A3XX_RBBM_PERFCTR_UCHE_1_HI 0xBD
#define A3XX_RBBM_PERFCTR_UCHE_2_LO 0xBE
#define A3XX_RBBM_PERFCTR_UCHE_2_HI 0xBF
#define A3XX_RBBM_PERFCTR_UCHE_3_LO 0xC0
#define A3XX_RBBM_PERFCTR_UCHE_3_HI 0xC1
#define A3XX_RBBM_PERFCTR_UCHE_4_LO 0xC2
#define A3XX_RBBM_PERFCTR_UCHE_4_HI 0xC3
#define A3XX_RBBM_PERFCTR_UCHE_5_LO 0xC4
#define A3XX_RBBM_PERFCTR_UCHE_5_HI 0xC5
#define A3XX_RBBM_PERFCTR_TP_0_LO 0xC6
#define A3XX_RBBM_PERFCTR_TP_0_HI 0xC7
#define A3XX_RBBM_PERFCTR_TP_1_LO 0xC8
#define A3XX_RBBM_PERFCTR_TP_1_HI 0xC9
#define A3XX_RBBM_PERFCTR_TP_2_LO 0xCA
#define A3XX_RBBM_PERFCTR_TP_2_HI 0xCB
#define A3XX_RBBM_PERFCTR_TP_3_LO 0xCC
#define A3XX_RBBM_PERFCTR_TP_3_HI 0xCD
#define A3XX_RBBM_PERFCTR_TP_4_LO 0xCE
#define A3XX_RBBM_PERFCTR_TP_4_HI 0xCF
#define A3XX_RBBM_PERFCTR_TP_5_LO 0xD0
#define A3XX_RBBM_PERFCTR_TP_5_HI 0xD1
#define A3XX_RBBM_PERFCTR_SP_0_LO 0xD2
#define A3XX_RBBM_PERFCTR_SP_0_HI 0xD3
#define A3XX_RBBM_PERFCTR_SP_1_LO 0xD4
#define A3XX_RBBM_PERFCTR_SP_1_HI 0xD5
#define A3XX_RBBM_PERFCTR_SP_2_LO 0xD6
#define A3XX_RBBM_PERFCTR_SP_2_HI 0xD7
#define A3XX_RBBM_PERFCTR_SP_3_LO 0xD8
#define A3XX_RBBM_PERFCTR_SP_3_HI 0xD9
#define A3XX_RBBM_PERFCTR_SP_4_LO 0xDA
#define A3XX_RBBM_PERFCTR_SP_4_HI 0xDB
#define A3XX_RBBM_PERFCTR_SP_5_LO 0xDC
#define A3XX_RBBM_PERFCTR_SP_5_HI 0xDD
#define A3XX_RBBM_PERFCTR_SP_6_LO 0xDE
#define A3XX_RBBM_PERFCTR_SP_6_HI 0xDF
#define A3XX_RBBM_PERFCTR_SP_7_LO 0xE0
#define A3XX_RBBM_PERFCTR_SP_7_HI 0xE1
#define A3XX_RBBM_PERFCTR_RB_0_LO 0xE2
#define A3XX_RBBM_PERFCTR_RB_0_HI 0xE3
#define A3XX_RBBM_PERFCTR_RB_1_LO 0xE4
#define A3XX_RBBM_PERFCTR_RB_1_HI 0xE5
#define A3XX_RBBM_RBBM_CTL 0x100
#define A3XX_RBBM_PERFCTR_PWR_0_LO 0x0EA
#define A3XX_RBBM_PERFCTR_PWR_0_HI 0x0EB
#define A3XX_RBBM_PERFCTR_PWR_1_LO 0x0EC
#define A3XX_RBBM_PERFCTR_PWR_1_HI 0x0ED
#define A3XX_RBBM_DEBUG_BUS_CTL 0x111
#define A3XX_RBBM_DEBUG_BUS_DATA_STATUS 0x112
#define A3XX_RBBM_DEBUG_BUS_STB_CTL0 0x11B
#define A3XX_RBBM_DEBUG_BUS_STB_CTL1 0x11C
#define A3XX_RBBM_INT_TRACE_BUS_CTL 0x11D
#define A3XX_RBBM_EXT_TRACE_BUS_CTL 0x11E
#define A3XX_RBBM_EXT_TRACE_STOP_CNT 0x11F
#define A3XX_RBBM_EXT_TRACE_START_CNT 0x120
#define A3XX_RBBM_EXT_TRACE_PERIOD_CNT 0x121
#define A3XX_RBBM_EXT_TRACE_CMD 0x122
#define A3XX_CP_RB_BASE 0x01C0
#define A3XX_CP_RB_CNTL 0x01C1
#define A3XX_CP_RB_RPTR 0x01C4
#define A3XX_CP_RB_WPTR 0x01C5
/* Following two are same as on A2XX, just in a different place */
#define A3XX_CP_PFP_UCODE_ADDR 0x1C9
#define A3XX_CP_PFP_UCODE_DATA 0x1CA
#define A3XX_CP_ROQ_ADDR 0x1CC
#define A3XX_CP_ROQ_DATA 0x1CD
#define A3XX_CP_MERCIU_ADDR 0x1D1
#define A3XX_CP_MERCIU_DATA 0x1D2
#define A3XX_CP_MERCIU_DATA2 0x1D3
#define A3XX_CP_QUEUE_THRESHOLDS 0x01D5
#define A3XX_CP_MEQ_ADDR 0x1DA
#define A3XX_CP_MEQ_DATA 0x1DB
#define A3XX_CP_STATE_DEBUG_INDEX 0x01EC
#define A3XX_CP_STATE_DEBUG_DATA 0x01ED
#define A3XX_CP_CNTL 0x01F4
#define A3XX_CP_WFI_PEND_CTR 0x01F5
#define A3XX_CP_ME_CNTL 0x01F6
#define A3XX_CP_ME_STATUS 0x01F7
#define A3XX_CP_ME_RAM_WADDR 0x01F8
#define A3XX_CP_ME_RAM_RADDR 0x01F9
#define A3XX_CP_ME_RAM_DATA 0x01FA
#define A3XX_CP_DEBUG 0x01FC
#define A3XX_RBBM_PM_OVERRIDE2 0x039D
#define A3XX_CP_PERFCOUNTER_SELECT 0x445
#define A3XX_CP_IB1_BASE 0x0458
#define A3XX_CP_IB1_BUFSZ 0x0459
#define A3XX_CP_IB2_BASE 0x045A
#define A3XX_CP_IB2_BUFSZ 0x045B
#define A3XX_CP_HW_FAULT 0x45C
#define A3XX_CP_PROTECT_CTRL 0x45E
#define A3XX_CP_PROTECT_STATUS 0x45F
#define A3XX_CP_PROTECT_REG_0 0x460
#define A3XX_CP_STAT 0x047F
#define A3XX_CP_SCRATCH_REG0 0x578
#define A3XX_CP_SCRATCH_REG6 0x57E
#define A3XX_CP_SCRATCH_REG7 0x57F
#define A3XX_VSC_SIZE_ADDRESS 0xC02
#define A3XX_VSC_PIPE_DATA_ADDRESS_0 0xC07
#define A3XX_VSC_PIPE_DATA_LENGTH_0 0xC08
#define A3XX_VSC_PIPE_DATA_ADDRESS_1 0xC0A
#define A3XX_VSC_PIPE_DATA_LENGTH_1 0xC0B
#define A3XX_VSC_PIPE_DATA_ADDRESS_2 0xC0D
#define A3XX_VSC_PIPE_DATA_LENGTH_2 0xC0E
#define A3XX_VSC_PIPE_DATA_ADDRESS_3 0xC10
#define A3XX_VSC_PIPE_DATA_LENGTH_3 0xC11
#define A3XX_VSC_PIPE_DATA_ADDRESS_4 0xC13
#define A3XX_VSC_PIPE_DATA_LENGTH_4 0xC14
#define A3XX_VSC_PIPE_DATA_ADDRESS_5 0xC16
#define A3XX_VSC_PIPE_DATA_LENGTH_5 0xC17
#define A3XX_VSC_PIPE_DATA_ADDRESS_6 0xC19
#define A3XX_VSC_PIPE_DATA_LENGTH_6 0xC1A
#define A3XX_VSC_PIPE_DATA_ADDRESS_7 0xC1C
#define A3XX_VSC_PIPE_DATA_LENGTH_7 0xC1D
#define A3XX_PC_PERFCOUNTER0_SELECT 0xC48
#define A3XX_PC_PERFCOUNTER1_SELECT 0xC49
#define A3XX_PC_PERFCOUNTER2_SELECT 0xC4A
#define A3XX_PC_PERFCOUNTER3_SELECT 0xC4B
#define A3XX_GRAS_TSE_DEBUG_ECO 0xC81
#define A3XX_GRAS_PERFCOUNTER0_SELECT 0xC88
#define A3XX_GRAS_PERFCOUNTER1_SELECT 0xC89
#define A3XX_GRAS_PERFCOUNTER2_SELECT 0xC8A
#define A3XX_GRAS_PERFCOUNTER3_SELECT 0xC8B
#define A3XX_GRAS_CL_USER_PLANE_X0 0xCA0
#define A3XX_GRAS_CL_USER_PLANE_Y0 0xCA1
#define A3XX_GRAS_CL_USER_PLANE_Z0 0xCA2
#define A3XX_GRAS_CL_USER_PLANE_W0 0xCA3
#define A3XX_GRAS_CL_USER_PLANE_X1 0xCA4
#define A3XX_GRAS_CL_USER_PLANE_Y1 0xCA5
#define A3XX_GRAS_CL_USER_PLANE_Z1 0xCA6
#define A3XX_GRAS_CL_USER_PLANE_W1 0xCA7
#define A3XX_GRAS_CL_USER_PLANE_X2 0xCA8
#define A3XX_GRAS_CL_USER_PLANE_Y2 0xCA9
#define A3XX_GRAS_CL_USER_PLANE_Z2 0xCAA
#define A3XX_GRAS_CL_USER_PLANE_W2 0xCAB
#define A3XX_GRAS_CL_USER_PLANE_X3 0xCAC
#define A3XX_GRAS_CL_USER_PLANE_Y3 0xCAD
#define A3XX_GRAS_CL_USER_PLANE_Z3 0xCAE
#define A3XX_GRAS_CL_USER_PLANE_W3 0xCAF
#define A3XX_GRAS_CL_USER_PLANE_X4 0xCB0
#define A3XX_GRAS_CL_USER_PLANE_Y4 0xCB1
#define A3XX_GRAS_CL_USER_PLANE_Z4 0xCB2
#define A3XX_GRAS_CL_USER_PLANE_W4 0xCB3
#define A3XX_GRAS_CL_USER_PLANE_X5 0xCB4
#define A3XX_GRAS_CL_USER_PLANE_Y5 0xCB5
#define A3XX_GRAS_CL_USER_PLANE_Z5 0xCB6
#define A3XX_GRAS_CL_USER_PLANE_W5 0xCB7
#define A3XX_RB_GMEM_BASE_ADDR 0xCC0
#define A3XX_RB_DEBUG_ECO_CONTROLS_ADDR 0xCC1
#define A3XX_RB_PERFCOUNTER0_SELECT 0xCC6
#define A3XX_RB_PERFCOUNTER1_SELECT 0xCC7
#define A3XX_RB_FRAME_BUFFER_DIMENSION 0xCE0
#define A3XX_SQ_GPR_MANAGEMENT 0x0D00
#define A3XX_SQ_INST_STORE_MANAGEMENT 0x0D02
#define A3XX_HLSQ_PERFCOUNTER0_SELECT 0xE00
#define A3XX_HLSQ_PERFCOUNTER1_SELECT 0xE01
#define A3XX_HLSQ_PERFCOUNTER2_SELECT 0xE02
#define A3XX_HLSQ_PERFCOUNTER3_SELECT 0xE03
#define A3XX_HLSQ_PERFCOUNTER4_SELECT 0xE04
#define A3XX_HLSQ_PERFCOUNTER5_SELECT 0xE05
#define A3XX_TP0_CHICKEN 0x0E1E
#define A3XX_VFD_PERFCOUNTER0_SELECT 0xE44
#define A3XX_VFD_PERFCOUNTER1_SELECT 0xE45
#define A3XX_VPC_VPC_DEBUG_RAM_SEL 0xE61
#define A3XX_VPC_VPC_DEBUG_RAM_READ 0xE62
#define A3XX_VPC_PERFCOUNTER0_SELECT 0xE64
#define A3XX_VPC_PERFCOUNTER1_SELECT 0xE65
#define A3XX_UCHE_CACHE_MODE_CONTROL_REG 0xE82
#define A3XX_UCHE_PERFCOUNTER0_SELECT 0xE84
#define A3XX_UCHE_PERFCOUNTER1_SELECT 0xE85
#define A3XX_UCHE_PERFCOUNTER2_SELECT 0xE86
#define A3XX_UCHE_PERFCOUNTER3_SELECT 0xE87
#define A3XX_UCHE_PERFCOUNTER4_SELECT 0xE88
#define A3XX_UCHE_PERFCOUNTER5_SELECT 0xE89
#define A3XX_UCHE_CACHE_INVALIDATE0_REG 0xEA0
#define A3XX_UCHE_CACHE_INVALIDATE1_REG 0xEA1
#define A3XX_UCHE_CACHE_WAYS_VFD 0xEA6
#define A3XX_SP_PERFCOUNTER0_SELECT 0xEC4
#define A3XX_SP_PERFCOUNTER1_SELECT 0xEC5
#define A3XX_SP_PERFCOUNTER2_SELECT 0xEC6
#define A3XX_SP_PERFCOUNTER3_SELECT 0xEC7
#define A3XX_SP_PERFCOUNTER4_SELECT 0xEC8
#define A3XX_SP_PERFCOUNTER5_SELECT 0xEC9
#define A3XX_SP_PERFCOUNTER6_SELECT 0xECA
#define A3XX_SP_PERFCOUNTER7_SELECT 0xECB
#define A3XX_TP_PERFCOUNTER0_SELECT 0xF04
#define A3XX_TP_PERFCOUNTER1_SELECT 0xF05
#define A3XX_TP_PERFCOUNTER2_SELECT 0xF06
#define A3XX_TP_PERFCOUNTER3_SELECT 0xF07
#define A3XX_TP_PERFCOUNTER4_SELECT 0xF08
#define A3XX_TP_PERFCOUNTER5_SELECT 0xF09
#define A3XX_GRAS_CL_CLIP_CNTL 0x2040
#define A3XX_GRAS_CL_GB_CLIP_ADJ 0x2044
#define A3XX_GRAS_CL_VPORT_XOFFSET 0x2048
#define A3XX_GRAS_CL_VPORT_XSCALE 0x2049
#define A3XX_GRAS_CL_VPORT_YOFFSET 0x204A
#define A3XX_GRAS_CL_VPORT_YSCALE 0x204B
#define A3XX_GRAS_CL_VPORT_ZOFFSET 0x204C
#define A3XX_GRAS_CL_VPORT_ZSCALE 0x204D
#define A3XX_GRAS_SU_POINT_MINMAX 0x2068
#define A3XX_GRAS_SU_POINT_SIZE 0x2069
#define A3XX_GRAS_SU_POLY_OFFSET_SCALE 0x206C
#define A3XX_GRAS_SU_POLY_OFFSET_OFFSET 0x206D
#define A3XX_GRAS_SU_MODE_CONTROL 0x2070
#define A3XX_GRAS_SC_CONTROL 0x2072
#define A3XX_GRAS_SC_SCREEN_SCISSOR_TL 0x2074
#define A3XX_GRAS_SC_SCREEN_SCISSOR_BR 0x2075
#define A3XX_GRAS_SC_WINDOW_SCISSOR_TL 0x2079
#define A3XX_GRAS_SC_WINDOW_SCISSOR_BR 0x207A
#define A3XX_RB_MODE_CONTROL 0x20C0
#define A3XX_RB_RENDER_CONTROL 0x20C1
#define A3XX_RB_MSAA_CONTROL 0x20C2
#define A3XX_RB_ALPHA_REFERENCE 0x20C3
#define A3XX_RB_MRT_CONTROL0 0x20C4
#define A3XX_RB_MRT_BUF_INFO0 0x20C5
#define A3XX_RB_MRT_BUF_BASE0 0x20C6
#define A3XX_RB_MRT_BLEND_CONTROL0 0x20C7
#define A3XX_RB_MRT_CONTROL1 0x20C8
#define A3XX_RB_MRT_BUF_INFO1 0x20C9
#define A3XX_RB_MRT_BUF_BASE1 0x20CA
#define A3XX_RB_MRT_BLEND_CONTROL1 0x20CB
#define A3XX_RB_MRT_CONTROL2 0x20CC
#define A3XX_RB_MRT_BUF_INFO2 0x20CD
#define A3XX_RB_MRT_BUF_BASE2 0x20CE
#define A3XX_RB_MRT_BLEND_CONTROL2 0x20CF
#define A3XX_RB_MRT_CONTROL3 0x20D0
#define A3XX_RB_MRT_BUF_INFO3 0x20D1
#define A3XX_RB_MRT_BUF_BASE3 0x20D2
#define A3XX_RB_MRT_BLEND_CONTROL3 0x20D3
#define A3XX_RB_BLEND_RED 0x20E4
#define A3XX_RB_BLEND_GREEN 0x20E5
#define A3XX_RB_BLEND_BLUE 0x20E6
#define A3XX_RB_BLEND_ALPHA 0x20E7
#define A3XX_RB_CLEAR_COLOR_DW0 0x20E8
#define A3XX_RB_CLEAR_COLOR_DW1 0x20E9
#define A3XX_RB_CLEAR_COLOR_DW2 0x20EA
#define A3XX_RB_CLEAR_COLOR_DW3 0x20EB
#define A3XX_RB_COPY_CONTROL 0x20EC
#define A3XX_RB_COPY_DEST_BASE 0x20ED
#define A3XX_RB_COPY_DEST_PITCH 0x20EE
#define A3XX_RB_COPY_DEST_INFO 0x20EF
#define A3XX_RB_DEPTH_CONTROL 0x2100
#define A3XX_RB_DEPTH_CLEAR 0x2101
#define A3XX_RB_DEPTH_BUF_INFO 0x2102
#define A3XX_RB_DEPTH_BUF_PITCH 0x2103
#define A3XX_RB_STENCIL_CONTROL 0x2104
#define A3XX_RB_STENCIL_CLEAR 0x2105
#define A3XX_RB_STENCIL_BUF_INFO 0x2106
#define A3XX_RB_STENCIL_BUF_PITCH 0x2107
#define A3XX_RB_STENCIL_REF_MASK 0x2108
#define A3XX_RB_STENCIL_REF_MASK_BF 0x2109
#define A3XX_RB_LRZ_VSC_CONTROL 0x210C
#define A3XX_RB_WINDOW_OFFSET 0x210E
#define A3XX_RB_SAMPLE_COUNT_CONTROL 0x2110
#define A3XX_RB_SAMPLE_COUNT_ADDR 0x2111
#define A3XX_RB_Z_CLAMP_MIN 0x2114
#define A3XX_RB_Z_CLAMP_MAX 0x2115
#define A3XX_HLSQ_CONTROL_0_REG 0x2200
#define A3XX_HLSQ_CONTROL_1_REG 0x2201
#define A3XX_HLSQ_CONTROL_2_REG 0x2202
#define A3XX_HLSQ_CONTROL_3_REG 0x2203
#define A3XX_HLSQ_VS_CONTROL_REG 0x2204
#define A3XX_HLSQ_FS_CONTROL_REG 0x2205
#define A3XX_HLSQ_CONST_VSPRESV_RANGE_REG 0x2206
#define A3XX_HLSQ_CONST_FSPRESV_RANGE_REG 0x2207
#define A3XX_HLSQ_CL_NDRANGE_0_REG 0x220A
#define A3XX_HLSQ_CL_NDRANGE_1_REG 0x220B
#define A3XX_HLSQ_CL_NDRANGE_2_REG 0x220C
#define A3XX_HLSQ_CL_NDRANGE_3_REG 0x220D
#define A3XX_HLSQ_CL_NDRANGE_4_REG 0x220E
#define A3XX_HLSQ_CL_NDRANGE_5_REG 0x220F
#define A3XX_HLSQ_CL_NDRANGE_6_REG 0x2210
#define A3XX_HLSQ_CL_CONTROL_0_REG 0x2211
#define A3XX_HLSQ_CL_CONTROL_1_REG 0x2212
#define A3XX_HLSQ_CL_KERNEL_CONST_REG 0x2214
#define A3XX_HLSQ_CL_KERNEL_GROUP_X_REG 0x2215
#define A3XX_HLSQ_CL_KERNEL_GROUP_Y_REG 0x2216
#define A3XX_HLSQ_CL_KERNEL_GROUP_Z_REG 0x2217
#define A3XX_HLSQ_CL_WG_OFFSET_REG 0x221A
#define A3XX_VFD_FETCH_INSTR_1_0 0x2247
#define A3XX_VFD_FETCH_INSTR_1_1 0x2249
#define A3XX_VFD_FETCH_INSTR_1_2 0x224B
#define A3XX_VFD_FETCH_INSTR_1_3 0x224D
#define A3XX_VFD_FETCH_INSTR_1_4 0x224F
#define A3XX_VFD_FETCH_INSTR_1_5 0x2251
#define A3XX_VFD_FETCH_INSTR_1_6 0x2253
#define A3XX_VFD_FETCH_INSTR_1_7 0x2255
#define A3XX_VFD_FETCH_INSTR_1_8 0x2257
#define A3XX_VFD_FETCH_INSTR_1_9 0x2259
#define A3XX_VFD_FETCH_INSTR_1_A 0x225B
#define A3XX_VFD_FETCH_INSTR_1_B 0x225D
#define A3XX_VFD_FETCH_INSTR_1_C 0x225F
#define A3XX_VFD_FETCH_INSTR_1_D 0x2261
#define A3XX_VFD_FETCH_INSTR_1_E 0x2263
#define A3XX_VFD_FETCH_INSTR_1_F 0x2265
#define A3XX_SP_SP_CTRL_REG 0x22C0
#define A3XX_SP_VS_CTRL_REG0 0x22C4
#define A3XX_SP_VS_CTRL_REG1 0x22C5
#define A3XX_SP_VS_PARAM_REG 0x22C6
#define A3XX_SP_VS_OUT_REG_0 0x22C7
#define A3XX_SP_VS_OUT_REG_1 0x22C8
#define A3XX_SP_VS_OUT_REG_2 0x22C9
#define A3XX_SP_VS_OUT_REG_3 0x22CA
#define A3XX_SP_VS_OUT_REG_4 0x22CB
#define A3XX_SP_VS_OUT_REG_5 0x22CC
#define A3XX_SP_VS_OUT_REG_6 0x22CD
#define A3XX_SP_VS_OUT_REG_7 0x22CE
#define A3XX_SP_VS_VPC_DST_REG_0 0x22D0
#define A3XX_SP_VS_VPC_DST_REG_1 0x22D1
#define A3XX_SP_VS_VPC_DST_REG_2 0x22D2
#define A3XX_SP_VS_VPC_DST_REG_3 0x22D3
#define A3XX_SP_VS_OBJ_OFFSET_REG 0x22D4
#define A3XX_SP_VS_OBJ_START_REG 0x22D5
#define A3XX_SP_VS_PVT_MEM_PARAM_REG 0x22D6
#define A3XX_SP_VS_PVT_MEM_ADDR_REG 0x22D7
#define A3XX_SP_VS_PVT_MEM_SIZE_REG 0x22D8
#define A3XX_SP_VS_LENGTH_REG 0x22DF
#define A3XX_SP_FS_CTRL_REG0 0x22E0
#define A3XX_SP_FS_CTRL_REG1 0x22E1
#define A3XX_SP_FS_OBJ_OFFSET_REG 0x22E2
#define A3XX_SP_FS_OBJ_START_REG 0x22E3
#define A3XX_SP_FS_PVT_MEM_PARAM_REG 0x22E4
#define A3XX_SP_FS_PVT_MEM_ADDR_REG 0x22E5
#define A3XX_SP_FS_PVT_MEM_SIZE_REG 0x22E6
#define A3XX_SP_FS_FLAT_SHAD_MODE_REG_0 0x22E8
#define A3XX_SP_FS_FLAT_SHAD_MODE_REG_1 0x22E9
#define A3XX_SP_FS_OUTPUT_REG 0x22EC
#define A3XX_SP_FS_MRT_REG_0 0x22F0
#define A3XX_SP_FS_MRT_REG_1 0x22F1
#define A3XX_SP_FS_MRT_REG_2 0x22F2
#define A3XX_SP_FS_MRT_REG_3 0x22F3
#define A3XX_SP_FS_IMAGE_OUTPUT_REG_0 0x22F4
#define A3XX_SP_FS_IMAGE_OUTPUT_REG_1 0x22F5
#define A3XX_SP_FS_IMAGE_OUTPUT_REG_2 0x22F6
#define A3XX_SP_FS_IMAGE_OUTPUT_REG_3 0x22F7
#define A3XX_SP_FS_LENGTH_REG 0x22FF
#define A3XX_PA_SC_AA_CONFIG 0x2301
#define A3XX_VBIF_CLKON 0x3001
#define A3XX_VBIF_ABIT_SORT 0x301C
#define A3XX_VBIF_ABIT_SORT_CONF 0x301D
#define A3XX_VBIF_GATE_OFF_WRREQ_EN 0x302A
#define A3XX_VBIF_IN_RD_LIM_CONF0 0x302C
#define A3XX_VBIF_IN_RD_LIM_CONF1 0x302D
#define A3XX_VBIF_IN_WR_LIM_CONF0 0x3030
#define A3XX_VBIF_IN_WR_LIM_CONF1 0x3031
#define A3XX_VBIF_OUT_RD_LIM_CONF0 0x3034
#define A3XX_VBIF_OUT_WR_LIM_CONF0 0x3035
#define A3XX_VBIF_DDR_OUT_MAX_BURST 0x3036
#define A3XX_VBIF_ARB_CTL 0x303C
#define A3XX_VBIF_ROUND_ROBIN_QOS_ARB 0x3049
#define A3XX_VBIF_OUT_AXI_AOOO_EN 0x305E
#define A3XX_VBIF_OUT_AXI_AOOO 0x305F
#define A3XX_VBIF_PERF_CNT0_LO 0x3073
#define A3XX_VBIF_PERF_CNT0_HI 0x3074
#define A3XX_VBIF_PERF_CNT1_LO 0x3075
#define A3XX_VBIF_PERF_CNT1_HI 0x3076
#define A3XX_VBIF_PERF_PWR_CNT0_LO 0x3077
#define A3XX_VBIF_PERF_PWR_CNT0_HI 0x3078
#define A3XX_VBIF_PERF_PWR_CNT1_LO 0x3079
#define A3XX_VBIF_PERF_PWR_CNT1_HI 0x307a
#define A3XX_VBIF_PERF_PWR_CNT2_LO 0x307b
#define A3XX_VBIF_PERF_PWR_CNT2_HI 0x307c
#define A3XX_VBIF_XIN_HALT_CTRL0 0x3080
#define A3XX_VBIF_XIN_HALT_CTRL0_MASK 0x3F
#define A30X_VBIF_XIN_HALT_CTRL0_MASK 0x7
#define A3XX_VBIF_XIN_HALT_CTRL1 0x3081
/* VBIF register offsets for A306 */
#define A3XX_VBIF2_PERF_CNT_SEL0 0x30d0
#define A3XX_VBIF2_PERF_CNT_SEL1 0x30d1
#define A3XX_VBIF2_PERF_CNT_SEL2 0x30d2
#define A3XX_VBIF2_PERF_CNT_SEL3 0x30d3
#define A3XX_VBIF2_PERF_CNT_LOW0 0x30d8
#define A3XX_VBIF2_PERF_CNT_LOW1 0x30d9
#define A3XX_VBIF2_PERF_CNT_LOW2 0x30da
#define A3XX_VBIF2_PERF_CNT_LOW3 0x30db
#define A3XX_VBIF2_PERF_CNT_HIGH0 0x30e0
#define A3XX_VBIF2_PERF_CNT_HIGH1 0x30e1
#define A3XX_VBIF2_PERF_CNT_HIGH2 0x30e2
#define A3XX_VBIF2_PERF_CNT_HIGH3 0x30e3
#define A3XX_VBIF2_PERF_PWR_CNT_EN0 0x3100
#define A3XX_VBIF2_PERF_PWR_CNT_EN1 0x3101
#define A3XX_VBIF2_PERF_PWR_CNT_EN2 0x3102
#define A3XX_VBIF2_PERF_PWR_CNT_LOW0 0x3110
#define A3XX_VBIF2_PERF_PWR_CNT_LOW1 0x3111
#define A3XX_VBIF2_PERF_PWR_CNT_LOW2 0x3112
#define A3XX_VBIF2_PERF_PWR_CNT_HIGH0 0x3118
#define A3XX_VBIF2_PERF_PWR_CNT_HIGH1 0x3119
#define A3XX_VBIF2_PERF_PWR_CNT_HIGH2 0x311a
#define A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL0 0x3800
#define A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL1 0x3801
/* RBBM Debug bus block IDs */
#define RBBM_BLOCK_ID_CP 0x1
#define RBBM_BLOCK_ID_RBBM 0x2
#define RBBM_BLOCK_ID_VBIF 0x3
#define RBBM_BLOCK_ID_HLSQ 0x4
#define RBBM_BLOCK_ID_UCHE 0x5
#define RBBM_BLOCK_ID_PC 0x8
#define RBBM_BLOCK_ID_VFD 0x9
#define RBBM_BLOCK_ID_VPC 0xa
#define RBBM_BLOCK_ID_TSE 0xb
#define RBBM_BLOCK_ID_RAS 0xc
#define RBBM_BLOCK_ID_VSC 0xd
#define RBBM_BLOCK_ID_SP_0 0x10
#define RBBM_BLOCK_ID_SP_1 0x11
#define RBBM_BLOCK_ID_SP_2 0x12
#define RBBM_BLOCK_ID_SP_3 0x13
#define RBBM_BLOCK_ID_TPL1_0 0x18
#define RBBM_BLOCK_ID_TPL1_1 0x19
#define RBBM_BLOCK_ID_TPL1_2 0x1a
#define RBBM_BLOCK_ID_TPL1_3 0x1b
#define RBBM_BLOCK_ID_RB_0 0x20
#define RBBM_BLOCK_ID_RB_1 0x21
#define RBBM_BLOCK_ID_RB_2 0x22
#define RBBM_BLOCK_ID_RB_3 0x23
#define RBBM_BLOCK_ID_MARB_0 0x28
#define RBBM_BLOCK_ID_MARB_1 0x29
#define RBBM_BLOCK_ID_MARB_2 0x2a
#define RBBM_BLOCK_ID_MARB_3 0x2b
/* RBBM_CLOCK_CTL default value */
#define A3XX_RBBM_CLOCK_CTL_DEFAULT 0xAAAAAAAA
#define A320_RBBM_CLOCK_CTL_DEFAULT 0xBFFFFFFF
#define A330_RBBM_CLOCK_CTL_DEFAULT 0xBFFCFFFF
#define A330_RBBM_GPR0_CTL_DEFAULT 0x00000000
#define A330v2_RBBM_GPR0_CTL_DEFAULT 0x05515455
#define A310_RBBM_GPR0_CTL_DEFAULT 0x000000AA
/* COUNTABLE FOR SP PERFCOUNTER */
#define SP_ALU_ACTIVE_CYCLES 0x1D
#define SP0_ICL1_MISSES 0x1A
#define SP_FS_CFLOW_INSTRUCTIONS 0x0C
/* COUNTABLE FOR TSE PERFCOUNTER */
#define TSE_INPUT_PRIM_NUM 0x0
/* VBIF countables */
#define VBIF_AXI_TOTAL_BEATS 85
/* VBIF Recoverable HALT bit value */
#define VBIF_RECOVERABLE_HALT_CTRL 0x1
/*
* CP DEBUG settings for A3XX core:
* DYNAMIC_CLK_DISABLE [27] - turn off the dynamic clock control
* MIU_128BIT_WRITE_ENABLE [25] - Allow 128 bit writes to the VBIF
*/
#define A3XX_CP_DEBUG_DEFAULT ((1 << 27) | (1 << 25))
#endif

902
drivers/gpu/msm/a5xx_reg.h Normal file
View file

@ -0,0 +1,902 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2014-2016,2019, The Linux Foundation. All rights reserved.
*/
#ifndef _A5XX_REG_H
#define _A5XX_REG_H
/* A5XX interrupt bits */
#define A5XX_INT_RBBM_GPU_IDLE 0
#define A5XX_INT_RBBM_AHB_ERROR 1
#define A5XX_INT_RBBM_TRANSFER_TIMEOUT 2
#define A5XX_INT_RBBM_ME_MS_TIMEOUT 3
#define A5XX_INT_RBBM_PFP_MS_TIMEOUT 4
#define A5XX_INT_RBBM_ETS_MS_TIMEOUT 5
#define A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW 6
#define A5XX_INT_RBBM_GPC_ERROR 7
#define A5XX_INT_CP_SW 8
#define A5XX_INT_CP_HW_ERROR 9
#define A5XX_INT_CP_CCU_FLUSH_DEPTH_TS 10
#define A5XX_INT_CP_CCU_FLUSH_COLOR_TS 11
#define A5XX_INT_CP_CCU_RESOLVE_TS 12
#define A5XX_INT_CP_IB2 13
#define A5XX_INT_CP_IB1 14
#define A5XX_INT_CP_RB 15
#define A5XX_INT_CP_UNUSED_1 16
#define A5XX_INT_CP_RB_DONE_TS 17
#define A5XX_INT_CP_WT_DONE_TS 18
#define A5XX_INT_UNKNOWN_1 19
#define A5XX_INT_CP_CACHE_FLUSH_TS 20
#define A5XX_INT_UNUSED_2 21
#define A5XX_INT_RBBM_ATB_BUS_OVERFLOW 22
#define A5XX_INT_MISC_HANG_DETECT 23
#define A5XX_INT_UCHE_OOB_ACCESS 24
#define A5XX_INT_UCHE_TRAP_INTR 25
#define A5XX_INT_DEBBUS_INTR_0 26
#define A5XX_INT_DEBBUS_INTR_1 27
#define A5XX_INT_GPMU_VOLTAGE_DROOP 28
#define A5XX_INT_GPMU_FIRMWARE 29
#define A5XX_INT_ISDB_CPU_IRQ 30
#define A5XX_INT_ISDB_UNDER_DEBUG 31
/* CP Interrupt bits */
#define A5XX_CP_OPCODE_ERROR 0
#define A5XX_CP_RESERVED_BIT_ERROR 1
#define A5XX_CP_HW_FAULT_ERROR 2
#define A5XX_CP_DMA_ERROR 3
#define A5XX_CP_REGISTER_PROTECTION_ERROR 4
#define A5XX_CP_AHB_ERROR 5
/* CP registers */
#define A5XX_CP_RB_BASE 0x800
#define A5XX_CP_RB_BASE_HI 0x801
#define A5XX_CP_RB_CNTL 0x802
#define A5XX_CP_RB_RPTR_ADDR_LO 0x804
#define A5XX_CP_RB_RPTR_ADDR_HI 0x805
#define A5XX_CP_RB_RPTR 0x806
#define A5XX_CP_RB_WPTR 0x807
#define A5XX_CP_PFP_STAT_ADDR 0x808
#define A5XX_CP_PFP_STAT_DATA 0x809
#define A5XX_CP_DRAW_STATE_ADDR 0x80B
#define A5XX_CP_DRAW_STATE_DATA 0x80C
#define A5XX_CP_CRASH_SCRIPT_BASE_LO 0x817
#define A5XX_CP_CRASH_SCRIPT_BASE_HI 0x818
#define A5XX_CP_CRASH_DUMP_CNTL 0x819
#define A5XX_CP_ME_STAT_ADDR 0x81A
#define A5XX_CP_ROQ_THRESHOLDS_1 0x81F
#define A5XX_CP_ROQ_THRESHOLDS_2 0x820
#define A5XX_CP_ROQ_DBG_ADDR 0x821
#define A5XX_CP_ROQ_DBG_DATA 0x822
#define A5XX_CP_MEQ_DBG_ADDR 0x823
#define A5XX_CP_MEQ_DBG_DATA 0x824
#define A5XX_CP_MEQ_THRESHOLDS 0x825
#define A5XX_CP_MERCIU_SIZE 0x826
#define A5XX_CP_MERCIU_DBG_ADDR 0x827
#define A5XX_CP_MERCIU_DBG_DATA_1 0x828
#define A5XX_CP_MERCIU_DBG_DATA_2 0x829
#define A5XX_CP_PFP_UCODE_DBG_ADDR 0x82A
#define A5XX_CP_PFP_UCODE_DBG_DATA 0x82B
#define A5XX_CP_ME_UCODE_DBG_ADDR 0x82F
#define A5XX_CP_ME_UCODE_DBG_DATA 0x830
#define A5XX_CP_CNTL 0x831
#define A5XX_CP_ME_CNTL 0x832
#define A5XX_CP_CHICKEN_DBG 0x833
#define A5XX_CP_PFP_INSTR_BASE_LO 0x835
#define A5XX_CP_PFP_INSTR_BASE_HI 0x836
#define A5XX_CP_PM4_INSTR_BASE_LO 0x838
#define A5XX_CP_PM4_INSTR_BASE_HI 0x839
#define A5XX_CP_CONTEXT_SWITCH_CNTL 0x83B
#define A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO 0x83C
#define A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI 0x83D
#define A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO 0x83E
#define A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI 0x83F
#define A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO 0x840
#define A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI 0x841
#define A5XX_CP_ADDR_MODE_CNTL 0x860
#define A5XX_CP_ME_STAT_DATA 0xB14
#define A5XX_CP_WFI_PEND_CTR 0xB15
#define A5XX_CP_INTERRUPT_STATUS 0xB18
#define A5XX_CP_HW_FAULT 0xB1A
#define A5XX_CP_PROTECT_STATUS 0xB1C
#define A5XX_CP_IB1_BASE 0xB1F
#define A5XX_CP_IB1_BASE_HI 0xB20
#define A5XX_CP_IB1_BUFSZ 0xB21
#define A5XX_CP_IB2_BASE 0xB22
#define A5XX_CP_IB2_BASE_HI 0xB23
#define A5XX_CP_IB2_BUFSZ 0xB24
#define A5XX_CP_PROTECT_REG_0 0x880
#define A5XX_CP_PROTECT_CNTL 0x8A0
#define A5XX_CP_AHB_FAULT 0xB1B
#define A5XX_CP_PERFCTR_CP_SEL_0 0xBB0
#define A5XX_CP_PERFCTR_CP_SEL_1 0xBB1
#define A5XX_CP_PERFCTR_CP_SEL_2 0xBB2
#define A5XX_CP_PERFCTR_CP_SEL_3 0xBB3
#define A5XX_CP_PERFCTR_CP_SEL_4 0xBB4
#define A5XX_CP_PERFCTR_CP_SEL_5 0xBB5
#define A5XX_CP_PERFCTR_CP_SEL_6 0xBB6
#define A5XX_CP_PERFCTR_CP_SEL_7 0xBB7
#define A5XX_VSC_ADDR_MODE_CNTL 0xBC1
/* CP Power Counter Registers Select */
#define A5XX_CP_POWERCTR_CP_SEL_0 0xBBA
#define A5XX_CP_POWERCTR_CP_SEL_1 0xBBB
#define A5XX_CP_POWERCTR_CP_SEL_2 0xBBC
#define A5XX_CP_POWERCTR_CP_SEL_3 0xBBD
/* RBBM registers */
#define A5XX_RBBM_CFG_DBGBUS_SEL_A 0x4
#define A5XX_RBBM_CFG_DBGBUS_SEL_B 0x5
#define A5XX_RBBM_CFG_DBGBUS_SEL_C 0x6
#define A5XX_RBBM_CFG_DBGBUS_SEL_D 0x7
#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_INDEX_SHIFT 0x0
#define A5XX_RBBM_CFG_DBGBUS_SEL_PING_BLK_SEL_SHIFT 0x8
#define A5XX_RBBM_CFG_DBGBUS_CNTLT 0x8
#define A5XX_RBBM_CFG_DBGBUS_CNTLM 0x9
#define A5XX_RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT 0x18
#define A5XX_RBBM_CFG_DBGBUS_OPL 0xA
#define A5XX_RBBM_CFG_DBGBUS_OPE 0xB
#define A5XX_RBBM_CFG_DBGBUS_IVTL_0 0xC
#define A5XX_RBBM_CFG_DBGBUS_IVTL_1 0xD
#define A5XX_RBBM_CFG_DBGBUS_IVTL_2 0xE
#define A5XX_RBBM_CFG_DBGBUS_IVTL_3 0xF
#define A5XX_RBBM_CFG_DBGBUS_MASKL_0 0x10
#define A5XX_RBBM_CFG_DBGBUS_MASKL_1 0x11
#define A5XX_RBBM_CFG_DBGBUS_MASKL_2 0x12
#define A5XX_RBBM_CFG_DBGBUS_MASKL_3 0x13
#define A5XX_RBBM_CFG_DBGBUS_BYTEL_0 0x14
#define A5XX_RBBM_CFG_DBGBUS_BYTEL_1 0x15
#define A5XX_RBBM_CFG_DBGBUS_IVTE_0 0x16
#define A5XX_RBBM_CFG_DBGBUS_IVTE_1 0x17
#define A5XX_RBBM_CFG_DBGBUS_IVTE_2 0x18
#define A5XX_RBBM_CFG_DBGBUS_IVTE_3 0x19
#define A5XX_RBBM_CFG_DBGBUS_MASKE_0 0x1A
#define A5XX_RBBM_CFG_DBGBUS_MASKE_1 0x1B
#define A5XX_RBBM_CFG_DBGBUS_MASKE_2 0x1C
#define A5XX_RBBM_CFG_DBGBUS_MASKE_3 0x1D
#define A5XX_RBBM_CFG_DBGBUS_NIBBLEE 0x1E
#define A5XX_RBBM_CFG_DBGBUS_PTRC0 0x1F
#define A5XX_RBBM_CFG_DBGBUS_PTRC1 0x20
#define A5XX_RBBM_CFG_DBGBUS_LOADREG 0x21
#define A5XX_RBBM_CFG_DBGBUS_IDX 0x22
#define A5XX_RBBM_CFG_DBGBUS_CLRC 0x23
#define A5XX_RBBM_CFG_DBGBUS_LOADIVT 0x24
#define A5XX_RBBM_INTERFACE_HANG_INT_CNTL 0x2F
#define A5XX_RBBM_INT_CLEAR_CMD 0x37
#define A5XX_RBBM_INT_0_MASK 0x38
#define A5XX_RBBM_AHB_DBG_CNTL 0x3F
#define A5XX_RBBM_EXT_VBIF_DBG_CNTL 0x41
#define A5XX_RBBM_SW_RESET_CMD 0x43
#define A5XX_RBBM_BLOCK_SW_RESET_CMD 0x45
#define A5XX_RBBM_BLOCK_SW_RESET_CMD2 0x46
#define A5XX_RBBM_DBG_LO_HI_GPIO 0x48
#define A5XX_RBBM_EXT_TRACE_BUS_CNTL 0x49
#define A5XX_RBBM_CLOCK_CNTL_TP0 0x4A
#define A5XX_RBBM_CLOCK_CNTL_TP1 0x4B
#define A5XX_RBBM_CLOCK_CNTL_TP2 0x4C
#define A5XX_RBBM_CLOCK_CNTL_TP3 0x4D
#define A5XX_RBBM_CLOCK_CNTL2_TP0 0x4E
#define A5XX_RBBM_CLOCK_CNTL2_TP1 0x4F
#define A5XX_RBBM_CLOCK_CNTL2_TP2 0x50
#define A5XX_RBBM_CLOCK_CNTL2_TP3 0x51
#define A5XX_RBBM_CLOCK_CNTL3_TP0 0x52
#define A5XX_RBBM_CLOCK_CNTL3_TP1 0x53
#define A5XX_RBBM_CLOCK_CNTL3_TP2 0x54
#define A5XX_RBBM_CLOCK_CNTL3_TP3 0x55
#define A5XX_RBBM_READ_AHB_THROUGH_DBG 0x59
#define A5XX_RBBM_CLOCK_CNTL_UCHE 0x5A
#define A5XX_RBBM_CLOCK_CNTL2_UCHE 0x5B
#define A5XX_RBBM_CLOCK_CNTL3_UCHE 0x5C
#define A5XX_RBBM_CLOCK_CNTL4_UCHE 0x5D
#define A5XX_RBBM_CLOCK_HYST_UCHE 0x5E
#define A5XX_RBBM_CLOCK_DELAY_UCHE 0x5F
#define A5XX_RBBM_CLOCK_MODE_GPC 0x60
#define A5XX_RBBM_CLOCK_DELAY_GPC 0x61
#define A5XX_RBBM_CLOCK_HYST_GPC 0x62
#define A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM 0x63
#define A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM 0x64
#define A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM 0x65
#define A5XX_RBBM_CLOCK_DELAY_HLSQ 0x66
#define A5XX_RBBM_CLOCK_CNTL 0x67
#define A5XX_RBBM_CLOCK_CNTL_SP0 0x68
#define A5XX_RBBM_CLOCK_CNTL_SP1 0x69
#define A5XX_RBBM_CLOCK_CNTL_SP2 0x6A
#define A5XX_RBBM_CLOCK_CNTL_SP3 0x6B
#define A5XX_RBBM_CLOCK_CNTL2_SP0 0x6C
#define A5XX_RBBM_CLOCK_CNTL2_SP1 0x6D
#define A5XX_RBBM_CLOCK_CNTL2_SP2 0x6E
#define A5XX_RBBM_CLOCK_CNTL2_SP3 0x6F
#define A5XX_RBBM_CLOCK_HYST_SP0 0x70
#define A5XX_RBBM_CLOCK_HYST_SP1 0x71
#define A5XX_RBBM_CLOCK_HYST_SP2 0x72
#define A5XX_RBBM_CLOCK_HYST_SP3 0x73
#define A5XX_RBBM_CLOCK_DELAY_SP0 0x74
#define A5XX_RBBM_CLOCK_DELAY_SP1 0x75
#define A5XX_RBBM_CLOCK_DELAY_SP2 0x76
#define A5XX_RBBM_CLOCK_DELAY_SP3 0x77
#define A5XX_RBBM_CLOCK_CNTL_RB0 0x78
#define A5XX_RBBM_CLOCK_CNTL_RB1 0x79
#define A5XX_RBBM_CLOCK_CNTL_RB2 0x7a
#define A5XX_RBBM_CLOCK_CNTL_RB3 0x7B
#define A5XX_RBBM_CLOCK_CNTL2_RB0 0x7C
#define A5XX_RBBM_CLOCK_CNTL2_RB1 0x7D
#define A5XX_RBBM_CLOCK_CNTL2_RB2 0x7E
#define A5XX_RBBM_CLOCK_CNTL2_RB3 0x7F
#define A5XX_RBBM_CLOCK_HYST_RAC 0x80
#define A5XX_RBBM_CLOCK_DELAY_RAC 0x81
#define A5XX_RBBM_CLOCK_CNTL_CCU0 0x82
#define A5XX_RBBM_CLOCK_CNTL_CCU1 0x83
#define A5XX_RBBM_CLOCK_CNTL_CCU2 0x84
#define A5XX_RBBM_CLOCK_CNTL_CCU3 0x85
#define A5XX_RBBM_CLOCK_HYST_RB_CCU0 0x86
#define A5XX_RBBM_CLOCK_HYST_RB_CCU1 0x87
#define A5XX_RBBM_CLOCK_HYST_RB_CCU2 0x88
#define A5XX_RBBM_CLOCK_HYST_RB_CCU3 0x89
#define A5XX_RBBM_CLOCK_CNTL_RAC 0x8A
#define A5XX_RBBM_CLOCK_CNTL2_RAC 0x8B
#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0 0x8C
#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1 0x8D
#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2 0x8E
#define A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3 0x8F
#define A5XX_RBBM_CLOCK_HYST_VFD 0x90
#define A5XX_RBBM_CLOCK_MODE_VFD 0x91
#define A5XX_RBBM_CLOCK_DELAY_VFD 0x92
#define A5XX_RBBM_AHB_CNTL0 0x93
#define A5XX_RBBM_AHB_CNTL1 0x94
#define A5XX_RBBM_AHB_CNTL2 0x95
#define A5XX_RBBM_AHB_CMD 0x96
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11 0x9C
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12 0x9D
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13 0x9E
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14 0x9F
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15 0xA0
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16 0xA1
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17 0xA2
#define A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18 0xA3
#define A5XX_RBBM_CLOCK_DELAY_TP0 0xA4
#define A5XX_RBBM_CLOCK_DELAY_TP1 0xA5
#define A5XX_RBBM_CLOCK_DELAY_TP2 0xA6
#define A5XX_RBBM_CLOCK_DELAY_TP3 0xA7
#define A5XX_RBBM_CLOCK_DELAY2_TP0 0xA8
#define A5XX_RBBM_CLOCK_DELAY2_TP1 0xA9
#define A5XX_RBBM_CLOCK_DELAY2_TP2 0xAA
#define A5XX_RBBM_CLOCK_DELAY2_TP3 0xAB
#define A5XX_RBBM_CLOCK_DELAY3_TP0 0xAC
#define A5XX_RBBM_CLOCK_DELAY3_TP1 0xAD
#define A5XX_RBBM_CLOCK_DELAY3_TP2 0xAE
#define A5XX_RBBM_CLOCK_DELAY3_TP3 0xAF
#define A5XX_RBBM_CLOCK_HYST_TP0 0xB0
#define A5XX_RBBM_CLOCK_HYST_TP1 0xB1
#define A5XX_RBBM_CLOCK_HYST_TP2 0xB2
#define A5XX_RBBM_CLOCK_HYST_TP3 0xB3
#define A5XX_RBBM_CLOCK_HYST2_TP0 0xB4
#define A5XX_RBBM_CLOCK_HYST2_TP1 0xB5
#define A5XX_RBBM_CLOCK_HYST2_TP2 0xB6
#define A5XX_RBBM_CLOCK_HYST2_TP3 0xB7
#define A5XX_RBBM_CLOCK_HYST3_TP0 0xB8
#define A5XX_RBBM_CLOCK_HYST3_TP1 0xB9
#define A5XX_RBBM_CLOCK_HYST3_TP2 0xBA
#define A5XX_RBBM_CLOCK_HYST3_TP3 0xBB
#define A5XX_RBBM_CLOCK_CNTL_GPMU 0xC8
#define A5XX_RBBM_CLOCK_DELAY_GPMU 0xC9
#define A5XX_RBBM_CLOCK_HYST_GPMU 0xCA
#define A5XX_RBBM_PERFCTR_CP_0_LO 0x3A0
#define A5XX_RBBM_PERFCTR_CP_0_HI 0x3A1
#define A5XX_RBBM_PERFCTR_CP_1_LO 0x3A2
#define A5XX_RBBM_PERFCTR_CP_1_HI 0x3A3
#define A5XX_RBBM_PERFCTR_CP_2_LO 0x3A4
#define A5XX_RBBM_PERFCTR_CP_2_HI 0x3A5
#define A5XX_RBBM_PERFCTR_CP_3_LO 0x3A6
#define A5XX_RBBM_PERFCTR_CP_3_HI 0x3A7
#define A5XX_RBBM_PERFCTR_CP_4_LO 0x3A8
#define A5XX_RBBM_PERFCTR_CP_4_HI 0x3A9
#define A5XX_RBBM_PERFCTR_CP_5_LO 0x3AA
#define A5XX_RBBM_PERFCTR_CP_5_HI 0x3AB
#define A5XX_RBBM_PERFCTR_CP_6_LO 0x3AC
#define A5XX_RBBM_PERFCTR_CP_6_HI 0x3AD
#define A5XX_RBBM_PERFCTR_CP_7_LO 0x3AE
#define A5XX_RBBM_PERFCTR_CP_7_HI 0x3AF
#define A5XX_RBBM_PERFCTR_RBBM_0_LO 0x3B0
#define A5XX_RBBM_PERFCTR_RBBM_0_HI 0x3B1
#define A5XX_RBBM_PERFCTR_RBBM_1_LO 0x3B2
#define A5XX_RBBM_PERFCTR_RBBM_1_HI 0x3B3
#define A5XX_RBBM_PERFCTR_RBBM_2_LO 0x3B4
#define A5XX_RBBM_PERFCTR_RBBM_2_HI 0x3B5
#define A5XX_RBBM_PERFCTR_RBBM_3_LO 0x3B6
#define A5XX_RBBM_PERFCTR_RBBM_3_HI 0x3B7
#define A5XX_RBBM_PERFCTR_PC_0_LO 0x3B8
#define A5XX_RBBM_PERFCTR_PC_0_HI 0x3B9
#define A5XX_RBBM_PERFCTR_PC_1_LO 0x3BA
#define A5XX_RBBM_PERFCTR_PC_1_HI 0x3BB
#define A5XX_RBBM_PERFCTR_PC_2_LO 0x3BC
#define A5XX_RBBM_PERFCTR_PC_2_HI 0x3BD
#define A5XX_RBBM_PERFCTR_PC_3_LO 0x3BE
#define A5XX_RBBM_PERFCTR_PC_3_HI 0x3BF
#define A5XX_RBBM_PERFCTR_PC_4_LO 0x3C0
#define A5XX_RBBM_PERFCTR_PC_4_HI 0x3C1
#define A5XX_RBBM_PERFCTR_PC_5_LO 0x3C2
#define A5XX_RBBM_PERFCTR_PC_5_HI 0x3C3
#define A5XX_RBBM_PERFCTR_PC_6_LO 0x3C4
#define A5XX_RBBM_PERFCTR_PC_6_HI 0x3C5
#define A5XX_RBBM_PERFCTR_PC_7_LO 0x3C6
#define A5XX_RBBM_PERFCTR_PC_7_HI 0x3C7
#define A5XX_RBBM_PERFCTR_VFD_0_LO 0x3C8
#define A5XX_RBBM_PERFCTR_VFD_0_HI 0x3C9
#define A5XX_RBBM_PERFCTR_VFD_1_LO 0x3CA
#define A5XX_RBBM_PERFCTR_VFD_1_HI 0x3CB
#define A5XX_RBBM_PERFCTR_VFD_2_LO 0x3CC
#define A5XX_RBBM_PERFCTR_VFD_2_HI 0x3CD
#define A5XX_RBBM_PERFCTR_VFD_3_LO 0x3CE
#define A5XX_RBBM_PERFCTR_VFD_3_HI 0x3CF
#define A5XX_RBBM_PERFCTR_VFD_4_LO 0x3D0
#define A5XX_RBBM_PERFCTR_VFD_4_HI 0x3D1
#define A5XX_RBBM_PERFCTR_VFD_5_LO 0x3D2
#define A5XX_RBBM_PERFCTR_VFD_5_HI 0x3D3
#define A5XX_RBBM_PERFCTR_VFD_6_LO 0x3D4
#define A5XX_RBBM_PERFCTR_VFD_6_HI 0x3D5
#define A5XX_RBBM_PERFCTR_VFD_7_LO 0x3D6
#define A5XX_RBBM_PERFCTR_VFD_7_HI 0x3D7
#define A5XX_RBBM_PERFCTR_HLSQ_0_LO 0x3D8
#define A5XX_RBBM_PERFCTR_HLSQ_0_HI 0x3D9
#define A5XX_RBBM_PERFCTR_HLSQ_1_LO 0x3DA
#define A5XX_RBBM_PERFCTR_HLSQ_1_HI 0x3DB
#define A5XX_RBBM_PERFCTR_HLSQ_2_LO 0x3DC
#define A5XX_RBBM_PERFCTR_HLSQ_2_HI 0x3DD
#define A5XX_RBBM_PERFCTR_HLSQ_3_LO 0x3DE
#define A5XX_RBBM_PERFCTR_HLSQ_3_HI 0x3DF
#define A5XX_RBBM_PERFCTR_HLSQ_4_LO 0x3E0
#define A5XX_RBBM_PERFCTR_HLSQ_4_HI 0x3E1
#define A5XX_RBBM_PERFCTR_HLSQ_5_LO 0x3E2
#define A5XX_RBBM_PERFCTR_HLSQ_5_HI 0x3E3
#define A5XX_RBBM_PERFCTR_HLSQ_6_LO 0x3E4
#define A5XX_RBBM_PERFCTR_HLSQ_6_HI 0x3E5
#define A5XX_RBBM_PERFCTR_HLSQ_7_LO 0x3E6
#define A5XX_RBBM_PERFCTR_HLSQ_7_HI 0x3E7
#define A5XX_RBBM_PERFCTR_VPC_0_LO 0x3E8
#define A5XX_RBBM_PERFCTR_VPC_0_HI 0x3E9
#define A5XX_RBBM_PERFCTR_VPC_1_LO 0x3EA
#define A5XX_RBBM_PERFCTR_VPC_1_HI 0x3EB
#define A5XX_RBBM_PERFCTR_VPC_2_LO 0x3EC
#define A5XX_RBBM_PERFCTR_VPC_2_HI 0x3ED
#define A5XX_RBBM_PERFCTR_VPC_3_LO 0x3EE
#define A5XX_RBBM_PERFCTR_VPC_3_HI 0x3EF
#define A5XX_RBBM_PERFCTR_CCU_0_LO 0x3F0
#define A5XX_RBBM_PERFCTR_CCU_0_HI 0x3F1
#define A5XX_RBBM_PERFCTR_CCU_1_LO 0x3F2
#define A5XX_RBBM_PERFCTR_CCU_1_HI 0x3F3
#define A5XX_RBBM_PERFCTR_CCU_2_LO 0x3F4
#define A5XX_RBBM_PERFCTR_CCU_2_HI 0x3F5
#define A5XX_RBBM_PERFCTR_CCU_3_LO 0x3F6
#define A5XX_RBBM_PERFCTR_CCU_3_HI 0x3F7
#define A5XX_RBBM_PERFCTR_TSE_0_LO 0x3F8
#define A5XX_RBBM_PERFCTR_TSE_0_HI 0x3F9
#define A5XX_RBBM_PERFCTR_TSE_1_LO 0x3FA
#define A5XX_RBBM_PERFCTR_TSE_1_HI 0x3FB
#define A5XX_RBBM_PERFCTR_TSE_2_LO 0x3FC
#define A5XX_RBBM_PERFCTR_TSE_2_HI 0x3FD
#define A5XX_RBBM_PERFCTR_TSE_3_LO 0x3FE
#define A5XX_RBBM_PERFCTR_TSE_3_HI 0x3FF
#define A5XX_RBBM_PERFCTR_RAS_0_LO 0x400
#define A5XX_RBBM_PERFCTR_RAS_0_HI 0x401
#define A5XX_RBBM_PERFCTR_RAS_1_LO 0x402
#define A5XX_RBBM_PERFCTR_RAS_1_HI 0x403
#define A5XX_RBBM_PERFCTR_RAS_2_LO 0x404
#define A5XX_RBBM_PERFCTR_RAS_2_HI 0x405
#define A5XX_RBBM_PERFCTR_RAS_3_LO 0x406
#define A5XX_RBBM_PERFCTR_RAS_3_HI 0x407
#define A5XX_RBBM_PERFCTR_UCHE_0_LO 0x408
#define A5XX_RBBM_PERFCTR_UCHE_0_HI 0x409
#define A5XX_RBBM_PERFCTR_UCHE_1_LO 0x40A
#define A5XX_RBBM_PERFCTR_UCHE_1_HI 0x40B
#define A5XX_RBBM_PERFCTR_UCHE_2_LO 0x40C
#define A5XX_RBBM_PERFCTR_UCHE_2_HI 0x40D
#define A5XX_RBBM_PERFCTR_UCHE_3_LO 0x40E
#define A5XX_RBBM_PERFCTR_UCHE_3_HI 0x40F
#define A5XX_RBBM_PERFCTR_UCHE_4_LO 0x410
#define A5XX_RBBM_PERFCTR_UCHE_4_HI 0x411
#define A5XX_RBBM_PERFCTR_UCHE_5_LO 0x412
#define A5XX_RBBM_PERFCTR_UCHE_5_HI 0x413
#define A5XX_RBBM_PERFCTR_UCHE_6_LO 0x414
#define A5XX_RBBM_PERFCTR_UCHE_6_HI 0x415
#define A5XX_RBBM_PERFCTR_UCHE_7_LO 0x416
#define A5XX_RBBM_PERFCTR_UCHE_7_HI 0x417
#define A5XX_RBBM_PERFCTR_TP_0_LO 0x418
#define A5XX_RBBM_PERFCTR_TP_0_HI 0x419
#define A5XX_RBBM_PERFCTR_TP_1_LO 0x41A
#define A5XX_RBBM_PERFCTR_TP_1_HI 0x41B
#define A5XX_RBBM_PERFCTR_TP_2_LO 0x41C
#define A5XX_RBBM_PERFCTR_TP_2_HI 0x41D
#define A5XX_RBBM_PERFCTR_TP_3_LO 0x41E
#define A5XX_RBBM_PERFCTR_TP_3_HI 0x41F
#define A5XX_RBBM_PERFCTR_TP_4_LO 0x420
#define A5XX_RBBM_PERFCTR_TP_4_HI 0x421
#define A5XX_RBBM_PERFCTR_TP_5_LO 0x422
#define A5XX_RBBM_PERFCTR_TP_5_HI 0x423
#define A5XX_RBBM_PERFCTR_TP_6_LO 0x424
#define A5XX_RBBM_PERFCTR_TP_6_HI 0x425
#define A5XX_RBBM_PERFCTR_TP_7_LO 0x426
#define A5XX_RBBM_PERFCTR_TP_7_HI 0x427
#define A5XX_RBBM_PERFCTR_SP_0_LO 0x428
#define A5XX_RBBM_PERFCTR_SP_0_HI 0x429
#define A5XX_RBBM_PERFCTR_SP_1_LO 0x42A
#define A5XX_RBBM_PERFCTR_SP_1_HI 0x42B
#define A5XX_RBBM_PERFCTR_SP_2_LO 0x42C
#define A5XX_RBBM_PERFCTR_SP_2_HI 0x42D
#define A5XX_RBBM_PERFCTR_SP_3_LO 0x42E
#define A5XX_RBBM_PERFCTR_SP_3_HI 0x42F
#define A5XX_RBBM_PERFCTR_SP_4_LO 0x430
#define A5XX_RBBM_PERFCTR_SP_4_HI 0x431
#define A5XX_RBBM_PERFCTR_SP_5_LO 0x432
#define A5XX_RBBM_PERFCTR_SP_5_HI 0x433
#define A5XX_RBBM_PERFCTR_SP_6_LO 0x434
#define A5XX_RBBM_PERFCTR_SP_6_HI 0x435
#define A5XX_RBBM_PERFCTR_SP_7_LO 0x436
#define A5XX_RBBM_PERFCTR_SP_7_HI 0x437
#define A5XX_RBBM_PERFCTR_SP_8_LO 0x438
#define A5XX_RBBM_PERFCTR_SP_8_HI 0x439
#define A5XX_RBBM_PERFCTR_SP_9_LO 0x43A
#define A5XX_RBBM_PERFCTR_SP_9_HI 0x43B
#define A5XX_RBBM_PERFCTR_SP_10_LO 0x43C
#define A5XX_RBBM_PERFCTR_SP_10_HI 0x43D
#define A5XX_RBBM_PERFCTR_SP_11_LO 0x43E
#define A5XX_RBBM_PERFCTR_SP_11_HI 0x43F
#define A5XX_RBBM_PERFCTR_RB_0_LO 0x440
#define A5XX_RBBM_PERFCTR_RB_0_HI 0x441
#define A5XX_RBBM_PERFCTR_RB_1_LO 0x442
#define A5XX_RBBM_PERFCTR_RB_1_HI 0x443
#define A5XX_RBBM_PERFCTR_RB_2_LO 0x444
#define A5XX_RBBM_PERFCTR_RB_2_HI 0x445
#define A5XX_RBBM_PERFCTR_RB_3_LO 0x446
#define A5XX_RBBM_PERFCTR_RB_3_HI 0x447
#define A5XX_RBBM_PERFCTR_RB_4_LO 0x448
#define A5XX_RBBM_PERFCTR_RB_4_HI 0x449
#define A5XX_RBBM_PERFCTR_RB_5_LO 0x44A
#define A5XX_RBBM_PERFCTR_RB_5_HI 0x44B
#define A5XX_RBBM_PERFCTR_RB_6_LO 0x44C
#define A5XX_RBBM_PERFCTR_RB_6_HI 0x44D
#define A5XX_RBBM_PERFCTR_RB_7_LO 0x44E
#define A5XX_RBBM_PERFCTR_RB_7_HI 0x44F
#define A5XX_RBBM_PERFCTR_VSC_0_LO 0x450
#define A5XX_RBBM_PERFCTR_VSC_0_HI 0x451
#define A5XX_RBBM_PERFCTR_VSC_1_LO 0x452
#define A5XX_RBBM_PERFCTR_VSC_1_HI 0x453
#define A5XX_RBBM_PERFCTR_LRZ_0_LO 0x454
#define A5XX_RBBM_PERFCTR_LRZ_0_HI 0x455
#define A5XX_RBBM_PERFCTR_LRZ_1_LO 0x456
#define A5XX_RBBM_PERFCTR_LRZ_1_HI 0x457
#define A5XX_RBBM_PERFCTR_LRZ_2_LO 0x458
#define A5XX_RBBM_PERFCTR_LRZ_2_HI 0x459
#define A5XX_RBBM_PERFCTR_LRZ_3_LO 0x45A
#define A5XX_RBBM_PERFCTR_LRZ_3_HI 0x45B
#define A5XX_RBBM_PERFCTR_CMP_0_LO 0x45C
#define A5XX_RBBM_PERFCTR_CMP_0_HI 0x45D
#define A5XX_RBBM_PERFCTR_CMP_1_LO 0x45E
#define A5XX_RBBM_PERFCTR_CMP_1_HI 0x45F
#define A5XX_RBBM_PERFCTR_CMP_2_LO 0x460
#define A5XX_RBBM_PERFCTR_CMP_2_HI 0x461
#define A5XX_RBBM_PERFCTR_CMP_3_LO 0x462
#define A5XX_RBBM_PERFCTR_CMP_3_HI 0x463
#define A5XX_RBBM_PERFCTR_RBBM_SEL_0 0x46B
#define A5XX_RBBM_PERFCTR_RBBM_SEL_1 0x46C
#define A5XX_RBBM_PERFCTR_RBBM_SEL_2 0x46D
#define A5XX_RBBM_PERFCTR_RBBM_SEL_3 0x46E
#define A5XX_RBBM_ALWAYSON_COUNTER_LO 0x4D2
#define A5XX_RBBM_ALWAYSON_COUNTER_HI 0x4D3
#define A5XX_RBBM_STATUS 0x4F5
#define A5XX_RBBM_STATUS3 0x530
#define A5XX_RBBM_INT_0_STATUS 0x4E1
#define A5XX_RBBM_AHB_ME_SPLIT_STATUS 0x4F0
#define A5XX_RBBM_AHB_PFP_SPLIT_STATUS 0x4F1
#define A5XX_RBBM_AHB_ERROR_STATUS 0x4F4
#define A5XX_RBBM_PERFCTR_CNTL 0x464
#define A5XX_RBBM_PERFCTR_LOAD_CMD0 0x465
#define A5XX_RBBM_PERFCTR_LOAD_CMD1 0x466
#define A5XX_RBBM_PERFCTR_LOAD_CMD2 0x467
#define A5XX_RBBM_PERFCTR_LOAD_CMD3 0x468
#define A5XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x469
#define A5XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x46A
#define A5XX_RBBM_PERFCTR_RBBM_SEL_0 0x46B
#define A5XX_RBBM_PERFCTR_RBBM_SEL_1 0x46C
#define A5XX_RBBM_PERFCTR_RBBM_SEL_2 0x46D
#define A5XX_RBBM_PERFCTR_RBBM_SEL_3 0x46E
#define A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED 0x46F
#define A5XX_RBBM_CFG_DBGBUS_EVENT_LOGIC 0x504
#define A5XX_RBBM_CFG_DBGBUS_OVER 0x505
#define A5XX_RBBM_CFG_DBGBUS_COUNT0 0x506
#define A5XX_RBBM_CFG_DBGBUS_COUNT1 0x507
#define A5XX_RBBM_CFG_DBGBUS_COUNT2 0x508
#define A5XX_RBBM_CFG_DBGBUS_COUNT3 0x509
#define A5XX_RBBM_CFG_DBGBUS_COUNT4 0x50A
#define A5XX_RBBM_CFG_DBGBUS_COUNT5 0x50B
#define A5XX_RBBM_CFG_DBGBUS_TRACE_ADDR 0x50C
#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF0 0x50D
#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1 0x50E
#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2 0x50F
#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF3 0x510
#define A5XX_RBBM_CFG_DBGBUS_TRACE_BUF4 0x511
#define A5XX_RBBM_CFG_DBGBUS_MISR0 0x512
#define A5XX_RBBM_CFG_DBGBUS_MISR1 0x513
#define A5XX_RBBM_ISDB_CNT 0x533
#define A5XX_RBBM_SECVID_TRUST_CONFIG 0xF000
#define A5XX_RBBM_SECVID_TRUST_CNTL 0xF400
#define A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO 0xF800
#define A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI 0xF801
#define A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE 0xF802
#define A5XX_RBBM_SECVID_TSB_CNTL 0xF803
#define A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL 0xF810
/* VSC registers */
#define A5XX_VSC_PERFCTR_VSC_SEL_0 0xC60
#define A5XX_VSC_PERFCTR_VSC_SEL_1 0xC61
#define A5XX_GRAS_ADDR_MODE_CNTL 0xC81
/* TSE registers */
#define A5XX_GRAS_PERFCTR_TSE_SEL_0 0xC90
#define A5XX_GRAS_PERFCTR_TSE_SEL_1 0xC91
#define A5XX_GRAS_PERFCTR_TSE_SEL_2 0xC92
#define A5XX_GRAS_PERFCTR_TSE_SEL_3 0xC93
/* RAS registers */
#define A5XX_GRAS_PERFCTR_RAS_SEL_0 0xC94
#define A5XX_GRAS_PERFCTR_RAS_SEL_1 0xC95
#define A5XX_GRAS_PERFCTR_RAS_SEL_2 0xC96
#define A5XX_GRAS_PERFCTR_RAS_SEL_3 0xC97
/* LRZ registers */
#define A5XX_GRAS_PERFCTR_LRZ_SEL_0 0xC98
#define A5XX_GRAS_PERFCTR_LRZ_SEL_1 0xC99
#define A5XX_GRAS_PERFCTR_LRZ_SEL_2 0xC9A
#define A5XX_GRAS_PERFCTR_LRZ_SEL_3 0xC9B
/* RB registers */
#define A5XX_RB_DBG_ECO_CNT 0xCC4
#define A5XX_RB_ADDR_MODE_CNTL 0xCC5
#define A5XX_RB_MODE_CNTL 0xCC6
#define A5XX_RB_PERFCTR_RB_SEL_0 0xCD0
#define A5XX_RB_PERFCTR_RB_SEL_1 0xCD1
#define A5XX_RB_PERFCTR_RB_SEL_2 0xCD2
#define A5XX_RB_PERFCTR_RB_SEL_3 0xCD3
#define A5XX_RB_PERFCTR_RB_SEL_4 0xCD4
#define A5XX_RB_PERFCTR_RB_SEL_5 0xCD5
#define A5XX_RB_PERFCTR_RB_SEL_6 0xCD6
#define A5XX_RB_PERFCTR_RB_SEL_7 0xCD7
/* CCU registers */
#define A5XX_RB_PERFCTR_CCU_SEL_0 0xCD8
#define A5XX_RB_PERFCTR_CCU_SEL_1 0xCD9
#define A5XX_RB_PERFCTR_CCU_SEL_2 0xCDA
#define A5XX_RB_PERFCTR_CCU_SEL_3 0xCDB
/* RB Power Counter RB Registers Select */
#define A5XX_RB_POWERCTR_RB_SEL_0 0xCE0
#define A5XX_RB_POWERCTR_RB_SEL_1 0xCE1
#define A5XX_RB_POWERCTR_RB_SEL_2 0xCE2
#define A5XX_RB_POWERCTR_RB_SEL_3 0xCE3
/* RB Power Counter CCU Registers Select */
#define A5XX_RB_POWERCTR_CCU_SEL_0 0xCE4
#define A5XX_RB_POWERCTR_CCU_SEL_1 0xCE5
/* CMP registers */
#define A5XX_RB_PERFCTR_CMP_SEL_0 0xCEC
#define A5XX_RB_PERFCTR_CMP_SEL_1 0xCED
#define A5XX_RB_PERFCTR_CMP_SEL_2 0xCEE
#define A5XX_RB_PERFCTR_CMP_SEL_3 0xCEF
/* PC registers */
#define A5XX_PC_DBG_ECO_CNTL 0xD00
#define A5XX_PC_ADDR_MODE_CNTL 0xD01
#define A5XX_PC_PERFCTR_PC_SEL_0 0xD10
#define A5XX_PC_PERFCTR_PC_SEL_1 0xD11
#define A5XX_PC_PERFCTR_PC_SEL_2 0xD12
#define A5XX_PC_PERFCTR_PC_SEL_3 0xD13
#define A5XX_PC_PERFCTR_PC_SEL_4 0xD14
#define A5XX_PC_PERFCTR_PC_SEL_5 0xD15
#define A5XX_PC_PERFCTR_PC_SEL_6 0xD16
#define A5XX_PC_PERFCTR_PC_SEL_7 0xD17
/* HLSQ registers */
#define A5XX_HLSQ_DBG_ECO_CNTL 0xE04
#define A5XX_HLSQ_ADDR_MODE_CNTL 0xE05
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_0 0xE10
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_1 0xE11
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_2 0xE12
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_3 0xE13
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_4 0xE14
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_5 0xE15
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_6 0xE16
#define A5XX_HLSQ_PERFCTR_HLSQ_SEL_7 0xE17
#define A5XX_HLSQ_DBG_READ_SEL 0xBC00
#define A5XX_HLSQ_DBG_AHB_READ_APERTURE 0xA000
/* VFD registers */
#define A5XX_VFD_ADDR_MODE_CNTL 0xE41
#define A5XX_VFD_PERFCTR_VFD_SEL_0 0xE50
#define A5XX_VFD_PERFCTR_VFD_SEL_1 0xE51
#define A5XX_VFD_PERFCTR_VFD_SEL_2 0xE52
#define A5XX_VFD_PERFCTR_VFD_SEL_3 0xE53
#define A5XX_VFD_PERFCTR_VFD_SEL_4 0xE54
#define A5XX_VFD_PERFCTR_VFD_SEL_5 0xE55
#define A5XX_VFD_PERFCTR_VFD_SEL_6 0xE56
#define A5XX_VFD_PERFCTR_VFD_SEL_7 0xE57
/* VPC registers */
#define A5XX_VPC_DBG_ECO_CNTL 0xE60
#define A5XX_VPC_ADDR_MODE_CNTL 0xE61
#define A5XX_VPC_PERFCTR_VPC_SEL_0 0xE64
#define A5XX_VPC_PERFCTR_VPC_SEL_1 0xE65
#define A5XX_VPC_PERFCTR_VPC_SEL_2 0xE66
#define A5XX_VPC_PERFCTR_VPC_SEL_3 0xE67
/* UCHE registers */
#define A5XX_UCHE_ADDR_MODE_CNTL 0xE80
#define A5XX_UCHE_MODE_CNTL 0xE81
#define A5XX_UCHE_WRITE_THRU_BASE_LO 0xE87
#define A5XX_UCHE_WRITE_THRU_BASE_HI 0xE88
#define A5XX_UCHE_TRAP_BASE_LO 0xE89
#define A5XX_UCHE_TRAP_BASE_HI 0xE8A
#define A5XX_UCHE_GMEM_RANGE_MIN_LO 0xE8B
#define A5XX_UCHE_GMEM_RANGE_MIN_HI 0xE8C
#define A5XX_UCHE_GMEM_RANGE_MAX_LO 0xE8D
#define A5XX_UCHE_GMEM_RANGE_MAX_HI 0xE8E
#define A5XX_UCHE_DBG_ECO_CNTL_2 0xE8F
#define A5XX_UCHE_INVALIDATE0 0xE95
#define A5XX_UCHE_CACHE_WAYS 0xE96
#define A5XX_UCHE_PERFCTR_UCHE_SEL_0 0xEA0
#define A5XX_UCHE_PERFCTR_UCHE_SEL_1 0xEA1
#define A5XX_UCHE_PERFCTR_UCHE_SEL_2 0xEA2
#define A5XX_UCHE_PERFCTR_UCHE_SEL_3 0xEA3
#define A5XX_UCHE_PERFCTR_UCHE_SEL_4 0xEA4
#define A5XX_UCHE_PERFCTR_UCHE_SEL_5 0xEA5
#define A5XX_UCHE_PERFCTR_UCHE_SEL_6 0xEA6
#define A5XX_UCHE_PERFCTR_UCHE_SEL_7 0xEA7
/* UCHE Power Counter UCHE Registers Select */
#define A5XX_UCHE_POWERCTR_UCHE_SEL_0 0xEA8
#define A5XX_UCHE_POWERCTR_UCHE_SEL_1 0xEA9
#define A5XX_UCHE_POWERCTR_UCHE_SEL_2 0xEAA
#define A5XX_UCHE_POWERCTR_UCHE_SEL_3 0xEAB
/* SP registers */
#define A5XX_SP_DBG_ECO_CNTL 0xEC0
#define A5XX_SP_ADDR_MODE_CNTL 0xEC1
#define A5XX_SP_PERFCTR_SP_SEL_0 0xED0
#define A5XX_SP_PERFCTR_SP_SEL_1 0xED1
#define A5XX_SP_PERFCTR_SP_SEL_2 0xED2
#define A5XX_SP_PERFCTR_SP_SEL_3 0xED3
#define A5XX_SP_PERFCTR_SP_SEL_4 0xED4
#define A5XX_SP_PERFCTR_SP_SEL_5 0xED5
#define A5XX_SP_PERFCTR_SP_SEL_6 0xED6
#define A5XX_SP_PERFCTR_SP_SEL_7 0xED7
#define A5XX_SP_PERFCTR_SP_SEL_8 0xED8
#define A5XX_SP_PERFCTR_SP_SEL_9 0xED9
#define A5XX_SP_PERFCTR_SP_SEL_10 0xEDA
#define A5XX_SP_PERFCTR_SP_SEL_11 0xEDB
/* SP Power Counter SP Registers Select */
#define A5XX_SP_POWERCTR_SP_SEL_0 0xEDC
#define A5XX_SP_POWERCTR_SP_SEL_1 0xEDD
#define A5XX_SP_POWERCTR_SP_SEL_2 0xEDE
#define A5XX_SP_POWERCTR_SP_SEL_3 0xEDF
/* TP registers */
#define A5XX_TPL1_ADDR_MODE_CNTL 0xF01
#define A5XX_TPL1_MODE_CNTL 0xF02
#define A5XX_TPL1_PERFCTR_TP_SEL_0 0xF10
#define A5XX_TPL1_PERFCTR_TP_SEL_1 0xF11
#define A5XX_TPL1_PERFCTR_TP_SEL_2 0xF12
#define A5XX_TPL1_PERFCTR_TP_SEL_3 0xF13
#define A5XX_TPL1_PERFCTR_TP_SEL_4 0xF14
#define A5XX_TPL1_PERFCTR_TP_SEL_5 0xF15
#define A5XX_TPL1_PERFCTR_TP_SEL_6 0xF16
#define A5XX_TPL1_PERFCTR_TP_SEL_7 0xF17
/* TP Power Counter TP Registers Select */
#define A5XX_TPL1_POWERCTR_TP_SEL_0 0xF18
#define A5XX_TPL1_POWERCTR_TP_SEL_1 0xF19
#define A5XX_TPL1_POWERCTR_TP_SEL_2 0xF1A
#define A5XX_TPL1_POWERCTR_TP_SEL_3 0xF1B
/* VBIF registers */
#define A5XX_VBIF_VERSION 0x3000
#define A5XX_VBIF_CLKON 0x3001
#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_MASK 0x1
#define A5XX_VBIF_CLKON_FORCE_ON_TESTBUS_SHIFT 0x1
#define A5XX_VBIF_ROUND_ROBIN_QOS_ARB 0x3049
#define A5XX_VBIF_GATE_OFF_WRREQ_EN 0x302A
#define A5XX_VBIF_XIN_HALT_CTRL0 0x3080
#define A5XX_VBIF_XIN_HALT_CTRL0_MASK 0xF
#define A510_VBIF_XIN_HALT_CTRL0_MASK 0x7
#define A5XX_VBIF_XIN_HALT_CTRL1 0x3081
#define A5XX_VBIF_TEST_BUS_OUT_CTRL 0x3084
#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_MASK 0x1
#define A5XX_VBIF_TEST_BUS_OUT_CTRL_EN_SHIFT 0x0
#define A5XX_VBIF_TEST_BUS1_CTRL0 0x3085
#define A5XX_VBIF_TEST_BUS1_CTRL1 0x3086
#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_MASK 0xF
#define A5XX_VBIF_TEST_BUS1_CTRL1_DATA_SEL_SHIFT 0x0
#define A5XX_VBIF_TEST_BUS2_CTRL0 0x3087
#define A5XX_VBIF_TEST_BUS2_CTRL1 0x3088
#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_MASK 0x1FF
#define A5XX_VBIF_TEST_BUS2_CTRL1_DATA_SEL_SHIFT 0x0
#define A5XX_VBIF_TEST_BUS_OUT 0x308c
#define A5XX_VBIF_PERF_CNT_SEL0 0x30D0
#define A5XX_VBIF_PERF_CNT_SEL1 0x30D1
#define A5XX_VBIF_PERF_CNT_SEL2 0x30D2
#define A5XX_VBIF_PERF_CNT_SEL3 0x30D3
#define A5XX_VBIF_PERF_CNT_LOW0 0x30D8
#define A5XX_VBIF_PERF_CNT_LOW1 0x30D9
#define A5XX_VBIF_PERF_CNT_LOW2 0x30DA
#define A5XX_VBIF_PERF_CNT_LOW3 0x30DB
#define A5XX_VBIF_PERF_CNT_HIGH0 0x30E0
#define A5XX_VBIF_PERF_CNT_HIGH1 0x30E1
#define A5XX_VBIF_PERF_CNT_HIGH2 0x30E2
#define A5XX_VBIF_PERF_CNT_HIGH3 0x30E3
#define A5XX_VBIF_PERF_PWR_CNT_EN0 0x3100
#define A5XX_VBIF_PERF_PWR_CNT_EN1 0x3101
#define A5XX_VBIF_PERF_PWR_CNT_EN2 0x3102
#define A5XX_VBIF_PERF_PWR_CNT_LOW0 0x3110
#define A5XX_VBIF_PERF_PWR_CNT_LOW1 0x3111
#define A5XX_VBIF_PERF_PWR_CNT_LOW2 0x3112
#define A5XX_VBIF_PERF_PWR_CNT_HIGH0 0x3118
#define A5XX_VBIF_PERF_PWR_CNT_HIGH1 0x3119
#define A5XX_VBIF_PERF_PWR_CNT_HIGH2 0x311A
/* GPMU registers */
#define A5XX_GPMU_INST_RAM_BASE 0x8800
#define A5XX_GPMU_DATA_RAM_BASE 0x9800
#define A5XX_GPMU_SP_POWER_CNTL 0xA881
#define A5XX_GPMU_RBCCU_CLOCK_CNTL 0xA886
#define A5XX_GPMU_RBCCU_POWER_CNTL 0xA887
#define A5XX_GPMU_SP_PWR_CLK_STATUS 0xA88B
#define A5XX_GPMU_RBCCU_PWR_CLK_STATUS 0xA88D
#define A5XX_GPMU_PWR_COL_STAGGER_DELAY 0xA891
#define A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL 0xA892
#define A5XX_GPMU_PWR_COL_INTER_FRAME_HYST 0xA893
#define A5XX_GPMU_PWR_COL_BINNING_CTRL 0xA894
#define A5XX_GPMU_CLOCK_THROTTLE_CTRL 0xA8A3
#define A5XX_GPMU_WFI_CONFIG 0xA8C1
#define A5XX_GPMU_RBBM_INTR_INFO 0xA8D6
#define A5XX_GPMU_CM3_SYSRESET 0xA8D8
#define A5XX_GPMU_GENERAL_0 0xA8E0
#define A5XX_GPMU_GENERAL_1 0xA8E1
/* COUNTABLE FOR SP PERFCOUNTER */
#define A5XX_SP_ALU_ACTIVE_CYCLES 0x1
#define A5XX_SP0_ICL1_MISSES 0x35
#define A5XX_SP_FS_CFLOW_INSTRUCTIONS 0x27
/* COUNTABLE FOR TSE PERFCOUNTER */
#define A5XX_TSE_INPUT_PRIM_NUM 0x6
/* COUNTABLE FOR RBBM PERFCOUNTER */
#define A5XX_RBBM_ALWAYS_COUNT 0x0
/* GPMU POWER COUNTERS */
#define A5XX_SP_POWER_COUNTER_0_LO 0xA840
#define A5XX_SP_POWER_COUNTER_0_HI 0xA841
#define A5XX_SP_POWER_COUNTER_1_LO 0xA842
#define A5XX_SP_POWER_COUNTER_1_HI 0xA843
#define A5XX_SP_POWER_COUNTER_2_LO 0xA844
#define A5XX_SP_POWER_COUNTER_2_HI 0xA845
#define A5XX_SP_POWER_COUNTER_3_LO 0xA846
#define A5XX_SP_POWER_COUNTER_3_HI 0xA847
#define A5XX_TP_POWER_COUNTER_0_LO 0xA848
#define A5XX_TP_POWER_COUNTER_0_HI 0xA849
#define A5XX_TP_POWER_COUNTER_1_LO 0xA84A
#define A5XX_TP_POWER_COUNTER_1_HI 0xA84B
#define A5XX_TP_POWER_COUNTER_2_LO 0xA84C
#define A5XX_TP_POWER_COUNTER_2_HI 0xA84D
#define A5XX_TP_POWER_COUNTER_3_LO 0xA84E
#define A5XX_TP_POWER_COUNTER_3_HI 0xA84F
#define A5XX_RB_POWER_COUNTER_0_LO 0xA850
#define A5XX_RB_POWER_COUNTER_0_HI 0xA851
#define A5XX_RB_POWER_COUNTER_1_LO 0xA852
#define A5XX_RB_POWER_COUNTER_1_HI 0xA853
#define A5XX_RB_POWER_COUNTER_2_LO 0xA854
#define A5XX_RB_POWER_COUNTER_2_HI 0xA855
#define A5XX_RB_POWER_COUNTER_3_LO 0xA856
#define A5XX_RB_POWER_COUNTER_3_HI 0xA857
#define A5XX_CCU_POWER_COUNTER_0_LO 0xA858
#define A5XX_CCU_POWER_COUNTER_0_HI 0xA859
#define A5XX_CCU_POWER_COUNTER_1_LO 0xA85A
#define A5XX_CCU_POWER_COUNTER_1_HI 0xA85B
#define A5XX_UCHE_POWER_COUNTER_0_LO 0xA85C
#define A5XX_UCHE_POWER_COUNTER_0_HI 0xA85D
#define A5XX_UCHE_POWER_COUNTER_1_LO 0xA85E
#define A5XX_UCHE_POWER_COUNTER_1_HI 0xA85F
#define A5XX_UCHE_POWER_COUNTER_2_LO 0xA860
#define A5XX_UCHE_POWER_COUNTER_2_HI 0xA861
#define A5XX_UCHE_POWER_COUNTER_3_LO 0xA862
#define A5XX_UCHE_POWER_COUNTER_3_HI 0xA863
#define A5XX_CP_POWER_COUNTER_0_LO 0xA864
#define A5XX_CP_POWER_COUNTER_0_HI 0xA865
#define A5XX_CP_POWER_COUNTER_1_LO 0xA866
#define A5XX_CP_POWER_COUNTER_1_HI 0xA867
#define A5XX_CP_POWER_COUNTER_2_LO 0xA868
#define A5XX_CP_POWER_COUNTER_2_HI 0xA869
#define A5XX_CP_POWER_COUNTER_3_LO 0xA86A
#define A5XX_CP_POWER_COUNTER_3_HI 0xA86B
#define A5XX_GPMU_POWER_COUNTER_0_LO 0xA86C
#define A5XX_GPMU_POWER_COUNTER_0_HI 0xA86D
#define A5XX_GPMU_POWER_COUNTER_1_LO 0xA86E
#define A5XX_GPMU_POWER_COUNTER_1_HI 0xA86F
#define A5XX_GPMU_POWER_COUNTER_2_LO 0xA870
#define A5XX_GPMU_POWER_COUNTER_2_HI 0xA871
#define A5XX_GPMU_POWER_COUNTER_3_LO 0xA872
#define A5XX_GPMU_POWER_COUNTER_3_HI 0xA873
#define A5XX_GPMU_POWER_COUNTER_4_LO 0xA874
#define A5XX_GPMU_POWER_COUNTER_4_HI 0xA875
#define A5XX_GPMU_POWER_COUNTER_5_LO 0xA876
#define A5XX_GPMU_POWER_COUNTER_5_HI 0xA877
#define A5XX_GPMU_POWER_COUNTER_ENABLE 0xA878
#define A5XX_GPMU_ALWAYS_ON_COUNTER_LO 0xA879
#define A5XX_GPMU_ALWAYS_ON_COUNTER_HI 0xA87A
#define A5XX_GPMU_ALWAYS_ON_COUNTER_RESET 0xA87B
#define A5XX_GPMU_POWER_COUNTER_SELECT_0 0xA87C
#define A5XX_GPMU_POWER_COUNTER_SELECT_1 0xA87D
#define A5XX_GPMU_GPMU_SP_CLOCK_CONTROL 0xA880
#define A5XX_GPMU_CLOCK_THROTTLE_CTRL 0xA8A3
#define A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL 0xA8A8
#define A5XX_GPMU_TEMP_SENSOR_ID 0xAC00
#define A5XX_GPMU_TEMP_SENSOR_CONFIG 0xAC01
#define A5XX_GPMU_DELTA_TEMP_THRESHOLD 0xAC03
#define A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK 0xAC06
#define A5XX_GPMU_LEAKAGE_TEMP_COEFF_0_1 0xAC40
#define A5XX_GPMU_LEAKAGE_TEMP_COEFF_2_3 0xAC41
#define A5XX_GPMU_LEAKAGE_VTG_COEFF_0_1 0xAC42
#define A5XX_GPMU_LEAKAGE_VTG_COEFF_2_3 0xAC43
#define A5XX_GPMU_BASE_LEAKAGE 0xAC46
#define A5XX_GPMU_GPMU_VOLTAGE 0xAC60
#define A5XX_GPMU_GPMU_VOLTAGE_INTR_STATUS 0xAC61
#define A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK 0xAC62
#define A5XX_GPMU_GPMU_PWR_THRESHOLD 0xAC80
#define A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL 0xACC4
#define A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS 0xACC5
#define A5XX_GPMU_GPMU_ISENSE_CTRL 0xACD0
#define A5XX_GDPM_CONFIG1 0xB80C
#define A5XX_GDPM_INT_EN 0xB80F
#define A5XX_GDPM_INT_MASK 0xB811
#define A5XX_GPMU_BEC_ENABLE 0xB9A0
/* ISENSE registers */
#define A5XX_GPU_CS_DECIMAL_ALIGN 0xC16A
#define A5XX_GPU_CS_SENSOR_PARAM_CORE_1 0xC126
#define A5XX_GPU_CS_SENSOR_PARAM_CORE_2 0xC127
#define A5XX_GPU_CS_SW_OV_FUSE_EN 0xC168
#define A5XX_GPU_CS_SENSOR_GENERAL_STATUS 0xC41A
#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_0 0xC41D
#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_2 0xC41F
#define A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_4 0xC421
#define A5XX_GPU_CS_ENABLE_REG 0xC520
#define A5XX_GPU_CS_AMP_CALIBRATION_CONTROL1 0xC557
#define A5XX_GPU_CS_AMP_CALIBRATION_DONE 0xC565
#define A5XX_GPU_CS_ENDPOINT_CALIBRATION_DONE 0xC556
#endif /* _A5XX_REG_H */

1144
drivers/gpu/msm/a6xx_reg.h Normal file
View file

File diff suppressed because it is too large Load diff

1433
drivers/gpu/msm/adreno-gpulist.h Normal file
View file

File diff suppressed because it is too large Load diff

3893
drivers/gpu/msm/adreno.c Normal file
View file

File diff suppressed because it is too large Load diff

1751
drivers/gpu/msm/adreno.h Normal file
View file

File diff suppressed because it is too large Load diff

1450
drivers/gpu/msm/adreno_a3xx.c Normal file
View file

File diff suppressed because it is too large Load diff

44
drivers/gpu/msm/adreno_a3xx.h Normal file
View file

@ -0,0 +1,44 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2013-2016, 2019, The Linux Foundation. All rights reserved.
*/
#ifndef __A3XX_H
#define __A3XX_H
#include "a3xx_reg.h"
/**
* struct adreno_a3xx_core - a3xx specific GPU core definitions
*/
struct adreno_a3xx_core {
/** @base: Container for the generic &struct adreno_gpu_core */
struct adreno_gpu_core base;
/** pm4fw_name: Name of the PM4 microcode file */
const char *pm4fw_name;
/** pfpfw_name: Name of the PFP microcode file */
const char *pfpfw_name;
/** @vbif: List of registers and values to write for VBIF */
const struct adreno_reglist *vbif;
/** @vbif_count: Number of registers in @vbif */
u32 vbif_count;
};
/**
* to_a3xx_core - return the a3xx specific GPU core struct
* @adreno_dev: An Adreno GPU device handle
*
* Returns:
* A pointer to the a3xx specific GPU core struct
*/
static inline const struct adreno_a3xx_core *
to_a3xx_core(struct adreno_device *adreno_dev)
{
const struct adreno_gpu_core *core = adreno_dev->gpucore;
return container_of(core, struct adreno_a3xx_core, base);
}
unsigned int a3xx_irq_pending(struct adreno_device *adreno_dev);
void a3xx_snapshot(struct adreno_device *adreno_dev,
struct kgsl_snapshot *snapshot);
#endif /*__A3XX_H */

412
drivers/gpu/msm/adreno_a3xx_snapshot.c Normal file
View file

@ -0,0 +1,412 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2012-2017,2019, The Linux Foundation. All rights reserved.
*/
#include <linux/io.h>
#include "adreno.h"
#include "adreno_a3xx.h"
#include "adreno_snapshot.h"
#include "kgsl_device.h"
/*
* Set of registers to dump for A3XX on snapshot.
* Registers in pairs - first value is the start offset, second
* is the stop offset (inclusive)
*/
static const unsigned int a3xx_registers[] = {
0x0000, 0x0002, 0x0010, 0x0012, 0x0018, 0x0018, 0x0020, 0x0027,
0x0029, 0x002b, 0x002e, 0x0033, 0x0040, 0x0042, 0x0050, 0x005c,
0x0060, 0x006c, 0x0080, 0x0082, 0x0084, 0x0088, 0x0090, 0x00e5,
0x00ea, 0x00ed, 0x0100, 0x0100, 0x0110, 0x0123, 0x01c0, 0x01c1,
0x01c3, 0x01c5, 0x01c7, 0x01c7, 0x01d5, 0x01d9, 0x01dc, 0x01dd,
0x01ea, 0x01ea, 0x01ee, 0x01f1, 0x01f5, 0x01f6, 0x01f8, 0x01f9,
0x01fc, 0x01ff,
0x0440, 0x0440, 0x0443, 0x0443, 0x0445, 0x0445, 0x044d, 0x044f,
0x0452, 0x0452, 0x0454, 0x046f, 0x047c, 0x047c, 0x047f, 0x047f,
0x0578, 0x057f, 0x0600, 0x0602, 0x0605, 0x0607, 0x060a, 0x060e,
0x0612, 0x0614, 0x0c01, 0x0c02, 0x0c06, 0x0c1d, 0x0c3d, 0x0c3f,
0x0c48, 0x0c4b, 0x0c80, 0x0c80, 0x0c88, 0x0c8b, 0x0ca0, 0x0cb7,
0x0cc0, 0x0cc1, 0x0cc6, 0x0cc7, 0x0ce4, 0x0ce5,
0x0e41, 0x0e45, 0x0e64, 0x0e65,
0x0e80, 0x0e82, 0x0e84, 0x0e89, 0x0ea0, 0x0ea1, 0x0ea4, 0x0ea7,
0x0ec4, 0x0ecb, 0x0ee0, 0x0ee0, 0x0f00, 0x0f01, 0x0f03, 0x0f09,
0x2040, 0x2040, 0x2044, 0x2044, 0x2048, 0x204d, 0x2068, 0x2069,
0x206c, 0x206d, 0x2070, 0x2070, 0x2072, 0x2072, 0x2074, 0x2075,
0x2079, 0x207a, 0x20c0, 0x20d3, 0x20e4, 0x20ef, 0x2100, 0x2109,
0x210c, 0x210c, 0x210e, 0x210e, 0x2110, 0x2111, 0x2114, 0x2115,
0x21e4, 0x21e4, 0x21ea, 0x21ea, 0x21ec, 0x21ed, 0x21f0, 0x21f0,
0x2240, 0x227e,
0x2280, 0x228b, 0x22c0, 0x22c0, 0x22c4, 0x22ce, 0x22d0, 0x22d8,
0x22df, 0x22e6, 0x22e8, 0x22e9, 0x22ec, 0x22ec, 0x22f0, 0x22f7,
0x22ff, 0x22ff, 0x2340, 0x2343,
0x2440, 0x2440, 0x2444, 0x2444, 0x2448, 0x244d,
0x2468, 0x2469, 0x246c, 0x246d, 0x2470, 0x2470, 0x2472, 0x2472,
0x2474, 0x2475, 0x2479, 0x247a, 0x24c0, 0x24d3, 0x24e4, 0x24ef,
0x2500, 0x2509, 0x250c, 0x250c, 0x250e, 0x250e, 0x2510, 0x2511,
0x2514, 0x2515, 0x25e4, 0x25e4, 0x25ea, 0x25ea, 0x25ec, 0x25ed,
0x25f0, 0x25f0,
0x2640, 0x267e, 0x2680, 0x268b, 0x26c0, 0x26c0, 0x26c4, 0x26ce,
0x26d0, 0x26d8, 0x26df, 0x26e6, 0x26e8, 0x26e9, 0x26ec, 0x26ec,
0x26f0, 0x26f7, 0x26ff, 0x26ff, 0x2740, 0x2743,
0x300C, 0x300E, 0x301C, 0x301D,
0x302A, 0x302A, 0x302C, 0x302D, 0x3030, 0x3031, 0x3034, 0x3036,
0x303C, 0x303C, 0x305E, 0x305F,
};
/* Removed the following HLSQ register ranges from being read during
* fault tolerance since reading the registers may cause the device to hang:
*/
static const unsigned int a3xx_hlsq_registers[] = {
0x0e00, 0x0e05, 0x0e0c, 0x0e0c, 0x0e22, 0x0e23,
0x2200, 0x2212, 0x2214, 0x2217, 0x221a, 0x221a,
0x2600, 0x2612, 0x2614, 0x2617, 0x261a, 0x261a,
};
/* The set of additional registers to be dumped for A330 */
static const unsigned int a330_registers[] = {
0x1d0, 0x1d0, 0x1d4, 0x1d4, 0x453, 0x453,
};
/* Shader memory size in words */
#define SHADER_MEMORY_SIZE 0x4000
/**
* _rbbm_debug_bus_read - Helper function to read data from the RBBM
* debug bus.
* @device - GPU device to read/write registers
* @block_id - Debug bus block to read from
* @index - Index in the debug bus block to read
* @ret - Value of the register read
*/
static void _rbbm_debug_bus_read(struct kgsl_device *device,
unsigned int block_id, unsigned int index, unsigned int *val)
{
unsigned int block = (block_id << 8) | 1 << 16;
kgsl_regwrite(device, A3XX_RBBM_DEBUG_BUS_CTL, block | index);
kgsl_regread(device, A3XX_RBBM_DEBUG_BUS_DATA_STATUS, val);
}
/**
* a3xx_snapshot_shader_memory - Helper function to dump the GPU shader
* memory to the snapshot buffer.
* @device: GPU device whose shader memory is to be dumped
* @buf: Pointer to binary snapshot data blob being made
* @remain: Number of remaining bytes in the snapshot blob
* @priv: Unused parameter
*
*/
static size_t a3xx_snapshot_shader_memory(struct kgsl_device *device,
u8 *buf, size_t remain, void *priv)
{
struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
void *data = buf + sizeof(*header);
unsigned int shader_read_len = SHADER_MEMORY_SIZE;
if (remain < DEBUG_SECTION_SZ(shader_read_len)) {
SNAPSHOT_ERR_NOMEM(device, "SHADER MEMORY");
return 0;
}
header->type = SNAPSHOT_DEBUG_SHADER_MEMORY;
header->size = shader_read_len;
/* Map shader memory to kernel, for dumping */
if (IS_ERR_OR_NULL(device->shader_mem_virt)) {
struct resource *res;
res = platform_get_resource_byname(device->pdev,
IORESOURCE_MEM, "kgsl_3d0_shader_memory");
if (res)
device->shader_mem_virt =
devm_ioremap_resource(&device->pdev->dev, res);
}
if (IS_ERR_OR_NULL(device->shader_mem_virt)) {
dev_err(device->dev, "Unable to map the shader memory\n");
return 0;
}
memcpy_fromio(data, device->shader_mem_virt, shader_read_len << 2);
return DEBUG_SECTION_SZ(shader_read_len);
}
static size_t a3xx_snapshot_debugbus_block(struct kgsl_device *device,
u8 *buf, size_t remain, void *priv)
{
struct kgsl_snapshot_debugbus *header
= (struct kgsl_snapshot_debugbus *)buf;
struct adreno_debugbus_block *block = priv;
int i;
unsigned int *data = (unsigned int *)(buf + sizeof(*header));
size_t size;
size = (0x40 * sizeof(unsigned int)) + sizeof(*header);
if (remain < size) {
SNAPSHOT_ERR_NOMEM(device, "DEBUGBUS");
return 0;
}
header->id = block->block_id;
header->count = 0x40;
for (i = 0; i < 0x40; i++)
_rbbm_debug_bus_read(device, block->block_id, i, &data[i]);
return size;
}
static struct adreno_debugbus_block debugbus_blocks[] = {
{ RBBM_BLOCK_ID_CP, 0x52, },
{ RBBM_BLOCK_ID_RBBM, 0x40, },
{ RBBM_BLOCK_ID_VBIF, 0x40, },
{ RBBM_BLOCK_ID_HLSQ, 0x40, },
{ RBBM_BLOCK_ID_UCHE, 0x40, },
{ RBBM_BLOCK_ID_PC, 0x40, },
{ RBBM_BLOCK_ID_VFD, 0x40, },
{ RBBM_BLOCK_ID_VPC, 0x40, },
{ RBBM_BLOCK_ID_TSE, 0x40, },
{ RBBM_BLOCK_ID_RAS, 0x40, },
{ RBBM_BLOCK_ID_VSC, 0x40, },
{ RBBM_BLOCK_ID_SP_0, 0x40, },
{ RBBM_BLOCK_ID_SP_1, 0x40, },
{ RBBM_BLOCK_ID_SP_2, 0x40, },
{ RBBM_BLOCK_ID_SP_3, 0x40, },
{ RBBM_BLOCK_ID_TPL1_0, 0x40, },
{ RBBM_BLOCK_ID_TPL1_1, 0x40, },
{ RBBM_BLOCK_ID_TPL1_2, 0x40, },
{ RBBM_BLOCK_ID_TPL1_3, 0x40, },
{ RBBM_BLOCK_ID_RB_0, 0x40, },
{ RBBM_BLOCK_ID_RB_1, 0x40, },
{ RBBM_BLOCK_ID_RB_2, 0x40, },
{ RBBM_BLOCK_ID_RB_3, 0x40, },
{ RBBM_BLOCK_ID_MARB_0, 0x40, },
{ RBBM_BLOCK_ID_MARB_1, 0x40, },
{ RBBM_BLOCK_ID_MARB_2, 0x40, },
{ RBBM_BLOCK_ID_MARB_3, 0x40, },
};
static void a3xx_snapshot_debugbus(struct kgsl_device *device,
struct kgsl_snapshot *snapshot)
{
int i;
for (i = 0; i < ARRAY_SIZE(debugbus_blocks); i++) {
kgsl_snapshot_add_section(device,
KGSL_SNAPSHOT_SECTION_DEBUGBUS, snapshot,
a3xx_snapshot_debugbus_block,
(void *) &debugbus_blocks[i]);
}
}
static void _snapshot_hlsq_regs(struct kgsl_device *device,
struct kgsl_snapshot *snapshot)
{
unsigned int next_pif = 0;
/*
* Trying to read HLSQ registers when the HLSQ block is busy
* will cause the device to hang. The RBBM_DEBUG_BUS has information
* that will tell us if the HLSQ block is busy or not. Read values
* from the debug bus to ensure the HLSQ block is not busy (this
* is hardware dependent). If the HLSQ block is busy do not
* dump the registers, otherwise dump the HLSQ registers.
*/
/*
* tpif status bits: RBBM_BLOCK_ID_HLSQ index 4 [4:0]
* spif status bits: RBBM_BLOCK_ID_HLSQ index 7 [5:0]
*
* if ((tpif == 0, 1, 28) && (spif == 0, 1, 10))
* then dump HLSQ registers
*/
/* check tpif */
_rbbm_debug_bus_read(device, RBBM_BLOCK_ID_HLSQ, 4, &next_pif);
next_pif &= 0x1f;
if (next_pif != 0 && next_pif != 1 && next_pif != 28)
return;
/* check spif */
_rbbm_debug_bus_read(device, RBBM_BLOCK_ID_HLSQ, 7, &next_pif);
next_pif &= 0x3f;
if (next_pif != 0 && next_pif != 1 && next_pif != 10)
return;
SNAPSHOT_REGISTERS(device, snapshot, a3xx_hlsq_registers);
}
#define VPC_MEM_SIZE 512
static size_t a3xx_snapshot_vpc_memory(struct kgsl_device *device, u8 *buf,
size_t remain, void *priv)
{
struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
unsigned int *data = (unsigned int *)(buf + sizeof(*header));
size_t size = 4 * VPC_MEM_SIZE;
int bank, addr, i = 0;
if (remain < DEBUG_SECTION_SZ(size)) {
SNAPSHOT_ERR_NOMEM(device, "VPC MEMORY");
return 0;
}
header->type = SNAPSHOT_DEBUG_VPC_MEMORY;
header->size = size;
for (bank = 0; bank < 4; bank++) {
for (addr = 0; addr < VPC_MEM_SIZE; addr++) {
unsigned int val = bank | (addr << 4);
kgsl_regwrite(device, A3XX_VPC_VPC_DEBUG_RAM_SEL, val);
kgsl_regread(device, A3XX_VPC_VPC_DEBUG_RAM_READ,
&data[i++]);
}
}
return DEBUG_SECTION_SZ(size);
}
static size_t a3xx_snapshot_cp_pm4_ram(struct kgsl_device *device, u8 *buf,
size_t remain, void *priv)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
unsigned int *data = (unsigned int *)(buf + sizeof(*header));
int i;
struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4);
size_t size = fw->size - 1;
if (remain < DEBUG_SECTION_SZ(size)) {
SNAPSHOT_ERR_NOMEM(device, "CP PM4 RAM DEBUG");
return 0;
}
header->type = SNAPSHOT_DEBUG_CP_PM4_RAM;
header->size = size;
/*
* Read the firmware from the GPU rather than use our cache in order to
* try to catch mis-programming or corruption in the hardware. We do
* use the cached version of the size, however, instead of trying to
* maintain always changing hardcoded constants
*/
kgsl_regwrite(device, A3XX_CP_ME_RAM_RADDR, 0x0);
for (i = 0; i < size; i++)
kgsl_regread(device, A3XX_CP_ME_RAM_DATA, &data[i]);
return DEBUG_SECTION_SZ(size);
}
static size_t a3xx_snapshot_cp_pfp_ram(struct kgsl_device *device, u8 *buf,
size_t remain, void *priv)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct kgsl_snapshot_debug *header = (struct kgsl_snapshot_debug *)buf;
unsigned int *data = (unsigned int *)(buf + sizeof(*header));
int i;
struct adreno_firmware *fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP);
int size = fw->size - 1;
if (remain < DEBUG_SECTION_SZ(size)) {
SNAPSHOT_ERR_NOMEM(device, "CP PFP RAM DEBUG");
return 0;
}
header->type = SNAPSHOT_DEBUG_CP_PFP_RAM;
header->size = size;
/*
* Read the firmware from the GPU rather than use our cache in order to
* try to catch mis-programming or corruption in the hardware. We do
* use the cached version of the size, however, instead of trying to
* maintain always changing hardcoded constants
*/
kgsl_regwrite(device, A3XX_CP_PFP_UCODE_ADDR, 0x0);
for (i = 0; i < size; i++)
kgsl_regread(device, A3XX_CP_PFP_UCODE_DATA, &data[i]);
return DEBUG_SECTION_SZ(size);
}
/*
* a3xx_snapshot() - A3XX GPU snapshot function
* @adreno_dev: Device being snapshotted
* @snapshot: Snapshot meta data
* @remain: Amount of space left in snapshot memory
*
* This is where all of the A3XX specific bits and pieces are grabbed
* into the snapshot memory
*/
void a3xx_snapshot(struct adreno_device *adreno_dev,
struct kgsl_snapshot *snapshot)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned int reg, val;
/* Disable Clock gating temporarily for the debug bus to work */
adreno_writereg(adreno_dev, ADRENO_REG_RBBM_CLOCK_CTL, 0x00);
SNAPSHOT_REGISTERS(device, snapshot, a3xx_registers);
_snapshot_hlsq_regs(device, snapshot);
kgsl_snapshot_indexed_registers(device, snapshot,
A3XX_CP_STATE_DEBUG_INDEX, A3XX_CP_STATE_DEBUG_DATA, 0, 0x14);
/* CP_ME indexed registers */
kgsl_snapshot_indexed_registers(device, snapshot,
A3XX_CP_ME_CNTL, A3XX_CP_ME_STATUS, 64, 44);
/* VPC memory */
kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG,
snapshot, a3xx_snapshot_vpc_memory, NULL);
/* CP MEQ */
val = 16;
kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG, snapshot,
adreno_snapshot_cp_meq, &val);
/* Shader working/shadow memory */
kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG,
snapshot, a3xx_snapshot_shader_memory, NULL);
/* CP PFP and PM4 */
/*
* Reading the microcode while the CP is running will
* basically move the CP instruction pointer to
* whatever address we read. Big badaboom ensues. Stop the CP
* (if it isn't already stopped) to ensure that we are safe.
* We do this here and not earlier to avoid corrupting the RBBM
* status and CP registers - by the time we get here we don't
* care about the contents of the CP anymore.
*/
adreno_readreg(adreno_dev, ADRENO_REG_CP_ME_CNTL, &reg);
reg |= (1 << 27) | (1 << 28);
adreno_writereg(adreno_dev, ADRENO_REG_CP_ME_CNTL, reg);
kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG,
snapshot, a3xx_snapshot_cp_pfp_ram, NULL);
kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG,
snapshot, a3xx_snapshot_cp_pm4_ram, NULL);
/* CP ROQ */
val = 128;
kgsl_snapshot_add_section(device, KGSL_SNAPSHOT_SECTION_DEBUG,
snapshot, adreno_snapshot_cp_roq, &val);
a3xx_snapshot_debugbus(device, snapshot);
}

3086
drivers/gpu/msm/adreno_a5xx.c Normal file
View file

File diff suppressed because it is too large Load diff

245
drivers/gpu/msm/adreno_a5xx.h Normal file
View file

@ -0,0 +1,245 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2015-2017,2019 The Linux Foundation. All rights reserved.
*/
#ifndef _ADRENO_A5XX_H_
#define _ADRENO_A5XX_H_
#include "a5xx_reg.h"
/**
* struct adreno_a5xx_core - a5xx specific GPU core definitions
*/
struct adreno_a5xx_core {
/** @base: Container for the generic &struct adreno_gpu_core */
struct adreno_gpu_core base;
/** @gpmu_tsens: ID for the temperature sensor used by the GPMU */
unsigned int gpmu_tsens;
/** @max_power: Max possible power draw of a core */
unsigned int max_power;
/** pm4fw_name: Name of the PM4 microcode file */
const char *pm4fw_name;
/** pfpfw_name: Name of the PFP microcode file */
const char *pfpfw_name;
/** gpmufw_name: Name of the GPMU microcode file */
const char *gpmufw_name;
/** @regfw_name: Filename for the LM registers if applicable */
const char *regfw_name;
/** @zap_name: Name of the CPZ zap file */
const char *zap_name;
/** @hwcg: List of registers and values to write for HWCG */
const struct adreno_reglist *hwcg;
/** @hwcg_count: Number of registers in @hwcg */
u32 hwcg_count;
/** @vbif: List of registers and values to write for VBIF */
const struct adreno_reglist *vbif;
/** @vbif_count: Number of registers in @vbif */
u32 vbif_count;
};
#define A5XX_CP_CTXRECORD_MAGIC_REF 0x27C4BAFCUL
/* Size of each CP preemption record */
#define A5XX_CP_CTXRECORD_SIZE_IN_BYTES 0x10000
/* Size of the preemption counter block (in bytes) */
#define A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE (16 * 4)
/**
* struct a5xx_cp_preemption_record - CP context record for
* preemption.
* @magic: (00) Value at this offset must be equal to
* A5XX_CP_CTXRECORD_MAGIC_REF.
* @info: (04) Type of record. Written non-zero (usually) by CP.
* we must set to zero for all ringbuffers.
* @data: (08) DATA field in SET_RENDER_MODE or checkpoint packets.
* Written by CP when switching out. Not used on switch-in.
* we must initialize to zero.
* @cntl: (12) RB_CNTL, saved and restored by CP.
* @rptr: (16) RB_RPTR, saved and restored by CP.
* @wptr: (20) RB_WPTR, saved and restored by CP.
* @rptr_addr: (24) RB_RPTR_ADDR_LO|HI saved and restored.
* rbase: (32) RB_BASE_LO|HI saved and restored.
* counter: (40) Pointer to preemption counter
*/
struct a5xx_cp_preemption_record {
uint32_t magic;
uint32_t info;
uint32_t data;
uint32_t cntl;
uint32_t rptr;
uint32_t wptr;
uint64_t rptr_addr;
uint64_t rbase;
uint64_t counter;
};
#define A5XX_CP_SMMU_INFO_MAGIC_REF 0x3618CDA3UL
/**
* struct a5xx_cp_smmu_info - CP preemption SMMU info.
* @magic: (00) The value at this offset must be equal to
* A5XX_CP_SMMU_INFO_MAGIC_REF.
* @_pad4: (04) Reserved/padding
* @ttbr0: (08) Base address of the page table for the
* incoming context.
* @context_idr: (16) Context Identification Register value.
*/
struct a5xx_cp_smmu_info {
uint32_t magic;
uint32_t _pad4;
uint64_t ttbr0;
uint32_t asid;
uint32_t context_idr;
};
void a5xx_snapshot(struct adreno_device *adreno_dev,
struct kgsl_snapshot *snapshot);
unsigned int a5xx_num_registers(void);
void a5xx_crashdump_init(struct adreno_device *adreno_dev);
void a5xx_hwcg_set(struct adreno_device *adreno_dev, bool on);
#define A5XX_CP_RB_CNTL_DEFAULT (((ilog2(4) << 8) & 0x1F00) | \
(ilog2(KGSL_RB_DWORDS >> 1) & 0x3F))
/* GPMU interrupt multiplexor */
#define FW_INTR_INFO (0)
#define LLM_ACK_ERR_INTR (1)
#define ISENS_TRIM_ERR_INTR (2)
#define ISENS_ERR_INTR (3)
#define ISENS_IDLE_ERR_INTR (4)
#define ISENS_PWR_ON_ERR_INTR (5)
#define WDOG_EXPITED (31)
#define VALID_GPMU_IRQ (\
BIT(FW_INTR_INFO) | \
BIT(LLM_ACK_ERR_INTR) | \
BIT(ISENS_TRIM_ERR_INTR) | \
BIT(ISENS_ERR_INTR) | \
BIT(ISENS_IDLE_ERR_INTR) | \
BIT(ISENS_PWR_ON_ERR_INTR) | \
BIT(WDOG_EXPITED))
/* A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL */
#define STATE_OF_CHILD GENMASK(5, 4)
#define STATE_OF_CHILD_01 BIT(4)
#define STATE_OF_CHILD_11 (BIT(4) | BIT(5))
#define IDLE_FULL_LM_SLEEP BIT(0)
/* A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS */
#define WAKEUP_ACK BIT(1)
#define IDLE_FULL_ACK BIT(0)
/* A5XX_GPMU_GPMU_ISENSE_CTRL */
#define ISENSE_CGC_EN_DISABLE BIT(0)
/* A5XX_GPMU_TEMP_SENSOR_CONFIG */
#define GPMU_BCL_ENABLED BIT(4)
#define GPMU_LLM_ENABLED BIT(9)
#define GPMU_ISENSE_STATUS GENMASK(3, 0)
#define GPMU_ISENSE_END_POINT_CAL_ERR BIT(0)
#define AMP_CALIBRATION_RETRY_CNT 3
#define AMP_CALIBRATION_TIMEOUT 6
/* A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK */
#define VOLTAGE_INTR_EN BIT(0)
/* A5XX_GPMU_GPMU_PWR_THRESHOLD */
#define PWR_THRESHOLD_VALID 0x80000000
/* A5XX_GPMU_GPMU_SP_CLOCK_CONTROL */
#define CNTL_IP_CLK_ENABLE BIT(0)
/* AGC */
#define AGC_INIT_BASE A5XX_GPMU_DATA_RAM_BASE
#define AGC_INIT_MSG_MAGIC (AGC_INIT_BASE + 5)
#define AGC_MSG_BASE (AGC_INIT_BASE + 7)
#define AGC_MSG_STATE (AGC_MSG_BASE + 0)
#define AGC_MSG_COMMAND (AGC_MSG_BASE + 1)
#define AGC_MSG_PAYLOAD_SIZE (AGC_MSG_BASE + 3)
#define AGC_MSG_PAYLOAD (AGC_MSG_BASE + 5)
#define AGC_INIT_MSG_VALUE 0xBABEFACE
#define AGC_POWER_CONFIG_PRODUCTION_ID 1
#define AGC_LM_CONFIG (136/4)
#define AGC_LM_CONFIG_ENABLE_GPMU_ADAPTIVE (1)
#define AGC_LM_CONFIG_ENABLE_ERROR (3 << 4)
#define AGC_LM_CONFIG_ISENSE_ENABLE (1 << 4)
#define AGC_THROTTLE_SEL_DCS (1 << 8)
#define AGC_THROTTLE_DISABLE (2 << 8)
#define AGC_LLM_ENABLED (1 << 16)
#define AGC_GPU_VERSION_MASK GENMASK(18, 17)
#define AGC_GPU_VERSION_SHIFT 17
#define AGC_BCL_DISABLED (1 << 24)
#define AGC_LEVEL_CONFIG (140/4)
#define LM_DCVS_LIMIT 1
/* FW file tages */
#define GPMU_FIRMWARE_ID 2
#define GPMU_SEQUENCE_ID 3
#define GPMU_INST_RAM_SIZE 0xFFF
#define HEADER_MAJOR 1
#define HEADER_MINOR 2
#define HEADER_DATE 3
#define HEADER_TIME 4
#define HEADER_SEQUENCE 5
#define MAX_HEADER_SIZE 10
#define LM_SEQUENCE_ID 1
#define MAX_SEQUENCE_ID 3
#define GPMU_ISENSE_SAVE (A5XX_GPMU_DATA_RAM_BASE + 200/4)
/* LM defaults */
#define LM_DEFAULT_LIMIT 6000
#define A530_DEFAULT_LEAKAGE 0x004E001A
static inline bool lm_on(struct adreno_device *adreno_dev)
{
return ADRENO_FEATURE(adreno_dev, ADRENO_LM) &&
test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag);
}
/**
* to_a5xx_core - return the a5xx specific GPU core struct
* @adreno_dev: An Adreno GPU device handle
*
* Returns:
* A pointer to the a5xx specific GPU core struct
*/
static inline const struct adreno_a5xx_core *
to_a5xx_core(struct adreno_device *adreno_dev)
{
const struct adreno_gpu_core *core = adreno_dev->gpucore;
return container_of(core, struct adreno_a5xx_core, base);
}
/* Preemption functions */
void a5xx_preemption_trigger(struct adreno_device *adreno_dev);
void a5xx_preemption_schedule(struct adreno_device *adreno_dev);
void a5xx_preemption_start(struct adreno_device *adreno_dev);
int a5xx_preemption_init(struct adreno_device *adreno_dev);
void a5xx_preemption_close(struct adreno_device *adreno_dev);
int a5xx_preemption_yield_enable(unsigned int *cmds);
unsigned int a5xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev,
unsigned int *cmds);
unsigned int a5xx_preemption_pre_ibsubmit(
struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb,
unsigned int *cmds, struct kgsl_context *context);
void a5xx_preempt_callback(struct adreno_device *adreno_dev, int bit);
#endif

1406
drivers/gpu/msm/adreno_a5xx_packets.h Normal file
View file

File diff suppressed because it is too large Load diff

632
drivers/gpu/msm/adreno_a5xx_preempt.c Normal file
View file

@ -0,0 +1,632 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2014-2017,2019 The Linux Foundation. All rights reserved.
*/
#include "adreno.h"
#include "adreno_a5xx.h"
#include "adreno_pm4types.h"
#include "adreno_trace.h"
#define PREEMPT_RECORD(_field) \
offsetof(struct a5xx_cp_preemption_record, _field)
#define PREEMPT_SMMU_RECORD(_field) \
offsetof(struct a5xx_cp_smmu_info, _field)
static void _update_wptr(struct adreno_device *adreno_dev, bool reset_timer)
{
struct adreno_ringbuffer *rb = adreno_dev->cur_rb;
unsigned int wptr;
unsigned long flags;
spin_lock_irqsave(&rb->preempt_lock, flags);
adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr);
if (wptr != rb->wptr) {
adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR,
rb->wptr);
/*
* In case something got submitted while preemption was on
* going, reset the timer.
*/
reset_timer = true;
}
if (reset_timer)
rb->dispatch_q.expires = jiffies +
msecs_to_jiffies(adreno_drawobj_timeout);
spin_unlock_irqrestore(&rb->preempt_lock, flags);
}
static inline bool adreno_move_preempt_state(struct adreno_device *adreno_dev,
enum adreno_preempt_states old, enum adreno_preempt_states new)
{
return (atomic_cmpxchg(&adreno_dev->preempt.state, old, new) == old);
}
static void _a5xx_preemption_done(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned int status;
/*
* In the very unlikely case that the power is off, do nothing - the
* state will be reset on power up and everybody will be happy
*/
if (!kgsl_state_is_awake(device))
return;
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status);
if (status != 0) {
dev_err(device->dev,
"Preemption not complete: status=%X cur=%d R/W=%X/%X next=%d R/W=%X/%X\n",
status, adreno_dev->cur_rb->id,
adreno_get_rptr(adreno_dev->cur_rb),
adreno_dev->cur_rb->wptr,
adreno_dev->next_rb->id,
adreno_get_rptr(adreno_dev->next_rb),
adreno_dev->next_rb->wptr);
/* Set a fault and restart */
adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);
adreno_dispatcher_schedule(device);
return;
}
del_timer_sync(&adreno_dev->preempt.timer);
trace_adreno_preempt_done(adreno_dev->cur_rb, adreno_dev->next_rb, 0);
/* Clean up all the bits */
adreno_dev->prev_rb = adreno_dev->cur_rb;
adreno_dev->cur_rb = adreno_dev->next_rb;
adreno_dev->next_rb = NULL;
/* Update the wptr for the new command queue */
_update_wptr(adreno_dev, true);
/* Update the dispatcher timer for the new command queue */
mod_timer(&adreno_dev->dispatcher.timer,
adreno_dev->cur_rb->dispatch_q.expires);
/* Clear the preempt state */
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
}
static void _a5xx_preemption_fault(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned int status;
/*
* If the power is on check the preemption status one more time - if it
* was successful then just transition to the complete state
*/
if (kgsl_state_is_awake(device)) {
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status);
if (status == 0) {
adreno_set_preempt_state(adreno_dev,
ADRENO_PREEMPT_COMPLETE);
adreno_dispatcher_schedule(device);
return;
}
}
dev_err(device->dev,
"Preemption timed out: cur=%d R/W=%X/%X, next=%d R/W=%X/%X\n",
adreno_dev->cur_rb->id,
adreno_get_rptr(adreno_dev->cur_rb),
adreno_dev->cur_rb->wptr,
adreno_dev->next_rb->id,
adreno_get_rptr(adreno_dev->next_rb),
adreno_dev->next_rb->wptr);
adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);
adreno_dispatcher_schedule(device);
}
static void _a5xx_preemption_worker(struct work_struct *work)
{
struct adreno_preemption *preempt = container_of(work,
struct adreno_preemption, work);
struct adreno_device *adreno_dev = container_of(preempt,
struct adreno_device, preempt);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
/* Need to take the mutex to make sure that the power stays on */
mutex_lock(&device->mutex);
if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_FAULTED))
_a5xx_preemption_fault(adreno_dev);
mutex_unlock(&device->mutex);
}
static void _a5xx_preemption_timer(struct timer_list *t)
{
struct adreno_preemption *preempt = from_timer(preempt, t, timer);
struct adreno_device *adreno_dev = container_of(preempt,
struct adreno_device, preempt);
/* We should only be here from a triggered state */
if (!adreno_move_preempt_state(adreno_dev,
ADRENO_PREEMPT_TRIGGERED, ADRENO_PREEMPT_FAULTED))
return;
/* Schedule the worker to take care of the details */
queue_work(system_unbound_wq, &adreno_dev->preempt.work);
}
/* Find the highest priority active ringbuffer */
static struct adreno_ringbuffer *a5xx_next_ringbuffer(
struct adreno_device *adreno_dev)
{
struct adreno_ringbuffer *rb;
unsigned long flags;
unsigned int i;
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
bool empty;
spin_lock_irqsave(&rb->preempt_lock, flags);
empty = adreno_rb_empty(rb);
spin_unlock_irqrestore(&rb->preempt_lock, flags);
if (!empty)
return rb;
}
return NULL;
}
void a5xx_preemption_trigger(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
struct adreno_ringbuffer *next;
uint64_t ttbr0;
unsigned int contextidr;
unsigned long flags;
/* Put ourselves into a possible trigger state */
if (!adreno_move_preempt_state(adreno_dev,
ADRENO_PREEMPT_NONE, ADRENO_PREEMPT_START))
return;
/* Get the next ringbuffer to preempt in */
next = a5xx_next_ringbuffer(adreno_dev);
/*
* Nothing to do if every ringbuffer is empty or if the current
* ringbuffer is the only active one
*/
if (next == NULL || next == adreno_dev->cur_rb) {
/*
* Update any critical things that might have been skipped while
* we were looking for a new ringbuffer
*/
if (next != NULL) {
_update_wptr(adreno_dev, false);
mod_timer(&adreno_dev->dispatcher.timer,
adreno_dev->cur_rb->dispatch_q.expires);
}
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
return;
}
/* Turn off the dispatcher timer */
del_timer(&adreno_dev->dispatcher.timer);
/*
* This is the most critical section - we need to take care not to race
* until we have programmed the CP for the switch
*/
spin_lock_irqsave(&next->preempt_lock, flags);
/*
* Get the pagetable from the pagetable info.
* The pagetable_desc is allocated and mapped at probe time, and
* preemption_desc at init time, so no need to check if
* sharedmem accesses to these memdescs succeed.
*/
kgsl_sharedmem_readq(&next->pagetable_desc, &ttbr0,
PT_INFO_OFFSET(ttbr0));
kgsl_sharedmem_readl(&next->pagetable_desc, &contextidr,
PT_INFO_OFFSET(contextidr));
kgsl_sharedmem_writel(device, &next->preemption_desc,
PREEMPT_RECORD(wptr), next->wptr);
spin_unlock_irqrestore(&next->preempt_lock, flags);
/* And write it to the smmu info */
kgsl_sharedmem_writeq(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(ttbr0), ttbr0);
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(context_idr), contextidr);
kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
lower_32_bits(next->preemption_desc.gpuaddr));
kgsl_regwrite(device, A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
upper_32_bits(next->preemption_desc.gpuaddr));
adreno_dev->next_rb = next;
/* Start the timer to detect a stuck preemption */
mod_timer(&adreno_dev->preempt.timer,
jiffies + msecs_to_jiffies(ADRENO_PREEMPT_TIMEOUT));
trace_adreno_preempt_trigger(adreno_dev->cur_rb, adreno_dev->next_rb,
1);
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_TRIGGERED);
/* Trigger the preemption */
adreno_writereg(adreno_dev, ADRENO_REG_CP_PREEMPT, 1);
}
void a5xx_preempt_callback(struct adreno_device *adreno_dev, int bit)
{
unsigned int status;
if (!adreno_move_preempt_state(adreno_dev,
ADRENO_PREEMPT_TRIGGERED, ADRENO_PREEMPT_PENDING))
return;
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status);
if (status != 0) {
dev_err(KGSL_DEVICE(adreno_dev)->dev,
"preempt interrupt with non-zero status: %X\n",
status);
/*
* Under the assumption that this is a race between the
* interrupt and the register, schedule the worker to clean up.
* If the status still hasn't resolved itself by the time we get
* there then we have to assume something bad happened
*/
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE);
adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev));
return;
}
del_timer(&adreno_dev->preempt.timer);
trace_adreno_preempt_done(adreno_dev->cur_rb, adreno_dev->next_rb, 0);
adreno_dev->prev_rb = adreno_dev->cur_rb;
adreno_dev->cur_rb = adreno_dev->next_rb;
adreno_dev->next_rb = NULL;
/* Update the wptr if it changed while preemption was ongoing */
_update_wptr(adreno_dev, true);
/* Update the dispatcher timer for the new command queue */
mod_timer(&adreno_dev->dispatcher.timer,
adreno_dev->cur_rb->dispatch_q.expires);
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
a5xx_preemption_trigger(adreno_dev);
}
void a5xx_preemption_schedule(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
if (!adreno_is_preemption_enabled(adreno_dev))
return;
mutex_lock(&device->mutex);
if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE))
_a5xx_preemption_done(adreno_dev);
a5xx_preemption_trigger(adreno_dev);
mutex_unlock(&device->mutex);
}
unsigned int a5xx_preemption_pre_ibsubmit(
struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb,
unsigned int *cmds, struct kgsl_context *context)
{
unsigned int *cmds_orig = cmds;
uint64_t gpuaddr = rb->preemption_desc.gpuaddr;
unsigned int preempt_style = 0;
if (context) {
/*
* Preemption from secure to unsecure needs Zap shader to be
* run to clear all secure content. CP does not know during
* preemption if it is switching between secure and unsecure
* contexts so restrict Secure contexts to be preempted at
* ringbuffer level.
*/
if (context->flags & KGSL_CONTEXT_SECURE)
preempt_style = KGSL_CONTEXT_PREEMPT_STYLE_RINGBUFFER;
else
preempt_style = ADRENO_PREEMPT_STYLE(context->flags);
}
/*
* CP_PREEMPT_ENABLE_GLOBAL(global preemption) can only be set by KMD
* in ringbuffer.
* 1) set global preemption to 0x0 to disable global preemption.
* Only RB level preemption is allowed in this mode
* 2) Set global preemption to defer(0x2) for finegrain preemption.
* when global preemption is set to defer(0x2),
* CP_PREEMPT_ENABLE_LOCAL(local preemption) determines the
* preemption point. Local preemption
* can be enabled by both UMD(within IB) and KMD.
*/
*cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_GLOBAL, 1);
*cmds++ = ((preempt_style == KGSL_CONTEXT_PREEMPT_STYLE_FINEGRAIN)
? 2 : 0);
/* Turn CP protection OFF */
cmds += cp_protected_mode(adreno_dev, cmds, 0);
/*
* CP during context switch will save context switch info to
* a5xx_cp_preemption_record pointed by CONTEXT_SWITCH_SAVE_ADDR
*/
*cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 1);
*cmds++ = lower_32_bits(gpuaddr);
*cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI, 1);
*cmds++ = upper_32_bits(gpuaddr);
/* Turn CP protection ON */
cmds += cp_protected_mode(adreno_dev, cmds, 1);
/*
* Enable local preemption for finegrain preemption in case of
* a misbehaving IB
*/
if (preempt_style == KGSL_CONTEXT_PREEMPT_STYLE_FINEGRAIN) {
*cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1);
*cmds++ = 1;
} else {
*cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1);
*cmds++ = 0;
}
/* Enable CP_CONTEXT_SWITCH_YIELD packets in the IB2s */
*cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1);
*cmds++ = 2;
return (unsigned int) (cmds - cmds_orig);
}
int a5xx_preemption_yield_enable(unsigned int *cmds)
{
/*
* SRM -- set render mode (ex binning, direct render etc)
* SRM is set by UMD usually at start of IB to tell CP the type of
* preemption.
* KMD needs to set SRM to NULL to indicate CP that rendering is
* done by IB.
*/
*cmds++ = cp_type7_packet(CP_SET_RENDER_MODE, 5);
*cmds++ = 0;
*cmds++ = 0;
*cmds++ = 0;
*cmds++ = 0;
*cmds++ = 0;
*cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1);
*cmds++ = 1;
return 8;
}
unsigned int a5xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev,
unsigned int *cmds)
{
int dwords = 0;
cmds[dwords++] = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4);
/* Write NULL to the address to skip the data write */
dwords += cp_gpuaddr(adreno_dev, &cmds[dwords], 0x0);
cmds[dwords++] = 1;
/* generate interrupt on preemption completion */
cmds[dwords++] = 1;
return dwords;
}
void a5xx_preemption_start(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
struct adreno_ringbuffer *rb;
unsigned int i;
if (!adreno_is_preemption_enabled(adreno_dev))
return;
/* Force the state to be clear */
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
/* smmu_info is allocated and mapped in a5xx_preemption_iommu_init */
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(magic), A5XX_CP_SMMU_INFO_MAGIC_REF);
kgsl_sharedmem_writeq(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(ttbr0), MMU_DEFAULT_TTBR0(device));
/* The CP doesn't use the asid record, so poison it */
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(asid), 0xDECAFBAD);
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(context_idr),
MMU_DEFAULT_CONTEXTIDR(device));
adreno_writereg64(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI,
iommu->smmu_info.gpuaddr);
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
/*
* preemption_desc is allocated and mapped at init time,
* so no need to check sharedmem_writel return value
*/
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(rptr), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(wptr), 0);
adreno_ringbuffer_set_pagetable(rb,
device->mmu.defaultpagetable);
}
}
static int a5xx_preemption_ringbuffer_init(struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb, uint64_t counteraddr)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
ret = kgsl_allocate_global(device, &rb->preemption_desc,
A5XX_CP_CTXRECORD_SIZE_IN_BYTES, 0, KGSL_MEMDESC_PRIVILEGED,
"preemption_desc");
if (ret)
return ret;
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(magic), A5XX_CP_CTXRECORD_MAGIC_REF);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(info), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(data), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(cntl), A5XX_CP_RB_CNTL_DEFAULT);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(rptr), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(wptr), 0);
kgsl_sharedmem_writeq(device, &rb->preemption_desc,
PREEMPT_RECORD(rptr_addr), SCRATCH_RPTR_GPU_ADDR(device,
rb->id));
kgsl_sharedmem_writeq(device, &rb->preemption_desc,
PREEMPT_RECORD(rbase), rb->buffer_desc.gpuaddr);
kgsl_sharedmem_writeq(device, &rb->preemption_desc,
PREEMPT_RECORD(counter), counteraddr);
return 0;
}
#if IS_ENABLED(CONFIG_ARM_SMMU)
static int a5xx_preemption_iommu_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
/* Allocate mem for storing preemption smmu record */
return kgsl_allocate_global(device, &iommu->smmu_info, PAGE_SIZE,
KGSL_MEMFLAGS_GPUREADONLY, KGSL_MEMDESC_PRIVILEGED,
"smmu_info");
}
static void a5xx_preemption_iommu_close(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
kgsl_free_global(device, &iommu->smmu_info);
}
#else
static int a5xx_preemption_iommu_init(struct adreno_device *adreno_dev)
{
return -ENODEV;
}
static void a5xx_preemption_iommu_close(struct adreno_device *adreno_dev)
{
}
#endif
static void _preemption_close(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_preemption *preempt = &adreno_dev->preempt;
struct adreno_ringbuffer *rb;
unsigned int i;
del_timer(&preempt->timer);
kgsl_free_global(device, &preempt->counters);
a5xx_preemption_iommu_close(adreno_dev);
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
kgsl_free_global(device, &rb->preemption_desc);
}
}
void a5xx_preemption_close(struct adreno_device *adreno_dev)
{
if (!test_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv))
return;
_preemption_close(adreno_dev);
}
int a5xx_preemption_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_preemption *preempt = &adreno_dev->preempt;
struct adreno_ringbuffer *rb;
int ret;
unsigned int i;
uint64_t addr;
/* We are dependent on IOMMU to make preemption go on the CP side */
if (kgsl_mmu_get_mmutype(device) != KGSL_MMU_TYPE_IOMMU)
return -ENODEV;
INIT_WORK(&preempt->work, _a5xx_preemption_worker);
timer_setup(&preempt->timer, _a5xx_preemption_timer, 0);
/* Allocate mem for storing preemption counters */
ret = kgsl_allocate_global(device, &preempt->counters,
adreno_dev->num_ringbuffers *
A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE, 0, 0,
"preemption_counters");
if (ret)
goto err;
addr = preempt->counters.gpuaddr;
/* Allocate mem for storing preemption switch record */
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
ret = a5xx_preemption_ringbuffer_init(adreno_dev, rb, addr);
if (ret)
goto err;
addr += A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE;
}
ret = a5xx_preemption_iommu_init(adreno_dev);
err:
if (ret)
_preemption_close(adreno_dev);
return ret;
}

1166
drivers/gpu/msm/adreno_a5xx_snapshot.c Normal file
View file

File diff suppressed because it is too large Load diff

2619
drivers/gpu/msm/adreno_a6xx.c Normal file
View file

File diff suppressed because it is too large Load diff

300
drivers/gpu/msm/adreno_a6xx.h Normal file
View file

@ -0,0 +1,300 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
*/
#ifndef _ADRENO_A6XX_H_
#define _ADRENO_A6XX_H_
#include <linux/delay.h>
#include "a6xx_reg.h"
/**
* struct a6xx_protected_regs - container for a protect register span
*/
struct a6xx_protected_regs {
/** @reg: Physical protected mode register to write to */
u32 reg;
/** @start: Dword offset of the starting register in the range */
u32 start;
/**
* @end: Dword offset of the ending register in the range
* (inclusive)
*/
u32 end;
/**
* @noaccess: 1 if the register should not be accessible from
* userspace, 0 if it can be read (but not written)
*/
u32 noaccess;
};
/**
* struct adreno_a6xx_core - a6xx specific GPU core definitions
*/
struct adreno_a6xx_core {
/** @base: Container for the generic GPU definitions */
struct adreno_gpu_core base;
/** @gmu_major: The maximum GMU version supported by the core */
u32 gmu_major;
/** @gmu_minor: The minimum GMU version supported by the core */
u32 gmu_minor;
/** @prim_fifo_threshold: target specific value for PC_DBG_ECO_CNTL */
unsigned int prim_fifo_threshold;
/** @pdc_address_offset: Offset for the PDC region for the target */
unsigned int pdc_address_offset;
/** @sqefw_name: Name of the SQE microcode file */
const char *sqefw_name;
/** @gmufw_name: Name of the GMU firmware file */
const char *gmufw_name;
/** @zap_name: Name of the CPZ zap file */
const char *zap_name;
/** @hwcg: List of registers and values to write for HWCG */
const struct adreno_reglist *hwcg;
/** @hwcg_count: Number of registers in @hwcg */
u32 hwcg_count;
/** @vbif: List of registers and values to write for VBIF */
const struct adreno_reglist *vbif;
/** @vbif_count: Number of registers in @vbif */
u32 vbif_count;
/** @veto_fal10: veto status for fal10 feature */
bool veto_fal10;
/** @pdc_in_aop: True if PDC programmed in AOP */
bool pdc_in_aop;
/** @hang_detect_cycles: Hang detect counter timeout value */
u32 hang_detect_cycles;
/** @protected_regs: Array of protected registers for the target */
const struct a6xx_protected_regs *protected_regs;
/** @disable_tseskip: True if TSESkip logic is disabled */
bool disable_tseskip;
};
#define CP_CLUSTER_FE 0x0
#define CP_CLUSTER_SP_VS 0x1
#define CP_CLUSTER_PC_VS 0x2
#define CP_CLUSTER_GRAS 0x3
#define CP_CLUSTER_SP_PS 0x4
#define CP_CLUSTER_PS 0x5
#define CP_CLUSTER_VPC_PS 0x6
/**
* struct a6xx_cp_preemption_record - CP context record for
* preemption.
* @magic: (00) Value at this offset must be equal to
* A6XX_CP_CTXRECORD_MAGIC_REF.
* @info: (04) Type of record. Written non-zero (usually) by CP.
* we must set to zero for all ringbuffers.
* @errno: (08) Error code. Initialize this to A6XX_CP_CTXRECORD_ERROR_NONE.
* CP will update to another value if a preemption error occurs.
* @data: (12) DATA field in YIELD and SET_MARKER packets.
* Written by CP when switching out. Not used on switch-in. Initialized to 0.
* @cntl: (16) RB_CNTL, saved and restored by CP. We must initialize this.
* @rptr: (20) RB_RPTR, saved and restored by CP. We must initialize this.
* @wptr: (24) RB_WPTR, saved and restored by CP. We must initialize this.
* @_pad28: (28) Reserved/padding.
* @rptr_addr: (32) RB_RPTR_ADDR_LO|HI saved and restored. We must initialize.
* rbase: (40) RB_BASE_LO|HI saved and restored.
* counter: (48) Pointer to preemption counter.
*/
struct a6xx_cp_preemption_record {
uint32_t magic;
uint32_t info;
uint32_t errno;
uint32_t data;
uint32_t cntl;
uint32_t rptr;
uint32_t wptr;
uint32_t _pad28;
uint64_t rptr_addr;
uint64_t rbase;
uint64_t counter;
};
/**
* struct a6xx_cp_smmu_info - CP preemption SMMU info.
* @magic: (00) The value at this offset must be equal to
* A6XX_CP_SMMU_INFO_MAGIC_REF.
* @_pad4: (04) Reserved/padding
* @ttbr0: (08) Base address of the page table for the
* incoming context.
* @context_idr: (16) Context Identification Register value.
*/
struct a6xx_cp_smmu_info {
uint32_t magic;
uint32_t _pad4;
uint64_t ttbr0;
uint32_t asid;
uint32_t context_idr;
};
#define A6XX_CP_SMMU_INFO_MAGIC_REF 0x241350D5UL
/**
* struct cpu_gpu_spinlock - CP spinlock structure for power up list
* @flag_ucode: flag value set by CP
* @flag_kmd: flag value set by KMD
* @turn: turn variable set by both CP and KMD
* @list_length: this tells CP the last dword in the list:
* 16 + (4 * (List_Length - 1))
* @list_offset: this tells CP the start of preemption only list:
* 16 + (4 * List_Offset)
*/
struct cpu_gpu_lock {
uint32_t flag_ucode;
uint32_t flag_kmd;
uint32_t turn;
uint16_t list_length;
uint16_t list_offset;
};
#define A6XX_CP_CTXRECORD_MAGIC_REF 0xAE399D6EUL
/* Size of each CP preemption record */
#define A6XX_CP_CTXRECORD_SIZE_IN_BYTES (2112 * 1024)
/* Size of the user context record block (in bytes) */
#define A6XX_CP_CTXRECORD_USER_RESTORE_SIZE (192 * 1024)
/* Size of the performance counter save/restore block (in bytes) */
#define A6XX_CP_PERFCOUNTER_SAVE_RESTORE_SIZE (4 * 1024)
#define A6XX_CP_RB_CNTL_DEFAULT (((ilog2(4) << 8) & 0x1F00) | \
(ilog2(KGSL_RB_DWORDS >> 1) & 0x3F))
/**
* to_a6xx_core - return the a6xx specific GPU core struct
* @adreno_dev: An Adreno GPU device handle
*
* Returns:
* A pointer to the a6xx specific GPU core struct
*/
static inline const struct adreno_a6xx_core *
to_a6xx_core(struct adreno_device *adreno_dev)
{
const struct adreno_gpu_core *core = adreno_dev->gpucore;
return container_of(core, struct adreno_a6xx_core, base);
}
/*
* timed_poll_check() - polling *gmu* register at given offset until
* its value changed to match expected value. The function times
* out and returns after given duration if register is not updated
* as expected.
*
* @device: Pointer to KGSL device
* @offset: Register offset
* @expected_ret: expected register value that stops polling
* @timout: number of jiffies to abort the polling
* @mask: bitmask to filter register value to match expected_ret
*/
static inline int timed_poll_check(struct kgsl_device *device,
unsigned int offset, unsigned int expected_ret,
unsigned int timeout, unsigned int mask)
{
unsigned long t;
unsigned int value;
t = jiffies + msecs_to_jiffies(timeout);
do {
gmu_core_regread(device, offset, &value);
if ((value & mask) == expected_ret)
return 0;
/* Wait 100us to reduce unnecessary AHB bus traffic */
usleep_range(10, 100);
} while (!time_after(jiffies, t));
/* Double check one last time */
gmu_core_regread(device, offset, &value);
if ((value & mask) == expected_ret)
return 0;
return -ETIMEDOUT;
}
static inline int timed_poll_check_rscc(struct kgsl_device *device,
unsigned int offset, unsigned int expected_ret,
unsigned int timeout, unsigned int mask)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned long t;
unsigned int value;
t = jiffies + msecs_to_jiffies(timeout);
do {
if (adreno_is_a650_family(adreno_dev))
adreno_rscc_regread(adreno_dev, offset, &value);
else
gmu_core_regread(device, offset + RSCC_OFFSET_LEGACY,
&value);
if ((value & mask) == expected_ret)
return 0;
/* Wait 100us to reduce unnecessary AHB bus traffic */
usleep_range(10, 100);
} while (!time_after(jiffies, t));
/* Double check one last time */
if (adreno_is_a650_family(adreno_dev))
adreno_rscc_regread(adreno_dev, offset, &value);
else
gmu_core_regread(device, offset + RSCC_OFFSET_LEGACY, &value);
if ((value & mask) == expected_ret)
return 0;
return -ETIMEDOUT;
}
/*
* read_AO_counter() - Returns the 64bit always on counter value
*
* @device: Pointer to KGSL device
*/
static inline uint64_t read_AO_counter(struct kgsl_device *device)
{
unsigned int l, h, h1;
gmu_core_regread(device, A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_H, &h);
gmu_core_regread(device, A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_L, &l);
gmu_core_regread(device, A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_H, &h1);
/*
* If there's no change in COUNTER_H we have no overflow so return,
* otherwise read COUNTER_L again
*/
if (h == h1)
return (uint64_t) l | ((uint64_t) h << 32);
gmu_core_regread(device, A6XX_GMU_CX_GMU_ALWAYS_ON_COUNTER_L, &l);
return (uint64_t) l | ((uint64_t) h1 << 32);
}
/* Preemption functions */
void a6xx_preemption_trigger(struct adreno_device *adreno_dev);
void a6xx_preemption_schedule(struct adreno_device *adreno_dev);
void a6xx_preemption_start(struct adreno_device *adreno_dev);
int a6xx_preemption_init(struct adreno_device *adreno_dev);
void a6xx_preemption_close(struct adreno_device *adreno_dev);
unsigned int a6xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev,
unsigned int *cmds);
unsigned int a6xx_preemption_pre_ibsubmit(struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb,
unsigned int *cmds, struct kgsl_context *context);
unsigned int a6xx_set_marker(unsigned int *cmds,
enum adreno_cp_marker_type type);
void a6xx_preemption_callback(struct adreno_device *adreno_dev, int bit);
int a6xx_preemption_context_init(struct kgsl_context *context);
void a6xx_preemption_context_destroy(struct kgsl_context *context);
void a6xx_snapshot(struct adreno_device *adreno_dev,
struct kgsl_snapshot *snapshot);
void a6xx_crashdump_init(struct adreno_device *adreno_dev);
int a6xx_gmu_sptprac_enable(struct adreno_device *adreno_dev);
void a6xx_gmu_sptprac_disable(struct adreno_device *adreno_dev);
bool a6xx_gmu_sptprac_is_on(struct adreno_device *adreno_dev);
#endif

1744
drivers/gpu/msm/adreno_a6xx_gmu.c Normal file
View file

File diff suppressed because it is too large Load diff

792
drivers/gpu/msm/adreno_a6xx_preempt.c Normal file
View file

@ -0,0 +1,792 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
*/
#include "adreno.h"
#include "adreno_a6xx.h"
#include "adreno_pm4types.h"
#include "adreno_trace.h"
#define PREEMPT_RECORD(_field) \
offsetof(struct a6xx_cp_preemption_record, _field)
#define PREEMPT_SMMU_RECORD(_field) \
offsetof(struct a6xx_cp_smmu_info, _field)
enum {
SET_PSEUDO_REGISTER_SAVE_REGISTER_SMMU_INFO = 0,
SET_PSEUDO_REGISTER_SAVE_REGISTER_PRIV_NON_SECURE_SAVE_ADDR,
SET_PSEUDO_REGISTER_SAVE_REGISTER_PRIV_SECURE_SAVE_ADDR,
SET_PSEUDO_REGISTER_SAVE_REGISTER_NON_PRIV_SAVE_ADDR,
SET_PSEUDO_REGISTER_SAVE_REGISTER_COUNTER,
};
static void _update_wptr(struct adreno_device *adreno_dev, bool reset_timer)
{
struct adreno_ringbuffer *rb = adreno_dev->cur_rb;
unsigned long flags;
int ret = 0;
spin_lock_irqsave(&rb->preempt_lock, flags);
if (in_interrupt() == 0) {
/*
* We might have skipped updating the wptr in case we are in
* dispatcher context. Do it now.
*/
if (rb->skip_inline_wptr) {
ret = adreno_gmu_fenced_write(adreno_dev,
ADRENO_REG_CP_RB_WPTR, rb->wptr,
FENCE_STATUS_WRITEDROPPED0_MASK);
reset_timer = true;
rb->skip_inline_wptr = false;
}
} else {
unsigned int wptr;
adreno_readreg(adreno_dev, ADRENO_REG_CP_RB_WPTR, &wptr);
if (wptr != rb->wptr) {
adreno_writereg(adreno_dev, ADRENO_REG_CP_RB_WPTR,
rb->wptr);
reset_timer = true;
}
}
if (reset_timer)
rb->dispatch_q.expires = jiffies +
msecs_to_jiffies(adreno_drawobj_timeout);
spin_unlock_irqrestore(&rb->preempt_lock, flags);
if (in_interrupt() == 0) {
/* If WPTR update fails, set the fault and trigger recovery */
if (ret) {
adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT);
adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev));
}
}
}
static inline bool adreno_move_preempt_state(struct adreno_device *adreno_dev,
enum adreno_preempt_states old, enum adreno_preempt_states new)
{
return (atomic_cmpxchg(&adreno_dev->preempt.state, old, new) == old);
}
static void _a6xx_preemption_done(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned int status;
/*
* In the very unlikely case that the power is off, do nothing - the
* state will be reset on power up and everybody will be happy
*/
if (!kgsl_state_is_awake(device))
return;
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status);
if (status & 0x1) {
dev_err(device->dev,
"Preemption not complete: status=%X cur=%d R/W=%X/%X next=%d R/W=%X/%X\n",
status, adreno_dev->cur_rb->id,
adreno_get_rptr(adreno_dev->cur_rb),
adreno_dev->cur_rb->wptr,
adreno_dev->next_rb->id,
adreno_get_rptr(adreno_dev->next_rb),
adreno_dev->next_rb->wptr);
/* Set a fault and restart */
adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);
adreno_dispatcher_schedule(device);
return;
}
adreno_dev->preempt.count++;
del_timer_sync(&adreno_dev->preempt.timer);
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_LEVEL_STATUS, &status);
trace_adreno_preempt_done(adreno_dev->cur_rb, adreno_dev->next_rb,
status);
/* Clean up all the bits */
adreno_dev->prev_rb = adreno_dev->cur_rb;
adreno_dev->cur_rb = adreno_dev->next_rb;
adreno_dev->next_rb = NULL;
/* Update the wptr for the new command queue */
_update_wptr(adreno_dev, true);
/* Update the dispatcher timer for the new command queue */
mod_timer(&adreno_dev->dispatcher.timer,
adreno_dev->cur_rb->dispatch_q.expires);
/* Clear the preempt state */
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
}
static void _a6xx_preemption_fault(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned int status;
/*
* If the power is on check the preemption status one more time - if it
* was successful then just transition to the complete state
*/
if (kgsl_state_is_awake(device)) {
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status);
if (status == 0) {
adreno_set_preempt_state(adreno_dev,
ADRENO_PREEMPT_COMPLETE);
adreno_dispatcher_schedule(device);
return;
}
}
dev_err(device->dev,
"Preemption timed out: cur=%d R/W=%X/%X, next=%d R/W=%X/%X\n",
adreno_dev->cur_rb->id,
adreno_get_rptr(adreno_dev->cur_rb),
adreno_dev->cur_rb->wptr,
adreno_dev->next_rb->id,
adreno_get_rptr(adreno_dev->next_rb),
adreno_dev->next_rb->wptr);
adreno_set_gpu_fault(adreno_dev, ADRENO_PREEMPT_FAULT);
adreno_dispatcher_schedule(device);
}
static void _a6xx_preemption_worker(struct work_struct *work)
{
struct adreno_preemption *preempt = container_of(work,
struct adreno_preemption, work);
struct adreno_device *adreno_dev = container_of(preempt,
struct adreno_device, preempt);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
/* Need to take the mutex to make sure that the power stays on */
mutex_lock(&device->mutex);
if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_FAULTED))
_a6xx_preemption_fault(adreno_dev);
mutex_unlock(&device->mutex);
}
static void _a6xx_preemption_timer(struct timer_list *t)
{
struct adreno_preemption *preempt = from_timer(preempt, t, timer);
struct adreno_device *adreno_dev = container_of(preempt,
struct adreno_device, preempt);
/* We should only be here from a triggered state */
if (!adreno_move_preempt_state(adreno_dev,
ADRENO_PREEMPT_TRIGGERED, ADRENO_PREEMPT_FAULTED))
return;
/* Schedule the worker to take care of the details */
queue_work(system_unbound_wq, &adreno_dev->preempt.work);
}
/* Find the highest priority active ringbuffer */
static struct adreno_ringbuffer *a6xx_next_ringbuffer(
struct adreno_device *adreno_dev)
{
struct adreno_ringbuffer *rb;
unsigned long flags;
unsigned int i;
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
bool empty;
spin_lock_irqsave(&rb->preempt_lock, flags);
empty = adreno_rb_empty(rb);
spin_unlock_irqrestore(&rb->preempt_lock, flags);
if (!empty)
return rb;
}
return NULL;
}
void a6xx_preemption_trigger(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
struct adreno_ringbuffer *next;
uint64_t ttbr0, gpuaddr;
unsigned int contextidr, cntl;
unsigned long flags;
struct adreno_preemption *preempt = &adreno_dev->preempt;
cntl = (((preempt->preempt_level << 6) & 0xC0) |
((preempt->skipsaverestore << 9) & 0x200) |
((preempt->usesgmem << 8) & 0x100) | 0x1);
/* Put ourselves into a possible trigger state */
if (!adreno_move_preempt_state(adreno_dev,
ADRENO_PREEMPT_NONE, ADRENO_PREEMPT_START))
return;
/* Get the next ringbuffer to preempt in */
next = a6xx_next_ringbuffer(adreno_dev);
/*
* Nothing to do if every ringbuffer is empty or if the current
* ringbuffer is the only active one
*/
if (next == NULL || next == adreno_dev->cur_rb) {
/*
* Update any critical things that might have been skipped while
* we were looking for a new ringbuffer
*/
if (next != NULL) {
_update_wptr(adreno_dev, false);
mod_timer(&adreno_dev->dispatcher.timer,
adreno_dev->cur_rb->dispatch_q.expires);
}
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
return;
}
/* Turn off the dispatcher timer */
del_timer(&adreno_dev->dispatcher.timer);
/*
* This is the most critical section - we need to take care not to race
* until we have programmed the CP for the switch
*/
spin_lock_irqsave(&next->preempt_lock, flags);
/*
* Get the pagetable from the pagetable info.
* The pagetable_desc is allocated and mapped at probe time, and
* preemption_desc at init time, so no need to check if
* sharedmem accesses to these memdescs succeed.
*/
kgsl_sharedmem_readq(&next->pagetable_desc, &ttbr0,
PT_INFO_OFFSET(ttbr0));
kgsl_sharedmem_readl(&next->pagetable_desc, &contextidr,
PT_INFO_OFFSET(contextidr));
kgsl_sharedmem_writel(device, &next->preemption_desc,
PREEMPT_RECORD(wptr), next->wptr);
spin_unlock_irqrestore(&next->preempt_lock, flags);
/* And write it to the smmu info */
kgsl_sharedmem_writeq(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(ttbr0), ttbr0);
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(context_idr), contextidr);
kgsl_sharedmem_readq(&device->scratch, &gpuaddr,
SCRATCH_PREEMPTION_CTXT_RESTORE_ADDR_OFFSET(next->id));
/*
* Set a keepalive bit before the first preemption register write.
* This is required since while each individual write to the context
* switch registers will wake the GPU from collapse, it will not in
* itself cause GPU activity. Thus, the GPU could technically be
* re-collapsed between subsequent register writes leading to a
* prolonged preemption sequence. The keepalive bit prevents any
* further power collapse while it is set.
* It is more efficient to use a keepalive+wake-on-fence approach here
* rather than an OOB. Both keepalive and the fence are effectively
* free when the GPU is already powered on, whereas an OOB requires an
* unconditional handshake with the GMU.
*/
if (gmu_core_isenabled(device))
gmu_core_regrmw(device, A6XX_GMU_AO_SPARE_CNTL, 0x0, 0x2);
/*
* Fenced writes on this path will make sure the GPU is woken up
* in case it was power collapsed by the GMU.
*/
if (adreno_gmu_fenced_write(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO,
lower_32_bits(next->preemption_desc.gpuaddr),
FENCE_STATUS_WRITEDROPPED1_MASK))
goto err;
/*
* Above fence writes will make sure GMU comes out of
* IFPC state if its was in IFPC state but it doesn't
* guarantee that GMU FW actually moved to ACTIVE state
* i.e. wake-up from IFPC is complete.
* Wait for GMU to move to ACTIVE state before triggering
* preemption. This is require to make sure CP doesn't
* interrupt GMU during wake-up from IFPC.
*/
if (gmu_core_dev_wait_for_active_transition(device))
goto err;
if (adreno_gmu_fenced_write(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI,
upper_32_bits(next->preemption_desc.gpuaddr),
FENCE_STATUS_WRITEDROPPED1_MASK))
goto err;
if (adreno_gmu_fenced_write(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO,
lower_32_bits(next->secure_preemption_desc.gpuaddr),
FENCE_STATUS_WRITEDROPPED1_MASK))
goto err;
if (adreno_gmu_fenced_write(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI,
upper_32_bits(next->secure_preemption_desc.gpuaddr),
FENCE_STATUS_WRITEDROPPED1_MASK))
goto err;
if (adreno_gmu_fenced_write(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO,
lower_32_bits(gpuaddr),
FENCE_STATUS_WRITEDROPPED1_MASK))
goto err;
if (adreno_gmu_fenced_write(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI,
upper_32_bits(gpuaddr),
FENCE_STATUS_WRITEDROPPED1_MASK))
goto err;
adreno_dev->next_rb = next;
/* Start the timer to detect a stuck preemption */
mod_timer(&adreno_dev->preempt.timer,
jiffies + msecs_to_jiffies(ADRENO_PREEMPT_TIMEOUT));
trace_adreno_preempt_trigger(adreno_dev->cur_rb, adreno_dev->next_rb,
cntl);
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_TRIGGERED);
/* Trigger the preemption */
if (adreno_gmu_fenced_write(adreno_dev, ADRENO_REG_CP_PREEMPT, cntl,
FENCE_STATUS_WRITEDROPPED1_MASK)) {
adreno_dev->next_rb = NULL;
del_timer(&adreno_dev->preempt.timer);
goto err;
}
return;
err:
/* If fenced write fails, set the fault and trigger recovery */
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT);
adreno_dispatcher_schedule(device);
/* Clear the keep alive */
if (gmu_core_isenabled(device))
gmu_core_regrmw(device, A6XX_GMU_AO_SPARE_CNTL, 0x2, 0x0);
}
void a6xx_preemption_callback(struct adreno_device *adreno_dev, int bit)
{
unsigned int status;
if (!adreno_move_preempt_state(adreno_dev,
ADRENO_PREEMPT_TRIGGERED, ADRENO_PREEMPT_PENDING))
return;
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT, &status);
if (status & 0x1) {
dev_err(KGSL_DEVICE(adreno_dev)->dev,
"preempt interrupt with non-zero status: %X\n",
status);
/*
* Under the assumption that this is a race between the
* interrupt and the register, schedule the worker to clean up.
* If the status still hasn't resolved itself by the time we get
* there then we have to assume something bad happened
*/
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE);
adreno_dispatcher_schedule(KGSL_DEVICE(adreno_dev));
return;
}
adreno_dev->preempt.count++;
/*
* We can now safely clear the preemption keepalive bit, allowing
* power collapse to resume its regular activity.
*/
if (gmu_core_isenabled(KGSL_DEVICE(adreno_dev)))
gmu_core_regrmw(KGSL_DEVICE(adreno_dev),
A6XX_GMU_AO_SPARE_CNTL, 0x2, 0x0);
del_timer(&adreno_dev->preempt.timer);
adreno_readreg(adreno_dev, ADRENO_REG_CP_PREEMPT_LEVEL_STATUS, &status);
trace_adreno_preempt_done(adreno_dev->cur_rb, adreno_dev->next_rb,
status);
adreno_dev->prev_rb = adreno_dev->cur_rb;
adreno_dev->cur_rb = adreno_dev->next_rb;
adreno_dev->next_rb = NULL;
/* Update the wptr if it changed while preemption was ongoing */
_update_wptr(adreno_dev, true);
/* Update the dispatcher timer for the new command queue */
mod_timer(&adreno_dev->dispatcher.timer,
adreno_dev->cur_rb->dispatch_q.expires);
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
a6xx_preemption_trigger(adreno_dev);
}
void a6xx_preemption_schedule(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
if (!adreno_is_preemption_enabled(adreno_dev))
return;
mutex_lock(&device->mutex);
if (adreno_in_preempt_state(adreno_dev, ADRENO_PREEMPT_COMPLETE))
_a6xx_preemption_done(adreno_dev);
a6xx_preemption_trigger(adreno_dev);
mutex_unlock(&device->mutex);
}
unsigned int a6xx_preemption_pre_ibsubmit(
struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb,
unsigned int *cmds, struct kgsl_context *context)
{
unsigned int *cmds_orig = cmds;
uint64_t gpuaddr = 0;
if (context) {
gpuaddr = context->user_ctxt_record->memdesc.gpuaddr;
*cmds++ = cp_type7_packet(CP_SET_PSEUDO_REGISTER, 15);
} else {
*cmds++ = cp_type7_packet(CP_SET_PSEUDO_REGISTER, 12);
}
/* NULL SMMU_INFO buffer - we track in KMD */
*cmds++ = SET_PSEUDO_REGISTER_SAVE_REGISTER_SMMU_INFO;
cmds += cp_gpuaddr(adreno_dev, cmds, 0x0);
*cmds++ = SET_PSEUDO_REGISTER_SAVE_REGISTER_PRIV_NON_SECURE_SAVE_ADDR;
cmds += cp_gpuaddr(adreno_dev, cmds, rb->preemption_desc.gpuaddr);
*cmds++ = SET_PSEUDO_REGISTER_SAVE_REGISTER_PRIV_SECURE_SAVE_ADDR;
cmds += cp_gpuaddr(adreno_dev, cmds,
rb->secure_preemption_desc.gpuaddr);
if (context) {
*cmds++ = SET_PSEUDO_REGISTER_SAVE_REGISTER_NON_PRIV_SAVE_ADDR;
cmds += cp_gpuaddr(adreno_dev, cmds, gpuaddr);
}
/*
* There is no need to specify this address when we are about to
* trigger preemption. This is because CP internally stores this
* address specified here in the CP_SET_PSEUDO_REGISTER payload to
* the context record and thus knows from where to restore
* the saved perfcounters for the new ringbuffer.
*/
*cmds++ = SET_PSEUDO_REGISTER_SAVE_REGISTER_COUNTER;
cmds += cp_gpuaddr(adreno_dev, cmds,
rb->perfcounter_save_restore_desc.gpuaddr);
if (context) {
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
struct adreno_ringbuffer *rb = drawctxt->rb;
uint64_t dest =
SCRATCH_PREEMPTION_CTXT_RESTORE_GPU_ADDR(device,
rb->id);
*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2);
cmds += cp_gpuaddr(adreno_dev, cmds, dest);
*cmds++ = lower_32_bits(gpuaddr);
*cmds++ = upper_32_bits(gpuaddr);
}
return (unsigned int) (cmds - cmds_orig);
}
unsigned int a6xx_preemption_post_ibsubmit(struct adreno_device *adreno_dev,
unsigned int *cmds)
{
unsigned int *cmds_orig = cmds;
struct adreno_ringbuffer *rb = adreno_dev->cur_rb;
if (rb) {
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
uint64_t dest = SCRATCH_PREEMPTION_CTXT_RESTORE_GPU_ADDR(device,
rb->id);
*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 2);
cmds += cp_gpuaddr(adreno_dev, cmds, dest);
*cmds++ = 0;
*cmds++ = 0;
}
*cmds++ = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4);
cmds += cp_gpuaddr(adreno_dev, cmds, 0x0);
*cmds++ = 1;
*cmds++ = 0;
return (unsigned int) (cmds - cmds_orig);
}
void a6xx_preemption_start(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
struct adreno_ringbuffer *rb;
unsigned int i;
if (!adreno_is_preemption_enabled(adreno_dev))
return;
/* Force the state to be clear */
adreno_set_preempt_state(adreno_dev, ADRENO_PREEMPT_NONE);
/* smmu_info is allocated and mapped in a6xx_preemption_iommu_init */
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(magic), A6XX_CP_SMMU_INFO_MAGIC_REF);
kgsl_sharedmem_writeq(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(ttbr0), MMU_DEFAULT_TTBR0(device));
/* The CP doesn't use the asid record, so poison it */
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(asid), 0xDECAFBAD);
kgsl_sharedmem_writel(device, &iommu->smmu_info,
PREEMPT_SMMU_RECORD(context_idr),
MMU_DEFAULT_CONTEXTIDR(device));
adreno_writereg64(adreno_dev,
ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI,
iommu->smmu_info.gpuaddr);
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
/*
* preemption_desc is allocated and mapped at init time,
* so no need to check sharedmem_writel return value
*/
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(rptr), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(wptr), 0);
adreno_ringbuffer_set_pagetable(rb,
device->mmu.defaultpagetable);
}
}
static int a6xx_preemption_ringbuffer_init(struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret;
ret = kgsl_allocate_global(device, &rb->preemption_desc,
A6XX_CP_CTXRECORD_SIZE_IN_BYTES, 0, KGSL_MEMDESC_PRIVILEGED,
"preemption_desc");
if (ret)
return ret;
ret = kgsl_allocate_user(device, &rb->secure_preemption_desc,
A6XX_CP_CTXRECORD_SIZE_IN_BYTES,
KGSL_MEMFLAGS_SECURE | KGSL_MEMDESC_PRIVILEGED);
if (ret)
return ret;
ret = kgsl_iommu_map_global_secure_pt_entry(device,
&rb->secure_preemption_desc);
if (ret)
return ret;
ret = kgsl_allocate_global(device, &rb->perfcounter_save_restore_desc,
A6XX_CP_PERFCOUNTER_SAVE_RESTORE_SIZE, 0,
KGSL_MEMDESC_PRIVILEGED, "perfcounter_save_restore_desc");
if (ret)
return ret;
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(magic), A6XX_CP_CTXRECORD_MAGIC_REF);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(info), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(data), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(cntl), A6XX_CP_RB_CNTL_DEFAULT);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(rptr), 0);
kgsl_sharedmem_writel(device, &rb->preemption_desc,
PREEMPT_RECORD(wptr), 0);
kgsl_sharedmem_writeq(device, &rb->preemption_desc,
PREEMPT_RECORD(rptr_addr), SCRATCH_RPTR_GPU_ADDR(device,
rb->id));
kgsl_sharedmem_writeq(device, &rb->preemption_desc,
PREEMPT_RECORD(rbase), rb->buffer_desc.gpuaddr);
kgsl_sharedmem_writeq(device, &rb->preemption_desc,
PREEMPT_RECORD(counter), 0);
return 0;
}
#if IS_ENABLED(CONFIG_ARM_SMMU)
static int a6xx_preemption_iommu_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
/* Allocate mem for storing preemption smmu record */
return kgsl_allocate_global(device, &iommu->smmu_info, PAGE_SIZE,
KGSL_MEMFLAGS_GPUREADONLY, KGSL_MEMDESC_PRIVILEGED,
"smmu_info");
}
static void a6xx_preemption_iommu_close(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
kgsl_free_global(device, &iommu->smmu_info);
}
#else
static int a6xx_preemption_iommu_init(struct adreno_device *adreno_dev)
{
return -ENODEV;
}
static void a6xx_preemption_iommu_close(struct adreno_device *adreno_dev)
{
}
#endif
static void _preemption_close(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_preemption *preempt = &adreno_dev->preempt;
struct adreno_ringbuffer *rb;
unsigned int i;
del_timer(&preempt->timer);
a6xx_preemption_iommu_close(adreno_dev);
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
kgsl_free_global(device, &rb->preemption_desc);
kgsl_free_global(device, &rb->perfcounter_save_restore_desc);
kgsl_iommu_unmap_global_secure_pt_entry(device,
&rb->secure_preemption_desc);
kgsl_sharedmem_free(&rb->secure_preemption_desc);
}
}
void a6xx_preemption_close(struct adreno_device *adreno_dev)
{
if (!test_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv))
return;
_preemption_close(adreno_dev);
}
int a6xx_preemption_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_preemption *preempt = &adreno_dev->preempt;
struct adreno_ringbuffer *rb;
int ret;
unsigned int i;
/* We are dependent on IOMMU to make preemption go on the CP side */
if (kgsl_mmu_get_mmutype(device) != KGSL_MMU_TYPE_IOMMU)
return -ENODEV;
INIT_WORK(&preempt->work, _a6xx_preemption_worker);
timer_setup(&preempt->timer, _a6xx_preemption_timer, 0);
/* Allocate mem for storing preemption switch record */
FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
ret = a6xx_preemption_ringbuffer_init(adreno_dev, rb);
if (ret)
goto err;
}
ret = a6xx_preemption_iommu_init(adreno_dev);
err:
if (ret)
_preemption_close(adreno_dev);
return ret;
}
void a6xx_preemption_context_destroy(struct kgsl_context *context)
{
struct kgsl_device *device = context->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (!adreno_is_preemption_enabled(adreno_dev))
return;
gpumem_free_entry(context->user_ctxt_record);
/* Put the extra ref from gpumem_alloc_entry() */
kgsl_mem_entry_put(context->user_ctxt_record);
}
int a6xx_preemption_context_init(struct kgsl_context *context)
{
struct kgsl_device *device = context->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
uint64_t flags = 0;
if (!adreno_is_preemption_enabled(adreno_dev))
return 0;
if (context->flags & KGSL_CONTEXT_SECURE)
flags |= KGSL_MEMFLAGS_SECURE;
if (kgsl_is_compat_task())
flags |= KGSL_MEMFLAGS_FORCE_32BIT;
/*
* gpumem_alloc_entry takes an extra refcount. Put it only when
* destroying the context to keep the context record valid
*/
context->user_ctxt_record = gpumem_alloc_entry(context->dev_priv,
A6XX_CP_CTXRECORD_USER_RESTORE_SIZE, flags);
if (IS_ERR(context->user_ctxt_record)) {
int ret = PTR_ERR(context->user_ctxt_record);
context->user_ctxt_record = NULL;
return ret;
}
return 0;
}

578
drivers/gpu/msm/adreno_a6xx_rgmu.c Normal file
View file

@ -0,0 +1,578 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/firmware.h>
#include <linux/regulator/consumer.h>
#include "adreno.h"
#include "adreno_a6xx.h"
#include "adreno_snapshot.h"
#include "kgsl_rgmu.h"
#include "kgsl_trace.h"
/* RGMU timeouts */
#define RGMU_IDLE_TIMEOUT 100 /* ms */
#define RGMU_START_TIMEOUT 100 /* ms */
#define GPU_START_TIMEOUT 100 /* ms */
#define GLM_SLEEP_TIMEOUT 10 /* ms */
static const unsigned int a6xx_rgmu_registers[] = {
/* GMU CX */
0x1F80F, 0x1F83D, 0x1F840, 0x1F8D8, 0x1F990, 0x1F99E, 0x1F9C0, 0x1F9CC,
/* GMU AO */
0x23B03, 0x23B16, 0x23B80, 0x23B82,
/* GPU CC */
0x24000, 0x24012, 0x24040, 0x24052, 0x24400, 0x24404, 0x24407, 0x2440B,
0x24415, 0x2441C, 0x2441E, 0x2442D, 0x2443C, 0x2443D, 0x2443F, 0x24440,
0x24442, 0x24449, 0x24458, 0x2445A, 0x24540, 0x2455E, 0x24800, 0x24802,
0x24C00, 0x24C02, 0x25400, 0x25402, 0x25800, 0x25802, 0x25C00, 0x25C02,
0x26000, 0x26002,
};
irqreturn_t rgmu_irq_handler(int irq, void *data)
{
struct kgsl_device *device = data;
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int status = 0;
adreno_read_gmureg(adreno_dev,
ADRENO_REG_GMU_AO_HOST_INTERRUPT_STATUS, &status);
if (status & RGMU_AO_IRQ_FENCE_ERR) {
unsigned int fence_status;
adreno_read_gmureg(adreno_dev,
ADRENO_REG_GMU_AHB_FENCE_STATUS, &fence_status);
adreno_write_gmureg(adreno_dev,
ADRENO_REG_GMU_AO_HOST_INTERRUPT_CLR, status);
dev_err_ratelimited(&rgmu->pdev->dev,
"FENCE error interrupt received %x\n", fence_status);
}
if (status & ~RGMU_AO_IRQ_MASK)
dev_err_ratelimited(&rgmu->pdev->dev,
"Unhandled RGMU interrupts 0x%lx\n",
status & ~RGMU_AO_IRQ_MASK);
return IRQ_HANDLED;
}
irqreturn_t oob_irq_handler(int irq, void *data)
{
struct kgsl_device *device = data;
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int status = 0;
adreno_read_gmureg(adreno_dev,
ADRENO_REG_GMU_GMU2HOST_INTR_INFO, &status);
if (status & RGMU_OOB_IRQ_ERR_MSG) {
adreno_write_gmureg(adreno_dev,
ADRENO_REG_GMU_GMU2HOST_INTR_CLR, status);
dev_err_ratelimited(&rgmu->pdev->dev,
"RGMU oob irq error\n");
adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT);
adreno_dispatcher_schedule(device);
}
if (status & ~RGMU_OOB_IRQ_MASK)
dev_err_ratelimited(&rgmu->pdev->dev,
"Unhandled OOB interrupts 0x%lx\n",
status & ~RGMU_OOB_IRQ_MASK);
return IRQ_HANDLED;
}
/*
* a6xx_rgmu_oob_set() - Set OOB interrupt to RGMU
* @adreno_dev: Pointer to adreno device
* @req: Which of the OOB bits to request
*/
static int a6xx_rgmu_oob_set(struct kgsl_device *device,
enum oob_request req)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
int ret, set, check;
set = BIT(req + 16);
check = BIT(req + 16);
gmu_core_regwrite(device, A6XX_GMU_HOST2GMU_INTR_SET, set);
ret = timed_poll_check(device,
A6XX_GMU_GMU2HOST_INTR_INFO,
check,
GPU_START_TIMEOUT,
check);
if (ret) {
unsigned int status;
gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, &status);
dev_err(&rgmu->pdev->dev,
"Timed out while setting OOB req:%s status:0x%x\n",
gmu_core_oob_type_str(req), status);
return ret;
}
gmu_core_regwrite(device, A6XX_GMU_GMU2HOST_INTR_CLR, check);
trace_kgsl_gmu_oob_set(set);
return 0;
}
/*
* a6xx_rgmu_oob_clear() - Clear a previously set OOB request.
* @adreno_dev: Pointer to the adreno device that has the RGMU
* @req: Which of the OOB bits to clear
*/
static inline void a6xx_rgmu_oob_clear(struct kgsl_device *device,
enum oob_request req)
{
gmu_core_regwrite(device, A6XX_GMU_HOST2GMU_INTR_SET, BIT(req + 24));
trace_kgsl_gmu_oob_clear(BIT(req + 24));
}
static void a6xx_rgmu_bcl_config(struct kgsl_device *device, bool on)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
if (on) {
/* Enable BCL CRC HW i/f */
gmu_core_regwrite(device,
A6XX_GMU_AO_RGMU_GLM_HW_CRC_DISABLE, 0);
} else {
/* Disable CRC HW i/f */
gmu_core_regwrite(device,
A6XX_GMU_AO_RGMU_GLM_HW_CRC_DISABLE, 1);
/* Wait for HW CRC disable ACK */
if (timed_poll_check(device,
A6XX_GMU_AO_RGMU_GLM_SLEEP_STATUS,
BIT(1), GLM_SLEEP_TIMEOUT, BIT(1)))
dev_err_ratelimited(&rgmu->pdev->dev,
"Timed out waiting for HW CRC disable acknowledgment\n");
/* Pull down the valid RGMU_GLM_SLEEP_CTRL[7] to 0 */
gmu_core_regrmw(device, A6XX_GMU_AO_RGMU_GLM_SLEEP_CTRL,
BIT(7), 0);
}
}
static void a6xx_rgmu_irq_enable(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
/* Clear pending IRQs and Unmask needed IRQs */
adreno_gmu_clear_and_unmask_irqs(ADRENO_DEVICE(device));
/* Enable all IRQs on host */
enable_irq(rgmu->oob_interrupt_num);
enable_irq(rgmu->rgmu_interrupt_num);
}
static void a6xx_rgmu_irq_disable(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
/* Disable all IRQs on host */
disable_irq(rgmu->rgmu_interrupt_num);
disable_irq(rgmu->oob_interrupt_num);
/* Mask all IRQs and clear pending IRQs */
adreno_gmu_mask_and_clear_irqs(ADRENO_DEVICE(device));
}
static int a6xx_rgmu_ifpc_store(struct kgsl_device *device,
unsigned int val)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
unsigned int requested_idle_level;
if (!ADRENO_FEATURE(adreno_dev, ADRENO_IFPC))
return -EINVAL;
if (val)
requested_idle_level = GPU_HW_IFPC;
else
requested_idle_level = GPU_HW_ACTIVE;
if (requested_idle_level == rgmu->idle_level)
return 0;
mutex_lock(&device->mutex);
/* Power down the GPU before changing the idle level */
kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND);
rgmu->idle_level = requested_idle_level;
kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER);
mutex_unlock(&device->mutex);
return 0;
}
static unsigned int a6xx_rgmu_ifpc_show(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
return rgmu->idle_level == GPU_HW_IFPC;
}
static void a6xx_rgmu_prepare_stop(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
/* Turn off GX_MEM retention */
kgsl_regwrite(KGSL_DEVICE(adreno_dev),
A6XX_RBBM_BLOCK_GX_RETENTION_CNTL, 0);
}
#define GX_GDSC_POWER_OFF BIT(6)
/*
* a6xx_rgmu_gx_is_on() - Check if GX is on using pwr status register
* @adreno_dev - Pointer to adreno_device
* This check should only be performed if the keepalive bit is set or it
* can be guaranteed that the power state of the GPU will remain unchanged
*/
static bool a6xx_rgmu_gx_is_on(struct kgsl_device *device)
{
unsigned int val;
gmu_core_regread(device, A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &val);
return !(val & GX_GDSC_POWER_OFF);
}
static int a6xx_rgmu_wait_for_lowest_idle(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
unsigned int reg[10] = {0};
unsigned long t;
uint64_t ts1, ts2, ts3;
if (rgmu->idle_level != GPU_HW_IFPC)
return 0;
ts1 = read_AO_counter(device);
t = jiffies + msecs_to_jiffies(RGMU_IDLE_TIMEOUT);
do {
gmu_core_regread(device,
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &reg[0]);
if (reg[0] & GX_GDSC_POWER_OFF)
return 0;
/* Wait 10us to reduce unnecessary AHB bus traffic */
usleep_range(10, 100);
} while (!time_after(jiffies, t));
ts2 = read_AO_counter(device);
/* Do one last read incase it succeeds */
gmu_core_regread(device,
A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, &reg[0]);
if (reg[0] & GX_GDSC_POWER_OFF)
return 0;
ts3 = read_AO_counter(device);
/* Collect abort data to help with debugging */
gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, &reg[1]);
gmu_core_regread(device, A6XX_RGMU_CX_PCC_STATUS, &reg[2]);
gmu_core_regread(device, A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, &reg[3]);
kgsl_regread(device, A6XX_CP_STATUS_1, &reg[4]);
gmu_core_regread(device, A6XX_GMU_RBBM_INT_UNMASKED_STATUS, &reg[5]);
gmu_core_regread(device, A6XX_GMU_GMU_PWR_COL_KEEPALIVE, &reg[6]);
kgsl_regread(device, A6XX_CP_CP2GMU_STATUS, &reg[7]);
kgsl_regread(device, A6XX_CP_CONTEXT_SWITCH_CNTL, &reg[8]);
gmu_core_regread(device, A6XX_GMU_AO_SPARE_CNTL, &reg[9]);
dev_err(&rgmu->pdev->dev,
"----------------------[ RGMU error ]----------------------\n");
dev_err(&rgmu->pdev->dev, "Timeout waiting for lowest idle level\n");
dev_err(&rgmu->pdev->dev,
"Timestamps: %llx %llx %llx\n", ts1, ts2, ts3);
dev_err(&rgmu->pdev->dev,
"SPTPRAC_PWR_CLK_STATUS=%x PCC_DEBUG=%x PCC_STATUS=%x\n",
reg[0], reg[1], reg[2]);
dev_err(&rgmu->pdev->dev,
"CX_BUSY_STATUS=%x CP_STATUS_1=%x\n", reg[3], reg[4]);
dev_err(&rgmu->pdev->dev,
"RBBM_INT_UNMASKED_STATUS=%x PWR_COL_KEEPALIVE=%x\n",
reg[5], reg[6]);
dev_err(&rgmu->pdev->dev,
"CP2GMU_STATUS=%x CONTEXT_SWITCH_CNTL=%x AO_SPARE_CNTL=%x\n",
reg[7], reg[8], reg[9]);
WARN_ON(1);
return -ETIMEDOUT;
}
/*
* The lowest 16 bits of this value are the number of XO clock cycles
* for main hysteresis. This is the first hysteresis. Here we set it
* to 0x1680 cycles, or 300 us. The highest 16 bits of this value are
* the number of XO clock cycles for short hysteresis. This happens
* after main hysteresis. Here we set it to 0xA cycles, or 0.5 us.
*/
#define RGMU_PWR_COL_HYST 0x000A1680
/* HOSTTOGMU and TIMER0/1 interrupt mask: 0x20060 */
#define RGMU_INTR_EN_MASK (BIT(5) | BIT(6) | BIT(17))
/* RGMU FENCE RANGE MASK */
#define RGMU_FENCE_RANGE_MASK ((0x1 << 31) | ((0xA << 2) << 18) | (0x8A0))
/*
* a6xx_rgmu_fw_start() - set up GMU and start FW
* @device: Pointer to KGSL device
* @boot_state: State of the rgmu being started
*/
static int a6xx_rgmu_fw_start(struct kgsl_device *device,
unsigned int boot_state)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
unsigned int status;
int i;
switch (boot_state) {
case GMU_COLD_BOOT:
case GMU_WARM_BOOT:
/* Turn on TCM retention */
gmu_core_regwrite(device, A6XX_GMU_GENERAL_7, 1);
/* Load RGMU FW image via AHB bus */
for (i = 0; i < rgmu->fw_size; i++)
gmu_core_regwrite(device, A6XX_GMU_CM3_ITCM_START + i,
rgmu->fw_hostptr[i]);
/*
* Enable power counter because it was disabled before
* slumber.
*/
gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE,
1);
break;
}
/* IFPC Feature Enable */
if (rgmu->idle_level == GPU_HW_IFPC) {
gmu_core_regwrite(device, A6XX_GMU_PWR_COL_INTER_FRAME_HYST,
RGMU_PWR_COL_HYST);
gmu_core_regwrite(device, A6XX_GMU_PWR_COL_INTER_FRAME_CTRL,
BIT(0));
}
/* For RGMU CX interrupt */
gmu_core_regwrite(device, A6XX_RGMU_CX_INTR_GEN_EN, RGMU_INTR_EN_MASK);
/* Enable GMU AO to host interrupt */
gmu_core_regwrite(device, A6XX_GMU_AO_INTERRUPT_EN, RGMU_AO_IRQ_MASK);
/* For OOB */
gmu_core_regwrite(device, A6XX_GMU_HOST2GMU_INTR_EN_2, 0x00FF0000);
gmu_core_regwrite(device, A6XX_GMU_HOST2GMU_INTR_EN_3, 0xFF000000);
/* Fence Address range configuration */
gmu_core_regwrite(device, A6XX_GMU_AHB_FENCE_RANGE_0,
RGMU_FENCE_RANGE_MASK);
/* During IFPC RGMU will put fence in drop mode so we would
* need to put fence allow mode during slumber out sequence.
*/
gmu_core_regwrite(device, A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
/* BCL ON Sequence */
a6xx_rgmu_bcl_config(device, true);
/* Write 0 first to make sure that rgmu is reset */
gmu_core_regwrite(device, A6XX_RGMU_CX_PCC_CTRL, 0);
/* Make sure putting in reset doesn't happen after writing 1 */
wmb();
/* Bring rgmu out of reset */
gmu_core_regwrite(device, A6XX_RGMU_CX_PCC_CTRL, 1);
if (timed_poll_check(device, A6XX_RGMU_CX_PCC_INIT_RESULT,
BIT(0), RGMU_START_TIMEOUT, BIT(0))) {
gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, &status);
dev_err(&rgmu->pdev->dev,
"rgmu boot Failed. status:%08x\n", status);
return -ETIMEDOUT;
}
/* Read the RGMU firmware version from registers */
gmu_core_regread(device, A6XX_GMU_GENERAL_0, &rgmu->ver);
return 0;
}
static int a6xx_rgmu_suspend(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
int ret = 0;
/* Check GX GDSC is status */
if (a6xx_rgmu_gx_is_on(device)) {
/* Switch gx gdsc control from RGMU to CPU
* force non-zero reference count in clk driver
* so next disable call will turn
* off the GDSC
*/
ret = regulator_enable(rgmu->gx_gdsc);
if (ret)
dev_err(&rgmu->pdev->dev,
"Fail to enable gx gdsc, error:%d\n", ret);
ret = regulator_disable(rgmu->gx_gdsc);
if (ret)
dev_err(&rgmu->pdev->dev,
"Fail to disable gx gdsc, error:%d\n", ret);
if (a6xx_rgmu_gx_is_on(device))
dev_err(&rgmu->pdev->dev, "gx is stuck on\n");
}
return ret;
}
/*
* a6xx_rgmu_gpu_pwrctrl() - GPU power control via rgmu interface
* @adreno_dev: Pointer to adreno device
* @mode: requested power mode
* @arg1: first argument for mode control
* @arg2: second argument for mode control
*/
static int a6xx_rgmu_gpu_pwrctrl(struct kgsl_device *device,
unsigned int mode, unsigned int arg1, unsigned int arg2)
{
int ret = 0;
switch (mode) {
case GMU_FW_START:
ret = a6xx_rgmu_fw_start(device, arg1);
break;
case GMU_SUSPEND:
ret = a6xx_rgmu_suspend(device);
break;
case GMU_NOTIFY_SLUMBER:
/* Disable the power counter so that the RGMU is not busy */
gmu_core_regwrite(device, A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE,
0);
/* BCL OFF Sequence */
a6xx_rgmu_bcl_config(device, false);
break;
default:
ret = -EINVAL;
break;
}
return ret;
}
/*
* a6xx_rgmu_load_firmware() - Load the ucode into the RGMU TCM
* @device: Pointer to KGSL device
*/
static int a6xx_rgmu_load_firmware(struct kgsl_device *device)
{
const struct firmware *fw = NULL;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev);
int ret;
/* RGMU fw already saved and verified so do nothing new */
if (rgmu->fw_hostptr)
return 0;
ret = request_firmware(&fw, a6xx_core->gmufw_name, device->dev);
if (ret < 0) {
pr_err("request_firmware (%s) failed: %d\n",
a6xx_core->gmufw_name, ret);
return ret;
}
rgmu->fw_hostptr = devm_kmemdup(&rgmu->pdev->dev, fw->data,
fw->size, GFP_KERNEL);
if (rgmu->fw_hostptr)
rgmu->fw_size = (fw->size / sizeof(u32));
release_firmware(fw);
return rgmu->fw_hostptr ? 0 : -ENOMEM;
}
/* Halt RGMU execution */
static void a6xx_rgmu_halt_execution(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
unsigned int index, status, fence;
gmu_core_regread(device, A6XX_RGMU_CX_PCC_DEBUG, &index);
gmu_core_regread(device, A6XX_RGMU_CX_PCC_STATUS, &status);
gmu_core_regread(device, A6XX_GMU_AO_AHB_FENCE_CTRL, &fence);
dev_err(&rgmu->pdev->dev,
"RGMU Fault PCC_DEBUG:0x%x PCC_STATUS:0x%x FENCE_CTRL:0x%x\n",
index, status, fence);
/*
* Write 0 to halt RGMU execution. We halt it in GMU/GPU fault and
* re start PCC execution in recovery path.
*/
gmu_core_regwrite(device, A6XX_RGMU_CX_PCC_CTRL, 0);
/*
* Ensure that fence is in allow mode after halting RGMU.
* After halting RGMU we dump snapshot.
*/
gmu_core_regwrite(device, A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
}
/*
* a6xx_rgmu_snapshot() - A6XX GMU snapshot function
* @adreno_dev: Device being snapshotted
* @snapshot: Pointer to the snapshot instance
*
* This is where all of the A6XX GMU specific bits and pieces are grabbed
* into the snapshot memory
*/
static void a6xx_rgmu_snapshot(struct kgsl_device *device,
struct kgsl_snapshot *snapshot)
{
adreno_snapshot_registers(device, snapshot, a6xx_rgmu_registers,
ARRAY_SIZE(a6xx_rgmu_registers) / 2);
}
struct gmu_dev_ops adreno_a6xx_rgmudev = {
.load_firmware = a6xx_rgmu_load_firmware,
.oob_set = a6xx_rgmu_oob_set,
.oob_clear = a6xx_rgmu_oob_clear,
.irq_enable = a6xx_rgmu_irq_enable,
.irq_disable = a6xx_rgmu_irq_disable,
.rpmh_gpu_pwrctrl = a6xx_rgmu_gpu_pwrctrl,
.gx_is_on = a6xx_rgmu_gx_is_on,
.prepare_stop = a6xx_rgmu_prepare_stop,
.wait_for_lowest_idle = a6xx_rgmu_wait_for_lowest_idle,
.ifpc_store = a6xx_rgmu_ifpc_store,
.ifpc_show = a6xx_rgmu_ifpc_show,
.snapshot = a6xx_rgmu_snapshot,
.halt_execution = a6xx_rgmu_halt_execution,
.gmu2host_intr_mask = RGMU_OOB_IRQ_MASK,
.gmu_ao_intr_mask = RGMU_AO_IRQ_MASK,
};

2217
drivers/gpu/msm/adreno_a6xx_snapshot.c Normal file
View file

File diff suppressed because it is too large Load diff

191
drivers/gpu/msm/adreno_compat.c Normal file
View file

@ -0,0 +1,191 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
*/
#include "adreno.h"
#include "adreno_compat.h"
#include "kgsl_compat.h"
int adreno_getproperty_compat(struct kgsl_device *device,
struct kgsl_device_getproperty *param)
{
int status = -EINVAL;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
switch (param->type) {
case KGSL_PROP_DEVICE_INFO:
{
struct kgsl_devinfo_compat devinfo;
if (param->sizebytes != sizeof(devinfo)) {
status = -EINVAL;
break;
}
memset(&devinfo, 0, sizeof(devinfo));
devinfo.device_id = device->id + 1;
devinfo.chip_id = adreno_dev->chipid;
devinfo.mmu_enabled =
MMU_FEATURE(&device->mmu, KGSL_MMU_PAGED);
devinfo.gmem_gpubaseaddr =
adreno_dev->gpucore->gmem_base;
devinfo.gmem_sizebytes =
adreno_dev->gpucore->gmem_size;
if (copy_to_user(param->value, &devinfo,
sizeof(devinfo))) {
status = -EFAULT;
break;
}
status = 0;
}
break;
case KGSL_PROP_DEVICE_SHADOW:
{
struct kgsl_shadowprop_compat shadowprop;
if (param->sizebytes != sizeof(shadowprop)) {
status = -EINVAL;
break;
}
memset(&shadowprop, 0, sizeof(shadowprop));
if (device->memstore.hostptr) {
/*
* NOTE: with mmu enabled, gpuaddr doesn't mean
* anything to mmap().
* NOTE: shadowprop.gpuaddr is uint32
* (because legacy) and the memstore gpuaddr is
* 64 bit. Cast the memstore gpuaddr to uint32.
*/
shadowprop.gpuaddr =
(unsigned int) device->memstore.gpuaddr;
shadowprop.size =
(unsigned int) device->memstore.size;
/*
* GSL needs this to be set, even if it
* appears to be meaningless
*/
shadowprop.flags = KGSL_FLAGS_INITIALIZED |
KGSL_FLAGS_PER_CONTEXT_TIMESTAMPS;
}
if (copy_to_user(param->value, &shadowprop,
sizeof(shadowprop))) {
status = -EFAULT;
break;
}
status = 0;
}
break;
default:
status = device->ftbl->getproperty(device, param);
}
return status;
}
int adreno_setproperty_compat(struct kgsl_device_private *dev_priv,
unsigned int type,
void __user *value,
unsigned int sizebytes)
{
int status = -EINVAL;
struct kgsl_device *device = dev_priv->device;
switch (type) {
case KGSL_PROP_PWR_CONSTRAINT:
case KGSL_PROP_L3_PWR_CONSTRAINT: {
struct kgsl_device_constraint_compat constraint32;
struct kgsl_device_constraint constraint;
struct kgsl_context *context;
if (sizebytes != sizeof(constraint32))
break;
if (copy_from_user(&constraint32, value,
sizeof(constraint32))) {
status = -EFAULT;
break;
}
/* Populate the real constraint type from the compat */
constraint.type = constraint32.type;
constraint.context_id = constraint32.context_id;
constraint.data = compat_ptr(constraint32.data);
constraint.size = (size_t)constraint32.size;
context = kgsl_context_get_owner(dev_priv,
constraint.context_id);
if (context == NULL)
break;
status = adreno_set_constraint(device, context,
&constraint);
kgsl_context_put(context);
}
break;
default:
/*
* Call adreno_setproperty in case the property type was
* KGSL_PROP_PWRCTRL
*/
status = device->ftbl->setproperty(dev_priv, type, value,
sizebytes);
}
return status;
}
static long adreno_ioctl_perfcounter_query_compat(
struct kgsl_device_private *dev_priv, unsigned int cmd,
void *data)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_priv->device);
struct kgsl_perfcounter_query_compat *query32 = data;
struct kgsl_perfcounter_query query;
long result;
query.groupid = query32->groupid;
query.countables = compat_ptr(query32->countables);
query.count = query32->count;
query.max_counters = query32->max_counters;
result = adreno_perfcounter_query_group(adreno_dev,
query.groupid, query.countables,
query.count, &query.max_counters);
query32->max_counters = query.max_counters;
return result;
}
static long adreno_ioctl_perfcounter_read_compat(
struct kgsl_device_private *dev_priv, unsigned int cmd,
void *data)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_priv->device);
struct kgsl_perfcounter_read_compat *read32 = data;
struct kgsl_perfcounter_read read;
read.reads = (struct kgsl_perfcounter_read_group __user *)
(uintptr_t)read32->reads;
read.count = read32->count;
return adreno_perfcounter_read_group(adreno_dev, read.reads,
read.count);
}
static struct kgsl_ioctl adreno_compat_ioctl_funcs[] = {
{ IOCTL_KGSL_PERFCOUNTER_GET, adreno_ioctl_perfcounter_get },
{ IOCTL_KGSL_PERFCOUNTER_PUT, adreno_ioctl_perfcounter_put },
{ IOCTL_KGSL_PERFCOUNTER_QUERY_COMPAT,
adreno_ioctl_perfcounter_query_compat },
{ IOCTL_KGSL_PERFCOUNTER_READ_COMPAT,
adreno_ioctl_perfcounter_read_compat },
};
long adreno_compat_ioctl(struct kgsl_device_private *dev_priv,
unsigned int cmd, unsigned long arg)
{
return adreno_ioctl_helper(dev_priv, cmd, arg,
adreno_compat_ioctl_funcs,
ARRAY_SIZE(adreno_compat_ioctl_funcs));
}

46
drivers/gpu/msm/adreno_compat.h Normal file
View file

@ -0,0 +1,46 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2013-2015, 2017, 2019 The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_COMPAT_H
#define __ADRENO_COMPAT_H
#ifdef CONFIG_COMPAT
struct kgsl_device;
struct kgsl_device_private;
int adreno_getproperty_compat(struct kgsl_device *device,
struct kgsl_device_getproperty *param);
int adreno_setproperty_compat(struct kgsl_device_private *dev_priv,
unsigned int type,
void __user *value,
unsigned int sizebytes);
long adreno_compat_ioctl(struct kgsl_device_private *dev_priv,
unsigned int cmd, unsigned long arg);
#else
static inline int adreno_getproperty_compat(struct kgsl_device *device,
struct kgsl_device_getproperty *param)
{
return -EINVAL;
}
static inline int adreno_setproperty_compat(struct kgsl_device_private
*dev_priv, unsigned int type,
void __user *value, unsigned int sizebytes)
{
return -EINVAL;
}
static inline long adreno_compat_ioctl(struct kgsl_device_private *dev_priv,
unsigned int cmd, unsigned long arg)
{
return -EINVAL;
}
#endif /* CONFIG_COMPAT */
#endif /* __ADRENO_COMPAT_H */

460
drivers/gpu/msm/adreno_coresight.c Normal file
View file

@ -0,0 +1,460 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/coresight.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include "adreno.h"
#define TO_ADRENO_CORESIGHT_ATTR(_attr) \
container_of(_attr, struct adreno_coresight_attr, attr)
static int adreno_coresight_identify(const char *name)
{
if (!strcmp(name, "coresight-gfx"))
return GPU_CORESIGHT_GX;
else if (!strcmp(name, "coresight-gfx-cx"))
return GPU_CORESIGHT_CX;
else
return -EINVAL;
}
ssize_t adreno_coresight_show_register(struct device *dev,
struct device_attribute *attr, char *buf)
{
unsigned int val = 0;
struct kgsl_device *device = dev_get_drvdata(dev->parent);
struct adreno_device *adreno_dev;
struct adreno_coresight_attr *cattr = TO_ADRENO_CORESIGHT_ATTR(attr);
bool is_cx;
if (device == NULL)
return -EINVAL;
adreno_dev = ADRENO_DEVICE(device);
if (cattr->reg == NULL)
return -EINVAL;
is_cx = adreno_is_cx_dbgc_register(device, cattr->reg->offset);
/*
* Return the current value of the register if coresight is enabled,
* otherwise report 0
*/
mutex_lock(&device->mutex);
if ((is_cx && test_bit(ADRENO_DEVICE_CORESIGHT_CX, &adreno_dev->priv))
|| (!is_cx && test_bit(ADRENO_DEVICE_CORESIGHT,
&adreno_dev->priv))) {
/*
* If the device isn't power collapsed read the actual value
* from the hardware - otherwise return the cached value
*/
if (device->state == KGSL_STATE_ACTIVE ||
device->state == KGSL_STATE_NAP) {
if (!kgsl_active_count_get(device)) {
if (!is_cx)
kgsl_regread(device, cattr->reg->offset,
&cattr->reg->value);
else
adreno_cx_dbgc_regread(device,
cattr->reg->offset,
&cattr->reg->value);
kgsl_active_count_put(device);
}
}
val = cattr->reg->value;
}
mutex_unlock(&device->mutex);
return scnprintf(buf, PAGE_SIZE, "0x%X\n", val);
}
ssize_t adreno_coresight_store_register(struct device *dev,
struct device_attribute *attr, const char *buf, size_t size)
{
struct kgsl_device *device = dev_get_drvdata(dev->parent);
struct adreno_device *adreno_dev;
struct adreno_coresight_attr *cattr = TO_ADRENO_CORESIGHT_ATTR(attr);
unsigned long val;
int ret, is_cx;
if (device == NULL)
return -EINVAL;
adreno_dev = ADRENO_DEVICE(device);
if (cattr->reg == NULL)
return -EINVAL;
is_cx = adreno_is_cx_dbgc_register(device, cattr->reg->offset);
ret = kstrtoul(buf, 0, &val);
if (ret)
return ret;
mutex_lock(&device->mutex);
/* Ignore writes while coresight is off */
if (!((is_cx && test_bit(ADRENO_DEVICE_CORESIGHT_CX, &adreno_dev->priv))
|| (!is_cx && test_bit(ADRENO_DEVICE_CORESIGHT,
&adreno_dev->priv))))
goto out;
cattr->reg->value = val;
/* Program the hardware if it is not power collapsed */
if (device->state == KGSL_STATE_ACTIVE ||
device->state == KGSL_STATE_NAP) {
if (!kgsl_active_count_get(device)) {
if (!is_cx)
kgsl_regwrite(device, cattr->reg->offset,
cattr->reg->value);
else
adreno_cx_dbgc_regwrite(device,
cattr->reg->offset,
cattr->reg->value);
kgsl_active_count_put(device);
}
}
out:
mutex_unlock(&device->mutex);
return size;
}
/**
* adreno_coresight_disable() - Generic function to disable coresight debugging
* @csdev: Pointer to coresight's device struct
*
* This is a generic function to disable coresight debug bus on adreno
* devices. This should be used in all cases of disabling
* coresight debug bus for adreno devices. This function in turn calls
* the adreno device specific function through the gpudev hook.
* This function is registered as the coresight disable function
* with coresight driver. It should only be called through coresight driver
* as that would ensure that the necessary setup required to be done on
* coresight driver's part is also done.
*/
static void adreno_coresight_disable(struct coresight_device *csdev,
struct perf_event *event)
{
struct kgsl_device *device = dev_get_drvdata(csdev->dev.parent);
struct adreno_device *adreno_dev;
struct adreno_gpudev *gpudev;
struct adreno_coresight *coresight;
int i, cs_id;
if (device == NULL)
return;
adreno_dev = ADRENO_DEVICE(device);
gpudev = ADRENO_GPU_DEVICE(adreno_dev);
cs_id = adreno_coresight_identify(dev_name(&csdev->dev));
if (cs_id < 0)
return;
coresight = gpudev->coresight[cs_id];
if (coresight == NULL)
return;
mutex_lock(&device->mutex);
if (!kgsl_active_count_get(device)) {
if (cs_id == GPU_CORESIGHT_GX)
for (i = 0; i < coresight->count; i++)
kgsl_regwrite(device,
coresight->registers[i].offset, 0);
else if (cs_id == GPU_CORESIGHT_CX)
for (i = 0; i < coresight->count; i++)
adreno_cx_dbgc_regwrite(device,
coresight->registers[i].offset, 0);
kgsl_active_count_put(device);
}
if (cs_id == GPU_CORESIGHT_GX)
clear_bit(ADRENO_DEVICE_CORESIGHT, &adreno_dev->priv);
else if (cs_id == GPU_CORESIGHT_CX)
clear_bit(ADRENO_DEVICE_CORESIGHT_CX, &adreno_dev->priv);
mutex_unlock(&device->mutex);
}
/**
* _adreno_coresight_get_and_clear(): Save the current value of coresight
* registers and clear the registers subsequently. Clearing registers
* has the effect of disabling coresight.
* @adreno_dev: Pointer to adreno device struct
*/
static int _adreno_coresight_get_and_clear(struct adreno_device *adreno_dev,
int cs_id)
{
int i;
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_coresight *coresight = gpudev->coresight[cs_id];
if (coresight == NULL)
return -ENODEV;
kgsl_pre_hwaccess(device);
/*
* Save the current value of each coresight register
* and then clear each register
*/
if (cs_id == GPU_CORESIGHT_GX) {
for (i = 0; i < coresight->count; i++) {
kgsl_regread(device, coresight->registers[i].offset,
&coresight->registers[i].value);
kgsl_regwrite(device, coresight->registers[i].offset,
0);
}
} else if (cs_id == GPU_CORESIGHT_CX) {
for (i = 0; i < coresight->count; i++) {
adreno_cx_dbgc_regread(device,
coresight->registers[i].offset,
&coresight->registers[i].value);
adreno_cx_dbgc_regwrite(device,
coresight->registers[i].offset, 0);
}
}
return 0;
}
static int _adreno_coresight_set(struct adreno_device *adreno_dev, int cs_id)
{
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_coresight *coresight = gpudev->coresight[cs_id];
int i;
if (coresight == NULL)
return -ENODEV;
if (cs_id == GPU_CORESIGHT_GX) {
for (i = 0; i < coresight->count; i++)
kgsl_regwrite(device, coresight->registers[i].offset,
coresight->registers[i].value);
} else if (cs_id == GPU_CORESIGHT_CX) {
for (i = 0; i < coresight->count; i++)
adreno_cx_dbgc_regwrite(device,
coresight->registers[i].offset,
coresight->registers[i].value);
}
return 0;
}
/**
* adreno_coresight_enable() - Generic function to enable coresight debugging
* @csdev: Pointer to coresight's device struct
*
* This is a generic function to enable coresight debug bus on adreno
* devices. This should be used in all cases of enabling
* coresight debug bus for adreno devices. This function is registered as the
* coresight enable function with coresight driver. It should only be called
* through coresight driver as that would ensure that the necessary setup
* required to be done on coresight driver's part is also done.
*/
static int adreno_coresight_enable(struct coresight_device *csdev,
struct perf_event *event, u32 mode)
{
struct kgsl_device *device = dev_get_drvdata(csdev->dev.parent);
struct adreno_device *adreno_dev;
struct adreno_gpudev *gpudev;
struct adreno_coresight *coresight;
int ret = 0, adreno_dev_flag = -EINVAL, cs_id;
if (device == NULL)
return -ENODEV;
adreno_dev = ADRENO_DEVICE(device);
gpudev = ADRENO_GPU_DEVICE(adreno_dev);
cs_id = adreno_coresight_identify(dev_name(&csdev->dev));
if (cs_id < 0)
return -ENODEV;
coresight = gpudev->coresight[cs_id];
if (coresight == NULL)
return -ENODEV;
if (cs_id == GPU_CORESIGHT_GX)
adreno_dev_flag = ADRENO_DEVICE_CORESIGHT;
else if (cs_id == GPU_CORESIGHT_CX)
adreno_dev_flag = ADRENO_DEVICE_CORESIGHT_CX;
else
return -ENODEV;
mutex_lock(&device->mutex);
if (!test_and_set_bit(adreno_dev_flag, &adreno_dev->priv)) {
int i;
/* Reset all the debug registers to their default values */
for (i = 0; i < coresight->count; i++)
coresight->registers[i].value =
coresight->registers[i].initial;
if (kgsl_state_is_awake(device)) {
ret = kgsl_active_count_get(device);
if (!ret) {
ret = _adreno_coresight_set(adreno_dev, cs_id);
kgsl_active_count_put(device);
}
}
}
mutex_unlock(&device->mutex);
return ret;
}
/**
* adreno_coresight_stop() - Reprogram coresight registers after power collapse
* @adreno_dev: Pointer to the adreno device structure
*
* Cache the current coresight register values so they can be restored after
* power collapse
*/
void adreno_coresight_stop(struct adreno_device *adreno_dev)
{
int i, adreno_dev_flag = -EINVAL;
for (i = 0; i < GPU_CORESIGHT_MAX; ++i) {
if (i == GPU_CORESIGHT_GX)
adreno_dev_flag = ADRENO_DEVICE_CORESIGHT;
else if (i == GPU_CORESIGHT_CX)
adreno_dev_flag = ADRENO_DEVICE_CORESIGHT_CX;
else
return;
if (test_bit(adreno_dev_flag, &adreno_dev->priv))
_adreno_coresight_get_and_clear(adreno_dev, i);
}
}
/**
* adreno_coresight_start() - Reprogram coresight registers after power collapse
* @adreno_dev: Pointer to the adreno device structure
*
* Reprogram the cached values to the coresight registers on power up
*/
void adreno_coresight_start(struct adreno_device *adreno_dev)
{
int i, adreno_dev_flag = -EINVAL;
for (i = 0; i < GPU_CORESIGHT_MAX; ++i) {
if (i == GPU_CORESIGHT_GX)
adreno_dev_flag = ADRENO_DEVICE_CORESIGHT;
else if (i == GPU_CORESIGHT_CX)
adreno_dev_flag = ADRENO_DEVICE_CORESIGHT_CX;
else
return;
if (test_bit(adreno_dev_flag, &adreno_dev->priv))
_adreno_coresight_set(adreno_dev, i);
}
}
static int adreno_coresight_trace_id(struct coresight_device *csdev)
{
struct kgsl_device *device = dev_get_drvdata(csdev->dev.parent);
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(ADRENO_DEVICE(device));
int cs_id;
cs_id = adreno_coresight_identify(dev_name(&csdev->dev));
if (cs_id < 0)
return -ENODEV;
return gpudev->coresight[cs_id]->atid;
}
static const struct coresight_ops_source adreno_coresight_source_ops = {
.trace_id = adreno_coresight_trace_id,
.enable = adreno_coresight_enable,
.disable = adreno_coresight_disable,
};
static const struct coresight_ops adreno_coresight_ops = {
.source_ops = &adreno_coresight_source_ops,
};
void adreno_coresight_remove(struct adreno_device *adreno_dev)
{
int i;
for (i = 0; i < ARRAY_SIZE(adreno_dev->csdev); i++) {
if (!IS_ERR_OR_NULL(adreno_dev->csdev[i])) {
coresight_unregister(adreno_dev->csdev[i]);
adreno_dev->csdev[i] = NULL;
}
}
}
static struct coresight_device *
adreno_coresight_dev_probe(struct kgsl_device *device,
struct adreno_coresight *coresight, struct device_node *node)
{
struct platform_device *pdev = of_find_device_by_node(node);
struct coresight_desc desc;
u32 atid;
if (!pdev)
return ERR_PTR(-ENODEV);
if (of_property_read_u32(node, "coresight-atid", &atid))
return ERR_PTR(-ENODEV);
desc.pdata = coresight_get_platform_data(&pdev->dev);
platform_device_put(pdev);
if (IS_ERR(desc.pdata))
return ERR_CAST(desc.pdata);
desc.type = CORESIGHT_DEV_TYPE_SOURCE;
desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE;
desc.ops = &adreno_coresight_ops;
desc.dev = &device->pdev->dev;
desc.groups = coresight->groups;
coresight->atid = atid;
return coresight_register(&desc);
}
void adreno_coresight_init(struct adreno_device *adreno_dev)
{
int ret = 0;
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int i = 0;
struct device_node *node, *child;
node = of_find_compatible_node(device->pdev->dev.of_node,
NULL, "qcom,gpu-coresight");
if (!node)
return;
for_each_child_of_node(node, child) {
adreno_dev->csdev[i] = adreno_coresight_dev_probe(device,
gpudev->coresight[i], child);
i++;
}
of_node_put(node);
return ret;
}

1047
drivers/gpu/msm/adreno_cp_parser.c Normal file
View file

File diff suppressed because it is too large Load diff

175
drivers/gpu/msm/adreno_cp_parser.h Normal file
View file

@ -0,0 +1,175 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2013-2014, 2017, 2019, The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_IB_PARSER__
#define __ADRENO_IB_PARSER__
#include "adreno.h"
extern const unsigned int a3xx_cp_addr_regs[];
extern const unsigned int a4xx_cp_addr_regs[];
/*
* struct adreno_ib_object - Structure containing information about an
* address range found in an IB
* @gpuaddr: The starting gpuaddress of the range
* @size: Size of the range
* @snapshot_obj_type - Type of range used in snapshot
* @entry: The memory entry in which this range is found
*/
struct adreno_ib_object {
uint64_t gpuaddr;
uint64_t size;
int snapshot_obj_type;
struct kgsl_mem_entry *entry;
};
/*
* struct adreno_ib_object_list - List of address ranges found in IB
* @obj_list: The address range list
* @num_objs: Number of objects in list
*/
struct adreno_ib_object_list {
struct adreno_ib_object *obj_list;
int num_objs;
};
/*
* adreno registers used during IB parsing, there contain addresses
* and sizes of the addresses that present in an IB
*/
enum adreno_cp_addr_regs {
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_0 = 0,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_0,
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_1,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_1,
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_2,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_2,
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_3,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_3,
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_4,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_4,
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_5,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_5,
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_6,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_6,
ADRENO_CP_ADDR_VSC_PIPE_DATA_ADDRESS_7,
ADRENO_CP_ADDR_VSC_PIPE_DATA_LENGTH_7,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_0,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_1,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_2,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_3,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_4,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_5,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_6,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_7,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_8,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_9,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_10,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_11,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_12,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_13,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_14,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_15,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_16,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_17,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_18,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_19,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_20,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_21,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_22,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_23,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_24,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_25,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_26,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_27,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_28,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_29,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_30,
ADRENO_CP_ADDR_VFD_FETCH_INSTR_1_31,
ADRENO_CP_ADDR_VSC_SIZE_ADDRESS,
ADRENO_CP_ADDR_SP_VS_PVT_MEM_ADDR,
ADRENO_CP_ADDR_SP_FS_PVT_MEM_ADDR,
ADRENO_CP_ADDR_SP_VS_OBJ_START_REG,
ADRENO_CP_ADDR_SP_FS_OBJ_START_REG,
ADRENO_CP_UCHE_INVALIDATE0,
ADRENO_CP_UCHE_INVALIDATE1,
ADRENO_CP_ADDR_MAX,
};
/*
* adreno_ib_init_ib_obj() - Create an ib object structure and initialize it
* with gpuaddress and size
* @gpuaddr: gpuaddr with which to initialize the object with
* @size: Size in bytes with which the object is initialized
* @ib_type: The IB type used by snapshot
*
* Returns the object pointer on success else error code in the pointer
*/
static inline void adreno_ib_init_ib_obj(uint64_t gpuaddr,
uint64_t size, int obj_type,
struct kgsl_mem_entry *entry,
struct adreno_ib_object *ib_obj)
{
ib_obj->gpuaddr = gpuaddr;
ib_obj->size = size;
ib_obj->snapshot_obj_type = obj_type;
ib_obj->entry = entry;
}
/*
* adreno_cp_parser_getreg() - Returns the value of register offset
* @adreno_dev: The adreno device being operated upon
* @reg_enum: Enum index of the register whose offset is returned
*/
static inline int adreno_cp_parser_getreg(struct adreno_device *adreno_dev,
enum adreno_cp_addr_regs reg_enum)
{
if (reg_enum == ADRENO_CP_ADDR_MAX)
return -EEXIST;
if (!adreno_is_a3xx(adreno_dev))
return -EEXIST;
return a3xx_cp_addr_regs[reg_enum];
}
/*
* adreno_cp_parser_regindex() - Returns enum index for a given register offset
* @adreno_dev: The adreno device being operated upon
* @offset: Register offset
* @start: The start index to search from
* @end: The last index to search
*
* Checks the list of registers defined for the device and returns the index
* whose offset value matches offset parameter.
*/
static inline int adreno_cp_parser_regindex(struct adreno_device *adreno_dev,
unsigned int offset,
enum adreno_cp_addr_regs start,
enum adreno_cp_addr_regs end)
{
int i;
const unsigned int *regs;
if (!adreno_is_a3xx(adreno_dev))
return -EEXIST;
regs = a3xx_cp_addr_regs;
for (i = start; i <= end && i < ADRENO_CP_ADDR_MAX; i++)
if (regs[i] == offset)
return i;
return -EEXIST;
}
int adreno_ib_create_object_list(
struct kgsl_device *device,
struct kgsl_process_private *process,
uint64_t gpuaddr, uint64_t dwords, uint64_t ib2base,
struct adreno_ib_object_list **out_ib_obj_list);
void adreno_ib_destroy_obj_list(struct adreno_ib_object_list *ib_obj_list);
#endif

388
drivers/gpu/msm/adreno_debugfs.c Normal file
View file

@ -0,0 +1,388 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2002,2008-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/debugfs.h>
#include "adreno.h"
extern struct dentry *kgsl_debugfs_dir;
static int _isdb_set(void *data, u64 val)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
/* Once ISDB goes enabled it stays enabled */
if (test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv))
return 0;
mutex_lock(&device->mutex);
/*
* Bring down the GPU so we can bring it back up with the correct power
* and clock settings
*/
kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND);
set_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv);
kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER);
mutex_unlock(&device->mutex);
return 0;
}
static int _isdb_get(void *data, u64 *val)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
*val = (u64) test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv);
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(_isdb_fops, _isdb_get, _isdb_set, "%llu\n");
static int _lm_limit_set(void *data, u64 val)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM))
return 0;
/* assure value is between 3A and 10A */
if (val > 10000)
val = 10000;
else if (val < 3000)
val = 3000;
adreno_dev->lm_limit = val;
if (test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag)) {
mutex_lock(&device->mutex);
kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND);
kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER);
mutex_unlock(&device->mutex);
}
return 0;
}
static int _lm_limit_get(void *data, u64 *val)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM))
*val = 0;
*val = (u64) adreno_dev->lm_limit;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(_lm_limit_fops, _lm_limit_get,
_lm_limit_set, "%llu\n");
static int _lm_threshold_count_get(void *data, u64 *val)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (!ADRENO_FEATURE(adreno_dev, ADRENO_LM))
*val = 0;
else
*val = (u64) adreno_dev->lm_threshold_cross;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(_lm_threshold_fops, _lm_threshold_count_get,
NULL, "%llu\n");
static int _active_count_get(void *data, u64 *val)
{
struct kgsl_device *device = data;
unsigned int i = atomic_read(&device->active_cnt);
*val = (u64) i;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(_active_count_fops, _active_count_get, NULL, "%llu\n");
static int _coop_reset_set(void *data, u64 val)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (ADRENO_FEATURE(adreno_dev, ADRENO_COOP_RESET))
adreno_dev->cooperative_reset = val ? true : false;
return 0;
}
static int _coop_reset_get(void *data, u64 *val)
{
struct kgsl_device *device = data;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
*val = (u64) adreno_dev->cooperative_reset;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(_coop_reset_fops, _coop_reset_get,
_coop_reset_set, "%llu\n");
typedef void (*reg_read_init_t)(struct kgsl_device *device);
typedef void (*reg_read_fill_t)(struct kgsl_device *device, int i,
unsigned int *vals, int linec);
static void sync_event_print(struct seq_file *s,
struct kgsl_drawobj_sync_event *sync_event)
{
switch (sync_event->type) {
case KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP: {
seq_printf(s, "sync: ctx: %u ts: %u",
sync_event->context->id, sync_event->timestamp);
break;
}
case KGSL_CMD_SYNCPOINT_TYPE_FENCE: {
int i;
for (i = 0; i < sync_event->info.num_fences; i++)
seq_printf(s, "sync: %s",
sync_event->info.fences[i].name);
break;
}
default:
seq_printf(s, "sync: type: %d", sync_event->type);
break;
}
}
struct flag_entry {
unsigned long mask;
const char *str;
};
static void _print_flags(struct seq_file *s, const struct flag_entry *table,
unsigned long flags)
{
int i;
int first = 1;
for (i = 0; table[i].str; i++) {
if (flags & table[i].mask) {
seq_printf(s, "%c%s", first ? '\0' : '|', table[i].str);
flags &= ~(table[i].mask);
first = 0;
}
}
if (flags) {
seq_printf(s, "%c0x%lx", first ? '\0' : '|', flags);
first = 0;
}
if (first)
seq_puts(s, "None");
}
#define print_flags(_s, _flag, _array...) \
({ \
const struct flag_entry symbols[] = \
{ _array, { -1, NULL } }; \
_print_flags(_s, symbols, _flag); \
})
static void syncobj_print(struct seq_file *s,
struct kgsl_drawobj_sync *syncobj)
{
struct kgsl_drawobj_sync_event *event;
unsigned int i;
seq_puts(s, " syncobj ");
for (i = 0; i < syncobj->numsyncs; i++) {
event = &syncobj->synclist[i];
if (!kgsl_drawobj_event_pending(syncobj, i))
continue;
sync_event_print(s, event);
seq_puts(s, "\n");
}
}
static void cmdobj_print(struct seq_file *s,
struct kgsl_drawobj_cmd *cmdobj)
{
struct kgsl_drawobj *drawobj = DRAWOBJ(cmdobj);
if (drawobj->type == CMDOBJ_TYPE)
seq_puts(s, " cmdobj ");
else
seq_puts(s, " markerobj ");
seq_printf(s, "\t %u ", drawobj->timestamp);
seq_puts(s, " priv: ");
print_flags(s, cmdobj->priv,
{ CMDOBJ_SKIP, "skip"},
{ CMDOBJ_FORCE_PREAMBLE, "force_preamble"},
{ CMDOBJ_WFI, "wait_for_idle" });
}
static void drawobj_print(struct seq_file *s,
struct kgsl_drawobj *drawobj)
{
if (!kref_get_unless_zero(&drawobj->refcount))
return;
if (drawobj->type == SYNCOBJ_TYPE)
syncobj_print(s, SYNCOBJ(drawobj));
else if ((drawobj->type == CMDOBJ_TYPE) ||
(drawobj->type == MARKEROBJ_TYPE))
cmdobj_print(s, CMDOBJ(drawobj));
seq_puts(s, " flags: ");
print_flags(s, drawobj->flags, KGSL_DRAWOBJ_FLAGS),
kgsl_drawobj_put(drawobj);
seq_puts(s, "\n");
}
static int ctx_print(struct seq_file *s, void *unused)
{
struct adreno_context *drawctxt = s->private;
unsigned int i;
struct kgsl_event *event;
unsigned int queued = 0, consumed = 0, retired = 0;
seq_printf(s, "id: %u type: %s priority: %d process: %s (%d) tid: %d\n",
drawctxt->base.id,
kgsl_context_type(drawctxt->type),
drawctxt->base.priority,
drawctxt->base.proc_priv->comm,
drawctxt->base.proc_priv->pid,
drawctxt->base.tid);
seq_puts(s, "flags: ");
print_flags(s, drawctxt->base.flags & ~(KGSL_CONTEXT_PRIORITY_MASK
| KGSL_CONTEXT_TYPE_MASK), KGSL_CONTEXT_FLAGS);
seq_puts(s, " priv: ");
print_flags(s, drawctxt->base.priv,
{ KGSL_CONTEXT_PRIV_SUBMITTED, "submitted"},
{ KGSL_CONTEXT_PRIV_DETACHED, "detached"},
{ KGSL_CONTEXT_PRIV_INVALID, "invalid"},
{ KGSL_CONTEXT_PRIV_PAGEFAULT, "pagefault"},
{ ADRENO_CONTEXT_FAULT, "fault"},
{ ADRENO_CONTEXT_GPU_HANG, "gpu_hang"},
{ ADRENO_CONTEXT_GPU_HANG_FT, "gpu_hang_ft"},
{ ADRENO_CONTEXT_SKIP_EOF, "skip_end_of_frame" },
{ ADRENO_CONTEXT_FORCE_PREAMBLE, "force_preamble"});
seq_puts(s, "\n");
seq_puts(s, "timestamps: ");
kgsl_readtimestamp(drawctxt->base.device, &drawctxt->base,
KGSL_TIMESTAMP_QUEUED, &queued);
kgsl_readtimestamp(drawctxt->base.device, &drawctxt->base,
KGSL_TIMESTAMP_CONSUMED, &consumed);
kgsl_readtimestamp(drawctxt->base.device, &drawctxt->base,
KGSL_TIMESTAMP_RETIRED, &retired);
seq_printf(s, "queued: %u consumed: %u retired: %u global:%u\n",
queued, consumed, retired,
drawctxt->internal_timestamp);
seq_puts(s, "drawqueue:\n");
spin_lock(&drawctxt->lock);
for (i = drawctxt->drawqueue_head;
i != drawctxt->drawqueue_tail;
i = DRAWQUEUE_NEXT(i, ADRENO_CONTEXT_DRAWQUEUE_SIZE))
drawobj_print(s, drawctxt->drawqueue[i]);
spin_unlock(&drawctxt->lock);
seq_puts(s, "events:\n");
spin_lock(&drawctxt->base.events.lock);
list_for_each_entry(event, &drawctxt->base.events.events, node)
seq_printf(s, "\t%d: %pS created: %u\n", event->timestamp,
event->func, event->created);
spin_unlock(&drawctxt->base.events.lock);
return 0;
}
static int ctx_open(struct inode *inode, struct file *file)
{
int ret;
unsigned int id = (unsigned int)(unsigned long)inode->i_private;
struct kgsl_context *context;
context = kgsl_context_get(kgsl_get_device(KGSL_DEVICE_3D0), id);
if (context == NULL)
return -ENODEV;
ret = single_open(file, ctx_print, context);
if (ret)
kgsl_context_put(context);
return ret;
}
static int ctx_release(struct inode *inode, struct file *file)
{
struct kgsl_context *context;
context = ((struct seq_file *)file->private_data)->private;
kgsl_context_put(context);
return single_release(inode, file);
}
static const struct file_operations ctx_fops = {
.open = ctx_open,
.read = seq_read,
.llseek = seq_lseek,
.release = ctx_release,
};
void
adreno_context_debugfs_init(struct adreno_device *adreno_dev,
struct adreno_context *ctx)
{
unsigned char name[16];
snprintf(name, sizeof(name), "%d", ctx->base.id);
ctx->debug_root = debugfs_create_file(name, 0444,
adreno_dev->ctx_d_debugfs,
(void *)(unsigned long)ctx->base.id, &ctx_fops);
}
void adreno_debugfs_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct dentry *snapshot_dir;
if (IS_ERR_OR_NULL(device->d_debugfs))
return;
debugfs_create_file("active_cnt", 0444, device->d_debugfs, device,
&_active_count_fops);
adreno_dev->ctx_d_debugfs = debugfs_create_dir("ctx",
device->d_debugfs);
snapshot_dir = debugfs_lookup("snapshot", kgsl_debugfs_dir);
if (!IS_ERR_OR_NULL(snapshot_dir))
debugfs_create_file("coop_reset", 0644, snapshot_dir, device,
&_coop_reset_fops);
if (ADRENO_FEATURE(adreno_dev, ADRENO_LM)) {
debugfs_create_file("lm_limit", 0644, device->d_debugfs, device,
&_lm_limit_fops);
debugfs_create_file("lm_threshold_count", 0444,
device->d_debugfs, device, &_lm_threshold_fops);
}
if (adreno_is_a5xx(adreno_dev))
debugfs_create_file("isdb", 0644, device->d_debugfs,
device, &_isdb_fops);
}

2930
drivers/gpu/msm/adreno_dispatch.c Normal file
View file

File diff suppressed because it is too large Load diff

110
drivers/gpu/msm/adreno_dispatch.h Normal file
View file

@ -0,0 +1,110 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2008-2019, The Linux Foundation. All rights reserved.
*/
#ifndef ____ADRENO_DISPATCHER_H
#define ____ADRENO_DISPATCHER_H
#include <linux/kobject.h>
#include <linux/kthread.h>
extern unsigned int adreno_drawobj_timeout;
/*
* Maximum size of the dispatcher ringbuffer - the actual inflight size will be
* smaller then this but this size will allow for a larger range of inflight
* sizes that can be chosen at runtime
*/
#define ADRENO_DISPATCH_DRAWQUEUE_SIZE 128
#define DRAWQUEUE_NEXT(_i, _s) (((_i) + 1) % (_s))
/**
* struct adreno_dispatcher_drawqueue - List of commands for a RB level
* @cmd_q: List of command obj's submitted to dispatcher
* @inflight: Number of commands inflight in this q
* @head: Head pointer to the q
* @tail: Queues tail pointer
* @active_context_count: Number of active contexts seen in this rb drawqueue
* @expires: The jiffies value at which this drawqueue has run too long
*/
struct adreno_dispatcher_drawqueue {
struct kgsl_drawobj_cmd *cmd_q[ADRENO_DISPATCH_DRAWQUEUE_SIZE];
unsigned int inflight;
unsigned int head;
unsigned int tail;
int active_context_count;
unsigned long expires;
};
/**
* struct adreno_dispatcher - container for the adreno GPU dispatcher
* @mutex: Mutex to protect the structure
* @state: Current state of the dispatcher (active or paused)
* @timer: Timer to monitor the progress of the drawobjs
* @inflight: Number of drawobj operations pending in the ringbuffer
* @fault: Non-zero if a fault was detected.
* @pending: Priority list of contexts waiting to submit drawobjs
* @plist_lock: Spin lock to protect the pending queue
* @work: work_struct to put the dispatcher in a work queue
* @kobj: kobject for the dispatcher directory in the device sysfs node
* @idle_gate: Gate to wait on for dispatcher to idle
*/
struct adreno_dispatcher {
struct mutex mutex;
unsigned long priv;
struct timer_list timer;
struct timer_list fault_timer;
unsigned int inflight;
atomic_t fault;
struct plist_head pending;
spinlock_t plist_lock;
struct kthread_work work;
struct kobject kobj;
struct completion idle_gate;
};
enum adreno_dispatcher_flags {
ADRENO_DISPATCHER_POWER = 0,
ADRENO_DISPATCHER_ACTIVE = 1,
};
struct adreno_device;
struct adreno_context;
struct kgsl_context;
struct kgsl_device;
struct kgsl_device_private;
void adreno_dispatcher_start(struct kgsl_device *device);
void adreno_dispatcher_halt(struct kgsl_device *device);
void adreno_dispatcher_unhalt(struct kgsl_device *device);
int adreno_dispatcher_init(struct adreno_device *adreno_dev);
void adreno_dispatcher_close(struct adreno_device *adreno_dev);
int adreno_dispatcher_idle(struct adreno_device *adreno_dev);
void adreno_dispatcher_irq_fault(struct adreno_device *adreno_dev);
void adreno_dispatcher_stop(struct adreno_device *adreno_dev);
void adreno_dispatcher_stop_fault_timer(struct kgsl_device *device);
struct kgsl_drawobj;
int adreno_dispatcher_queue_cmds(struct kgsl_device_private *dev_priv,
struct kgsl_context *context, struct kgsl_drawobj *drawobj[],
uint32_t count, uint32_t *timestamp);
void adreno_dispatcher_schedule(struct kgsl_device *device);
void adreno_dispatcher_pause(struct adreno_device *adreno_dev);
void adreno_dispatcher_queue_context(struct kgsl_device *device,
struct adreno_context *drawctxt);
void adreno_dispatcher_preempt_callback(struct adreno_device *adreno_dev,
int bit);
void adreno_preempt_process_dispatch_queue(struct adreno_device *adreno_dev,
struct adreno_dispatcher_drawqueue *dispatch_q);
static inline bool adreno_drawqueue_is_empty(
struct adreno_dispatcher_drawqueue *drawqueue)
{
return (drawqueue != NULL && drawqueue->head == drawqueue->tail);
}
#endif /* __ADRENO_DISPATCHER_H */

645
drivers/gpu/msm/adreno_drawctxt.c Normal file
View file

@ -0,0 +1,645 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/debugfs.h>
#include "adreno.h"
#include "adreno_iommu.h"
#include "adreno_trace.h"
static void wait_callback(struct kgsl_device *device,
struct kgsl_event_group *group, void *priv, int result)
{
struct adreno_context *drawctxt = priv;
wake_up_all(&drawctxt->waiting);
}
static int _check_context_timestamp(struct kgsl_device *device,
struct kgsl_context *context, unsigned int timestamp)
{
/* Bail if the drawctxt has been invalidated or destroyed */
if (kgsl_context_detached(context) || kgsl_context_invalid(context))
return 1;
return kgsl_check_timestamp(device, context, timestamp);
}
/**
* adreno_drawctxt_dump() - dump information about a draw context
* @device: KGSL device that owns the context
* @context: KGSL context to dump information about
*
* Dump specific information about the context to the kernel log. Used for
* fence timeout callbacks
*/
void adreno_drawctxt_dump(struct kgsl_device *device,
struct kgsl_context *context)
{
unsigned int queue, start, retire;
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
int index, pos;
char buf[120];
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_QUEUED, &queue);
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_CONSUMED, &start);
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED, &retire);
/*
* We may have kgsl sync obj timer running, which also uses same
* lock, take a lock with software interrupt disabled (bh)
* to avoid spin lock recursion.
*
* Use Spin trylock because dispatcher can acquire drawctxt->lock
* if context is pending and the fence it is waiting on just got
* signalled. Dispatcher acquires drawctxt->lock and tries to
* delete the sync obj timer using del_timer_sync().
* del_timer_sync() waits till timer and its pending handlers
* are deleted. But if the timer expires at the same time,
* timer handler could be waiting on drawctxt->lock leading to a
* deadlock. To prevent this use spin_trylock_bh.
*/
if (!spin_trylock_bh(&drawctxt->lock)) {
dev_err(device->dev, " context[%u]: could not get lock\n",
context->id);
return;
}
dev_err(device->dev,
" context[%u]: queue=%u, submit=%u, start=%u, retire=%u\n",
context->id, queue, drawctxt->submitted_timestamp,
start, retire);
if (drawctxt->drawqueue_head != drawctxt->drawqueue_tail) {
struct kgsl_drawobj *drawobj =
drawctxt->drawqueue[drawctxt->drawqueue_head];
if (test_bit(ADRENO_CONTEXT_FENCE_LOG, &context->priv)) {
dev_err(device->dev,
" possible deadlock. Context %u might be blocked for itself\n",
context->id);
goto stats;
}
if (!kref_get_unless_zero(&drawobj->refcount))
goto stats;
if (drawobj->type == SYNCOBJ_TYPE) {
struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj);
if (kgsl_drawobj_events_pending(syncobj)) {
dev_err(device->dev,
" context[%u] (ts=%u) Active sync points:\n",
context->id, drawobj->timestamp);
kgsl_dump_syncpoints(device, syncobj);
}
}
kgsl_drawobj_put(drawobj);
}
stats:
memset(buf, 0, sizeof(buf));
pos = 0;
for (index = 0; index < SUBMIT_RETIRE_TICKS_SIZE; index++) {
uint64_t msecs;
unsigned int usecs;
if (!drawctxt->submit_retire_ticks[index])
continue;
msecs = drawctxt->submit_retire_ticks[index] * 10;
usecs = do_div(msecs, 192);
usecs = do_div(msecs, 1000);
pos += scnprintf(buf + pos, sizeof(buf) - pos, "%u.%0u ",
(unsigned int)msecs, usecs);
}
dev_err(device->dev, " context[%u]: submit times: %s\n",
context->id, buf);
spin_unlock_bh(&drawctxt->lock);
}
/**
* adreno_drawctxt_wait() - sleep until a timestamp expires
* @adreno_dev: pointer to the adreno_device struct
* @drawctxt: Pointer to the draw context to sleep for
* @timetamp: Timestamp to wait on
* @timeout: Number of jiffies to wait (0 for infinite)
*
* Register an event to wait for a timestamp on a context and sleep until it
* has past. Returns < 0 on error, -ETIMEDOUT if the timeout expires or 0
* on success
*/
int adreno_drawctxt_wait(struct adreno_device *adreno_dev,
struct kgsl_context *context,
uint32_t timestamp, unsigned int timeout)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
int ret;
long ret_temp;
if (kgsl_context_detached(context))
return -ENOENT;
if (kgsl_context_invalid(context))
return -EDEADLK;
trace_adreno_drawctxt_wait_start(-1, context->id, timestamp);
ret = kgsl_add_event(device, &context->events, timestamp,
wait_callback, (void *) drawctxt);
if (ret)
goto done;
/*
* If timeout is 0, wait forever. msecs_to_jiffies will force
* values larger than INT_MAX to an infinite timeout.
*/
if (timeout == 0)
timeout = UINT_MAX;
ret_temp = wait_event_interruptible_timeout(drawctxt->waiting,
_check_context_timestamp(device, context, timestamp),
msecs_to_jiffies(timeout));
if (ret_temp == 0) {
ret = -ETIMEDOUT;
goto done;
} else if (ret_temp < 0) {
ret = (int) ret_temp;
goto done;
}
ret = 0;
/* -EDEADLK if the context was invalidated while we were waiting */
if (kgsl_context_invalid(context))
ret = -EDEADLK;
/* Return -EINVAL if the context was detached while we were waiting */
if (kgsl_context_detached(context))
ret = -ENOENT;
done:
trace_adreno_drawctxt_wait_done(-1, context->id, timestamp, ret);
return ret;
}
/**
* adreno_drawctxt_wait_rb() - Wait for the last RB timestamp at which this
* context submitted a command to the corresponding RB
* @adreno_dev: The device on which the timestamp is active
* @context: The context which subbmitted command to RB
* @timestamp: The RB timestamp of last command submitted to RB by context
* @timeout: Timeout value for the wait
* Caller must hold the device mutex
*/
static int adreno_drawctxt_wait_rb(struct adreno_device *adreno_dev,
struct kgsl_context *context,
uint32_t timestamp, unsigned int timeout)
{
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
int ret = 0;
/*
* If the context is invalid (OR) not submitted commands to GPU
* then return immediately - we may end up waiting for a timestamp
* that will never come
*/
if (kgsl_context_invalid(context) ||
!test_bit(KGSL_CONTEXT_PRIV_SUBMITTED, &context->priv))
goto done;
trace_adreno_drawctxt_wait_start(drawctxt->rb->id, context->id,
timestamp);
ret = adreno_ringbuffer_waittimestamp(drawctxt->rb, timestamp, timeout);
done:
trace_adreno_drawctxt_wait_done(drawctxt->rb->id, context->id,
timestamp, ret);
return ret;
}
static int drawctxt_detach_drawobjs(struct adreno_context *drawctxt,
struct kgsl_drawobj **list)
{
int count = 0;
while (drawctxt->drawqueue_head != drawctxt->drawqueue_tail) {
struct kgsl_drawobj *drawobj =
drawctxt->drawqueue[drawctxt->drawqueue_head];
drawctxt->drawqueue_head = (drawctxt->drawqueue_head + 1) %
ADRENO_CONTEXT_DRAWQUEUE_SIZE;
list[count++] = drawobj;
}
return count;
}
/**
* adreno_drawctxt_invalidate() - Invalidate an adreno draw context
* @device: Pointer to the KGSL device structure for the GPU
* @context: Pointer to the KGSL context structure
*
* Invalidate the context and remove all queued commands and cancel any pending
* waiters
*/
void adreno_drawctxt_invalidate(struct kgsl_device *device,
struct kgsl_context *context)
{
struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
struct kgsl_drawobj *list[ADRENO_CONTEXT_DRAWQUEUE_SIZE];
int i, count;
trace_adreno_drawctxt_invalidate(drawctxt);
spin_lock(&drawctxt->lock);
set_bit(KGSL_CONTEXT_PRIV_INVALID, &context->priv);
/*
* set the timestamp to the last value since the context is invalidated
* and we want the pending events for this context to go away
*/
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
/* Get rid of commands still waiting in the queue */
count = drawctxt_detach_drawobjs(drawctxt, list);
spin_unlock(&drawctxt->lock);
for (i = 0; i < count; i++) {
kgsl_cancel_events_timestamp(device, &context->events,
list[i]->timestamp);
kgsl_drawobj_destroy(list[i]);
}
/* Make sure all pending events are processed or cancelled */
kgsl_flush_event_group(device, &context->events);
/* Give the bad news to everybody waiting around */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
wake_up_all(&drawctxt->timeout);
}
/*
* Set the priority of the context based on the flags passed into context
* create. If the priority is not set in the flags, then the kernel can
* assign any priority it desires for the context.
*/
#define KGSL_CONTEXT_PRIORITY_MED 0x8
static inline void _set_context_priority(struct adreno_context *drawctxt)
{
/* If the priority is not set by user, set it for them */
if ((drawctxt->base.flags & KGSL_CONTEXT_PRIORITY_MASK) ==
KGSL_CONTEXT_PRIORITY_UNDEF)
drawctxt->base.flags |= (KGSL_CONTEXT_PRIORITY_MED <<
KGSL_CONTEXT_PRIORITY_SHIFT);
/* Store the context priority */
drawctxt->base.priority =
(drawctxt->base.flags & KGSL_CONTEXT_PRIORITY_MASK) >>
KGSL_CONTEXT_PRIORITY_SHIFT;
}
/**
* adreno_drawctxt_create - create a new adreno draw context
* @dev_priv: the owner of the context
* @flags: flags for the context (passed from user space)
*
* Create and return a new draw context for the 3D core.
*/
struct kgsl_context *
adreno_drawctxt_create(struct kgsl_device_private *dev_priv,
uint32_t *flags)
{
struct adreno_context *drawctxt;
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
int ret;
unsigned int local;
local = *flags & (KGSL_CONTEXT_PREAMBLE |
KGSL_CONTEXT_NO_GMEM_ALLOC |
KGSL_CONTEXT_PER_CONTEXT_TS |
KGSL_CONTEXT_USER_GENERATED_TS |
KGSL_CONTEXT_NO_FAULT_TOLERANCE |
KGSL_CONTEXT_INVALIDATE_ON_FAULT |
KGSL_CONTEXT_CTX_SWITCH |
KGSL_CONTEXT_PRIORITY_MASK |
KGSL_CONTEXT_TYPE_MASK |
KGSL_CONTEXT_PWR_CONSTRAINT |
KGSL_CONTEXT_IFH_NOP |
KGSL_CONTEXT_SECURE |
KGSL_CONTEXT_PREEMPT_STYLE_MASK |
KGSL_CONTEXT_NO_SNAPSHOT |
KGSL_CONTEXT_SPARSE);
/* Check for errors before trying to initialize */
/* If preemption is not supported, ignore preemption request */
if (!test_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv))
local &= ~KGSL_CONTEXT_PREEMPT_STYLE_MASK;
/* We no longer support legacy context switching */
if ((local & KGSL_CONTEXT_PREAMBLE) == 0 ||
(local & KGSL_CONTEXT_NO_GMEM_ALLOC) == 0) {
dev_err_once(device->dev,
"legacy context switch not supported\n");
return ERR_PTR(-EINVAL);
}
/* Make sure that our target can support secure contexts if requested */
if (!kgsl_mmu_is_secured(&dev_priv->device->mmu) &&
(local & KGSL_CONTEXT_SECURE)) {
dev_err_once(device->dev, "Secure context not supported\n");
return ERR_PTR(-EOPNOTSUPP);
}
drawctxt = kzalloc(sizeof(struct adreno_context), GFP_KERNEL);
if (drawctxt == NULL)
return ERR_PTR(-ENOMEM);
drawctxt->timestamp = 0;
drawctxt->base.flags = local;
/* Always enable per-context timestamps */
drawctxt->base.flags |= KGSL_CONTEXT_PER_CONTEXT_TS;
drawctxt->type = (drawctxt->base.flags & KGSL_CONTEXT_TYPE_MASK)
>> KGSL_CONTEXT_TYPE_SHIFT;
spin_lock_init(&drawctxt->lock);
init_waitqueue_head(&drawctxt->wq);
init_waitqueue_head(&drawctxt->waiting);
init_waitqueue_head(&drawctxt->timeout);
/* Set the context priority */
_set_context_priority(drawctxt);
/* set the context ringbuffer */
drawctxt->rb = adreno_ctx_get_rb(adreno_dev, drawctxt);
/*
* Set up the plist node for the dispatcher. Insert the node into the
* drawctxt pending list based on priority.
*/
plist_node_init(&drawctxt->pending, drawctxt->base.priority);
/*
* Now initialize the common part of the context. This allocates the
* context id, and then possibly another thread could look it up.
* So we want all of our initializtion that doesn't require the context
* id to be done before this call.
*/
ret = kgsl_context_init(dev_priv, &drawctxt->base);
if (ret != 0) {
kfree(drawctxt);
return ERR_PTR(ret);
}
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, soptimestamp),
0);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(drawctxt->base.id, eoptimestamp),
0);
adreno_context_debugfs_init(ADRENO_DEVICE(device), drawctxt);
INIT_LIST_HEAD(&drawctxt->active_node);
if (gpudev->preemption_context_init) {
ret = gpudev->preemption_context_init(&drawctxt->base);
if (ret != 0) {
kgsl_context_detach(&drawctxt->base);
return ERR_PTR(ret);
}
}
/* copy back whatever flags we dediced were valid */
*flags = drawctxt->base.flags;
return &drawctxt->base;
}
/**
* adreno_drawctxt_sched() - Schedule a previously blocked context
* @device: pointer to a KGSL device
* @drawctxt: drawctxt to rechedule
*
* This function is called by the core when it knows that a previously blocked
* context has been unblocked. The default adreno response is to reschedule the
* context on the dispatcher
*/
void adreno_drawctxt_sched(struct kgsl_device *device,
struct kgsl_context *context)
{
adreno_dispatcher_queue_context(device, ADRENO_CONTEXT(context));
}
/**
* adreno_drawctxt_detach(): detach a context from the GPU
* @context: Generic KGSL context container for the context
*
*/
void adreno_drawctxt_detach(struct kgsl_context *context)
{
struct kgsl_device *device;
struct adreno_device *adreno_dev;
struct adreno_gpudev *gpudev;
struct adreno_context *drawctxt;
struct adreno_ringbuffer *rb;
int ret, count, i;
struct kgsl_drawobj *list[ADRENO_CONTEXT_DRAWQUEUE_SIZE];
if (context == NULL)
return;
device = context->device;
adreno_dev = ADRENO_DEVICE(device);
gpudev = ADRENO_GPU_DEVICE(adreno_dev);
drawctxt = ADRENO_CONTEXT(context);
rb = drawctxt->rb;
spin_lock(&adreno_dev->active_list_lock);
list_del_init(&drawctxt->active_node);
spin_unlock(&adreno_dev->active_list_lock);
spin_lock(&drawctxt->lock);
count = drawctxt_detach_drawobjs(drawctxt, list);
spin_unlock(&drawctxt->lock);
for (i = 0; i < count; i++) {
/*
* If the context is deteached while we are waiting for
* the next command in GFT SKIP CMD, print the context
* detached status here.
*/
adreno_fault_skipcmd_detached(adreno_dev, drawctxt, list[i]);
kgsl_drawobj_destroy(list[i]);
}
debugfs_remove_recursive(drawctxt->debug_root);
/*
* internal_timestamp is set in adreno_ringbuffer_addcmds,
* which holds the device mutex.
*/
mutex_lock(&device->mutex);
/*
* Wait for the last global timestamp to pass before continuing.
* The maxumum wait time is 30s, some large IB's can take longer
* than 10s and if hang happens then the time for the context's
* commands to retire will be greater than 10s. 30s should be sufficient
* time to wait for the commands even if a hang happens.
*/
ret = adreno_drawctxt_wait_rb(adreno_dev, context,
drawctxt->internal_timestamp, 30 * 1000);
/*
* If the wait for global fails due to timeout then mark it as
* context detach timeout fault and schedule dispatcher to kick
* in GPU recovery. For a ADRENO_CTX_DETATCH_TIMEOUT_FAULT we clear
* the policy and invalidate the context. If EAGAIN error is returned
* then recovery will kick in and there will be no more commands in the
* RB pipe from this context which is what we are waiting for, so ignore
* -EAGAIN error.
*/
if (ret && ret != -EAGAIN) {
dev_err(device->dev,
"Wait for global ctx=%u ts=%u type=%d error=%d\n",
drawctxt->base.id, drawctxt->internal_timestamp,
drawctxt->type, ret);
adreno_set_gpu_fault(adreno_dev,
ADRENO_CTX_DETATCH_TIMEOUT_FAULT);
mutex_unlock(&device->mutex);
/* Schedule dispatcher to kick in recovery */
adreno_dispatcher_schedule(device);
/* Wait for context to be invalidated and release context */
wait_event_interruptible_timeout(drawctxt->timeout,
kgsl_context_invalid(&drawctxt->base),
msecs_to_jiffies(5000));
return;
}
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, soptimestamp),
drawctxt->timestamp);
kgsl_sharedmem_writel(device, &device->memstore,
KGSL_MEMSTORE_OFFSET(context->id, eoptimestamp),
drawctxt->timestamp);
adreno_profile_process_results(adreno_dev);
mutex_unlock(&device->mutex);
if (gpudev->preemption_context_destroy)
gpudev->preemption_context_destroy(context);
/* wake threads waiting to submit commands from this context */
wake_up_all(&drawctxt->waiting);
wake_up_all(&drawctxt->wq);
}
void adreno_drawctxt_destroy(struct kgsl_context *context)
{
struct adreno_context *drawctxt;
if (context == NULL)
return;
drawctxt = ADRENO_CONTEXT(context);
kfree(drawctxt);
}
static void _drawctxt_switch_wait_callback(struct kgsl_device *device,
struct kgsl_event_group *group,
void *priv, int result)
{
struct adreno_context *drawctxt = (struct adreno_context *) priv;
kgsl_context_put(&drawctxt->base);
}
/**
* adreno_drawctxt_switch - switch the current draw context in a given RB
* @adreno_dev - The 3D device that owns the context
* @rb: The ringubffer pointer on which the current context is being changed
* @drawctxt - the 3D context to switch to
*
* Switch the current draw context in given RB
*/
int adreno_drawctxt_switch(struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb,
struct adreno_context *drawctxt)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pagetable *new_pt;
int ret = 0;
/* We always expect a valid rb */
if (!rb)
return -EINVAL;
/* already current? */
if (rb->drawctxt_active == drawctxt)
return ret;
/*
* Submitting pt switch commands from a detached context can
* lead to a race condition where the pt is destroyed before
* the pt switch commands get executed by the GPU, leading to
* pagefaults.
*/
if (drawctxt != NULL && kgsl_context_detached(&drawctxt->base))
return -ENOENT;
trace_adreno_drawctxt_switch(rb, drawctxt);
/* Get a refcount to the new instance */
if (drawctxt) {
if (!_kgsl_context_get(&drawctxt->base))
return -ENOENT;
new_pt = drawctxt->base.proc_priv->pagetable;
} else {
/* No context - set the default pagetable and thats it. */
new_pt = device->mmu.defaultpagetable;
}
ret = adreno_iommu_set_pt_ctx(rb, new_pt, drawctxt);
if (ret)
return ret;
if (rb->drawctxt_active) {
/* Wait for the timestamp to expire */
if (kgsl_add_event(device, &rb->events, rb->timestamp,
_drawctxt_switch_wait_callback,
rb->drawctxt_active)) {
kgsl_context_put(&rb->drawctxt_active->base);
}
}
rb->drawctxt_active = drawctxt;
return 0;
}

133
drivers/gpu/msm/adreno_drawctxt.h Normal file
View file

@ -0,0 +1,133 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_DRAWCTXT_H
#define __ADRENO_DRAWCTXT_H
#include <linux/types.h>
#include "kgsl_device.h"
struct adreno_context_type {
unsigned int type;
const char *str;
};
#define ADRENO_CONTEXT_DRAWQUEUE_SIZE 128
#define SUBMIT_RETIRE_TICKS_SIZE 7
struct kgsl_device;
struct adreno_device;
struct kgsl_device_private;
/**
* struct adreno_context - Adreno GPU draw context
* @timestamp: Last issued context-specific timestamp
* @internal_timestamp: Global timestamp of the last issued command
* NOTE: guarded by device->mutex, not drawctxt->mutex!
* @type: Context type (GL, CL, RS)
* @mutex: Mutex to protect the drawqueue
* @drawqueue: Queue of drawobjs waiting to be dispatched for this
* context
* @drawqueue_head: Head of the drawqueue queue
* @drawqueue_tail: Tail of the drawqueue queue
* @pending: Priority list node for the dispatcher list of pending contexts
* @wq: Workqueue structure for contexts to sleep pending room in the queue
* @waiting: Workqueue structure for contexts waiting for a timestamp or event
* @timeout: Workqueue structure for contexts waiting to invalidate
* @queued: Number of commands queued in the drawqueue
* @fault_policy: GFT fault policy set in _skip_cmd();
* @debug_root: debugfs entry for this context.
* @queued_timestamp: The last timestamp that was queued on this context
* @rb: The ringbuffer in which this context submits commands.
* @submitted_timestamp: The last timestamp that was submitted for this context
* @submit_retire_ticks: Array to hold command obj execution times from submit
* to retire
* @ticks_index: The index into submit_retire_ticks[] where the new delta will
* be written.
* @active_node: Linkage for nodes in active_list
* @active_time: Time when this context last seen
*/
struct adreno_context {
struct kgsl_context base;
unsigned int timestamp;
unsigned int internal_timestamp;
unsigned int type;
spinlock_t lock;
/* Dispatcher */
struct kgsl_drawobj *drawqueue[ADRENO_CONTEXT_DRAWQUEUE_SIZE];
unsigned int drawqueue_head;
unsigned int drawqueue_tail;
struct plist_node pending;
wait_queue_head_t wq;
wait_queue_head_t waiting;
wait_queue_head_t timeout;
int queued;
unsigned int fault_policy;
struct dentry *debug_root;
unsigned int queued_timestamp;
struct adreno_ringbuffer *rb;
unsigned int submitted_timestamp;
uint64_t submit_retire_ticks[SUBMIT_RETIRE_TICKS_SIZE];
int ticks_index;
struct list_head active_node;
unsigned long active_time;
};
/* Flag definitions for flag field in adreno_context */
/**
* enum adreno_context_priv - Private flags for an adreno draw context
* @ADRENO_CONTEXT_FAULT - set if the context has faulted (and recovered)
* @ADRENO_CONTEXT_GPU_HANG - Context has caused a GPU hang
* @ADRENO_CONTEXT_GPU_HANG_FT - Context has caused a GPU hang
* and fault tolerance was successful
* @ADRENO_CONTEXT_SKIP_EOF - Context skip IBs until the next end of frame
* marker.
* @ADRENO_CONTEXT_FORCE_PREAMBLE - Force the preamble for the next submission.
* @ADRENO_CONTEXT_SKIP_CMD - Context's drawobj's skipped during
fault tolerance.
* @ADRENO_CONTEXT_FENCE_LOG - Dump fences on this context.
*/
enum adreno_context_priv {
ADRENO_CONTEXT_FAULT = KGSL_CONTEXT_PRIV_DEVICE_SPECIFIC,
ADRENO_CONTEXT_GPU_HANG,
ADRENO_CONTEXT_GPU_HANG_FT,
ADRENO_CONTEXT_SKIP_EOF,
ADRENO_CONTEXT_FORCE_PREAMBLE,
ADRENO_CONTEXT_SKIP_CMD,
ADRENO_CONTEXT_FENCE_LOG,
};
struct kgsl_context *adreno_drawctxt_create(
struct kgsl_device_private *dev_priv,
uint32_t *flags);
void adreno_drawctxt_detach(struct kgsl_context *context);
void adreno_drawctxt_destroy(struct kgsl_context *context);
void adreno_drawctxt_sched(struct kgsl_device *device,
struct kgsl_context *context);
struct adreno_ringbuffer;
int adreno_drawctxt_switch(struct adreno_device *adreno_dev,
struct adreno_ringbuffer *rb,
struct adreno_context *drawctxt);
int adreno_drawctxt_wait(struct adreno_device *adreno_dev,
struct kgsl_context *context,
uint32_t timestamp, unsigned int timeout);
void adreno_drawctxt_invalidate(struct kgsl_device *device,
struct kgsl_context *context);
void adreno_drawctxt_dump(struct kgsl_device *device,
struct kgsl_context *context);
#endif /* __ADRENO_DRAWCTXT_H */

230
drivers/gpu/msm/adreno_ioctl.c Normal file
View file

@ -0,0 +1,230 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/slab.h>
#include "adreno.h"
#include "adreno_a5xx.h"
/*
* Add a perfcounter to the per-fd list.
* Call with the device mutex held
*/
static int adreno_process_perfcounter_add(struct kgsl_device_private *dev_priv,
unsigned int groupid, unsigned int countable)
{
struct adreno_device_private *adreno_priv = container_of(dev_priv,
struct adreno_device_private, dev_priv);
struct adreno_perfcounter_list_node *perfctr;
perfctr = kmalloc(sizeof(*perfctr), GFP_KERNEL);
if (!perfctr)
return -ENOMEM;
perfctr->groupid = groupid;
perfctr->countable = countable;
/* add the pair to process perfcounter list */
list_add(&perfctr->node, &adreno_priv->perfcounter_list);
return 0;
}
/*
* Remove a perfcounter from the per-fd list.
* Call with the device mutex held
*/
static int adreno_process_perfcounter_del(struct kgsl_device_private *dev_priv,
unsigned int groupid, unsigned int countable)
{
struct adreno_device_private *adreno_priv = container_of(dev_priv,
struct adreno_device_private, dev_priv);
struct adreno_perfcounter_list_node *p;
list_for_each_entry(p, &adreno_priv->perfcounter_list, node) {
if (p->groupid == groupid && p->countable == countable) {
list_del(&p->node);
kfree(p);
return 0;
}
}
return -ENODEV;
}
long adreno_ioctl_perfcounter_get(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct kgsl_perfcounter_get *get = data;
int result;
mutex_lock(&device->mutex);
/*
* adreno_perfcounter_get() is called by kernel clients
* during start(), so it is not safe to take an
* active count inside that function.
*/
result = adreno_perfcntr_active_oob_get(device);
if (result) {
mutex_unlock(&device->mutex);
return (long)result;
}
result = adreno_perfcounter_get(adreno_dev,
get->groupid, get->countable, &get->offset,
&get->offset_hi, PERFCOUNTER_FLAG_NONE);
/* Add the perfcounter into the list */
if (!result) {
result = adreno_process_perfcounter_add(dev_priv, get->groupid,
get->countable);
if (result)
adreno_perfcounter_put(adreno_dev, get->groupid,
get->countable, PERFCOUNTER_FLAG_NONE);
}
adreno_perfcntr_active_oob_put(device);
mutex_unlock(&device->mutex);
return (long) result;
}
long adreno_ioctl_perfcounter_put(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_device *device = dev_priv->device;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct kgsl_perfcounter_put *put = data;
int result;
mutex_lock(&device->mutex);
/* Delete the perfcounter from the process list */
result = adreno_process_perfcounter_del(dev_priv, put->groupid,
put->countable);
/* Put the perfcounter refcount */
if (!result)
adreno_perfcounter_put(adreno_dev, put->groupid,
put->countable, PERFCOUNTER_FLAG_NONE);
mutex_unlock(&device->mutex);
return (long) result;
}
static long adreno_ioctl_perfcounter_query(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_priv->device);
struct kgsl_perfcounter_query *query = data;
return (long) adreno_perfcounter_query_group(adreno_dev, query->groupid,
query->countables, query->count, &query->max_counters);
}
static long adreno_ioctl_perfcounter_read(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_priv->device);
struct kgsl_perfcounter_read *read = data;
return (long) adreno_perfcounter_read_group(adreno_dev, read->reads,
read->count);
}
static long adreno_ioctl_preemption_counters_query(
struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_priv->device);
struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
struct kgsl_preemption_counters_query *read = data;
int size_level = A5XX_CP_CTXRECORD_PREEMPTION_COUNTER_SIZE;
int levels_to_copy;
if (!adreno_is_a5xx(adreno_dev) ||
!adreno_is_preemption_enabled(adreno_dev))
return -EOPNOTSUPP;
if (read->size_user < size_level)
return -EINVAL;
/* Calculate number of preemption counter levels to copy to userspace */
levels_to_copy = (read->size_user / size_level);
if (levels_to_copy > gpudev->num_prio_levels)
levels_to_copy = gpudev->num_prio_levels;
if (copy_to_user(u64_to_user_ptr(read->counters),
adreno_dev->preempt.counters.hostptr,
levels_to_copy * size_level))
return -EFAULT;
read->max_priority_level = levels_to_copy;
read->size_priority_level = size_level;
return 0;
}
long adreno_ioctl_helper(struct kgsl_device_private *dev_priv,
unsigned int cmd, unsigned long arg,
const struct kgsl_ioctl *cmds, int len)
{
unsigned char data[128] = { 0 };
long ret;
int i;
for (i = 0; i < len; i++) {
if (_IOC_NR(cmd) == _IOC_NR(cmds[i].cmd))
break;
}
if (i == len) {
dev_err(dev_priv->device->dev,
"invalid ioctl code 0x%08X\n", cmd);
return -ENOIOCTLCMD;
}
if (_IOC_SIZE(cmds[i].cmd > sizeof(data))) {
dev_err_ratelimited(dev_priv->device->dev,
"data too big for ioctl 0x%08x: %d/%zu\n",
cmd, _IOC_SIZE(cmds[i].cmd), sizeof(data));
return -EINVAL;
}
if (_IOC_SIZE(cmds[i].cmd)) {
ret = kgsl_ioctl_copy_in(cmds[i].cmd, cmd, arg, data);
if (ret)
return ret;
} else {
memset(data, 0, sizeof(data));
}
ret = cmds[i].func(dev_priv, cmd, data);
if (ret == 0 && _IOC_SIZE(cmds[i].cmd))
ret = kgsl_ioctl_copy_out(cmds[i].cmd, cmd, arg, data);
return ret;
}
static struct kgsl_ioctl adreno_ioctl_funcs[] = {
{ IOCTL_KGSL_PERFCOUNTER_GET, adreno_ioctl_perfcounter_get },
{ IOCTL_KGSL_PERFCOUNTER_PUT, adreno_ioctl_perfcounter_put },
{ IOCTL_KGSL_PERFCOUNTER_QUERY, adreno_ioctl_perfcounter_query },
{ IOCTL_KGSL_PERFCOUNTER_READ, adreno_ioctl_perfcounter_read },
{ IOCTL_KGSL_PREEMPTIONCOUNTER_QUERY,
adreno_ioctl_preemption_counters_query },
};
long adreno_ioctl(struct kgsl_device_private *dev_priv,
unsigned int cmd, unsigned long arg)
{
return adreno_ioctl_helper(dev_priv, cmd, arg,
adreno_ioctl_funcs, ARRAY_SIZE(adreno_ioctl_funcs));
}

501
drivers/gpu/msm/adreno_iommu.c Normal file
View file

@ -0,0 +1,501 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/slab.h>
#include "a3xx_reg.h"
#include "adreno.h"
#include "adreno_iommu.h"
#include "adreno_pm4types.h"
/*
* a3xx_wait_reg() - make CP poll on a register
* @cmds: Pointer to memory where commands are to be added
* @addr: Register address to poll for
* @val: Value to poll for
* @mask: The value against which register value is masked
* @interval: wait interval
*/
static unsigned int a3xx_wait_reg(struct adreno_device *adreno_dev,
unsigned int *cmds, unsigned int addr,
unsigned int val, unsigned int mask,
unsigned int interval)
{
unsigned int *start = cmds;
*cmds++ = cp_packet(adreno_dev, CP_WAIT_REG_EQ, 4);
*cmds++ = addr;
*cmds++ = val;
*cmds++ = mask;
*cmds++ = interval;
return cmds - start;
}
static unsigned int a3xx_vbif_lock(struct adreno_device *adreno_dev,
unsigned int *cmds)
{
unsigned int *start = cmds;
/*
* glue commands together until next
* WAIT_FOR_ME
*/
cmds += a3xx_wait_reg(adreno_dev, cmds, A3XX_CP_WFI_PEND_CTR,
1, 0xFFFFFFFF, 0xF);
/* MMU-500 VBIF stall */
*cmds++ = cp_packet(adreno_dev, CP_REG_RMW, 3);
*cmds++ = A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL0;
/* AND to unmask the HALT bit */
*cmds++ = ~(VBIF_RECOVERABLE_HALT_CTRL);
/* OR to set the HALT bit */
*cmds++ = 0x1;
/* Wait for acknowledgment */
cmds += a3xx_wait_reg(adreno_dev, cmds,
A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL1,
1, 0xFFFFFFFF, 0xF);
return cmds - start;
}
static unsigned int a3xx_vbif_unlock(struct adreno_device *adreno_dev,
unsigned int *cmds)
{
unsigned int *start = cmds;
/* MMU-500 VBIF unstall */
*cmds++ = cp_packet(adreno_dev, CP_REG_RMW, 3);
*cmds++ = A3XX_VBIF_DDR_OUTPUT_RECOVERABLE_HALT_CTRL0;
/* AND to unmask the HALT bit */
*cmds++ = ~(VBIF_RECOVERABLE_HALT_CTRL);
/* OR to reset the HALT bit */
*cmds++ = 0;
/* release all commands since _vbif_lock() with wait_for_me */
cmds += cp_wait_for_me(adreno_dev, cmds);
return cmds - start;
}
#define A3XX_GPU_OFFSET 0xa000
/* This function is only needed for A3xx targets */
static unsigned int a3xx_cp_smmu_reg(struct adreno_device *adreno_dev,
unsigned int *cmds,
enum kgsl_iommu_reg_map reg,
unsigned int num)
{
unsigned int *start = cmds;
unsigned int offset = (A3XX_GPU_OFFSET + kgsl_iommu_reg_list[reg]) >> 2;
*cmds++ = cp_packet(adreno_dev, CP_REG_WR_NO_CTXT, num + 1);
*cmds++ = offset;
return cmds - start;
}
/* This function is only needed for A3xx targets */
static unsigned int a3xx_tlbiall(struct adreno_device *adreno_dev,
unsigned int *cmds)
{
unsigned int *start = cmds;
unsigned int tlbstatus = (A3XX_GPU_OFFSET +
kgsl_iommu_reg_list[KGSL_IOMMU_CTX_TLBSTATUS]) >> 2;
cmds += a3xx_cp_smmu_reg(adreno_dev, cmds, KGSL_IOMMU_CTX_TLBIALL, 1);
*cmds++ = 1;
cmds += a3xx_cp_smmu_reg(adreno_dev, cmds, KGSL_IOMMU_CTX_TLBSYNC, 1);
*cmds++ = 0;
cmds += a3xx_wait_reg(adreno_dev, cmds, tlbstatus, 0,
KGSL_IOMMU_CTX_TLBSTATUS_SACTIVE, 0xF);
return cmds - start;
}
/**
* _adreno_iommu_add_idle_cmds - Add pm4 packets for GPU idle
* @adreno_dev - Pointer to device structure
* @cmds - Pointer to memory where idle commands need to be added
*/
static inline int _adreno_iommu_add_idle_cmds(struct adreno_device *adreno_dev,
unsigned int *cmds)
{
unsigned int *start = cmds;
cmds += cp_wait_for_idle(adreno_dev, cmds);
if (adreno_is_a3xx(adreno_dev))
cmds += cp_wait_for_me(adreno_dev, cmds);
return cmds - start;
}
/**
* adreno_iommu_set_apriv() - Generate commands to set/reset the APRIV
* @adreno_dev: Device on which the commands will execute
* @cmds: The memory pointer where commands are generated
* @set: If set then APRIV is set else reset
*
* Returns the number of commands generated
*/
static unsigned int adreno_iommu_set_apriv(struct adreno_device *adreno_dev,
unsigned int *cmds, int set)
{
unsigned int *cmds_orig = cmds;
/* adreno 3xx doesn't have the CP_CNTL.APRIV field */
if (adreno_is_a3xx(adreno_dev))
return 0;
/* Targets with apriv control do not need to explicitly set the bit */
if (ADRENO_FEATURE(adreno_dev, ADRENO_APRIV))
return 0;
cmds += cp_wait_for_idle(adreno_dev, cmds);
cmds += cp_wait_for_me(adreno_dev, cmds);
*cmds++ = cp_register(adreno_dev, adreno_getreg(adreno_dev,
ADRENO_REG_CP_CNTL), 1);
if (set)
*cmds++ = 1;
else
*cmds++ = 0;
return cmds - cmds_orig;
}
static inline int _adreno_iommu_add_idle_indirect_cmds(
struct adreno_device *adreno_dev,
unsigned int *cmds, uint64_t nop_gpuaddr)
{
unsigned int *start = cmds;
/*
* Adding an indirect buffer ensures that the prefetch stalls until
* the commands in indirect buffer have completed. We need to stall
* prefetch with a nop indirect buffer when updating pagetables
* because it provides stabler synchronization.
*/
cmds += cp_wait_for_me(adreno_dev, cmds);
*cmds++ = cp_mem_packet(adreno_dev, CP_INDIRECT_BUFFER_PFE, 2, 1);
cmds += cp_gpuaddr(adreno_dev, cmds, nop_gpuaddr);
*cmds++ = 2;
cmds += cp_wait_for_idle(adreno_dev, cmds);
return cmds - start;
}
static unsigned int _adreno_iommu_set_pt_v2_a3xx(struct kgsl_device *device,
unsigned int *cmds_orig,
u64 ttbr0, u32 contextidr)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int *cmds = cmds_orig;
cmds += _adreno_iommu_add_idle_cmds(adreno_dev, cmds);
cmds += a3xx_vbif_lock(adreno_dev, cmds);
cmds += a3xx_cp_smmu_reg(adreno_dev, cmds, KGSL_IOMMU_CTX_TTBR0, 2);
*cmds++ = lower_32_bits(ttbr0);
*cmds++ = upper_32_bits(ttbr0);
cmds += a3xx_cp_smmu_reg(adreno_dev, cmds, KGSL_IOMMU_CTX_CONTEXTIDR,
1);
*cmds++ = contextidr;
cmds += a3xx_vbif_unlock(adreno_dev, cmds);
cmds += a3xx_tlbiall(adreno_dev, cmds);
/* wait for me to finish the TLBI */
cmds += cp_wait_for_me(adreno_dev, cmds);
cmds += _adreno_iommu_add_idle_cmds(adreno_dev, cmds);
return cmds - cmds_orig;
}
static unsigned int _adreno_iommu_set_pt_v2_a5xx(struct kgsl_device *device,
unsigned int *cmds_orig,
u64 ttbr0, u32 contextidr,
struct adreno_ringbuffer *rb)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int *cmds = cmds_orig;
cmds += _adreno_iommu_add_idle_cmds(adreno_dev, cmds);
cmds += cp_wait_for_me(adreno_dev, cmds);
/* CP switches the pagetable and flushes the Caches */
*cmds++ = cp_packet(adreno_dev, CP_SMMU_TABLE_UPDATE, 3);
*cmds++ = lower_32_bits(ttbr0);
*cmds++ = upper_32_bits(ttbr0);
*cmds++ = contextidr;
*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 4, 1);
cmds += cp_gpuaddr(adreno_dev, cmds, (rb->pagetable_desc.gpuaddr +
PT_INFO_OFFSET(ttbr0)));
*cmds++ = lower_32_bits(ttbr0);
*cmds++ = upper_32_bits(ttbr0);
*cmds++ = contextidr;
/* release all commands with wait_for_me */
cmds += cp_wait_for_me(adreno_dev, cmds);
cmds += _adreno_iommu_add_idle_cmds(adreno_dev, cmds);
return cmds - cmds_orig;
}
static unsigned int _adreno_iommu_set_pt_v2_a6xx(struct kgsl_device *device,
unsigned int *cmds_orig,
u64 ttbr0, u32 contextidr,
struct adreno_ringbuffer *rb,
unsigned int cb_num)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int *cmds = cmds_orig;
cmds += _adreno_iommu_add_idle_cmds(adreno_dev, cmds);
cmds += cp_wait_for_me(adreno_dev, cmds);
/* CP switches the pagetable and flushes the Caches */
*cmds++ = cp_packet(adreno_dev, CP_SMMU_TABLE_UPDATE, 4);
*cmds++ = lower_32_bits(ttbr0);
*cmds++ = upper_32_bits(ttbr0);
*cmds++ = contextidr;
*cmds++ = cb_num;
*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 4, 1);
cmds += cp_gpuaddr(adreno_dev, cmds, (rb->pagetable_desc.gpuaddr +
PT_INFO_OFFSET(ttbr0)));
*cmds++ = lower_32_bits(ttbr0);
*cmds++ = upper_32_bits(ttbr0);
*cmds++ = contextidr;
/* release all commands with wait_for_me */
cmds += cp_wait_for_me(adreno_dev, cmds);
cmds += _adreno_iommu_add_idle_cmds(adreno_dev, cmds);
return cmds - cmds_orig;
}
/**
* adreno_iommu_set_pt_generate_cmds() - Generate commands to change pagetable
* @rb: The RB pointer in which these commaands are to be submitted
* @cmds: The pointer where the commands are placed
* @pt: The pagetable to switch to
*/
unsigned int adreno_iommu_set_pt_generate_cmds(
struct adreno_ringbuffer *rb,
unsigned int *cmds,
struct kgsl_pagetable *pt)
{
struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
struct kgsl_iommu_context *ctx = &iommu->ctx[KGSL_IOMMU_CONTEXT_USER];
u64 ttbr0;
u32 contextidr;
unsigned int *cmds_orig = cmds;
ttbr0 = kgsl_mmu_pagetable_get_ttbr0(pt);
contextidr = kgsl_mmu_pagetable_get_contextidr(pt);
cmds += adreno_iommu_set_apriv(adreno_dev, cmds, 1);
cmds += _adreno_iommu_add_idle_indirect_cmds(adreno_dev, cmds,
iommu->setstate.gpuaddr + KGSL_IOMMU_SETSTATE_NOP_OFFSET);
if (adreno_is_a6xx(adreno_dev))
cmds += _adreno_iommu_set_pt_v2_a6xx(device, cmds,
ttbr0, contextidr, rb,
ctx->cb_num);
else if (adreno_is_a5xx(adreno_dev))
cmds += _adreno_iommu_set_pt_v2_a5xx(device, cmds,
ttbr0, contextidr, rb);
else if (adreno_is_a3xx(adreno_dev))
cmds += _adreno_iommu_set_pt_v2_a3xx(device, cmds,
ttbr0, contextidr);
/* invalidate all base pointers */
cmds += cp_invalidate_state(adreno_dev, cmds);
cmds += adreno_iommu_set_apriv(adreno_dev, cmds, 0);
return cmds - cmds_orig;
}
/**
* __add_curr_ctxt_cmds() - Add commands to set a context id in memstore
* @rb: The RB in which the commands will be added for execution
* @cmds: Pointer to memory where commands are added
* @drawctxt: The context whose id is being set in memstore
*
* Returns the number of dwords
*/
static unsigned int __add_curr_ctxt_cmds(struct adreno_ringbuffer *rb,
unsigned int *cmds,
struct adreno_context *drawctxt)
{
struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
unsigned int *cmds_orig = cmds;
/* write the context identifier to memstore memory */
cmds += cp_identifier(adreno_dev, cmds, CONTEXT_TO_MEM_IDENTIFIER);
*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1);
cmds += cp_gpuaddr(adreno_dev, cmds,
MEMSTORE_RB_GPU_ADDR(device, rb, current_context));
*cmds++ = (drawctxt ? drawctxt->base.id : 0);
*cmds++ = cp_mem_packet(adreno_dev, CP_MEM_WRITE, 2, 1);
cmds += cp_gpuaddr(adreno_dev, cmds,
MEMSTORE_ID_GPU_ADDR(device,
KGSL_MEMSTORE_GLOBAL, current_context));
*cmds++ = (drawctxt ? drawctxt->base.id : 0);
/* Invalidate UCHE for new context */
if (adreno_is_a6xx(adreno_dev)) {
*cmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1);
*cmds++ = 0x31; /* CACHE_INVALIDATE */
} else if (adreno_is_a5xx(adreno_dev)) {
*cmds++ = cp_register(adreno_dev,
adreno_getreg(adreno_dev,
ADRENO_REG_UCHE_INVALIDATE0), 1);
*cmds++ = 0x12;
} else if (adreno_is_a3xx(adreno_dev)) {
*cmds++ = cp_register(adreno_dev,
adreno_getreg(adreno_dev,
ADRENO_REG_UCHE_INVALIDATE0), 2);
*cmds++ = 0;
*cmds++ = 0x90000000;
} else
WARN_ONCE(1, "GPU UCHE invalidate sequence not defined\n");
return cmds - cmds_orig;
}
/**
* _set_ctxt_gpu() - Add commands to set the current context in memstore
* @rb: The ringbuffer in which commands to set memstore are added
* @drawctxt: The context whose id is being set in memstore
*/
static int _set_ctxt_gpu(struct adreno_ringbuffer *rb,
struct adreno_context *drawctxt)
{
unsigned int link[15], *cmds;
int result;
cmds = &link[0];
cmds += __add_curr_ctxt_cmds(rb, cmds, drawctxt);
result = adreno_ringbuffer_issue_internal_cmds(rb, 0, link,
(unsigned int)(cmds - link));
return result;
}
/**
* _set_pagetable_gpu() - Use GPU to switch the pagetable
* @rb: The rb in which commands to switch pagetable are to be
* submitted
* @new_pt: The pagetable to switch to
*/
static int _set_pagetable_gpu(struct adreno_ringbuffer *rb,
struct kgsl_pagetable *new_pt)
{
struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb);
unsigned int *link = NULL, count;
int result;
link = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (link == NULL)
return -ENOMEM;
/* If we are in a fault the MMU will be reset soon */
if (test_bit(ADRENO_DEVICE_FAULT, &adreno_dev->priv)) {
kfree(link);
return 0;
}
count = adreno_iommu_set_pt_generate_cmds(rb, link, new_pt);
WARN(count > (PAGE_SIZE / sizeof(unsigned int)),
"Temp command buffer overflow\n");
/*
* This returns the per context timestamp but we need to
* use the global timestamp for iommu clock disablement
*/
result = adreno_ringbuffer_issue_internal_cmds(rb,
KGSL_CMD_FLAGS_PMODE, link, count);
kfree(link);
return result;
}
/**
* adreno_iommu_init() - Adreno iommu init
* @adreno_dev: Adreno device
*/
void adreno_iommu_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_iommu *iommu = KGSL_IOMMU_PRIV(device);
if (kgsl_mmu_get_mmutype(device) == KGSL_MMU_TYPE_NONE)
return;
/*
* A nop is required in an indirect buffer when switching
* pagetables in-stream
*/
kgsl_sharedmem_writel(device, &iommu->setstate,
KGSL_IOMMU_SETSTATE_NOP_OFFSET,
cp_packet(adreno_dev, CP_NOP, 1));
/* Enable guard page MMU feature for A3xx and A4xx targets only */
if (adreno_is_a3xx(adreno_dev))
device->mmu.features |= KGSL_MMU_NEED_GUARD_PAGE;
}
/**
* adreno_iommu_set_pt_ctx() - Change the pagetable of the current RB
* @device: Pointer to device to which the rb belongs
* @rb: The RB pointer on which pagetable is to be changed
* @new_pt: The new pt the device will change to
* @drawctxt: The context whose pagetable the ringbuffer is switching to,
* NULL means KGSL_CONTEXT_GLOBAL
*
* Returns 0 on success else error code.
*/
int adreno_iommu_set_pt_ctx(struct adreno_ringbuffer *rb,
struct kgsl_pagetable *new_pt,
struct adreno_context *drawctxt)
{
struct adreno_device *adreno_dev = ADRENO_RB_DEVICE(rb);
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_pagetable *cur_pt = device->mmu.defaultpagetable;
int result = 0;
/* Switch the page table if a MMU is attached */
if (kgsl_mmu_get_mmutype(device) != KGSL_MMU_TYPE_NONE) {
if (rb->drawctxt_active)
cur_pt = rb->drawctxt_active->base.proc_priv->pagetable;
/* Pagetable switch */
if (new_pt != cur_pt)
result = _set_pagetable_gpu(rb, new_pt);
if (result)
return result;
}
/* Context switch */
return _set_ctxt_gpu(rb, drawctxt);
}

39
drivers/gpu/msm/adreno_iommu.h Normal file
View file

@ -0,0 +1,39 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2015-2016,2019 The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_IOMMU_H
#define __ADRENO_IOMMU_H
#if IS_ENABLED(CONFIG_ARM_SMMU)
int adreno_iommu_set_pt_ctx(struct adreno_ringbuffer *rb,
struct kgsl_pagetable *new_pt,
struct adreno_context *drawctxt);
void adreno_iommu_init(struct adreno_device *adreno_dev);
unsigned int adreno_iommu_set_pt_generate_cmds(
struct adreno_ringbuffer *rb,
unsigned int *cmds,
struct kgsl_pagetable *pt);
#else
static inline void adreno_iommu_init(struct adreno_device *adreno_dev) { }
static inline int adreno_iommu_set_pt_ctx(struct adreno_ringbuffer *rb,
struct kgsl_pagetable *new_pt,
struct adreno_context *drawctxt)
{
return 0;
}
static inline unsigned int adreno_iommu_set_pt_generate_cmds(
struct adreno_ringbuffer *rb,
unsigned int *cmds,
struct kgsl_pagetable *pt)
{
return 0;
}
#endif
#endif

1071
drivers/gpu/msm/adreno_perfcounter.c Normal file
View file

File diff suppressed because it is too large Load diff

120
drivers/gpu/msm/adreno_perfcounter.h Normal file
View file

@ -0,0 +1,120 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2008-2015,2017,2019 The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_PERFCOUNTER_H
#define __ADRENO_PERFCOUNTER_H
struct adreno_device;
/* ADRENO_PERFCOUNTERS - Given an adreno device, return the perfcounters list */
#define ADRENO_PERFCOUNTERS(_a) \
(ADRENO_GPU_DEVICE(_a) ? ADRENO_GPU_DEVICE(_a)->perfcounters : NULL)
#define PERFCOUNTER_FLAG_NONE 0x0
#define PERFCOUNTER_FLAG_KERNEL 0x1
/* Structs to maintain the list of active performance counters */
/**
* struct adreno_perfcount_register: register state
* @countable: countable the register holds
* @kernelcount: number of user space users of the register
* @usercount: number of kernel users of the register
* @offset: register hardware offset
* @load_bit: The bit number in LOAD register which corresponds to this counter
* @select: The countable register offset
* @value: The 64 bit countable register value
*/
struct adreno_perfcount_register {
unsigned int countable;
unsigned int kernelcount;
unsigned int usercount;
unsigned int offset;
unsigned int offset_hi;
int load_bit;
unsigned int select;
uint64_t value;
};
/**
* struct adreno_perfcount_group: registers for a hardware group
* @regs: available registers for this group
* @reg_count: total registers for this group
* @name: group name for this group
*/
struct adreno_perfcount_group {
struct adreno_perfcount_register *regs;
unsigned int reg_count;
const char *name;
unsigned long flags;
};
/*
* ADRENO_PERFCOUNTER_GROUP_FIXED indicates that a perfcounter group is fixed -
* instead of having configurable countables like the other groups, registers in
* fixed groups have a hardwired countable. So when the user requests a
* countable in one of these groups, that countable should be used as the
* register offset to return
*/
#define ADRENO_PERFCOUNTER_GROUP_FIXED BIT(0)
/*
* ADRENO_PERFCOUNTER_GROUP_RESTORE indicates CP needs to restore the select
* registers of this perfcounter group as part of preemption and IFPC
*/
#define ADRENO_PERFCOUNTER_GROUP_RESTORE BIT(1)
/**
* adreno_perfcounts: all available perfcounter groups
* @groups: available groups for this device
* @group_count: total groups for this device
*/
struct adreno_perfcounters {
struct adreno_perfcount_group *groups;
unsigned int group_count;
};
#define ADRENO_PERFCOUNTER_GROUP_FLAGS(core, offset, name, flags) \
[KGSL_PERFCOUNTER_GROUP_##offset] = { core##_perfcounters_##name, \
ARRAY_SIZE(core##_perfcounters_##name), __stringify(name), flags }
#define ADRENO_PERFCOUNTER_GROUP(core, offset, name) \
ADRENO_PERFCOUNTER_GROUP_FLAGS(core, offset, name, 0)
#define ADRENO_POWER_COUNTER_GROUP(core, offset, name) \
[KGSL_PERFCOUNTER_GROUP_##offset##_PWR] = { core##_pwrcounters_##name, \
ARRAY_SIZE(core##_pwrcounters_##name), __stringify(name##_pwr), 0}
int adreno_perfcounter_query_group(struct adreno_device *adreno_dev,
unsigned int groupid, unsigned int __user *countables,
unsigned int count, unsigned int *max_counters);
int adreno_perfcounter_read_group(struct adreno_device *adreno_dev,
struct kgsl_perfcounter_read_group __user *reads, unsigned int count);
void adreno_perfcounter_restore(struct adreno_device *adreno_dev);
void adreno_perfcounter_save(struct adreno_device *adreno_dev);
void adreno_perfcounter_start(struct adreno_device *adreno_dev);
int adreno_perfcounter_get_groupid(struct adreno_device *adreno_dev,
const char *name);
uint64_t adreno_perfcounter_read(struct adreno_device *adreno_dev,
unsigned int group, unsigned int counter);
const char *adreno_perfcounter_get_name(struct adreno_device
*adreno_dev, unsigned int groupid);
int adreno_perfcounter_get(struct adreno_device *adreno_dev,
unsigned int groupid, unsigned int countable, unsigned int *offset,
unsigned int *offset_hi, unsigned int flags);
int adreno_perfcounter_put(struct adreno_device *adreno_dev,
unsigned int groupid, unsigned int countable, unsigned int flags);
#endif /* __ADRENO_PERFCOUNTER_H */

405
drivers/gpu/msm/adreno_pm4types.h Normal file
View file

@ -0,0 +1,405 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_PM4TYPES_H
#define __ADRENO_PM4TYPES_H
#include "adreno.h"
#define CP_TYPE0_PKT (0 << 30)
#define CP_TYPE3_PKT (3 << 30)
#define CP_TYPE4_PKT (4 << 28)
#define CP_TYPE7_PKT (7 << 28)
#define PM4_TYPE4_PKT_SIZE_MAX 128
/* type3 packets */
/* Enable preemption flag */
#define CP_PREEMPT_ENABLE 0x1C
/* Preemption token command on which preemption occurs */
#define CP_PREEMPT_TOKEN 0x1E
/* Bit to set in CP_PREEMPT_TOKEN ordinal for interrupt on preemption */
#define CP_PREEMPT_ORDINAL_INTERRUPT 24
/* Wait for memory writes to complete */
#define CP_WAIT_MEM_WRITES 0x12
/* initialize CP's micro-engine */
#define CP_ME_INIT 0x48
/* skip N 32-bit words to get to the next packet */
#define CP_NOP 0x10
/* indirect buffer dispatch. same as IB, but init is pipelined */
#define CP_INDIRECT_BUFFER_PFD 0x37
/* wait for the IDLE state of the engine */
#define CP_WAIT_FOR_IDLE 0x26
/* wait until a register or memory location is a specific value */
#define CP_WAIT_REG_MEM 0x3c
/* wait until a register location is equal to a specific value */
#define CP_WAIT_REG_EQ 0x52
/* switches SMMU pagetable, used on a5xx only */
#define CP_SMMU_TABLE_UPDATE 0x53
/* Set internal CP registers, used to indicate context save data addresses */
#define CP_SET_PSEUDO_REGISTER 0x56
/* Tell CP the current operation mode, indicates save and restore procedure */
#define CP_SET_MARKER 0x65
/* register read/modify/write */
#define CP_REG_RMW 0x21
/* Set binning configuration registers */
#define CP_SET_BIN_DATA 0x2f
/* reads register in chip and writes to memory */
#define CP_REG_TO_MEM 0x3e
/* write N 32-bit words to memory */
#define CP_MEM_WRITE 0x3d
/* conditional execution of a sequence of packets */
#define CP_COND_EXEC 0x44
/* conditional write to memory or register */
#define CP_COND_WRITE 0x45
/* generate an event that creates a write to memory when completed */
#define CP_EVENT_WRITE 0x46
/* initiate fetch of index buffer and draw */
#define CP_DRAW_INDX 0x22
/* New draw packets defined for A4XX */
#define CP_DRAW_INDX_OFFSET 0x38
#define CP_DRAW_INDIRECT 0x28
#define CP_DRAW_INDX_INDIRECT 0x29
#define CP_DRAW_AUTO 0x24
/* load constant into chip and to memory */
#define CP_SET_CONSTANT 0x2d
/* selective invalidation of state pointers */
#define CP_INVALIDATE_STATE 0x3b
/* generate interrupt from the command stream */
#define CP_INTERRUPT 0x40
/* A5XX Enable yield in RB only */
#define CP_YIELD_ENABLE 0x1C
/* Enable/Disable/Defer A5x global preemption model */
#define CP_PREEMPT_ENABLE_GLOBAL 0x69
/* Enable/Disable A5x local preemption model */
#define CP_PREEMPT_ENABLE_LOCAL 0x6A
/* Yeild token on a5xx similar to CP_PREEMPT on a4xx */
#define CP_CONTEXT_SWITCH_YIELD 0x6B
/* Inform CP about current render mode (needed for a5xx preemption) */
#define CP_SET_RENDER_MODE 0x6C
/* Write register, ignoring context state for context sensitive registers */
#define CP_REG_WR_NO_CTXT 0x78
/*
* for A4xx
* Write to register with address that does not fit into type-0 pkt
*/
#define CP_WIDE_REG_WRITE 0x74
/* PFP waits until the FIFO between the PFP and the ME is empty */
#define CP_WAIT_FOR_ME 0x13
#define CP_SET_PROTECTED_MODE 0x5f /* sets the register protection mode */
/* Used to switch GPU between secure and non-secure modes */
#define CP_SET_SECURE_MODE 0x66
#define CP_BOOTSTRAP_UCODE 0x6f /* bootstraps microcode */
/*
* for a3xx
*/
#define CP_LOAD_STATE 0x30 /* load high level sequencer command */
/* Conditionally load a IB based on a flag */
#define CP_COND_INDIRECT_BUFFER_PFE 0x3A /* prefetch enabled */
#define CP_COND_INDIRECT_BUFFER_PFD 0x32 /* prefetch disabled */
/* Load a buffer with pre-fetch enabled */
#define CP_INDIRECT_BUFFER_PFE 0x3F
#define CP_EXEC_CL 0x31
/* (A4x) save PM4 stream pointers to execute upon a visible draw */
#define CP_SET_DRAW_STATE 0x43
#define CP_LOADSTATE_DSTOFFSET_SHIFT 0x00000000
#define CP_LOADSTATE_STATESRC_SHIFT 0x00000010
#define CP_LOADSTATE_STATEBLOCKID_SHIFT 0x00000013
#define CP_LOADSTATE_NUMOFUNITS_SHIFT 0x00000016
#define CP_LOADSTATE_STATETYPE_SHIFT 0x00000000
#define CP_LOADSTATE_EXTSRCADDR_SHIFT 0x00000002
static inline uint pm4_calc_odd_parity_bit(uint val)
{
return (0x9669 >> (0xf & ((val) ^
((val) >> 4) ^ ((val) >> 8) ^ ((val) >> 12) ^
((val) >> 16) ^ ((val) >> 20) ^ ((val) >> 24) ^
((val) >> 28)))) & 1;
}
/*
* PM4 packet header functions
* For all the packet functions the passed in count should be the size of the
* payload excluding the header
*/
static inline uint cp_type0_packet(uint regindx, uint cnt)
{
return CP_TYPE0_PKT | ((cnt-1) << 16) | ((regindx) & 0x7FFF);
}
static inline uint cp_type3_packet(uint opcode, uint cnt)
{
return CP_TYPE3_PKT | ((cnt-1) << 16) | (((opcode) & 0xFF) << 8);
}
static inline uint cp_type4_packet(uint opcode, uint cnt)
{
return CP_TYPE4_PKT | ((cnt) << 0) |
(pm4_calc_odd_parity_bit(cnt) << 7) |
(((opcode) & 0x3FFFF) << 8) |
((pm4_calc_odd_parity_bit(opcode) << 27));
}
static inline uint cp_type7_packet(uint opcode, uint cnt)
{
return CP_TYPE7_PKT | ((cnt) << 0) |
(pm4_calc_odd_parity_bit(cnt) << 15) |
(((opcode) & 0x7F) << 16) |
((pm4_calc_odd_parity_bit(opcode) << 23));
}
#define pkt_is_type0(pkt) (((pkt) & 0XC0000000) == CP_TYPE0_PKT)
#define type0_pkt_size(pkt) ((((pkt) >> 16) & 0x3FFF) + 1)
#define type0_pkt_offset(pkt) ((pkt) & 0x7FFF)
/*
* Check both for the type3 opcode and make sure that the reserved bits [1:7]
* and 15 are 0
*/
#define pkt_is_type3(pkt) \
((((pkt) & 0xC0000000) == CP_TYPE3_PKT) && \
(((pkt) & 0x80FE) == 0))
#define cp_type3_opcode(pkt) (((pkt) >> 8) & 0xFF)
#define type3_pkt_size(pkt) ((((pkt) >> 16) & 0x3FFF) + 1)
#define pkt_is_type4(pkt) \
((((pkt) & 0xF0000000) == CP_TYPE4_PKT) && \
((((pkt) >> 27) & 0x1) == \
pm4_calc_odd_parity_bit(cp_type4_base_index_one_reg_wr(pkt))) \
&& ((((pkt) >> 7) & 0x1) == \
pm4_calc_odd_parity_bit(type4_pkt_size(pkt))))
#define cp_type4_base_index_one_reg_wr(pkt) (((pkt) >> 8) & 0x7FFFF)
#define type4_pkt_size(pkt) ((pkt) & 0x7F)
#define pkt_is_type7(pkt) \
((((pkt) & 0xF0000000) == CP_TYPE7_PKT) && \
(((pkt) & 0x0F000000) == 0) && \
((((pkt) >> 23) & 0x1) == \
pm4_calc_odd_parity_bit(cp_type7_opcode(pkt))) \
&& ((((pkt) >> 15) & 0x1) == \
pm4_calc_odd_parity_bit(type7_pkt_size(pkt))))
#define cp_type7_opcode(pkt) (((pkt) >> 16) & 0x7F)
#define type7_pkt_size(pkt) ((pkt) & 0x3FFF)
/* dword base address of the GFX decode space */
#define SUBBLOCK_OFFSET(reg) ((unsigned int)((reg) - (0x2000)))
/* gmem command buffer length */
#define CP_REG(reg) ((0x4 << 16) | (SUBBLOCK_OFFSET(reg)))
/* Return true if the hardware uses the legacy (A4XX and older) PM4 format */
#define ADRENO_LEGACY_PM4(_d) (ADRENO_GPUREV(_d) < 500)
/**
* cp_packet - Generic CP packet to support different opcodes on
* different GPU cores.
* @adreno_dev: The adreno device
* @opcode: Operation for cp packet
* @size: size for cp packet
*/
static inline uint cp_packet(struct adreno_device *adreno_dev,
int opcode, uint size)
{
if (ADRENO_LEGACY_PM4(adreno_dev))
return cp_type3_packet(opcode, size);
return cp_type7_packet(opcode, size);
}
/**
* cp_mem_packet - Generic CP memory packet to support different
* opcodes on different GPU cores.
* @adreno_dev: The adreno device
* @opcode: mem operation for cp packet
* @size: size for cp packet
* @num_mem: num of mem access
*/
static inline uint cp_mem_packet(struct adreno_device *adreno_dev,
int opcode, uint size, uint num_mem)
{
if (ADRENO_LEGACY_PM4(adreno_dev))
return cp_type3_packet(opcode, size);
return cp_type7_packet(opcode, size + num_mem);
}
/* Return 1 if the command is an indirect buffer of any kind */
static inline int adreno_cmd_is_ib(struct adreno_device *adreno_dev,
unsigned int cmd)
{
return cmd == cp_mem_packet(adreno_dev,
CP_INDIRECT_BUFFER_PFE, 2, 1) ||
cmd == cp_mem_packet(adreno_dev,
CP_INDIRECT_BUFFER_PFD, 2, 1) ||
cmd == cp_mem_packet(adreno_dev,
CP_COND_INDIRECT_BUFFER_PFE, 2, 1) ||
cmd == cp_mem_packet(adreno_dev,
CP_COND_INDIRECT_BUFFER_PFD, 2, 1);
}
/**
* cp_gpuaddr - Generic function to add 64bit and 32bit gpuaddr
* to pm4 commands
* @adreno_dev: The adreno device
* @cmds: command pointer to add gpuaddr
* @gpuaddr: gpuaddr to add
*/
static inline uint cp_gpuaddr(struct adreno_device *adreno_dev,
uint *cmds, uint64_t gpuaddr)
{
uint *start = cmds;
if (ADRENO_LEGACY_PM4(adreno_dev))
*cmds++ = (uint)gpuaddr;
else {
*cmds++ = lower_32_bits(gpuaddr);
*cmds++ = upper_32_bits(gpuaddr);
}
return cmds - start;
}
/**
* cp_register - Generic function for gpu register operation
* @adreno_dev: The adreno device
* @reg: GPU register
* @size: count for PM4 operation
*/
static inline uint cp_register(struct adreno_device *adreno_dev,
unsigned int reg, unsigned int size)
{
if (ADRENO_LEGACY_PM4(adreno_dev))
return cp_type0_packet(reg, size);
return cp_type4_packet(reg, size);
}
/**
* cp_wait_for_me - common function for WAIT_FOR_ME
* @adreno_dev: The adreno device
* @cmds: command pointer to add gpuaddr
*/
static inline uint cp_wait_for_me(struct adreno_device *adreno_dev,
uint *cmds)
{
uint *start = cmds;
if (ADRENO_LEGACY_PM4(adreno_dev)) {
*cmds++ = cp_type3_packet(CP_WAIT_FOR_ME, 1);
*cmds++ = 0;
} else
*cmds++ = cp_type7_packet(CP_WAIT_FOR_ME, 0);
return cmds - start;
}
/**
* cp_wait_for_idle - common function for WAIT_FOR_IDLE
* @adreno_dev: The adreno device
* @cmds: command pointer to add gpuaddr
*/
static inline uint cp_wait_for_idle(struct adreno_device *adreno_dev,
uint *cmds)
{
uint *start = cmds;
if (ADRENO_LEGACY_PM4(adreno_dev)) {
*cmds++ = cp_type3_packet(CP_WAIT_FOR_IDLE, 1);
*cmds++ = 0;
} else
*cmds++ = cp_type7_packet(CP_WAIT_FOR_IDLE, 0);
return cmds - start;
}
/**
* cp_invalidate_state - common function for invalidating cp
* state
* @adreno_dev: The adreno device
* @cmds: command pointer to add gpuaddr
*/
static inline uint cp_invalidate_state(struct adreno_device *adreno_dev,
uint *cmds)
{
uint *start = cmds;
if (ADRENO_GPUREV(adreno_dev) < 500) {
*cmds++ = cp_type3_packet(CP_INVALIDATE_STATE, 1);
*cmds++ = 0x7fff;
} else {
*cmds++ = cp_type7_packet(CP_SET_DRAW_STATE, 3);
*cmds++ = 0x40000;
*cmds++ = 0;
*cmds++ = 0;
}
return cmds - start;
}
static inline u32 cp_protected_mode(struct adreno_device *adreno_dev,
u32 *cmds, int on)
{
cmds[0] = cp_packet(adreno_dev, CP_SET_PROTECTED_MODE, 1);
cmds[1] = on;
return 2;
}
static inline u32 cp_identifier(struct adreno_device *adreno_dev,
u32 *cmds, u32 id)
{
cmds[0] = cp_packet(adreno_dev, CP_NOP, 1);
cmds[1] = id;
return 2;
}
#endif /* __ADRENO_PM4TYPES_H */

1183
drivers/gpu/msm/adreno_profile.c Normal file
View file

File diff suppressed because it is too large Load diff

102
drivers/gpu/msm/adreno_profile.h Normal file
View file

@ -0,0 +1,102 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2013-2014,2019 The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_PROFILE_H
#define __ADRENO_PROFILE_H
/**
* struct adreno_profile_assigns_list: linked list for assigned perf counters
* @list: linkage for nodes in list
* @name: group name or GPU name name
* @groupid: group id
* @countable: countable assigned to perfcounter
* @offset: perfcounter register address offset
*/
struct adreno_profile_assigns_list {
struct list_head list;
char name[25];
unsigned int groupid;
unsigned int countable;
unsigned int offset; /* LO offset */
unsigned int offset_hi; /* HI offset */
};
struct adreno_profile {
struct list_head assignments_list; /* list of all assignments */
unsigned int assignment_count; /* Number of assigned counters */
unsigned int *log_buffer;
unsigned int *log_head;
unsigned int *log_tail;
bool enabled;
/* counter, pre_ib, and post_ib held in one large circular buffer
* shared between kgsl and GPU
* counter entry 0
* pre_ib entry 0
* post_ib entry 0
* ...
* counter entry N
* pre_ib entry N
* post_ib entry N
*/
struct kgsl_memdesc shared_buffer;
unsigned int shared_head;
unsigned int shared_tail;
unsigned int shared_size;
};
#define ADRENO_PROFILE_SHARED_BUF_SIZE_DWORDS (48 * 4096 / sizeof(uint))
/* sized @ 48 pages should allow for over 50 outstanding IBs minimum, 1755 max*/
#define ADRENO_PROFILE_LOG_BUF_SIZE (1024 * 920)
/* sized for 1024 entries of fully assigned 45 cnters in log buffer, 230 pages*/
#define ADRENO_PROFILE_LOG_BUF_SIZE_DWORDS (ADRENO_PROFILE_LOG_BUF_SIZE / \
sizeof(unsigned int))
#ifdef CONFIG_DEBUG_FS
void adreno_profile_init(struct adreno_device *adreno_dev);
void adreno_profile_close(struct adreno_device *adreno_dev);
int adreno_profile_process_results(struct adreno_device *adreno_dev);
void adreno_profile_preib_processing(struct adreno_device *adreno_dev,
struct adreno_context *drawctxt, unsigned int *cmd_flags,
unsigned int **rbptr);
void adreno_profile_postib_processing(struct adreno_device *adreno_dev,
unsigned int *cmd_flags, unsigned int **rbptr);
#else
static inline void adreno_profile_init(struct adreno_device *adreno_dev) { }
static inline void adreno_profile_close(struct adreno_device *adreno_dev) { }
static inline int adreno_profile_process_results(
struct adreno_device *adreno_dev)
{
return 0;
}
static inline void adreno_profile_preib_processing(
struct adreno_device *adreno_dev,
struct adreno_context *drawctxt, unsigned int *cmd_flags,
unsigned int **rbptr) { }
static inline void adreno_profile_postib_processing(
struct adreno_device *adreno_dev,
unsigned int *cmd_flags, unsigned int **rbptr) { }
#endif
static inline bool adreno_profile_enabled(struct adreno_profile *profile)
{
return profile->enabled;
}
static inline bool adreno_profile_has_assignments(
struct adreno_profile *profile)
{
return list_empty(&profile->assignments_list) ? false : true;
}
static inline bool adreno_profile_assignments_ready(
struct adreno_profile *profile)
{
return adreno_profile_enabled(profile) &&
adreno_profile_has_assignments(profile);
}
#endif

1226
drivers/gpu/msm/adreno_ringbuffer.c Normal file
View file

File diff suppressed because it is too large Load diff

210
drivers/gpu/msm/adreno_ringbuffer.h Normal file
View file

@ -0,0 +1,210 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_RINGBUFFER_H
#define __ADRENO_RINGBUFFER_H
/* Given a ringbuffer, return the adreno device that owns it */
#define _RB_OFFSET(_id) (offsetof(struct adreno_device, ringbuffers) + \
((_id) * sizeof(struct adreno_ringbuffer)))
#define ADRENO_RB_DEVICE(_rb) \
((struct adreno_device *) (((void *) (_rb)) - _RB_OFFSET((_rb)->id)))
/* Adreno ringbuffer size in bytes */
#define KGSL_RB_SIZE (32 * 1024)
/*
* A handy macro to convert the RB size to dwords since most ringbuffer
* operations happen in dword increments
*/
#define KGSL_RB_DWORDS (KGSL_RB_SIZE >> 2)
struct kgsl_device;
struct kgsl_device_private;
/**
* struct adreno_submit_time - utility structure to store the wall clock / GPU
* ticks at command submit time
* @ticks: GPU ticks at submit time (from the 19.2Mhz timer)
* @ktime: local clock time (in nanoseconds)
* @utime: Wall clock time
* @drawobj: the object that we want to profile
*/
struct adreno_submit_time {
uint64_t ticks;
u64 ktime;
struct timespec utime;
struct kgsl_drawobj *drawobj;
};
/**
* struct adreno_ringbuffer_pagetable_info - Contains fields used during a
* pagetable switch.
* @current_global_ptname: The current pagetable id being used by the GPU.
* Only the ringbuffers[0] current_global_ptname is used to keep track of
* the current pagetable id
* @current_rb_ptname: The current pagetable active on the given RB
* @incoming_ptname: Contains the incoming pagetable we are switching to. After
* switching of pagetable this value equals current_rb_ptname.
* @switch_pt_enable: Flag used during pagetable switch to check if pt
* switch can be skipped
* @ttbr0: value to program into TTBR0 during pagetable switch.
* @contextidr: value to program into CONTEXTIDR during pagetable switch.
*/
struct adreno_ringbuffer_pagetable_info {
int current_global_ptname;
int current_rb_ptname;
int incoming_ptname;
int switch_pt_enable;
uint64_t ttbr0;
unsigned int contextidr;
};
#define PT_INFO_OFFSET(_field) \
offsetof(struct adreno_ringbuffer_pagetable_info, _field)
/**
* struct adreno_ringbuffer - Definition for an adreno ringbuffer object
* @flags: Internal control flags for the ringbuffer
* @buffer_desc: Pointer to the ringbuffer memory descriptor
* @_wptr: The next value of wptr to be written to the hardware on submit
* @wptr: Local copy of the wptr offset last written to hardware
* @last_wptr: offset of the last wptr that was written to CFF
* @rb_ctx: The context that represents a ringbuffer
* @id: Priority level of the ringbuffer, also used as an ID
* @fault_detect_ts: The last retired global timestamp read during fault detect
* @timestamp: The RB's global timestamp
* @events: A kgsl_event_group for this context - contains the list of GPU
* events
* @drawctxt_active: The last pagetable that this ringbuffer is set to
* @preemption_desc: The memory descriptor containing
* preemption info written/read by CP
* @secure_preemption_desc: The memory descriptor containing
* preemption info written/read by CP for secure contexts
* @perfcounter_save_restore_desc: Used by CP to save/restore the perfcounter
* values across preemption
* @pagetable_desc: Memory to hold information about the pagetables being used
* and the commands to switch pagetable on the RB
* @dispatch_q: The dispatcher side queue for this ringbuffer
* @ts_expire_waitq: Wait queue to wait for rb timestamp to expire
* @ts_expire_waitq: Wait q to wait for rb timestamp to expire
* @wptr_preempt_end: Used during preemption to check that preemption occurred
* at the right rptr
* @gpr11: The gpr11 value of this RB
* @preempted_midway: Indicates that the RB was preempted before rptr = wptr
* @preempt_lock: Lock to protect the wptr pointer while it is being updated
* @skip_inline_wptr: Used during preemption to make sure wptr is updated in
* hardware
*/
struct adreno_ringbuffer {
uint32_t flags;
struct kgsl_memdesc buffer_desc;
unsigned int _wptr;
unsigned int wptr;
unsigned int last_wptr;
int id;
unsigned int fault_detect_ts;
unsigned int timestamp;
struct kgsl_event_group events;
struct adreno_context *drawctxt_active;
struct kgsl_memdesc preemption_desc;
struct kgsl_memdesc secure_preemption_desc;
struct kgsl_memdesc perfcounter_save_restore_desc;
struct kgsl_memdesc pagetable_desc;
struct adreno_dispatcher_drawqueue dispatch_q;
wait_queue_head_t ts_expire_waitq;
unsigned int wptr_preempt_end;
unsigned int gpr11;
int preempted_midway;
spinlock_t preempt_lock;
bool skip_inline_wptr;
/**
* @profile_desc: global memory to construct IB1s to do user side
* profiling
*/
struct kgsl_memdesc profile_desc;
/**
* @profile_index: Pointer to the next "slot" in profile_desc for a user
* profiling IB1. This allows for PAGE_SIZE / 16 = 256 simultaneous
* commands per ringbuffer with user profiling enabled
* enough.
*/
u32 profile_index;
};
/* Returns the current ringbuffer */
#define ADRENO_CURRENT_RINGBUFFER(a) ((a)->cur_rb)
int cp_secure_mode(struct adreno_device *adreno_dev, uint *cmds, int set);
int adreno_ringbuffer_issueibcmds(struct kgsl_device_private *dev_priv,
struct kgsl_context *context,
struct kgsl_drawobj *drawobj,
uint32_t *timestamp);
int adreno_ringbuffer_submitcmd(struct adreno_device *adreno_dev,
struct kgsl_drawobj_cmd *cmdobj,
struct adreno_submit_time *time);
int adreno_ringbuffer_probe(struct adreno_device *adreno_dev);
int adreno_ringbuffer_start(struct adreno_device *adreno_dev);
void adreno_ringbuffer_stop(struct adreno_device *adreno_dev);
void adreno_ringbuffer_close(struct adreno_device *adreno_dev);
int adreno_ringbuffer_issue_internal_cmds(struct adreno_ringbuffer *rb,
unsigned int flags,
unsigned int *cmdaddr,
int sizedwords);
void adreno_ringbuffer_submit(struct adreno_ringbuffer *rb,
struct adreno_submit_time *time);
int adreno_ringbuffer_submit_spin(struct adreno_ringbuffer *rb,
struct adreno_submit_time *time, unsigned int timeout);
void kgsl_cp_intrcallback(struct kgsl_device *device);
unsigned int *adreno_ringbuffer_allocspace(struct adreno_ringbuffer *rb,
unsigned int numcmds);
void adreno_ringbuffer_read_pfp_ucode(struct kgsl_device *device);
void adreno_ringbuffer_read_pm4_ucode(struct kgsl_device *device);
int adreno_ringbuffer_waittimestamp(struct adreno_ringbuffer *rb,
unsigned int timestamp,
unsigned int msecs);
int adreno_rb_readtimestamp(struct adreno_device *adreno_dev,
void *priv, enum kgsl_timestamp_type type,
unsigned int *timestamp);
static inline int adreno_ringbuffer_count(struct adreno_ringbuffer *rb,
unsigned int rptr)
{
if (rb->wptr >= rptr)
return rb->wptr - rptr;
return rb->wptr + KGSL_RB_DWORDS - rptr;
}
/* Increment a value by 4 bytes with wrap-around based on size */
static inline unsigned int adreno_ringbuffer_inc_wrapped(unsigned int val,
unsigned int size)
{
return (val + sizeof(unsigned int)) % size;
}
/* Decrement a value by 4 bytes with wrap-around based on size */
static inline unsigned int adreno_ringbuffer_dec_wrapped(unsigned int val,
unsigned int size)
{
return (val + size - sizeof(unsigned int)) % size;
}
#endif /* __ADRENO_RINGBUFFER_H */

1062
drivers/gpu/msm/adreno_snapshot.c Normal file
View file

File diff suppressed because it is too large Load diff

42
drivers/gpu/msm/adreno_snapshot.h Normal file
View file

@ -0,0 +1,42 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2013-2015,2019, The Linux Foundation. All rights reserved.
*/
#ifndef __ADRENO_SNAPSHOT_H
#define __ADRENO_SNAPSHOT_H
#include "kgsl_snapshot.h"
#define CP_CRASH_DUMPER_TIMEOUT 1000
#define DEBUG_SECTION_SZ(_dwords) (((_dwords) * sizeof(unsigned int)) \
+ sizeof(struct kgsl_snapshot_debug))
#define SHADER_SECTION_SZ(_dwords) (((_dwords) * sizeof(unsigned int)) \
+ sizeof(struct kgsl_snapshot_shader))
/* Section sizes for A320 */
#define A320_SNAPSHOT_CP_STATE_SECTION_SIZE 0x2e
#define A320_SNAPSHOT_ROQ_SECTION_SIZE 512
#define A320_SNAPSHOT_CP_MERCIU_SECTION_SIZE 32
/* Macro to make it super easy to dump registers */
#define SNAPSHOT_REGISTERS(_d, _s, _r) \
adreno_snapshot_registers((_d), (_s), \
(unsigned int *) _r, ARRAY_SIZE(_r) / 2)
size_t adreno_snapshot_cp_roq(struct kgsl_device *device, u8 *buf,
size_t remain, void *priv);
size_t adreno_snapshot_cp_meq(struct kgsl_device *device, u8 *buf,
size_t remain, void *priv);
void adreno_snapshot_registers(struct kgsl_device *device,
struct kgsl_snapshot *snapshot,
const unsigned int *regs, unsigned int count);
void adreno_snapshot_vbif_registers(struct kgsl_device *device,
struct kgsl_snapshot *snapshot,
const struct adreno_vbif_snapshot_registers *list,
unsigned int count);
#endif /*__ADRENO_SNAPSHOT_H */

452
drivers/gpu/msm/adreno_sysfs.c Normal file
View file

@ -0,0 +1,452 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2014-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/sysfs.h>
#include "adreno.h"
struct adreno_sysfs_attribute {
struct device_attribute attr;
unsigned int (*show)(struct adreno_device *adreno_dev);
int (*store)(struct adreno_device *adreno_dev, unsigned int val);
};
#define _ADRENO_SYSFS_ATTR(_name, __show, __store) \
struct adreno_sysfs_attribute adreno_attr_##_name = { \
.attr = __ATTR(_name, 0644, __show, __store), \
.show = _ ## _name ## _show, \
.store = _ ## _name ## _store, \
}
#define _ADRENO_SYSFS_ATTR_RO(_name, __show) \
struct adreno_sysfs_attribute adreno_attr_##_name = { \
.attr = __ATTR(_name, 0444, __show, NULL), \
.show = _ ## _name ## _show, \
.store = NULL, \
}
#define ADRENO_SYSFS_ATTR(_a) \
container_of((_a), struct adreno_sysfs_attribute, attr)
static int _ft_policy_store(struct adreno_device *adreno_dev,
unsigned int val)
{
adreno_dev->ft_policy = val & KGSL_FT_POLICY_MASK;
return 0;
}
static unsigned int _ft_policy_show(struct adreno_device *adreno_dev)
{
return adreno_dev->ft_policy;
}
static int _preempt_level_store(struct adreno_device *adreno_dev,
unsigned int val)
{
struct adreno_preemption *preempt = &adreno_dev->preempt;
if (val <= 2)
preempt->preempt_level = val;
return 0;
}
static unsigned int _preempt_level_show(struct adreno_device *adreno_dev)
{
struct adreno_preemption *preempt = &adreno_dev->preempt;
return preempt->preempt_level;
}
static int _usesgmem_store(struct adreno_device *adreno_dev,
unsigned int val)
{
struct adreno_preemption *preempt = &adreno_dev->preempt;
preempt->usesgmem = val ? 1 : 0;
return 0;
}
static unsigned int _usesgmem_show(struct adreno_device *adreno_dev)
{
struct adreno_preemption *preempt = &adreno_dev->preempt;
return preempt->usesgmem;
}
static int _skipsaverestore_store(struct adreno_device *adreno_dev,
unsigned int val)
{
struct adreno_preemption *preempt = &adreno_dev->preempt;
preempt->skipsaverestore = val ? 1 : 0;
return 0;
}
static unsigned int _skipsaverestore_show(struct adreno_device *adreno_dev)
{
struct adreno_preemption *preempt = &adreno_dev->preempt;
return preempt->skipsaverestore;
}
static int _ft_pagefault_policy_store(struct adreno_device *adreno_dev,
unsigned int val)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
int ret = 0;
mutex_lock(&device->mutex);
val &= KGSL_FT_PAGEFAULT_MASK;
if (test_bit(ADRENO_DEVICE_STARTED, &adreno_dev->priv))
ret = kgsl_mmu_set_pagefault_policy(&device->mmu,
(unsigned long) val);
if (ret == 0)
adreno_dev->ft_pf_policy = val;
mutex_unlock(&device->mutex);
return 0;
}
static unsigned int _ft_pagefault_policy_show(struct adreno_device *adreno_dev)
{
return adreno_dev->ft_pf_policy;
}
static int _gpu_llc_slice_enable_store(struct adreno_device *adreno_dev,
unsigned int val)
{
adreno_dev->gpu_llc_slice_enable = val ? true : false;
return 0;
}
static unsigned int _gpu_llc_slice_enable_show(struct adreno_device *adreno_dev)
{
return adreno_dev->gpu_llc_slice_enable;
}
static int _gpuhtw_llc_slice_enable_store(struct adreno_device *adreno_dev,
unsigned int val)
{
adreno_dev->gpuhtw_llc_slice_enable = val ? true : false;
return 0;
}
static unsigned int
_gpuhtw_llc_slice_enable_show(struct adreno_device *adreno_dev)
{
return adreno_dev->gpuhtw_llc_slice_enable;
}
static int _ft_long_ib_detect_store(struct adreno_device *adreno_dev,
unsigned int val)
{
adreno_dev->long_ib_detect = val;
return 0;
}
static unsigned int _ft_long_ib_detect_show(struct adreno_device *adreno_dev)
{
return adreno_dev->long_ib_detect;
}
static unsigned int _ft_hang_intr_status_show(struct adreno_device *adreno_dev)
{
/* Hang interrupt is always on on all targets */
return 1;
}
static int _pwrctrl_store(struct adreno_device *adreno_dev,
unsigned int val, unsigned int flag)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
if (val == test_bit(flag, &adreno_dev->pwrctrl_flag))
return 0;
mutex_lock(&device->mutex);
/* Power down the GPU before changing the state */
kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND);
change_bit(flag, &adreno_dev->pwrctrl_flag);
kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER);
mutex_unlock(&device->mutex);
return 0;
}
static int _preemption_store(struct adreno_device *adreno_dev,
unsigned int val)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
struct kgsl_context *context;
struct adreno_context *drawctxt;
int id;
mutex_lock(&device->mutex);
if (!(ADRENO_FEATURE(adreno_dev, ADRENO_PREEMPTION)) ||
(test_bit(ADRENO_DEVICE_PREEMPTION,
&adreno_dev->priv) == val)) {
mutex_unlock(&device->mutex);
return 0;
}
kgsl_pwrctrl_change_state(device, KGSL_STATE_SUSPEND);
change_bit(ADRENO_DEVICE_PREEMPTION, &adreno_dev->priv);
adreno_dev->cur_rb = &(adreno_dev->ringbuffers[0]);
/* Update the ringbuffer for each draw context */
write_lock(&device->context_lock);
idr_for_each_entry(&device->context_idr, context, id) {
drawctxt = ADRENO_CONTEXT(context);
drawctxt->rb = adreno_ctx_get_rb(adreno_dev, drawctxt);
}
write_unlock(&device->context_lock);
kgsl_pwrctrl_change_state(device, KGSL_STATE_SLUMBER);
mutex_unlock(&device->mutex);
return 0;
}
static unsigned int _preemption_show(struct adreno_device *adreno_dev)
{
return adreno_is_preemption_enabled(adreno_dev);
}
static int _hwcg_store(struct adreno_device *adreno_dev,
unsigned int val)
{
return _pwrctrl_store(adreno_dev, val, ADRENO_HWCG_CTRL);
}
static unsigned int _hwcg_show(struct adreno_device *adreno_dev)
{
return test_bit(ADRENO_HWCG_CTRL, &adreno_dev->pwrctrl_flag);
}
static int _throttling_store(struct adreno_device *adreno_dev,
unsigned int val)
{
return _pwrctrl_store(adreno_dev, val, ADRENO_THROTTLING_CTRL);
}
static unsigned int _throttling_show(struct adreno_device *adreno_dev)
{
return test_bit(ADRENO_THROTTLING_CTRL, &adreno_dev->pwrctrl_flag);
}
static int _sptp_pc_store(struct adreno_device *adreno_dev,
unsigned int val)
{
return _pwrctrl_store(adreno_dev, val, ADRENO_SPTP_PC_CTRL);
}
static unsigned int _sptp_pc_show(struct adreno_device *adreno_dev)
{
return test_bit(ADRENO_SPTP_PC_CTRL, &adreno_dev->pwrctrl_flag);
}
static int _lm_store(struct adreno_device *adreno_dev, unsigned int val)
{
return _pwrctrl_store(adreno_dev, val, ADRENO_LM_CTRL);
}
static unsigned int _lm_show(struct adreno_device *adreno_dev)
{
return test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag);
}
static int _ifpc_store(struct adreno_device *adreno_dev, unsigned int val)
{
return gmu_core_dev_ifpc_store(KGSL_DEVICE(adreno_dev), val);
}
static unsigned int _ifpc_show(struct adreno_device *adreno_dev)
{
return gmu_core_dev_ifpc_show(KGSL_DEVICE(adreno_dev));
}
static unsigned int _ifpc_count_show(struct adreno_device *adreno_dev)
{
return adreno_dev->ifpc_count;
}
static unsigned int _preempt_count_show(struct adreno_device *adreno_dev)
{
struct adreno_preemption *preempt = &adreno_dev->preempt;
return preempt->count;
}
static unsigned int _acd_show(struct adreno_device *adreno_dev)
{
return test_bit(ADRENO_ACD_CTRL, &adreno_dev->pwrctrl_flag);
}
static int _acd_store(struct adreno_device *adreno_dev, unsigned int val)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
if (test_bit(ADRENO_ACD_CTRL, &adreno_dev->pwrctrl_flag) == val)
return 0;
return gmu_core_acd_set(device, val);
}
static ssize_t _sysfs_store_u32(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_get_drvdata(dev));
struct adreno_sysfs_attribute *_attr = ADRENO_SYSFS_ATTR(attr);
unsigned int val = 0;
int ret;
ret = kgsl_sysfs_store(buf, &val);
if (!ret && _attr->store)
ret = _attr->store(adreno_dev, val);
return (ssize_t) ret < 0 ? ret : count;
}
static ssize_t _sysfs_show_u32(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_get_drvdata(dev));
struct adreno_sysfs_attribute *_attr = ADRENO_SYSFS_ATTR(attr);
unsigned int val = 0;
if (_attr->show)
val = _attr->show(adreno_dev);
return scnprintf(buf, PAGE_SIZE, "0x%X\n", val);
}
static ssize_t _sysfs_store_bool(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_get_drvdata(dev));
struct adreno_sysfs_attribute *_attr = ADRENO_SYSFS_ATTR(attr);
unsigned int val = 0;
int ret;
ret = kgsl_sysfs_store(buf, &val);
if (!ret && _attr->store)
ret = _attr->store(adreno_dev, val ? 1 : 0);
return (ssize_t) ret < 0 ? ret : count;
}
static ssize_t _sysfs_show_bool(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(dev_get_drvdata(dev));
struct adreno_sysfs_attribute *_attr = ADRENO_SYSFS_ATTR(attr);
unsigned int val = 0;
if (_attr->show)
val = _attr->show(adreno_dev);
return scnprintf(buf, PAGE_SIZE, "%d\n", val);
}
#define ADRENO_SYSFS_BOOL(_name) \
_ADRENO_SYSFS_ATTR(_name, _sysfs_show_bool, _sysfs_store_bool)
#define ADRENO_SYSFS_RO_BOOL(_name) \
_ADRENO_SYSFS_ATTR_RO(_name, _sysfs_show_bool)
#define ADRENO_SYSFS_U32(_name) \
_ADRENO_SYSFS_ATTR(_name, _sysfs_show_u32, _sysfs_store_u32)
#define ADRENO_SYSFS_RO_U32(_name) \
_ADRENO_SYSFS_ATTR_RO(_name, _sysfs_show_u32)
static ADRENO_SYSFS_U32(ft_policy);
static ADRENO_SYSFS_U32(ft_pagefault_policy);
static ADRENO_SYSFS_U32(preempt_level);
static ADRENO_SYSFS_RO_U32(preempt_count);
static ADRENO_SYSFS_BOOL(usesgmem);
static ADRENO_SYSFS_BOOL(skipsaverestore);
static ADRENO_SYSFS_BOOL(ft_long_ib_detect);
static ADRENO_SYSFS_RO_BOOL(ft_hang_intr_status);
static ADRENO_SYSFS_BOOL(gpu_llc_slice_enable);
static ADRENO_SYSFS_BOOL(gpuhtw_llc_slice_enable);
static DEVICE_INT_ATTR(wake_nice, 0644, adreno_wake_nice);
static DEVICE_INT_ATTR(wake_timeout, 0644, adreno_wake_timeout);
static ADRENO_SYSFS_BOOL(sptp_pc);
static ADRENO_SYSFS_BOOL(lm);
static ADRENO_SYSFS_BOOL(preemption);
static ADRENO_SYSFS_BOOL(hwcg);
static ADRENO_SYSFS_BOOL(throttling);
static ADRENO_SYSFS_BOOL(ifpc);
static ADRENO_SYSFS_RO_U32(ifpc_count);
static ADRENO_SYSFS_BOOL(acd);
static const struct attribute *_attr_list[] = {
&adreno_attr_ft_policy.attr.attr,
&adreno_attr_ft_pagefault_policy.attr.attr,
&adreno_attr_ft_long_ib_detect.attr.attr,
&adreno_attr_ft_hang_intr_status.attr.attr,
&dev_attr_wake_nice.attr.attr,
&dev_attr_wake_timeout.attr.attr,
&adreno_attr_sptp_pc.attr.attr,
&adreno_attr_lm.attr.attr,
&adreno_attr_preemption.attr.attr,
&adreno_attr_hwcg.attr.attr,
&adreno_attr_throttling.attr.attr,
&adreno_attr_gpu_llc_slice_enable.attr.attr,
&adreno_attr_gpuhtw_llc_slice_enable.attr.attr,
&adreno_attr_preempt_level.attr.attr,
&adreno_attr_usesgmem.attr.attr,
&adreno_attr_skipsaverestore.attr.attr,
&adreno_attr_ifpc.attr.attr,
&adreno_attr_ifpc_count.attr.attr,
&adreno_attr_preempt_count.attr.attr,
&adreno_attr_acd.attr.attr,
NULL,
};
/**
* adreno_sysfs_close() - Take down the adreno sysfs files
* @adreno_dev: Pointer to the adreno device
*
* Take down the sysfs files on when the device goes away
*/
void adreno_sysfs_close(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
sysfs_remove_files(&device->dev->kobj, _attr_list);
}
/**
* adreno_sysfs_init() - Initialize adreno sysfs files
* @adreno_dev: Pointer to the adreno device
*
* Initialize many of the adreno specific sysfs files especially for fault
* tolerance and power control
*/
int adreno_sysfs_init(struct adreno_device *adreno_dev)
{
struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
return sysfs_create_files(&device->dev->kobj, _attr_list);
}

10
drivers/gpu/msm/adreno_trace.c Normal file
View file

@ -0,0 +1,10 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2013-2014, 2019, The Linux Foundation. All rights reserved.
*/
#include "adreno.h"
/* Instantiate tracepoints */
#define CREATE_TRACE_POINTS
#include "adreno_trace.h"

676
drivers/gpu/msm/adreno_trace.h Normal file
View file

@ -0,0 +1,676 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
*/
#if !defined(_ADRENO_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
#define _ADRENO_TRACE_H
#undef TRACE_SYSTEM
#define TRACE_SYSTEM kgsl
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE adreno_trace
#include <linux/tracepoint.h>
#include "adreno_a3xx.h"
#include "adreno_a5xx.h"
#define ADRENO_FT_TYPES \
{ BIT(KGSL_FT_OFF), "off" }, \
{ BIT(KGSL_FT_REPLAY), "replay" }, \
{ BIT(KGSL_FT_SKIPIB), "skipib" }, \
{ BIT(KGSL_FT_SKIPFRAME), "skipframe" }, \
{ BIT(KGSL_FT_DISABLE), "disable" }, \
{ BIT(KGSL_FT_TEMP_DISABLE), "temp" }, \
{ BIT(KGSL_FT_THROTTLE), "throttle"}, \
{ BIT(KGSL_FT_SKIPCMD), "skipcmd" }
TRACE_EVENT(adreno_cmdbatch_queued,
TP_PROTO(struct kgsl_drawobj *drawobj, unsigned int queued),
TP_ARGS(drawobj, queued),
TP_STRUCT__entry(
__field(unsigned int, id)
__field(unsigned int, timestamp)
__field(unsigned int, queued)
__field(unsigned int, flags)
__field(unsigned int, prio)
),
TP_fast_assign(
__entry->id = drawobj->context->id;
__entry->timestamp = drawobj->timestamp;
__entry->queued = queued;
__entry->flags = drawobj->flags;
__entry->prio = drawobj->context->priority;
),
TP_printk(
"ctx=%u ctx_prio=%u ts=%u queued=%u flags=%s",
__entry->id, __entry->prio,
__entry->timestamp, __entry->queued,
__entry->flags ? __print_flags(__entry->flags, "|",
KGSL_DRAWOBJ_FLAGS) : "none"
)
);
TRACE_EVENT(adreno_cmdbatch_submitted,
TP_PROTO(struct kgsl_drawobj *drawobj, int inflight, uint64_t ticks,
unsigned long secs, unsigned long usecs,
struct adreno_ringbuffer *rb, unsigned int rptr),
TP_ARGS(drawobj, inflight, ticks, secs, usecs, rb, rptr),
TP_STRUCT__entry(
__field(unsigned int, id)
__field(unsigned int, timestamp)
__field(int, inflight)
__field(unsigned int, flags)
__field(uint64_t, ticks)
__field(unsigned long, secs)
__field(unsigned long, usecs)
__field(int, prio)
__field(int, rb_id)
__field(unsigned int, rptr)
__field(unsigned int, wptr)
__field(int, q_inflight)
),
TP_fast_assign(
__entry->id = drawobj->context->id;
__entry->timestamp = drawobj->timestamp;
__entry->inflight = inflight;
__entry->flags = drawobj->flags;
__entry->ticks = ticks;
__entry->secs = secs;
__entry->usecs = usecs;
__entry->prio = drawobj->context->priority;
__entry->rb_id = rb->id;
__entry->rptr = rptr;
__entry->wptr = rb->wptr;
__entry->q_inflight = rb->dispatch_q.inflight;
),
TP_printk(
"ctx=%u ctx_prio=%d ts=%u inflight=%d flags=%s ticks=%lld time=%lu.%0lu rb_id=%d r/w=%x/%x, q_inflight=%d",
__entry->id, __entry->prio, __entry->timestamp,
__entry->inflight,
__entry->flags ? __print_flags(__entry->flags, "|",
KGSL_DRAWOBJ_FLAGS) : "none",
__entry->ticks, __entry->secs, __entry->usecs,
__entry->rb_id, __entry->rptr, __entry->wptr,
__entry->q_inflight
)
);
TRACE_EVENT(adreno_cmdbatch_retired,
TP_PROTO(struct kgsl_drawobj *drawobj, int inflight,
uint64_t start, uint64_t retire,
struct adreno_ringbuffer *rb, unsigned int rptr,
unsigned long fault_recovery),
TP_ARGS(drawobj, inflight, start, retire, rb, rptr, fault_recovery),
TP_STRUCT__entry(
__field(unsigned int, id)
__field(unsigned int, timestamp)
__field(int, inflight)
__field(unsigned int, recovery)
__field(unsigned int, flags)
__field(uint64_t, start)
__field(uint64_t, retire)
__field(int, prio)
__field(int, rb_id)
__field(unsigned int, rptr)
__field(unsigned int, wptr)
__field(int, q_inflight)
__field(unsigned long, fault_recovery)
),
TP_fast_assign(
__entry->id = drawobj->context->id;
__entry->timestamp = drawobj->timestamp;
__entry->inflight = inflight;
__entry->recovery = fault_recovery;
__entry->flags = drawobj->flags;
__entry->start = start;
__entry->retire = retire;
__entry->prio = drawobj->context->priority;
__entry->rb_id = rb->id;
__entry->rptr = rptr;
__entry->wptr = rb->wptr;
__entry->q_inflight = rb->dispatch_q.inflight;
),
TP_printk(
"ctx=%u ctx_prio=%d ts=%u inflight=%d recovery=%s flags=%s start=%lld retire=%lld rb_id=%d, r/w=%x/%x, q_inflight=%d",
__entry->id, __entry->prio, __entry->timestamp,
__entry->inflight,
__entry->recovery ?
__print_flags(__entry->recovery, "|",
ADRENO_FT_TYPES) : "none",
__entry->flags ? __print_flags(__entry->flags, "|",
KGSL_DRAWOBJ_FLAGS) : "none",
__entry->start,
__entry->retire,
__entry->rb_id, __entry->rptr, __entry->wptr,
__entry->q_inflight
)
);
TRACE_EVENT(adreno_cmdbatch_sync,
TP_PROTO(struct adreno_context *drawctxt,
uint64_t ticks),
TP_ARGS(drawctxt, ticks),
TP_STRUCT__entry(
__field(unsigned int, id)
__field(unsigned int, timestamp)
__field(uint64_t, ticks)
__field(int, prio)
),
TP_fast_assign(
__entry->id = drawctxt->base.id;
__entry->timestamp = drawctxt->timestamp;
__entry->ticks = ticks;
__entry->prio = drawctxt->base.priority;
),
TP_printk(
"ctx=%u ctx_prio=%d ts=%u ticks=%lld",
__entry->id, __entry->prio, __entry->timestamp,
__entry->ticks
)
);
TRACE_EVENT(adreno_cmdbatch_fault,
TP_PROTO(struct kgsl_drawobj_cmd *cmdobj, unsigned int fault),
TP_ARGS(cmdobj, fault),
TP_STRUCT__entry(
__field(unsigned int, id)
__field(unsigned int, timestamp)
__field(unsigned int, fault)
),
TP_fast_assign(
__entry->id = cmdobj->base.context->id;
__entry->timestamp = cmdobj->base.timestamp;
__entry->fault = fault;
),
TP_printk(
"ctx=%u ts=%u type=%s",
__entry->id, __entry->timestamp,
__print_symbolic(__entry->fault,
{ 0, "none" },
{ ADRENO_SOFT_FAULT, "soft" },
{ ADRENO_HARD_FAULT, "hard" },
{ ADRENO_TIMEOUT_FAULT, "timeout" })
)
);
TRACE_EVENT(adreno_cmdbatch_recovery,
TP_PROTO(struct kgsl_drawobj_cmd *cmdobj, unsigned int action),
TP_ARGS(cmdobj, action),
TP_STRUCT__entry(
__field(unsigned int, id)
__field(unsigned int, timestamp)
__field(unsigned int, action)
),
TP_fast_assign(
__entry->id = cmdobj->base.context->id;
__entry->timestamp = cmdobj->base.timestamp;
__entry->action = action;
),
TP_printk(
"ctx=%u ts=%u action=%s",
__entry->id, __entry->timestamp,
__print_symbolic(__entry->action, ADRENO_FT_TYPES)
)
);
DECLARE_EVENT_CLASS(adreno_drawctxt_template,
TP_PROTO(struct adreno_context *drawctxt),
TP_ARGS(drawctxt),
TP_STRUCT__entry(
__field(unsigned int, id)
__field(unsigned int, priority)
),
TP_fast_assign(
__entry->id = drawctxt->base.id;
__entry->priority = drawctxt->base.priority;
),
TP_printk("ctx=%u priority=%u", __entry->id, __entry->priority)
);
DEFINE_EVENT(adreno_drawctxt_template, adreno_drawctxt_sleep,
TP_PROTO(struct adreno_context *drawctxt),
TP_ARGS(drawctxt)
);
DEFINE_EVENT(adreno_drawctxt_template, adreno_drawctxt_wake,
TP_PROTO(struct adreno_context *drawctxt),
TP_ARGS(drawctxt)
);
DEFINE_EVENT(adreno_drawctxt_template, dispatch_queue_context,
TP_PROTO(struct adreno_context *drawctxt),
TP_ARGS(drawctxt)
);
DEFINE_EVENT(adreno_drawctxt_template, adreno_drawctxt_invalidate,
TP_PROTO(struct adreno_context *drawctxt),
TP_ARGS(drawctxt)
);
TRACE_EVENT(adreno_drawctxt_wait_start,
TP_PROTO(unsigned int rb_id, unsigned int ctx_id, unsigned int ts),
TP_ARGS(rb_id, ctx_id, ts),
TP_STRUCT__entry(
__field(unsigned int, rb_id)
__field(unsigned int, ctx_id)
__field(unsigned int, ts)
),
TP_fast_assign(
__entry->rb_id = rb_id;
__entry->ctx_id = ctx_id;
__entry->ts = ts;
),
TP_printk(
"rb=%u ctx=%u ts=%u",
__entry->rb_id, __entry->ctx_id, __entry->ts
)
);
TRACE_EVENT(adreno_drawctxt_wait_done,
TP_PROTO(unsigned int rb_id, unsigned int ctx_id,
unsigned int ts, int status),
TP_ARGS(rb_id, ctx_id, ts, status),
TP_STRUCT__entry(
__field(unsigned int, rb_id)
__field(unsigned int, ctx_id)
__field(unsigned int, ts)
__field(int, status)
),
TP_fast_assign(
__entry->rb_id = rb_id;
__entry->ctx_id = ctx_id;
__entry->ts = ts;
__entry->status = status;
),
TP_printk(
"rb=%u ctx=%u ts=%u status=%d",
__entry->rb_id, __entry->ctx_id, __entry->ts, __entry->status
)
);
TRACE_EVENT(adreno_drawctxt_switch,
TP_PROTO(struct adreno_ringbuffer *rb,
struct adreno_context *newctx),
TP_ARGS(rb, newctx),
TP_STRUCT__entry(
__field(int, rb_level)
__field(unsigned int, oldctx)
__field(unsigned int, newctx)
__field(unsigned int, flags)
),
TP_fast_assign(
__entry->rb_level = rb->id;
__entry->oldctx = rb->drawctxt_active ?
rb->drawctxt_active->base.id : 0;
__entry->newctx = newctx ? newctx->base.id : 0;
),
TP_printk(
"rb level=%d oldctx=%u newctx=%u",
__entry->rb_level, __entry->oldctx, __entry->newctx
)
);
TRACE_EVENT(adreno_gpu_fault,
TP_PROTO(unsigned int ctx, unsigned int ts,
unsigned int status, unsigned int rptr, unsigned int wptr,
unsigned int ib1base, unsigned int ib1size,
unsigned int ib2base, unsigned int ib2size, int rb_id),
TP_ARGS(ctx, ts, status, rptr, wptr, ib1base, ib1size, ib2base,
ib2size, rb_id),
TP_STRUCT__entry(
__field(unsigned int, ctx)
__field(unsigned int, ts)
__field(unsigned int, status)
__field(unsigned int, rptr)
__field(unsigned int, wptr)
__field(unsigned int, ib1base)
__field(unsigned int, ib1size)
__field(unsigned int, ib2base)
__field(unsigned int, ib2size)
__field(int, rb_id)
),
TP_fast_assign(
__entry->ctx = ctx;
__entry->ts = ts;
__entry->status = status;
__entry->rptr = rptr;
__entry->wptr = wptr;
__entry->ib1base = ib1base;
__entry->ib1size = ib1size;
__entry->ib2base = ib2base;
__entry->ib2size = ib2size;
__entry->rb_id = rb_id;
),
TP_printk(
"ctx=%d ts=%d rb_id=%d status=%X RB=%X/%X IB1=%X/%X IB2=%X/%X",
__entry->ctx, __entry->ts, __entry->rb_id, __entry->status,
__entry->wptr, __entry->rptr, __entry->ib1base,
__entry->ib1size, __entry->ib2base, __entry->ib2size)
);
TRACE_EVENT(adreno_sp_tp,
TP_PROTO(unsigned long ip),
TP_ARGS(ip),
TP_STRUCT__entry(
__field(unsigned long, ip)
),
TP_fast_assign(
__entry->ip = ip;
),
TP_printk(
"func=%pS", (void *) __entry->ip
)
);
/*
* Tracepoint for a3xx irq. Includes status info
*/
TRACE_EVENT(kgsl_a3xx_irq_status,
TP_PROTO(struct adreno_device *adreno_dev, unsigned int status),
TP_ARGS(adreno_dev, status),
TP_STRUCT__entry(
__string(device_name, adreno_dev->dev.name)
__field(unsigned int, status)
),
TP_fast_assign(
__assign_str(device_name, adreno_dev->dev.name);
__entry->status = status;
),
TP_printk(
"d_name=%s status=%s",
__get_str(device_name),
__entry->status ? __print_flags(__entry->status, "|",
{ BIT(A3XX_INT_RBBM_GPU_IDLE), "RBBM_GPU_IDLE" },
{ BIT(A3XX_INT_RBBM_AHB_ERROR), "RBBM_AHB_ERR" },
{ BIT(A3XX_INT_RBBM_REG_TIMEOUT), "RBBM_REG_TIMEOUT" },
{ BIT(A3XX_INT_RBBM_ME_MS_TIMEOUT),
"RBBM_ME_MS_TIMEOUT" },
{ BIT(A3XX_INT_RBBM_PFP_MS_TIMEOUT),
"RBBM_PFP_MS_TIMEOUT" },
{ BIT(A3XX_INT_RBBM_ATB_BUS_OVERFLOW),
"RBBM_ATB_BUS_OVERFLOW" },
{ BIT(A3XX_INT_VFD_ERROR), "RBBM_VFD_ERROR" },
{ BIT(A3XX_INT_CP_SW_INT), "CP_SW" },
{ BIT(A3XX_INT_CP_T0_PACKET_IN_IB),
"CP_T0_PACKET_IN_IB" },
{ BIT(A3XX_INT_CP_OPCODE_ERROR), "CP_OPCODE_ERROR" },
{ BIT(A3XX_INT_CP_RESERVED_BIT_ERROR),
"CP_RESERVED_BIT_ERROR" },
{ BIT(A3XX_INT_CP_HW_FAULT), "CP_HW_FAULT" },
{ BIT(A3XX_INT_CP_DMA), "CP_DMA" },
{ BIT(A3XX_INT_CP_IB2_INT), "CP_IB2_INT" },
{ BIT(A3XX_INT_CP_IB1_INT), "CP_IB1_INT" },
{ BIT(A3XX_INT_CP_RB_INT), "CP_RB_INT" },
{ BIT(A3XX_INT_CP_REG_PROTECT_FAULT),
"CP_REG_PROTECT_FAULT" },
{ BIT(A3XX_INT_CP_RB_DONE_TS), "CP_RB_DONE_TS" },
{ BIT(A3XX_INT_CP_VS_DONE_TS), "CP_VS_DONE_TS" },
{ BIT(A3XX_INT_CP_PS_DONE_TS), "CP_PS_DONE_TS" },
{ BIT(A3XX_INT_CACHE_FLUSH_TS), "CACHE_FLUSH_TS" },
{ BIT(A3XX_INT_CP_AHB_ERROR_HALT),
"CP_AHB_ERROR_HALT" },
{ BIT(A3XX_INT_MISC_HANG_DETECT), "MISC_HANG_DETECT" },
{ BIT(A3XX_INT_UCHE_OOB_ACCESS), "UCHE_OOB_ACCESS" })
: "None"
)
);
/*
* Tracepoint for a5xx irq. Includes status info
*/
TRACE_EVENT(kgsl_a5xx_irq_status,
TP_PROTO(struct adreno_device *adreno_dev, unsigned int status),
TP_ARGS(adreno_dev, status),
TP_STRUCT__entry(
__string(device_name, adreno_dev->dev.name)
__field(unsigned int, status)
),
TP_fast_assign(
__assign_str(device_name, adreno_dev->dev.name);
__entry->status = status;
),
TP_printk(
"d_name=%s status=%s",
__get_str(device_name),
__entry->status ? __print_flags(__entry->status, "|",
{ BIT(A5XX_INT_RBBM_GPU_IDLE), "RBBM_GPU_IDLE" },
{ BIT(A5XX_INT_RBBM_AHB_ERROR), "RBBM_AHB_ERR" },
{ BIT(A5XX_INT_RBBM_TRANSFER_TIMEOUT),
"RBBM_TRANSFER_TIMEOUT" },
{ BIT(A5XX_INT_RBBM_ME_MS_TIMEOUT),
"RBBM_ME_MS_TIMEOUT" },
{ BIT(A5XX_INT_RBBM_PFP_MS_TIMEOUT),
"RBBM_PFP_MS_TIMEOUT" },
{ BIT(A5XX_INT_RBBM_ETS_MS_TIMEOUT),
"RBBM_ETS_MS_TIMEOUT" },
{ BIT(A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW),
"RBBM_ATB_ASYNC_OVERFLOW" },
{ BIT(A5XX_INT_RBBM_GPC_ERROR), "RBBM_GPC_ERR" },
{ BIT(A5XX_INT_CP_SW), "CP_SW" },
{ BIT(A5XX_INT_CP_HW_ERROR), "CP_OPCODE_ERROR" },
{ BIT(A5XX_INT_CP_CCU_FLUSH_DEPTH_TS),
"CP_CCU_FLUSH_DEPTH_TS" },
{ BIT(A5XX_INT_CP_CCU_FLUSH_COLOR_TS),
"CP_CCU_FLUSH_COLOR_TS" },
{ BIT(A5XX_INT_CP_CCU_RESOLVE_TS),
"CP_CCU_RESOLVE_TS" },
{ BIT(A5XX_INT_CP_IB2), "CP_IB2_INT" },
{ BIT(A5XX_INT_CP_IB1), "CP_IB1_INT" },
{ BIT(A5XX_INT_CP_RB), "CP_RB_INT" },
{ BIT(A5XX_INT_CP_UNUSED_1), "CP_UNUSED_1" },
{ BIT(A5XX_INT_CP_RB_DONE_TS), "CP_RB_DONE_TS" },
{ BIT(A5XX_INT_CP_WT_DONE_TS), "CP_WT_DONE_TS" },
{ BIT(A5XX_INT_UNKNOWN_1), "UNKNOWN_1" },
{ BIT(A5XX_INT_CP_CACHE_FLUSH_TS),
"CP_CACHE_FLUSH_TS" },
{ BIT(A5XX_INT_UNUSED_2), "UNUSED_2" },
{ BIT(A5XX_INT_RBBM_ATB_BUS_OVERFLOW),
"RBBM_ATB_BUS_OVERFLOW" },
{ BIT(A5XX_INT_MISC_HANG_DETECT), "MISC_HANG_DETECT" },
{ BIT(A5XX_INT_UCHE_OOB_ACCESS), "UCHE_OOB_ACCESS" },
{ BIT(A5XX_INT_UCHE_TRAP_INTR), "UCHE_TRAP_INTR" },
{ BIT(A5XX_INT_DEBBUS_INTR_0), "DEBBUS_INTR_0" },
{ BIT(A5XX_INT_DEBBUS_INTR_1), "DEBBUS_INTR_1" },
{ BIT(A5XX_INT_GPMU_VOLTAGE_DROOP),
"GPMU_VOLTAGE_DROOP" },
{ BIT(A5XX_INT_GPMU_FIRMWARE), "GPMU_FIRMWARE" },
{ BIT(A5XX_INT_ISDB_CPU_IRQ), "ISDB_CPU_IRQ" },
{ BIT(A5XX_INT_ISDB_UNDER_DEBUG), "ISDB_UNDER_DEBUG" })
: "None"
)
);
DECLARE_EVENT_CLASS(adreno_hw_preempt_template,
TP_PROTO(struct adreno_ringbuffer *cur_rb,
struct adreno_ringbuffer *new_rb,
unsigned int cur_rptr, unsigned int new_rptr),
TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr),
TP_STRUCT__entry(__field(int, cur_level)
__field(int, new_level)
__field(unsigned int, cur_rptr)
__field(unsigned int, new_rptr)
__field(unsigned int, cur_wptr)
__field(unsigned int, new_wptr)
__field(unsigned int, cur_rbbase)
__field(unsigned int, new_rbbase)
),
TP_fast_assign(__entry->cur_level = cur_rb->id;
__entry->new_level = new_rb->id;
__entry->cur_rptr = cur_rptr;
__entry->new_rptr = new_rptr;
__entry->cur_wptr = cur_rb->wptr;
__entry->new_wptr = new_rb->wptr;
__entry->cur_rbbase = cur_rb->buffer_desc.gpuaddr;
__entry->new_rbbase = new_rb->buffer_desc.gpuaddr;
),
TP_printk(
"cur_rb_lvl=%d rptr=%x wptr=%x rbbase=%x new_rb_lvl=%d rptr=%x wptr=%x rbbase=%x",
__entry->cur_level, __entry->cur_rptr,
__entry->cur_wptr, __entry->cur_rbbase,
__entry->new_level, __entry->new_rptr,
__entry->new_wptr, __entry->new_rbbase
)
);
DEFINE_EVENT(adreno_hw_preempt_template, adreno_hw_preempt_clear_to_trig,
TP_PROTO(struct adreno_ringbuffer *cur_rb,
struct adreno_ringbuffer *new_rb,
unsigned int cur_rptr, unsigned int new_rptr),
TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr)
);
DEFINE_EVENT(adreno_hw_preempt_template, adreno_hw_preempt_trig_to_comp,
TP_PROTO(struct adreno_ringbuffer *cur_rb,
struct adreno_ringbuffer *new_rb,
unsigned int cur_rptr, unsigned int new_rptr),
TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr)
);
DEFINE_EVENT(adreno_hw_preempt_template, adreno_hw_preempt_trig_to_comp_int,
TP_PROTO(struct adreno_ringbuffer *cur_rb,
struct adreno_ringbuffer *new_rb,
unsigned int cur_rptr, unsigned int new_rptr),
TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr)
);
TRACE_EVENT(adreno_hw_preempt_comp_to_clear,
TP_PROTO(struct adreno_ringbuffer *cur_rb,
struct adreno_ringbuffer *new_rb,
unsigned int cur_rptr, unsigned int new_rptr),
TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr),
TP_STRUCT__entry(__field(int, cur_level)
__field(int, new_level)
__field(unsigned int, cur_rptr)
__field(unsigned int, new_rptr)
__field(unsigned int, cur_wptr)
__field(unsigned int, new_wptr_end)
__field(unsigned int, new_wptr)
__field(unsigned int, cur_rbbase)
__field(unsigned int, new_rbbase)
),
TP_fast_assign(__entry->cur_level = cur_rb->id;
__entry->new_level = new_rb->id;
__entry->cur_rptr = cur_rptr;
__entry->new_rptr = new_rptr;
__entry->cur_wptr = cur_rb->wptr;
__entry->new_wptr_end = new_rb->wptr_preempt_end;
__entry->new_wptr = new_rb->wptr;
__entry->cur_rbbase = cur_rb->buffer_desc.gpuaddr;
__entry->new_rbbase = new_rb->buffer_desc.gpuaddr;
),
TP_printk(
"cur_rb_lvl=%d rptr=%x wptr=%x rbbase=%x prev_rb_lvl=%d rptr=%x wptr_preempt_end=%x wptr=%x rbbase=%x",
__entry->cur_level, __entry->cur_rptr,
__entry->cur_wptr, __entry->cur_rbbase,
__entry->new_level, __entry->new_rptr,
__entry->new_wptr_end, __entry->new_wptr, __entry->new_rbbase
)
);
TRACE_EVENT(adreno_hw_preempt_token_submit,
TP_PROTO(struct adreno_ringbuffer *cur_rb,
struct adreno_ringbuffer *new_rb,
unsigned int cur_rptr, unsigned int new_rptr),
TP_ARGS(cur_rb, new_rb, cur_rptr, new_rptr),
TP_STRUCT__entry(__field(int, cur_level)
__field(int, new_level)
__field(unsigned int, cur_rptr)
__field(unsigned int, new_rptr)
__field(unsigned int, cur_wptr)
__field(unsigned int, cur_wptr_end)
__field(unsigned int, new_wptr)
__field(unsigned int, cur_rbbase)
__field(unsigned int, new_rbbase)
),
TP_fast_assign(__entry->cur_level = cur_rb->id;
__entry->new_level = new_rb->id;
__entry->cur_rptr = cur_rptr;
__entry->new_rptr = new_rptr;
__entry->cur_wptr = cur_rb->wptr;
__entry->cur_wptr_end = cur_rb->wptr_preempt_end;
__entry->new_wptr = new_rb->wptr;
__entry->cur_rbbase = cur_rb->buffer_desc.gpuaddr;
__entry->new_rbbase = new_rb->buffer_desc.gpuaddr;
),
TP_printk(
"cur_rb_lvl=%d rptr=%x wptr_preempt_end=%x wptr=%x rbbase=%x new_rb_lvl=%d rptr=%x wptr=%x rbbase=%x",
__entry->cur_level, __entry->cur_rptr,
__entry->cur_wptr_end, __entry->cur_wptr,
__entry->cur_rbbase,
__entry->new_level, __entry->new_rptr,
__entry->new_wptr, __entry->new_rbbase
)
);
TRACE_EVENT(adreno_preempt_trigger,
TP_PROTO(struct adreno_ringbuffer *cur, struct adreno_ringbuffer *next,
unsigned int cntl),
TP_ARGS(cur, next, cntl),
TP_STRUCT__entry(
__field(unsigned int, cur)
__field(unsigned int, next)
__field(unsigned int, cntl)
),
TP_fast_assign(
__entry->cur = cur->id;
__entry->next = next->id;
__entry->cntl = cntl;
),
TP_printk("trigger from id=%d to id=%d cntl=%x",
__entry->cur, __entry->next, __entry->cntl
)
);
TRACE_EVENT(adreno_preempt_done,
TP_PROTO(struct adreno_ringbuffer *cur, struct adreno_ringbuffer *next,
unsigned int level),
TP_ARGS(cur, next, level),
TP_STRUCT__entry(
__field(unsigned int, cur)
__field(unsigned int, next)
__field(unsigned int, level)
),
TP_fast_assign(
__entry->cur = cur->id;
__entry->next = next->id;
__entry->level = level;
),
TP_printk("done switch to id=%d from id=%d level=%x",
__entry->next, __entry->cur, __entry->level
)
);
TRACE_EVENT(adreno_ifpc_count,
TP_PROTO(unsigned int ifpc_count),
TP_ARGS(ifpc_count),
TP_STRUCT__entry(
__field(unsigned int, ifpc_count)
),
TP_fast_assign(
__entry->ifpc_count = ifpc_count;
),
TP_printk("total times GMU entered IFPC = %d", __entry->ifpc_count)
);
#endif /* _ADRENO_TRACE_H */
/* This part must be outside protection */
#include <trace/define_trace.h>

5215
drivers/gpu/msm/kgsl.c Normal file
View file

File diff suppressed because it is too large Load diff

594
drivers/gpu/msm/kgsl.h Normal file
View file

@ -0,0 +1,594 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2008-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_H
#define __KGSL_H
#include <linux/cdev.h>
#include <linux/compat.h>
#include <linux/interrupt.h>
#include <linux/kthread.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include "kgsl_gmu_core.h"
#include "kgsl_pwrscale.h"
#include "uapi/msm_kgsl.h"
/*
* --- kgsl drawobj flags ---
* These flags are same as --- drawobj flags ---
* but renamed to reflect that cmdbatch is renamed to drawobj.
*/
#define KGSL_DRAWOBJ_MEMLIST KGSL_CMDBATCH_MEMLIST
#define KGSL_DRAWOBJ_MARKER KGSL_CMDBATCH_MARKER
#define KGSL_DRAWOBJ_SUBMIT_IB_LIST KGSL_CMDBATCH_SUBMIT_IB_LIST
#define KGSL_DRAWOBJ_CTX_SWITCH KGSL_CMDBATCH_CTX_SWITCH
#define KGSL_DRAWOBJ_PROFILING KGSL_CMDBATCH_PROFILING
#define KGSL_DRAWOBJ_PROFILING_KTIME KGSL_CMDBATCH_PROFILING_KTIME
#define KGSL_DRAWOBJ_END_OF_FRAME KGSL_CMDBATCH_END_OF_FRAME
#define KGSL_DRAWOBJ_SYNC KGSL_CMDBATCH_SYNC
#define KGSL_DRAWOBJ_PWR_CONSTRAINT KGSL_CMDBATCH_PWR_CONSTRAINT
#define KGSL_DRAWOBJ_SPARSE KGSL_CMDBATCH_SPARSE
#define kgsl_drawobj_profiling_buffer kgsl_cmdbatch_profiling_buffer
/* The number of memstore arrays limits the number of contexts allowed.
* If more contexts are needed, update multiple for MEMSTORE_SIZE
*/
#define KGSL_MEMSTORE_SIZE ((int)(PAGE_SIZE * 8))
#define KGSL_MEMSTORE_GLOBAL (0)
#define KGSL_PRIORITY_MAX_RB_LEVELS 4
#define KGSL_MEMSTORE_MAX (KGSL_MEMSTORE_SIZE / \
sizeof(struct kgsl_devmemstore) - 1 - KGSL_PRIORITY_MAX_RB_LEVELS)
#define KGSL_MAX_CONTEXTS_PER_PROC 200
#define MEMSTORE_RB_OFFSET(rb, field) \
KGSL_MEMSTORE_OFFSET(((rb)->id + KGSL_MEMSTORE_MAX), field)
#define MEMSTORE_ID_GPU_ADDR(dev, iter, field) \
((dev)->memstore.gpuaddr + KGSL_MEMSTORE_OFFSET(iter, field))
#define MEMSTORE_RB_GPU_ADDR(dev, rb, field) \
((dev)->memstore.gpuaddr + \
KGSL_MEMSTORE_OFFSET(((rb)->id + KGSL_MEMSTORE_MAX), field))
/*
* SCRATCH MEMORY: The scratch memory is one page worth of data that
* is mapped into the GPU. This allows for some 'shared' data between
* the GPU and CPU. For example, it will be used by the GPU to write
* each updated RPTR for each RB.
*
* Used Data:
* Offset: Length(bytes): What
* 0x0: 4 * KGSL_PRIORITY_MAX_RB_LEVELS: RB0 RPTR
* 0x10: 8 * KGSL_PRIORITY_MAX_RB_LEVELS: RB0 CTXT RESTORE ADDR
*/
/* Shadow global helpers */
#define SCRATCH_RPTR_OFFSET(id) ((id) * sizeof(unsigned int))
#define SCRATCH_RPTR_GPU_ADDR(dev, id) \
((dev)->scratch.gpuaddr + SCRATCH_RPTR_OFFSET(id))
#define SCRATCH_PREEMPTION_CTXT_RESTORE_ADDR_OFFSET(id) \
(SCRATCH_RPTR_OFFSET(KGSL_PRIORITY_MAX_RB_LEVELS) + \
((id) * sizeof(uint64_t)))
#define SCRATCH_PREEMPTION_CTXT_RESTORE_GPU_ADDR(dev, id) \
((dev)->scratch.gpuaddr + \
SCRATCH_PREEMPTION_CTXT_RESTORE_ADDR_OFFSET(id))
/* Timestamp window used to detect rollovers (half of integer range) */
#define KGSL_TIMESTAMP_WINDOW 0x80000000
/*
* A macro for memory statistics - add the new size to the stat and if
* the statisic is greater then _max, set _max
*/
static inline void KGSL_STATS_ADD(uint64_t size, atomic_long_t *stat,
atomic_long_t *max)
{
uint64_t ret = atomic_long_add_return(size, stat);
if (ret > atomic_long_read(max))
atomic_long_set(max, ret);
}
#define KGSL_MAX_NUMIBS 100000
#define KGSL_MAX_SYNCPOINTS 32
#define KGSL_MAX_SPARSE 1000
struct kgsl_device;
struct kgsl_context;
/**
* struct kgsl_driver - main container for global KGSL things
* @cdev: Character device struct
* @major: Major ID for the KGSL device
* @class: Pointer to the class struct for the core KGSL sysfs entries
* @virtdev: Virtual device for managing the core
* @ptkobj: kobject for storing the pagetable statistics
* @prockobj: kobject for storing the process statistics
* @devp: Array of pointers to the individual KGSL device structs
* @process_list: List of open processes
* @pagetable_list: LIst of open pagetables
* @ptlock: Lock for accessing the pagetable list
* @process_mutex: Mutex for accessing the process list
* @proclist_lock: Lock for accessing the process list
* @devlock: Mutex protecting the device list
* @stats: Struct containing atomic memory statistics
* @full_cache_threshold: the threshold that triggers a full cache flush
* @workqueue: Pointer to a single threaded workqueue
* @mem_workqueue: Pointer to a workqueue for deferring memory entries
*/
struct kgsl_driver {
struct cdev cdev;
dev_t major;
struct class *class;
struct device virtdev;
struct kobject *ptkobj;
struct kobject *prockobj;
struct kgsl_device *devp[1];
struct list_head process_list;
struct list_head pagetable_list;
spinlock_t ptlock;
struct mutex process_mutex;
spinlock_t proclist_lock;
struct mutex devlock;
struct {
atomic_long_t vmalloc;
atomic_long_t vmalloc_max;
atomic_long_t page_alloc;
atomic_long_t page_alloc_max;
atomic_long_t coherent;
atomic_long_t coherent_max;
atomic_long_t secure;
atomic_long_t secure_max;
atomic_long_t mapped;
atomic_long_t mapped_max;
} stats;
unsigned int full_cache_threshold;
struct workqueue_struct *workqueue;
struct workqueue_struct *mem_workqueue;
struct kthread_worker worker;
struct task_struct *worker_thread;
};
extern struct kgsl_driver kgsl_driver;
struct kgsl_pagetable;
struct kgsl_memdesc;
struct kgsl_memdesc_ops {
unsigned int vmflags;
int (*vmfault)(struct kgsl_memdesc *memdesc, struct vm_area_struct *vma,
struct vm_fault *vmf);
void (*free)(struct kgsl_memdesc *memdesc);
int (*map_kernel)(struct kgsl_memdesc *memdesc);
void (*unmap_kernel)(struct kgsl_memdesc *memdesc);
};
/* Internal definitions for memdesc->priv */
#define KGSL_MEMDESC_GUARD_PAGE BIT(0)
/* Set if the memdesc is mapped into all pagetables */
#define KGSL_MEMDESC_GLOBAL BIT(1)
/* The memdesc is frozen during a snapshot */
#define KGSL_MEMDESC_FROZEN BIT(2)
/* The memdesc is mapped into a pagetable */
#define KGSL_MEMDESC_MAPPED BIT(3)
/* The memdesc is secured for content protection */
#define KGSL_MEMDESC_SECURE BIT(4)
/* Memory is accessible in privileged mode */
#define KGSL_MEMDESC_PRIVILEGED BIT(6)
/* The memdesc is TZ locked content protection */
#define KGSL_MEMDESC_TZ_LOCKED BIT(7)
/* The memdesc is allocated through contiguous memory */
#define KGSL_MEMDESC_CONTIG BIT(8)
/* This is an instruction buffer */
#define KGSL_MEMDESC_UCODE BIT(9)
/* For global buffers, randomly assign an address from the region */
#define KGSL_MEMDESC_RANDOM BIT(10)
/**
* struct kgsl_memdesc - GPU memory object descriptor
* @pagetable: Pointer to the pagetable that the object is mapped in
* @hostptr: Kernel virtual address
* @hostptr_count: Number of threads using hostptr
* @useraddr: User virtual address (if applicable)
* @gpuaddr: GPU virtual address
* @physaddr: Physical address of the memory object
* @size: Size of the memory object
* @mapsize: Size of memory mapped in userspace
* @priv: Internal flags and settings
* @sgt: Scatter gather table for allocated pages
* @ops: Function hooks for the memdesc memory type
* @flags: Flags set from userspace
* @dev: Pointer to the struct device that owns this memory
* @attrs: dma attributes for this memory
* @pages: An array of pointers to allocated pages
* @page_count: Total number of pages allocated
* @cur_bindings: Number of sparse pages actively bound
*/
struct kgsl_memdesc {
struct kgsl_pagetable *pagetable;
void *hostptr;
unsigned int hostptr_count;
unsigned long useraddr;
uint64_t gpuaddr;
phys_addr_t physaddr;
uint64_t size;
uint64_t mapsize;
unsigned int priv;
struct sg_table *sgt;
struct kgsl_memdesc_ops *ops;
uint64_t flags;
struct device *dev;
unsigned long attrs;
struct page **pages;
unsigned int page_count;
unsigned int cur_bindings;
};
/*
* List of different memory entry types. The usermem enum
* starts at 0, which we use for allocated memory, so 1 is
* added to the enum values.
*/
#define KGSL_MEM_ENTRY_KERNEL 0
#define KGSL_MEM_ENTRY_USER (KGSL_USER_MEM_TYPE_ADDR + 1)
#define KGSL_MEM_ENTRY_ION (KGSL_USER_MEM_TYPE_ION + 1)
#define KGSL_MEM_ENTRY_MAX (KGSL_USER_MEM_TYPE_MAX + 1)
/* symbolic table for trace and debugfs */
/*
* struct kgsl_mem_entry - a userspace memory allocation
* @refcount: reference count. Currently userspace can only
* hold a single reference count, but the kernel may hold more.
* @memdesc: description of the memory
* @priv_data: type-specific data, such as the dma-buf attachment pointer.
* @node: rb_node for the gpu address lookup rb tree
* @id: idr index for this entry, can be used to find memory that does not have
* a valid GPU address.
* @priv: back pointer to the process that owns this memory
* @pending_free: if !0, userspace requested that his memory be freed, but there
* are still references to it.
* @dev_priv: back pointer to the device file that created this entry.
* @metadata: String containing user specified metadata for the entry
* @work: Work struct used to schedule a kgsl_mem_entry_put in atomic contexts
* @bind_lock: Lock for sparse memory bindings
* @bind_tree: RB Tree for sparse memory bindings
*/
struct kgsl_mem_entry {
struct kref refcount;
struct kgsl_memdesc memdesc;
void *priv_data;
struct rb_node node;
unsigned int id;
struct kgsl_process_private *priv;
int pending_free;
char metadata[KGSL_GPUOBJ_ALLOC_METADATA_MAX + 1];
struct work_struct work;
spinlock_t bind_lock;
struct rb_root bind_tree;
};
struct kgsl_device_private;
struct kgsl_event_group;
typedef void (*kgsl_event_func)(struct kgsl_device *, struct kgsl_event_group *,
void *, int);
/**
* struct kgsl_event - KGSL GPU timestamp event
* @device: Pointer to the KGSL device that owns the event
* @context: Pointer to the context that owns the event
* @timestamp: Timestamp for the event to expire
* @func: Callback function for for the event when it expires
* @priv: Private data passed to the callback function
* @node: List node for the kgsl_event_group list
* @created: Jiffies when the event was created
* @work: Work struct for dispatching the callback
* @result: KGSL event result type to pass to the callback
* group: The event group this event belongs to
*/
struct kgsl_event {
struct kgsl_device *device;
struct kgsl_context *context;
unsigned int timestamp;
kgsl_event_func func;
void *priv;
struct list_head node;
unsigned int created;
struct work_struct work;
int result;
struct kgsl_event_group *group;
};
typedef int (*readtimestamp_func)(struct kgsl_device *, void *,
enum kgsl_timestamp_type, unsigned int *);
/**
* struct event_group - A list of GPU events
* @context: Pointer to the active context for the events
* @lock: Spinlock for protecting the list
* @events: List of active GPU events
* @group: Node for the master group list
* @processed: Last processed timestamp
* @name: String name for the group (for the debugfs file)
* @readtimestamp: Function pointer to read a timestamp
* @priv: Priv member to pass to the readtimestamp function
*/
struct kgsl_event_group {
struct kgsl_context *context;
spinlock_t lock;
struct list_head events;
struct list_head group;
unsigned int processed;
char name[64];
readtimestamp_func readtimestamp;
void *priv;
};
/**
* struct sparse_bind_object - Bind metadata
* @node: Node for the rb tree
* @p_memdesc: Physical memdesc bound to
* @v_off: Offset of bind in the virtual entry
* @p_off: Offset of bind in the physical memdesc
* @size: Size of the bind
* @flags: Flags for the bind
*/
struct sparse_bind_object {
struct rb_node node;
struct kgsl_memdesc *p_memdesc;
uint64_t v_off;
uint64_t p_off;
uint64_t size;
uint64_t flags;
};
long kgsl_ioctl_device_getproperty(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_device_setproperty(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_device_waittimestamp_ctxtid(struct kgsl_device_private
*dev_priv, unsigned int cmd, void *data);
long kgsl_ioctl_rb_issueibcmds(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_submit_commands(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_cmdstream_readtimestamp_ctxtid(struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data);
long kgsl_ioctl_cmdstream_freememontimestamp_ctxtid(
struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data);
long kgsl_ioctl_drawctxt_create(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_drawctxt_destroy(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sharedmem_free(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpumem_free_id(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_map_user_mem(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpumem_sync_cache(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpumem_sync_cache_bulk(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sharedmem_flush_cache(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpumem_alloc(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpumem_alloc_id(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpumem_get_info(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_timestamp_event(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpuobj_alloc(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpuobj_free(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpuobj_info(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpuobj_import(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpuobj_sync(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpu_command(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpuobj_set_info(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sparse_phys_alloc(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sparse_phys_free(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sparse_virt_alloc(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sparse_virt_free(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sparse_bind(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_sparse_unbind(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_gpu_sparse_command(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
void kgsl_mem_entry_destroy(struct kref *kref);
void kgsl_get_egl_counts(struct kgsl_mem_entry *entry,
int *egl_surface_count, int *egl_image_count);
struct kgsl_mem_entry * __must_check
kgsl_sharedmem_find(struct kgsl_process_private *private, uint64_t gpuaddr);
struct kgsl_mem_entry * __must_check
kgsl_sharedmem_find_id(struct kgsl_process_private *process, unsigned int id);
extern const struct dev_pm_ops kgsl_pm_ops;
int kgsl_suspend_driver(struct platform_device *pdev, pm_message_t state);
int kgsl_resume_driver(struct platform_device *pdev);
struct kgsl_mem_entry *gpumem_alloc_entry(struct kgsl_device_private *dev_priv,
uint64_t size, uint64_t flags);
long gpumem_free_entry(struct kgsl_mem_entry *entry);
enum kgsl_mmutype kgsl_mmu_get_mmutype(struct kgsl_device *device);
void kgsl_mmu_add_global(struct kgsl_device *device,
struct kgsl_memdesc *memdesc, const char *name);
void kgsl_mmu_remove_global(struct kgsl_device *device,
struct kgsl_memdesc *memdesc);
/* Helper functions */
int kgsl_request_irq(struct platform_device *pdev, const char *name,
irq_handler_t handler, void *data);
int __init kgsl_core_init(void);
void kgsl_core_exit(void);
static inline int kgsl_gpuaddr_in_memdesc(const struct kgsl_memdesc *memdesc,
uint64_t gpuaddr, uint64_t size)
{
/* set a minimum size to search for */
if (!size)
size = 1;
/* don't overflow */
if (size > U64_MAX - gpuaddr)
return 0;
if (gpuaddr >= memdesc->gpuaddr &&
((gpuaddr + size) <= (memdesc->gpuaddr + memdesc->size))) {
return 1;
}
return 0;
}
static inline void *kgsl_memdesc_map(struct kgsl_memdesc *memdesc)
{
if (memdesc->ops && memdesc->ops->map_kernel)
memdesc->ops->map_kernel(memdesc);
return memdesc->hostptr;
}
static inline void kgsl_memdesc_unmap(struct kgsl_memdesc *memdesc)
{
if (memdesc->ops && memdesc->ops->unmap_kernel)
memdesc->ops->unmap_kernel(memdesc);
}
static inline void *kgsl_gpuaddr_to_vaddr(struct kgsl_memdesc *memdesc,
uint64_t gpuaddr)
{
void *hostptr = NULL;
if ((gpuaddr >= memdesc->gpuaddr) &&
(gpuaddr < (memdesc->gpuaddr + memdesc->size)))
hostptr = kgsl_memdesc_map(memdesc);
return hostptr != NULL ? hostptr + (gpuaddr - memdesc->gpuaddr) : NULL;
}
static inline int timestamp_cmp(unsigned int a, unsigned int b)
{
/* check for equal */
if (a == b)
return 0;
/* check for greater-than for non-rollover case */
if ((a > b) && (a - b < KGSL_TIMESTAMP_WINDOW))
return 1;
/* check for greater-than for rollover case
* note that <= is required to ensure that consistent
* results are returned for values whose difference is
* equal to the window size
*/
a += KGSL_TIMESTAMP_WINDOW;
b += KGSL_TIMESTAMP_WINDOW;
return ((a > b) && (a - b <= KGSL_TIMESTAMP_WINDOW)) ? 1 : -1;
}
/**
* kgsl_schedule_work() - Schedule a work item on the KGSL workqueue
* @work: work item to schedule
*/
static inline void kgsl_schedule_work(struct work_struct *work)
{
queue_work(kgsl_driver.workqueue, work);
}
static inline int
kgsl_mem_entry_get(struct kgsl_mem_entry *entry)
{
if (entry)
return kref_get_unless_zero(&entry->refcount);
return 0;
}
static inline void
kgsl_mem_entry_put(struct kgsl_mem_entry *entry)
{
if (entry)
kref_put(&entry->refcount, kgsl_mem_entry_destroy);
}
/*
* kgsl_addr_range_overlap() - Checks if 2 ranges overlap
* @gpuaddr1: Start of first address range
* @size1: Size of first address range
* @gpuaddr2: Start of second address range
* @size2: Size of second address range
*
* Function returns true if the 2 given address ranges overlap
* else false
*/
static inline bool kgsl_addr_range_overlap(uint64_t gpuaddr1,
uint64_t size1, uint64_t gpuaddr2, uint64_t size2)
{
if ((size1 > (U64_MAX - gpuaddr1)) || (size2 > (U64_MAX - gpuaddr2)))
return false;
return !(((gpuaddr1 + size1) <= gpuaddr2) ||
(gpuaddr1 >= (gpuaddr2 + size2)));
}
static inline int kgsl_copy_from_user(void *dest, void __user *src,
unsigned int ksize, unsigned int usize)
{
unsigned int copy = ksize < usize ? ksize : usize;
if (copy == 0)
return -EINVAL;
return copy_from_user(dest, src, copy) ? -EFAULT : 0;
}
static inline void kgsl_gpu_sysfs_add_link(struct kobject *dst,
struct kobject *src, const char *src_name,
const char *dst_name)
{
struct kernfs_node *old;
if (dst == NULL || src == NULL)
return;
old = sysfs_get_dirent(src->sd, src_name);
if (IS_ERR_OR_NULL(old))
return;
kernfs_create_link(dst->sd, dst_name, old);
}
static inline bool kgsl_is_compat_task(void)
{
return (BITS_PER_LONG == 32) || is_compat_task();
}
#endif /* __KGSL_H */

378
drivers/gpu/msm/kgsl_compat.c Normal file
View file

@ -0,0 +1,378 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
*/
#include "kgsl_device.h"
#include "kgsl_compat.h"
#include "kgsl_sync.h"
static long
kgsl_ioctl_device_getproperty_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_device_getproperty_compat *param32 = data;
struct kgsl_device_getproperty param;
param.type = param32->type;
param.value = compat_ptr(param32->value);
param.sizebytes = (size_t)param32->sizebytes;
return kgsl_ioctl_device_getproperty(dev_priv, cmd, &param);
}
static long
kgsl_ioctl_device_setproperty_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_device_getproperty_compat *param32 = data;
struct kgsl_device_getproperty param;
param.type = param32->type;
param.value = compat_ptr(param32->value);
param.sizebytes = (size_t)param32->sizebytes;
return kgsl_ioctl_device_setproperty(dev_priv, cmd, &param);
}
static long
kgsl_ioctl_submit_commands_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result;
struct kgsl_submit_commands_compat *param32 = data;
struct kgsl_submit_commands param;
param.context_id = param32->context_id;
param.flags = param32->flags;
param.cmdlist = compat_ptr(param32->cmdlist);
param.numcmds = param32->numcmds;
param.synclist = compat_ptr(param32->synclist);
param.numsyncs = param32->numsyncs;
param.timestamp = param32->timestamp;
result = kgsl_ioctl_submit_commands(dev_priv, cmd, &param);
param32->timestamp = param.timestamp;
return result;
}
static long
kgsl_ioctl_rb_issueibcmds_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result;
struct kgsl_ringbuffer_issueibcmds_compat *param32 = data;
struct kgsl_ringbuffer_issueibcmds param;
param.drawctxt_id = param32->drawctxt_id;
param.flags = param32->flags;
param.ibdesc_addr = (unsigned long)param32->ibdesc_addr;
param.numibs = param32->numibs;
param.timestamp = param32->timestamp;
result = kgsl_ioctl_rb_issueibcmds(dev_priv, cmd, &param);
param32->timestamp = param.timestamp;
return result;
}
static long kgsl_ioctl_cmdstream_freememontimestamp_ctxtid_compat(
struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data)
{
struct kgsl_cmdstream_freememontimestamp_ctxtid_compat *param32 = data;
struct kgsl_cmdstream_freememontimestamp_ctxtid param;
param.context_id = param32->context_id;
param.gpuaddr = (unsigned long)param32->gpuaddr;
param.type = param32->type;
param.timestamp = param32->timestamp;
return kgsl_ioctl_cmdstream_freememontimestamp_ctxtid(dev_priv, cmd,
&param);
}
static long kgsl_ioctl_sharedmem_free_compat(struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data)
{
struct kgsl_sharedmem_free_compat *param32 = data;
struct kgsl_sharedmem_free param;
param.gpuaddr = (unsigned long)param32->gpuaddr;
return kgsl_ioctl_sharedmem_free(dev_priv, cmd, &param);
}
static long kgsl_ioctl_map_user_mem_compat(struct kgsl_device_private
*dev_priv, unsigned int cmd,
void *data)
{
int result = 0;
struct kgsl_map_user_mem_compat *param32 = data;
struct kgsl_map_user_mem param;
param.fd = param32->fd;
param.gpuaddr = (unsigned long)param32->gpuaddr;
param.len = (size_t)param32->len;
param.offset = (size_t)param32->offset;
param.hostptr = (unsigned long)param32->hostptr;
param.memtype = param32->memtype;
param.flags = param32->flags;
result = kgsl_ioctl_map_user_mem(dev_priv, cmd, &param);
param32->gpuaddr = gpuaddr_to_compat(param.gpuaddr);
param32->flags = param.flags;
return result;
}
static long
kgsl_ioctl_gpumem_sync_cache_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_gpumem_sync_cache_compat *param32 = data;
struct kgsl_gpumem_sync_cache param;
param.gpuaddr = (unsigned long)param32->gpuaddr;
param.id = param32->id;
param.op = param32->op;
param.offset = (size_t)param32->offset;
param.length = (size_t)param32->length;
return kgsl_ioctl_gpumem_sync_cache(dev_priv, cmd, &param);
}
static long
kgsl_ioctl_gpumem_sync_cache_bulk_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_gpumem_sync_cache_bulk_compat *param32 = data;
struct kgsl_gpumem_sync_cache_bulk param;
param.id_list = compat_ptr(param32->id_list);
param.count = param32->count;
param.op = param32->op;
return kgsl_ioctl_gpumem_sync_cache_bulk(dev_priv, cmd, &param);
}
static long
kgsl_ioctl_sharedmem_flush_cache_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_sharedmem_free_compat *param32 = data;
struct kgsl_sharedmem_free param;
param.gpuaddr = (unsigned long)param32->gpuaddr;
return kgsl_ioctl_sharedmem_flush_cache(dev_priv, cmd, &param);
}
static long
kgsl_ioctl_gpumem_alloc_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result = 0;
struct kgsl_gpumem_alloc_compat *param32 = data;
struct kgsl_gpumem_alloc param;
param.gpuaddr = (unsigned long)param32->gpuaddr;
param.size = (size_t)param32->size;
param.flags = param32->flags;
/*
* Since this is a 32 bit application the page aligned size is expected
* to fit inside of 32 bits - check for overflow and return error if so
*/
if (PAGE_ALIGN(param.size) >= UINT_MAX)
return -EINVAL;
result = kgsl_ioctl_gpumem_alloc(dev_priv, cmd, &param);
param32->gpuaddr = gpuaddr_to_compat(param.gpuaddr);
param32->size = sizet_to_compat(param.size);
param32->flags = param.flags;
return result;
}
static long
kgsl_ioctl_gpumem_alloc_id_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result = 0;
struct kgsl_gpumem_alloc_id_compat *param32 = data;
struct kgsl_gpumem_alloc_id param;
param.id = param32->id;
param.flags = param32->flags;
param.size = (size_t)param32->size;
param.mmapsize = (size_t)param32->mmapsize;
param.gpuaddr = (unsigned long)param32->gpuaddr;
/*
* Since this is a 32 bit application the page aligned size is expected
* to fit inside of 32 bits - check for overflow and return error if so
*/
if (PAGE_ALIGN(param.size) >= UINT_MAX)
return -EINVAL;
result = kgsl_ioctl_gpumem_alloc_id(dev_priv, cmd, &param);
param32->id = param.id;
param32->flags = param.flags;
param32->size = sizet_to_compat(param.size);
param32->mmapsize = sizet_to_compat(param.mmapsize);
param32->gpuaddr = gpuaddr_to_compat(param.gpuaddr);
return result;
}
static long
kgsl_ioctl_gpumem_get_info_compat(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int result = 0;
struct kgsl_gpumem_get_info_compat *param32 = data;
struct kgsl_gpumem_get_info param;
param.gpuaddr = (unsigned long)param32->gpuaddr;
param.id = param32->id;
param.flags = param32->flags;
param.size = (size_t)param32->size;
param.mmapsize = (size_t)param32->mmapsize;
param.useraddr = (unsigned long)param32->useraddr;
result = kgsl_ioctl_gpumem_get_info(dev_priv, cmd, &param);
param32->gpuaddr = gpuaddr_to_compat(param.gpuaddr);
param32->id = param.id;
param32->flags = param.flags;
param32->size = sizet_to_compat(param.size);
param32->mmapsize = sizet_to_compat(param.mmapsize);
param32->useraddr = (compat_ulong_t)param.useraddr;
return result;
}
static long kgsl_ioctl_timestamp_event_compat(struct kgsl_device_private
*dev_priv, unsigned int cmd, void *data)
{
struct kgsl_timestamp_event_compat *param32 = data;
struct kgsl_timestamp_event param;
param.type = param32->type;
param.timestamp = param32->timestamp;
param.context_id = param32->context_id;
param.priv = compat_ptr(param32->priv);
param.len = (size_t)param32->len;
return kgsl_ioctl_timestamp_event(dev_priv, cmd, &param);
}
static const struct kgsl_ioctl kgsl_compat_ioctl_funcs[] = {
KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_GETPROPERTY_COMPAT,
kgsl_ioctl_device_getproperty_compat),
/* IOCTL_KGSL_DEVICE_WAITTIMESTAMP is no longer supported */
KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_WAITTIMESTAMP_CTXTID,
kgsl_ioctl_device_waittimestamp_ctxtid),
KGSL_IOCTL_FUNC(IOCTL_KGSL_RINGBUFFER_ISSUEIBCMDS_COMPAT,
kgsl_ioctl_rb_issueibcmds_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SUBMIT_COMMANDS_COMPAT,
kgsl_ioctl_submit_commands_compat),
/* IOCTL_KGSL_CMDSTREAM_READTIMESTAMP is no longer supported */
KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_READTIMESTAMP_CTXTID,
kgsl_ioctl_cmdstream_readtimestamp_ctxtid),
/* IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP is no longer supported */
KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_CTXTID_COMPAT,
kgsl_ioctl_cmdstream_freememontimestamp_ctxtid_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_CREATE,
kgsl_ioctl_drawctxt_create),
KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_DESTROY,
kgsl_ioctl_drawctxt_destroy),
KGSL_IOCTL_FUNC(IOCTL_KGSL_MAP_USER_MEM_COMPAT,
kgsl_ioctl_map_user_mem_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FREE_COMPAT,
kgsl_ioctl_sharedmem_free_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FLUSH_CACHE_COMPAT,
kgsl_ioctl_sharedmem_flush_cache_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_ALLOC_COMPAT,
kgsl_ioctl_gpumem_alloc_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMESTAMP_EVENT_COMPAT,
kgsl_ioctl_timestamp_event_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SETPROPERTY_COMPAT,
kgsl_ioctl_device_setproperty_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_ALLOC_ID_COMPAT,
kgsl_ioctl_gpumem_alloc_id_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_FREE_ID,
kgsl_ioctl_gpumem_free_id),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_GET_INFO_COMPAT,
kgsl_ioctl_gpumem_get_info_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_SYNC_CACHE_COMPAT,
kgsl_ioctl_gpumem_sync_cache_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_SYNC_CACHE_BULK_COMPAT,
kgsl_ioctl_gpumem_sync_cache_bulk_compat),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_CREATE,
kgsl_ioctl_syncsource_create),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_DESTROY,
kgsl_ioctl_syncsource_destroy),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_CREATE_FENCE,
kgsl_ioctl_syncsource_create_fence),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_SIGNAL_FENCE,
kgsl_ioctl_syncsource_signal_fence),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_ALLOC,
kgsl_ioctl_gpuobj_alloc),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_FREE,
kgsl_ioctl_gpuobj_free),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_INFO,
kgsl_ioctl_gpuobj_info),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_IMPORT,
kgsl_ioctl_gpuobj_import),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_SYNC,
kgsl_ioctl_gpuobj_sync),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_COMMAND,
kgsl_ioctl_gpu_command),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_SET_INFO,
kgsl_ioctl_gpuobj_set_info),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_ALLOC,
kgsl_ioctl_sparse_phys_alloc),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_FREE,
kgsl_ioctl_sparse_phys_free),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_ALLOC,
kgsl_ioctl_sparse_virt_alloc),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_FREE,
kgsl_ioctl_sparse_virt_free),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_BIND,
kgsl_ioctl_sparse_bind),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_SPARSE_COMMAND,
kgsl_ioctl_gpu_sparse_command),
};
long kgsl_compat_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
{
struct kgsl_device_private *dev_priv = filep->private_data;
struct kgsl_device *device = dev_priv->device;
long ret = kgsl_ioctl_helper(filep, cmd, arg, kgsl_compat_ioctl_funcs,
ARRAY_SIZE(kgsl_compat_ioctl_funcs));
/*
* If the command was unrecognized in the generic core, try the device
* specific function
*/
if (ret == -ENOIOCTLCMD) {
if (device->ftbl->compat_ioctl != NULL)
return device->ftbl->compat_ioctl(dev_priv, cmd, arg);
dev_err(device->dev, "invalid ioctl code 0x%08X\n", cmd);
}
return ret;
}

244
drivers/gpu/msm/kgsl_compat.h Normal file
View file

@ -0,0 +1,244 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2013-2017,2019 The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_COMPAT_H
#define __KGSL_COMPAT_H
#include <linux/compat.h>
#include "uapi/msm_kgsl.h"
#ifdef CONFIG_COMPAT
struct kgsl_ibdesc_compat {
compat_ulong_t gpuaddr;
unsigned int __pad;
compat_size_t sizedwords;
unsigned int ctrl;
};
struct kgsl_cmd_syncpoint_compat {
int type;
compat_uptr_t priv;
compat_size_t size;
};
struct kgsl_devinfo_compat {
unsigned int device_id;
unsigned int chip_id;
unsigned int mmu_enabled;
compat_ulong_t gmem_gpubaseaddr;
unsigned int gpu_id;
compat_size_t gmem_sizebytes;
};
struct kgsl_shadowprop_compat {
compat_ulong_t gpuaddr;
compat_size_t size;
unsigned int flags;
};
struct kgsl_device_constraint_compat {
unsigned int type;
unsigned int context_id;
compat_uptr_t data;
compat_size_t size;
};
struct kgsl_device_getproperty_compat {
unsigned int type;
compat_uptr_t value;
compat_size_t sizebytes;
};
#define IOCTL_KGSL_DEVICE_GETPROPERTY_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x2, struct kgsl_device_getproperty_compat)
#define IOCTL_KGSL_SETPROPERTY_COMPAT \
_IOW(KGSL_IOC_TYPE, 0x32, struct kgsl_device_getproperty_compat)
struct kgsl_submit_commands_compat {
unsigned int context_id;
unsigned int flags;
compat_uptr_t cmdlist;
unsigned int numcmds;
compat_uptr_t synclist;
unsigned int numsyncs;
unsigned int timestamp;
/* private: reserved for future use */
unsigned int __pad[4];
};
#define IOCTL_KGSL_SUBMIT_COMMANDS_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x3D, struct kgsl_submit_commands_compat)
struct kgsl_ringbuffer_issueibcmds_compat {
unsigned int drawctxt_id;
compat_ulong_t ibdesc_addr;
unsigned int numibs;
unsigned int timestamp; /* output param */
unsigned int flags;
};
#define IOCTL_KGSL_RINGBUFFER_ISSUEIBCMDS_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x10, struct kgsl_ringbuffer_issueibcmds_compat)
struct kgsl_cmdstream_freememontimestamp_ctxtid_compat {
unsigned int context_id;
compat_ulong_t gpuaddr;
unsigned int type;
unsigned int timestamp;
};
#define IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_CTXTID_COMPAT \
_IOW(KGSL_IOC_TYPE, 0x17, \
struct kgsl_cmdstream_freememontimestamp_ctxtid_compat)
struct kgsl_map_user_mem_compat {
int fd;
compat_ulong_t gpuaddr;
compat_size_t len;
compat_size_t offset;
compat_ulong_t hostptr;
enum kgsl_user_mem_type memtype;
unsigned int flags;
};
#define IOCTL_KGSL_MAP_USER_MEM_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x15, struct kgsl_map_user_mem_compat)
struct kgsl_sharedmem_free_compat {
compat_ulong_t gpuaddr;
};
#define IOCTL_KGSL_SHAREDMEM_FLUSH_CACHE_COMPAT \
_IOW(KGSL_IOC_TYPE, 0x24, struct kgsl_sharedmem_free_compat)
#define IOCTL_KGSL_SHAREDMEM_FREE_COMPAT \
_IOW(KGSL_IOC_TYPE, 0x21, struct kgsl_sharedmem_free_compat)
struct kgsl_gpumem_alloc_compat {
compat_ulong_t gpuaddr; /* output param */
compat_size_t size;
unsigned int flags;
};
#define IOCTL_KGSL_GPUMEM_ALLOC_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x2f, struct kgsl_gpumem_alloc_compat)
struct kgsl_cff_syncmem_compat {
compat_ulong_t gpuaddr;
compat_size_t len;
unsigned int __pad[2]; /* For future binary compatibility */
};
#define IOCTL_KGSL_CFF_SYNCMEM_COMPAT \
_IOW(KGSL_IOC_TYPE, 0x30, struct kgsl_cff_syncmem_compat)
struct kgsl_timestamp_event_compat {
int type; /* Type of event (see list below) */
unsigned int timestamp; /* Timestamp to trigger event on */
unsigned int context_id; /* Context for the timestamp */
compat_uptr_t priv; /* Pointer to the event specific blob */
compat_size_t len; /* Size of the event specific blob */
};
#define IOCTL_KGSL_TIMESTAMP_EVENT_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x33, struct kgsl_timestamp_event_compat)
struct kgsl_gpumem_alloc_id_compat {
unsigned int id;
unsigned int flags;
compat_size_t size;
compat_size_t mmapsize;
compat_ulong_t gpuaddr;
/* private: reserved for future use*/
unsigned int __pad[2];
};
#define IOCTL_KGSL_GPUMEM_ALLOC_ID_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x34, struct kgsl_gpumem_alloc_id_compat)
struct kgsl_gpumem_get_info_compat {
compat_ulong_t gpuaddr;
unsigned int id;
unsigned int flags;
compat_size_t size;
compat_size_t mmapsize;
compat_ulong_t useraddr;
/* private: reserved for future use*/
unsigned int __pad[4];
};
#define IOCTL_KGSL_GPUMEM_GET_INFO_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x36, struct kgsl_gpumem_get_info_compat)
struct kgsl_gpumem_sync_cache_compat {
compat_ulong_t gpuaddr;
unsigned int id;
unsigned int op;
compat_size_t offset;
compat_size_t length;
};
#define IOCTL_KGSL_GPUMEM_SYNC_CACHE_COMPAT \
_IOW(KGSL_IOC_TYPE, 0x37, struct kgsl_gpumem_sync_cache_compat)
struct kgsl_gpumem_sync_cache_bulk_compat {
compat_uptr_t id_list;
unsigned int count;
unsigned int op;
/* private: reserved for future use */
unsigned int __pad[2]; /* For future binary compatibility */
};
#define IOCTL_KGSL_GPUMEM_SYNC_CACHE_BULK_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x3C, struct kgsl_gpumem_sync_cache_bulk_compat)
struct kgsl_perfcounter_query_compat {
unsigned int groupid;
compat_uptr_t countables;
unsigned int count;
unsigned int max_counters;
unsigned int __pad[2];
};
#define IOCTL_KGSL_PERFCOUNTER_QUERY_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x3A, struct kgsl_perfcounter_query_compat)
struct kgsl_perfcounter_read_compat {
compat_uptr_t reads;
unsigned int count;
unsigned int __pad[2];
};
#define IOCTL_KGSL_PERFCOUNTER_READ_COMPAT \
_IOWR(KGSL_IOC_TYPE, 0x3B, struct kgsl_perfcounter_read_compat)
static inline compat_ulong_t gpuaddr_to_compat(unsigned long gpuaddr)
{
WARN(gpuaddr >> 32, "Top 32 bits of gpuaddr have been set\n");
return (compat_ulong_t)gpuaddr;
}
static inline compat_size_t sizet_to_compat(size_t size)
{
WARN(size >> 32, "Size greater than 4G\n");
return (compat_size_t)size;
}
long kgsl_compat_ioctl(struct file *filep, unsigned int cmd,
unsigned long arg);
#else
static inline long kgsl_compat_ioctl(struct file *filep, unsigned int cmd,
unsigned long arg)
{
return -EINVAL;
}
#endif /* CONFIG_COMPAT */
#endif /* __KGSL_COMPAT_H */

445
drivers/gpu/msm/kgsl_debugfs.c Normal file
View file

@ -0,0 +1,445 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2002,2008-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/debugfs.h>
#include <linux/io.h>
#include "kgsl_debugfs.h"
#include "kgsl_device.h"
#include "kgsl_sharedmem.h"
struct dentry *kgsl_debugfs_dir;
static struct dentry *proc_d_debugfs;
static int _strict_set(void *data, u64 val)
{
kgsl_sharedmem_set_noretry(val ? true : false);
return 0;
}
static int _strict_get(void *data, u64 *val)
{
*val = kgsl_sharedmem_get_noretry();
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(_strict_fops, _strict_get, _strict_set, "%llu\n");
static void kgsl_qdss_gfx_register_probe(struct kgsl_device *device)
{
struct resource *res;
res = platform_get_resource_byname(device->pdev, IORESOURCE_MEM,
"qdss_gfx");
if (res == NULL)
return;
device->qdss_gfx_virt = devm_ioremap(device->dev, res->start,
resource_size(res));
if (device->qdss_gfx_virt == NULL)
dev_warn(device->dev, "qdss_gfx ioremap failed\n");
}
static int _isdb_set(void *data, u64 val)
{
struct kgsl_device *device = data;
if (device->qdss_gfx_virt == NULL)
kgsl_qdss_gfx_register_probe(device);
device->set_isdb_breakpoint = val ? true : false;
return 0;
}
static int _isdb_get(void *data, u64 *val)
{
struct kgsl_device *device = data;
*val = device->set_isdb_breakpoint ? 1 : 0;
return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(_isdb_fops, _isdb_get, _isdb_set, "%llu\n");
void kgsl_device_debugfs_init(struct kgsl_device *device)
{
struct dentry *snapshot_dir;
if (IS_ERR_OR_NULL(kgsl_debugfs_dir))
return;
device->d_debugfs = debugfs_create_dir(device->name,
kgsl_debugfs_dir);
snapshot_dir = debugfs_create_dir("snapshot", kgsl_debugfs_dir);
debugfs_create_file("break_isdb", 0644, snapshot_dir, device,
&_isdb_fops);
}
void kgsl_device_debugfs_close(struct kgsl_device *device)
{
debugfs_remove_recursive(device->d_debugfs);
}
static const char *memtype_str(int memtype)
{
if (memtype == KGSL_MEM_ENTRY_KERNEL)
return "gpumem";
else if (memtype == KGSL_MEM_ENTRY_USER)
return "usermem";
else if (memtype == KGSL_MEM_ENTRY_ION)
return "ion";
return "unknown";
}
static char get_alignflag(const struct kgsl_memdesc *m)
{
int align = kgsl_memdesc_get_align(m);
if (align >= ilog2(SZ_1M))
return 'L';
else if (align >= ilog2(SZ_64K))
return 'l';
return '-';
}
static char get_cacheflag(const struct kgsl_memdesc *m)
{
static const char table[] = {
[KGSL_CACHEMODE_WRITECOMBINE] = '-',
[KGSL_CACHEMODE_UNCACHED] = 'u',
[KGSL_CACHEMODE_WRITEBACK] = 'b',
[KGSL_CACHEMODE_WRITETHROUGH] = 't',
};
return table[kgsl_memdesc_get_cachemode(m)];
}
static int print_mem_entry(void *data, void *ptr)
{
struct seq_file *s = data;
struct kgsl_mem_entry *entry = ptr;
char flags[10];
char usage[16];
struct kgsl_memdesc *m = &entry->memdesc;
unsigned int usermem_type = kgsl_memdesc_usermem_type(m);
int egl_surface_count = 0, egl_image_count = 0;
if (m->flags & KGSL_MEMFLAGS_SPARSE_VIRT)
return 0;
flags[0] = kgsl_memdesc_is_global(m) ? 'g' : '-';
flags[1] = '-';
flags[2] = !(m->flags & KGSL_MEMFLAGS_GPUREADONLY) ? 'w' : '-';
flags[3] = get_alignflag(m);
flags[4] = get_cacheflag(m);
flags[5] = kgsl_memdesc_use_cpu_map(m) ? 'p' : '-';
flags[6] = (m->useraddr) ? 'Y' : 'N';
flags[7] = kgsl_memdesc_is_secured(m) ? 's' : '-';
flags[8] = m->flags & KGSL_MEMFLAGS_SPARSE_PHYS ? 'P' : '-';
flags[9] = '\0';
kgsl_get_memory_usage(usage, sizeof(usage), m->flags);
if (usermem_type == KGSL_MEM_ENTRY_ION)
kgsl_get_egl_counts(entry, &egl_surface_count,
&egl_image_count);
seq_printf(s, "%pK %pK %16llu %5d %9s %10s %16s %5d %16llu %6d %6d",
(uint64_t *)(uintptr_t) m->gpuaddr,
(unsigned long *) m->useraddr,
m->size, entry->id, flags,
memtype_str(usermem_type),
usage, (m->sgt ? m->sgt->nents : 0), m->mapsize,
egl_surface_count, egl_image_count);
if (entry->metadata[0] != 0)
seq_printf(s, " %s", entry->metadata);
seq_putc(s, '\n');
return 0;
}
static struct kgsl_mem_entry *process_mem_seq_find(struct seq_file *s,
void *ptr, loff_t pos)
{
struct kgsl_mem_entry *entry = ptr;
struct kgsl_process_private *private = s->private;
int id = 0;
loff_t temp_pos = 1;
if (entry != SEQ_START_TOKEN)
id = entry->id + 1;
spin_lock(&private->mem_lock);
for (entry = idr_get_next(&private->mem_idr, &id); entry;
id++, entry = idr_get_next(&private->mem_idr, &id),
temp_pos++) {
if (temp_pos == pos && kgsl_mem_entry_get(entry)) {
spin_unlock(&private->mem_lock);
goto found;
}
}
spin_unlock(&private->mem_lock);
entry = NULL;
found:
if (ptr != SEQ_START_TOKEN)
kgsl_mem_entry_put(ptr);
return entry;
}
static void *process_mem_seq_start(struct seq_file *s, loff_t *pos)
{
loff_t seq_file_offset = *pos;
if (seq_file_offset == 0)
return SEQ_START_TOKEN;
else
return process_mem_seq_find(s, SEQ_START_TOKEN,
seq_file_offset);
}
static void process_mem_seq_stop(struct seq_file *s, void *ptr)
{
if (ptr && ptr != SEQ_START_TOKEN)
kgsl_mem_entry_put(ptr);
}
static void *process_mem_seq_next(struct seq_file *s, void *ptr,
loff_t *pos)
{
++*pos;
return process_mem_seq_find(s, ptr, 1);
}
static int process_mem_seq_show(struct seq_file *s, void *ptr)
{
if (ptr == SEQ_START_TOKEN) {
seq_printf(s, "%16s %16s %16s %5s %9s %10s %16s %5s %16s %6s %6s\n",
"gpuaddr", "useraddr", "size", "id", "flags", "type",
"usage", "sglen", "mapsize", "eglsrf", "eglimg");
return 0;
} else
return print_mem_entry(s, ptr);
}
static const struct seq_operations process_mem_seq_fops = {
.start = process_mem_seq_start,
.stop = process_mem_seq_stop,
.next = process_mem_seq_next,
.show = process_mem_seq_show,
};
static int process_mem_open(struct inode *inode, struct file *file)
{
int ret;
pid_t pid = (pid_t) (unsigned long) inode->i_private;
struct seq_file *s = NULL;
struct kgsl_process_private *private = NULL;
private = kgsl_process_private_find(pid);
if (!private)
return -ENODEV;
ret = seq_open(file, &process_mem_seq_fops);
if (ret)
kgsl_process_private_put(private);
else {
s = file->private_data;
s->private = private;
}
return ret;
}
static int process_mem_release(struct inode *inode, struct file *file)
{
struct kgsl_process_private *private =
((struct seq_file *)file->private_data)->private;
if (private)
kgsl_process_private_put(private);
return seq_release(inode, file);
}
static const struct file_operations process_mem_fops = {
.open = process_mem_open,
.read = seq_read,
.llseek = seq_lseek,
.release = process_mem_release,
};
static int print_sparse_mem_entry(int id, void *ptr, void *data)
{
struct seq_file *s = data;
struct kgsl_mem_entry *entry = ptr;
struct kgsl_memdesc *m = &entry->memdesc;
struct rb_node *node;
if (!(m->flags & KGSL_MEMFLAGS_SPARSE_VIRT))
return 0;
spin_lock(&entry->bind_lock);
node = rb_first(&entry->bind_tree);
while (node != NULL) {
struct sparse_bind_object *obj = rb_entry(node,
struct sparse_bind_object, node);
seq_printf(s, "%5d %16llx %16llx %16llx %16llx\n",
entry->id, entry->memdesc.gpuaddr,
obj->v_off, obj->size, obj->p_off);
node = rb_next(node);
}
spin_unlock(&entry->bind_lock);
seq_putc(s, '\n');
return 0;
}
static int process_sparse_mem_print(struct seq_file *s, void *unused)
{
struct kgsl_process_private *private = s->private;
seq_printf(s, "%5s %16s %16s %16s %16s\n",
"v_id", "gpuaddr", "v_offset", "v_size", "p_offset");
spin_lock(&private->mem_lock);
idr_for_each(&private->mem_idr, print_sparse_mem_entry, s);
spin_unlock(&private->mem_lock);
return 0;
}
static int process_sparse_mem_open(struct inode *inode, struct file *file)
{
int ret;
pid_t pid = (pid_t) (unsigned long) inode->i_private;
struct kgsl_process_private *private = NULL;
private = kgsl_process_private_find(pid);
if (!private)
return -ENODEV;
ret = single_open(file, process_sparse_mem_print, private);
if (ret)
kgsl_process_private_put(private);
return ret;
}
static const struct file_operations process_sparse_mem_fops = {
.open = process_sparse_mem_open,
.read = seq_read,
.llseek = seq_lseek,
.release = process_mem_release,
};
static int globals_print(struct seq_file *s, void *unused)
{
kgsl_print_global_pt_entries(s);
return 0;
}
static int globals_open(struct inode *inode, struct file *file)
{
return single_open(file, globals_print, NULL);
}
static int globals_release(struct inode *inode, struct file *file)
{
return single_release(inode, file);
}
static const struct file_operations global_fops = {
.open = globals_open,
.read = seq_read,
.llseek = seq_lseek,
.release = globals_release,
};
/**
* kgsl_process_init_debugfs() - Initialize debugfs for a process
* @private: Pointer to process private structure created for the process
*
* kgsl_process_init_debugfs() is called at the time of creating the
* process struct when a process opens kgsl device for the first time.
* This function is not fatal - all we do is print a warning message if
* the files can't be created
*/
void kgsl_process_init_debugfs(struct kgsl_process_private *private)
{
unsigned char name[16];
struct dentry *dentry;
snprintf(name, sizeof(name), "%d", private->pid);
private->debug_root = debugfs_create_dir(name, proc_d_debugfs);
/*
* Both debugfs_create_dir() and debugfs_create_file() return
* ERR_PTR(-ENODEV) if debugfs is disabled in the kernel but return
* NULL on error when it is enabled. For both usages we need to check
* for ERROR or NULL and only print a warning on an actual failure
* (i.e. - when the return value is NULL)
*/
if (IS_ERR_OR_NULL(private->debug_root)) {
WARN((private->debug_root == NULL),
"Unable to create debugfs dir for %s\n", name);
private->debug_root = NULL;
return;
}
dentry = debugfs_create_file("mem", 0444, private->debug_root,
(void *) ((unsigned long) private->pid), &process_mem_fops);
if (IS_ERR_OR_NULL(dentry))
WARN((dentry == NULL),
"Unable to create 'mem' file for %s\n", name);
dentry = debugfs_create_file("sparse_mem", 0444, private->debug_root,
(void *) ((unsigned long) private->pid),
&process_sparse_mem_fops);
if (IS_ERR_OR_NULL(dentry))
WARN((dentry == NULL),
"Unable to create 'sparse_mem' file for %s\n", name);
}
void kgsl_core_debugfs_init(void)
{
struct dentry *debug_dir;
kgsl_debugfs_dir = debugfs_create_dir("kgsl", NULL);
if (IS_ERR_OR_NULL(kgsl_debugfs_dir))
return;
debugfs_create_file("globals", 0444, kgsl_debugfs_dir, NULL,
&global_fops);
debug_dir = debugfs_create_dir("debug", kgsl_debugfs_dir);
debugfs_create_file("strict_memory", 0644, debug_dir, NULL,
&_strict_fops);
proc_d_debugfs = debugfs_create_dir("proc", kgsl_debugfs_dir);
}
void kgsl_core_debugfs_close(void)
{
debugfs_remove_recursive(kgsl_debugfs_dir);
}

36
drivers/gpu/msm/kgsl_debugfs.h Normal file
View file

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2002,2008-2011,2013,2015,2017,2019, The Linux Foundation. All rights reserved.
*/
#ifndef _KGSL_DEBUGFS_H
#define _KGSL_DEBUGFS_H
struct kgsl_device;
struct kgsl_process_private;
#ifdef CONFIG_DEBUG_FS
void kgsl_core_debugfs_init(void);
void kgsl_core_debugfs_close(void);
void kgsl_device_debugfs_init(struct kgsl_device *device);
void kgsl_device_debugfs_close(struct kgsl_device *device);
extern struct dentry *kgsl_debugfs_dir;
static inline struct dentry *kgsl_get_debugfs_dir(void)
{
return kgsl_debugfs_dir;
}
void kgsl_process_init_debugfs(struct kgsl_process_private *priv);
#else
static inline void kgsl_core_debugfs_init(void) { }
static inline void kgsl_device_debugfs_init(struct kgsl_device *device) { }
static inline void kgsl_device_debugfs_close(struct kgsl_device *device) { }
static inline void kgsl_core_debugfs_close(void) { }
static inline struct dentry *kgsl_get_debugfs_dir(void) { return NULL; }
static inline void kgsl_process_init_debugfs(struct kgsl_process_private *priv)
{
}
#endif
#endif

961
drivers/gpu/msm/kgsl_device.h Normal file
View file

@ -0,0 +1,961 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_DEVICE_H
#define __KGSL_DEVICE_H
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include "kgsl.h"
#include "kgsl_drawobj.h"
#include "kgsl_mmu.h"
#define KGSL_IOCTL_FUNC(_cmd, _func) \
[_IOC_NR((_cmd))] = \
{ .cmd = (_cmd), .func = (_func) }
/*
* KGSL device state is initialized to INIT when platform_probe *
* successfully initialized the device. Once a device has been opened *
* (started) it becomes active. NAP implies that only low latency *
* resources (for now clocks on some platforms) are off. SLEEP implies *
* that the KGSL module believes a device is idle (has been inactive *
* past its timer) and all system resources are released. SUSPEND is *
* requested by the kernel and will be enforced upon all open devices. *
* RESET indicates that GPU or GMU hang happens. KGSL is handling *
* snapshot or recover GPU from hang. *
*/
#define KGSL_STATE_NONE 0x00000000
#define KGSL_STATE_INIT 0x00000001
#define KGSL_STATE_ACTIVE 0x00000002
#define KGSL_STATE_NAP 0x00000004
#define KGSL_STATE_SUSPEND 0x00000010
#define KGSL_STATE_AWARE 0x00000020
#define KGSL_STATE_SLUMBER 0x00000080
#define KGSL_STATE_RESET 0x00000100
/**
* enum kgsl_event_results - result codes passed to an event callback when the
* event is retired or cancelled
* @KGSL_EVENT_RETIRED: The timestamp associated with the event retired
* successflly
* @KGSL_EVENT_CANCELLED: The event was cancelled before the event was fired
*/
enum kgsl_event_results {
KGSL_EVENT_RETIRED = 1,
KGSL_EVENT_CANCELLED = 2,
};
#define KGSL_FLAG_WAKE_ON_TOUCH BIT(0)
#define KGSL_FLAG_SPARSE BIT(1)
/*
* "list" of event types for ftrace symbolic magic
*/
#define KGSL_CONTEXT_FLAGS \
{ KGSL_CONTEXT_NO_GMEM_ALLOC, "NO_GMEM_ALLOC" }, \
{ KGSL_CONTEXT_PREAMBLE, "PREAMBLE" }, \
{ KGSL_CONTEXT_TRASH_STATE, "TRASH_STATE" }, \
{ KGSL_CONTEXT_CTX_SWITCH, "CTX_SWITCH" }, \
{ KGSL_CONTEXT_PER_CONTEXT_TS, "PER_CONTEXT_TS" }, \
{ KGSL_CONTEXT_USER_GENERATED_TS, "USER_TS" }, \
{ KGSL_CONTEXT_NO_FAULT_TOLERANCE, "NO_FT" }, \
{ KGSL_CONTEXT_INVALIDATE_ON_FAULT, "INVALIDATE_ON_FAULT" }, \
{ KGSL_CONTEXT_PWR_CONSTRAINT, "PWR" }, \
{ KGSL_CONTEXT_SAVE_GMEM, "SAVE_GMEM" }, \
{ KGSL_CONTEXT_IFH_NOP, "IFH_NOP" }, \
{ KGSL_CONTEXT_SECURE, "SECURE" }, \
{ KGSL_CONTEXT_NO_SNAPSHOT, "NO_SNAPSHOT" }, \
{ KGSL_CONTEXT_SPARSE, "SPARSE" }
#define KGSL_CONTEXT_ID(_context) \
((_context != NULL) ? (_context)->id : KGSL_MEMSTORE_GLOBAL)
/* Allocate 600K for the snapshot static region*/
#define KGSL_SNAPSHOT_MEMSIZE (600 * 1024)
#define MAX_L3_LEVELS 3
struct kgsl_device;
struct platform_device;
struct kgsl_device_private;
struct kgsl_context;
struct kgsl_power_stats;
struct kgsl_event;
struct kgsl_snapshot;
struct kgsl_functable {
/* Mandatory functions - these functions must be implemented
* by the client device. The driver will not check for a NULL
* pointer before calling the hook.
*/
void (*regread)(struct kgsl_device *device,
unsigned int offsetwords, unsigned int *value);
void (*regwrite)(struct kgsl_device *device,
unsigned int offsetwords, unsigned int value);
int (*idle)(struct kgsl_device *device);
bool (*isidle)(struct kgsl_device *device);
int (*suspend_context)(struct kgsl_device *device);
int (*init)(struct kgsl_device *device);
int (*start)(struct kgsl_device *device, int priority);
int (*stop)(struct kgsl_device *device);
int (*getproperty)(struct kgsl_device *device,
struct kgsl_device_getproperty *param);
int (*getproperty_compat)(struct kgsl_device *device,
struct kgsl_device_getproperty *param);
int (*waittimestamp)(struct kgsl_device *device,
struct kgsl_context *context, unsigned int timestamp,
unsigned int msecs);
int (*readtimestamp)(struct kgsl_device *device, void *priv,
enum kgsl_timestamp_type type, unsigned int *timestamp);
int (*queue_cmds)(struct kgsl_device_private *dev_priv,
struct kgsl_context *context, struct kgsl_drawobj *drawobj[],
uint32_t count, uint32_t *timestamp);
void (*power_stats)(struct kgsl_device *device,
struct kgsl_power_stats *stats);
unsigned int (*gpuid)(struct kgsl_device *device, unsigned int *chipid);
void (*snapshot)(struct kgsl_device *device,
struct kgsl_snapshot *snapshot, struct kgsl_context *context);
irqreturn_t (*irq_handler)(struct kgsl_device *device);
int (*drain)(struct kgsl_device *device);
struct kgsl_device_private * (*device_private_create)(void);
void (*device_private_destroy)(struct kgsl_device_private *dev_priv);
/*
* Optional functions - these functions are not mandatory. The
* driver will check that the function pointer is not NULL before
* calling the hook
*/
struct kgsl_context *(*drawctxt_create)
(struct kgsl_device_private *dev_priv,
uint32_t *flags);
void (*drawctxt_detach)(struct kgsl_context *context);
void (*drawctxt_destroy)(struct kgsl_context *context);
void (*drawctxt_dump)(struct kgsl_device *device,
struct kgsl_context *context);
long (*ioctl)(struct kgsl_device_private *dev_priv,
unsigned int cmd, unsigned long arg);
long (*compat_ioctl)(struct kgsl_device_private *dev_priv,
unsigned int cmd, unsigned long arg);
int (*setproperty)(struct kgsl_device_private *dev_priv,
unsigned int type, void __user *value,
unsigned int sizebytes);
int (*setproperty_compat)(struct kgsl_device_private *dev_priv,
unsigned int type, void __user *value,
unsigned int sizebytes);
void (*drawctxt_sched)(struct kgsl_device *device,
struct kgsl_context *context);
void (*resume)(struct kgsl_device *device);
int (*regulator_enable)(struct kgsl_device *device);
bool (*is_hw_collapsible)(struct kgsl_device *device);
void (*regulator_disable)(struct kgsl_device *device);
void (*pwrlevel_change_settings)(struct kgsl_device *device,
unsigned int prelevel, unsigned int postlevel, bool post);
void (*regulator_disable_poll)(struct kgsl_device *device);
void (*clk_set_options)(struct kgsl_device *device,
const char *name, struct clk *clk, bool on);
void (*gpu_model)(struct kgsl_device *device, char *str,
size_t bufsz);
void (*stop_fault_timer)(struct kgsl_device *device);
void (*dispatcher_halt)(struct kgsl_device *device);
void (*dispatcher_unhalt)(struct kgsl_device *device);
/**
* @query_property_list: query the list of properties
* supported by the device. If 'list' is NULL just return the total
* number of properties available otherwise copy up to 'count' items
* into the list and return the total number of items copied.
*/
int (*query_property_list)(struct kgsl_device *device, u32 *list,
u32 count);
bool (*is_hwcg_on)(struct kgsl_device *device);
};
struct kgsl_ioctl {
unsigned int cmd;
long (*func)(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
};
long kgsl_ioctl_helper(struct file *filep, unsigned int cmd, unsigned long arg,
const struct kgsl_ioctl *cmds, int len);
/* Flag to mark the memobj_node as a preamble */
#define MEMOBJ_PREAMBLE BIT(0)
/* Flag to mark that the memobj_node should not go to the hadrware */
#define MEMOBJ_SKIP BIT(1)
/**
* struct kgsl_memobj_node - Memory object descriptor
* @node: Local list node for the object
* @id: GPU memory ID for the object
* offset: Offset within the object
* @gpuaddr: GPU address for the object
* @flags: External flags passed by the user
* @priv: Internal flags set by the driver
*/
struct kgsl_memobj_node {
struct list_head node;
unsigned int id;
uint64_t offset;
uint64_t gpuaddr;
uint64_t size;
unsigned long flags;
unsigned long priv;
};
/**
* struct kgsl_sparseobj_node - Sparse object descriptor
* @node: Local list node for the sparse cmdbatch
* @virt_id: Virtual ID to bind/unbind
* @obj: struct kgsl_sparse_binding_object
*/
struct kgsl_sparseobj_node {
struct list_head node;
unsigned int virt_id;
struct kgsl_sparse_binding_object obj;
};
struct kgsl_device {
struct device *dev;
const char *name;
uint32_t flags;
u32 id;
/* Starting physical address for GPU registers */
unsigned long reg_phys;
/* Starting Kernel virtual address for GPU registers */
void __iomem *reg_virt;
/* Total memory size for all GPU registers */
unsigned int reg_len;
/* Kernel virtual address for GPU shader memory */
void __iomem *shader_mem_virt;
/* Starting kernel virtual address for QDSS GFX DBG register block */
void __iomem *qdss_gfx_virt;
struct kgsl_memdesc memstore;
struct kgsl_memdesc scratch;
const char *iomemname;
struct kgsl_mmu mmu;
struct gmu_core_device gmu_core;
struct completion hwaccess_gate;
struct completion halt_gate;
const struct kgsl_functable *ftbl;
struct work_struct idle_check_ws;
struct timer_list idle_timer;
struct kgsl_pwrctrl pwrctrl;
int open_count;
/* For GPU inline submission */
uint32_t submit_now;
spinlock_t submit_lock;
bool slumber;
struct mutex mutex;
uint32_t state;
uint32_t requested_state;
atomic_t active_cnt;
wait_queue_head_t wait_queue;
wait_queue_head_t active_cnt_wq;
struct platform_device *pdev;
struct dentry *d_debugfs;
struct idr context_idr;
rwlock_t context_lock;
struct {
void *ptr;
u32 size;
} snapshot_memory;
struct kgsl_snapshot *snapshot;
u32 snapshot_faultcount; /* Total number of faults since boot */
bool force_panic; /* Force panic after snapshot dump */
bool skip_ib_capture; /* Skip IB capture after snapshot */
bool prioritize_unrecoverable; /* Overwrite with new GMU snapshots */
bool set_isdb_breakpoint; /* Set isdb registers before snapshot */
/* Use CP Crash dumper to get GPU snapshot*/
bool snapshot_crashdumper;
/* Use HOST side register reads to get GPU snapshot*/
bool snapshot_legacy;
struct kobject snapshot_kobj;
struct kobject ppd_kobj;
struct kgsl_pwrscale pwrscale;
int reset_counter; /* Track how many GPU core resets have occurred */
struct workqueue_struct *events_wq;
struct device *busmondev; /* pseudo dev for GPU BW voting governor */
/* Number of active contexts seen globally for this device */
int active_context_count;
struct kobject *gpu_sysfs_kobj;
struct clk *l3_clk;
unsigned int l3_freq[MAX_L3_LEVELS];
unsigned int num_l3_pwrlevels;
/* store current L3 vote to determine if we should change our vote */
unsigned int cur_l3_pwrlevel;
};
#define KGSL_MMU_DEVICE(_mmu) \
container_of((_mmu), struct kgsl_device, mmu)
#define KGSL_DEVICE_COMMON_INIT(_dev) \
.hwaccess_gate = COMPLETION_INITIALIZER((_dev).hwaccess_gate),\
.halt_gate = COMPLETION_INITIALIZER((_dev).halt_gate),\
.idle_check_ws = __WORK_INITIALIZER((_dev).idle_check_ws,\
kgsl_idle_check),\
.context_idr = IDR_INIT((_dev).context_idr),\
.wait_queue = __WAIT_QUEUE_HEAD_INITIALIZER((_dev).wait_queue),\
.active_cnt_wq = __WAIT_QUEUE_HEAD_INITIALIZER((_dev).active_cnt_wq),\
.mutex = __MUTEX_INITIALIZER((_dev).mutex),\
.state = KGSL_STATE_NONE
/**
* enum bits for struct kgsl_context.priv
* @KGSL_CONTEXT_PRIV_SUBMITTED - The context has submitted commands to gpu.
* @KGSL_CONTEXT_PRIV_DETACHED - The context has been destroyed by userspace
* and is no longer using the gpu.
* @KGSL_CONTEXT_PRIV_INVALID - The context has been destroyed by the kernel
* because it caused a GPU fault.
* @KGSL_CONTEXT_PRIV_PAGEFAULT - The context has caused a page fault.
* @KGSL_CONTEXT_PRIV_DEVICE_SPECIFIC - this value and higher values are
* reserved for devices specific use.
*/
enum kgsl_context_priv {
KGSL_CONTEXT_PRIV_SUBMITTED = 0,
KGSL_CONTEXT_PRIV_DETACHED,
KGSL_CONTEXT_PRIV_INVALID,
KGSL_CONTEXT_PRIV_PAGEFAULT,
KGSL_CONTEXT_PRIV_DEVICE_SPECIFIC = 16,
};
struct kgsl_process_private;
/**
* struct kgsl_context - The context fields that are valid for a user defined
* context
* @refcount: kref object for reference counting the context
* @id: integer identifier for the context
* @priority; The context's priority to submit commands to GPU
* @tid: task that created this context.
* @dev_priv: pointer to the owning device instance
* @proc_priv: pointer to process private, the process that allocated the
* context
* @priv: in-kernel context flags, use KGSL_CONTEXT_* values
* @reset_status: status indication whether a gpu reset occurred and whether
* this context was responsible for causing it
* @timeline: sync timeline used to create fences that can be signaled when a
* sync_pt timestamp expires
* @events: A kgsl_event_group for this context - contains the list of GPU
* events
* @flags: flags from userspace controlling the behavior of this context
* @pwr_constraint: power constraint from userspace for this context
* @fault_count: number of times gpu hanged in last _context_throttle_time ms
* @fault_time: time of the first gpu hang in last _context_throttle_time ms
* @user_ctxt_record: memory descriptor used by CP to save/restore VPC data
* across preemption
* @total_fault_count: number of times gpu faulted in this context
* @last_faulted_cmd_ts: last faulted command batch timestamp
*/
struct kgsl_context {
struct kref refcount;
uint32_t id;
uint32_t priority;
pid_t tid;
struct kgsl_device_private *dev_priv;
struct kgsl_process_private *proc_priv;
unsigned long priv;
struct kgsl_device *device;
unsigned int reset_status;
struct kgsl_sync_timeline *ktimeline;
struct kgsl_event_group events;
unsigned int flags;
struct kgsl_pwr_constraint pwr_constraint;
struct kgsl_pwr_constraint l3_pwr_constraint;
unsigned int fault_count;
unsigned long fault_time;
struct kgsl_mem_entry *user_ctxt_record;
unsigned int total_fault_count;
unsigned int last_faulted_cmd_ts;
};
#define _context_comm(_c) \
(((_c) && (_c)->proc_priv) ? (_c)->proc_priv->comm : "unknown")
/*
* Print log messages with the context process name/pid:
* [...] kgsl kgsl-3d0: kgsl-api-test[22182]:
*/
#define pr_context(_d, _c, fmt, args...) \
dev_err((_d)->dev, "%s[%d]: " fmt, \
_context_comm((_c)), \
(_c)->proc_priv->pid, ##args)
/**
* struct kgsl_process_private - Private structure for a KGSL process (across
* all devices)
* @priv: Internal flags, use KGSL_PROCESS_* values
* @pid: ID for the task owner of the process
* @comm: task name of the process
* @mem_lock: Spinlock to protect the process memory lists
* @refcount: kref object for reference counting the process
* @idr: Iterator for assigning IDs to memory allocations
* @pagetable: Pointer to the pagetable owned by this process
* @kobj: Pointer to a kobj for the sysfs directory for this process
* @debug_root: Pointer to the debugfs root for this process
* @stats: Memory allocation statistics for this process
* @gpumem_mapped: KGSL memory mapped in the process address space
* @syncsource_idr: sync sources created by this process
* @syncsource_lock: Spinlock to protect the syncsource idr
* @fd_count: Counter for the number of FDs for this process
* @ctxt_count: Count for the number of contexts for this process
* @ctxt_count_lock: Spinlock to protect ctxt_count
*/
struct kgsl_process_private {
unsigned long priv;
pid_t pid;
char comm[TASK_COMM_LEN];
spinlock_t mem_lock;
struct kref refcount;
struct idr mem_idr;
struct kgsl_pagetable *pagetable;
struct list_head list;
struct kobject kobj;
struct dentry *debug_root;
struct {
uint64_t cur;
uint64_t max;
} stats[KGSL_MEM_ENTRY_MAX];
uint64_t gpumem_mapped;
struct idr syncsource_idr;
spinlock_t syncsource_lock;
int fd_count;
atomic_t ctxt_count;
spinlock_t ctxt_count_lock;
};
/**
* enum kgsl_process_priv_flags - Private flags for kgsl_process_private
* @KGSL_PROCESS_INIT: Set if the process structure has been set up
*/
enum kgsl_process_priv_flags {
KGSL_PROCESS_INIT = 0,
};
struct kgsl_device_private {
struct kgsl_device *device;
struct kgsl_process_private *process_priv;
};
/**
* struct kgsl_snapshot - details for a specific snapshot instance
* @ib1base: Active IB1 base address at the time of fault
* @ib2base: Active IB2 base address at the time of fault
* @ib1size: Number of DWORDS pending in IB1 at the time of fault
* @ib2size: Number of DWORDS pending in IB2 at the time of fault
* @ib1dumped: Active IB1 dump status to sansphot binary
* @ib2dumped: Active IB2 dump status to sansphot binary
* @start: Pointer to the start of the static snapshot region
* @size: Size of the current snapshot instance
* @ptr: Pointer to the next block of memory to write to during snapshotting
* @remain: Bytes left in the snapshot region
* @timestamp: Timestamp of the snapshot instance (in seconds since boot)
* @mempool: Pointer to the memory pool for storing memory objects
* @mempool_size: Size of the memory pool
* @obj_list: List of frozen GPU buffers that are waiting to be dumped.
* @cp_list: List of IB's to be dumped.
* @work: worker to dump the frozen memory
* @dump_gate: completion gate signaled by worker when it is finished.
* @process: the process that caused the hang, if known.
* @sysfs_read: Count of current reads via sysfs
* @first_read: True until the snapshot read is started
* @gmu_fault: Snapshot collected when GMU fault happened
* @recovered: True if GPU was recovered after previous snapshot
*/
struct kgsl_snapshot {
uint64_t ib1base;
uint64_t ib2base;
unsigned int ib1size;
unsigned int ib2size;
bool ib1dumped;
bool ib2dumped;
u8 *start;
size_t size;
u8 *ptr;
size_t remain;
unsigned long timestamp;
u8 *mempool;
size_t mempool_size;
struct list_head obj_list;
struct list_head cp_list;
struct work_struct work;
struct completion dump_gate;
struct kgsl_process_private *process;
unsigned int sysfs_read;
bool first_read;
bool gmu_fault;
bool recovered;
struct kgsl_device *device;
};
/**
* struct kgsl_snapshot_object - GPU memory in the snapshot
* @gpuaddr: The GPU address identified during snapshot
* @size: The buffer size identified during snapshot
* @offset: offset from start of the allocated kgsl_mem_entry
* @type: SNAPSHOT_OBJ_TYPE_* identifier.
* @entry: the reference counted memory entry for this buffer
* @node: node for kgsl_snapshot.obj_list
*/
struct kgsl_snapshot_object {
uint64_t gpuaddr;
uint64_t size;
uint64_t offset;
int type;
struct kgsl_mem_entry *entry;
struct list_head node;
};
struct kgsl_device *kgsl_get_device(int dev_idx);
static inline void kgsl_process_add_stats(struct kgsl_process_private *priv,
unsigned int type, uint64_t size)
{
priv->stats[type].cur += size;
if (priv->stats[type].max < priv->stats[type].cur)
priv->stats[type].max = priv->stats[type].cur;
add_mm_counter(current->mm, MM_UNRECLAIMABLE, (size >> PAGE_SHIFT));
}
static inline void kgsl_process_sub_stats(struct kgsl_process_private *priv,
unsigned int type, uint64_t size)
{
struct pid *pid_struct;
struct task_struct *task;
struct mm_struct *mm;
priv->stats[type].cur -= size;
pid_struct = find_get_pid(priv->pid);
if (pid_struct) {
task = get_pid_task(pid_struct, PIDTYPE_PID);
if (task) {
mm = get_task_mm(task);
if (mm) {
add_mm_counter(mm, MM_UNRECLAIMABLE,
-(size >> PAGE_SHIFT));
mmput(mm);
}
put_task_struct(task);
}
put_pid(pid_struct);
}
}
static inline bool kgsl_is_register_offset(struct kgsl_device *device,
unsigned int offsetwords)
{
return ((offsetwords * sizeof(uint32_t)) < device->reg_len);
}
static inline void kgsl_regread(struct kgsl_device *device,
unsigned int offsetwords,
unsigned int *value)
{
if (kgsl_is_register_offset(device, offsetwords))
device->ftbl->regread(device, offsetwords, value);
else if (gmu_core_is_register_offset(device, offsetwords))
gmu_core_regread(device, offsetwords, value);
else {
WARN(1, "Out of bounds register read: 0x%x\n", offsetwords);
*value = 0;
}
}
static inline void kgsl_regwrite(struct kgsl_device *device,
unsigned int offsetwords,
unsigned int value)
{
if (kgsl_is_register_offset(device, offsetwords))
device->ftbl->regwrite(device, offsetwords, value);
else if (gmu_core_is_register_offset(device, offsetwords))
gmu_core_regwrite(device, offsetwords, value);
else
WARN(1, "Out of bounds register write: 0x%x\n", offsetwords);
}
static inline void kgsl_regrmw(struct kgsl_device *device,
unsigned int offsetwords,
unsigned int mask, unsigned int bits)
{
unsigned int val = 0;
kgsl_regread(device, offsetwords, &val);
val &= ~mask;
kgsl_regwrite(device, offsetwords, val | bits);
}
static inline int kgsl_idle(struct kgsl_device *device)
{
return device->ftbl->idle(device);
}
static inline unsigned int kgsl_gpuid(struct kgsl_device *device,
unsigned int *chipid)
{
return device->ftbl->gpuid(device, chipid);
}
static inline int kgsl_state_is_awake(struct kgsl_device *device)
{
if (device->state == KGSL_STATE_ACTIVE ||
device->state == KGSL_STATE_AWARE)
return true;
else if (gmu_core_isenabled(device) &&
test_bit(GMU_CLK_ON, &device->gmu_core.flags))
return true;
else
return false;
}
int kgsl_readtimestamp(struct kgsl_device *device, void *priv,
enum kgsl_timestamp_type type, unsigned int *timestamp);
int kgsl_check_timestamp(struct kgsl_device *device,
struct kgsl_context *context, unsigned int timestamp);
int kgsl_device_platform_probe(struct kgsl_device *device);
void kgsl_device_platform_remove(struct kgsl_device *device);
const char *kgsl_pwrstate_to_str(unsigned int state);
int kgsl_device_snapshot_init(struct kgsl_device *device);
void kgsl_device_snapshot(struct kgsl_device *device,
struct kgsl_context *context, bool gmu_fault);
void kgsl_device_snapshot_close(struct kgsl_device *device);
void kgsl_events_init(void);
void kgsl_events_exit(void);
void kgsl_context_detach(struct kgsl_context *context);
void kgsl_del_event_group(struct kgsl_event_group *group);
void kgsl_add_event_group(struct kgsl_event_group *group,
struct kgsl_context *context, readtimestamp_func readtimestamp,
void *priv, const char *fmt, ...);
void kgsl_cancel_events_timestamp(struct kgsl_device *device,
struct kgsl_event_group *group, unsigned int timestamp);
void kgsl_cancel_events(struct kgsl_device *device,
struct kgsl_event_group *group);
void kgsl_cancel_event(struct kgsl_device *device,
struct kgsl_event_group *group, unsigned int timestamp,
kgsl_event_func func, void *priv);
bool kgsl_event_pending(struct kgsl_device *device,
struct kgsl_event_group *group, unsigned int timestamp,
kgsl_event_func func, void *priv);
int kgsl_add_event(struct kgsl_device *device, struct kgsl_event_group *group,
unsigned int timestamp, kgsl_event_func func, void *priv);
void kgsl_process_event_group(struct kgsl_device *device,
struct kgsl_event_group *group);
void kgsl_flush_event_group(struct kgsl_device *device,
struct kgsl_event_group *group);
void kgsl_process_event_groups(struct kgsl_device *device);
void kgsl_context_destroy(struct kref *kref);
int kgsl_context_init(struct kgsl_device_private *dev_priv,
struct kgsl_context *context);
void kgsl_context_dump(struct kgsl_context *context);
int kgsl_memfree_find_entry(pid_t ptname, uint64_t *gpuaddr,
uint64_t *size, uint64_t *flags, pid_t *pid);
long kgsl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
long kgsl_ioctl_copy_in(unsigned int kernel_cmd, unsigned int user_cmd,
unsigned long arg, unsigned char *ptr);
long kgsl_ioctl_copy_out(unsigned int kernel_cmd, unsigned int user_cmd,
unsigned long arg, unsigned char *ptr);
void kgsl_sparse_bind(struct kgsl_process_private *private,
struct kgsl_drawobj_sparse *sparse);
/**
* kgsl_context_type - Return a symbolic string for the context type
* @type: Context type
*
* Return: Symbolic string representing the context type
*/
const char *kgsl_context_type(int type);
/**
* kgsl_context_put() - Release context reference count
* @context: Pointer to the KGSL context to be released
*
* Reduce the reference count on a KGSL context and destroy it if it is no
* longer needed
*/
static inline void
kgsl_context_put(struct kgsl_context *context)
{
if (context)
kref_put(&context->refcount, kgsl_context_destroy);
}
/**
* kgsl_context_detached() - check if a context is detached
* @context: the context
*
* Check if a context has been destroyed by userspace and is only waiting
* for reference counts to go away. This check is used to weed out
* contexts that shouldn't use the gpu so NULL is considered detached.
*/
static inline bool kgsl_context_detached(struct kgsl_context *context)
{
return (context == NULL || test_bit(KGSL_CONTEXT_PRIV_DETACHED,
&context->priv));
}
/**
* kgsl_context_invalid() - check if a context is invalid
* @context: the context
*
* Check if a context has been invalidated by the kernel and may no
* longer use the GPU.
*/
static inline bool kgsl_context_invalid(struct kgsl_context *context)
{
return (context == NULL || test_bit(KGSL_CONTEXT_PRIV_INVALID,
&context->priv));
}
/**
* kgsl_context_get() - get a pointer to a KGSL context
* @device: Pointer to the KGSL device that owns the context
* @id: Context ID
*
* Find the context associated with the given ID number, increase the reference
* count on it and return it. The caller must make sure that this call is
* paired with a kgsl_context_put. This function is for internal use because it
* doesn't validate the ownership of the context with the calling process - use
* kgsl_context_get_owner for that
*/
static inline struct kgsl_context *kgsl_context_get(struct kgsl_device *device,
uint32_t id)
{
int result = 0;
struct kgsl_context *context = NULL;
read_lock(&device->context_lock);
context = idr_find(&device->context_idr, id);
/* Don't return a context that has been detached */
if (kgsl_context_detached(context))
context = NULL;
else
result = kref_get_unless_zero(&context->refcount);
read_unlock(&device->context_lock);
if (!result)
return NULL;
return context;
}
/**
* _kgsl_context_get() - lightweight function to just increment the ref count
* @context: Pointer to the KGSL context
*
* Get a reference to the specified KGSL context structure. This is a
* lightweight way to just increase the refcount on a known context rather than
* walking through kgsl_context_get and searching the iterator
*/
static inline int _kgsl_context_get(struct kgsl_context *context)
{
int ret = 0;
if (context)
ret = kref_get_unless_zero(&context->refcount);
return ret;
}
/**
* kgsl_context_get_owner() - get a pointer to a KGSL context in a specific
* process
* @dev_priv: Pointer to the process struct
* @id: Context ID to return
*
* Find the context associated with the given ID number, increase the reference
* count on it and return it. The caller must make sure that this call is
* paired with a kgsl_context_put. This function validates that the context id
* given is owned by the dev_priv instancet that is passed in. See
* kgsl_context_get for the internal version that doesn't do the check
*/
static inline struct kgsl_context *kgsl_context_get_owner(
struct kgsl_device_private *dev_priv, uint32_t id)
{
struct kgsl_context *context;
context = kgsl_context_get(dev_priv->device, id);
/* Verify that the context belongs to current calling fd. */
if (context != NULL && context->dev_priv != dev_priv) {
kgsl_context_put(context);
return NULL;
}
return context;
}
/**
* kgsl_process_private_get() - increment the refcount on a
* kgsl_process_private struct
* @process: Pointer to the KGSL process_private
*
* Returns 0 if the structure is invalid and a reference count could not be
* obtained, nonzero otherwise.
*/
static inline int kgsl_process_private_get(struct kgsl_process_private *process)
{
int ret = 0;
if (process != NULL)
ret = kref_get_unless_zero(&process->refcount);
return ret;
}
void kgsl_process_private_put(struct kgsl_process_private *private);
struct kgsl_process_private *kgsl_process_private_find(pid_t pid);
/**
* kgsl_sysfs_store() - parse a string from a sysfs store function
* @buf: Incoming string to parse
* @ptr: Pointer to an unsigned int to store the value
*/
static inline int kgsl_sysfs_store(const char *buf, unsigned int *ptr)
{
unsigned int val;
int rc;
rc = kstrtou32(buf, 0, &val);
if (rc)
return rc;
if (ptr)
*ptr = val;
return 0;
}
/*
* A helper macro to print out "not enough memory functions" - this
* makes it easy to standardize the messages as well as cut down on
* the number of strings in the binary
*/
#define SNAPSHOT_ERR_NOMEM(_d, _s) \
dev_err((_d)->dev, \
"snapshot: not enough snapshot memory for section %s\n", (_s))
/**
* struct kgsl_snapshot_registers - list of registers to snapshot
* @regs: Pointer to an array of register ranges
* @count: Number of entries in the array
*/
struct kgsl_snapshot_registers {
const unsigned int *regs;
unsigned int count;
};
size_t kgsl_snapshot_dump_registers(struct kgsl_device *device, u8 *buf,
size_t remain, void *priv);
void kgsl_snapshot_indexed_registers(struct kgsl_device *device,
struct kgsl_snapshot *snapshot, unsigned int index,
unsigned int data, unsigned int start, unsigned int count);
int kgsl_snapshot_get_object(struct kgsl_snapshot *snapshot,
struct kgsl_process_private *process, uint64_t gpuaddr,
uint64_t size, unsigned int type);
int kgsl_snapshot_have_object(struct kgsl_snapshot *snapshot,
struct kgsl_process_private *process,
uint64_t gpuaddr, uint64_t size);
struct adreno_ib_object_list;
int kgsl_snapshot_add_ib_obj_list(struct kgsl_snapshot *snapshot,
struct adreno_ib_object_list *ib_obj_list);
void kgsl_snapshot_add_section(struct kgsl_device *device, u16 id,
struct kgsl_snapshot *snapshot,
size_t (*func)(struct kgsl_device *, u8 *, size_t, void *),
void *priv);
/**
* kgsl_of_property_read_ddrtype - Get property from devicetree based on
* the type of DDR.
* @node: Devicetree node
* @base: prefix string of the property
* @ptr: Pointer to store the value of the property
*
* First look up the devicetree property based on the prefix string and DDR
* type. If property is not specified per DDR type, then look for the property
* based on prefix string only.
*
* Return: 0 on success or error code on failure.
*/
int kgsl_of_property_read_ddrtype(struct device_node *node, const char *base,
u32 *ptr);
/**
* kgsl_query_property_list - Get a list of valid properties
* @device: A KGSL device handle
* @list: Pointer to a list of u32s
* @count: Number of items in @list
*
* Populate a list with the IDs for supported properties. If @list is NULL,
* just return the number of properties available, otherwise fill up to @count
* items in the list with property identifiers.
*
* Returns the number of total properties if @list is NULL or the number of
* properties copied to @list.
*/
int kgsl_query_property_list(struct kgsl_device *device, u32 *list, u32 count);
/**
* struct kgsl_pwr_limit - limit structure for each client
* @node: Local list node for the limits list
* @level: requested power level
* @device: pointer to the device structure
*/
struct kgsl_pwr_limit {
struct list_head node;
unsigned int level;
struct kgsl_device *device;
};
#endif /* __KGSL_DEVICE_H */

1166
drivers/gpu/msm/kgsl_drawobj.c Normal file
View file

File diff suppressed because it is too large Load diff

243
drivers/gpu/msm/kgsl_drawobj.h Normal file
View file

@ -0,0 +1,243 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_DRAWOBJ_H
#define __KGSL_DRAWOBJ_H
#include <linux/kref.h>
#define DRAWOBJ(obj) (&obj->base)
#define SYNCOBJ(obj) \
container_of(obj, struct kgsl_drawobj_sync, base)
#define CMDOBJ(obj) \
container_of(obj, struct kgsl_drawobj_cmd, base)
#define SPARSEOBJ(obj) \
container_of(obj, struct kgsl_drawobj_sparse, base)
#define CMDOBJ_TYPE BIT(0)
#define MARKEROBJ_TYPE BIT(1)
#define SYNCOBJ_TYPE BIT(2)
#define SPARSEOBJ_TYPE BIT(3)
/**
* struct kgsl_drawobj - KGSL drawobj descriptor
* @device: KGSL GPU device that the command was created for
* @context: KGSL context that created the command
* @type: Object type
* @timestamp: Timestamp assigned to the command
* @flags: flags
* @refcount: kref structure to maintain the reference count
*/
struct kgsl_drawobj {
struct kgsl_device *device;
struct kgsl_context *context;
uint32_t type;
uint32_t timestamp;
unsigned long flags;
struct kref refcount;
};
/**
* struct kgsl_drawobj_cmd - KGSL command obj, This covers marker
* cmds also since markers are special form of cmds that do not
* need their cmds to be executed.
* @base: Base kgsl_drawobj, this needs to be the first entry
* @priv: Internal flags
* @global_ts: The ringbuffer timestamp corresponding to this
* command obj
* @fault_policy: Internal policy describing how to handle this command in case
* of a fault
* @fault_recovery: recovery actions actually tried for this batch
* be hung
* @refcount: kref structure to maintain the reference count
* @cmdlist: List of IBs to issue
* @memlist: List of all memory used in this command batch
* @marker_timestamp: For markers, the timestamp of the last "real" command that
* was queued
* @profiling_buf_entry: Mem entry containing the profiling buffer
* @profiling_buffer_gpuaddr: GPU virt address of the profile buffer added here
* for easy access
* @profile_index: Index to store the start/stop ticks in the kernel profiling
* buffer
* @submit_ticks: Variable to hold ticks at the time of
* command obj submit.
*/
struct kgsl_drawobj_cmd {
struct kgsl_drawobj base;
unsigned long priv;
unsigned int global_ts;
unsigned long fault_policy;
unsigned long fault_recovery;
struct list_head cmdlist;
struct list_head memlist;
unsigned int marker_timestamp;
struct kgsl_mem_entry *profiling_buf_entry;
uint64_t profiling_buffer_gpuaddr;
unsigned int profile_index;
uint64_t submit_ticks;
};
/**
* struct kgsl_drawobj_sync - KGSL sync object
* @base: Base kgsl_drawobj, this needs to be the first entry
* @synclist: Array of context/timestamp tuples to wait for before issuing
* @numsyncs: Number of sync entries in the array
* @pending: Bitmask of sync events that are active
* @timer: a timer used to track possible sync timeouts for this
* sync obj
* @timeout_jiffies: For a sync obj the jiffies at
* which the timer will expire
*/
struct kgsl_drawobj_sync {
struct kgsl_drawobj base;
struct kgsl_drawobj_sync_event *synclist;
unsigned int numsyncs;
unsigned long pending;
struct timer_list timer;
unsigned long timeout_jiffies;
};
#define KGSL_FENCE_NAME_LEN 74
struct fence_info {
char name[KGSL_FENCE_NAME_LEN];
};
struct event_fence_info {
struct fence_info *fences;
int num_fences;
};
/**
* struct kgsl_drawobj_sync_event
* @id: identifer (positiion within the pending bitmap)
* @type: Syncpoint type
* @syncobj: Pointer to the syncobj that owns the sync event
* @context: KGSL context for whose timestamp we want to
* register this event
* @timestamp: Pending timestamp for the event
* @handle: Pointer to a sync fence handle
* @device: Pointer to the KGSL device
* @info: structure to hold info about the fence
*/
struct kgsl_drawobj_sync_event {
unsigned int id;
int type;
struct kgsl_drawobj_sync *syncobj;
struct kgsl_context *context;
unsigned int timestamp;
struct kgsl_sync_fence_cb *handle;
struct kgsl_device *device;
struct event_fence_info info;
};
/**
* struct kgsl_drawobj_sparse - KGSl sparse obj descriptor
* @base: Base kgsl_obj, this needs to be the first entry
* @id: virtual id of the bind/unbind
* @sparselist: list of binds/unbinds
* @size: Size of kgsl_sparse_bind_object
* @count: Number of elements in list
*/
struct kgsl_drawobj_sparse {
struct kgsl_drawobj base;
unsigned int id;
struct list_head sparselist;
unsigned int size;
unsigned int count;
};
#define KGSL_DRAWOBJ_FLAGS \
{ KGSL_DRAWOBJ_MARKER, "MARKER" }, \
{ KGSL_DRAWOBJ_CTX_SWITCH, "CTX_SWITCH" }, \
{ KGSL_DRAWOBJ_SYNC, "SYNC" }, \
{ KGSL_DRAWOBJ_END_OF_FRAME, "EOF" }, \
{ KGSL_DRAWOBJ_PWR_CONSTRAINT, "PWR_CONSTRAINT" }, \
{ KGSL_DRAWOBJ_SUBMIT_IB_LIST, "IB_LIST" }
/**
* enum kgsl_drawobj_cmd_priv - Internal command obj flags
* @CMDOBJ_SKIP - skip the entire command obj
* @CMDOBJ_FORCE_PREAMBLE - Force the preamble on for
* command obj
* @CMDOBJ_WFI - Force wait-for-idle for the submission
* @CMDOBJ_PROFILE - store the start / retire ticks for
* the command obj in the profiling buffer
*/
enum kgsl_drawobj_cmd_priv {
CMDOBJ_SKIP = 0,
CMDOBJ_FORCE_PREAMBLE,
CMDOBJ_WFI,
CMDOBJ_PROFILE,
};
struct kgsl_ibdesc;
struct kgsl_cmd_syncpoint;
struct kgsl_drawobj_cmd *kgsl_drawobj_cmd_create(struct kgsl_device *device,
struct kgsl_context *context, unsigned int flags,
unsigned int type);
int kgsl_drawobj_cmd_add_ibdesc(struct kgsl_device *device,
struct kgsl_drawobj_cmd *cmdobj, struct kgsl_ibdesc *ibdesc);
int kgsl_drawobj_cmd_add_ibdesc_list(struct kgsl_device *device,
struct kgsl_drawobj_cmd *cmdobj, void __user *ptr, int count);
int kgsl_drawobj_cmd_add_cmdlist(struct kgsl_device *device,
struct kgsl_drawobj_cmd *cmdobj, void __user *ptr,
unsigned int size, unsigned int count);
int kgsl_drawobj_cmd_add_memlist(struct kgsl_device *device,
struct kgsl_drawobj_cmd *cmdobj, void __user *ptr,
unsigned int size, unsigned int count);
struct kgsl_drawobj_sync *kgsl_drawobj_sync_create(struct kgsl_device *device,
struct kgsl_context *context);
int kgsl_drawobj_sync_add_syncpoints(struct kgsl_device *device,
struct kgsl_drawobj_sync *syncobj, void __user *ptr,
int count);
int kgsl_drawobj_sync_add_synclist(struct kgsl_device *device,
struct kgsl_drawobj_sync *syncobj, void __user *ptr,
unsigned int size, unsigned int count);
int kgsl_drawobj_sync_add_sync(struct kgsl_device *device,
struct kgsl_drawobj_sync *syncobj,
struct kgsl_cmd_syncpoint *sync);
struct kgsl_drawobj_sparse *kgsl_drawobj_sparse_create(
struct kgsl_device *device,
struct kgsl_context *context, unsigned int flags);
int kgsl_drawobj_sparse_add_sparselist(struct kgsl_device *device,
struct kgsl_drawobj_sparse *sparseobj, unsigned int id,
void __user *ptr, unsigned int size, unsigned int count);
int kgsl_drawobjs_cache_init(void);
void kgsl_drawobjs_cache_exit(void);
void kgsl_dump_syncpoints(struct kgsl_device *device,
struct kgsl_drawobj_sync *syncobj);
void kgsl_drawobj_destroy(struct kgsl_drawobj *drawobj);
void kgsl_drawobj_destroy_object(struct kref *kref);
static inline bool kgsl_drawobj_events_pending(
struct kgsl_drawobj_sync *syncobj)
{
return !bitmap_empty(&syncobj->pending, KGSL_MAX_SYNCPOINTS);
}
static inline bool kgsl_drawobj_event_pending(
struct kgsl_drawobj_sync *syncobj, unsigned int bit)
{
if (bit >= KGSL_MAX_SYNCPOINTS)
return false;
return test_bit(bit, &syncobj->pending);
}
static inline void kgsl_drawobj_put(struct kgsl_drawobj *drawobj)
{
if (drawobj)
kref_put(&drawobj->refcount, kgsl_drawobj_destroy_object);
}
#endif /* __KGSL_DRAWOBJ_H */

440
drivers/gpu/msm/kgsl_events.c Normal file
View file

@ -0,0 +1,440 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2011-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/debugfs.h>
#include "kgsl_debugfs.h"
#include "kgsl_device.h"
#include "kgsl_trace.h"
/*
* Define an kmem cache for the event structures since we allocate and free them
* so frequently
*/
static struct kmem_cache *events_cache;
static struct dentry *events_dentry;
static inline void signal_event(struct kgsl_device *device,
struct kgsl_event *event, int result)
{
list_del(&event->node);
event->result = result;
queue_work(device->events_wq, &event->work);
}
/**
* _kgsl_event_worker() - Work handler for processing GPU event callbacks
* @work: Pointer to the work_struct for the event
*
* Each event callback has its own work struct and is run on a event specific
* workqeuue. This is the worker that queues up the event callback function.
*/
static void _kgsl_event_worker(struct work_struct *work)
{
struct kgsl_event *event = container_of(work, struct kgsl_event, work);
int id = KGSL_CONTEXT_ID(event->context);
trace_kgsl_fire_event(id, event->timestamp, event->result,
jiffies - event->created, event->func);
event->func(event->device, event->group, event->priv, event->result);
kgsl_context_put(event->context);
kmem_cache_free(events_cache, event);
}
/* return true if the group needs to be processed */
static bool _do_process_group(unsigned int processed, unsigned int cur)
{
if (processed == cur)
return false;
/*
* This ensures that the timestamp didn't slip back accidently, maybe
* due to a memory barrier issue. This is highly unlikely but we've
* been burned here in the past.
*/
if ((cur < processed) && ((processed - cur) < KGSL_TIMESTAMP_WINDOW))
return false;
return true;
}
static void _process_event_group(struct kgsl_device *device,
struct kgsl_event_group *group, bool flush)
{
struct kgsl_event *event, *tmp;
unsigned int timestamp;
struct kgsl_context *context;
if (group == NULL)
return;
context = group->context;
/*
* Sanity check to be sure that we we aren't racing with the context
* getting destroyed
*/
if (WARN_ON(context != NULL && !_kgsl_context_get(context)))
return;
spin_lock(&group->lock);
group->readtimestamp(device, group->priv, KGSL_TIMESTAMP_RETIRED,
&timestamp);
if (!flush && !_do_process_group(group->processed, timestamp))
goto out;
list_for_each_entry_safe(event, tmp, &group->events, node) {
if (timestamp_cmp(event->timestamp, timestamp) <= 0)
signal_event(device, event, KGSL_EVENT_RETIRED);
else if (flush)
signal_event(device, event, KGSL_EVENT_CANCELLED);
}
group->processed = timestamp;
out:
spin_unlock(&group->lock);
kgsl_context_put(context);
}
/**
* kgsl_process_event_group() - Handle all the retired events in a group
* @device: Pointer to a KGSL device
* @group: Pointer to a GPU events group to process
*/
void kgsl_process_event_group(struct kgsl_device *device,
struct kgsl_event_group *group)
{
_process_event_group(device, group, false);
}
/**
* kgsl_flush_event_group() - flush all the events in a group by retiring the
* ones can be retired and cancelling the ones that are pending
* @device: Pointer to a KGSL device
* @group: Pointer to a GPU events group to process
*/
void kgsl_flush_event_group(struct kgsl_device *device,
struct kgsl_event_group *group)
{
_process_event_group(device, group, true);
}
/**
* kgsl_cancel_events_timestamp() - Cancel pending events for a given timestamp
* @device: Pointer to a KGSL device
* @group: Ponter to the GPU event group that owns the event
* @timestamp: Registered expiry timestamp for the event
*/
void kgsl_cancel_events_timestamp(struct kgsl_device *device,
struct kgsl_event_group *group, unsigned int timestamp)
{
struct kgsl_event *event, *tmp;
spin_lock(&group->lock);
list_for_each_entry_safe(event, tmp, &group->events, node) {
if (timestamp_cmp(timestamp, event->timestamp) == 0)
signal_event(device, event, KGSL_EVENT_CANCELLED);
}
spin_unlock(&group->lock);
}
/**
* kgsl_cancel_events() - Cancel all pending events in the group
* @device: Pointer to a KGSL device
* @group: Pointer to a kgsl_events_group
*/
void kgsl_cancel_events(struct kgsl_device *device,
struct kgsl_event_group *group)
{
struct kgsl_event *event, *tmp;
spin_lock(&group->lock);
list_for_each_entry_safe(event, tmp, &group->events, node)
signal_event(device, event, KGSL_EVENT_CANCELLED);
spin_unlock(&group->lock);
}
/**
* kgsl_cancel_event() - Cancel a specific event from a group
* @device: Pointer to a KGSL device
* @group: Pointer to the group that contains the events
* @timestamp: Registered expiry timestamp for the event
* @func: Registered callback for the function
* @priv: Registered priv data for the function
*/
void kgsl_cancel_event(struct kgsl_device *device,
struct kgsl_event_group *group, unsigned int timestamp,
kgsl_event_func func, void *priv)
{
struct kgsl_event *event, *tmp;
spin_lock(&group->lock);
list_for_each_entry_safe(event, tmp, &group->events, node) {
if (timestamp == event->timestamp && func == event->func &&
event->priv == priv)
signal_event(device, event, KGSL_EVENT_CANCELLED);
}
spin_unlock(&group->lock);
}
/**
* kgsl_event_pending() - Searches for an event in an event group
* @device: Pointer to a KGSL device
* @group: Pointer to the group that contains the events
* @timestamp: Registered expiry timestamp for the event
* @func: Registered callback for the function
* @priv: Registered priv data for the function
*/
bool kgsl_event_pending(struct kgsl_device *device,
struct kgsl_event_group *group,
unsigned int timestamp, kgsl_event_func func, void *priv)
{
struct kgsl_event *event;
bool result = false;
spin_lock(&group->lock);
list_for_each_entry(event, &group->events, node) {
if (timestamp == event->timestamp && func == event->func &&
event->priv == priv) {
result = true;
break;
}
}
spin_unlock(&group->lock);
return result;
}
/**
* kgsl_add_event() - Add a new GPU event to a group
* @device: Pointer to a KGSL device
* @group: Pointer to the group to add the event to
* @timestamp: Timestamp that the event will expire on
* @func: Callback function for the event
* @priv: Private data to send to the callback function
*/
int kgsl_add_event(struct kgsl_device *device, struct kgsl_event_group *group,
unsigned int timestamp, kgsl_event_func func, void *priv)
{
unsigned int queued;
struct kgsl_context *context = group->context;
struct kgsl_event *event;
unsigned int retired;
if (!func)
return -EINVAL;
/*
* If the caller is creating their own timestamps, let them schedule
* events in the future. Otherwise only allow timestamps that have been
* queued.
*/
if (!context || !(context->flags & KGSL_CONTEXT_USER_GENERATED_TS)) {
group->readtimestamp(device, group->priv, KGSL_TIMESTAMP_QUEUED,
&queued);
if (timestamp_cmp(timestamp, queued) > 0)
return -EINVAL;
}
event = kmem_cache_alloc(events_cache, GFP_KERNEL);
if (event == NULL)
return -ENOMEM;
/* Get a reference to the context while the event is active */
if (context != NULL && !_kgsl_context_get(context)) {
kmem_cache_free(events_cache, event);
return -ENOENT;
}
event->device = device;
event->context = context;
event->timestamp = timestamp;
event->priv = priv;
event->func = func;
event->created = jiffies;
event->group = group;
INIT_WORK(&event->work, _kgsl_event_worker);
trace_kgsl_register_event(KGSL_CONTEXT_ID(context), timestamp, func);
spin_lock(&group->lock);
/*
* Check to see if the requested timestamp has already retired. If so,
* schedule the callback right away
*/
group->readtimestamp(device, group->priv, KGSL_TIMESTAMP_RETIRED,
&retired);
if (timestamp_cmp(retired, timestamp) >= 0) {
event->result = KGSL_EVENT_RETIRED;
queue_work(device->events_wq, &event->work);
spin_unlock(&group->lock);
return 0;
}
/* Add the event to the group list */
list_add_tail(&event->node, &group->events);
spin_unlock(&group->lock);
return 0;
}
static DEFINE_RWLOCK(group_lock);
static LIST_HEAD(group_list);
void kgsl_process_event_groups(struct kgsl_device *device)
{
struct kgsl_event_group *group;
read_lock(&group_lock);
list_for_each_entry(group, &group_list, group)
_process_event_group(device, group, false);
read_unlock(&group_lock);
}
/**
* kgsl_del_event_group() - Remove a GPU event group
* @group: GPU event group to remove
*/
void kgsl_del_event_group(struct kgsl_event_group *group)
{
/* Make sure that all the events have been deleted from the list */
WARN_ON(!list_empty(&group->events));
write_lock(&group_lock);
list_del(&group->group);
write_unlock(&group_lock);
}
/**
* kgsl_add_event_group() - Add a new GPU event group
* @group: Pointer to the new group to add to the list
* @context: Context that owns the group (or NULL for global)
* @readtimestamp: Function pointer to the readtimestamp function to call when
* processing events
* @priv: Priv member to pass to the readtimestamp function
* @fmt: The format string to use to build the event name
* @...: Arguments for the format string
*/
void kgsl_add_event_group(struct kgsl_event_group *group,
struct kgsl_context *context, readtimestamp_func readtimestamp,
void *priv, const char *fmt, ...)
{
va_list args;
WARN_ON(readtimestamp == NULL);
spin_lock_init(&group->lock);
INIT_LIST_HEAD(&group->events);
group->context = context;
group->readtimestamp = readtimestamp;
group->priv = priv;
if (fmt) {
va_start(args, fmt);
vsnprintf(group->name, sizeof(group->name), fmt, args);
va_end(args);
}
write_lock(&group_lock);
list_add_tail(&group->group, &group_list);
write_unlock(&group_lock);
}
static void events_debugfs_print_group(struct seq_file *s,
struct kgsl_event_group *group)
{
struct kgsl_event *event;
unsigned int retired;
spin_lock(&group->lock);
seq_printf(s, "%s: last=%d\n", group->name, group->processed);
list_for_each_entry(event, &group->events, node) {
group->readtimestamp(event->device, group->priv,
KGSL_TIMESTAMP_RETIRED, &retired);
seq_printf(s, "\t%u:%u age=%lu func=%ps [retired=%u]\n",
group->context ? group->context->id :
KGSL_MEMSTORE_GLOBAL,
event->timestamp, jiffies - event->created,
event->func, retired);
}
spin_unlock(&group->lock);
}
static int events_debugfs_print(struct seq_file *s, void *unused)
{
struct kgsl_event_group *group;
seq_puts(s, "event groups:\n");
seq_puts(s, "--------------\n");
read_lock(&group_lock);
list_for_each_entry(group, &group_list, group) {
events_debugfs_print_group(s, group);
seq_puts(s, "\n");
}
read_unlock(&group_lock);
return 0;
}
static int events_debugfs_open(struct inode *inode, struct file *file)
{
return single_open(file, events_debugfs_print, NULL);
}
static const struct file_operations events_fops = {
.open = events_debugfs_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
/**
* kgsl_events_exit() - Destroy the event kmem cache on module exit
*/
void kgsl_events_exit(void)
{
kmem_cache_destroy(events_cache);
debugfs_remove(events_dentry);
}
/**
* kgsl_events_init() - Create the event kmem cache on module start
*/
void __init kgsl_events_init(void)
{
struct dentry *debugfs_dir = kgsl_get_debugfs_dir();
events_cache = KMEM_CACHE(kgsl_event, 0);
events_dentry = debugfs_create_file("events", 0444, debugfs_dir, NULL,
&events_fops);
/* Failure to create a debugfs entry is non fatal */
if (IS_ERR(events_dentry))
events_dentry = NULL;
}

1736
drivers/gpu/msm/kgsl_gmu.c Normal file
View file

File diff suppressed because it is too large Load diff

227
drivers/gpu/msm/kgsl_gmu.h Normal file
View file

@ -0,0 +1,227 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_GMU_H
#define __KGSL_GMU_H
#include "kgsl_gmu_core.h"
#include "kgsl_hfi.h"
#define GMU_PWR_LEVELS 2
#define GMU_FREQUENCY 200000000
#define MAX_GMUFW_SIZE 0x8000 /* in bytes */
#define BWMEM_SIZE (12 + (4 * NUM_BW_LEVELS)) /*in bytes*/
#define GMU_VER_MAJOR(ver) (((ver) >> 28) & 0xF)
#define GMU_VER_MINOR(ver) (((ver) >> 16) & 0xFFF)
#define GMU_VER_STEP(ver) ((ver) & 0xFFFF)
#define GMU_VERSION(major, minor) \
((((major) & 0xF) << 28) | (((minor) & 0xFFF) << 16))
#define GMU_INT_WDOG_BITE BIT(0)
#define GMU_INT_RSCC_COMP BIT(1)
#define GMU_INT_FENCE_ERR BIT(3)
#define GMU_INT_DBD_WAKEUP BIT(4)
#define GMU_INT_HOST_AHB_BUS_ERR BIT(5)
#define GMU_AO_INT_MASK \
(GMU_INT_WDOG_BITE | \
GMU_INT_FENCE_ERR | \
GMU_INT_HOST_AHB_BUS_ERR)
/* Bitmask for GPU low power mode enabling and hysterisis*/
#define SPTP_ENABLE_MASK (BIT(2) | BIT(0))
#define IFPC_ENABLE_MASK (BIT(1) | BIT(0))
#define HW_NAP_ENABLE_MASK BIT(0)
#define MIN_BW_ENABLE_MASK BIT(12)
#define MIN_BW_HYST 0xFA0
/* Bitmask for RPMH capability enabling */
#define RPMH_INTERFACE_ENABLE BIT(0)
#define LLC_VOTE_ENABLE BIT(4)
#define DDR_VOTE_ENABLE BIT(8)
#define MX_VOTE_ENABLE BIT(9)
#define CX_VOTE_ENABLE BIT(10)
#define GFX_VOTE_ENABLE BIT(11)
#define RPMH_ENABLE_MASK (RPMH_INTERFACE_ENABLE | \
LLC_VOTE_ENABLE | \
DDR_VOTE_ENABLE | \
MX_VOTE_ENABLE | \
CX_VOTE_ENABLE | \
GFX_VOTE_ENABLE)
/* Constants for GMU OOBs */
#define OOB_BOOT_OPTION 0
#define OOB_SLUMBER_OPTION 1
/* Gmu FW block header format */
struct gmu_block_header {
uint32_t addr;
uint32_t size;
uint32_t type;
uint32_t value;
};
/* GMU Block types */
#define GMU_BLK_TYPE_DATA 0
#define GMU_BLK_TYPE_PREALLOC_REQ 1
#define GMU_BLK_TYPE_CORE_VER 2
#define GMU_BLK_TYPE_CORE_DEV_VER 3
#define GMU_BLK_TYPE_PWR_VER 4
#define GMU_BLK_TYPE_PWR_DEV_VER 5
#define GMU_BLK_TYPE_HFI_VER 6
#define GMU_BLK_TYPE_PREALLOC_PERSIST_REQ 7
/* For GMU Logs*/
#define LOGMEM_SIZE SZ_4K
/* GMU memdesc entries */
#define GMU_KERNEL_ENTRIES 16
extern struct gmu_dev_ops adreno_a6xx_gmudev;
#define KGSL_GMU_DEVICE(_a) ((struct gmu_device *)((_a)->gmu_core.ptr))
enum gmu_mem_type {
GMU_ITCM = 0,
GMU_ICACHE,
GMU_CACHE = GMU_ICACHE,
GMU_DTCM,
GMU_DCACHE,
GMU_NONCACHED_KERNEL,
GMU_NONCACHED_USER,
GMU_MEM_TYPE_MAX,
};
enum gmu_context_index {
GMU_CONTEXT_USER = 0,
GMU_CONTEXT_KERNEL,
};
/**
* struct gmu_memdesc - Gmu shared memory object descriptor
* @hostptr: Kernel virtual address
* @gmuaddr: GPU virtual address
* @physaddr: Physical address of the memory object
* @size: Size of the memory object
* @mem_type: memory type for this memory
* @ctx_idx: GMU IOMMU context idx
*/
struct gmu_memdesc {
void *hostptr;
uint64_t gmuaddr;
phys_addr_t physaddr;
uint64_t size;
enum gmu_mem_type mem_type;
enum gmu_context_index ctx_idx;
};
struct gmu_bw_votes {
uint32_t cmds_wait_bitmask;
uint32_t cmds_per_bw_vote;
uint32_t cmd_addrs[MAX_BW_CMDS];
uint32_t cmd_data[MAX_GX_LEVELS][MAX_BW_CMDS];
};
struct rpmh_votes_t {
uint32_t gx_votes[MAX_GX_LEVELS];
uint32_t cx_votes[MAX_CX_LEVELS];
struct gmu_bw_votes ddr_votes;
struct gmu_bw_votes cnoc_votes;
};
enum gmu_load_mode {
CACHED_LOAD_BOOT,
CACHED_BOOT,
TCM_BOOT,
TCM_LOAD_BOOT,
INVALID_LOAD
};
struct kgsl_mailbox {
struct mbox_client *client;
struct mbox_chan *channel;
};
struct icc_path;
/**
* struct gmu_device - GMU device structure
* @ver: GMU Version information
* @reg_phys: GMU CSR physical address
* @reg_len: GMU CSR range
* @gmu_interrupt_num: GMU interrupt number
* @fw_image: GMU FW image
* @hfi_mem: pointer to HFI shared memory
* @dump_mem: pointer to GMU debug dump memory
* @gmu_log: gmu event log memory
* @hfi: HFI controller
* @gpu_freqs: GPU frequency table with lowest freq at index 0
* @num_gpupwrlevels: number GPU frequencies in GPU freq table
* @num_bwlevel: number of GPU BW levels
* @num_cnocbwlevel: number CNOC BW levels
* @rpmh_votes: RPMh TCS command set for GPU, GMU voltage and bw scaling
* @cx_gdsc: CX headswitch that controls power of GMU and
subsystem peripherals
* @gx_gdsc: GX headswitch that controls power of GPU subsystem
* @clks: GPU subsystem clocks required for GMU functionality
* @load_mode: GMU FW load/boot mode
* @wakeup_pwrlevel: GPU wake up power/DCVS level in case different
* than default power level
* @pcl: GPU BW scaling client
* @ccl: CNOC BW scaling client
* @idle_level: Minimal GPU idle power level
* @fault_count: GMU fault count
* @mailbox: Messages to AOP for ACD enable/disable go through this
* @log_wptr_retention: Store the log wptr offset on slumber
*/
struct gmu_device {
struct {
u32 core;
u32 core_dev;
u32 pwr;
u32 pwr_dev;
u32 hfi;
} ver;
struct platform_device *pdev;
unsigned long reg_phys;
unsigned int reg_len;
int gmu_interrupt_num;
const struct firmware *fw_image;
struct gmu_memdesc *hfi_mem;
struct gmu_memdesc *dump_mem;
struct gmu_memdesc *gmu_log;
struct kgsl_hfi hfi;
unsigned int gpu_freqs[MAX_GX_LEVELS];
unsigned int num_gpupwrlevels;
unsigned int num_bwlevels;
unsigned int num_cnocbwlevels;
struct rpmh_votes_t rpmh_votes;
struct regulator *cx_gdsc;
struct regulator *gx_gdsc;
struct clk *clks[MAX_GMU_CLKS];
enum gmu_load_mode load_mode;
unsigned int wakeup_pwrlevel;
unsigned int pcl;
unsigned int ccl;
unsigned int idle_level;
unsigned int fault_count;
struct kgsl_mailbox mailbox;
bool preallocations;
struct gmu_memdesc kmem_entries[GMU_KERNEL_ENTRIES];
unsigned long kmem_bitmap;
const struct gmu_vma_entry *vma;
unsigned int log_wptr_retention;
/** @icc_path: Interconnect path for the GMU */
struct icc_path *icc_path;
};
struct gmu_memdesc *gmu_get_memdesc(struct gmu_device *gmu,
unsigned int addr, unsigned int size);
unsigned int gmu_get_memtype_base(struct gmu_device *gmu,
enum gmu_mem_type type);
int gmu_prealloc_req(struct kgsl_device *device, struct gmu_block_header *blk);
int gmu_cache_finalize(struct kgsl_device *device);
#endif /* __KGSL_GMU_H */

379
drivers/gpu/msm/kgsl_gmu_core.c Normal file
View file

@ -0,0 +1,379 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/of.h>
#include "adreno.h"
#include "kgsl_device.h"
#include "kgsl_gmu_core.h"
#include "kgsl_trace.h"
static const struct {
char *compat;
struct gmu_core_ops *core_ops;
enum gmu_coretype type;
} gmu_subtypes[] = {
{"qcom,gpu-gmu", &gmu_ops, GMU_CORE_TYPE_CM3},
{"qcom,gpu-rgmu", &rgmu_ops, GMU_CORE_TYPE_PCC},
};
struct oob_entry {
enum oob_request req;
const char *str;
};
const char *gmu_core_oob_type_str(enum oob_request req)
{
int i;
struct oob_entry table[] = {
{ oob_gpu, "oob_gpu"},
{ oob_perfcntr, "oob_perfcntr"},
{ oob_boot_slumber, "oob_boot_slumber"},
{ oob_dcvs, "oob_dcvs"},
};
for (i = 0; i < ARRAY_SIZE(table); i++)
if (req == table[i].req)
return table[i].str;
return "UNKNOWN";
}
int gmu_core_probe(struct kgsl_device *device)
{
struct device_node *node;
struct gmu_core_ops *gmu_core_ops;
int i = 0, ret = -ENXIO;
device->gmu_core.flags = ADRENO_FEATURE(ADRENO_DEVICE(device),
ADRENO_GPMU) ? BIT(GMU_GPMU) : 0;
for (i = 0; i < ARRAY_SIZE(gmu_subtypes); i++) {
node = of_find_compatible_node(device->pdev->dev.of_node,
NULL, gmu_subtypes[i].compat);
if (node != NULL) {
gmu_core_ops = gmu_subtypes[i].core_ops;
device->gmu_core.type = gmu_subtypes[i].type;
break;
}
}
/* No GMU in dt, no worries...hopefully */
if (node == NULL) {
/* If we are trying to use GPMU and no GMU, that's bad */
if (device->gmu_core.flags & BIT(GMU_GPMU))
return ret;
/* Otherwise it's ok and nothing to do */
return 0;
}
if (gmu_core_ops && gmu_core_ops->probe) {
ret = gmu_core_ops->probe(device, node);
if (ret == 0)
device->gmu_core.core_ops = gmu_core_ops;
}
return ret;
}
void gmu_core_remove(struct kgsl_device *device)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->remove)
gmu_core_ops->remove(device);
}
bool gmu_core_isenabled(struct kgsl_device *device)
{
return test_bit(GMU_ENABLED, &device->gmu_core.flags);
}
bool gmu_core_gpmu_isenabled(struct kgsl_device *device)
{
return test_bit(GMU_GPMU, &device->gmu_core.flags);
}
bool gmu_core_scales_bandwidth(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
if (device->gmu_core.type == GMU_CORE_TYPE_PCC)
return false;
return gmu_core_gpmu_isenabled(device) &&
(ADRENO_GPUREV(adreno_dev) >= ADRENO_REV_A640);
}
int gmu_core_init(struct kgsl_device *device)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->init)
return gmu_core_ops->init(device);
return 0;
}
int gmu_core_start(struct kgsl_device *device)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->start)
return gmu_core_ops->start(device);
return -EINVAL;
}
void gmu_core_stop(struct kgsl_device *device)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->stop)
gmu_core_ops->stop(device);
}
int gmu_core_suspend(struct kgsl_device *device)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->suspend)
return gmu_core_ops->suspend(device);
return -EINVAL;
}
void gmu_core_snapshot(struct kgsl_device *device)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->snapshot)
gmu_core_ops->snapshot(device);
}
int gmu_core_dcvs_set(struct kgsl_device *device, unsigned int gpu_pwrlevel,
unsigned int bus_level)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->dcvs_set)
return gmu_core_ops->dcvs_set(device, gpu_pwrlevel, bus_level);
return -EINVAL;
}
int gmu_core_acd_set(struct kgsl_device *device, unsigned int val)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->acd_set)
return gmu_core_ops->acd_set(device, val);
return -EINVAL;
}
bool gmu_core_regulator_isenabled(struct kgsl_device *device)
{
struct gmu_core_ops *gmu_core_ops = GMU_CORE_OPS(device);
if (gmu_core_ops && gmu_core_ops->regulator_isenabled)
return gmu_core_ops->regulator_isenabled(device);
return false;
}
bool gmu_core_is_register_offset(struct kgsl_device *device,
unsigned int offsetwords)
{
return (gmu_core_isenabled(device) &&
(offsetwords >= device->gmu_core.gmu2gpu_offset) &&
((offsetwords - device->gmu_core.gmu2gpu_offset) *
sizeof(uint32_t) < device->gmu_core.reg_len));
}
void gmu_core_regread(struct kgsl_device *device, unsigned int offsetwords,
unsigned int *value)
{
void __iomem *reg;
if (WARN(!gmu_core_is_register_offset(device, offsetwords),
"Out of bounds register read: 0x%x\n", offsetwords))
return;
offsetwords -= device->gmu_core.gmu2gpu_offset;
reg = device->gmu_core.reg_virt + (offsetwords << 2);
*value = __raw_readl(reg);
/*
* ensure this read finishes before the next one.
* i.e. act like normal readl()
*/
rmb();
}
void gmu_core_regwrite(struct kgsl_device *device, unsigned int offsetwords,
unsigned int value)
{
void __iomem *reg;
if (WARN(!gmu_core_is_register_offset(device, offsetwords),
"Out of bounds register write: 0x%x\n", offsetwords))
return;
trace_kgsl_regwrite(device, offsetwords, value);
offsetwords -= device->gmu_core.gmu2gpu_offset;
reg = device->gmu_core.reg_virt + (offsetwords << 2);
/*
* ensure previous writes post before this one,
* i.e. act like normal writel()
*/
wmb();
__raw_writel(value, reg);
}
void gmu_core_blkwrite(struct kgsl_device *device, unsigned int offsetwords,
const void *buffer, size_t size)
{
void __iomem *base;
if (WARN_ON(!gmu_core_is_register_offset(device, offsetwords)))
return;
offsetwords -= device->gmu_core.gmu2gpu_offset;
base = device->gmu_core.reg_virt + (offsetwords << 2);
memcpy_toio(base, buffer, size);
}
void gmu_core_regrmw(struct kgsl_device *device,
unsigned int offsetwords,
unsigned int mask, unsigned int bits)
{
unsigned int val = 0;
if (WARN(!gmu_core_is_register_offset(device, offsetwords),
"Out of bounds register rmw: 0x%x\n", offsetwords))
return;
gmu_core_regread(device, offsetwords, &val);
val &= ~mask;
gmu_core_regwrite(device, offsetwords, val | bits);
}
int gmu_core_dev_oob_set(struct kgsl_device *device, enum oob_request req)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->oob_set)
return ops->oob_set(device, req);
return 0;
}
void gmu_core_dev_oob_clear(struct kgsl_device *device, enum oob_request req)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->oob_clear)
ops->oob_clear(device, req);
}
int gmu_core_dev_hfi_start_msg(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->hfi_start_msg)
return ops->hfi_start_msg(device);
return 0;
}
int gmu_core_dev_wait_for_lowest_idle(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->wait_for_lowest_idle)
ops->wait_for_lowest_idle(device);
return 0;
}
void gmu_core_dev_enable_lm(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->enable_lm)
ops->enable_lm(device);
}
void gmu_core_dev_snapshot(struct kgsl_device *device,
struct kgsl_snapshot *snapshot)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->snapshot)
ops->snapshot(device, snapshot);
}
void gmu_core_dev_cooperative_reset(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->cooperative_reset)
ops->cooperative_reset(device);
}
bool gmu_core_dev_gx_is_on(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->gx_is_on)
return ops->gx_is_on(device);
return true;
}
int gmu_core_dev_ifpc_show(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->ifpc_show)
return ops->ifpc_show(device);
return 0;
}
int gmu_core_dev_ifpc_store(struct kgsl_device *device, unsigned int val)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->ifpc_store)
return ops->ifpc_store(device, val);
return -EINVAL;
}
void gmu_core_dev_prepare_stop(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->prepare_stop)
ops->prepare_stop(device);
}
int gmu_core_dev_wait_for_active_transition(struct kgsl_device *device)
{
struct gmu_dev_ops *ops = GMU_DEVICE_OPS(device);
if (ops && ops->wait_for_active_transition)
return ops->wait_for_active_transition(device);
return 0;
}

232
drivers/gpu/msm/kgsl_gmu_core.h Normal file
View file

@ -0,0 +1,232 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_GMU_CORE_H
#define __KGSL_GMU_CORE_H
/* GMU_DEVICE - Given an KGSL device return the GMU specific struct */
#define GMU_DEVICE_OPS(_a) ((_a)->gmu_core.dev_ops)
#define GMU_CORE_OPS(_a) ((_a)->gmu_core.core_ops)
#define NUM_BW_LEVELS 100
#define MAX_GX_LEVELS 16
#define MAX_CX_LEVELS 4
#define MAX_CNOC_LEVELS 2
#define MAX_CNOC_CMDS 6
#define MAX_BW_CMDS 8
#define INVALID_DCVS_IDX 0xFF
#if MAX_CNOC_LEVELS > MAX_GX_LEVELS
#error "CNOC levels cannot exceed GX levels"
#endif
#define MAX_GMU_CLKS 6
/*
* These are the different ways the GMU can boot. GMU_WARM_BOOT is waking up
* from slumber. GMU_COLD_BOOT is booting for the first time. GMU_RESET
* is a soft reset of the GMU.
*/
enum gmu_core_boot {
GMU_WARM_BOOT = 0,
GMU_COLD_BOOT = 1,
GMU_RESET = 2
};
/* Bits for the flags field in the gmu structure */
enum gmu_core_flags {
GMU_BOOT_INIT_DONE = 0,
GMU_CLK_ON,
GMU_HFI_ON,
GMU_FAULT,
GMU_DCVS_REPLAY,
GMU_GPMU,
GMU_ENABLED,
GMU_RSCC_SLEEP_SEQ_DONE,
};
/* GMU Types */
enum gmu_coretype {
GMU_CORE_TYPE_CM3 = 1, /* Cortex M3 core */
GMU_CORE_TYPE_PCC = 2, /* Power collapsible controller */
GMU_CORE_TYPE_NONE, /* No GMU */
};
/*
* OOB requests values. These range from 0 to 7 and then
* the BIT() offset into the actual value is calculated
* later based on the request. This keeps the math clean
* and easy to ensure not reaching over/under the range
* of 8 bits.
*/
enum oob_request {
oob_gpu = 0,
oob_perfcntr = 1,
oob_boot_slumber = 6, /* reserved special case */
oob_dcvs = 7, /* reserved special case */
};
enum gmu_pwrctrl_mode {
GMU_FW_START,
GMU_FW_STOP,
GMU_SUSPEND,
GMU_DCVS_NOHFI,
GMU_NOTIFY_SLUMBER,
INVALID_POWER_CTRL
};
enum gpu_idle_level {
GPU_HW_ACTIVE = 0x0,
GPU_HW_SPTP_PC = 0x2,
GPU_HW_IFPC = 0x3,
GPU_HW_NAP = 0x4,
GPU_HW_MIN_VOLT = 0x5,
GPU_HW_MIN_DDR = 0x6,
GPU_HW_SLUMBER = 0xF
};
/*
* Wait time before trying to write the register again.
* Hopefully the GMU has finished waking up during this delay.
* This delay must be less than the IFPC main hysteresis or
* the GMU will start shutting down before we try again.
*/
#define GMU_CORE_WAKEUP_DELAY_US 10
/* Max amount of tries to wake up the GMU. The short retry
* limit is half of the long retry limit. After the short
* number of retries, we print an informational message to say
* exiting IFPC is taking longer than expected. We continue
* to retry after this until the long retry limit.
*/
#define GMU_CORE_SHORT_WAKEUP_RETRY_LIMIT 100
#define GMU_CORE_LONG_WAKEUP_RETRY_LIMIT 200
#define FENCE_STATUS_WRITEDROPPED0_MASK 0x1
#define FENCE_STATUS_WRITEDROPPED1_MASK 0x2
struct device_node;
struct kgsl_device;
struct kgsl_snapshot;
struct gmu_core_ops {
int (*probe)(struct kgsl_device *device, struct device_node *node);
void (*remove)(struct kgsl_device *device);
int (*dcvs_set)(struct kgsl_device *device,
unsigned int gpu_pwrlevel, unsigned int bus_level);
int (*init)(struct kgsl_device *device);
int (*start)(struct kgsl_device *device);
void (*stop)(struct kgsl_device *device);
void (*snapshot)(struct kgsl_device *device);
bool (*regulator_isenabled)(struct kgsl_device *device);
int (*suspend)(struct kgsl_device *device);
int (*acd_set)(struct kgsl_device *device, unsigned int val);
};
struct gmu_dev_ops {
int (*load_firmware)(struct kgsl_device *device);
int (*oob_set)(struct kgsl_device *device, enum oob_request req);
void (*oob_clear)(struct kgsl_device *device, enum oob_request req);
void (*bcl_config)(struct kgsl_device *device, bool on);
void (*irq_enable)(struct kgsl_device *device);
void (*irq_disable)(struct kgsl_device *device);
int (*hfi_start_msg)(struct kgsl_device *device);
void (*enable_lm)(struct kgsl_device *device);
int (*rpmh_gpu_pwrctrl)(struct kgsl_device *device, unsigned int ops,
unsigned int arg1, unsigned int arg2);
int (*wait_for_lowest_idle)(struct kgsl_device *device);
int (*wait_for_gmu_idle)(struct kgsl_device *device);
bool (*gx_is_on)(struct kgsl_device *device);
void (*prepare_stop)(struct kgsl_device *device);
int (*ifpc_store)(struct kgsl_device *device, unsigned int val);
unsigned int (*ifpc_show)(struct kgsl_device *device);
void (*snapshot)(struct kgsl_device *device,
struct kgsl_snapshot *shapshot);
void (*cooperative_reset)(struct kgsl_device *device);
void (*halt_execution)(struct kgsl_device *device);
int (*wait_for_active_transition)(struct kgsl_device *device);
const unsigned int gmu2host_intr_mask;
const unsigned int gmu_ao_intr_mask;
};
/**
* struct gmu_core_device - GMU Core device structure
* @ptr: Pointer to GMU device structure
* @gmu2gpu_offset: address difference between GMU register set
* and GPU register set, the offset will be used when accessing
* gmu registers using offset defined in GPU register space.
* @reg_len: GMU registers length
* @reg_virt: GMU CSR virtual address
* @core_ops: Pointer to gmu core operations
* @dev_ops: Pointer to gmu device operations
* @flags: GMU flags
*/
struct gmu_core_device {
void *ptr;
unsigned int gmu2gpu_offset;
unsigned int reg_len;
void __iomem *reg_virt;
struct gmu_core_ops *core_ops;
struct gmu_dev_ops *dev_ops;
unsigned long flags;
enum gmu_coretype type;
};
extern struct gmu_core_ops gmu_ops;
extern struct gmu_core_ops rgmu_ops;
/* GMU core functions */
int gmu_core_probe(struct kgsl_device *device);
void gmu_core_remove(struct kgsl_device *device);
int gmu_core_init(struct kgsl_device *device);
int gmu_core_start(struct kgsl_device *device);
void gmu_core_stop(struct kgsl_device *device);
int gmu_core_suspend(struct kgsl_device *device);
void gmu_core_snapshot(struct kgsl_device *device);
bool gmu_core_gpmu_isenabled(struct kgsl_device *device);
bool gmu_core_scales_bandwidth(struct kgsl_device *device);
bool gmu_core_isenabled(struct kgsl_device *device);
int gmu_core_dcvs_set(struct kgsl_device *device, unsigned int gpu_pwrlevel,
unsigned int bus_level);
int gmu_core_acd_set(struct kgsl_device *device, unsigned int val);
bool gmu_core_regulator_isenabled(struct kgsl_device *device);
bool gmu_core_is_register_offset(struct kgsl_device *device,
unsigned int offsetwords);
void gmu_core_regread(struct kgsl_device *device, unsigned int offsetwords,
unsigned int *value);
void gmu_core_regwrite(struct kgsl_device *device, unsigned int offsetwords,
unsigned int value);
/**
* gmu_core_blkwrite - Do a bulk I/O write to GMU
* @device: Pointer to the kgsl device
* @offsetwords: Destination dword offset
* @buffer: Pointer to the source buffer
* @size: Number of bytes to copy
*
* Write a series of GMU registers quickly without bothering to spend time
* logging the register writes. The logging of these writes causes extra
* delays that could allow IRQs arrive and be serviced before finishing
* all the writes.
*/
void gmu_core_blkwrite(struct kgsl_device *device, unsigned int offsetwords,
const void *buffer, size_t size);
void gmu_core_regrmw(struct kgsl_device *device, unsigned int offsetwords,
unsigned int mask, unsigned int bits);
const char *gmu_core_oob_type_str(enum oob_request req);
int gmu_core_dev_oob_set(struct kgsl_device *device, enum oob_request req);
void gmu_core_dev_oob_clear(struct kgsl_device *device, enum oob_request req);
int gmu_core_dev_hfi_start_msg(struct kgsl_device *device);
int gmu_core_dev_wait_for_lowest_idle(struct kgsl_device *device);
void gmu_core_dev_enable_lm(struct kgsl_device *device);
void gmu_core_dev_snapshot(struct kgsl_device *device,
struct kgsl_snapshot *snapshot);
bool gmu_core_dev_gx_is_on(struct kgsl_device *device);
int gmu_core_dev_ifpc_show(struct kgsl_device *device);
int gmu_core_dev_ifpc_store(struct kgsl_device *device, unsigned int val);
void gmu_core_dev_prepare_stop(struct kgsl_device *device);
int gmu_core_dev_wait_for_active_transition(struct kgsl_device *device);
void gmu_core_dev_cooperative_reset(struct kgsl_device *device);
#endif /* __KGSL_GMU_CORE_H */

868
drivers/gpu/msm/kgsl_hfi.c Normal file
View file

@ -0,0 +1,868 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/delay.h>
#include "adreno.h"
#include "adreno_a6xx.h"
#include "kgsl_device.h"
#include "kgsl_gmu.h"
#include "kgsl_hfi.h"
#include "kgsl_trace.h"
#define HFI_QUEUE_OFFSET(i) \
(ALIGN(sizeof(struct hfi_queue_table), SZ_16) + \
((i) * HFI_QUEUE_SIZE))
#define HOST_QUEUE_START_ADDR(hfi_mem, i) \
((hfi_mem)->hostptr + HFI_QUEUE_OFFSET(i))
#define GMU_QUEUE_START_ADDR(hfi_mem, i) \
((hfi_mem)->gmuaddr + HFI_QUEUE_OFFSET(i))
#define MSG_HDR_GET_ID(hdr) ((hdr) & 0xFF)
#define MSG_HDR_GET_SIZE(hdr) (((hdr) >> 8) & 0xFF)
#define MSG_HDR_GET_TYPE(hdr) (((hdr) >> 16) & 0xF)
#define MSG_HDR_GET_SEQNUM(hdr) (((hdr) >> 20) & 0xFFF)
/* Size is converted from Bytes to DWords */
#define CREATE_MSG_HDR(id, size, type) \
(((type) << 16) | ((((size) >> 2) & 0xFF) << 8) | ((id) & 0xFF))
#define CMD_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_CMD)
#define ACK_MSG_HDR(id, size) CREATE_MSG_HDR(id, size, HFI_MSG_ACK)
static void hfi_process_queue(struct gmu_device *gmu, uint32_t queue_idx,
struct pending_cmd *ret_cmd);
/* Size in below functions are in unit of dwords */
static int hfi_queue_read(struct gmu_device *gmu, uint32_t queue_idx,
unsigned int *output, unsigned int max_size)
{
struct gmu_memdesc *mem_addr = gmu->hfi_mem;
struct hfi_queue_table *tbl = mem_addr->hostptr;
struct hfi_queue_header *hdr = &tbl->qhdr[queue_idx];
uint32_t *queue;
uint32_t msg_hdr;
uint32_t i, read;
uint32_t size;
int result = 0;
if (hdr->status == HFI_QUEUE_STATUS_DISABLED)
return -EINVAL;
if (hdr->read_index == hdr->write_index)
return -ENODATA;
/* Clear the output data before populating */
memset(output, 0, max_size);
queue = HOST_QUEUE_START_ADDR(mem_addr, queue_idx);
msg_hdr = queue[hdr->read_index];
size = MSG_HDR_GET_SIZE(msg_hdr);
if (size > (max_size >> 2)) {
dev_err(&gmu->pdev->dev,
"HFI message too big: hdr:0x%x rd idx=%d\n",
msg_hdr, hdr->read_index);
result = -EMSGSIZE;
goto done;
}
read = hdr->read_index;
if (read < hdr->queue_size) {
for (i = 0; i < size && i < (max_size >> 2); i++) {
output[i] = queue[read];
read = (read + 1)%hdr->queue_size;
}
result = size;
} else {
/* In case FW messed up */
dev_err(&gmu->pdev->dev,
"Read index %d greater than queue size %d\n",
hdr->read_index, hdr->queue_size);
result = -ENODATA;
}
if (GMU_VER_MAJOR(gmu->ver.hfi) >= 2)
read = ALIGN(read, SZ_4) % hdr->queue_size;
hdr->read_index = read;
done:
return result;
}
/* Size in below functions are in unit of dwords */
static int hfi_queue_write(struct gmu_device *gmu, uint32_t queue_idx,
uint32_t *msg)
{
struct hfi_queue_table *tbl = gmu->hfi_mem->hostptr;
struct hfi_queue_header *hdr = &tbl->qhdr[queue_idx];
uint32_t *queue;
struct kgsl_hfi *hfi = &gmu->hfi;
uint32_t i, write, empty_space;
uint32_t size = MSG_HDR_GET_SIZE(*msg);
uint32_t id = MSG_HDR_GET_ID(*msg);
if (hdr->status == HFI_QUEUE_STATUS_DISABLED)
return -EINVAL;
if (size > HFI_MAX_MSG_SIZE) {
dev_err(&gmu->pdev->dev,
"Message too big to send: sz=%d, id=%d\n",
size, id);
return -EINVAL;
}
queue = HOST_QUEUE_START_ADDR(gmu->hfi_mem, queue_idx);
trace_kgsl_hfi_send(id, size, MSG_HDR_GET_SEQNUM(*msg));
mutex_lock(&hfi->cmdq_mutex);
empty_space = (hdr->write_index >= hdr->read_index) ?
(hdr->queue_size - (hdr->write_index - hdr->read_index))
: (hdr->read_index - hdr->write_index);
if (empty_space < size) {
dev_err(&gmu->pdev->dev,
"Insufficient bufsize %d for msg id=%d of size %d\n",
empty_space, id, size);
mutex_unlock(&hfi->cmdq_mutex);
return -ENOSPC;
}
write = hdr->write_index;
for (i = 0; i < size; i++) {
queue[write] = msg[i];
write = (write + 1) % hdr->queue_size;
}
/* Cookify any non used data at the end of the write buffer */
if (GMU_VER_MAJOR(gmu->ver.hfi) >= 2) {
for (; write % 4; write = (write + 1) % hdr->queue_size)
queue[write] = 0xFAFAFAFA;
}
hdr->write_index = write;
mutex_unlock(&hfi->cmdq_mutex);
/*
* Memory barrier to make sure packet and write index are written before
* an interrupt is raised
*/
wmb();
/* Send interrupt to GMU to receive the message */
adreno_write_gmureg(ADRENO_DEVICE(hfi->kgsldev),
ADRENO_REG_GMU_HOST2GMU_INTR_SET, 0x1);
return 0;
}
#define QUEUE_HDR_TYPE(id, prio, rtype, stype) \
(((id) & 0xFF) | (((prio) & 0xFF) << 8) | \
(((rtype) & 0xFF) << 16) | (((stype) & 0xFF) << 24))
/* Sizes of the queue and message are in unit of dwords */
void hfi_init(struct gmu_device *gmu)
{
struct kgsl_hfi *hfi = &gmu->hfi;
struct adreno_device *adreno_dev = ADRENO_DEVICE(hfi->kgsldev);
struct gmu_memdesc *mem_addr = gmu->hfi_mem;
int i;
struct hfi_queue_table *tbl;
struct hfi_queue_header *hdr;
struct {
unsigned int idx;
unsigned int pri;
unsigned int status;
} queue[HFI_QUEUE_MAX] = {
{ HFI_CMD_IDX, HFI_CMD_PRI, HFI_QUEUE_STATUS_ENABLED },
{ HFI_MSG_IDX, HFI_MSG_PRI, HFI_QUEUE_STATUS_ENABLED },
{ HFI_DBG_IDX, HFI_DBG_PRI, HFI_QUEUE_STATUS_ENABLED },
{ HFI_DSP_IDX_0, HFI_DSP_PRI_0, HFI_QUEUE_STATUS_DISABLED },
};
/*
* Overwrite the queue IDs for A630, A615 and A616 as they use
* legacy firmware. Legacy firmware has different queue IDs for
* message, debug and dispatch queues.
*/
if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) {
queue[HFI_MSG_ID].idx = HFI_MSG_IDX_LEGACY;
queue[HFI_DBG_ID].idx = HFI_DBG_IDX_LEGACY;
queue[HFI_DSP_ID_0].idx = HFI_DSP_IDX_0_LEGACY;
}
/* Fill Table Header */
tbl = mem_addr->hostptr;
tbl->qtbl_hdr.version = 0;
tbl->qtbl_hdr.size = sizeof(struct hfi_queue_table) >> 2;
tbl->qtbl_hdr.qhdr0_offset = sizeof(struct hfi_queue_table_header) >> 2;
tbl->qtbl_hdr.qhdr_size = sizeof(struct hfi_queue_header) >> 2;
tbl->qtbl_hdr.num_q = HFI_QUEUE_MAX;
tbl->qtbl_hdr.num_active_q = HFI_QUEUE_MAX;
memset(&tbl->qhdr[0], 0, sizeof(tbl->qhdr));
/* Fill Individual Queue Headers */
for (i = 0; i < HFI_QUEUE_MAX; i++) {
hdr = &tbl->qhdr[i];
hdr->start_addr = GMU_QUEUE_START_ADDR(mem_addr, i);
hdr->type = QUEUE_HDR_TYPE(queue[i].idx, queue[i].pri, 0, 0);
hdr->status = queue[i].status;
hdr->queue_size = HFI_QUEUE_SIZE >> 2; /* convert to dwords */
}
mutex_init(&hfi->cmdq_mutex);
}
#define HDR_CMP_SEQNUM(out_hdr, in_hdr) \
(MSG_HDR_GET_SEQNUM(out_hdr) == MSG_HDR_GET_SEQNUM(in_hdr))
static void receive_ack_cmd(struct gmu_device *gmu, void *rcvd,
struct pending_cmd *ret_cmd)
{
uint32_t *ack = rcvd;
uint32_t hdr = ack[0];
uint32_t req_hdr = ack[1];
struct kgsl_hfi *hfi = &gmu->hfi;
if (ret_cmd == NULL)
return;
trace_kgsl_hfi_receive(MSG_HDR_GET_ID(req_hdr),
MSG_HDR_GET_SIZE(req_hdr),
MSG_HDR_GET_SEQNUM(req_hdr));
if (HDR_CMP_SEQNUM(ret_cmd->sent_hdr, req_hdr)) {
memcpy(&ret_cmd->results, ack, MSG_HDR_GET_SIZE(hdr) << 2);
return;
}
/* Didn't find the sender, list the waiter */
dev_err_ratelimited(&gmu->pdev->dev,
"HFI ACK: Cannot find sender for 0x%8.8x Waiter: 0x%8.8x\n",
req_hdr, ret_cmd->sent_hdr);
adreno_set_gpu_fault(ADRENO_DEVICE(hfi->kgsldev), ADRENO_GMU_FAULT);
adreno_dispatcher_schedule(hfi->kgsldev);
}
#define MSG_HDR_SET_SEQNUM(hdr, num) \
(((hdr) & 0xFFFFF) | ((num) << 20))
static int poll_adreno_gmu_reg(struct adreno_device *adreno_dev,
enum adreno_regs offset_name, unsigned int expected_val,
unsigned int mask, unsigned int timeout_ms)
{
unsigned int val;
unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
while (time_is_after_jiffies(timeout)) {
adreno_read_gmureg(adreno_dev, offset_name, &val);
if ((val & mask) == expected_val)
return 0;
usleep_range(10, 100);
}
/* Check one last time */
adreno_read_gmureg(adreno_dev, offset_name, &val);
if ((val & mask) == expected_val)
return 0;
return -ETIMEDOUT;
}
static int hfi_send_cmd(struct gmu_device *gmu, uint32_t queue_idx,
void *data, struct pending_cmd *ret_cmd)
{
int rc;
uint32_t *cmd = data;
struct kgsl_hfi *hfi = &gmu->hfi;
unsigned int seqnum = atomic_inc_return(&hfi->seqnum);
struct adreno_device *adreno_dev = ADRENO_DEVICE(hfi->kgsldev);
*cmd = MSG_HDR_SET_SEQNUM(*cmd, seqnum);
if (ret_cmd == NULL)
return hfi_queue_write(gmu, queue_idx, cmd);
ret_cmd->sent_hdr = cmd[0];
rc = hfi_queue_write(gmu, queue_idx, cmd);
if (rc)
return rc;
rc = poll_adreno_gmu_reg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_INFO,
HFI_IRQ_MSGQ_MASK, HFI_IRQ_MSGQ_MASK, HFI_RSP_TIMEOUT);
if (rc) {
dev_err(&gmu->pdev->dev,
"Timed out waiting on ack for 0x%8.8x (id %d, sequence %d)\n",
cmd[0], MSG_HDR_GET_ID(*cmd), MSG_HDR_GET_SEQNUM(*cmd));
return rc;
}
/* Clear the interrupt */
adreno_write_gmureg(adreno_dev, ADRENO_REG_GMU_GMU2HOST_INTR_CLR,
HFI_IRQ_MSGQ_MASK);
hfi_process_queue(gmu, HFI_MSG_ID, ret_cmd);
return rc;
}
#define HFI_ACK_ERROR 0xffffffff
static int hfi_send_generic_req(struct gmu_device *gmu, uint32_t queue,
void *cmd)
{
struct pending_cmd ret_cmd;
int rc;
memset(&ret_cmd, 0, sizeof(ret_cmd));
rc = hfi_send_cmd(gmu, queue, cmd, &ret_cmd);
if (!rc && ret_cmd.results[2] == HFI_ACK_ERROR) {
dev_err(&gmu->pdev->dev, "HFI ACK failure: Req 0x%8.8X\n",
ret_cmd.results[1]);
return -EINVAL;
}
return rc;
}
static int hfi_send_gmu_init(struct gmu_device *gmu, uint32_t boot_state)
{
struct hfi_gmu_init_cmd cmd = {
.hdr = CMD_MSG_HDR(H2F_MSG_INIT, sizeof(cmd)),
.seg_id = 0,
.dbg_buffer_addr = (unsigned int) gmu->dump_mem->gmuaddr,
.dbg_buffer_size = (unsigned int) gmu->dump_mem->size,
.boot_state = boot_state,
};
return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd);
}
static int hfi_get_fw_version(struct gmu_device *gmu,
uint32_t expected_ver, uint32_t *ver)
{
struct hfi_fw_version_cmd cmd = {
.hdr = CMD_MSG_HDR(H2F_MSG_FW_VER, sizeof(cmd)),
.supported_ver = expected_ver,
};
int rc;
struct pending_cmd ret_cmd;
memset(&ret_cmd, 0, sizeof(ret_cmd));
rc = hfi_send_cmd(gmu, HFI_CMD_ID, &cmd, &ret_cmd);
if (rc)
return rc;
rc = ret_cmd.results[2];
if (!rc)
*ver = ret_cmd.results[3];
else
dev_err(&gmu->pdev->dev,
"gmu get fw ver failed with error=%d\n", rc);
return rc;
}
static int hfi_send_core_fw_start(struct gmu_device *gmu)
{
struct hfi_core_fw_start_cmd cmd = {
.hdr = CMD_MSG_HDR(H2F_MSG_CORE_FW_START, sizeof(cmd)),
.handle = 0x0,
};
return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd);
}
static const char * const hfi_features[] = {
[HFI_FEATURE_ECP] = "ECP",
[HFI_FEATURE_ACD] = "ACD",
[HFI_FEATURE_LM] = "LM",
};
static const char *feature_to_string(uint32_t feature)
{
if (feature < ARRAY_SIZE(hfi_features) && hfi_features[feature])
return hfi_features[feature];
return "unknown";
}
static int hfi_send_feature_ctrl(struct gmu_device *gmu,
uint32_t feature, uint32_t enable, uint32_t data)
{
struct hfi_feature_ctrl_cmd cmd = {
.hdr = CMD_MSG_HDR(H2F_MSG_FEATURE_CTRL, sizeof(cmd)),
.feature = feature,
.enable = enable,
.data = data,
};
int ret;
ret = hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd);
if (ret)
dev_err(&gmu->pdev->dev,
"Unable to %s feature %s (%d)\n",
enable ? "enable" : "disable",
feature_to_string(feature),
feature);
return ret;
}
static int hfi_send_dcvstbl_v1(struct gmu_device *gmu)
{
struct hfi_dcvstable_v1_cmd cmd = {
.hdr = CMD_MSG_HDR(H2F_MSG_PERF_TBL, sizeof(cmd)),
.gpu_level_num = gmu->num_gpupwrlevels,
.gmu_level_num = GMU_PWR_LEVELS,
};
int i;
for (i = 0; i < gmu->num_gpupwrlevels; i++) {
cmd.gx_votes[i].vote = gmu->rpmh_votes.gx_votes[i];
/* Divide by 1000 to convert to kHz */
cmd.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
}
cmd.cx_votes[0].vote = gmu->rpmh_votes.cx_votes[0];
cmd.cx_votes[0].freq = 0;
cmd.cx_votes[1].vote = gmu->rpmh_votes.cx_votes[1];
cmd.cx_votes[1].freq = GMU_FREQUENCY / 1000;
return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd);
}
static int hfi_send_get_value(struct gmu_device *gmu,
struct hfi_get_value_req *req)
{
struct hfi_get_value_cmd *cmd = &req->cmd;
struct pending_cmd ret_cmd;
struct hfi_get_value_reply_cmd *reply =
(struct hfi_get_value_reply_cmd *)ret_cmd.results;
int rc;
cmd->hdr = CMD_MSG_HDR(H2F_MSG_GET_VALUE, sizeof(*cmd));
rc = hfi_send_cmd(gmu, HFI_CMD_ID, cmd, &ret_cmd);
if (rc)
return rc;
memset(&req->data, 0, sizeof(req->data));
memcpy(&req->data, &reply->data,
(MSG_HDR_GET_SIZE(reply->hdr) - 2) << 2);
return 0;
}
static int hfi_send_dcvstbl(struct gmu_device *gmu)
{
struct hfi_dcvstable_cmd cmd = {
.hdr = CMD_MSG_HDR(H2F_MSG_PERF_TBL, sizeof(cmd)),
.gpu_level_num = gmu->num_gpupwrlevels,
.gmu_level_num = GMU_PWR_LEVELS,
};
int i;
for (i = 0; i < gmu->num_gpupwrlevels; i++) {
cmd.gx_votes[i].vote = gmu->rpmh_votes.gx_votes[i];
/* Hardcode this to the max threshold since it is not used */
cmd.gx_votes[i].acd = 0xFFFFFFFF;
/* Divide by 1000 to convert to kHz */
cmd.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
}
cmd.cx_votes[0].vote = gmu->rpmh_votes.cx_votes[0];
cmd.cx_votes[0].freq = 0;
cmd.cx_votes[1].vote = gmu->rpmh_votes.cx_votes[1];
cmd.cx_votes[1].freq = GMU_FREQUENCY / 1000;
return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd);
}
static int hfi_send_bwtbl(struct gmu_device *gmu)
{
struct hfi_bwtable_cmd *cmd = &gmu->hfi.bwtbl_cmd;
cmd->hdr = CMD_MSG_HDR(H2F_MSG_BW_VOTE_TBL, sizeof(*cmd));
return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd);
}
static int hfi_send_acd_tbl(struct gmu_device *gmu)
{
struct hfi_acd_table_cmd *cmd = &gmu->hfi.acd_tbl_cmd;
cmd->hdr = CMD_MSG_HDR(H2F_MSG_ACD_TBL, sizeof(*cmd));
return hfi_send_generic_req(gmu, HFI_CMD_IDX, cmd);
}
static int hfi_send_test(struct gmu_device *gmu)
{
struct hfi_test_cmd cmd = {
.hdr = CMD_MSG_HDR(H2F_MSG_TEST, sizeof(cmd)),
};
return hfi_send_generic_req(gmu, HFI_CMD_ID, &cmd);
}
static void receive_err_req(struct gmu_device *gmu, void *rcvd)
{
struct hfi_err_cmd *cmd = rcvd;
dev_err(&gmu->pdev->dev, "HFI Error Received: %d %d %s\n",
((cmd->error_code >> 16) & 0xFFFF),
(cmd->error_code & 0xFFFF),
(char *) cmd->data);
}
static void receive_debug_req(struct gmu_device *gmu, void *rcvd)
{
struct hfi_debug_cmd *cmd = rcvd;
dev_dbg(&gmu->pdev->dev, "HFI Debug Received: %d %d %d\n",
cmd->type, cmd->timestamp, cmd->data);
}
static void hfi_v1_receiver(struct gmu_device *gmu, uint32_t *rcvd,
struct pending_cmd *ret_cmd)
{
/* V1 ACK Handler */
if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_V1_MSG_ACK) {
receive_ack_cmd(gmu, rcvd, ret_cmd);
return;
}
/* V1 Request Handler */
switch (MSG_HDR_GET_ID(rcvd[0])) {
case F2H_MSG_ERR: /* No Reply */
receive_err_req(gmu, rcvd);
break;
case F2H_MSG_DEBUG: /* No Reply */
receive_debug_req(gmu, rcvd);
break;
default: /* No Reply */
dev_err(&gmu->pdev->dev,
"HFI V1 request %d not supported\n",
MSG_HDR_GET_ID(rcvd[0]));
break;
}
}
static void hfi_process_queue(struct gmu_device *gmu, uint32_t queue_idx,
struct pending_cmd *ret_cmd)
{
uint32_t rcvd[MAX_RCVD_SIZE];
while (hfi_queue_read(gmu, queue_idx, rcvd, sizeof(rcvd)) > 0) {
/* Special case if we're v1 */
if (GMU_VER_MAJOR(gmu->ver.hfi) < 2) {
hfi_v1_receiver(gmu, rcvd, ret_cmd);
continue;
}
/* V2 ACK Handler */
if (MSG_HDR_GET_TYPE(rcvd[0]) == HFI_MSG_ACK) {
receive_ack_cmd(gmu, rcvd, ret_cmd);
continue;
}
/* V2 Request Handler */
switch (MSG_HDR_GET_ID(rcvd[0])) {
case F2H_MSG_ERR: /* No Reply */
receive_err_req(gmu, rcvd);
break;
case F2H_MSG_DEBUG: /* No Reply */
receive_debug_req(gmu, rcvd);
break;
default: /* No Reply */
dev_err(&gmu->pdev->dev,
"HFI request %d not supported\n",
MSG_HDR_GET_ID(rcvd[0]));
break;
}
}
}
void hfi_receiver(unsigned long data)
{
/* Process all asynchronous read (firmware to host) queues */
hfi_process_queue((struct gmu_device *) data, HFI_DBG_ID, NULL);
}
static int hfi_verify_fw_version(struct kgsl_device *device,
struct gmu_device *gmu)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
const struct adreno_a6xx_core *a6xx_core = to_a6xx_core(adreno_dev);
int result;
unsigned int ver, major, minor;
/* GMU version is already known, so don't waste time finding again */
if (gmu->ver.core != 0)
return 0;
major = a6xx_core->gmu_major;
minor = a6xx_core->gmu_minor;
result = hfi_get_fw_version(gmu, GMU_VERSION(major, minor), &ver);
if (result) {
dev_err_once(&gmu->pdev->dev,
"Failed to get FW version via HFI\n");
return result;
}
/* For now, warn once. Could return error later if needed */
if (major != GMU_VER_MAJOR(ver))
dev_err_once(&gmu->pdev->dev,
"FW Major Error: Wanted %d, got %d\n",
major, GMU_VER_MAJOR(ver));
if (minor > GMU_VER_MINOR(ver))
dev_err_once(&gmu->pdev->dev,
"FW Minor Error: Wanted < %d, got %d\n",
GMU_VER_MINOR(ver), minor);
/* Save the gmu version information */
gmu->ver.core = ver;
return 0;
}
static int hfi_send_lm_feature_ctrl(struct gmu_device *gmu,
struct adreno_device *adreno_dev)
{
struct hfi_set_value_cmd req = {
.type = HFI_VALUE_LM_CS0,
.subtype = 0,
.data = adreno_dev->lm_slope,
};
struct kgsl_device *device = &adreno_dev->dev;
int ret;
if (!test_bit(ADRENO_LM_CTRL, &adreno_dev->pwrctrl_flag))
return 0;
ret = hfi_send_feature_ctrl(gmu, HFI_FEATURE_LM, 1,
device->pwrctrl.throttle_mask);
if (!ret)
ret = hfi_send_req(gmu, H2F_MSG_SET_VALUE, &req);
return ret;
}
static int hfi_send_acd_feature_ctrl(struct gmu_device *gmu,
struct adreno_device *adreno_dev)
{
int ret = 0;
if (test_bit(ADRENO_ACD_CTRL, &adreno_dev->pwrctrl_flag)) {
ret = hfi_send_acd_tbl(gmu);
if (!ret)
ret = hfi_send_feature_ctrl(gmu, HFI_FEATURE_ACD, 1, 0);
}
return ret;
}
int hfi_start(struct kgsl_device *device,
struct gmu_device *gmu, uint32_t boot_state)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct gmu_memdesc *mem_addr = gmu->hfi_mem;
struct hfi_queue_table *tbl = mem_addr->hostptr;
struct hfi_queue_header *hdr;
int result, i;
if (test_bit(GMU_HFI_ON, &device->gmu_core.flags))
return 0;
/* Force read_index to the write_index no matter what */
for (i = 0; i < HFI_QUEUE_MAX; i++) {
hdr = &tbl->qhdr[i];
if (hdr->status == HFI_QUEUE_STATUS_DISABLED)
continue;
if (hdr->read_index != hdr->write_index) {
dev_err(&gmu->pdev->dev,
"HFI Q[%d] Index Error: read:0x%X write:0x%X\n",
i, hdr->read_index, hdr->write_index);
hdr->read_index = hdr->write_index;
}
}
/* This is legacy HFI message for A630 and A615 family firmware */
if (adreno_is_a630(adreno_dev) || adreno_is_a615_family(adreno_dev)) {
result = hfi_send_gmu_init(gmu, boot_state);
if (result)
return result;
}
result = hfi_verify_fw_version(device, gmu);
if (result)
return result;
if (GMU_VER_MAJOR(gmu->ver.hfi) < 2)
result = hfi_send_dcvstbl_v1(gmu);
else
result = hfi_send_dcvstbl(gmu);
if (result)
return result;
result = hfi_send_bwtbl(gmu);
if (result)
return result;
/*
* If quirk is enabled send H2F_MSG_TEST and tell the GMU
* we are sending no more HFIs until the next boot otherwise
* send H2F_MSG_CORE_FW_START and features for A640 devices
*/
if (GMU_VER_MAJOR(gmu->ver.hfi) >= 2) {
if (ADRENO_FEATURE(adreno_dev, ADRENO_ECP)) {
result = hfi_send_feature_ctrl(gmu,
HFI_FEATURE_ECP, 1, 0);
if (result)
return result;
}
result = hfi_send_acd_feature_ctrl(gmu, adreno_dev);
if (result)
return result;
result = hfi_send_lm_feature_ctrl(gmu, adreno_dev);
if (result)
return result;
result = hfi_send_core_fw_start(gmu);
if (result)
return result;
} else {
if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_HFI_USE_REG)) {
result = hfi_send_test(gmu);
if (result)
return result;
}
}
set_bit(GMU_HFI_ON, &device->gmu_core.flags);
return 0;
}
void hfi_stop(struct gmu_device *gmu)
{
struct gmu_memdesc *mem_addr = gmu->hfi_mem;
struct hfi_queue_table *tbl = mem_addr->hostptr;
struct hfi_queue_header *hdr;
struct kgsl_hfi *hfi = &gmu->hfi;
struct kgsl_device *device = hfi->kgsldev;
unsigned int i;
if (!test_bit(GMU_HFI_ON, &device->gmu_core.flags))
return;
/* Flush HFI queues */
for (i = 0; i < HFI_QUEUE_MAX; i++) {
hdr = &tbl->qhdr[i];
if (hdr->status == HFI_QUEUE_STATUS_DISABLED)
continue;
if (hdr->read_index != hdr->write_index)
dev_err(&gmu->pdev->dev,
"HFI queue[%d] is not empty before close: rd=%d,wt=%d\n",
i, hdr->read_index, hdr->write_index);
}
clear_bit(GMU_HFI_ON, &device->gmu_core.flags);
}
/* Entry point for external HFI requests */
int hfi_send_req(struct gmu_device *gmu, unsigned int id, void *data)
{
switch (id) {
case H2F_MSG_GX_BW_PERF_VOTE: {
struct hfi_gx_bw_perf_vote_cmd *cmd = data;
cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd));
return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd);
}
case H2F_MSG_PREPARE_SLUMBER: {
struct hfi_prep_slumber_cmd *cmd = data;
if (cmd->freq >= MAX_GX_LEVELS || cmd->bw >= MAX_GX_LEVELS)
return -EINVAL;
cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd));
return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd);
}
case H2F_MSG_START: {
struct hfi_start_cmd *cmd = data;
cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd));
return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd);
}
case H2F_MSG_GET_VALUE: {
return hfi_send_get_value(gmu, data);
}
case H2F_MSG_SET_VALUE: {
struct hfi_set_value_cmd *cmd = data;
cmd->hdr = CMD_MSG_HDR(id, sizeof(*cmd));
return hfi_send_generic_req(gmu, HFI_CMD_ID, cmd);
}
default:
break;
}
return -EINVAL;
}
/* HFI interrupt handler */
irqreturn_t hfi_irq_handler(int irq, void *data)
{
struct kgsl_device *device = data;
struct gmu_device *gmu = KGSL_GMU_DEVICE(device);
struct kgsl_hfi *hfi = &gmu->hfi;
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
unsigned int status = 0;
adreno_read_gmureg(ADRENO_DEVICE(device),
ADRENO_REG_GMU_GMU2HOST_INTR_INFO, &status);
adreno_write_gmureg(ADRENO_DEVICE(device),
ADRENO_REG_GMU_GMU2HOST_INTR_CLR, HFI_IRQ_MASK);
if (status & HFI_IRQ_DBGQ_MASK)
tasklet_hi_schedule(&hfi->tasklet);
if (status & HFI_IRQ_CM3_FAULT_MASK) {
dev_err_ratelimited(&gmu->pdev->dev,
"GMU CM3 fault interrupt received\n");
adreno_set_gpu_fault(adreno_dev, ADRENO_GMU_FAULT);
adreno_dispatcher_schedule(device);
}
if (status & ~HFI_IRQ_MASK)
dev_err_ratelimited(&gmu->pdev->dev,
"Unhandled HFI interrupts 0x%lx\n",
status & ~HFI_IRQ_MASK);
return IRQ_HANDLED;
}

623
drivers/gpu/msm/kgsl_hfi.h Normal file
View file

@ -0,0 +1,623 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_HFI_H
#define __KGSL_HFI_H
#define HFI_QUEUE_SIZE SZ_4K /* bytes, must be base 4dw */
#define MAX_RCVD_PAYLOAD_SIZE 16 /* dwords */
#define MAX_RCVD_SIZE (MAX_RCVD_PAYLOAD_SIZE + 3) /* dwords */
#define HFI_MAX_MSG_SIZE (SZ_1K>>2) /* dwords */
/* Below section is for all structures related to HFI queues */
#define HFI_QUEUE_DEFAULT_CNT 3
#define HFI_QUEUE_DISPATCH_CNT 1
#define HFI_QUEUE_MAX (HFI_QUEUE_DEFAULT_CNT + HFI_QUEUE_DISPATCH_CNT)
struct hfi_queue_table;
/* Total header sizes + queue sizes + 16 for alignment */
#define HFIMEM_SIZE (sizeof(struct hfi_queue_table) + 16 + \
(HFI_QUEUE_SIZE * HFI_QUEUE_MAX))
#define HFI_CMD_ID 0
#define HFI_MSG_ID 1
#define HFI_DBG_ID 2
#define HFI_DSP_ID_0 3
#define HFI_CMD_IDX 0
#define HFI_MSG_IDX 1
#define HFI_DBG_IDX 2
#define HFI_DSP_IDX_BASE 3
#define HFI_DSP_IDX_0 3
#define HFI_CMD_IDX_LEGACY 0
#define HFI_DSP_IDX_0_LEGACY 1
#define HFI_MSG_IDX_LEGACY 4
#define HFI_DBG_IDX_LEGACY 5
#define HFI_QUEUE_STATUS_DISABLED 0
#define HFI_QUEUE_STATUS_ENABLED 1
/* HTOF queue priority, 1 is highest priority */
#define HFI_CMD_PRI 10
#define HFI_MSG_PRI 10
#define HFI_DBG_PRI 40
#define HFI_DSP_PRI_0 20
#define HFI_RSP_TIMEOUT 100 /* msec */
#define HFI_H2F_CMD_IRQ_MASK BIT(0)
#define HFI_IRQ_MSGQ_MASK BIT(0)
#define HFI_IRQ_SIDEMSGQ_MASK BIT(1)
#define HFI_IRQ_DBGQ_MASK BIT(2)
#define HFI_IRQ_CM3_FAULT_MASK BIT(15)
#define HFI_IRQ_OOB_MASK GENMASK(31, 16)
#define HFI_IRQ_MASK (HFI_IRQ_SIDEMSGQ_MASK |\
HFI_IRQ_DBGQ_MASK |\
HFI_IRQ_CM3_FAULT_MASK)
#define CLKSET_OPTION_DEFAULT 0
#define CLKSET_OPTION_CLOSEST 1
#define CLKSET_OPTION_ATMOST 2
#define CLKSET_OPTION_ATLEAST 3
#define DCVS_ACK_NONBLOCK 0
#define DCVS_ACK_BLOCK 1
#define HFI_FEATURE_DCVS 0
#define HFI_FEATURE_ECP 1
#define HFI_FEATURE_PREEMPTION 2
#define HFI_FEATURE_CLOCKS_ON 3
#define HFI_FEATURE_BUS_ON 4
#define HFI_FEATURE_RAIL_ON 5
#define HFI_FEATURE_HWCG 6
#define HFI_FEATURE_LM 7
#define HFI_FEATURE_THROTTLE 8
#define HFI_FEATURE_IFPC 9
#define HFI_FEATURE_NAP 10
#define HFI_FEATURE_BCL 11
#define HFI_FEATURE_ACD 12
#define HFI_FEATURE_DIDT 13
#define HFI_VALUE_FT_POLICY 100
#define HFI_VALUE_RB_MAX_CMDS 101
#define HFI_VALUE_CTX_MAX_CMDS 102
#define HFI_VALUE_ADDRESS 103
#define HFI_VALUE_MAX_GPU_PERF_INDEX 104
#define HFI_VALUE_MIN_GPU_PERF_INDEX 105
#define HFI_VALUE_MAX_BW_PERF_INDEX 106
#define HFI_VALUE_MIN_BW_PERF_INDEX 107
#define HFI_VALUE_MAX_GPU_THERMAL_INDEX 108
#define HFI_VALUE_GPUCLK 109
#define HFI_VALUE_CLK_TIME 110
#define HFI_VALUE_LOG_LEVEL 111
#define HFI_VALUE_LOG_EVENT_ON 112
#define HFI_VALUE_LOG_EVENT_OFF 113
#define HFI_VALUE_DCVS_OBJ 114
#define HFI_VALUE_LM_CS0 115
#define HFI_VALUE_GLOBAL_TOKEN 0xFFFFFFFF
/**
* struct hfi_queue_table_header - HFI queue table structure
* @version: HFI protocol version
* @size: queue table size in dwords
* @qhdr0_offset: first queue header offset (dwords) in this table
* @qhdr_size: queue header size
* @num_q: number of queues defined in this table
* @num_active_q: number of active queues
*/
struct hfi_queue_table_header {
uint32_t version;
uint32_t size;
uint32_t qhdr0_offset;
uint32_t qhdr_size;
uint32_t num_q;
uint32_t num_active_q;
};
/**
* struct hfi_queue_header - HFI queue header structure
* @status: active: 1; inactive: 0
* @start_addr: starting address of the queue in GMU VA space
* @type: queue type encoded the priority, ID and send/recevie types
* @queue_size: size of the queue
* @msg_size: size of the message if each message has fixed size.
* Otherwise, 0 means variable size of message in the queue.
* @read_index: read index of the queue
* @write_index: write index of the queue
*/
struct hfi_queue_header {
uint32_t status;
uint32_t start_addr;
uint32_t type;
uint32_t queue_size;
uint32_t msg_size;
uint32_t unused0;
uint32_t unused1;
uint32_t unused2;
uint32_t unused3;
uint32_t unused4;
uint32_t read_index;
uint32_t write_index;
};
struct hfi_queue_table {
struct hfi_queue_table_header qtbl_hdr;
struct hfi_queue_header qhdr[HFI_QUEUE_MAX];
};
enum hfi_msg_type {
HFI_MSG_CMD = 0, /* V1 and V2 */
HFI_MSG_ACK = 1, /* V2 only */
HFI_V1_MSG_POST = 1, /* V1 only */
HFI_V1_MSG_ACK = 2, /* V1 only */
};
#define H2F_MSG_INIT 0
#define H2F_MSG_FW_VER 1
#define H2F_MSG_LM_CFG 2
#define H2F_MSG_BW_VOTE_TBL 3
#define H2F_MSG_PERF_TBL 4
#define H2F_MSG_TEST 5
#define H2F_MSG_ACD_TBL 7
#define H2F_MSG_START 10
#define H2F_MSG_FEATURE_CTRL 11
#define H2F_MSG_GET_VALUE 12
#define H2F_MSG_SET_VALUE 13
#define H2F_MSG_CORE_FW_START 14
#define F2H_MSG_MEM_ALLOC 20
#define H2F_MSG_GX_BW_PERF_VOTE 30
#define H2F_MSG_FW_HALT 32
#define H2F_MSG_PREPARE_SLUMBER 33
#define F2H_MSG_ERR 100
#define F2H_MSG_DEBUG 101
#define F2H_MSG_GMU_CNTR_REGISTER 110
#define F2H_MSG_GMU_CNTR_RELEASE 111
#define F2H_MSG_ACK 126 /* Deprecated for v2.0*/
#define H2F_MSG_ACK 127 /* Deprecated for v2.0*/
#define H2F_MSG_REGISTER_CONTEXT 128
#define H2F_MSG_UNREGISTER_CONTEXT 129
#define H2F_MSG_ISSUE_CMD 130
#define H2F_MSG_ISSUE_CMD_RAW 131
#define H2F_MSG_TS_NOTIFY 132
#define F2H_MSG_TS_RETIRE 133
#define H2F_MSG_CONTEXT_POINTERS 134
#define H2F_MSG_CONTEXT_RULE 140 /* AKA constraint */
#define F2H_MSG_CONTEXT_BAD 150
/* H2F */
struct hfi_gmu_init_cmd {
uint32_t hdr;
uint32_t seg_id;
uint32_t dbg_buffer_addr;
uint32_t dbg_buffer_size;
uint32_t boot_state;
};
/* H2F */
struct hfi_fw_version_cmd {
uint32_t hdr;
uint32_t supported_ver;
};
#define ARC_VOTE_GET_PRI(_v) ((_v) & 0xFF)
#define ARC_VOTE_GET_SEC(_v) (((_v) >> 8) & 0xFF)
#define ARC_VOTE_GET_VLVL(_v) (((_v) >> 16) & 0xFFFF)
#define ARC_VOTE_SET(pri, sec, vlvl) \
((((vlvl) & 0xFFFF) << 16) | (((sec) & 0xFF) << 8) | ((pri) & 0xFF))
/* H2F */
struct hfi_bwtable_cmd {
uint32_t hdr;
uint32_t bw_level_num;
uint32_t cnoc_cmds_num;
uint32_t ddr_cmds_num;
uint32_t cnoc_wait_bitmask;
uint32_t ddr_wait_bitmask;
uint32_t cnoc_cmd_addrs[MAX_CNOC_CMDS];
uint32_t cnoc_cmd_data[MAX_CNOC_LEVELS][MAX_CNOC_CMDS];
uint32_t ddr_cmd_addrs[MAX_BW_CMDS];
uint32_t ddr_cmd_data[MAX_GX_LEVELS][MAX_BW_CMDS];
};
struct opp_gx_desc {
uint32_t vote;
uint32_t acd;
uint32_t freq;
};
struct opp_desc {
uint32_t vote;
uint32_t freq;
};
/* H2F */
struct hfi_dcvstable_v1_cmd {
uint32_t hdr;
uint32_t gpu_level_num;
uint32_t gmu_level_num;
struct opp_desc gx_votes[MAX_GX_LEVELS];
struct opp_desc cx_votes[MAX_CX_LEVELS];
};
/* H2F */
struct hfi_dcvstable_cmd {
uint32_t hdr;
uint32_t gpu_level_num;
uint32_t gmu_level_num;
struct opp_gx_desc gx_votes[MAX_GX_LEVELS];
struct opp_desc cx_votes[MAX_CX_LEVELS];
};
#define HFI_ACD_INIT_VERSION 1
#define MAX_ACD_STRIDE 2
#define MAX_ACD_NUM_LEVELS 6
/* H2F */
struct hfi_acd_table_cmd {
uint32_t hdr;
uint32_t version;
uint32_t enable_by_level;
uint32_t stride;
uint32_t num_levels;
uint32_t data[MAX_ACD_NUM_LEVELS * MAX_ACD_STRIDE];
};
/* H2F */
struct hfi_test_cmd {
uint32_t hdr;
uint32_t data;
};
/* H2F */
struct hfi_start_cmd {
uint32_t hdr;
};
/* H2F */
struct hfi_feature_ctrl_cmd {
uint32_t hdr;
uint32_t feature;
uint32_t enable;
uint32_t data;
};
/* H2F */
struct hfi_get_value_cmd {
uint32_t hdr;
uint32_t type;
uint32_t subtype;
};
/* Internal */
struct hfi_get_value_req {
struct hfi_get_value_cmd cmd;
uint32_t data[16];
};
/* F2H */
struct hfi_get_value_reply_cmd {
uint32_t hdr;
uint32_t req_hdr;
uint32_t data[16];
};
/* H2F */
struct hfi_set_value_cmd {
uint32_t hdr;
uint32_t type;
uint32_t subtype;
uint32_t data;
};
/* H2F */
struct hfi_core_fw_start_cmd {
uint32_t hdr;
uint32_t handle;
};
/* CP/GFX pipeline can access, The mem_kind may imply restrictions for non-CP */
#define MEMFLAG_GFX_ACC BIT(0)
/* Buffer has APRIV protection in GFX PTEs */
#define MEMFLAG_GFX_PRIV BIT(1)
/* Buffer is read-write for GFX PTEs. A 0 indicates read-only */
#define MEMFLAG_GFX_WRITEABLE BIT(2)
/* GMU can access */
#define MEMFLAG_GMU_ACC BIT(3)
/* Buffer has APRIV protection in GMU PTEs */
#define MEMFLAG_GMU_PRIV BIT(4)
/* Buffer is read-write for GMU PTEs. A 0 indicates read-only */
#define MEMFLAG_GMU_WRITEABLE BIT(5)
/* Buffer is located in GMU's non-cached bufferable VA range */
#define MEMFLAG_GMU_BUFFERABLE BIT(6)
/* Buffer is located in GMU's cacheable VA range */
#define MEMFLAG_GMU_CACHEABLE BIT(7)
/* Host can access */
#define MEMFLAG_HOST_ACC BIT(8)
/*
* Request that Host initialize the buffer.
* Implies zero-init, unless Memkind implies otherwise
*/
#define MEMFLAG_HOST_INIT BIT(9)
#define HFI_MEMKIND_GENERIC 0
#define HFI_MEMKIND_RB 1
#define HFI_MEMKIND_MEMSTORE 2
#define HFI_MEMKIND_CSW_SMMU_INFO 3
#define HFI_MEMKIND_CSW_PRIV_NON_SECURE 4
#define HFI_MEMKIND_CSW_PRIV_SECURE 5
#define HFI_MEMKIND_CSW_NON_PRIV 6
#define HFI_MEMKIND_CSW_COUNTER 7
#define HFI_MEMKIND_CTXTREC_PERF_CNTR_SAVE_RESTORE 8
#define HFI_MEMKIND_CTXTREC_PREEMPT_CNTR 9
#define HFI_MEMKIND_SYS_LOG 10
#define HFI_MEMKIND_CRASH_DUMP 11
#define HFI_MEMKIND_MMIO_DPU 12
#define HFI_MEMKIND_MMIO_TCSR 13
#define HFI_MEMKIND_MMIO_QDSS_STM 14
struct hfi_mem_alloc_desc {
uint64_t gpu_addr;
uint32_t flags;
uint32_t mem_kind;
uint32_t host_mem_handle;
uint32_t gmu_mem_handle;
uint32_t gmu_addr;
uint32_t size; /* Bytes */
};
/* F2H */
struct hfi_mem_alloc_cmd {
uint32_t hdr;
uint32_t reserved; /* Padding to ensure alignment of 'desc' below */
struct hfi_mem_alloc_desc desc;
};
/* H2F */
struct hfi_mem_alloc_reply_cmd {
uint32_t hdr;
uint32_t req_hdr;
struct hfi_mem_alloc_desc desc;
};
/* H2F */
struct hfi_gx_bw_perf_vote_cmd {
uint32_t hdr;
uint32_t ack_type;
uint32_t freq;
uint32_t bw;
};
/* H2F */
struct hfi_fw_halt_cmd {
uint32_t hdr;
uint32_t en_halt;
};
/* H2F */
struct hfi_prep_slumber_cmd {
uint32_t hdr;
uint32_t bw;
uint32_t freq;
};
/* F2H */
struct hfi_err_cmd {
uint32_t hdr;
uint32_t error_code;
uint32_t data[16];
};
/* F2H */
struct hfi_debug_cmd {
uint32_t hdr;
uint32_t type;
uint32_t timestamp;
uint32_t data;
};
/* F2H */
struct hfi_gmu_cntr_register_cmd {
uint32_t hdr;
uint32_t group_id;
uint32_t countable;
};
/* H2F */
struct hfi_gmu_cntr_register_reply_cmd {
uint32_t hdr;
uint32_t req_hdr;
uint32_t group_id;
uint32_t countable;
uint64_t counter_addr;
};
/* F2H */
struct hfi_gmu_cntr_release_cmd {
uint32_t hdr;
uint32_t group_id;
uint32_t countable;
};
#define CTXT_FLAG_PMODE 0x00000001
#define CTXT_FLAG_SWITCH_INTERNAL 0x00000002
#define CTXT_FLAG_SWITCH 0x00000008
#define CTXT_FLAG_NOTIFY 0x00000020
#define CTXT_FLAG_NO_FAULT_TOLERANCE 0x00000200
#define CTXT_FLAG_PWR_RULE 0x00000800
#define CTXT_FLAG_PRIORITY_MASK 0x0000F000
#define CTXT_FLAG_IFH_NOP 0x00010000
#define CTXT_FLAG_SECURE 0x00020000
#define CTXT_FLAG_TYPE_MASK 0x01F00000
#define CTXT_FLAG_TYPE_SHIFT 20
#define CTXT_FLAG_TYPE_ANY 0
#define CTXT_FLAG_TYPE_GL 1
#define CTXT_FLAG_TYPE_CL 2
#define CTXT_FLAG_TYPE_C2D 3
#define CTXT_FLAG_TYPE_RS 4
#define CTXT_FLAG_TYPE_UNKNOWN 0x1E
#define CTXT_FLAG_PREEMPT_STYLE_MASK 0x0E000000
#define CTXT_FLAG_PREEMPT_STYLE_SHIFT 25
#define CTXT_FLAG_PREEMPT_STYLE_ANY 0
#define CTXT_FLAG_PREEMPT_STYLE_RB 1
#define CTXT_FLAG_PREEMPT_STYLE_FG 2
/* H2F */
struct hfi_register_ctxt_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint32_t flags;
uint64_t pt_addr;
uint32_t ctxt_idr;
uint32_t ctxt_bank;
};
/* H2F */
struct hfi_unregister_ctxt_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint32_t ts;
};
#define CMDBATCH_SWITCH CTXT_FLAG_SWITCH
#define CMDBATCH_NOTIFY CTXT_FLAG_NOTIFY
#define CMDBATCH_PROFILING 0x00000010
#define CMDBATCH_EOF 0x00000100
#define CMDBATCH_PWR_STRICT CTXT_FLAG_PWR_RULE
struct hfi_issue_ib {
uint64_t addr;
uint32_t size;
};
/* H2F */
struct hfi_issue_cmd_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint32_t flags;
uint32_t ts;
uint32_t count;
struct hfi_issue_ib *ibs[];
};
/* Internal */
struct hfi_issue_cmd_req {
uint32_t queue;
uint32_t ctxt_id;
struct hfi_issue_cmd_cmd cmd;
};
/* H2F */
/* The length of *buf will be embedded in the hdr */
struct hfi_issue_cmd_raw_cmd {
uint32_t hdr;
uint32_t *buf;
};
/* Internal */
struct hfi_issue_cmd_raw_req {
uint32_t queue;
uint32_t ctxt_id;
uint32_t len;
uint32_t *buf;
};
/* H2F */
struct hfi_ts_notify_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint32_t ts;
};
#define TS_RETIRE_FLUSH 1
#define TS_RETIRE_ERROR 2
#define TS_RETIRE_PAST 3
#define TS_RETIRE_DONE 4
/* F2H */
struct hfi_ts_retire_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint32_t ts;
uint32_t type;
};
/* H2F */
struct hfi_context_pointers_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint64_t sop_addr;
uint64_t eop_addr;
};
/* H2F */
struct hfi_context_rule_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint32_t type;
uint32_t status;
};
/* F2H */
struct hfi_context_bad_cmd {
uint32_t hdr;
uint32_t ctxt_id;
uint32_t status;
uint32_t error;
};
/* H2F */
struct hfi_context_bad_reply_cmd {
uint32_t hdr;
uint32_t req_hdr;
};
/**
* struct pending_cmd - data structure to track outstanding HFI
* command messages
* @sent_hdr: copy of outgoing header for response comparison
* @results: the payload of received return message (ACK)
*/
struct pending_cmd {
uint32_t sent_hdr;
uint32_t results[MAX_RCVD_SIZE];
};
/**
* struct kgsl_hfi - HFI control structure
* @kgsldev: Point to the kgsl device
* @hfi_interrupt_num: number of GMU asserted HFI interrupt
* @cmdq_mutex: mutex to protect command queue access from multiple senders
* @tasklet: the thread handling received messages from GMU
* @seqnum: atomic counter that is incremented for each message sent. The
* value of the counter is used as sequence number for HFI message
* @bwtbl_cmd: HFI BW table buffer
* @acd_tbl_cmd: HFI table for ACD data
*/
struct kgsl_hfi {
struct kgsl_device *kgsldev;
int hfi_interrupt_num;
struct mutex cmdq_mutex;
struct tasklet_struct tasklet;
atomic_t seqnum;
struct hfi_bwtable_cmd bwtbl_cmd;
struct hfi_acd_table_cmd acd_tbl_cmd;
};
struct gmu_device;
struct gmu_memdesc;
irqreturn_t hfi_irq_handler(int irq, void *data);
int hfi_start(struct kgsl_device *device, struct gmu_device *gmu,
uint32_t boot_state);
void hfi_stop(struct gmu_device *gmu);
void hfi_receiver(unsigned long data);
void hfi_init(struct gmu_device *gmu);
/* hfi_send_req is only for external (to HFI) requests */
int hfi_send_req(struct gmu_device *gmu, unsigned int id, void *data);
#endif /* __KGSL_HFI_H */

175
drivers/gpu/msm/kgsl_ioctl.c Normal file
View file

@ -0,0 +1,175 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2008-2019, The Linux Foundation. All rights reserved.
*/
#include "kgsl_device.h"
#include "kgsl_sync.h"
static const struct kgsl_ioctl kgsl_ioctl_funcs[] = {
KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_GETPROPERTY,
kgsl_ioctl_device_getproperty),
/* IOCTL_KGSL_DEVICE_WAITTIMESTAMP is no longer supported */
KGSL_IOCTL_FUNC(IOCTL_KGSL_DEVICE_WAITTIMESTAMP_CTXTID,
kgsl_ioctl_device_waittimestamp_ctxtid),
KGSL_IOCTL_FUNC(IOCTL_KGSL_RINGBUFFER_ISSUEIBCMDS,
kgsl_ioctl_rb_issueibcmds),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SUBMIT_COMMANDS,
kgsl_ioctl_submit_commands),
/* IOCTL_KGSL_CMDSTREAM_READTIMESTAMP is no longer supported */
KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_READTIMESTAMP_CTXTID,
kgsl_ioctl_cmdstream_readtimestamp_ctxtid),
/* IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP is no longer supported */
KGSL_IOCTL_FUNC(IOCTL_KGSL_CMDSTREAM_FREEMEMONTIMESTAMP_CTXTID,
kgsl_ioctl_cmdstream_freememontimestamp_ctxtid),
KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_CREATE,
kgsl_ioctl_drawctxt_create),
KGSL_IOCTL_FUNC(IOCTL_KGSL_DRAWCTXT_DESTROY,
kgsl_ioctl_drawctxt_destroy),
KGSL_IOCTL_FUNC(IOCTL_KGSL_MAP_USER_MEM,
kgsl_ioctl_map_user_mem),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FROM_PMEM,
kgsl_ioctl_map_user_mem),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FREE,
kgsl_ioctl_sharedmem_free),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SHAREDMEM_FLUSH_CACHE,
kgsl_ioctl_sharedmem_flush_cache),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_ALLOC,
kgsl_ioctl_gpumem_alloc),
KGSL_IOCTL_FUNC(IOCTL_KGSL_TIMESTAMP_EVENT,
kgsl_ioctl_timestamp_event),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SETPROPERTY,
kgsl_ioctl_device_setproperty),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_ALLOC_ID,
kgsl_ioctl_gpumem_alloc_id),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_FREE_ID,
kgsl_ioctl_gpumem_free_id),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_GET_INFO,
kgsl_ioctl_gpumem_get_info),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_SYNC_CACHE,
kgsl_ioctl_gpumem_sync_cache),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUMEM_SYNC_CACHE_BULK,
kgsl_ioctl_gpumem_sync_cache_bulk),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_CREATE,
kgsl_ioctl_syncsource_create),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_DESTROY,
kgsl_ioctl_syncsource_destroy),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_CREATE_FENCE,
kgsl_ioctl_syncsource_create_fence),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SYNCSOURCE_SIGNAL_FENCE,
kgsl_ioctl_syncsource_signal_fence),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_ALLOC,
kgsl_ioctl_gpuobj_alloc),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_FREE,
kgsl_ioctl_gpuobj_free),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_INFO,
kgsl_ioctl_gpuobj_info),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_IMPORT,
kgsl_ioctl_gpuobj_import),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_SYNC,
kgsl_ioctl_gpuobj_sync),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_COMMAND,
kgsl_ioctl_gpu_command),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_SET_INFO,
kgsl_ioctl_gpuobj_set_info),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_ALLOC,
kgsl_ioctl_sparse_phys_alloc),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_FREE,
kgsl_ioctl_sparse_phys_free),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_ALLOC,
kgsl_ioctl_sparse_virt_alloc),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_FREE,
kgsl_ioctl_sparse_virt_free),
KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_BIND,
kgsl_ioctl_sparse_bind),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPU_SPARSE_COMMAND,
kgsl_ioctl_gpu_sparse_command),
};
long kgsl_ioctl_copy_in(unsigned int kernel_cmd, unsigned int user_cmd,
unsigned long arg, unsigned char *ptr)
{
unsigned int usize = _IOC_SIZE(user_cmd);
unsigned int ksize = _IOC_SIZE(kernel_cmd);
unsigned int copy = ksize < usize ? ksize : usize;
if ((kernel_cmd & IOC_IN) && (user_cmd & IOC_IN)) {
if (copy > 0 && copy_from_user(ptr, (void __user *) arg, copy))
return -EFAULT;
}
return 0;
}
long kgsl_ioctl_copy_out(unsigned int kernel_cmd, unsigned int user_cmd,
unsigned long arg, unsigned char *ptr)
{
unsigned int usize = _IOC_SIZE(user_cmd);
unsigned int ksize = _IOC_SIZE(kernel_cmd);
unsigned int copy = ksize < usize ? ksize : usize;
if ((kernel_cmd & IOC_OUT) && (user_cmd & IOC_OUT)) {
if (copy > 0 && copy_to_user((void __user *) arg, ptr, copy))
return -EFAULT;
}
return 0;
}
long kgsl_ioctl_helper(struct file *filep, unsigned int cmd, unsigned long arg,
const struct kgsl_ioctl *cmds, int len)
{
struct kgsl_device_private *dev_priv = filep->private_data;
unsigned char data[128] = { 0 };
unsigned int nr = _IOC_NR(cmd);
long ret;
if (nr >= len || cmds[nr].func == NULL)
return -ENOIOCTLCMD;
if (_IOC_SIZE(cmds[nr].cmd) > sizeof(data)) {
dev_err_ratelimited(dev_priv->device->dev,
"data too big for ioctl 0x%08x: %d/%zu\n",
cmd, _IOC_SIZE(cmds[nr].cmd), sizeof(data));
return -EINVAL;
}
if (_IOC_SIZE(cmds[nr].cmd)) {
ret = kgsl_ioctl_copy_in(cmds[nr].cmd, cmd, arg, data);
if (ret)
return ret;
}
ret = cmds[nr].func(dev_priv, cmd, data);
if (ret == 0 && _IOC_SIZE(cmds[nr].cmd))
ret = kgsl_ioctl_copy_out(cmds[nr].cmd, cmd, arg, data);
return ret;
}
long kgsl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
{
struct kgsl_device_private *dev_priv = filep->private_data;
struct kgsl_device *device = dev_priv->device;
long ret;
ret = kgsl_ioctl_helper(filep, cmd, arg, kgsl_ioctl_funcs,
ARRAY_SIZE(kgsl_ioctl_funcs));
/*
* If the command was unrecognized in the generic core, try the device
* specific function
*/
if (ret == -ENOIOCTLCMD) {
if (is_compat_task() && device->ftbl->compat_ioctl != NULL)
return device->ftbl->compat_ioctl(dev_priv, cmd, arg);
else if (device->ftbl->ioctl != NULL)
return device->ftbl->ioctl(dev_priv, cmd, arg);
dev_err(device->dev, "invalid ioctl code 0x%08X\n", cmd);
}
return ret;
}

2676
drivers/gpu/msm/kgsl_iommu.c Normal file
View file

File diff suppressed because it is too large Load diff

193
drivers/gpu/msm/kgsl_iommu.h Normal file
View file

@ -0,0 +1,193 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_IOMMU_H
#define __KGSL_IOMMU_H
#include "kgsl_mmu.h"
/*
* These defines control the address range for allocations that
* are mapped into all pagetables.
*/
#define KGSL_IOMMU_GLOBAL_MEM_SIZE (20 * SZ_1M)
#define KGSL_IOMMU_GLOBAL_MEM_BASE32 0xf8000000
#define KGSL_IOMMU_GLOBAL_MEM_BASE64 0xfc000000
#define KGSL_IOMMU_GLOBAL_MEM_BASE(__mmu) \
(MMU_FEATURE(__mmu, KGSL_MMU_64BIT) ? \
KGSL_IOMMU_GLOBAL_MEM_BASE64 : KGSL_IOMMU_GLOBAL_MEM_BASE32)
#define KGSL_IOMMU_SECURE_SIZE SZ_256M
#define KGSL_IOMMU_SECURE_END(_mmu) KGSL_IOMMU_GLOBAL_MEM_BASE(_mmu)
#define KGSL_IOMMU_SECURE_BASE(_mmu) \
(KGSL_IOMMU_GLOBAL_MEM_BASE(_mmu) - KGSL_IOMMU_SECURE_SIZE)
#define KGSL_IOMMU_SVM_BASE32 0x300000
#define KGSL_IOMMU_SVM_END32 (0xC0000000 - SZ_16M)
#define KGSL_IOMMU_VA_BASE64 0x500000000ULL
#define KGSL_IOMMU_VA_END64 0x600000000ULL
/*
* Note: currently we only support 36 bit addresses,
* but the CPU supports 39. Eventually this range
* should change to high part of the 39 bit address
* space just like the CPU.
*/
#define KGSL_IOMMU_SVM_BASE64 0x700000000ULL
#define KGSL_IOMMU_SVM_END64 0x800000000ULL
#define CP_APERTURE_REG 0
#define CP_SMMU_APERTURE_ID 0x1B
/* TLBSTATUS register fields */
#define KGSL_IOMMU_CTX_TLBSTATUS_SACTIVE BIT(0)
/* SCTLR fields */
#define KGSL_IOMMU_SCTLR_HUPCF_SHIFT 8
#define KGSL_IOMMU_SCTLR_CFCFG_SHIFT 7
#define KGSL_IOMMU_SCTLR_CFIE_SHIFT 6
/* FSR fields */
#define KGSL_IOMMU_FSR_SS_SHIFT 30
enum kgsl_iommu_reg_map {
KGSL_IOMMU_CTX_SCTLR = 0,
KGSL_IOMMU_CTX_TTBR0,
KGSL_IOMMU_CTX_CONTEXTIDR,
KGSL_IOMMU_CTX_FSR,
KGSL_IOMMU_CTX_FAR,
KGSL_IOMMU_CTX_TLBIALL,
KGSL_IOMMU_CTX_RESUME,
KGSL_IOMMU_CTX_FSYNR0,
KGSL_IOMMU_CTX_FSYNR1,
KGSL_IOMMU_CTX_TLBSYNC,
KGSL_IOMMU_CTX_TLBSTATUS,
KGSL_IOMMU_REG_MAX
};
/* Max number of iommu clks per IOMMU unit */
#define KGSL_IOMMU_MAX_CLKS 5
enum kgsl_iommu_context_id {
KGSL_IOMMU_CONTEXT_USER = 0,
KGSL_IOMMU_CONTEXT_SECURE = 1,
KGSL_IOMMU_CONTEXT_MAX,
};
/* offset at which a nop command is placed in setstate */
#define KGSL_IOMMU_SETSTATE_NOP_OFFSET 1024
/*
* struct kgsl_iommu_context - Structure holding data about an iommu context
* bank
* @dev: pointer to the iommu context's device
* @name: context name
* @id: The id of the context, used for deciding how it is used.
* @cb_num: The hardware context bank number, used for calculating register
* offsets.
* @kgsldev: The kgsl device that uses this context.
* @stalled_on_fault: Flag when set indicates that this iommu device is stalled
* on a page fault
* @default_pt: The default pagetable for this context,
* it may be changed by self programming.
*/
struct kgsl_iommu_context {
struct device *dev;
const char *name;
enum kgsl_iommu_context_id id;
unsigned int cb_num;
struct kgsl_device *kgsldev;
bool stalled_on_fault;
void __iomem *regbase;
struct kgsl_pagetable *default_pt;
};
/*
* struct kgsl_iommu - Structure holding iommu data for kgsl driver
* @ctx: Array of kgsl_iommu_context structs
* @regbase: Virtual address of the IOMMU register base
* @regstart: Physical address of the iommu registers
* @regsize: Length of the iommu register region.
* @setstate: Scratch GPU memory for IOMMU operations
* @clk_enable_count: The ref count of clock enable calls
* @clks: Array of pointers to IOMMU clocks
* @smmu_info: smmu info used in a5xx preemption
*/
struct kgsl_iommu {
struct kgsl_iommu_context ctx[KGSL_IOMMU_CONTEXT_MAX];
void __iomem *regbase;
unsigned long regstart;
unsigned int regsize;
struct kgsl_memdesc setstate;
atomic_t clk_enable_count;
struct clk *clks[KGSL_IOMMU_MAX_CLKS];
struct kgsl_memdesc smmu_info;
};
/*
* struct kgsl_iommu_pt - Iommu pagetable structure private to kgsl driver
* @domain: Pointer to the iommu domain that contains the iommu pagetable
* @ttbr0: register value to set when using this pagetable
* @contextidr: register value to set when using this pagetable
* @attached: is the pagetable attached?
* @rbtree: all buffers mapped into the pagetable, indexed by gpuaddr
* @va_start: Start of virtual range used in this pagetable.
* @va_end: End of virtual range.
* @svm_start: Start of shared virtual memory range. Addresses in this
* range are also valid in the process's CPU address space.
* @svm_end: End of the shared virtual memory range.
* @svm_start: 32 bit compatible range, for old clients who lack bits
* @svm_end: end of 32 bit compatible range
*/
struct kgsl_iommu_pt {
struct iommu_domain *domain;
u64 ttbr0;
u32 contextidr;
bool attached;
struct rb_root rbtree;
uint64_t va_start;
uint64_t va_end;
uint64_t svm_start;
uint64_t svm_end;
uint64_t compat_va_start;
uint64_t compat_va_end;
};
/*
* offset of context bank 0 from the start of the SMMU register space.
*/
#define KGSL_IOMMU_CB0_OFFSET 0x8000
/* size of each context bank's register space */
#define KGSL_IOMMU_CB_SHIFT 12
/* Macros to read/write IOMMU registers */
extern const unsigned int kgsl_iommu_reg_list[KGSL_IOMMU_REG_MAX];
/*
* Don't use this function directly. Use the macros below to read/write
* IOMMU registers.
*/
static inline void __iomem *
kgsl_iommu_reg(struct kgsl_iommu_context *ctx, enum kgsl_iommu_reg_map reg)
{
return ctx->regbase + kgsl_iommu_reg_list[reg];
}
#define KGSL_IOMMU_SET_CTX_REG_Q(_ctx, REG, val) \
writeq_relaxed((val), \
kgsl_iommu_reg((_ctx), KGSL_IOMMU_CTX_##REG))
#define KGSL_IOMMU_GET_CTX_REG_Q(_ctx, REG) \
readq_relaxed(kgsl_iommu_reg((_ctx), KGSL_IOMMU_CTX_##REG))
#define KGSL_IOMMU_SET_CTX_REG(_ctx, REG, val) \
writel_relaxed((val), \
kgsl_iommu_reg((_ctx), KGSL_IOMMU_CTX_##REG))
#define KGSL_IOMMU_GET_CTX_REG(_ctx, REG) \
readl_relaxed(kgsl_iommu_reg((_ctx), KGSL_IOMMU_CTX_##REG))
#endif

718
drivers/gpu/msm/kgsl_mmu.c Normal file
View file

@ -0,0 +1,718 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/slab.h>
#include "kgsl_device.h"
#include "kgsl_mmu.h"
#include "kgsl_sharedmem.h"
static void pagetable_remove_sysfs_objects(struct kgsl_pagetable *pagetable);
static void _deferred_destroy(struct work_struct *ws)
{
struct kgsl_pagetable *pagetable = container_of(ws,
struct kgsl_pagetable, destroy_ws);
if (PT_OP_VALID(pagetable, mmu_destroy_pagetable))
pagetable->pt_ops->mmu_destroy_pagetable(pagetable);
kfree(pagetable);
}
static void kgsl_destroy_pagetable(struct kref *kref)
{
struct kgsl_pagetable *pagetable = container_of(kref,
struct kgsl_pagetable, refcount);
kgsl_mmu_detach_pagetable(pagetable);
kgsl_schedule_work(&pagetable->destroy_ws);
}
static inline void kgsl_put_pagetable(struct kgsl_pagetable *pagetable)
{
if (pagetable)
kref_put(&pagetable->refcount, kgsl_destroy_pagetable);
}
struct kgsl_pagetable *
kgsl_get_pagetable(unsigned long name)
{
struct kgsl_pagetable *pt, *ret = NULL;
unsigned long flags;
spin_lock_irqsave(&kgsl_driver.ptlock, flags);
list_for_each_entry(pt, &kgsl_driver.pagetable_list, list) {
if (name == pt->name && kref_get_unless_zero(&pt->refcount)) {
ret = pt;
break;
}
}
spin_unlock_irqrestore(&kgsl_driver.ptlock, flags);
return ret;
}
static struct kgsl_pagetable *
_get_pt_from_kobj(struct kobject *kobj)
{
unsigned int ptname;
if (!kobj)
return NULL;
if (kstrtou32(kobj->name, 0, &ptname))
return NULL;
return kgsl_get_pagetable(ptname);
}
static ssize_t
sysfs_show_entries(struct kobject *kobj,
struct kobj_attribute *attr,
char *buf)
{
struct kgsl_pagetable *pt;
int ret = 0;
pt = _get_pt_from_kobj(kobj);
if (pt) {
unsigned int val = atomic_read(&pt->stats.entries);
ret += scnprintf(buf, PAGE_SIZE, "%d\n", val);
}
kgsl_put_pagetable(pt);
return ret;
}
static ssize_t
sysfs_show_mapped(struct kobject *kobj,
struct kobj_attribute *attr,
char *buf)
{
struct kgsl_pagetable *pt;
int ret = 0;
pt = _get_pt_from_kobj(kobj);
if (pt) {
uint64_t val = atomic_long_read(&pt->stats.mapped);
ret += scnprintf(buf, PAGE_SIZE, "%llu\n", val);
}
kgsl_put_pagetable(pt);
return ret;
}
static ssize_t
sysfs_show_max_mapped(struct kobject *kobj,
struct kobj_attribute *attr,
char *buf)
{
struct kgsl_pagetable *pt;
int ret = 0;
pt = _get_pt_from_kobj(kobj);
if (pt) {
uint64_t val = atomic_long_read(&pt->stats.max_mapped);
ret += scnprintf(buf, PAGE_SIZE, "%llu\n", val);
}
kgsl_put_pagetable(pt);
return ret;
}
static struct kobj_attribute attr_entries = {
.attr = { .name = "entries", .mode = 0444 },
.show = sysfs_show_entries,
.store = NULL,
};
static struct kobj_attribute attr_mapped = {
.attr = { .name = "mapped", .mode = 0444 },
.show = sysfs_show_mapped,
.store = NULL,
};
static struct kobj_attribute attr_max_mapped = {
.attr = { .name = "max_mapped", .mode = 0444 },
.show = sysfs_show_max_mapped,
.store = NULL,
};
static struct attribute *pagetable_attrs[] = {
&attr_entries.attr,
&attr_mapped.attr,
&attr_max_mapped.attr,
NULL,
};
static struct attribute_group pagetable_attr_group = {
.attrs = pagetable_attrs,
};
static void
pagetable_remove_sysfs_objects(struct kgsl_pagetable *pagetable)
{
if (pagetable->kobj)
sysfs_remove_group(pagetable->kobj,
&pagetable_attr_group);
kobject_put(pagetable->kobj);
pagetable->kobj = NULL;
}
static int
pagetable_add_sysfs_objects(struct kgsl_pagetable *pagetable)
{
char ptname[16];
int ret = -ENOMEM;
snprintf(ptname, sizeof(ptname), "%d", pagetable->name);
pagetable->kobj = kobject_create_and_add(ptname,
kgsl_driver.ptkobj);
if (pagetable->kobj == NULL)
goto err;
ret = sysfs_create_group(pagetable->kobj, &pagetable_attr_group);
err:
if (ret) {
if (pagetable->kobj)
kobject_put(pagetable->kobj);
pagetable->kobj = NULL;
}
return ret;
}
void
kgsl_mmu_detach_pagetable(struct kgsl_pagetable *pagetable)
{
unsigned long flags;
spin_lock_irqsave(&kgsl_driver.ptlock, flags);
if (!list_empty(&pagetable->list))
list_del_init(&pagetable->list);
spin_unlock_irqrestore(&kgsl_driver.ptlock, flags);
pagetable_remove_sysfs_objects(pagetable);
}
struct kgsl_pagetable *kgsl_mmu_get_pt_from_ptname(struct kgsl_mmu *mmu,
int ptname)
{
struct kgsl_pagetable *pt;
spin_lock(&kgsl_driver.ptlock);
list_for_each_entry(pt, &kgsl_driver.pagetable_list, list) {
if (pt->name == ptname) {
spin_unlock(&kgsl_driver.ptlock);
return pt;
}
}
spin_unlock(&kgsl_driver.ptlock);
return NULL;
}
unsigned int
kgsl_mmu_log_fault_addr(struct kgsl_mmu *mmu, u64 pt_base,
uint64_t addr)
{
struct kgsl_pagetable *pt;
unsigned int ret = 0;
if (!MMU_OP_VALID(mmu, mmu_pt_equal))
return 0;
spin_lock(&kgsl_driver.ptlock);
list_for_each_entry(pt, &kgsl_driver.pagetable_list, list) {
if (mmu->mmu_ops->mmu_pt_equal(mmu, pt, pt_base)) {
if ((addr & ~(PAGE_SIZE-1)) == pt->fault_addr) {
ret = 1;
break;
}
pt->fault_addr = (addr & ~(PAGE_SIZE-1));
ret = 0;
break;
}
}
spin_unlock(&kgsl_driver.ptlock);
return ret;
}
int kgsl_mmu_start(struct kgsl_device *device)
{
struct kgsl_mmu *mmu = &device->mmu;
if (MMU_OP_VALID(mmu, mmu_start))
return mmu->mmu_ops->mmu_start(mmu);
return 0;
}
struct kgsl_pagetable *
kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu, unsigned int name)
{
int status = 0;
struct kgsl_pagetable *pagetable = NULL;
unsigned long flags;
pagetable = kzalloc(sizeof(struct kgsl_pagetable), GFP_KERNEL);
if (pagetable == NULL)
return ERR_PTR(-ENOMEM);
kref_init(&pagetable->refcount);
spin_lock_init(&pagetable->lock);
INIT_WORK(&pagetable->destroy_ws, _deferred_destroy);
pagetable->mmu = mmu;
pagetable->name = name;
atomic_set(&pagetable->stats.entries, 0);
atomic_long_set(&pagetable->stats.mapped, 0);
atomic_long_set(&pagetable->stats.max_mapped, 0);
if (MMU_OP_VALID(mmu, mmu_init_pt)) {
status = mmu->mmu_ops->mmu_init_pt(mmu, pagetable);
if (status) {
kfree(pagetable);
return ERR_PTR(status);
}
}
spin_lock_irqsave(&kgsl_driver.ptlock, flags);
list_add(&pagetable->list, &kgsl_driver.pagetable_list);
spin_unlock_irqrestore(&kgsl_driver.ptlock, flags);
/* Create the sysfs entries */
pagetable_add_sysfs_objects(pagetable);
return pagetable;
}
void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable)
{
kgsl_put_pagetable(pagetable);
}
/**
* kgsl_mmu_find_svm_region() - Find a empty spot in the SVM region
* @pagetable: KGSL pagetable to search
* @start: start of search range, must be within kgsl_mmu_svm_range()
* @end: end of search range, must be within kgsl_mmu_svm_range()
* @size: Size of the region to find
* @align: Desired alignment of the address
*/
uint64_t kgsl_mmu_find_svm_region(struct kgsl_pagetable *pagetable,
uint64_t start, uint64_t end, uint64_t size,
uint64_t align)
{
if (PT_OP_VALID(pagetable, find_svm_region))
return pagetable->pt_ops->find_svm_region(pagetable, start,
end, size, align);
return -ENOMEM;
}
/**
* kgsl_mmu_set_svm_region() - Check if a region is empty and reserve it if so
* @pagetable: KGSL pagetable to search
* @gpuaddr: GPU address to check/reserve
* @size: Size of the region to check/reserve
*/
int kgsl_mmu_set_svm_region(struct kgsl_pagetable *pagetable, uint64_t gpuaddr,
uint64_t size)
{
if (PT_OP_VALID(pagetable, set_svm_region))
return pagetable->pt_ops->set_svm_region(pagetable, gpuaddr,
size);
return -ENOMEM;
}
/**
* kgsl_mmu_get_gpuaddr() - Assign a GPU address to the memdesc
* @pagetable: GPU pagetable to assign the address in
* @memdesc: mem descriptor to assign the memory to
*/
int
kgsl_mmu_get_gpuaddr(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc)
{
if (PT_OP_VALID(pagetable, get_gpuaddr))
return pagetable->pt_ops->get_gpuaddr(pagetable, memdesc);
return -ENOMEM;
}
int
kgsl_mmu_map(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc)
{
int size;
if (!memdesc->gpuaddr)
return -EINVAL;
if (!(memdesc->flags & (KGSL_MEMFLAGS_SPARSE_VIRT |
KGSL_MEMFLAGS_SPARSE_PHYS))) {
/* Only global mappings should be mapped multiple times */
if (!kgsl_memdesc_is_global(memdesc) &&
(KGSL_MEMDESC_MAPPED & memdesc->priv))
return -EINVAL;
}
size = kgsl_memdesc_footprint(memdesc);
if (PT_OP_VALID(pagetable, mmu_map)) {
int ret;
ret = pagetable->pt_ops->mmu_map(pagetable, memdesc);
if (ret)
return ret;
atomic_inc(&pagetable->stats.entries);
KGSL_STATS_ADD(size, &pagetable->stats.mapped,
&pagetable->stats.max_mapped);
/* This is needed for non-sparse mappings */
memdesc->priv |= KGSL_MEMDESC_MAPPED;
}
return 0;
}
/**
* kgsl_mmu_put_gpuaddr() - Remove a GPU address from a pagetable
* @pagetable: Pagetable to release the memory from
* @memdesc: Memory descriptor containing the GPU address to free
*/
void kgsl_mmu_put_gpuaddr(struct kgsl_memdesc *memdesc)
{
struct kgsl_pagetable *pagetable = memdesc->pagetable;
int unmap_fail = 0;
if (memdesc->size == 0 || memdesc->gpuaddr == 0)
return;
if (!kgsl_memdesc_is_global(memdesc))
unmap_fail = kgsl_mmu_unmap(pagetable, memdesc);
/*
* Do not free the gpuaddr/size if unmap fails. Because if we
* try to map this range in future, the iommu driver will throw
* a BUG_ON() because it feels we are overwriting a mapping.
*/
if (PT_OP_VALID(pagetable, put_gpuaddr) && (unmap_fail == 0))
pagetable->pt_ops->put_gpuaddr(memdesc);
if (!kgsl_memdesc_is_global(memdesc))
memdesc->gpuaddr = 0;
memdesc->pagetable = NULL;
}
/**
* kgsl_mmu_svm_range() - Return the range for SVM (if applicable)
* @pagetable: Pagetable to query the range from
* @lo: Pointer to store the start of the SVM range
* @hi: Pointer to store the end of the SVM range
* @memflags: Flags from the buffer we are mapping
*/
int kgsl_mmu_svm_range(struct kgsl_pagetable *pagetable,
uint64_t *lo, uint64_t *hi, uint64_t memflags)
{
if (PT_OP_VALID(pagetable, svm_range))
return pagetable->pt_ops->svm_range(pagetable, lo, hi,
memflags);
return -ENODEV;
}
int
kgsl_mmu_unmap(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc)
{
int ret = 0;
if (memdesc->size == 0)
return -EINVAL;
if (!(memdesc->flags & (KGSL_MEMFLAGS_SPARSE_VIRT |
KGSL_MEMFLAGS_SPARSE_PHYS))) {
/* Only global mappings should be mapped multiple times */
if (!(KGSL_MEMDESC_MAPPED & memdesc->priv))
return -EINVAL;
}
if (PT_OP_VALID(pagetable, mmu_unmap)) {
uint64_t size;
size = kgsl_memdesc_footprint(memdesc);
ret = pagetable->pt_ops->mmu_unmap(pagetable, memdesc);
atomic_dec(&pagetable->stats.entries);
atomic_long_sub(size, &pagetable->stats.mapped);
if (!kgsl_memdesc_is_global(memdesc))
memdesc->priv &= ~KGSL_MEMDESC_MAPPED;
}
return ret;
}
int kgsl_mmu_map_offset(struct kgsl_pagetable *pagetable,
uint64_t virtaddr, uint64_t virtoffset,
struct kgsl_memdesc *memdesc, uint64_t physoffset,
uint64_t size, uint64_t flags)
{
if (PT_OP_VALID(pagetable, mmu_map_offset)) {
int ret;
ret = pagetable->pt_ops->mmu_map_offset(pagetable, virtaddr,
virtoffset, memdesc, physoffset, size, flags);
if (ret)
return ret;
atomic_inc(&pagetable->stats.entries);
KGSL_STATS_ADD(size, &pagetable->stats.mapped,
&pagetable->stats.max_mapped);
}
return 0;
}
int kgsl_mmu_unmap_offset(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc, uint64_t addr, uint64_t offset,
uint64_t size)
{
if (PT_OP_VALID(pagetable, mmu_unmap_offset)) {
int ret;
ret = pagetable->pt_ops->mmu_unmap_offset(pagetable, memdesc,
addr, offset, size);
if (ret)
return ret;
atomic_dec(&pagetable->stats.entries);
atomic_long_sub(size, &pagetable->stats.mapped);
}
return 0;
}
int kgsl_mmu_sparse_dummy_map(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc, uint64_t offset, uint64_t size)
{
if (PT_OP_VALID(pagetable, mmu_sparse_dummy_map)) {
int ret;
ret = pagetable->pt_ops->mmu_sparse_dummy_map(pagetable,
memdesc, offset, size);
if (ret)
return ret;
atomic_dec(&pagetable->stats.entries);
atomic_long_sub(size, &pagetable->stats.mapped);
}
return 0;
}
void kgsl_mmu_remove_global(struct kgsl_device *device,
struct kgsl_memdesc *memdesc)
{
struct kgsl_mmu *mmu = &device->mmu;
if (MMU_OP_VALID(mmu, mmu_remove_global))
mmu->mmu_ops->mmu_remove_global(mmu, memdesc);
}
void kgsl_mmu_add_global(struct kgsl_device *device,
struct kgsl_memdesc *memdesc, const char *name)
{
struct kgsl_mmu *mmu = &device->mmu;
if (MMU_OP_VALID(mmu, mmu_add_global))
mmu->mmu_ops->mmu_add_global(mmu, memdesc, name);
}
void kgsl_mmu_close(struct kgsl_device *device)
{
struct kgsl_mmu *mmu = &(device->mmu);
if (MMU_OP_VALID(mmu, mmu_close))
mmu->mmu_ops->mmu_close(mmu);
}
enum kgsl_mmutype kgsl_mmu_get_mmutype(struct kgsl_device *device)
{
return device ? device->mmu.type : KGSL_MMU_TYPE_NONE;
}
bool kgsl_mmu_gpuaddr_in_range(struct kgsl_pagetable *pagetable,
uint64_t gpuaddr)
{
if (PT_OP_VALID(pagetable, addr_in_range))
return pagetable->pt_ops->addr_in_range(pagetable, gpuaddr);
return false;
}
struct kgsl_memdesc *kgsl_mmu_get_qdss_global_entry(struct kgsl_device *device)
{
struct kgsl_mmu *mmu = &device->mmu;
if (MMU_OP_VALID(mmu, mmu_get_qdss_global_entry))
return mmu->mmu_ops->mmu_get_qdss_global_entry();
return NULL;
}
struct kgsl_memdesc *kgsl_mmu_get_qtimer_global_entry(
struct kgsl_device *device)
{
struct kgsl_mmu *mmu = &device->mmu;
if (MMU_OP_VALID(mmu, mmu_get_qtimer_global_entry))
return mmu->mmu_ops->mmu_get_qtimer_global_entry();
return NULL;
}
/*
* NOMMU definitions - NOMMU really just means that the MMU is kept in pass
* through and the GPU directly accesses physical memory. Used in debug mode
* and when a real MMU isn't up and running yet.
*/
static bool nommu_gpuaddr_in_range(struct kgsl_pagetable *pagetable,
uint64_t gpuaddr)
{
return (gpuaddr != 0) ? true : false;
}
static int nommu_get_gpuaddr(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc)
{
if (WARN_ONCE(memdesc->sgt->nents > 1,
"Attempt to map non-contiguous memory with NOMMU\n"))
return -EINVAL;
memdesc->gpuaddr = (uint64_t) sg_phys(memdesc->sgt->sgl);
if (memdesc->gpuaddr) {
memdesc->pagetable = pagetable;
return 0;
}
return -ENOMEM;
}
static struct kgsl_mmu_pt_ops nommu_pt_ops = {
.get_gpuaddr = nommu_get_gpuaddr,
.addr_in_range = nommu_gpuaddr_in_range,
};
static void nommu_add_global(struct kgsl_mmu *mmu,
struct kgsl_memdesc *memdesc, const char *name)
{
memdesc->gpuaddr = (uint64_t) sg_phys(memdesc->sgt->sgl);
}
static void nommu_remove_global(struct kgsl_mmu *mmu,
struct kgsl_memdesc *memdesc)
{
memdesc->gpuaddr = 0;
}
static int nommu_init_pt(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt)
{
if (pt == NULL)
return -EINVAL;
pt->pt_ops = &nommu_pt_ops;
return 0;
}
static struct kgsl_pagetable *nommu_getpagetable(struct kgsl_mmu *mmu,
unsigned long name)
{
struct kgsl_pagetable *pagetable;
pagetable = kgsl_get_pagetable(KGSL_MMU_GLOBAL_PT);
if (pagetable == NULL)
pagetable = kgsl_mmu_createpagetableobject(mmu,
KGSL_MMU_GLOBAL_PT);
return pagetable;
}
static int nommu_init(struct kgsl_mmu *mmu)
{
mmu->features |= KGSL_MMU_GLOBAL_PAGETABLE;
return 0;
}
static int nommu_probe(struct kgsl_device *device)
{
/* NOMMU always exists */
return 0;
}
static struct kgsl_mmu_ops kgsl_nommu_ops = {
.mmu_init = nommu_init,
.mmu_add_global = nommu_add_global,
.mmu_remove_global = nommu_remove_global,
.mmu_init_pt = nommu_init_pt,
.mmu_getpagetable = nommu_getpagetable,
.probe = nommu_probe,
};
static struct {
const char *name;
unsigned int type;
struct kgsl_mmu_ops *ops;
} kgsl_mmu_subtypes[] = {
#if IS_ENABLED(CONFIG_ARM_SMMU)
{ "iommu", KGSL_MMU_TYPE_IOMMU, &kgsl_iommu_ops },
#endif
{ "nommu", KGSL_MMU_TYPE_NONE, &kgsl_nommu_ops },
};
int kgsl_mmu_probe(struct kgsl_device *device)
{
struct kgsl_mmu *mmu = &device->mmu;
int ret, i;
for (i = 0; i < ARRAY_SIZE(kgsl_mmu_subtypes); i++) {
ret = kgsl_mmu_subtypes[i].ops->probe(device);
if (ret == 0) {
mmu->type = kgsl_mmu_subtypes[i].type;
mmu->mmu_ops = kgsl_mmu_subtypes[i].ops;
if (MMU_OP_VALID(mmu, mmu_init))
return mmu->mmu_ops->mmu_init(mmu);
return 0;
}
}
dev_err(device->dev, "mmu: couldn't detect any known MMU types\n");
return -ENODEV;
}

354
drivers/gpu/msm/kgsl_mmu.h Normal file
View file

@ -0,0 +1,354 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_MMU_H
#define __KGSL_MMU_H
#include <linux/platform_device.h>
#include "kgsl_iommu.h"
/* Identifier for the global page table */
/*
* Per process page tables will probably pass in the thread group
* as an identifier
*/
#define KGSL_MMU_GLOBAL_PT 0
#define KGSL_MMU_SECURE_PT 1
#define MMU_DEFAULT_TTBR0(_d) \
(kgsl_mmu_pagetable_get_ttbr0((_d)->mmu.defaultpagetable))
#define MMU_DEFAULT_CONTEXTIDR(_d) \
(kgsl_mmu_pagetable_get_contextidr((_d)->mmu.defaultpagetable))
struct kgsl_device;
enum kgsl_mmutype {
KGSL_MMU_TYPE_IOMMU = 0,
KGSL_MMU_TYPE_NONE
};
#define KGSL_IOMMU_SMMU_V500 1
struct kgsl_pagetable {
spinlock_t lock;
struct kref refcount;
struct list_head list;
unsigned int name;
struct kobject *kobj;
struct work_struct destroy_ws;
struct {
atomic_t entries;
atomic_long_t mapped;
atomic_long_t max_mapped;
} stats;
const struct kgsl_mmu_pt_ops *pt_ops;
uint64_t fault_addr;
void *priv;
struct kgsl_mmu *mmu;
};
struct kgsl_mmu;
struct kgsl_mmu_ops {
int (*probe)(struct kgsl_device *device);
int (*mmu_init)(struct kgsl_mmu *mmu);
void (*mmu_close)(struct kgsl_mmu *mmu);
int (*mmu_start)(struct kgsl_mmu *mmu);
int (*mmu_set_pt)(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt);
uint64_t (*mmu_get_current_ttbr0)(struct kgsl_mmu *mmu);
void (*mmu_pagefault_resume)(struct kgsl_mmu *mmu);
void (*mmu_clear_fsr)(struct kgsl_mmu *mmu);
void (*mmu_enable_clk)(struct kgsl_mmu *mmu);
void (*mmu_disable_clk)(struct kgsl_mmu *mmu);
bool (*mmu_pt_equal)(struct kgsl_mmu *mmu,
struct kgsl_pagetable *pt, u64 ttbr0);
int (*mmu_set_pf_policy)(struct kgsl_mmu *mmu, unsigned long pf_policy);
int (*mmu_init_pt)(struct kgsl_mmu *mmu, struct kgsl_pagetable *pt);
void (*mmu_add_global)(struct kgsl_mmu *mmu,
struct kgsl_memdesc *memdesc, const char *name);
void (*mmu_remove_global)(struct kgsl_mmu *mmu,
struct kgsl_memdesc *memdesc);
struct kgsl_pagetable * (*mmu_getpagetable)(struct kgsl_mmu *mmu,
unsigned long name);
struct kgsl_memdesc* (*mmu_get_qdss_global_entry)(void);
struct kgsl_memdesc* (*mmu_get_qtimer_global_entry)(void);
};
struct kgsl_mmu_pt_ops {
int (*mmu_map)(struct kgsl_pagetable *pt,
struct kgsl_memdesc *memdesc);
int (*mmu_unmap)(struct kgsl_pagetable *pt,
struct kgsl_memdesc *memdesc);
void (*mmu_destroy_pagetable)(struct kgsl_pagetable *pt);
u64 (*get_ttbr0)(struct kgsl_pagetable *pt);
u32 (*get_contextidr)(struct kgsl_pagetable *pt);
int (*get_gpuaddr)(struct kgsl_pagetable *pt,
struct kgsl_memdesc *memdesc);
void (*put_gpuaddr)(struct kgsl_memdesc *memdesc);
uint64_t (*find_svm_region)(struct kgsl_pagetable *pt, uint64_t start,
uint64_t end, uint64_t size, uint64_t align);
int (*set_svm_region)(struct kgsl_pagetable *pt,
uint64_t gpuaddr, uint64_t size);
int (*svm_range)(struct kgsl_pagetable *pt, uint64_t *lo, uint64_t *hi,
uint64_t memflags);
bool (*addr_in_range)(struct kgsl_pagetable *pagetable,
uint64_t gpuaddr);
int (*mmu_map_offset)(struct kgsl_pagetable *pt,
uint64_t virtaddr, uint64_t virtoffset,
struct kgsl_memdesc *memdesc, uint64_t physoffset,
uint64_t size, uint64_t flags);
int (*mmu_unmap_offset)(struct kgsl_pagetable *pt,
struct kgsl_memdesc *memdesc, uint64_t addr,
uint64_t offset, uint64_t size);
int (*mmu_sparse_dummy_map)(struct kgsl_pagetable *pt,
struct kgsl_memdesc *memdesc, uint64_t offset,
uint64_t size);
};
/*
* MMU_FEATURE - return true if the specified feature is supported by the GPU
* MMU
*/
#define MMU_FEATURE(_mmu, _bit) \
((_mmu)->features & (_bit))
/* MMU requires the TLB to be flushed on map */
#define KGSL_MMU_FLUSH_TLB_ON_MAP BIT(2)
/* MMU uses global pagetable */
#define KGSL_MMU_GLOBAL_PAGETABLE BIT(3)
/* Force 32 bit, even if the MMU can do 64 bit */
#define KGSL_MMU_FORCE_32BIT BIT(4)
/* 64 bit address is live */
#define KGSL_MMU_64BIT BIT(5)
/* The MMU supports non-contigious pages */
#define KGSL_MMU_PAGED BIT(6)
/* The device requires a guard page */
#define KGSL_MMU_NEED_GUARD_PAGE BIT(7)
/* The device supports IO coherency */
#define KGSL_MMU_IO_COHERENT BIT(8)
/**
* struct kgsl_mmu - Master definition for KGSL MMU devices
* @flags: MMU device flags
* @type: Type of MMU that is attached
* @subtype: Sub Type of MMU that is attached
* @defaultpagetable: Default pagetable object for the MMU
* @securepagetable: Default secure pagetable object for the MMU
* @mmu_ops: Function pointers for the MMU sub-type
* @secured: True if the MMU needs to be secured
* @feature: Static list of MMU features
* @priv: Union of sub-device specific members
*/
struct kgsl_mmu {
unsigned long flags;
enum kgsl_mmutype type;
u32 subtype;
struct kgsl_pagetable *defaultpagetable;
struct kgsl_pagetable *securepagetable;
const struct kgsl_mmu_ops *mmu_ops;
bool secured;
unsigned long features;
union {
struct kgsl_iommu iommu;
} priv;
};
#define KGSL_IOMMU_PRIV(_device) (&((_device)->mmu.priv.iommu))
extern struct kgsl_mmu_ops kgsl_iommu_ops;
int kgsl_mmu_probe(struct kgsl_device *device);
int kgsl_mmu_start(struct kgsl_device *device);
struct kgsl_pagetable *kgsl_mmu_getpagetable_ptbase(struct kgsl_mmu *mmu,
u64 ptbase);
int kgsl_iommu_map_global_secure_pt_entry(struct kgsl_device *device,
struct kgsl_memdesc *memdesc);
void kgsl_iommu_unmap_global_secure_pt_entry(struct kgsl_device *device,
struct kgsl_memdesc *memdesc);
void kgsl_print_global_pt_entries(struct seq_file *s);
void kgsl_mmu_putpagetable(struct kgsl_pagetable *pagetable);
int kgsl_mmu_get_gpuaddr(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc);
int kgsl_mmu_map(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc);
int kgsl_mmu_unmap(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc);
void kgsl_mmu_put_gpuaddr(struct kgsl_memdesc *memdesc);
unsigned int kgsl_virtaddr_to_physaddr(void *virtaddr);
unsigned int kgsl_mmu_log_fault_addr(struct kgsl_mmu *mmu,
u64 ttbr0, uint64_t addr);
bool kgsl_mmu_gpuaddr_in_range(struct kgsl_pagetable *pt, uint64_t gpuaddr);
int kgsl_mmu_get_region(struct kgsl_pagetable *pagetable,
uint64_t gpuaddr, uint64_t size);
int kgsl_mmu_find_region(struct kgsl_pagetable *pagetable,
uint64_t region_start, uint64_t region_end,
uint64_t *gpuaddr, uint64_t size, unsigned int align);
struct kgsl_pagetable *kgsl_mmu_get_pt_from_ptname(struct kgsl_mmu *mmu,
int ptname);
void kgsl_mmu_close(struct kgsl_device *device);
uint64_t kgsl_mmu_find_svm_region(struct kgsl_pagetable *pagetable,
uint64_t start, uint64_t end, uint64_t size,
uint64_t alignment);
int kgsl_mmu_set_svm_region(struct kgsl_pagetable *pagetable, uint64_t gpuaddr,
uint64_t size);
void kgsl_mmu_detach_pagetable(struct kgsl_pagetable *pagetable);
int kgsl_mmu_svm_range(struct kgsl_pagetable *pagetable,
uint64_t *lo, uint64_t *hi, uint64_t memflags);
struct kgsl_pagetable *kgsl_get_pagetable(unsigned long name);
struct kgsl_pagetable *
kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu, unsigned int name);
int kgsl_mmu_map_offset(struct kgsl_pagetable *pagetable,
uint64_t virtaddr, uint64_t virtoffset,
struct kgsl_memdesc *memdesc, uint64_t physoffset,
uint64_t size, uint64_t flags);
int kgsl_mmu_unmap_offset(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc, uint64_t addr, uint64_t offset,
uint64_t size);
struct kgsl_memdesc *kgsl_mmu_get_qdss_global_entry(struct kgsl_device *device);
struct kgsl_memdesc *kgsl_mmu_get_qtimer_global_entry(
struct kgsl_device *device);
int kgsl_mmu_sparse_dummy_map(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc, uint64_t offset, uint64_t size);
/*
* Static inline functions of MMU that simply call the SMMU specific
* function using a function pointer. These functions can be thought
* of as wrappers around the actual function
*/
#define MMU_OP_VALID(_mmu, _field) \
(((_mmu) != NULL) && \
((_mmu)->mmu_ops != NULL) && \
((_mmu)->mmu_ops->_field != NULL))
#define PT_OP_VALID(_pt, _field) \
(((_pt) != NULL) && \
((_pt)->pt_ops != NULL) && \
((_pt)->pt_ops->_field != NULL))
static inline u64 kgsl_mmu_get_current_ttbr0(struct kgsl_mmu *mmu)
{
if (MMU_OP_VALID(mmu, mmu_get_current_ttbr0))
return mmu->mmu_ops->mmu_get_current_ttbr0(mmu);
return 0;
}
static inline struct kgsl_pagetable *kgsl_mmu_getpagetable(struct kgsl_mmu *mmu,
unsigned long name)
{
if (MMU_OP_VALID(mmu, mmu_getpagetable))
return mmu->mmu_ops->mmu_getpagetable(mmu, name);
return NULL;
}
static inline int kgsl_mmu_set_pt(struct kgsl_mmu *mmu,
struct kgsl_pagetable *pagetable)
{
if (MMU_OP_VALID(mmu, mmu_set_pt))
return mmu->mmu_ops->mmu_set_pt(mmu, pagetable);
return 0;
}
static inline bool kgsl_mmu_pt_equal(struct kgsl_mmu *mmu,
struct kgsl_pagetable *pt, u64 ttbr0)
{
if (MMU_OP_VALID(mmu, mmu_pt_equal))
return mmu->mmu_ops->mmu_pt_equal(mmu, pt, ttbr0);
return false;
}
static inline void kgsl_mmu_enable_clk(struct kgsl_mmu *mmu)
{
if (MMU_OP_VALID(mmu, mmu_enable_clk))
mmu->mmu_ops->mmu_enable_clk(mmu);
}
static inline void kgsl_mmu_disable_clk(struct kgsl_mmu *mmu)
{
if (MMU_OP_VALID(mmu, mmu_disable_clk))
mmu->mmu_ops->mmu_disable_clk(mmu);
}
static inline int kgsl_mmu_set_pagefault_policy(struct kgsl_mmu *mmu,
unsigned long pf_policy)
{
if (MMU_OP_VALID(mmu, mmu_set_pf_policy))
return mmu->mmu_ops->mmu_set_pf_policy(mmu, pf_policy);
return 0;
}
static inline void kgsl_mmu_pagefault_resume(struct kgsl_mmu *mmu)
{
if (MMU_OP_VALID(mmu, mmu_pagefault_resume))
return mmu->mmu_ops->mmu_pagefault_resume(mmu);
}
static inline void kgsl_mmu_clear_fsr(struct kgsl_mmu *mmu)
{
if (MMU_OP_VALID(mmu, mmu_clear_fsr))
return mmu->mmu_ops->mmu_clear_fsr(mmu);
}
static inline int kgsl_mmu_is_perprocess(struct kgsl_mmu *mmu)
{
return MMU_FEATURE(mmu, KGSL_MMU_GLOBAL_PAGETABLE) ? 0 : 1;
}
static inline int kgsl_mmu_use_cpu_map(struct kgsl_mmu *mmu)
{
return kgsl_mmu_is_perprocess(mmu);
}
static inline int kgsl_mmu_is_secured(struct kgsl_mmu *mmu)
{
return mmu && (mmu->secured) && (mmu->securepagetable);
}
static inline u64
kgsl_mmu_pagetable_get_ttbr0(struct kgsl_pagetable *pagetable)
{
if (PT_OP_VALID(pagetable, get_ttbr0))
return pagetable->pt_ops->get_ttbr0(pagetable);
return 0;
}
static inline u32
kgsl_mmu_pagetable_get_contextidr(struct kgsl_pagetable *pagetable)
{
if (PT_OP_VALID(pagetable, get_contextidr))
return pagetable->pt_ops->get_contextidr(pagetable);
return 0;
}
static inline struct device *kgsl_mmu_get_ctx(const char *name)
{
return ERR_PTR(-ENODEV);
}
#endif /* __KGSL_MMU_H */

602
drivers/gpu/msm/kgsl_pool.c Normal file
View file

@ -0,0 +1,602 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
*/
#include <asm/cacheflush.h>
#include <linux/highmem.h>
#include <linux/of.h>
#include <linux/scatterlist.h>
#include "kgsl_device.h"
#include "kgsl_pool.h"
#include "kgsl_sharedmem.h"
#define KGSL_MAX_POOLS 4
#define KGSL_MAX_POOL_ORDER 8
#define KGSL_MAX_RESERVED_PAGES 4096
/**
* struct kgsl_page_pool - Structure to hold information for the pool
* @pool_order: Page order describing the size of the page
* @page_count: Number of pages currently present in the pool
* @reserved_pages: Number of pages reserved at init for the pool
* @allocation_allowed: Tells if reserved pool gets exhausted, can we allocate
* from system memory
* @list_lock: Spinlock for page list in the pool
* @page_list: List of pages held/reserved in this pool
*/
struct kgsl_page_pool {
unsigned int pool_order;
int page_count;
unsigned int reserved_pages;
bool allocation_allowed;
spinlock_t list_lock;
struct list_head page_list;
};
static struct kgsl_page_pool kgsl_pools[KGSL_MAX_POOLS];
static int kgsl_num_pools;
static int kgsl_pool_max_pages;
/* Returns KGSL pool corresponding to input page order*/
static struct kgsl_page_pool *
_kgsl_get_pool_from_order(unsigned int order)
{
int i;
for (i = 0; i < kgsl_num_pools; i++) {
if (kgsl_pools[i].pool_order == order)
return &kgsl_pools[i];
}
return NULL;
}
/* Map the page into kernel and zero it out */
static void
_kgsl_pool_zero_page(struct page *p, unsigned int pool_order)
{
int i;
for (i = 0; i < (1 << pool_order); i++) {
struct page *page = nth_page(p, i);
void *addr = kmap_atomic(page);
memset(addr, 0, PAGE_SIZE);
dmac_flush_range(addr, addr + PAGE_SIZE);
kunmap_atomic(addr);
}
}
/* Add a page to specified pool */
static void
_kgsl_pool_add_page(struct kgsl_page_pool *pool, struct page *p)
{
_kgsl_pool_zero_page(p, pool->pool_order);
spin_lock(&pool->list_lock);
list_add_tail(&p->lru, &pool->page_list);
pool->page_count++;
spin_unlock(&pool->list_lock);
mod_node_page_state(page_pgdat(p), NR_INDIRECTLY_RECLAIMABLE_BYTES,
(PAGE_SIZE << pool->pool_order));
}
/* Returns a page from specified pool */
static struct page *
_kgsl_pool_get_page(struct kgsl_page_pool *pool)
{
struct page *p = NULL;
spin_lock(&pool->list_lock);
if (pool->page_count) {
p = list_first_entry(&pool->page_list, struct page, lru);
pool->page_count--;
list_del(&p->lru);
}
spin_unlock(&pool->list_lock);
mod_node_page_state(page_pgdat(p), NR_INDIRECTLY_RECLAIMABLE_BYTES,
-(PAGE_SIZE << pool->pool_order));
return p;
}
/* Returns the number of pages in specified pool */
static int
kgsl_pool_size(struct kgsl_page_pool *kgsl_pool)
{
int size;
spin_lock(&kgsl_pool->list_lock);
size = kgsl_pool->page_count * (1 << kgsl_pool->pool_order);
spin_unlock(&kgsl_pool->list_lock);
return size;
}
/* Returns the number of pages in all kgsl page pools */
static int kgsl_pool_size_total(void)
{
int i;
int total = 0;
for (i = 0; i < kgsl_num_pools; i++)
total += kgsl_pool_size(&kgsl_pools[i]);
return total;
}
/*
* This will shrink the specified pool by num_pages or its pool_size,
* whichever is smaller.
*/
static unsigned int
_kgsl_pool_shrink(struct kgsl_page_pool *pool, int num_pages)
{
int j;
unsigned int pcount = 0;
if (pool == NULL || num_pages <= 0)
return pcount;
for (j = 0; j < num_pages >> pool->pool_order; j++) {
struct page *page = _kgsl_pool_get_page(pool);
if (page != NULL) {
__free_pages(page, pool->pool_order);
pcount += (1 << pool->pool_order);
} else {
/* Break as this pool is empty */
break;
}
}
return pcount;
}
/*
* This function reduces the total pool size
* to number of pages specified by target_pages.
*
* If target_pages are greater than current pool size
* nothing needs to be done otherwise remove
* (current_pool_size - target_pages) pages from pool
* starting from higher order pool.
*/
static unsigned long
kgsl_pool_reduce(unsigned int target_pages, bool exit)
{
int total_pages = 0;
int i;
int nr_removed;
struct kgsl_page_pool *pool;
unsigned long pcount = 0;
total_pages = kgsl_pool_size_total();
for (i = (kgsl_num_pools - 1); i >= 0; i--) {
pool = &kgsl_pools[i];
/*
* Only reduce the pool sizes for pools which are allowed to
* allocate memory unless we are at close, in which case the
* reserved memory for all pools needs to be freed
*/
if (!pool->allocation_allowed && !exit)
continue;
total_pages -= pcount;
nr_removed = total_pages - target_pages;
if (nr_removed <= 0)
return pcount;
/* Round up to integral number of pages in this pool */
nr_removed = ALIGN(nr_removed, 1 << pool->pool_order);
/* Remove nr_removed pages from this pool*/
pcount += _kgsl_pool_shrink(pool, nr_removed);
}
return pcount;
}
/**
* kgsl_pool_free_sgt() - Free scatter-gather list
* @sgt: pointer of the sg list
*
* Free the sg list by collapsing any physical adjacent pages.
* Pages are added back to the pool, if pool has sufficient space
* otherwise they are given back to system.
*/
void kgsl_pool_free_sgt(struct sg_table *sgt)
{
int i;
struct scatterlist *sg;
for_each_sg(sgt->sgl, sg, sgt->nents, i) {
/*
* sg_alloc_table_from_pages() will collapse any physically
* adjacent pages into a single scatterlist entry. We cannot
* just call __free_pages() on the entire set since we cannot
* ensure that the size is a whole order. Instead, free each
* page or compound page group individually.
*/
struct page *p = sg_page(sg), *next;
unsigned int count;
unsigned int j = 0;
while (j < (sg->length/PAGE_SIZE)) {
count = 1 << compound_order(p);
next = nth_page(p, count);
kgsl_pool_free_page(p);
p = next;
j += count;
}
}
}
/**
* kgsl_pool_free_pages() - Free pages in the pages array
* @pages: pointer of the pages array
*
* Free the pages by collapsing any physical adjacent pages.
* Pages are added back to the pool, if pool has sufficient space
* otherwise they are given back to system.
*/
void kgsl_pool_free_pages(struct page **pages, unsigned int pcount)
{
int i;
if (pages == NULL || pcount == 0)
return;
if (WARN(!kern_addr_valid((unsigned long)pages),
"Address of pages=%pK is not valid\n", pages))
return;
for (i = 0; i < pcount;) {
/*
* Free each page or compound page group individually.
*/
struct page *p = pages[i];
if (WARN(!kern_addr_valid((unsigned long)p),
"Address of page=%pK is not valid\n", p))
return;
i += 1 << compound_order(p);
kgsl_pool_free_page(p);
}
}
static int kgsl_pool_idx_lookup(unsigned int order)
{
int i;
for (i = 0; i < kgsl_num_pools; i++)
if (order == kgsl_pools[i].pool_order)
return i;
return -ENOMEM;
}
static int kgsl_pool_get_retry_order(unsigned int order)
{
int i;
for (i = kgsl_num_pools-1; i > 0; i--)
if (order >= kgsl_pools[i].pool_order)
return kgsl_pools[i].pool_order;
return 0;
}
static unsigned int kgsl_gfp_mask(unsigned int page_order)
{
unsigned int gfp_mask = __GFP_HIGHMEM;
if (page_order > 0) {
gfp_mask |= __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN;
gfp_mask &= ~__GFP_RECLAIM;
} else
gfp_mask |= GFP_KERNEL;
if (kgsl_sharedmem_get_noretry())
gfp_mask |= __GFP_NORETRY | __GFP_NOWARN;
return gfp_mask;
}
/**
* kgsl_pool_alloc_page() - Allocate a page of requested size
* @page_size: Size of the page to be allocated
* @pages: pointer to hold list of pages, should be big enough to hold
* requested page
* @len: Length of array pages.
*
* Return total page count on success and negative value on failure
*/
int kgsl_pool_alloc_page(int *page_size, struct page **pages,
unsigned int pages_len, unsigned int *align)
{
int j;
int pcount = 0;
struct kgsl_page_pool *pool;
struct page *page = NULL;
struct page *p = NULL;
int order = get_order(*page_size);
int pool_idx;
size_t size = 0;
if ((pages == NULL) || pages_len < (*page_size >> PAGE_SHIFT))
return -EINVAL;
/* If the pool is not configured get pages from the system */
if (!kgsl_num_pools) {
gfp_t gfp_mask = kgsl_gfp_mask(order);
page = alloc_pages(gfp_mask, order);
if (page == NULL) {
/* Retry with lower order pages */
if (order > 0) {
size = PAGE_SIZE << --order;
goto eagain;
} else
return -ENOMEM;
}
_kgsl_pool_zero_page(page, order);
goto done;
}
pool = _kgsl_get_pool_from_order(order);
if (pool == NULL) {
/* Retry with lower order pages */
if (order > 0) {
size = PAGE_SIZE << kgsl_pool_get_retry_order(order);
goto eagain;
} else {
/*
* Fall back to direct allocation in case
* pool with zero order is not present
*/
gfp_t gfp_mask = kgsl_gfp_mask(order);
page = alloc_pages(gfp_mask, order);
if (page == NULL)
return -ENOMEM;
_kgsl_pool_zero_page(page, order);
goto done;
}
}
pool_idx = kgsl_pool_idx_lookup(order);
page = _kgsl_pool_get_page(pool);
/* Allocate a new page if not allocated from pool */
if (page == NULL) {
gfp_t gfp_mask = kgsl_gfp_mask(order);
/* Only allocate non-reserved memory for certain pools */
if (!pool->allocation_allowed && pool_idx > 0) {
size = PAGE_SIZE <<
kgsl_pools[pool_idx-1].pool_order;
goto eagain;
}
page = alloc_pages(gfp_mask, order);
if (!page) {
if (pool_idx > 0) {
/* Retry with lower order pages */
size = PAGE_SIZE <<
kgsl_pools[pool_idx-1].pool_order;
goto eagain;
} else
return -ENOMEM;
}
_kgsl_pool_zero_page(page, order);
}
done:
for (j = 0; j < (*page_size >> PAGE_SHIFT); j++) {
p = nth_page(page, j);
pages[pcount] = p;
pcount++;
}
mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES,
(1 << order));
return pcount;
eagain:
*page_size = kgsl_get_page_size(size,
ilog2(size));
*align = ilog2(*page_size);
return -EAGAIN;
}
void kgsl_pool_free_page(struct page *page)
{
struct kgsl_page_pool *pool;
int page_order;
if (page == NULL)
return;
page_order = compound_order(page);
mod_node_page_state(page_pgdat(page), NR_UNRECLAIMABLE_PAGES,
-(1 << page_order));
if (!kgsl_pool_max_pages ||
(kgsl_pool_size_total() < kgsl_pool_max_pages)) {
pool = _kgsl_get_pool_from_order(page_order);
if (pool != NULL) {
_kgsl_pool_add_page(pool, page);
return;
}
}
/* Give back to system as not added to pool */
__free_pages(page, page_order);
}
/*
* Return true if the pool of specified page size is supported
* or no pools are supported otherwise return false.
*/
bool kgsl_pool_avaialable(int page_size)
{
int i;
if (!kgsl_num_pools)
return true;
for (i = 0; i < kgsl_num_pools; i++)
if (ilog2(page_size >> PAGE_SHIFT) == kgsl_pools[i].pool_order)
return true;
return false;
}
static void kgsl_pool_reserve_pages(void)
{
int i, j;
for (i = 0; i < kgsl_num_pools; i++) {
struct page *page;
for (j = 0; j < kgsl_pools[i].reserved_pages; j++) {
int order = kgsl_pools[i].pool_order;
gfp_t gfp_mask = kgsl_gfp_mask(order);
page = alloc_pages(gfp_mask, order);
if (page != NULL)
_kgsl_pool_add_page(&kgsl_pools[i], page);
}
}
}
/* Functions for the shrinker */
static unsigned long
kgsl_pool_shrink_scan_objects(struct shrinker *shrinker,
struct shrink_control *sc)
{
/* nr represents number of pages to be removed*/
int nr = sc->nr_to_scan;
int total_pages = kgsl_pool_size_total();
/* Target pages represents new pool size */
int target_pages = (nr > total_pages) ? 0 : (total_pages - nr);
/* Reduce pool size to target_pages */
return kgsl_pool_reduce(target_pages, false);
}
static unsigned long
kgsl_pool_shrink_count_objects(struct shrinker *shrinker,
struct shrink_control *sc)
{
/* Return total pool size as everything in pool can be freed */
return kgsl_pool_size_total();
}
/* Shrinker callback data*/
static struct shrinker kgsl_pool_shrinker = {
.count_objects = kgsl_pool_shrink_count_objects,
.scan_objects = kgsl_pool_shrink_scan_objects,
.seeks = DEFAULT_SEEKS,
.batch = 0,
};
static void kgsl_pool_config(unsigned int order, unsigned int reserved_pages,
bool allocation_allowed)
{
#ifdef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS
if (order > 0) {
pr_err("kgsl: pool order:%d not supprted\n", order);
return;
}
#endif
if ((order > KGSL_MAX_POOL_ORDER) ||
(reserved_pages > KGSL_MAX_RESERVED_PAGES))
return;
kgsl_pools[kgsl_num_pools].pool_order = order;
kgsl_pools[kgsl_num_pools].reserved_pages = reserved_pages;
kgsl_pools[kgsl_num_pools].allocation_allowed = allocation_allowed;
spin_lock_init(&kgsl_pools[kgsl_num_pools].list_lock);
INIT_LIST_HEAD(&kgsl_pools[kgsl_num_pools].page_list);
kgsl_num_pools++;
}
static void kgsl_of_parse_mempools(struct device_node *node)
{
struct device_node *child;
unsigned int page_size, reserved_pages = 0;
bool allocation_allowed;
for_each_child_of_node(node, child) {
unsigned int index;
if (of_property_read_u32(child, "reg", &index))
return;
if (index >= KGSL_MAX_POOLS)
continue;
if (of_property_read_u32(child, "qcom,mempool-page-size",
&page_size))
return;
of_property_read_u32(child, "qcom,mempool-reserved",
&reserved_pages);
allocation_allowed = of_property_read_bool(child,
"qcom,mempool-allocate");
kgsl_pool_config(ilog2(page_size >> PAGE_SHIFT), reserved_pages,
allocation_allowed);
}
}
static void kgsl_of_get_mempools(struct device_node *parent)
{
struct device_node *node;
node = of_find_compatible_node(parent, NULL, "qcom,gpu-mempools");
if (node != NULL) {
/* Get Max pages limit for mempool */
of_property_read_u32(node, "qcom,mempool-max-pages",
&kgsl_pool_max_pages);
kgsl_of_parse_mempools(node);
}
}
void kgsl_init_page_pools(struct platform_device *pdev)
{
/* Get GPU mempools data and configure pools */
kgsl_of_get_mempools(pdev->dev.of_node);
/* Reserve the appropriate number of pages for each pool */
kgsl_pool_reserve_pages();
/* Initialize shrinker */
register_shrinker(&kgsl_pool_shrinker);
}
void kgsl_exit_page_pools(void)
{
/* Release all pages in pools, if any.*/
kgsl_pool_reduce(0, true);
/* Unregister shrinker */
unregister_shrinker(&kgsl_pool_shrinker);
}

17
drivers/gpu/msm/kgsl_pool.h Normal file
View file

@ -0,0 +1,17 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016-2017,2019 The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_POOL_H
#define __KGSL_POOL_H
void kgsl_pool_free_sgt(struct sg_table *sgt);
void kgsl_pool_free_pages(struct page **pages, unsigned int page_count);
void kgsl_init_page_pools(struct platform_device *pdev);
void kgsl_exit_page_pools(void);
int kgsl_pool_alloc_page(int *page_size, struct page **pages,
unsigned int pages_len, unsigned int *align);
void kgsl_pool_free_page(struct page *p);
bool kgsl_pool_avaialable(int size);
#endif /* __KGSL_POOL_H */

2638
drivers/gpu/msm/kgsl_pwrctrl.c Normal file
View file

File diff suppressed because it is too large Load diff

208
drivers/gpu/msm/kgsl_pwrctrl.h Normal file
View file

@ -0,0 +1,208 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2010-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_PWRCTRL_H
#define __KGSL_PWRCTRL_H
#include <linux/clk.h>
/*****************************************************************************
* power flags
****************************************************************************/
#define KGSL_PWRFLAGS_ON 1
#define KGSL_PWRFLAGS_OFF 0
#define KGSL_PWRLEVEL_TURBO 0
#define KGSL_PWR_ON 0xFFFF
#define KGSL_MAX_CLKS 17
#define KGSL_MAX_REGULATORS 2
#define KGSL_MAX_PWRLEVELS 10
/* Only two supported levels, min & max */
#define KGSL_CONSTRAINT_PWR_MAXLEVELS 2
#define KGSL_XO_CLK_FREQ 19200000
#define KGSL_RBBMTIMER_CLK_FREQ KGSL_XO_CLK_FREQ
#define KGSL_ISENSE_CLK_FREQ 200000000
enum kgsl_pwrctrl_timer_type {
KGSL_PWR_IDLE_TIMER,
};
struct platform_device;
struct kgsl_clk_stats {
unsigned int busy;
unsigned int total;
unsigned int busy_old;
unsigned int total_old;
};
struct kgsl_pwr_constraint {
unsigned int type;
unsigned int sub_type;
union {
struct {
unsigned int level;
} pwrlevel;
} hint;
unsigned long expires;
uint32_t owner_id;
};
/**
* struct kgsl_pwrlevel - Struct holding different pwrlevel info obtained from
* from dtsi file
* @gpu_freq: GPU frequency vote in Hz
* @bus_freq: Bus bandwidth vote index
* @bus_min: Min bus index @gpu_freq
* @bus_max: Max bus index @gpu_freq
*/
struct kgsl_pwrlevel {
unsigned int gpu_freq;
unsigned int bus_freq;
unsigned int bus_min;
unsigned int bus_max;
unsigned int acd_level;
};
struct kgsl_regulator {
struct regulator *reg;
char name[8];
};
/**
* struct kgsl_pwrctrl - Power control settings for a KGSL device
* @interrupt_num - The interrupt number for the device
* @grp_clks - Array of clocks structures that we control
* @power_flags - Control flags for power
* @pwrlevels - List of supported power levels
* @nb - Notifier block to receive GPU OPP change event
* @active_pwrlevel - The currently active power level
* @previous_pwrlevel - The power level before transition
* @thermal_pwrlevel - maximum powerlevel constraint from thermal
* @thermal_pwrlevel_floor - minimum powerlevel constraint from thermal
* @default_pwrlevel - device wake up power level
* @max_pwrlevel - maximum allowable powerlevel per the user
* @min_pwrlevel - minimum allowable powerlevel per the user
* @num_pwrlevels - number of available power levels
* @throttle_mask - LM throttle mask
* @interval_timeout - timeout in jiffies to be idle before a power event
* @clock_times - Each GPU frequency's accumulated active time in us
* @regulators - array of pointers to kgsl_regulator structs
* @pcl - bus scale identifier
* @irq_name - resource name for the IRQ
* @clk_stats - structure of clock statistics
* @input_disable - To disable GPU wakeup on touch input event
* @bus_control - true if the bus calculation is independent
* @bus_mod - modifier from the current power level for the bus vote
* @bus_percent_ab - current percent of total possible bus usage
* @bus_width - target specific bus width in number of bytes
* @bus_ab_mbytes - AB vote in Mbytes for current bus usage
* @constraint - currently active power constraint
* @superfast - Boolean flag to indicate that the GPU start should be run in the
* higher priority thread
* isense_clk_indx - index of isense clock, 0 if no isense
* isense_clk_on_level - isense clock rate is XO rate below this level.
* tzone_name - pointer to thermal zone name of GPU temperature sensor
*/
struct kgsl_pwrctrl {
int interrupt_num;
struct clk *grp_clks[KGSL_MAX_CLKS];
struct clk *gpu_bimc_int_clk;
int isense_clk_indx;
int isense_clk_on_level;
unsigned long power_flags;
unsigned long ctrl_flags;
struct kgsl_pwrlevel pwrlevels[KGSL_MAX_PWRLEVELS];
struct notifier_block nb;
unsigned int active_pwrlevel;
unsigned int previous_pwrlevel;
unsigned int thermal_pwrlevel;
unsigned int thermal_pwrlevel_floor;
unsigned int default_pwrlevel;
unsigned int wakeup_maxpwrlevel;
unsigned int max_pwrlevel;
unsigned int min_pwrlevel;
unsigned int num_pwrlevels;
unsigned int throttle_mask;
unsigned long interval_timeout;
u64 clock_times[KGSL_MAX_PWRLEVELS];
struct kgsl_regulator regulators[KGSL_MAX_REGULATORS];
uint32_t pcl;
const char *irq_name;
struct kgsl_clk_stats clk_stats;
bool input_disable;
bool bus_control;
int bus_mod;
unsigned int bus_percent_ab;
unsigned int bus_width;
unsigned long bus_ab_mbytes;
struct device *devbw;
/** @bus_ibs: List of the bus bandwidths in use by our target */
u32 *bus_ibs;
/** @bus_ibs_count: Number of objects in @bus_ibs */
int bus_ibs_count;
/** cur_buslevel: The last buslevel voted by the driver */
int cur_buslevel;
/** @bus_max: The maximum bandwidth available to the device */
unsigned long bus_max;
struct kgsl_pwr_constraint constraint;
bool superfast;
unsigned int gpu_bimc_int_clk_freq;
bool gpu_bimc_interface_enabled;
const char *tzone_name;
/** @icc_path: Interconnect path for the GPU (if applicable) */
struct icc_path *icc_path;
};
int kgsl_pwrctrl_init(struct kgsl_device *device);
void kgsl_pwrctrl_close(struct kgsl_device *device);
void kgsl_timer(struct timer_list *t);
void kgsl_idle_check(struct work_struct *work);
void kgsl_pre_hwaccess(struct kgsl_device *device);
void kgsl_pwrctrl_pwrlevel_change(struct kgsl_device *device,
unsigned int level);
void kgsl_pwrctrl_buslevel_update(struct kgsl_device *device,
bool on);
int kgsl_pwrctrl_init_sysfs(struct kgsl_device *device);
void kgsl_pwrctrl_uninit_sysfs(struct kgsl_device *device);
int kgsl_pwrctrl_change_state(struct kgsl_device *device, int state);
int kgsl_clk_set_rate(struct kgsl_device *device,
unsigned int pwrlevel);
unsigned int kgsl_pwrctrl_adjust_pwrlevel(struct kgsl_device *device,
unsigned int new_level);
static inline unsigned long kgsl_get_clkrate(struct clk *clk)
{
return (clk != NULL) ? clk_get_rate(clk) : 0;
}
/*
* kgsl_pwrctrl_active_freq - get currently configured frequency
* @pwr: kgsl_pwrctrl structure for the device
*
* Returns the currently configured frequency for the device.
*/
static inline unsigned long
kgsl_pwrctrl_active_freq(struct kgsl_pwrctrl *pwr)
{
return pwr->pwrlevels[pwr->active_pwrlevel].gpu_freq;
}
int __must_check kgsl_active_count_get(struct kgsl_device *device);
void kgsl_active_count_put(struct kgsl_device *device);
int kgsl_active_count_wait(struct kgsl_device *device, int count);
void kgsl_pwrctrl_busy_time(struct kgsl_device *device, u64 time, u64 busy);
void kgsl_pwrctrl_set_constraint(struct kgsl_device *device,
struct kgsl_pwr_constraint *pwrc, uint32_t id);
int kgsl_pwrctrl_set_default_gpu_pwrlevel(struct kgsl_device *device);
void kgsl_pwrctrl_disable_unused_opp(struct kgsl_device *device,
struct device *dev);
#endif /* __KGSL_PWRCTRL_H */

871
drivers/gpu/msm/kgsl_pwrscale.c Normal file
View file

@ -0,0 +1,871 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2010-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/devfreq_cooling.h>
#include <linux/slab.h>
#include "kgsl_device.h"
#include "kgsl_pwrscale.h"
#include "kgsl_trace.h"
/**
* struct kgsl_midframe_info - midframe power stats sampling info
* @timer - midframe sampling timer
* @timer_check_ws - Updates powerstats on midframe expiry
* @device - pointer to kgsl_device
*/
static struct kgsl_midframe_info {
struct hrtimer timer;
struct work_struct timer_check_ws;
struct kgsl_device *device;
} *kgsl_midframe = NULL;
static void do_devfreq_suspend(struct work_struct *work);
static void do_devfreq_resume(struct work_struct *work);
static void do_devfreq_notify(struct work_struct *work);
/*
* These variables are used to keep the latest data
* returned by kgsl_devfreq_get_dev_status
*/
static struct xstats last_xstats;
static struct devfreq_dev_status last_status = { .private_data = &last_xstats };
/*
* kgsl_pwrscale_sleep - notify governor that device is going off
* @device: The device
*
* Called shortly after all pending work is completed.
*/
void kgsl_pwrscale_sleep(struct kgsl_device *device)
{
if (!device->pwrscale.enabled)
return;
device->pwrscale.on_time = 0;
/* to call devfreq_suspend_device() from a kernel thread */
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_suspend_ws);
}
/*
* kgsl_pwrscale_wake - notify governor that device is going on
* @device: The device
*
* Called when the device is returning to an active state.
*/
void kgsl_pwrscale_wake(struct kgsl_device *device)
{
struct kgsl_power_stats stats;
struct kgsl_pwrscale *psc = &device->pwrscale;
if (!device->pwrscale.enabled)
return;
/* clear old stats before waking */
memset(&psc->accum_stats, 0, sizeof(psc->accum_stats));
memset(&last_xstats, 0, sizeof(last_xstats));
/* and any hw activity from waking up*/
device->ftbl->power_stats(device, &stats);
psc->time = ktime_get();
psc->next_governor_call = ktime_add_us(psc->time,
KGSL_GOVERNOR_CALL_INTERVAL);
/* to call devfreq_resume_device() from a kernel thread */
queue_work(psc->devfreq_wq, &psc->devfreq_resume_ws);
}
/*
* kgsl_pwrscale_busy - update pwrscale state for new work
* @device: The device
*
* Called when new work is submitted to the device.
* This function must be called with the device mutex locked.
*/
void kgsl_pwrscale_busy(struct kgsl_device *device)
{
if (!device->pwrscale.enabled)
return;
if (device->pwrscale.on_time == 0)
device->pwrscale.on_time = ktime_to_us(ktime_get());
}
/**
* kgsl_pwrscale_update_stats() - update device busy statistics
* @device: The device
*
* Read hardware busy counters and accumulate the results.
*/
void kgsl_pwrscale_update_stats(struct kgsl_device *device)
{
struct kgsl_pwrctrl *pwrctrl = &device->pwrctrl;
struct kgsl_pwrscale *psc = &device->pwrscale;
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (!psc->enabled)
return;
if (device->state == KGSL_STATE_ACTIVE) {
struct kgsl_power_stats stats;
device->ftbl->power_stats(device, &stats);
device->pwrscale.accum_stats.busy_time += stats.busy_time;
device->pwrscale.accum_stats.ram_time += stats.ram_time;
device->pwrscale.accum_stats.ram_wait += stats.ram_wait;
pwrctrl->clock_times[pwrctrl->active_pwrlevel] +=
stats.busy_time;
}
}
/**
* kgsl_pwrscale_update() - update device busy statistics
* @device: The device
*
* If enough time has passed schedule the next call to devfreq
* get_dev_status.
*/
void kgsl_pwrscale_update(struct kgsl_device *device)
{
ktime_t t;
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (!device->pwrscale.enabled)
return;
t = ktime_get();
if (ktime_compare(t, device->pwrscale.next_governor_call) < 0)
return;
device->pwrscale.next_governor_call = ktime_add_us(t,
KGSL_GOVERNOR_CALL_INTERVAL);
/* to call srcu_notifier_call_chain() from a kernel thread */
if (device->state != KGSL_STATE_SLUMBER)
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_notify_ws);
kgsl_pwrscale_midframe_timer_restart(device);
}
void kgsl_pwrscale_midframe_timer_restart(struct kgsl_device *device)
{
if (kgsl_midframe) {
WARN_ON(!mutex_is_locked(&device->mutex));
/* If the timer is already running, stop it */
if (hrtimer_active(&kgsl_midframe->timer))
hrtimer_cancel(
&kgsl_midframe->timer);
hrtimer_start(&kgsl_midframe->timer,
ns_to_ktime(KGSL_GOVERNOR_CALL_INTERVAL
* NSEC_PER_USEC), HRTIMER_MODE_REL);
}
}
void kgsl_pwrscale_midframe_timer_cancel(struct kgsl_device *device)
{
if (kgsl_midframe) {
WARN_ON(!mutex_is_locked(&device->mutex));
hrtimer_cancel(&kgsl_midframe->timer);
}
}
static void kgsl_pwrscale_midframe_timer_check(struct work_struct *work)
{
struct kgsl_device *device = kgsl_midframe->device;
mutex_lock(&device->mutex);
if (device->state == KGSL_STATE_ACTIVE)
kgsl_pwrscale_update(device);
mutex_unlock(&device->mutex);
}
static enum hrtimer_restart kgsl_pwrscale_midframe_timer(struct hrtimer *timer)
{
struct kgsl_device *device = kgsl_midframe->device;
queue_work(device->pwrscale.devfreq_wq,
&kgsl_midframe->timer_check_ws);
return HRTIMER_NORESTART;
}
/*
* kgsl_pwrscale_disable - temporarily disable the governor
* @device: The device
* @turbo: Indicates if pwrlevel should be forced to turbo
*
* Temporarily disable the governor, to prevent interference
* with profiling tools that expect a fixed clock frequency.
* This function must be called with the device mutex locked.
*/
void kgsl_pwrscale_disable(struct kgsl_device *device, bool turbo)
{
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (device->pwrscale.devfreqptr)
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_suspend_ws);
device->pwrscale.enabled = false;
if (turbo)
kgsl_pwrctrl_pwrlevel_change(device, KGSL_PWRLEVEL_TURBO);
}
/*
* kgsl_pwrscale_enable - re-enable the governor
* @device: The device
*
* Reenable the governor after a kgsl_pwrscale_disable() call.
* This function must be called with the device mutex locked.
*/
void kgsl_pwrscale_enable(struct kgsl_device *device)
{
if (WARN_ON(!mutex_is_locked(&device->mutex)))
return;
if (device->pwrscale.devfreqptr) {
queue_work(device->pwrscale.devfreq_wq,
&device->pwrscale.devfreq_resume_ws);
device->pwrscale.enabled = true;
} else {
/*
* Don't enable it if devfreq is not set and let the device
* run at default level;
*/
kgsl_pwrctrl_pwrlevel_change(device,
device->pwrctrl.default_pwrlevel);
device->pwrscale.enabled = false;
}
}
#ifdef DEVFREQ_FLAG_WAKEUP_MAXFREQ
static inline bool _check_maxfreq(u32 flags)
{
return (flags & DEVFREQ_FLAG_WAKEUP_MAXFREQ);
}
#else
static inline bool _check_maxfreq(u32 flags)
{
return false;
}
#endif
/*
* kgsl_devfreq_target - devfreq_dev_profile.target callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_devfreq_target(struct device *dev, unsigned long *freq, u32 flags)
{
struct kgsl_device *device = dev_get_drvdata(dev);
struct kgsl_pwrctrl *pwr;
struct kgsl_pwrlevel *pwr_level;
int level;
unsigned int i;
unsigned long cur_freq, rec_freq;
if (device == NULL)
return -ENODEV;
if (freq == NULL)
return -EINVAL;
if (!device->pwrscale.enabled)
return 0;
pwr = &device->pwrctrl;
if (_check_maxfreq(flags)) {
/*
* The GPU is about to get suspended,
* but it needs to be at the max power level when waking up
*/
pwr->wakeup_maxpwrlevel = 1;
return 0;
}
rec_freq = *freq;
mutex_lock(&device->mutex);
cur_freq = kgsl_pwrctrl_active_freq(pwr);
level = pwr->active_pwrlevel;
pwr_level = &pwr->pwrlevels[level];
/* If the governor recommends a new frequency, update it here */
if (rec_freq != cur_freq) {
level = pwr->max_pwrlevel;
/*
* Array index of pwrlevels[] should be within the permitted
* power levels, i.e., from max_pwrlevel to min_pwrlevel.
*/
for (i = pwr->min_pwrlevel; (i >= pwr->max_pwrlevel
&& i <= pwr->min_pwrlevel); i--)
if (rec_freq <= pwr->pwrlevels[i].gpu_freq) {
level = i;
break;
}
if (level != pwr->active_pwrlevel)
kgsl_pwrctrl_pwrlevel_change(device, level);
}
*freq = kgsl_pwrctrl_active_freq(pwr);
mutex_unlock(&device->mutex);
return 0;
}
/*
* kgsl_devfreq_get_dev_status - devfreq_dev_profile.get_dev_status callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_devfreq_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat)
{
struct kgsl_device *device = dev_get_drvdata(dev);
struct kgsl_pwrctrl *pwrctrl;
struct kgsl_pwrscale *pwrscale;
ktime_t tmp1, tmp2;
if (device == NULL)
return -ENODEV;
if (stat == NULL)
return -EINVAL;
pwrscale = &device->pwrscale;
pwrctrl = &device->pwrctrl;
mutex_lock(&device->mutex);
tmp1 = ktime_get();
/*
* If the GPU clock is on grab the latest power counter
* values. Otherwise the most recent ACTIVE values will
* already be stored in accum_stats.
*/
kgsl_pwrscale_update_stats(device);
tmp2 = ktime_get();
stat->total_time = ktime_us_delta(tmp2, pwrscale->time);
pwrscale->time = tmp1;
stat->busy_time = pwrscale->accum_stats.busy_time;
stat->current_frequency = kgsl_pwrctrl_active_freq(&device->pwrctrl);
stat->private_data = &device->active_context_count;
/*
* keep the latest devfreq_dev_status values
* and vbif counters data
* to be (re)used by kgsl_busmon_get_dev_status()
*/
if (pwrctrl->bus_control) {
struct xstats *last_b =
(struct xstats *)last_status.private_data;
last_status.total_time = stat->total_time;
last_status.busy_time = stat->busy_time;
last_b->ram_time = device->pwrscale.accum_stats.ram_time;
last_b->ram_wait = device->pwrscale.accum_stats.ram_wait;
last_b->mod = device->pwrctrl.bus_mod;
last_b->buslevel = device->pwrctrl.cur_buslevel;
}
kgsl_pwrctrl_busy_time(device, stat->total_time, stat->busy_time);
trace_kgsl_pwrstats(device, stat->total_time,
&pwrscale->accum_stats, device->active_context_count);
memset(&pwrscale->accum_stats, 0, sizeof(pwrscale->accum_stats));
mutex_unlock(&device->mutex);
return 0;
}
/*
* kgsl_devfreq_get_cur_freq - devfreq_dev_profile.get_cur_freq callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
{
struct kgsl_device *device = dev_get_drvdata(dev);
if (device == NULL)
return -ENODEV;
if (freq == NULL)
return -EINVAL;
mutex_lock(&device->mutex);
*freq = kgsl_pwrctrl_active_freq(&device->pwrctrl);
mutex_unlock(&device->mutex);
return 0;
}
/*
* kgsl_devfreq_add_notifier - add a fine grained notifier.
* @dev: The device
* @nb: Notifier block that will receive updates.
*
* Add a notifier to receive ADRENO_DEVFREQ_NOTIFY_* events
* from the device.
*/
int kgsl_devfreq_add_notifier(struct device *dev,
struct notifier_block *nb)
{
struct kgsl_device *device = dev_get_drvdata(dev);
if (device == NULL)
return -ENODEV;
if (nb == NULL)
return -EINVAL;
return srcu_notifier_chain_register(&device->pwrscale.nh, nb);
}
/*
* kgsl_devfreq_del_notifier - remove a fine grained notifier.
* @dev: The device
* @nb: The notifier block.
*
* Remove a notifier registered with kgsl_devfreq_add_notifier().
*/
int kgsl_devfreq_del_notifier(struct device *dev, struct notifier_block *nb)
{
struct kgsl_device *device = dev_get_drvdata(dev);
if (device == NULL)
return -ENODEV;
if (nb == NULL)
return -EINVAL;
return srcu_notifier_chain_unregister(&device->pwrscale.nh, nb);
}
/*
* kgsl_busmon_get_dev_status - devfreq_dev_profile.get_dev_status callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_busmon_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat)
{
struct xstats *b;
stat->total_time = last_status.total_time;
stat->busy_time = last_status.busy_time;
stat->current_frequency = last_status.current_frequency;
if (stat->private_data) {
struct xstats *last_b =
(struct xstats *)last_status.private_data;
b = (struct xstats *)stat->private_data;
b->ram_time = last_b->ram_time;
b->ram_wait = last_b->ram_wait;
b->buslevel = last_b->buslevel;
}
return 0;
}
#ifdef DEVFREQ_FLAG_FAST_HINT
static inline bool _check_fast_hint(u32 flags)
{
return (flags & DEVFREQ_FLAG_FAST_HINT);
}
#else
static inline bool _check_fast_hint(u32 flags)
{
return false;
}
#endif
#ifdef DEVFREQ_FLAG_SLOW_HINT
static inline bool _check_slow_hint(u32 flags)
{
return (flags & DEVFREQ_FLAG_SLOW_HINT);
}
#else
static inline bool _check_slow_hint(u32 flags)
{
return false;
}
#endif
/*
* kgsl_busmon_target - devfreq_dev_profile.target callback
* @dev: see devfreq.h
* @freq: see devfreq.h
* @flags: see devfreq.h
*
* This function expects the device mutex to be unlocked.
*/
int kgsl_busmon_target(struct device *dev, unsigned long *freq, u32 flags)
{
struct kgsl_device *device = dev_get_drvdata(dev);
struct kgsl_pwrctrl *pwr;
struct kgsl_pwrlevel *pwr_level;
int level, b;
u32 bus_flag;
unsigned long ab_mbytes;
if (device == NULL)
return -ENODEV;
if (freq == NULL)
return -EINVAL;
if (!device->pwrscale.enabled)
return 0;
pwr = &device->pwrctrl;
if (!pwr->bus_control)
return 0;
mutex_lock(&device->mutex);
level = pwr->active_pwrlevel;
pwr_level = &pwr->pwrlevels[level];
bus_flag = device->pwrscale.bus_profile.flag;
device->pwrscale.bus_profile.flag = 0;
ab_mbytes = device->pwrscale.bus_profile.ab_mbytes;
/*
* Bus devfreq governor has calculated its recomendations
* when gpu was running with *freq frequency.
* If the gpu frequency is different now it's better to
* ignore the call
*/
if (pwr_level->gpu_freq != *freq) {
mutex_unlock(&device->mutex);
return 0;
}
b = pwr->bus_mod;
if (_check_fast_hint(bus_flag))
pwr->bus_mod++;
else if (_check_slow_hint(bus_flag))
pwr->bus_mod--;
/* trim calculated change to fit range */
if (pwr_level->bus_freq + pwr->bus_mod < pwr_level->bus_min)
pwr->bus_mod = -(pwr_level->bus_freq - pwr_level->bus_min);
else if (pwr_level->bus_freq + pwr->bus_mod > pwr_level->bus_max)
pwr->bus_mod = pwr_level->bus_max - pwr_level->bus_freq;
/* Update bus vote if AB or IB is modified */
if ((pwr->bus_mod != b) || (pwr->bus_ab_mbytes != ab_mbytes)) {
pwr->bus_percent_ab = device->pwrscale.bus_profile.percent_ab;
pwr->bus_ab_mbytes = ab_mbytes;
kgsl_pwrctrl_buslevel_update(device, true);
}
mutex_unlock(&device->mutex);
return 0;
}
int kgsl_busmon_get_cur_freq(struct device *dev, unsigned long *freq)
{
return 0;
}
/*
* opp_notify - Callback function registered to receive OPP events.
* @nb: The notifier block
* @type: The event type. Two OPP events are expected in this function:
* - OPP_EVENT_ENABLE: an GPU OPP is enabled. The in_opp parameter
* contains the OPP that is enabled
* - OPP_EVENT_DISALBE: an GPU OPP is disabled. The in_opp parameter
* contains the OPP that is disabled.
* @in_opp: the GPU OPP whose status is changed and triggered the event
*
* GPU OPP event callback function. The function subscribe GPU OPP status
* change and update thermal power level accordingly.
*/
static int opp_notify(struct notifier_block *nb,
unsigned long type, void *in_opp)
{
int level, min_level, max_level;
struct kgsl_pwrctrl *pwr = container_of(nb, struct kgsl_pwrctrl, nb);
struct kgsl_device *device = container_of(pwr,
struct kgsl_device, pwrctrl);
struct device *dev = &device->pdev->dev;
struct dev_pm_opp *opp;
unsigned long min_freq = 0, max_freq = pwr->pwrlevels[0].gpu_freq;
if (type != OPP_EVENT_ENABLE && type != OPP_EVENT_DISABLE)
return -EINVAL;
opp = dev_pm_opp_find_freq_floor(dev, &max_freq);
if (IS_ERR(opp))
return PTR_ERR(opp);
dev_pm_opp_put(opp);
opp = dev_pm_opp_find_freq_ceil(dev, &min_freq);
if (IS_ERR(opp))
min_freq = pwr->pwrlevels[pwr->min_pwrlevel].gpu_freq;
else
dev_pm_opp_put(opp);
trace_kgsl_opp_notify(min_freq, max_freq);
mutex_lock(&device->mutex);
max_level = pwr->thermal_pwrlevel;
min_level = pwr->thermal_pwrlevel_floor;
/* Thermal limit cannot be lower than lowest non-zero operating freq */
for (level = 0; level < (pwr->num_pwrlevels - 1); level++) {
if (pwr->pwrlevels[level].gpu_freq == max_freq)
max_level = level;
if (pwr->pwrlevels[level].gpu_freq == min_freq)
min_level = level;
}
pwr->thermal_pwrlevel = max_level;
pwr->thermal_pwrlevel_floor = min_level;
/* Update the current level using the new limit */
kgsl_pwrctrl_pwrlevel_change(device, pwr->active_pwrlevel);
mutex_unlock(&device->mutex);
return 0;
}
int kgsl_pwrscale_init(struct kgsl_device *device, struct platform_device *pdev,
const char *governor)
{
struct kgsl_pwrscale *pwrscale;
struct kgsl_pwrctrl *pwr;
struct devfreq *devfreq;
struct devfreq *bus_devfreq = NULL;
struct msm_adreno_extended_profile *gpu_profile;
struct devfreq_dev_profile *profile;
struct devfreq_msm_adreno_tz_data *data;
int i, ret;
pwrscale = &device->pwrscale;
pwr = &device->pwrctrl;
gpu_profile = &pwrscale->gpu_profile;
profile = &pwrscale->gpu_profile.profile;
pwr->nb.notifier_call = opp_notify;
dev_pm_opp_register_notifier(&pdev->dev, &pwr->nb);
srcu_init_notifier_head(&pwrscale->nh);
profile->initial_freq =
pwr->pwrlevels[pwr->default_pwrlevel].gpu_freq;
/* Let's start with 10 ms and tune in later */
profile->polling_ms = 10;
/* do not include the 'off' level or duplicate freq. levels */
for (i = 0; i < (pwr->num_pwrlevels - 1); i++)
pwrscale->freq_table[out++] = pwr->pwrlevels[i].gpu_freq;
/*
* Max_state is the number of valid power levels.
* The valid power levels range from 0 - (max_state - 1)
*/
profile->max_state = pwr->num_pwrlevels - 1;
/* link storage array to the devfreq profile pointer */
profile->freq_table = pwrscale->freq_table;
/* if there is only 1 freq, no point in running a governor */
if (profile->max_state == 1)
governor = "performance";
/* initialize msm-adreno-tz governor specific data here */
data = gpu_profile->private_data;
data->disable_busy_time_burst =
of_property_read_bool(pdev->dev.of_node,
"qcom,disable-busy-time-burst");
if (pwrscale->ctxt_aware_enable) {
data->ctxt_aware_enable = pwrscale->ctxt_aware_enable;
data->bin.ctxt_aware_target_pwrlevel =
pwrscale->ctxt_aware_target_pwrlevel;
data->bin.ctxt_aware_busy_penalty =
pwrscale->ctxt_aware_busy_penalty;
}
if (of_property_read_bool(pdev->dev.of_node,
"qcom,enable-midframe-timer")) {
kgsl_midframe = kzalloc(
sizeof(struct kgsl_midframe_info), GFP_KERNEL);
if (kgsl_midframe) {
hrtimer_init(&kgsl_midframe->timer,
CLOCK_MONOTONIC, HRTIMER_MODE_REL);
kgsl_midframe->timer.function =
kgsl_pwrscale_midframe_timer;
kgsl_midframe->device = device;
} else
dev_err(device->dev,
"Failed to enable-midframe-timer feature\n");
}
/*
* If there is a separate GX power rail, allow
* independent modification to its voltage through
* the bus bandwidth vote.
*/
if (pwr->bus_control) {
data->bus.num = pwr->bus_ibs_count;
data->bus.ib_mbps = pwr->bus_ibs;
data->bus.width = pwr->bus_width;
if (!kgsl_of_property_read_ddrtype(device->pdev->dev.of_node,
"qcom,bus-accesses", &data->bus.max))
data->bus.floating = false;
} else
data->bus.num = 0;
devfreq = devfreq_add_device(&pdev->dev, &pwrscale->gpu_profile.profile,
governor, pwrscale->gpu_profile.private_data);
if (IS_ERR(devfreq)) {
device->pwrscale.enabled = false;
return PTR_ERR(devfreq);
}
pwrscale->devfreqptr = devfreq;
pwrscale->cooling_dev = of_devfreq_cooling_register(pdev->dev.of_node,
devfreq);
if (IS_ERR(pwrscale->cooling_dev))
pwrscale->cooling_dev = NULL;
pwrscale->gpu_profile.bus_devfreq = NULL;
if (data->bus.num) {
pwrscale->bus_profile.profile.max_state
= pwr->num_pwrlevels - 1;
pwrscale->bus_profile.profile.freq_table
= pwrscale->freq_table;
/*
* This is needed because devfreq expects the device
* to have an opp table handle to calculate the min/max
* frequency.
*/
ret = dev_pm_opp_of_add_table(device->busmondev);
/*
* Disable OPP which are not supported as per GPU freq plan.
* This is need to ensure freq_table specified in bus_profile
* above matches OPP table.
*/
kgsl_pwrctrl_disable_unused_opp(device, device->busmondev);
if (!ret)
bus_devfreq = devfreq_add_device(device->busmondev,
&pwrscale->bus_profile.profile, "gpubw_mon",
NULL);
if (IS_ERR_OR_NULL(bus_devfreq))
dev_err(device->dev, "Bus scaling not enabled\n");
else
pwrscale->gpu_profile.bus_devfreq = bus_devfreq;
}
ret = sysfs_create_link(&device->dev->kobj,
&devfreq->dev.kobj, "devfreq");
pwrscale->devfreq_wq = create_freezable_workqueue("kgsl_devfreq_wq");
INIT_WORK(&pwrscale->devfreq_suspend_ws, do_devfreq_suspend);
INIT_WORK(&pwrscale->devfreq_resume_ws, do_devfreq_resume);
INIT_WORK(&pwrscale->devfreq_notify_ws, do_devfreq_notify);
if (kgsl_midframe)
INIT_WORK(&kgsl_midframe->timer_check_ws,
kgsl_pwrscale_midframe_timer_check);
pwrscale->next_governor_call = ktime_add_us(ktime_get(),
KGSL_GOVERNOR_CALL_INTERVAL);
/* Add links to the devfreq sysfs nodes */
kgsl_gpu_sysfs_add_link(device->gpu_sysfs_kobj,
&pwrscale->devfreqptr->dev.kobj, "governor",
"gpu_governor");
kgsl_gpu_sysfs_add_link(device->gpu_sysfs_kobj,
&pwrscale->devfreqptr->dev.kobj,
"available_governors", "gpu_available_governor");
return 0;
}
/*
* kgsl_pwrscale_close - clean up pwrscale
* @device: the device
*
* This function should be called with the device mutex locked.
*/
void kgsl_pwrscale_close(struct kgsl_device *device)
{
struct kgsl_pwrscale *pwrscale;
struct kgsl_pwrctrl *pwr;
pwr = &device->pwrctrl;
pwrscale = &device->pwrscale;
if (!pwrscale->devfreqptr)
return;
if (pwrscale->cooling_dev)
devfreq_cooling_unregister(pwrscale->cooling_dev);
kgsl_pwrscale_midframe_timer_cancel(device);
flush_workqueue(pwrscale->devfreq_wq);
destroy_workqueue(pwrscale->devfreq_wq);
devfreq_remove_device(device->pwrscale.devfreqptr);
kfree(kgsl_midframe);
kgsl_midframe = NULL;
device->pwrscale.devfreqptr = NULL;
srcu_cleanup_notifier_head(&device->pwrscale.nh);
dev_pm_opp_unregister_notifier(&device->pdev->dev, &pwr->nb);
}
static void do_devfreq_suspend(struct work_struct *work)
{
struct kgsl_pwrscale *pwrscale = container_of(work,
struct kgsl_pwrscale, devfreq_suspend_ws);
struct devfreq *devfreq = pwrscale->devfreqptr;
devfreq_suspend_device(devfreq);
}
static void do_devfreq_resume(struct work_struct *work)
{
struct kgsl_pwrscale *pwrscale = container_of(work,
struct kgsl_pwrscale, devfreq_resume_ws);
struct devfreq *devfreq = pwrscale->devfreqptr;
devfreq_resume_device(devfreq);
}
static void do_devfreq_notify(struct work_struct *work)
{
struct kgsl_pwrscale *pwrscale = container_of(work,
struct kgsl_pwrscale, devfreq_notify_ws);
struct devfreq *devfreq = pwrscale->devfreqptr;
srcu_notifier_call_chain(&pwrscale->nh,
ADRENO_DEVFREQ_NOTIFY_RETIRE,
devfreq);
}

118
drivers/gpu/msm/kgsl_pwrscale.h Normal file
View file

@ -0,0 +1,118 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2010-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_PWRSCALE_H
#define __KGSL_PWRSCALE_H
#include "kgsl_pwrctrl.h"
#include "msm_adreno_devfreq.h"
/* devfreq governor call window in usec */
#define KGSL_GOVERNOR_CALL_INTERVAL 10000
struct kgsl_power_stats {
u64 busy_time;
u64 ram_time;
u64 ram_wait;
};
/**
* struct kgsl_pwrscale - Power scaling settings for a KGSL device
* @devfreqptr - Pointer to the devfreq device
* @gpu_profile - GPU profile data for the devfreq device
* @bus_profile - Bus specific data for the bus devfreq device
* @freq_table - GPU frequencies for the DCVS algorithm
* @last_governor - Prior devfreq governor
* @accum_stats - Accumulated statistics for various frequency calculations
* @enabled - Whether or not power scaling is enabled
* @time - Last submitted sample timestamp
* @on_time - Timestamp when gpu busy begins
* @nh - Notifier for the partner devfreq bus device
* @devfreq_wq - Main devfreq workqueue
* @devfreq_suspend_ws - Pass device suspension to devfreq
* @devfreq_resume_ws - Pass device resume to devfreq
* @devfreq_notify_ws - Notify devfreq to update sampling
* @next_governor_call - Timestamp after which the governor may be notified of
* a new sample
* @cooling_dev - Thermal cooling device handle
* @ctxt_aware_enable - Whether or not ctxt aware DCVS feature is enabled
* @ctxt_aware_busy_penalty - The time in microseconds required to trigger
* ctxt aware power level jump
* @ctxt_aware_target_pwrlevel - pwrlevel to jump on in case of ctxt aware
* power level jump
*/
struct kgsl_pwrscale {
struct devfreq *devfreqptr;
struct msm_adreno_extended_profile gpu_profile;
struct msm_busmon_extended_profile bus_profile;
unsigned long freq_table[KGSL_MAX_PWRLEVELS];
char last_governor[DEVFREQ_NAME_LEN];
struct kgsl_power_stats accum_stats;
bool enabled;
ktime_t time;
s64 on_time;
struct srcu_notifier_head nh;
struct workqueue_struct *devfreq_wq;
struct work_struct devfreq_suspend_ws;
struct work_struct devfreq_resume_ws;
struct work_struct devfreq_notify_ws;
ktime_t next_governor_call;
struct thermal_cooling_device *cooling_dev;
bool ctxt_aware_enable;
unsigned int ctxt_aware_target_pwrlevel;
unsigned int ctxt_aware_busy_penalty;
};
/**
* kgsl_pwrscale_init - Initialize the pwrscale subsystem
* @device: A GPU device handle
* @pdev: A pointer to the GPU platform device
* @governor: default devfreq governor to use for GPU frequency scaling
*
* Return: 0 on success or negative on failure
*/
int kgsl_pwrscale_init(struct kgsl_device *device, struct platform_device *pdev,
const char *governor);
void kgsl_pwrscale_close(struct kgsl_device *device);
void kgsl_pwrscale_update(struct kgsl_device *device);
void kgsl_pwrscale_update_stats(struct kgsl_device *device);
void kgsl_pwrscale_busy(struct kgsl_device *device);
void kgsl_pwrscale_sleep(struct kgsl_device *device);
void kgsl_pwrscale_wake(struct kgsl_device *device);
void kgsl_pwrscale_midframe_timer_restart(struct kgsl_device *device);
void kgsl_pwrscale_midframe_timer_cancel(struct kgsl_device *device);
void kgsl_pwrscale_enable(struct kgsl_device *device);
void kgsl_pwrscale_disable(struct kgsl_device *device, bool turbo);
int kgsl_devfreq_target(struct device *dev, unsigned long *freq, u32 flags);
int kgsl_devfreq_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat);
int kgsl_devfreq_get_cur_freq(struct device *dev, unsigned long *freq);
int kgsl_busmon_target(struct device *dev, unsigned long *freq, u32 flags);
int kgsl_busmon_get_dev_status(struct device *dev,
struct devfreq_dev_status *stat);
int kgsl_busmon_get_cur_freq(struct device *dev, unsigned long *freq);
#define KGSL_PWRSCALE_INIT(_priv_data) { \
.enabled = true, \
.gpu_profile = { \
.private_data = _priv_data, \
.profile = { \
.target = kgsl_devfreq_target, \
.get_dev_status = kgsl_devfreq_get_dev_status, \
.get_cur_freq = kgsl_devfreq_get_cur_freq, \
} }, \
.bus_profile = { \
.private_data = _priv_data, \
.profile = { \
.target = kgsl_busmon_target, \
.get_dev_status = kgsl_busmon_get_dev_status, \
.get_cur_freq = kgsl_busmon_get_cur_freq, \
} } }
#endif

480
drivers/gpu/msm/kgsl_rgmu.c Normal file
View file

@ -0,0 +1,480 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/clk-provider.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/regulator/consumer.h>
#include "adreno.h"
#include "kgsl_device.h"
#include "kgsl_rgmu.h"
#define RGMU_CLK_FREQ 200000000
static int rgmu_irq_probe(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
int ret;
rgmu->oob_interrupt_num = platform_get_irq_byname(rgmu->pdev,
"kgsl_oob");
ret = devm_request_irq(&rgmu->pdev->dev,
rgmu->oob_interrupt_num,
oob_irq_handler, IRQF_TRIGGER_HIGH,
"kgsl-oob", device);
if (ret) {
dev_err(&rgmu->pdev->dev,
"Request kgsl-oob interrupt failed:%d\n", ret);
return ret;
}
rgmu->rgmu_interrupt_num = platform_get_irq_byname(rgmu->pdev,
"kgsl_rgmu");
ret = devm_request_irq(&rgmu->pdev->dev,
rgmu->rgmu_interrupt_num,
rgmu_irq_handler, IRQF_TRIGGER_HIGH,
"kgsl-rgmu", device);
if (ret)
dev_err(&rgmu->pdev->dev,
"Request kgsl-rgmu interrupt failed:%d\n", ret);
return ret;
}
static int rgmu_regulators_probe(struct rgmu_device *rgmu,
struct device_node *node)
{
int ret;
rgmu->cx_gdsc = devm_regulator_get(&rgmu->pdev->dev, "vddcx");
if (IS_ERR_OR_NULL(rgmu->cx_gdsc)) {
ret = PTR_ERR(rgmu->cx_gdsc);
dev_err(&rgmu->pdev->dev,
"Couldn't get CX gdsc error:%d\n", ret);
rgmu->cx_gdsc = NULL;
return ret;
}
rgmu->gx_gdsc = devm_regulator_get(&rgmu->pdev->dev, "vdd");
if (IS_ERR_OR_NULL(rgmu->gx_gdsc)) {
ret = PTR_ERR(rgmu->gx_gdsc);
dev_err(&rgmu->pdev->dev,
"Couldn't get GX gdsc error:%d\n", ret);
rgmu->gx_gdsc = NULL;
return ret;
}
return 0;
}
static int rgmu_clocks_probe(struct rgmu_device *rgmu, struct device_node *node)
{
const char *cname;
struct property *prop;
struct clk *c;
int i = 0;
of_property_for_each_string(node, "clock-names", prop, cname) {
if (i >= ARRAY_SIZE(rgmu->clks)) {
dev_err(&rgmu->pdev->dev,
"dt: too many RGMU clocks defined\n");
return -EINVAL;
}
c = devm_clk_get(&rgmu->pdev->dev, cname);
if (IS_ERR_OR_NULL(c)) {
dev_err(&rgmu->pdev->dev,
"dt: Couldn't get clock: %s\n", cname);
return PTR_ERR(c);
}
/* Remember the key clocks that we need to control later */
if (!strcmp(cname, "core"))
rgmu->gpu_clk = c;
else if (!strcmp(cname, "gmu"))
rgmu->rgmu_clk = c;
rgmu->clks[i++] = c;
}
return 0;
}
static inline int rgmu_clk_set_rate(struct clk *grp_clk, unsigned int freq)
{
int ret = clk_set_rate(grp_clk, freq);
if (ret)
pr_err("%s set freq %d failed:%d\n",
__clk_get_name(grp_clk), freq, ret);
return ret;
}
static void rgmu_disable_clks(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device);
int j = 0, ret;
/* Check GX GDSC is status */
if (gmu_dev_ops->gx_is_on(device)) {
if (IS_ERR_OR_NULL(rgmu->gx_gdsc))
return;
/*
* Switch gx gdsc control from RGMU to CPU. Force non-zero
* reference count in clk driver so next disable call will
* turn off the GDSC.
*/
ret = regulator_enable(rgmu->gx_gdsc);
if (ret)
dev_err(&rgmu->pdev->dev,
"Fail to enable gx gdsc:%d\n", ret);
ret = regulator_disable(rgmu->gx_gdsc);
if (ret)
dev_err(&rgmu->pdev->dev,
"Fail to disable gx gdsc:%d\n", ret);
if (gmu_dev_ops->gx_is_on(device))
dev_err(&rgmu->pdev->dev, "gx is stuck on\n");
}
for (j = 0; j < ARRAY_SIZE(rgmu->clks); j++)
clk_disable_unprepare(rgmu->clks[j]);
clear_bit(GMU_CLK_ON, &device->gmu_core.flags);
}
static int rgmu_enable_clks(struct kgsl_device *device)
{
int ret, j = 0;
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
if (IS_ERR_OR_NULL(rgmu->rgmu_clk) ||
IS_ERR_OR_NULL(rgmu->gpu_clk))
return -EINVAL;
/* Let us set rgmu clk */
ret = rgmu_clk_set_rate(rgmu->rgmu_clk, RGMU_CLK_FREQ);
if (ret)
return ret;
/* Let us set gpu clk to default power level */
ret = rgmu_clk_set_rate(rgmu->gpu_clk,
rgmu->gpu_freqs[pwr->default_pwrlevel]);
if (ret)
return ret;
for (j = 0; j < ARRAY_SIZE(rgmu->clks); j++) {
ret = clk_prepare_enable(rgmu->clks[j]);
if (ret) {
dev_err(&rgmu->pdev->dev,
"Fail(%d) to enable gpucc clk idx %d\n",
ret, j);
return ret;
}
}
set_bit(GMU_CLK_ON, &device->gmu_core.flags);
return 0;
}
#define CX_GDSC_TIMEOUT 5000 /* ms */
static void rgmu_disable_gdsc(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
int ret = 0;
unsigned long t;
if (IS_ERR_OR_NULL(rgmu->cx_gdsc))
return;
ret = regulator_disable(rgmu->cx_gdsc);
if (ret) {
dev_err(&rgmu->pdev->dev,
"Failed to disable CX gdsc:%d\n", ret);
return;
}
/*
* After GX GDSC is off, CX GDSC must be off.
* Voting off alone from GPU driver cannot
* guarantee CX GDSC off. Polling with 5sec
* timeout to ensure CX GDSC is off.
*/
t = jiffies + msecs_to_jiffies(CX_GDSC_TIMEOUT);
do {
if (!regulator_is_enabled(rgmu->cx_gdsc))
return;
usleep_range(10, 100);
} while (!(time_after(jiffies, t)));
if (regulator_is_enabled(rgmu->cx_gdsc))
dev_err(&rgmu->pdev->dev, "RGMU CX gdsc off timeout\n");
}
static int rgmu_enable_gdsc(struct rgmu_device *rgmu)
{
int ret;
if (IS_ERR_OR_NULL(rgmu->cx_gdsc))
return 0;
ret = regulator_enable(rgmu->cx_gdsc);
if (ret)
dev_err(&rgmu->pdev->dev,
"Fail to enable CX gdsc:%d\n", ret);
return ret;
}
static void rgmu_snapshot(struct kgsl_device *device)
{
struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device);
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
/* Mask so there's no interrupt caused by NMI */
adreno_write_gmureg(adreno_dev,
ADRENO_REG_GMU_GMU2HOST_INTR_MASK, 0xFFFFFFFF);
/* Make sure the interrupt is masked */
wmb();
/*
* Halt RGMU execution so that GX will not
* be collapsed while dumping snapshot.
*/
gmu_dev_ops->halt_execution(device);
kgsl_device_snapshot(device, NULL, true);
adreno_write_gmureg(adreno_dev,
ADRENO_REG_GMU_GMU2HOST_INTR_CLR, 0xFFFFFFFF);
adreno_write_gmureg(adreno_dev,
ADRENO_REG_GMU_GMU2HOST_INTR_MASK,
~(gmu_dev_ops->gmu2host_intr_mask));
rgmu->fault_count++;
}
/* Caller shall ensure GPU is ready for SLUMBER */
static void rgmu_stop(struct kgsl_device *device)
{
struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device);
if (!test_bit(GMU_CLK_ON, &device->gmu_core.flags))
return;
/* Wait for the lowest idle level we requested */
if (gmu_dev_ops->wait_for_lowest_idle(device))
goto error;
gmu_dev_ops->rpmh_gpu_pwrctrl(device,
GMU_NOTIFY_SLUMBER, 0, 0);
gmu_dev_ops->irq_disable(device);
rgmu_disable_clks(device);
rgmu_disable_gdsc(device);
return;
error:
/*
* The power controller will change state to SLUMBER anyway
* Set GMU_FAULT flag to indicate to power contrller
* that hang recovery is needed to power on GPU
*/
set_bit(GMU_FAULT, &device->gmu_core.flags);
rgmu_snapshot(device);
}
/* Do not access any RGMU registers in RGMU probe function */
static int rgmu_probe(struct kgsl_device *device, struct device_node *node)
{
struct rgmu_device *rgmu;
struct platform_device *pdev = of_find_device_by_node(node);
struct kgsl_pwrctrl *pwr = &device->pwrctrl;
struct resource *res;
int i, ret = -ENXIO;
rgmu = devm_kzalloc(&pdev->dev, sizeof(*rgmu), GFP_KERNEL);
if (rgmu == NULL)
return -ENOMEM;
rgmu->pdev = pdev;
/* Set up RGMU regulators */
ret = rgmu_regulators_probe(rgmu, node);
if (ret)
return ret;
/* Set up RGMU clocks */
ret = rgmu_clocks_probe(rgmu, node);
if (ret)
return ret;
/* Map and reserve RGMU CSRs registers */
res = platform_get_resource_byname(rgmu->pdev,
IORESOURCE_MEM, "kgsl_rgmu");
if (res == NULL) {
dev_err(&rgmu->pdev->dev,
"platform_get_resource failed\n");
return -EINVAL;
}
if (res->start == 0 || resource_size(res) == 0) {
dev_err(&rgmu->pdev->dev,
"Register region is invalid\n");
return -EINVAL;
}
rgmu->reg_phys = res->start;
rgmu->reg_len = resource_size(res);
device->gmu_core.reg_virt = devm_ioremap(&rgmu->pdev->dev, res->start,
resource_size(res));
if (device->gmu_core.reg_virt == NULL) {
dev_err(&rgmu->pdev->dev, "Unable to remap rgmu registers\n");
return -ENODEV;
}
device->gmu_core.gmu2gpu_offset =
(rgmu->reg_phys - device->reg_phys) >> 2;
device->gmu_core.reg_len = rgmu->reg_len;
device->gmu_core.ptr = (void *)rgmu;
/* Initialize OOB and RGMU interrupts */
ret = rgmu_irq_probe(device);
if (ret)
return ret;
/* Don't enable RGMU interrupts until RGMU started */
/* We cannot use rgmu_irq_disable because it writes registers */
disable_irq(rgmu->rgmu_interrupt_num);
disable_irq(rgmu->oob_interrupt_num);
/* Retrieves GPU power level configurations */
for (i = 0; i < pwr->num_pwrlevels; i++)
rgmu->gpu_freqs[i] = pwr->pwrlevels[i].gpu_freq;
rgmu->num_gpupwrlevels = pwr->num_pwrlevels;
/* Set up RGMU idle states */
if (ADRENO_FEATURE(ADRENO_DEVICE(device), ADRENO_IFPC))
rgmu->idle_level = GPU_HW_IFPC;
else
rgmu->idle_level = GPU_HW_ACTIVE;
set_bit(GMU_ENABLED, &device->gmu_core.flags);
device->gmu_core.dev_ops = &adreno_a6xx_rgmudev;
return 0;
}
static int rgmu_suspend(struct kgsl_device *device)
{
struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device);
if (!test_bit(GMU_CLK_ON, &device->gmu_core.flags))
return 0;
gmu_dev_ops->irq_disable(device);
if (gmu_dev_ops->rpmh_gpu_pwrctrl(device, GMU_SUSPEND, 0, 0))
return -EINVAL;
rgmu_disable_clks(device);
rgmu_disable_gdsc(device);
return 0;
}
/* To be called to power on both GPU and RGMU */
static int rgmu_start(struct kgsl_device *device)
{
int ret = 0;
struct gmu_dev_ops *gmu_dev_ops = GMU_DEVICE_OPS(device);
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
switch (device->state) {
case KGSL_STATE_RESET:
ret = rgmu_suspend(device);
if (ret)
goto error_rgmu;
/* Fall-thru */
case KGSL_STATE_INIT:
case KGSL_STATE_SUSPEND:
case KGSL_STATE_SLUMBER:
rgmu_enable_gdsc(rgmu);
rgmu_enable_clks(device);
gmu_dev_ops->irq_enable(device);
ret = gmu_dev_ops->rpmh_gpu_pwrctrl(device,
GMU_FW_START, GMU_COLD_BOOT, 0);
if (ret)
goto error_rgmu;
break;
}
/* Request default DCVS level */
kgsl_pwrctrl_set_default_gpu_pwrlevel(device);
return 0;
error_rgmu:
set_bit(GMU_FAULT, &device->gmu_core.flags);
rgmu_snapshot(device);
return ret;
}
/*
* rgmu_dcvs_set() - Change GPU frequency and/or bandwidth.
* @rgmu: Pointer to RGMU device
* @pwrlevel: index to GPU DCVS table used by KGSL
* @bus_level: index to GPU bus table used by KGSL
*
* The function converts GPU power level and bus level index used by KGSL
* to index being used by GMU/RPMh.
*/
static int rgmu_dcvs_set(struct kgsl_device *device,
unsigned int pwrlevel, unsigned int bus_level)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
if (pwrlevel == INVALID_DCVS_IDX)
return -EINVAL;
return rgmu_clk_set_rate(rgmu->gpu_clk,
rgmu->gpu_freqs[pwrlevel]);
}
static bool rgmu_regulator_isenabled(struct kgsl_device *device)
{
struct rgmu_device *rgmu = KGSL_RGMU_DEVICE(device);
return (rgmu->gx_gdsc && regulator_is_enabled(rgmu->gx_gdsc));
}
struct gmu_core_ops rgmu_ops = {
.probe = rgmu_probe,
.remove = rgmu_stop,
.start = rgmu_start,
.stop = rgmu_stop,
.dcvs_set = rgmu_dcvs_set,
.snapshot = rgmu_snapshot,
.regulator_isenabled = rgmu_regulator_isenabled,
.suspend = rgmu_suspend,
};

64
drivers/gpu/msm/kgsl_rgmu.h Normal file
View file

@ -0,0 +1,64 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_RGMU_H
#define __KGSL_RGMU_H
#define RGMU_AO_IRQ_FENCE_ERR BIT(3)
#define RGMU_AO_IRQ_MASK RGMU_AO_IRQ_FENCE_ERR
#define RGMU_OOB_IRQ_ERR_MSG BIT(24)
#define RGMU_OOB_IRQ_ACK_MASK GENMASK(23, 16)
#define RGMU_OOB_IRQ_ERR_MSG_MASK GENMASK(31, 24)
#define RGMU_OOB_IRQ_MASK RGMU_OOB_IRQ_ERR_MSG_MASK
#define MAX_RGMU_CLKS 8
/**
* struct rgmu_device - rGMU device structure
* @ver: RGMU firmware version
* @reg_phys: RGMU CSR physical address
* @reg_virt: RGMU CSR virtual address
* @reg_len: RGMU CSR range
* @rgmu_interrupt_num: RGMU interrupt number
* @oob_interrupt_num: number of RGMU asserted OOB interrupt
* @fw_hostptr: Buffer which holds the RGMU firmware
* @fw_size: Size of RGMU firmware buffer
* @cx_gdsc: CX headswitch that controls power of RGMU and
subsystem peripherals
* @clks: RGMU clocks including the GPU
* @gpu_clk: Pointer to GPU core clock
* @rgmu_clk: Pointer to rgmu clock
* @gpu_freqs: GPU frequency table with lowest freq at index 0
* @num_gpupwrlevels: number GPU frequencies in GPU freq table
* @flags: RGMU flags
* @idle_level: Minimal GPU idle power level
* @fault_count: RGMU fault count
*/
struct rgmu_device {
u32 ver;
struct platform_device *pdev;
unsigned long reg_phys;
unsigned int reg_len;
unsigned int rgmu_interrupt_num;
unsigned int oob_interrupt_num;
unsigned int *fw_hostptr;
uint32_t fw_size;
struct regulator *cx_gdsc;
struct regulator *gx_gdsc;
struct clk *clks[MAX_RGMU_CLKS];
struct clk *gpu_clk;
struct clk *rgmu_clk;
unsigned int gpu_freqs[MAX_GX_LEVELS];
unsigned int num_gpupwrlevels;
unsigned int idle_level;
unsigned int fault_count;
};
extern struct gmu_dev_ops adreno_a6xx_rgmudev;
#define KGSL_RGMU_DEVICE(_a) ((struct rgmu_device *)((_a)->gmu_core.ptr))
irqreturn_t rgmu_irq_handler(int irq, void *data);
irqreturn_t oob_irq_handler(int irq, void *data);
#endif /* __KGSL_RGMU_H */

1320
drivers/gpu/msm/kgsl_sharedmem.c Normal file
View file

File diff suppressed because it is too large Load diff

352
drivers/gpu/msm/kgsl_sharedmem.h Normal file
View file

@ -0,0 +1,352 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2002,2007-2019, The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_SHAREDMEM_H
#define __KGSL_SHAREDMEM_H
#include <linux/dma-mapping.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include "kgsl.h"
#include "kgsl_mmu.h"
struct kgsl_device;
struct kgsl_process_private;
#define KGSL_CACHE_OP_INV 0x01
#define KGSL_CACHE_OP_FLUSH 0x02
#define KGSL_CACHE_OP_CLEAN 0x03
int kgsl_sharedmem_alloc_contig(struct kgsl_device *device,
struct kgsl_memdesc *memdesc,
uint64_t size);
void kgsl_sharedmem_free(struct kgsl_memdesc *memdesc);
int kgsl_sharedmem_readl(const struct kgsl_memdesc *memdesc,
uint32_t *dst,
uint64_t offsetbytes);
int kgsl_sharedmem_writel(struct kgsl_device *device,
const struct kgsl_memdesc *memdesc,
uint64_t offsetbytes,
uint32_t src);
int kgsl_sharedmem_readq(const struct kgsl_memdesc *memdesc,
uint64_t *dst,
uint64_t offsetbytes);
int kgsl_sharedmem_writeq(struct kgsl_device *device,
const struct kgsl_memdesc *memdesc,
uint64_t offsetbytes,
uint64_t src);
int kgsl_sharedmem_set(struct kgsl_device *device,
const struct kgsl_memdesc *memdesc,
uint64_t offsetbytes, unsigned int value,
uint64_t sizebytes);
int kgsl_cache_range_op(struct kgsl_memdesc *memdesc,
uint64_t offset, uint64_t size,
unsigned int op);
void kgsl_memdesc_init(struct kgsl_device *device,
struct kgsl_memdesc *memdesc, uint64_t flags);
void kgsl_process_init_sysfs(struct kgsl_device *device,
struct kgsl_process_private *private);
void kgsl_process_uninit_sysfs(struct kgsl_process_private *private);
int kgsl_sharedmem_init_sysfs(void);
void kgsl_sharedmem_uninit_sysfs(void);
int kgsl_allocate_user(struct kgsl_device *device,
struct kgsl_memdesc *memdesc,
uint64_t size, uint64_t flags);
void kgsl_get_memory_usage(char *str, size_t len, uint64_t memflags);
int kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
uint64_t size);
void kgsl_free_secure_page(struct page *page);
struct page *kgsl_alloc_secure_page(void);
#define MEMFLAGS(_flags, _mask, _shift) \
((unsigned int) (((_flags) & (_mask)) >> (_shift)))
/*
* kgsl_memdesc_get_align - Get alignment flags from a memdesc
* @memdesc - the memdesc
*
* Returns the alignment requested, as power of 2 exponent.
*/
static inline int
kgsl_memdesc_get_align(const struct kgsl_memdesc *memdesc)
{
return MEMFLAGS(memdesc->flags, KGSL_MEMALIGN_MASK,
KGSL_MEMALIGN_SHIFT);
}
/*
* kgsl_memdesc_get_pagesize - Get pagesize based on alignment
* @memdesc - the memdesc
*
* Returns the pagesize based on memdesc alignment
*/
static inline int
kgsl_memdesc_get_pagesize(const struct kgsl_memdesc *memdesc)
{
return (1 << kgsl_memdesc_get_align(memdesc));
}
/*
* kgsl_memdesc_get_cachemode - Get cache mode of a memdesc
* @memdesc: the memdesc
*
* Returns a KGSL_CACHEMODE* value.
*/
static inline int
kgsl_memdesc_get_cachemode(const struct kgsl_memdesc *memdesc)
{
return MEMFLAGS(memdesc->flags, KGSL_CACHEMODE_MASK,
KGSL_CACHEMODE_SHIFT);
}
static inline unsigned int
kgsl_memdesc_get_memtype(const struct kgsl_memdesc *memdesc)
{
return MEMFLAGS(memdesc->flags, KGSL_MEMTYPE_MASK,
KGSL_MEMTYPE_SHIFT);
}
/*
* kgsl_memdesc_set_align - Set alignment flags of a memdesc
* @memdesc - the memdesc
* @align - alignment requested, as a power of 2 exponent.
*/
static inline int
kgsl_memdesc_set_align(struct kgsl_memdesc *memdesc, unsigned int align)
{
if (align > 32)
align = 32;
memdesc->flags &= ~(uint64_t)KGSL_MEMALIGN_MASK;
memdesc->flags |= (uint64_t)((align << KGSL_MEMALIGN_SHIFT) &
KGSL_MEMALIGN_MASK);
return 0;
}
/**
* kgsl_memdesc_usermem_type - return buffer type
* @memdesc - the memdesc
*
* Returns a KGSL_MEM_ENTRY_* value for this buffer, which
* identifies if was allocated by us, or imported from
* another allocator.
*/
static inline unsigned int
kgsl_memdesc_usermem_type(const struct kgsl_memdesc *memdesc)
{
return MEMFLAGS(memdesc->flags, KGSL_MEMFLAGS_USERMEM_MASK,
KGSL_MEMFLAGS_USERMEM_SHIFT);
}
/**
* kgsl_memdesc_sg_dma - Turn a dma_addr (from CMA) into a sg table
* @memdesc: Pointer to a memory descriptor
* @addr: Physical address from the dma_alloc function
* @size: Size of the chunk
*
* Create a sg table for the contiguous chunk specified by addr and size.
*
* Return: 0 on success or negative on failure.
*/
int kgsl_memdesc_sg_dma(struct kgsl_memdesc *memdesc,
phys_addr_t addr, u64 size);
/*
* kgsl_memdesc_is_global - is this a globally mapped buffer?
* @memdesc: the memdesc
*
* Returns nonzero if this is a global mapping, 0 otherwise
*/
static inline int kgsl_memdesc_is_global(const struct kgsl_memdesc *memdesc)
{
return (memdesc->priv & KGSL_MEMDESC_GLOBAL) != 0;
}
/*
* kgsl_memdesc_is_secured - is this a secure buffer?
* @memdesc: the memdesc
*
* Returns true if this is a secure mapping, false otherwise
*/
static inline bool kgsl_memdesc_is_secured(const struct kgsl_memdesc *memdesc)
{
return memdesc && (memdesc->priv & KGSL_MEMDESC_SECURE);
}
/*
* kgsl_memdesc_has_guard_page - is the last page a guard page?
* @memdesc - the memdesc
*
* Returns nonzero if there is a guard page, 0 otherwise
*/
static inline int
kgsl_memdesc_has_guard_page(const struct kgsl_memdesc *memdesc)
{
return (memdesc->priv & KGSL_MEMDESC_GUARD_PAGE) != 0;
}
/*
* kgsl_memdesc_guard_page_size - returns guard page size
* @memdesc - the memdesc
*
* Returns guard page size
*/
static inline uint64_t
kgsl_memdesc_guard_page_size(const struct kgsl_memdesc *memdesc)
{
if (!kgsl_memdesc_has_guard_page(memdesc))
return 0;
return PAGE_SIZE;
}
/*
* kgsl_memdesc_use_cpu_map - use the same virtual mapping on CPU and GPU?
* @memdesc - the memdesc
*/
static inline int
kgsl_memdesc_use_cpu_map(const struct kgsl_memdesc *memdesc)
{
return (memdesc->flags & KGSL_MEMFLAGS_USE_CPU_MAP) != 0;
}
/*
* kgsl_memdesc_footprint - get the size of the mmap region
* @memdesc - the memdesc
*
* The entire memdesc must be mapped. Additionally if the
* CPU mapping is going to be mirrored, there must be room
* for the guard page to be mapped so that the address spaces
* match up.
*/
static inline uint64_t
kgsl_memdesc_footprint(const struct kgsl_memdesc *memdesc)
{
return ALIGN(memdesc->size + kgsl_memdesc_guard_page_size(memdesc),
PAGE_SIZE);
}
/*
* kgsl_allocate_global() - Allocate GPU accessible memory that will be global
* across all processes
* @device: The device pointer to which the memdesc belongs
* @memdesc: Pointer to a KGSL memory descriptor for the memory allocation
* @size: size of the allocation
* @flags: Allocation flags that control how the memory is mapped
* @priv: Priv flags that controls memory attributes
*
* Allocate contiguous memory for internal use and add the allocation to the
* list of global pagetable entries that will be mapped at the same address in
* all pagetables. This is for use for device wide GPU allocations such as
* ringbuffers.
*/
int kgsl_allocate_global(struct kgsl_device *device,
struct kgsl_memdesc *memdesc, uint64_t size, uint64_t flags,
unsigned int priv, const char *name);
/**
* kgsl_free_global() - Free a device wide GPU allocation and remove it from the
* global pagetable entry list
*
* @device: Pointer to the device
* @memdesc: Pointer to the GPU memory descriptor to free
*
* Remove the specific memory descriptor from the global pagetable entry list
* and free it
*/
void kgsl_free_global(struct kgsl_device *device, struct kgsl_memdesc *memdesc);
void kgsl_sharedmem_set_noretry(bool val);
bool kgsl_sharedmem_get_noretry(void);
/**
* kgsl_alloc_sgt_from_pages() - Allocate a sg table
*
* @memdesc: memory descriptor of the allocation
*
* Allocate and return pointer to a sg table
*/
static inline struct sg_table *kgsl_alloc_sgt_from_pages(
struct kgsl_memdesc *m)
{
int ret;
struct sg_table *sgt;
sgt = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
if (sgt == NULL)
return ERR_PTR(-ENOMEM);
ret = sg_alloc_table_from_pages(sgt, m->pages, m->page_count, 0,
m->size, GFP_KERNEL);
if (ret) {
kfree(sgt);
return ERR_PTR(ret);
}
return sgt;
}
/**
* kgsl_free_sgt() - Free a sg table structure
*
* @sgt: sg table pointer to be freed
*
* Free the sg table allocated using sgt and free the
* sgt structure itself
*/
static inline void kgsl_free_sgt(struct sg_table *sgt)
{
if (sgt != NULL) {
sg_free_table(sgt);
kfree(sgt);
}
}
#include "kgsl_pool.h"
/**
* kgsl_get_page_size() - Get supported pagesize
* @size: Size of the page
* @align: Desired alignment of the size
*
* Return supported pagesize
*/
#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS
static inline int kgsl_get_page_size(size_t size, unsigned int align)
{
if (align >= ilog2(SZ_1M) && size >= SZ_1M &&
kgsl_pool_avaialable(SZ_1M))
return SZ_1M;
else if (align >= ilog2(SZ_64K) && size >= SZ_64K &&
kgsl_pool_avaialable(SZ_64K))
return SZ_64K;
else if (align >= ilog2(SZ_8K) && size >= SZ_8K &&
kgsl_pool_avaialable(SZ_8K))
return SZ_8K;
else
return PAGE_SIZE;
}
#else
static inline int kgsl_get_page_size(size_t size, unsigned int align)
{
return PAGE_SIZE;
}
#endif
#endif /* __KGSL_SHAREDMEM_H */

1328
drivers/gpu/msm/kgsl_snapshot.c Normal file
View file

File diff suppressed because it is too large Load diff

288
drivers/gpu/msm/kgsl_snapshot.h Normal file
View file

@ -0,0 +1,288 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
*/
#ifndef _KGSL_SNAPSHOT_H_
#define _KGSL_SNAPSHOT_H_
#include <linux/types.h>
/* Snapshot header */
/* High word is static, low word is snapshot version ID */
#define SNAPSHOT_MAGIC 0x504D0002
/* GPU ID scheme:
* [16:31] - core identifer (0x0002 for 2D or 0x0003 for 3D)
* [00:16] - GPU specific identifier
*/
struct kgsl_snapshot_header {
__u32 magic; /* Magic identifier */
__u32 gpuid; /* GPU ID - see above */
/* Added in snapshot version 2 */
__u32 chipid; /* Chip ID from the GPU */
} __packed;
/* Section header */
#define SNAPSHOT_SECTION_MAGIC 0xABCD
struct kgsl_snapshot_section_header {
__u16 magic; /* Magic identifier */
__u16 id; /* Type of section */
__u32 size; /* Size of the section including this header */
} __packed;
/* Section identifiers */
#define KGSL_SNAPSHOT_SECTION_OS 0x0101
#define KGSL_SNAPSHOT_SECTION_REGS 0x0201
#define KGSL_SNAPSHOT_SECTION_RB 0x0301
#define KGSL_SNAPSHOT_SECTION_RB_V2 0x0302
#define KGSL_SNAPSHOT_SECTION_IB 0x0401
#define KGSL_SNAPSHOT_SECTION_IB_V2 0x0402
#define KGSL_SNAPSHOT_SECTION_INDEXED_REGS 0x0501
#define KGSL_SNAPSHOT_SECTION_ISTORE 0x0801
#define KGSL_SNAPSHOT_SECTION_DEBUG 0x0901
#define KGSL_SNAPSHOT_SECTION_DEBUGBUS 0x0A01
#define KGSL_SNAPSHOT_SECTION_GPU_OBJECT 0x0B01
#define KGSL_SNAPSHOT_SECTION_GPU_OBJECT_V2 0x0B02
#define KGSL_SNAPSHOT_SECTION_MEMLIST 0x0E01
#define KGSL_SNAPSHOT_SECTION_MEMLIST_V2 0x0E02
#define KGSL_SNAPSHOT_SECTION_SHADER 0x1201
#define KGSL_SNAPSHOT_SECTION_MVC 0x1501
#define KGSL_SNAPSHOT_SECTION_GMU 0x1601
#define KGSL_SNAPSHOT_SECTION_GMU_MEMORY 0x1701
#define KGSL_SNAPSHOT_SECTION_END 0xFFFF
/* OS sub-section header */
#define KGSL_SNAPSHOT_OS_LINUX 0x0001
#define KGSL_SNAPSHOT_OS_LINUX_V3 0x00000202
/* Linux OS specific information */
struct kgsl_snapshot_linux {
int osid; /* subsection OS identifier */
int state; /* 1 if the thread is running, 0 for hung */
__u32 seconds; /* Unix timestamp for the snapshot */
__u32 power_flags; /* Current power flags */
__u32 power_level; /* Current power level */
__u32 power_interval_timeout; /* Power interval timeout */
__u32 grpclk; /* Current GP clock value */
__u32 busclk; /* Current busclk value */
__u32 ptbase; /* Current ptbase */
__u32 pid; /* PID of the process that owns the PT */
__u32 current_context; /* ID of the current context */
__u32 ctxtcount; /* Number of contexts appended to section */
unsigned char release[32]; /* kernel release */
unsigned char version[32]; /* kernel version */
unsigned char comm[16]; /* Name of the process that owns the PT */
} __packed;
struct kgsl_snapshot_linux_v2 {
int osid; /* subsection OS identifier */
__u32 seconds; /* Unix timestamp for the snapshot */
__u32 power_flags; /* Current power flags */
__u32 power_level; /* Current power level */
__u32 power_interval_timeout; /* Power interval timeout */
__u32 grpclk; /* Current GP clock value */
__u32 busclk; /* Current busclk value */
__u64 ptbase; /* Current ptbase */
__u32 pid; /* PID of the process that owns the PT */
__u32 current_context; /* ID of the current context */
__u32 ctxtcount; /* Number of contexts appended to section */
unsigned char release[32]; /* kernel release */
unsigned char version[32]; /* kernel version */
unsigned char comm[16]; /* Name of the process that owns the PT */
} __packed;
/*
* This structure contains a record of an active context.
* These are appended one after another in the OS section below
* the header above
*/
struct kgsl_snapshot_linux_context {
__u32 id; /* The context ID */
__u32 timestamp_queued; /* The last queued timestamp */
__u32 timestamp_retired; /* The last timestamp retired by HW */
};
struct kgsl_snapshot_linux_context_v2 {
__u32 id; /* The context ID */
__u32 timestamp_queued; /* The last queued timestamp */
__u32 timestamp_consumed; /* The last timestamp consumed by HW */
__u32 timestamp_retired; /* The last timestamp retired by HW */
};
/* Ringbuffer sub-section header */
struct kgsl_snapshot_rb {
int start; /* dword at the start of the dump */
int end; /* dword at the end of the dump */
int rbsize; /* Size (in dwords) of the ringbuffer */
int wptr; /* Current index of the CPU write pointer */
int rptr; /* Current index of the GPU read pointer */
int count; /* Number of dwords in the dump */
__u32 timestamp_queued; /* The last queued timestamp */
__u32 timestamp_retired; /* The last timestamp retired by HW */
} __packed;
struct kgsl_snapshot_rb_v2 {
int start; /* dword at the start of the dump */
int end; /* dword at the end of the dump */
int rbsize; /* Size (in dwords) of the ringbuffer */
int wptr; /* Current index of the CPU write pointer */
int rptr; /* Current index of the GPU read pointer */
int count; /* Number of dwords in the dump */
__u32 timestamp_queued; /* The last queued timestamp */
__u32 timestamp_retired; /* The last timestamp retired by HW */
__u64 gpuaddr; /* The GPU address of the ringbuffer */
__u32 id; /* Ringbuffer identifier */
} __packed;
/* Replay or Memory list section, both sections have same header */
struct kgsl_snapshot_replay_mem_list {
/*
* Number of IBs to replay for replay section or
* number of memory list entries for mem list section
*/
int num_entries;
/* Pagetable base to which the replay IBs or memory entries belong */
__u32 ptbase;
} __packed;
/* Replay or Memory list section, both sections have same header */
struct kgsl_snapshot_mem_list_v2 {
/*
* Number of IBs to replay for replay section or
* number of memory list entries for mem list section
*/
int num_entries;
/* Pagetable base to which the replay IBs or memory entries belong */
__u64 ptbase;
} __packed;
/* Indirect buffer sub-section header */
struct kgsl_snapshot_ib {
__u32 gpuaddr; /* GPU address of the the IB */
__u32 ptbase; /* Base for the pagetable the GPU address is valid in */
int size; /* Size of the IB */
} __packed;
/* Indirect buffer sub-section header (v2) */
struct kgsl_snapshot_ib_v2 {
__u64 gpuaddr; /* GPU address of the the IB */
__u64 ptbase; /* Base for the pagetable the GPU address is valid in */
__u64 size; /* Size of the IB */
} __packed;
/* GMU memory ID's */
#define SNAPSHOT_GMU_MEM_UNKNOWN 0x00
#define SNAPSHOT_GMU_MEM_HFI 0x01
#define SNAPSHOT_GMU_MEM_LOG 0x02
#define SNAPSHOT_GMU_MEM_BWTABLE 0x03
#define SNAPSHOT_GMU_MEM_DEBUG 0x04
#define SNAPSHOT_GMU_MEM_BIN_BLOCK 0x05
/* GMU memory section data */
struct kgsl_snapshot_gmu_mem {
int type;
uint64_t hostaddr;
uint64_t gmuaddr;
uint64_t gpuaddr;
} __packed;
/* Register sub-section header */
struct kgsl_snapshot_regs {
__u32 count; /* Number of register pairs in the section */
} __packed;
/* Indexed register sub-section header */
struct kgsl_snapshot_indexed_regs {
__u32 index_reg; /* Offset of the index register for this section */
__u32 data_reg; /* Offset of the data register for this section */
int start; /* Starting index */
int count; /* Number of dwords in the data */
} __packed;
/* MVC register sub-section header */
struct kgsl_snapshot_mvc_regs {
int ctxt_id;
int cluster_id;
} __packed;
/* Istore sub-section header */
struct kgsl_snapshot_istore {
int count; /* Number of instructions in the istore */
} __packed;
/* Debug data sub-section header */
/* A2XX debug sections */
#define SNAPSHOT_DEBUG_SX 1
#define SNAPSHOT_DEBUG_CP 2
#define SNAPSHOT_DEBUG_SQ 3
#define SNAPSHOT_DEBUG_SQTHREAD 4
#define SNAPSHOT_DEBUG_MIU 5
/* A3XX debug sections */
#define SNAPSHOT_DEBUG_VPC_MEMORY 6
#define SNAPSHOT_DEBUG_CP_MEQ 7
#define SNAPSHOT_DEBUG_CP_PM4_RAM 8
#define SNAPSHOT_DEBUG_CP_PFP_RAM 9
#define SNAPSHOT_DEBUG_CP_ROQ 10
#define SNAPSHOT_DEBUG_SHADER_MEMORY 11
#define SNAPSHOT_DEBUG_CP_MERCIU 12
#define SNAPSHOT_DEBUG_SQE_VERSION 14
/* GMU Version information */
#define SNAPSHOT_DEBUG_GMU_CORE_VERSION 15
#define SNAPSHOT_DEBUG_GMU_CORE_DEV_VERSION 16
#define SNAPSHOT_DEBUG_GMU_PWR_VERSION 17
#define SNAPSHOT_DEBUG_GMU_PWR_DEV_VERSION 18
#define SNAPSHOT_DEBUG_GMU_HFI_VERSION 19
struct kgsl_snapshot_debug {
int type; /* Type identifier for the attached tata */
int size; /* Size of the section in dwords */
} __packed;
struct kgsl_snapshot_debugbus {
int id; /* Debug bus ID */
int count; /* Number of dwords in the dump */
} __packed;
struct kgsl_snapshot_shader {
int type; /* SP/TP statetype */
int index; /* SP/TP index */
int size; /* Number of dwords in the dump */
} __packed;
#define SNAPSHOT_GPU_OBJECT_SHADER 1
#define SNAPSHOT_GPU_OBJECT_IB 2
#define SNAPSHOT_GPU_OBJECT_GENERIC 3
#define SNAPSHOT_GPU_OBJECT_DRAW 4
#define SNAPSHOT_GPU_OBJECT_GLOBAL 5
struct kgsl_snapshot_gpu_object {
int type; /* Type of GPU object */
__u32 gpuaddr; /* GPU address of the the object */
__u32 ptbase; /* Base for the pagetable the GPU address is valid in */
int size; /* Size of the object (in dwords) */
};
struct kgsl_snapshot_gpu_object_v2 {
int type; /* Type of GPU object */
__u64 gpuaddr; /* GPU address of the the object */
__u64 ptbase; /* Base for the pagetable the GPU address is valid in */
__u64 size; /* Size of the object (in dwords) */
} __packed;
struct kgsl_device;
struct kgsl_process_private;
void kgsl_snapshot_push_object(struct kgsl_device *device,
struct kgsl_process_private *process,
uint64_t gpuaddr, uint64_t dwords);
#endif

862
drivers/gpu/msm/kgsl_sync.c Normal file
View file

@ -0,0 +1,862 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2012-2019, The Linux Foundation. All rights reserved.
*/
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/sync_file.h>
#include "kgsl_device.h"
#include "kgsl_sync.h"
static void kgsl_sync_timeline_signal(struct kgsl_sync_timeline *timeline,
unsigned int timestamp);
static const struct dma_fence_ops kgsl_sync_fence_ops;
static struct kgsl_sync_fence *kgsl_sync_fence_create(
struct kgsl_context *context,
unsigned int timestamp)
{
struct kgsl_sync_fence *kfence;
struct kgsl_sync_timeline *ktimeline = context->ktimeline;
unsigned long flags;
/* Get a refcount to the timeline. Put when released */
if (!kref_get_unless_zero(&ktimeline->kref))
return NULL;
kfence = kzalloc(sizeof(*kfence), GFP_KERNEL);
if (kfence == NULL) {
kgsl_sync_timeline_put(ktimeline);
return NULL;
}
kfence->parent = ktimeline;
kfence->context_id = context->id;
kfence->timestamp = timestamp;
dma_fence_init(&kfence->fence, &kgsl_sync_fence_ops, &ktimeline->lock,
ktimeline->fence_context, timestamp);
/*
* sync_file_create() takes a refcount to the fence. This refcount is
* put when the fence is signaled.
*/
kfence->sync_file = sync_file_create(&kfence->fence);
if (kfence->sync_file == NULL) {
kgsl_sync_timeline_put(ktimeline);
dev_err(context->device->dev, "Create sync_file failed\n");
kfree(kfence);
return NULL;
}
spin_lock_irqsave(&ktimeline->lock, flags);
list_add_tail(&kfence->child_list, &ktimeline->child_list_head);
spin_unlock_irqrestore(&ktimeline->lock, flags);
return kfence;
}
static void kgsl_sync_fence_release(struct dma_fence *fence)
{
struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
kgsl_sync_timeline_put(kfence->parent);
kfree(kfence);
}
/* Called with ktimeline->lock held */
static bool kgsl_sync_fence_has_signaled(struct dma_fence *fence)
{
struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
struct kgsl_sync_timeline *ktimeline = kfence->parent;
unsigned int ts = kfence->timestamp;
return (timestamp_cmp(ktimeline->last_timestamp, ts) >= 0);
}
static bool kgsl_enable_signaling(struct dma_fence *fence)
{
return !kgsl_sync_fence_has_signaled(fence);
}
struct kgsl_sync_fence_event_priv {
struct kgsl_context *context;
unsigned int timestamp;
};
/**
* kgsl_sync_fence_event_cb - Event callback for a fence timestamp event
* @device - The KGSL device that expired the timestamp
* @context- Pointer to the context that owns the event
* @priv: Private data for the callback
* @result - Result of the event (retired or canceled)
*
* Signal a fence following the expiration of a timestamp
*/
static void kgsl_sync_fence_event_cb(struct kgsl_device *device,
struct kgsl_event_group *group, void *priv, int result)
{
struct kgsl_sync_fence_event_priv *ev = priv;
kgsl_sync_timeline_signal(ev->context->ktimeline, ev->timestamp);
kgsl_context_put(ev->context);
kfree(ev);
}
static int _add_fence_event(struct kgsl_device *device,
struct kgsl_context *context, unsigned int timestamp)
{
struct kgsl_sync_fence_event_priv *event;
int ret;
event = kmalloc(sizeof(*event), GFP_KERNEL);
if (event == NULL)
return -ENOMEM;
/*
* Increase the refcount for the context to keep it through the
* callback
*/
if (!_kgsl_context_get(context)) {
kfree(event);
return -ENOENT;
}
event->context = context;
event->timestamp = timestamp;
ret = kgsl_add_event(device, &context->events, timestamp,
kgsl_sync_fence_event_cb, event);
if (ret) {
kgsl_context_put(context);
kfree(event);
}
return ret;
}
/* Only to be used if creating a related event failed */
static void kgsl_sync_cancel(struct kgsl_sync_fence *kfence)
{
spin_lock(&kfence->parent->lock);
if (!list_empty(&kfence->child_list)) {
list_del_init(&kfence->child_list);
dma_fence_put(&kfence->fence);
}
spin_unlock(&kfence->parent->lock);
}
/**
* kgsl_add_fence_event - Create a new fence event
* @device - KGSL device to create the event on
* @timestamp - Timestamp to trigger the event
* @data - Return fence fd stored in struct kgsl_timestamp_event_fence
* @len - length of the fence event
* @owner - driver instance that owns this event
* @returns 0 on success or error code on error
*
* Create a fence and register an event to signal the fence when
* the timestamp expires
*/
int kgsl_add_fence_event(struct kgsl_device *device,
u32 context_id, u32 timestamp, void __user *data, int len,
struct kgsl_device_private *owner)
{
struct kgsl_timestamp_event_fence priv;
struct kgsl_context *context;
struct kgsl_sync_fence *kfence = NULL;
int ret = -EINVAL;
unsigned int cur;
priv.fence_fd = -1;
if (len != sizeof(priv))
return -EINVAL;
context = kgsl_context_get_owner(owner, context_id);
if (context == NULL)
return -EINVAL;
if (test_bit(KGSL_CONTEXT_PRIV_INVALID, &context->priv))
goto out;
kfence = kgsl_sync_fence_create(context, timestamp);
if (kfence == NULL) {
ret = -ENOMEM;
goto out;
}
priv.fence_fd = get_unused_fd_flags(0);
if (priv.fence_fd < 0) {
dev_crit_ratelimited(device->dev,
"Unable to get a file descriptor: %d\n",
priv.fence_fd);
ret = priv.fence_fd;
goto out;
}
/*
* If the timestamp hasn't expired yet create an event to trigger it.
* Otherwise, just signal the fence - there is no reason to go through
* the effort of creating a fence we don't need.
*/
kgsl_readtimestamp(device, context, KGSL_TIMESTAMP_RETIRED, &cur);
if (timestamp_cmp(cur, timestamp) >= 0) {
ret = 0;
kgsl_sync_timeline_signal(context->ktimeline, cur);
} else {
ret = _add_fence_event(device, context, timestamp);
if (ret)
goto out;
}
if (copy_to_user(data, &priv, sizeof(priv))) {
ret = -EFAULT;
goto out;
}
fd_install(priv.fence_fd, kfence->sync_file->file);
out:
kgsl_context_put(context);
if (ret) {
if (priv.fence_fd >= 0)
put_unused_fd(priv.fence_fd);
if (kfence) {
kgsl_sync_cancel(kfence);
/*
* Put the refcount of sync file. This will release
* kfence->fence as well.
*/
fput(kfence->sync_file->file);
}
}
return ret;
}
static void kgsl_sync_timeline_value_str(struct dma_fence *fence,
char *str, int size)
{
struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
struct kgsl_sync_timeline *ktimeline = kfence->parent;
unsigned int timestamp_retired = 0;
unsigned int timestamp_queued = 0;
if (!kref_get_unless_zero(&ktimeline->kref))
return;
/*
* ktimeline->device might be NULL here but kgsl_readtimestamp()
* will handle that correctly
*/
kgsl_readtimestamp(ktimeline->device, ktimeline->context,
KGSL_TIMESTAMP_RETIRED, &timestamp_retired);
kgsl_readtimestamp(ktimeline->device, ktimeline->context,
KGSL_TIMESTAMP_QUEUED, &timestamp_queued);
snprintf(str, size, "%u queued:%u retired:%u",
ktimeline->last_timestamp,
timestamp_queued, timestamp_retired);
kgsl_sync_timeline_put(ktimeline);
}
static void kgsl_sync_fence_value_str(struct dma_fence *fence,
char *str, int size)
{
struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
snprintf(str, size, "%u", kfence->timestamp);
}
static const char *kgsl_sync_fence_driver_name(struct dma_fence *fence)
{
return "kgsl-timeline";
}
static const char *kgsl_sync_timeline_name(struct dma_fence *fence)
{
struct kgsl_sync_fence *kfence = (struct kgsl_sync_fence *)fence;
struct kgsl_sync_timeline *ktimeline = kfence->parent;
return ktimeline->name;
}
int kgsl_sync_timeline_create(struct kgsl_context *context)
{
struct kgsl_sync_timeline *ktimeline;
/* Put context when timeline is released */
if (!_kgsl_context_get(context))
return -ENOENT;
ktimeline = kzalloc(sizeof(*ktimeline), GFP_KERNEL);
if (ktimeline == NULL) {
kgsl_context_put(context);
return -ENOMEM;
}
kref_init(&ktimeline->kref);
snprintf(ktimeline->name, sizeof(ktimeline->name),
"%s_%d-%.15s(%d)-%.15s(%d)",
context->device->name, context->id,
current->group_leader->comm, current->group_leader->pid,
current->comm, current->pid);
ktimeline->fence_context = dma_fence_context_alloc(1);
ktimeline->last_timestamp = 0;
INIT_LIST_HEAD(&ktimeline->child_list_head);
spin_lock_init(&ktimeline->lock);
ktimeline->device = context->device;
ktimeline->context = context;
context->ktimeline = ktimeline;
return 0;
}
static void kgsl_sync_timeline_signal(struct kgsl_sync_timeline *ktimeline,
unsigned int timestamp)
{
unsigned long flags;
struct kgsl_sync_fence *kfence, *next;
if (!kref_get_unless_zero(&ktimeline->kref))
return;
spin_lock_irqsave(&ktimeline->lock, flags);
if (timestamp_cmp(timestamp, ktimeline->last_timestamp) > 0)
ktimeline->last_timestamp = timestamp;
list_for_each_entry_safe(kfence, next, &ktimeline->child_list_head,
child_list) {
if (dma_fence_is_signaled_locked(&kfence->fence)) {
list_del_init(&kfence->child_list);
dma_fence_put(&kfence->fence);
}
}
spin_unlock_irqrestore(&ktimeline->lock, flags);
kgsl_sync_timeline_put(ktimeline);
}
void kgsl_sync_timeline_destroy(struct kgsl_context *context)
{
kfree(context->ktimeline);
}
static void kgsl_sync_timeline_release(struct kref *kref)
{
struct kgsl_sync_timeline *ktimeline =
container_of(kref, struct kgsl_sync_timeline, kref);
/*
* Only put the context refcount here. The context destroy function
* will call kgsl_sync_timeline_destroy() to kfree it
*/
kgsl_context_put(ktimeline->context);
}
void kgsl_sync_timeline_put(struct kgsl_sync_timeline *ktimeline)
{
if (ktimeline)
kref_put(&ktimeline->kref, kgsl_sync_timeline_release);
}
static const struct dma_fence_ops kgsl_sync_fence_ops = {
.get_driver_name = kgsl_sync_fence_driver_name,
.get_timeline_name = kgsl_sync_timeline_name,
.enable_signaling = kgsl_enable_signaling,
.signaled = kgsl_sync_fence_has_signaled,
.wait = dma_fence_default_wait,
.release = kgsl_sync_fence_release,
.fence_value_str = kgsl_sync_fence_value_str,
.timeline_value_str = kgsl_sync_timeline_value_str,
};
static void kgsl_sync_fence_callback(struct dma_fence *fence,
struct dma_fence_cb *cb)
{
struct kgsl_sync_fence_cb *kcb = (struct kgsl_sync_fence_cb *)cb;
/*
* If the callback is marked for cancellation in a separate thread,
* let the other thread do the cleanup.
*/
if (kcb->func(kcb->priv)) {
dma_fence_put(kcb->fence);
kfree(kcb);
}
}
static void kgsl_get_fence_names(struct dma_fence *fence,
struct event_fence_info *info_ptr)
{
unsigned int num_fences;
struct dma_fence **fences;
struct dma_fence_array *array;
int i;
if (!info_ptr)
return;
array = to_dma_fence_array(fence);
if (array != NULL) {
num_fences = array->num_fences;
fences = array->fences;
} else {
num_fences = 1;
fences = &fence;
}
info_ptr->fences = kcalloc(num_fences, sizeof(struct fence_info),
GFP_ATOMIC);
if (info_ptr->fences == NULL)
return;
info_ptr->num_fences = num_fences;
for (i = 0; i < num_fences; i++) {
struct dma_fence *f = fences[i];
struct fence_info *fi = &info_ptr->fences[i];
int len;
len = scnprintf(fi->name, sizeof(fi->name), "%s %s",
f->ops->get_driver_name(f),
f->ops->get_timeline_name(f));
if (f->ops->fence_value_str) {
len += scnprintf(fi->name + len, sizeof(fi->name) - len,
": ");
f->ops->fence_value_str(f, fi->name + len,
sizeof(fi->name) - len);
}
}
}
struct kgsl_sync_fence_cb *kgsl_sync_fence_async_wait(int fd,
bool (*func)(void *priv), void *priv, struct event_fence_info *info_ptr)
{
struct kgsl_sync_fence_cb *kcb;
struct dma_fence *fence;
int status;
fence = sync_file_get_fence(fd);
if (fence == NULL)
return ERR_PTR(-EINVAL);
/* create the callback */
kcb = kzalloc(sizeof(*kcb), GFP_ATOMIC);
if (kcb == NULL) {
dma_fence_put(fence);
return ERR_PTR(-ENOMEM);
}
kcb->fence = fence;
kcb->priv = priv;
kcb->func = func;
kgsl_get_fence_names(fence, info_ptr);
/* if status then error or signaled */
status = dma_fence_add_callback(fence, &kcb->fence_cb,
kgsl_sync_fence_callback);
if (status) {
kfree(kcb);
if (!dma_fence_is_signaled(fence))
kcb = ERR_PTR(status);
else
kcb = NULL;
dma_fence_put(fence);
}
return kcb;
}
/*
* Cancel the fence async callback and do the cleanup. The caller must make
* sure that the callback (if run before cancelling) returns false, so that
* no other thread frees the pointer.
*/
void kgsl_sync_fence_async_cancel(struct kgsl_sync_fence_cb *kcb)
{
if (kcb == NULL)
return;
/*
* After fence_remove_callback() returns, the fence callback is
* either not called at all, or completed without freeing kcb.
* This thread can then put the fence refcount and free kcb.
*/
dma_fence_remove_callback(kcb->fence, &kcb->fence_cb);
dma_fence_put(kcb->fence);
kfree(kcb);
}
struct kgsl_syncsource {
struct kref refcount;
char name[32];
int id;
struct kgsl_process_private *private;
struct list_head child_list_head;
spinlock_t lock;
};
struct kgsl_syncsource_fence {
struct dma_fence fence;
struct kgsl_syncsource *parent;
struct list_head child_list;
};
static const struct dma_fence_ops kgsl_syncsource_fence_ops;
long kgsl_ioctl_syncsource_create(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_syncsource *syncsource = NULL;
struct kgsl_syncsource_create *param = data;
int ret = -EINVAL;
int id = 0;
struct kgsl_process_private *private = dev_priv->process_priv;
if (!kgsl_process_private_get(private))
return ret;
syncsource = kzalloc(sizeof(*syncsource), GFP_KERNEL);
if (syncsource == NULL) {
ret = -ENOMEM;
goto out;
}
kref_init(&syncsource->refcount);
snprintf(syncsource->name, sizeof(syncsource->name),
"kgsl-syncsource-pid-%d", current->group_leader->pid);
syncsource->private = private;
INIT_LIST_HEAD(&syncsource->child_list_head);
spin_lock_init(&syncsource->lock);
idr_preload(GFP_KERNEL);
spin_lock(&private->syncsource_lock);
id = idr_alloc(&private->syncsource_idr, syncsource, 1, 0, GFP_NOWAIT);
if (id > 0) {
syncsource->id = id;
param->id = id;
ret = 0;
} else {
ret = id;
}
spin_unlock(&private->syncsource_lock);
idr_preload_end();
out:
if (ret) {
kgsl_process_private_put(private);
kfree(syncsource);
}
return ret;
}
static struct kgsl_syncsource *
kgsl_syncsource_get(struct kgsl_process_private *private, int id)
{
int result = 0;
struct kgsl_syncsource *syncsource = NULL;
spin_lock(&private->syncsource_lock);
syncsource = idr_find(&private->syncsource_idr, id);
if (syncsource)
result = kref_get_unless_zero(&syncsource->refcount);
spin_unlock(&private->syncsource_lock);
return result ? syncsource : NULL;
}
static void kgsl_syncsource_destroy(struct kref *kref)
{
struct kgsl_syncsource *syncsource = container_of(kref,
struct kgsl_syncsource,
refcount);
struct kgsl_process_private *private = syncsource->private;
/* Done with process private. Release the refcount */
kgsl_process_private_put(private);
kfree(syncsource);
}
void kgsl_syncsource_put(struct kgsl_syncsource *syncsource)
{
if (syncsource)
kref_put(&syncsource->refcount, kgsl_syncsource_destroy);
}
static void kgsl_syncsource_cleanup(struct kgsl_process_private *private,
struct kgsl_syncsource *syncsource)
{
struct kgsl_syncsource_fence *sfence, *next;
/* Signal all fences to release any callbacks */
spin_lock(&syncsource->lock);
list_for_each_entry_safe(sfence, next, &syncsource->child_list_head,
child_list) {
dma_fence_signal_locked(&sfence->fence);
list_del_init(&sfence->child_list);
}
spin_unlock(&syncsource->lock);
/* put reference from syncsource creation */
kgsl_syncsource_put(syncsource);
}
long kgsl_ioctl_syncsource_destroy(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_syncsource_destroy *param = data;
struct kgsl_syncsource *syncsource = NULL;
struct kgsl_process_private *private = dev_priv->process_priv;
spin_lock(&private->syncsource_lock);
syncsource = idr_find(&private->syncsource_idr, param->id);
if (syncsource == NULL) {
spin_unlock(&private->syncsource_lock);
return -EINVAL;
}
if (syncsource->id != 0) {
idr_remove(&private->syncsource_idr, syncsource->id);
syncsource->id = 0;
}
spin_unlock(&private->syncsource_lock);
kgsl_syncsource_cleanup(private, syncsource);
return 0;
}
long kgsl_ioctl_syncsource_create_fence(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
struct kgsl_syncsource_create_fence *param = data;
struct kgsl_syncsource *syncsource = NULL;
int ret = -EINVAL;
struct kgsl_syncsource_fence *sfence = NULL;
struct sync_file *sync_file = NULL;
int fd = -1;
/*
* Take a refcount that is released when the fence is released
* (or if fence can't be added to the syncsource).
*/
syncsource = kgsl_syncsource_get(dev_priv->process_priv,
param->id);
if (syncsource == NULL)
goto out;
sfence = kzalloc(sizeof(*sfence), GFP_KERNEL);
if (sfence == NULL) {
ret = -ENOMEM;
goto out;
}
sfence->parent = syncsource;
/* Use a new fence context for each fence */
dma_fence_init(&sfence->fence, &kgsl_syncsource_fence_ops,
&syncsource->lock, dma_fence_context_alloc(1), 1);
sync_file = sync_file_create(&sfence->fence);
if (sync_file == NULL) {
dev_err(dev_priv->device->dev,
"Create sync_file failed\n");
ret = -ENOMEM;
goto out;
}
fd = get_unused_fd_flags(0);
if (fd < 0) {
ret = -EBADF;
goto out;
}
ret = 0;
fd_install(fd, sync_file->file);
param->fence_fd = fd;
spin_lock(&syncsource->lock);
list_add_tail(&sfence->child_list, &syncsource->child_list_head);
spin_unlock(&syncsource->lock);
out:
/*
* We're transferring ownership of the fence to the sync file.
* The sync file takes an extra refcount when it is created, so put
* our refcount.
*/
if (sync_file)
dma_fence_put(&sfence->fence);
if (ret) {
if (sync_file)
fput(sync_file->file);
else if (sfence)
dma_fence_put(&sfence->fence);
else
kgsl_syncsource_put(syncsource);
}
return ret;
}
static int kgsl_syncsource_signal(struct kgsl_syncsource *syncsource,
struct dma_fence *fence)
{
struct kgsl_syncsource_fence *sfence, *next;
int ret = -EINVAL;
spin_lock(&syncsource->lock);
list_for_each_entry_safe(sfence, next, &syncsource->child_list_head,
child_list) {
if (fence == &sfence->fence) {
dma_fence_signal_locked(fence);
list_del_init(&sfence->child_list);
ret = 0;
break;
}
}
spin_unlock(&syncsource->lock);
return ret;
}
long kgsl_ioctl_syncsource_signal_fence(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
int ret = -EINVAL;
struct kgsl_syncsource_signal_fence *param = data;
struct kgsl_syncsource *syncsource = NULL;
struct dma_fence *fence = NULL;
syncsource = kgsl_syncsource_get(dev_priv->process_priv,
param->id);
if (syncsource == NULL)
goto out;
fence = sync_file_get_fence(param->fence_fd);
if (fence == NULL) {
ret = -EBADF;
goto out;
}
ret = kgsl_syncsource_signal(syncsource, fence);
out:
if (fence)
dma_fence_put(fence);
if (syncsource)
kgsl_syncsource_put(syncsource);
return ret;
}
static void kgsl_syncsource_fence_release(struct dma_fence *fence)
{
struct kgsl_syncsource_fence *sfence =
(struct kgsl_syncsource_fence *)fence;
/* Signal if it's not signaled yet */
kgsl_syncsource_signal(sfence->parent, fence);
/* Release the refcount on the syncsource */
kgsl_syncsource_put(sfence->parent);
kfree(sfence);
}
void kgsl_syncsource_process_release_syncsources(
struct kgsl_process_private *private)
{
struct kgsl_syncsource *syncsource;
int next = 0;
while (1) {
spin_lock(&private->syncsource_lock);
syncsource = idr_get_next(&private->syncsource_idr, &next);
if (syncsource == NULL) {
spin_unlock(&private->syncsource_lock);
break;
}
if (syncsource->id != 0) {
idr_remove(&private->syncsource_idr, syncsource->id);
syncsource->id = 0;
}
spin_unlock(&private->syncsource_lock);
kgsl_syncsource_cleanup(private, syncsource);
next = next + 1;
}
}
static const char *kgsl_syncsource_get_timeline_name(struct dma_fence *fence)
{
struct kgsl_syncsource_fence *sfence =
(struct kgsl_syncsource_fence *)fence;
struct kgsl_syncsource *syncsource = sfence->parent;
return syncsource->name;
}
static bool kgsl_syncsource_enable_signaling(struct dma_fence *fence)
{
return true;
}
static const char *kgsl_syncsource_driver_name(struct dma_fence *fence)
{
return "kgsl-syncsource-timeline";
}
static void kgsl_syncsource_fence_value_str(struct dma_fence *fence,
char *str, int size)
{
/*
* Each fence is independent of the others on the same timeline.
* We use a different context for each of them.
*/
snprintf(str, size, "%llu", fence->context);
}
static const struct dma_fence_ops kgsl_syncsource_fence_ops = {
.get_driver_name = kgsl_syncsource_driver_name,
.get_timeline_name = kgsl_syncsource_get_timeline_name,
.enable_signaling = kgsl_syncsource_enable_signaling,
.wait = dma_fence_default_wait,
.release = kgsl_syncsource_fence_release,
.fence_value_str = kgsl_syncsource_fence_value_str,
};

183
drivers/gpu/msm/kgsl_sync.h Normal file
View file

@ -0,0 +1,183 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2012-2014,2018-2019 The Linux Foundation. All rights reserved.
*/
#ifndef __KGSL_SYNC_H
#define __KGSL_SYNC_H
#include <linux/dma-fence.h>
/**
* struct kgsl_sync_timeline - A sync timeline associated with a kgsl context
* @kref: Refcount to keep the struct alive until all its fences are released
* @name: String to describe this timeline
* @fence_context: Used by the fence driver to identify fences belonging to
* this context
* @child_list_head: List head for all fences on this timeline
* @lock: Spinlock to protect this timeline
* @last_timestamp: Last timestamp when signaling fences
* @device: kgsl device
* @context: kgsl context
*/
struct kgsl_sync_timeline {
struct kref kref;
char name[32];
u64 fence_context;
struct list_head child_list_head;
spinlock_t lock;
unsigned int last_timestamp;
struct kgsl_device *device;
struct kgsl_context *context;
};
/**
* struct kgsl_sync_fence - A struct containing a fence and other data
* associated with it
* @fence: The fence struct
* @sync_file: Pointer to the sync file
* @parent: Pointer to the kgsl sync timeline this fence is on
* @child_list: List of fences on the same timeline
* @context_id: kgsl context id
* @timestamp: Context timestamp that this fence is associated with
*/
struct kgsl_sync_fence {
struct dma_fence fence;
struct sync_file *sync_file;
struct kgsl_sync_timeline *parent;
struct list_head child_list;
u32 context_id;
unsigned int timestamp;
};
/**
* struct kgsl_sync_fence_cb - Used for fence callbacks
* fence_cb: Fence callback struct
* fence: Pointer to the fence for which the callback is done
* priv: Private data for the callback
* func: Pointer to the kgsl function to call. This function should return
* false if the sync callback is marked for cancellation in a separate thread.
*/
struct kgsl_sync_fence_cb {
struct dma_fence_cb fence_cb;
struct dma_fence *fence;
void *priv;
bool (*func)(void *priv);
};
struct kgsl_device_private;
struct kgsl_drawobj_sync_event;
struct event_fence_info;
struct kgsl_process_private;
struct kgsl_syncsource;
#if defined(CONFIG_SYNC_FILE)
int kgsl_add_fence_event(struct kgsl_device *device,
u32 context_id, u32 timestamp, void __user *data, int len,
struct kgsl_device_private *owner);
int kgsl_sync_timeline_create(struct kgsl_context *context);
void kgsl_sync_timeline_destroy(struct kgsl_context *context);
void kgsl_sync_timeline_put(struct kgsl_sync_timeline *ktimeline);
struct kgsl_sync_fence_cb *kgsl_sync_fence_async_wait(int fd,
bool (*func)(void *priv), void *priv,
struct event_fence_info *info_ptr);
void kgsl_sync_fence_async_cancel(struct kgsl_sync_fence_cb *kcb);
long kgsl_ioctl_syncsource_create(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_syncsource_destroy(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_syncsource_create_fence(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_syncsource_signal_fence(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
void kgsl_syncsource_put(struct kgsl_syncsource *syncsource);
void kgsl_syncsource_process_release_syncsources(
struct kgsl_process_private *private);
#else
static inline int kgsl_add_fence_event(struct kgsl_device *device,
u32 context_id, u32 timestamp, void __user *data, int len,
struct kgsl_device_private *owner)
{
return -EINVAL;
}
static inline int kgsl_sync_timeline_create(struct kgsl_context *context)
{
context->ktimeline = NULL;
return 0;
}
static inline void kgsl_sync_timeline_destroy(struct kgsl_context *context)
{
}
static inline void kgsl_sync_timeline_put(struct kgsl_sync_timeline *ktimeline)
{
}
static inline struct kgsl_sync_fence_cb *kgsl_sync_fence_async_wait(int fd,
bool (*func)(void *priv), void *priv,
struct event_fence_info *info_ptr)
{
return NULL;
}
static inline void
kgsl_sync_fence_async_cancel(struct kgsl_sync_fence_cb *kcb)
{
}
static inline long
kgsl_ioctl_syncsource_create(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
return -ENOIOCTLCMD;
}
static inline long
kgsl_ioctl_syncsource_destroy(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
return -ENOIOCTLCMD;
}
static inline long
kgsl_ioctl_syncsource_create_fence(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
return -ENOIOCTLCMD;
}
static inline long
kgsl_ioctl_syncsource_signal_fence(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
return -ENOIOCTLCMD;
}
static inline void kgsl_syncsource_put(struct kgsl_syncsource *syncsource)
{
}
static inline void kgsl_syncsource_process_release_syncsources(
struct kgsl_process_private *private)
{
}
#endif /* CONFIG_SYNC_FILE */
#endif /* __KGSL_SYNC_H */

12
drivers/gpu/msm/kgsl_trace.c Normal file
View file

@ -0,0 +1,12 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2011,2013,2015,2019 The Linux Foundation. All rights reserved.
*/
#include <linux/module.h>
#include "kgsl_device.h"
/* Instantiate tracepoints */
#define CREATE_TRACE_POINTS
#include "kgsl_trace.h"

1240
drivers/gpu/msm/kgsl_trace.h Normal file
View file

File diff suppressed because it is too large Load diff

91
drivers/gpu/msm/msm_adreno_devfreq.h Normal file
View file

@ -0,0 +1,91 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
*/
#ifndef MSM_ADRENO_DEVFREQ_H
#define MSM_ADRENO_DEVFREQ_H
#include <linux/devfreq.h>
#include <linux/notifier.h>
#define ADRENO_DEVFREQ_NOTIFY_SUBMIT 1
#define ADRENO_DEVFREQ_NOTIFY_RETIRE 2
#define ADRENO_DEVFREQ_NOTIFY_IDLE 3
#define DEVFREQ_FLAG_WAKEUP_MAXFREQ 0x2
#define DEVFREQ_FLAG_FAST_HINT 0x4
#define DEVFREQ_FLAG_SLOW_HINT 0x8
struct device;
int kgsl_devfreq_add_notifier(struct device *device,
struct notifier_block *block);
int kgsl_devfreq_del_notifier(struct device *device,
struct notifier_block *block);
/* same as KGSL_MAX_PWRLEVELS */
#define MSM_ADRENO_MAX_PWRLEVELS 10
struct xstats {
u64 ram_time;
u64 ram_wait;
int mod;
int buslevel;
};
struct devfreq_msm_adreno_tz_data {
struct notifier_block nb;
struct {
s64 total_time;
s64 busy_time;
u32 ctxt_aware_target_pwrlevel;
u32 ctxt_aware_busy_penalty;
} bin;
struct {
u64 total_time;
u64 ram_time;
u64 ram_wait;
u64 gpu_time;
u32 num;
u32 max;
u32 width;
u32 *up;
u32 *down;
s32 *p_up;
s32 *p_down;
u32 *ib_mbps;
bool floating;
} bus;
unsigned int device_id;
bool is_64;
bool disable_busy_time_burst;
bool ctxt_aware_enable;
};
struct msm_adreno_extended_profile {
struct devfreq_msm_adreno_tz_data *private_data;
struct devfreq *bus_devfreq;
struct workqueue_struct *partner_wq;
struct work_struct partner_start_event_ws;
struct work_struct partner_stop_event_ws;
struct work_struct partner_suspend_event_ws;
struct work_struct partner_resume_event_ws;
struct devfreq_dev_profile profile;
};
struct msm_busmon_extended_profile {
u32 flag;
unsigned long percent_ab;
unsigned long ab_mbytes;
struct devfreq_msm_adreno_tz_data *private_data;
struct devfreq_dev_profile profile;
};
typedef void(*getbw_func)(unsigned long *, unsigned long *, void *);
int devfreq_vbif_update_bw(void);
void devfreq_vbif_register_callback(getbw_func func, void *data);
#endif

24
include/uapi/linux/msm_kgsl.h → drivers/gpu/msm/uapi/msm_kgsl.h
View file

@ -359,7 +359,7 @@ struct kgsl_capabilities_properties {
#define KGSL_QUERY_CAPS_PROPERTIES 1
/*
* kgsl_capabilities allows the user to query kernel capabiilties. The 'data'
* kgsl_capabilities allows the user to query kernel capabilities. The 'data'
* type should be set appropriately for the querytype (see above). Pass 0 to
* 'size' and the kernel will set it to the expected size of 'data' that is
* appropriate for querytype (in bytes).
@ -1312,7 +1312,7 @@ struct kgsl_gpuobj_alloc {
*/
struct kgsl_gpuobj_free {
uint64_t flags;
uint64_t __user priv;
uint64_t priv;
unsigned int id;
unsigned int type;
unsigned int len;
@ -1375,7 +1375,7 @@ struct kgsl_gpuobj_info {
* @id: Returns the ID of the new GPU object
*/
struct kgsl_gpuobj_import {
uint64_t __user priv;
uint64_t priv;
uint64_t priv_len;
uint64_t flags;
unsigned int type;
@ -1424,7 +1424,7 @@ struct kgsl_gpuobj_sync_obj {
*/
struct kgsl_gpuobj_sync {
uint64_t __user objs;
uint64_t objs;
unsigned int obj_len;
unsigned int count;
};
@ -1455,7 +1455,7 @@ struct kgsl_command_object {
* @type: type of sync point defined here
*/
struct kgsl_command_syncpoint {
uint64_t __user priv;
uint64_t priv;
uint64_t size;
unsigned int type;
};
@ -1477,13 +1477,13 @@ struct kgsl_command_syncpoint {
*/
struct kgsl_gpu_command {
uint64_t flags;
uint64_t __user cmdlist;
uint64_t cmdlist;
unsigned int cmdsize;
unsigned int numcmds;
uint64_t __user objlist;
uint64_t objlist;
unsigned int objsize;
unsigned int numobjs;
uint64_t __user synclist;
uint64_t synclist;
unsigned int syncsize;
unsigned int numsyncs;
unsigned int context_id;
@ -1511,7 +1511,7 @@ struct kgsl_gpu_command {
* returned back.
*/
struct kgsl_preemption_counters_query {
uint64_t __user counters;
uint64_t counters;
unsigned int size_user;
unsigned int size_priority_level;
unsigned int max_priority_level;
@ -1628,7 +1628,7 @@ struct kgsl_sparse_binding_object {
*
*/
struct kgsl_sparse_bind {
uint64_t __user list;
uint64_t list;
unsigned int id;
unsigned int size;
unsigned int count;
@ -1653,8 +1653,8 @@ struct kgsl_sparse_bind {
*/
struct kgsl_gpu_sparse_command {
uint64_t flags;
uint64_t __user sparselist;
uint64_t __user synclist;
uint64_t sparselist;
uint64_t synclist;
unsigned int sparsesize;
unsigned int numsparse;
unsigned int syncsize;

2
drivers/video/Kconfig
View file

@ -19,6 +19,8 @@ source "drivers/gpu/ipu-v3/Kconfig"
source "drivers/gpu/drm/Kconfig"
source "drivers/gpu/msm/Kconfig"
menu "Frame buffer Devices"
source "drivers/video/fbdev/Kconfig"
endmenu