From d1b62d2e45b49ef7792d43a660d048d8618dee46 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Mon, 12 Aug 2024 15:30:09 +0800 Subject: [PATCH] MALI: rockchip: upgrade bifrost DDK to g25p0-00eac0, from g24p0-00eac0 mali_csffw.bin from Valhall DDK g25(r50) is included. Change-Id: Ic454428c384456a14b29d9651f537eb59c11284d Signed-off-by: Zhen Chen --- .../arm/arm,coresight-mali-source.yaml | 163 ++++ .../devicetree/bindings/arm/mali-bifrost.txt | 7 +- drivers/base/arm/Kconfig | 2 +- .../memory_group_manager.c | 20 +- drivers/gpu/arm/bifrost/Kbuild | 10 +- drivers/gpu/arm/bifrost/Kconfig | 13 +- drivers/gpu/arm/bifrost/Makefile | 2 +- .../arm/bifrost/arbiter/mali_kbase_arbif.c | 29 +- .../bifrost/arbiter/mali_kbase_arbiter_pm.c | 119 +-- .../gpu/mali_kbase_clk_rate_trace_mgr.c | 16 +- .../backend/gpu/mali_kbase_gpuprops_backend.c | 8 +- .../bifrost/backend/gpu/mali_kbase_jm_hw.c | 159 +--- .../bifrost/backend/gpu/mali_kbase_jm_rb.c | 25 +- .../backend/gpu/mali_kbase_js_backend.c | 6 +- .../backend/gpu/mali_kbase_model_dummy.c | 3 +- .../backend/gpu/mali_kbase_pm_backend.c | 41 +- .../bifrost/backend/gpu/mali_kbase_pm_ca.c | 18 +- .../bifrost/backend/gpu/mali_kbase_pm_defs.h | 32 +- .../backend/gpu/mali_kbase_pm_driver.c | 161 ++-- .../backend/gpu/mali_kbase_pm_internal.h | 41 +- .../backend/gpu/mali_kbase_pm_policy.c | 16 + .../arm/bifrost/backend/gpu/mali_kbase_time.c | 10 +- drivers/gpu/arm/bifrost/build.bp | 13 +- .../context/backend/mali_kbase_context_csf.c | 10 +- .../context/backend/mali_kbase_context_jm.c | 12 +- drivers/gpu/arm/bifrost/csf/Kbuild | 4 +- .../ipa_control/mali_kbase_csf_ipa_control.c | 5 +- drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c | 125 ++- .../bifrost/csf/mali_kbase_csf_csg_debugfs.c | 87 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_defs.h | 37 +- .../arm/bifrost/csf/mali_kbase_csf_firmware.c | 478 ++++++++--- .../arm/bifrost/csf/mali_kbase_csf_firmware.h | 34 +- .../csf/mali_kbase_csf_firmware_no_mali.c | 34 +- .../arm/bifrost/csf/mali_kbase_csf_fw_io.c | 251 ++++++ .../arm/bifrost/csf/mali_kbase_csf_fw_io.h | 362 +++++++++ .../csf/mali_kbase_csf_fw_io_no_mali.c | 294 +++++++ .../csf/mali_kbase_csf_heap_context_alloc.c | 7 +- .../gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c | 14 +- .../bifrost/csf/mali_kbase_csf_registers.h | 86 ++ .../bifrost/csf/mali_kbase_csf_reset_gpu.c | 2 - .../bifrost/csf/mali_kbase_csf_scheduler.c | 156 ++-- .../bifrost/csf/mali_kbase_csf_scheduler.h | 2 + .../bifrost/csf/mali_kbase_csf_tiler_heap.c | 16 +- .../bifrost/csf/mali_kbase_csf_trace_buffer.c | 7 +- .../backend/mali_kbase_debug_coresight_csf.c | 4 +- .../mali_kbase_debug_ktrace_codes_csf.h | 5 +- .../mali_kbase_debug_linux_ktrace_csf.h | 3 +- .../debug/mali_kbase_debug_ktrace_codes.h | 12 +- .../debug/mali_kbase_debug_linux_ktrace.h | 11 +- .../device/backend/mali_kbase_device_csf.c | 12 +- .../device/backend/mali_kbase_device_hw_csf.c | 33 +- .../device/backend/mali_kbase_device_hw_jm.c | 4 +- .../device/backend/mali_kbase_device_jm.c | 4 - .../arm/bifrost/device/mali_kbase_device.c | 37 +- .../bifrost/hw_access/mali_kbase_hw_access.c | 12 +- .../hw_access/mali_kbase_hw_access_regmap.h | 2 +- .../mali_kbase_hw_access_regmap_legacy.h | 2 +- .../regmap/mali_kbase_regmap_csf_macros.h | 1 + .../hwcnt/backend/mali_kbase_hwcnt_backend.h | 20 +- .../backend/mali_kbase_hwcnt_backend_csf.c | 18 + .../backend/mali_kbase_hwcnt_backend_csf_if.h | 20 + .../mali_kbase_hwcnt_backend_csf_if_fw.c | 22 + .../backend/mali_kbase_hwcnt_backend_jm.c | 12 +- .../mali_kbase_hwcnt_backend_jm_watchdog.c | 12 +- .../gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c | 8 +- .../arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h | 2 +- .../bifrost/hwcnt/mali_kbase_hwcnt_types.h | 61 +- .../arm/bifrost/ipa/mali_kbase_ipa_debugfs.c | 4 +- .../gpu/arm/bifrost/jm/mali_kbase_jm_defs.h | 81 +- drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h | 21 +- .../arm/bifrost/mali_base_hwconfig_features.h | 163 ---- .../arm/bifrost/mali_base_hwconfig_issues.h | 621 -------------- drivers/gpu/arm/bifrost/mali_csffw.bin | Bin 278528 -> 278528 bytes drivers/gpu/arm/bifrost/mali_kbase.h | 29 +- drivers/gpu/arm/bifrost/mali_kbase_caps.h | 97 ++- .../arm/bifrost/mali_kbase_config_defaults.h | 44 +- .../gpu/arm/bifrost/mali_kbase_core_linux.c | 239 +++--- .../arm/bifrost/mali_kbase_cs_experimental.h | 5 +- .../arm/bifrost/mali_kbase_debug_mem_allocs.c | 4 +- drivers/gpu/arm/bifrost/mali_kbase_defs.h | 104 ++- .../gpu/arm/bifrost/mali_kbase_dummy_job_wa.c | 6 +- .../gpu/arm/bifrost/mali_kbase_gpu_metrics.c | 5 +- drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c | 6 +- drivers/gpu/arm/bifrost/mali_kbase_gwt.c | 16 +- drivers/gpu/arm/bifrost/mali_kbase_hw.c | 14 +- drivers/gpu/arm/bifrost/mali_kbase_hw.h | 10 +- .../bifrost/mali_kbase_hwconfig_features.h | 158 ++++ .../arm/bifrost/mali_kbase_hwconfig_issues.h | 609 ++++++++++++++ .../arm/bifrost/mali_kbase_ioctl_helpers.h | 542 +++++++++++++ drivers/gpu/arm/bifrost/mali_kbase_jd.c | 78 +- drivers/gpu/arm/bifrost/mali_kbase_js.c | 762 ++---------------- .../gpu/arm/bifrost/mali_kbase_kinstr_jm.c | 10 +- drivers/gpu/arm/bifrost/mali_kbase_mem.c | 126 ++- drivers/gpu/arm/bifrost/mali_kbase_mem.h | 49 +- .../gpu/arm/bifrost/mali_kbase_mem_linux.c | 69 +- .../gpu/arm/bifrost/mali_kbase_mem_linux.h | 13 +- .../gpu/arm/bifrost/mali_kbase_mem_migrate.c | 6 + .../gpu/arm/bifrost/mali_kbase_native_mgm.c | 36 +- drivers/gpu/arm/bifrost/mali_kbase_pbha.c | 65 +- .../gpu/arm/bifrost/mali_kbase_pbha_debugfs.c | 2 +- drivers/gpu/arm/bifrost/mali_kbase_pm.c | 46 +- drivers/gpu/arm/bifrost/mali_kbase_pm.h | 8 +- .../gpu/arm/bifrost/mali_kbase_reg_track.c | 4 +- drivers/gpu/arm/bifrost/mali_kbase_softjobs.c | 19 +- .../bifrost/mmu/backend/mali_kbase_mmu_csf.c | 78 +- .../bifrost/mmu/backend/mali_kbase_mmu_jm.c | 57 +- drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c | 637 +++++++++++---- .../mmu/mali_kbase_mmu_faults_decoder.c | 8 +- .../gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h | 4 +- .../bifrost/mmu/mali_kbase_mmu_hw_direct.c | 15 +- .../arm/bifrost/mmu/mali_kbase_mmu_internal.h | 15 +- .../bifrost/mmu/mali_kbase_mmu_mode_aarch64.c | 4 +- drivers/gpu/arm/bifrost/tests/Kconfig | 2 +- .../kernel/mali_kutf_clk_rate_trace_test.c | 4 +- .../mali_kutf_mgm_integration_test_main.c | 6 +- .../tl/backend/mali_kbase_timeline_csf.c | 4 +- .../arm/bifrost/tl/mali_kbase_timeline_io.c | 5 +- .../arm/bifrost/tl/mali_kbase_tracepoints.c | 28 + .../arm/bifrost/tl/mali_kbase_tracepoints.h | 24 + drivers/hwtracing/coresight/mali/build.bp | 2 +- .../itm/coresight_mali_source_itm_core.c | 6 +- include/linux/mali_hw_access.h | 23 +- include/linux/memory_group_manager.h | 36 +- include/linux/version_compat_defs.h | 129 ++- .../arm/bifrost/csf/mali_base_csf_kernel.h | 10 +- .../arm/bifrost/csf/mali_kbase_csf_ioctl.h | 6 +- .../gpu/arm/bifrost/jm/mali_base_jm_kernel.h | 128 +-- .../gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h | 10 +- .../gpu/arm/bifrost/mali_base_common_kernel.h | 9 +- .../uapi/gpu/arm/bifrost/mali_base_kernel.h | 9 +- .../uapi/gpu/arm/bifrost/mali_kbase_ioctl.h | 6 +- .../mali_kbase_mem_profile_debugfs_buf_size.h | 4 +- 132 files changed, 5331 insertions(+), 3263 deletions(-) create mode 100644 Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml create mode 100644 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c create mode 100644 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h create mode 100644 drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c delete mode 100644 drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h delete mode 100644 drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h create mode 100644 drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h create mode 100644 drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h create mode 100644 drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml new file mode 100644 index 000000000000..d844ad10932c --- /dev/null +++ b/Documentation/devicetree/bindings/arm/arm,coresight-mali-source.yaml @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. +# +# This program is free software and is provided to you under the terms of the +# GNU General Public License version 2 as published by the Free Software +# Foundation, and any use by you of this program is subject to the terms +# of such GNU license. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, you can access it online at +# http://www.gnu.org/licenses/gpl-2.0.html. +# +# +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/arm/arm,coresight-mali-source.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ARM CoreSight Mali Source integration + +maintainers: + - ARM Ltd. + +description: | + See Documentation/trace/coresight/coresight.rst for detailed information + about Coresight. + + This documentation will cover Mali specific devicetree integration. + + References to Sink ports are given as examples. Access to Sink is specific + to an implementation and would require dedicated kernel modules. + + Arm Mali GPU are supporting 3 different sources: ITM, ETM, ELA + + ELA source configuration via SysFS entries: + + The register values used by CoreSight for ELA can be configured using SysFS + interfaces. This implicitly includes configuring the ELA for independent or + shared JCN request and response channels. + +properties: + compatible: + enum: + - arm,coresight-mali-source-itm + - arm,coresight-mali-source-etm + - arm,coresight-mali-source-ela + + gpu: + minItems: 1 + maxItems: 1 + description: + Phandle to a Mali GPU definition + + port: + description: + Output connection to CoreSight Sink Trace bus. + + Legacy binding between Coresight Sources and CoreSight Sink. + For Linux kernel < v4.20. + $ref: /schemas/graph.yaml#/properties/port + + out-ports: + description: + Binding between Coresight Sources and CoreSight Sink. + For Linux kernel >= v4.20. + $ref: /schemas/graph.yaml#/properties/ports + + properties: + port: + description: Output connection to CoreSight Sink Trace bus. + $ref: /schemas/graph.yaml#/properties/port + +required: + - compatible + - gpu + - port + - out-ports + +additionalProperties: false + +examples: + +# A Sink node without legacy CoreSight connections + - | + mali-source-itm { + compatible = "arm,coresight-mali-source-itm"; + gpu = <&gpu>; + + out-ports { + port { + mali_source_itm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port0>; + }; + }; + }; + }; + + mali-source-ela { + compatible = "arm,coresight-mali-source-ela"; + gpu = <&gpu>; + + out-ports { + port { + mali_source_ela_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port1>; + }; + }; + }; + }; + + mali-source-etm { + compatible = "arm,coresight-mali-source-etm"; + gpu = <&gpu>; + + out-ports { + port { + mali_source_etm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port2>; + }; + }; + }; + }; + +# A Sink node with legacy CoreSight connections + - | + mali-source-itm { + compatible = "arm,coresight-mali-source-itm"; + gpu = <&gpu>; + + port { + mali_source_itm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port0>; + }; + }; + }; + + mali-source-etm { + compatible = "arm,coresight-mali-source-etm"; + gpu = <&gpu>; + + port { + mali_source_etm_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port1>; + }; + }; + }; + + mali-source-ela { + compatible = "arm,coresight-mali-source-ela"; + gpu = <&gpu>; + + port { + mali_source_ela_out_port0: endpoint { + remote-endpoint = <&mali_sink_in_port2>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/arm/mali-bifrost.txt b/Documentation/devicetree/bindings/arm/mali-bifrost.txt index 3f80d97b0064..8ada052ebe56 100644 --- a/Documentation/devicetree/bindings/arm/mali-bifrost.txt +++ b/Documentation/devicetree/bindings/arm/mali-bifrost.txt @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -132,6 +132,10 @@ for details. set and the setting coresponding to the SYSC_ALLOC register. - propagate-bits: Used to write to L2_CONFIG.PBHA_HWU. This bitset establishes which PBHA bits are propagated on the AXI bus. +- mma-wa-id: Sets the PBHA ID to be used for the PBHA override based MMA violation workaround. + The read and write allocation override bits for the PBHA are set to NONCACHEABLE + and the driver encodes the PBHA ID in the PTEs where this workaround is to be applied. + Valid values are from 1 to 15. Example for a Mali GPU with 1 clock and 1 regulator: @@ -241,6 +245,7 @@ gpu@0xfc010000 { pbha { int-id-override = <2 0x32>, <9 0x05>, <16 0x32>; propagate-bits = /bits/ 8 <0x03>; + mma-wa-id = <2>; }; ... }; diff --git a/drivers/base/arm/Kconfig b/drivers/base/arm/Kconfig index e8bb8a40d2c5..c24a377723ca 100644 --- a/drivers/base/arm/Kconfig +++ b/drivers/base/arm/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/drivers/base/arm/memory_group_manager/memory_group_manager.c b/drivers/base/arm/memory_group_manager/memory_group_manager.c index c5fba5f1b522..da4a0c39e63a 100644 --- a/drivers/base/arm/memory_group_manager/memory_group_manager.c +++ b/drivers/base/arm/memory_group_manager/memory_group_manager.c @@ -299,7 +299,8 @@ static int example_mgm_get_import_memory_id(struct memory_group_manager_device * } static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const mgm_dev, - unsigned int const group_id, int const mmu_level, u64 pte) + unsigned int const group_id, unsigned int const pbha_id, + unsigned int pte_flags, int const mmu_level, u64 pte) { struct mgm_groups *const data = mgm_dev->data; @@ -309,7 +310,10 @@ static u64 example_mgm_update_gpu_pte(struct memory_group_manager_device *const if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return pte; - pte |= ((u64)group_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + if (pte_flags & BIT(MMA_VIOLATION)) { + pr_warn_once("MMA violation! Applying PBHA override workaround to PTE\n"); + pte |= ((u64)pbha_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + } /* Address could be translated into a different bus address here */ pte |= ((u64)1 << PTE_RES_BIT_MULTI_AS_SHIFT); @@ -362,6 +366,16 @@ static vm_fault_t example_mgm_vmf_insert_pfn_prot(struct memory_group_manager_de return fault; } +static bool example_mgm_get_import_memory_cached_access_permitted( + struct memory_group_manager_device *mgm_dev, + struct memory_group_manager_import_data *import_data) +{ + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(import_data); + + return true; +} + static int mgm_initialize_data(struct mgm_groups *mgm_data) { int i; @@ -408,6 +422,8 @@ static int memory_group_manager_probe(struct platform_device *pdev) mgm_dev->ops.mgm_vmf_insert_pfn_prot = example_mgm_vmf_insert_pfn_prot; mgm_dev->ops.mgm_update_gpu_pte = example_mgm_update_gpu_pte; mgm_dev->ops.mgm_pte_to_original_pte = example_mgm_pte_to_original_pte; + mgm_dev->ops.mgm_get_import_memory_cached_access_permitted = + example_mgm_get_import_memory_cached_access_permitted; mgm_data = kzalloc(sizeof(*mgm_data), GFP_KERNEL); if (!mgm_data) { diff --git a/drivers/gpu/arm/bifrost/Kbuild b/drivers/gpu/arm/bifrost/Kbuild index b19e4abf3e2b..d64c439fbabc 100644 --- a/drivers/gpu/arm/bifrost/Kbuild +++ b/drivers/gpu/arm/bifrost/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -69,7 +69,7 @@ endif # # Driver version string which is returned to userspace via an ioctl -MALI_RELEASE_NAME ?= '"g24p0-00eac0"' +MALI_RELEASE_NAME ?= '"g25p0-00eac0"' # Set up defaults if not defined by build system ifeq ($(CONFIG_MALI_BIFROST_DEBUG), y) MALI_UNIT_TEST = 1 @@ -104,7 +104,6 @@ endif # # Experimental features must default to disabled, e.g.: # MALI_EXPERIMENTAL_FEATURE ?= 0 -MALI_INCREMENTAL_RENDERING_JM ?= 0 # # ccflags @@ -117,7 +116,6 @@ ccflags-y = \ -DMALI_COVERAGE=$(MALI_COVERAGE) \ -DMALI_RELEASE_NAME=$(MALI_RELEASE_NAME) \ -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \ - -DMALI_INCREMENTAL_RENDERING_JM=$(MALI_INCREMENTAL_RENDERING_JM) \ -DMALI_PLATFORM_DIR=$(MALI_PLATFORM_DIR) @@ -212,6 +210,7 @@ endif INCLUDE_SUBDIR = \ + $(src)/arbiter/Kbuild \ $(src)/context/Kbuild \ $(src)/debug/Kbuild \ $(src)/device/Kbuild \ @@ -228,9 +227,6 @@ ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) INCLUDE_SUBDIR += $(src)/csf/Kbuild endif -ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y) - INCLUDE_SUBDIR += $(src)/arbiter/Kbuild -endif ifeq ($(CONFIG_MALI_BIFROST_DEVFREQ),y) ifeq ($(CONFIG_DEVFREQ_THERMAL),y) diff --git a/drivers/gpu/arm/bifrost/Kconfig b/drivers/gpu/arm/bifrost/Kconfig index 685ce4f423ad..b8ceff10e250 100644 --- a/drivers/gpu/arm/bifrost/Kconfig +++ b/drivers/gpu/arm/bifrost/Kconfig @@ -63,6 +63,8 @@ config MALI_BIFROST_NO_MALI All calls to the simulated hardware will complete immediately as if the hardware completed the task. +endchoice + config MALI_NO_MALI_DEFAULT_GPU string "Default GPU for No Mali" depends on MALI_BIFROST_NO_MALI @@ -70,7 +72,12 @@ config MALI_NO_MALI_DEFAULT_GPU help This option sets the default GPU to identify as for No Mali builds. -endchoice +config MALI_IS_FPGA + bool "Enable build of Mali kernel driver for FPGA" + depends on MALI_BIFROST + default n + help + This is the default HW backend. menu "Platform specific options" source "$(MALI_KCONFIG_EXT_PREFIX)drivers/gpu/arm/bifrost/platform/Kconfig" @@ -340,7 +347,7 @@ config MALI_PWRSOFT_765 changes have been backported say Y to avoid compilation errors. config MALI_HW_ERRATA_1485982_NOT_AFFECTED - bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" + bool "Disable workaround for KBASE_HW_ISSUE_GPU2017_1336" depends on MALI_BIFROST && MALI_BIFROST_EXPERT default n help @@ -352,7 +359,7 @@ config MALI_HW_ERRATA_1485982_NOT_AFFECTED coherency mode requires the L2 to be turned off. config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE - bool "Use alternative workaround for BASE_HW_ISSUE_GPU2017_1336" + bool "Use alternative workaround for KBASE_HW_ISSUE_GPU2017_1336" depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_HW_ERRATA_1485982_NOT_AFFECTED default n help diff --git a/drivers/gpu/arm/bifrost/Makefile b/drivers/gpu/arm/bifrost/Makefile index 9b636f58c6bc..e10033aabc57 100644 --- a/drivers/gpu/arm/bifrost/Makefile +++ b/drivers/gpu/arm/bifrost/Makefile @@ -156,7 +156,6 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_BIFROST \ CONFIG_MALI_CSF_SUPPORT \ CONFIG_MALI_BIFROST_GATOR_SUPPORT \ - CONFIG_MALI_ARBITER_SUPPORT \ CONFIG_MALI_ARBITRATION \ CONFIG_MALI_PARTITION_MANAGER \ CONFIG_MALI_REAL_HW \ @@ -170,6 +169,7 @@ ifeq ($(MALI_KCONFIG_EXT_PREFIX),) CONFIG_MALI_PWRSOFT_765 \ CONFIG_MALI_JOB_DUMP \ CONFIG_MALI_BIFROST_NO_MALI \ + CONFIG_MALI_IS_FPGA \ CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED \ CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE \ CONFIG_MALI_PRFCNT_SET_PRIMARY \ diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c index 8cdae33cf919..49b42a6ec2c0 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c @@ -108,6 +108,7 @@ static void on_gpu_stop(struct device *dev) } KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOP_REQUESTED, NULL, 0); kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); } @@ -133,6 +134,7 @@ static void on_gpu_granted(struct device *dev) } KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_GRANTED, NULL, 0); kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); } @@ -156,7 +158,8 @@ static void on_gpu_lost(struct device *dev) dev_err(dev, "%s(): kbdev is NULL", __func__); return; } - + KBASE_TLSTREAM_TL_ARBITER_LOST(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, 0); kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); } @@ -178,7 +181,7 @@ static int kbase_arbif_of_init(struct kbase_device *kbdev) if (!arbiter_if_node) arbiter_if_node = of_parse_phandle(kbdev->dev->of_node, "arbiter_if", 0); if (!arbiter_if_node) { - dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n"); + dev_dbg(kbdev->dev, "No arbiter_if in Device Tree"); /* no arbiter interface defined in device tree */ kbdev->arb.arb_dev = NULL; kbdev->arb.arb_if = NULL; @@ -187,19 +190,19 @@ static int kbase_arbif_of_init(struct kbase_device *kbdev) pdev = of_find_device_by_node(arbiter_if_node); if (!pdev) { - dev_err(kbdev->dev, "Failed to find arbiter_if device\n"); + dev_err(kbdev->dev, "Failed to find arbiter_if device"); return -EPROBE_DEFER; } if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) { - dev_err(kbdev->dev, "arbiter_if driver not available\n"); + dev_err(kbdev->dev, "arbiter_if driver not available"); put_device(&pdev->dev); return -EPROBE_DEFER; } kbdev->arb.arb_dev = &pdev->dev; arb_if = platform_get_drvdata(pdev); if (!arb_if) { - dev_err(kbdev->dev, "arbiter_if driver not ready\n"); + dev_err(kbdev->dev, "arbiter_if driver not ready"); module_put(pdev->dev.driver->owner); put_device(&pdev->dev); return -EPROBE_DEFER; @@ -243,6 +246,10 @@ int kbase_arbif_init(struct kbase_device *kbdev) /* Tries to init with 'arbiter-if' if present in devicetree */ err = kbase_arbif_of_init(kbdev); + if (err == -ENODEV) { + /* devicetree does not support arbitration */ + return -EPERM; + } if (err) return err; @@ -260,19 +267,19 @@ int kbase_arbif_init(struct kbase_device *kbdev) arb_if = kbdev->arb.arb_if; if (arb_if == NULL) { - dev_err(kbdev->dev, "No arbiter interface present\n"); + dev_err(kbdev->dev, "No arbiter interface present"); goto failure_term; } if (!arb_if->vm_ops.vm_arb_register_dev) { - dev_err(kbdev->dev, "arbiter_if registration callback not present\n"); + dev_err(kbdev->dev, "arbiter_if registration callback not present"); goto failure_term; } /* register kbase arbiter_if callbacks */ err = arb_if->vm_ops.vm_arb_register_dev(arb_if, kbdev->dev, &ops); if (err) { - dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)\n", err); + dev_err(kbdev->dev, "Failed to register with arbiter. (err = %d)", err); goto failure_term; } @@ -333,6 +340,7 @@ void kbase_arbif_gpu_request(struct kbase_device *kbdev) if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) { KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_REQUESTED, NULL, 0); arb_if->vm_ops.vm_arb_gpu_request(arb_if); } } @@ -349,8 +357,11 @@ void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required) if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) { KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev); - if (gpu_required) + KBASE_KTRACE_ADD(kbdev, ARB_GPU_STOPPED, NULL, 0); + if (gpu_required) { KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_REQUESTED, NULL, 0); + } arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required); } } diff --git a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c index a27085d0f4f4..9b8551609dc7 100644 --- a/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c +++ b/drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c @@ -199,6 +199,7 @@ static void kbase_arbiter_pm_resume_wq(struct work_struct *data) arb_vm_state->vm_arb_starting = false; mutex_unlock(&arb_vm_state->vm_state_lock); KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev); + KBASE_KTRACE_ADD(kbdev, ARB_GPU_STARTED, NULL, 0); dev_dbg(kbdev->dev, "<%s\n", __func__); } @@ -295,11 +296,13 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) err = kbase_arbif_init(kbdev); if (err) { - dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)\n", err); + if (err != -EPERM) + dev_err(kbdev->dev, "Failed to initialise arbif module. (err = %d)", err); + goto arbif_init_fail; } - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { kbase_arbif_gpu_request(kbdev); dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n"); @@ -345,6 +348,9 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) if (arb_vm_state == NULL) return; + if (!kbase_has_arbiter(kbdev)) + return; + kbase_arbiter_pm_release_interrupts(kbdev); cancel_request_timer(kbdev); @@ -475,6 +481,12 @@ int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev) if (!kbdev) return result; + /* If there is no Arbiter, then there is no virtualization + * and current VM always has access to GPU. + */ + if (!kbase_has_arbiter(kbdev)) + return 1; + /* First check the GPU_LOST state */ kbase_pm_lock(kbdev); if (kbase_pm_is_gpu_lost(kbdev)) { @@ -688,7 +700,7 @@ static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state(struct kbase_devic * @kbdev: The kbase device structure for the device * * Prepares OS to be in suspend state until it receives GRANT message - * from Arbiter asynchronously. + * from Arbiter asynchronously. This function assumes there is an active Arbiter. */ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) { @@ -696,10 +708,8 @@ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) enum kbase_vm_state prev_state; lockdep_assert_held(&arb_vm_state->vm_state_lock); - if (kbdev->arb.arb_if) { - if (kbdev->pm.arb_vm_state->vm_state == KBASE_VM_STATE_SUSPENDED) - return; - } + if (kbdev->pm.arb_vm_state->vm_state == KBASE_VM_STATE_SUSPENDED) + return; /* Block suspend OS function until we are in a stable state * with vm_state_lock */ @@ -791,7 +801,7 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, enum kbase_arbif_evt { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - if (!kbdev->arb.arb_if) + if (!kbase_has_arbiter(kbdev)) return; mutex_lock(&arb_vm_state->vm_state_lock); @@ -911,7 +921,8 @@ static inline bool kbase_arbiter_pm_vm_gpu_assigned_locked(struct kbase_device * * * This function handles a suspend event from the driver, * communicating with the arbiter and waiting synchronously for the GPU - * to be granted again depending on the VM state. + * to be granted again depending on the VM state. Returns immediately + * with success if there is no Arbiter. * * Return: 0 on success else 1 suspend handler isn not possible. */ @@ -921,58 +932,58 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; int res = 0; - if (kbdev->arb.arb_if) { - mutex_lock(&arb_vm_state->vm_state_lock); - while (!kbase_arbiter_pm_vm_gpu_assigned_locked(kbdev)) { - /* Update VM state since we have GPU work to do */ - if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_STOPPING_ACTIVE); - else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPED) { - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_STOPPED_GPU_REQUESTED); - kbase_arbif_gpu_request(kbdev); - start_request_timer(kbdev); - } else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) + if (!kbase_has_arbiter(kbdev)) + return res; + + mutex_lock(&arb_vm_state->vm_state_lock); + while (!kbase_arbiter_pm_vm_gpu_assigned_locked(kbdev)) { + /* Update VM state since we have GPU work to do */ + if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPING_ACTIVE); + else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPED) { + kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED_GPU_REQUESTED); + kbase_arbif_gpu_request(kbdev); + start_request_timer(kbdev); + } else if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) + break; + + if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { + /* In case of GPU lost, even if + * active_count > 0, we no longer have GPU + * access + */ + if (kbase_pm_is_gpu_lost(kbdev)) + res = 1; + + switch (suspend_handler) { + case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: + res = 1; break; - - if (suspend_handler != KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { - /* In case of GPU lost, even if - * active_count > 0, we no longer have GPU - * access - */ - if (kbase_pm_is_gpu_lost(kbdev)) + case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: + if (kbdev->pm.active_count == 0) res = 1; - - switch (suspend_handler) { - case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: - res = 1; - break; - case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: - if (kbdev->pm.active_count == 0) - res = 1; - break; - case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED: - break; - default: - WARN(1, "Unknown suspend_handler\n"); - res = 1; - break; - } + break; + case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED: + break; + default: + WARN(1, "Unknown suspend_handler\n"); + res = 1; break; } - - /* Need to synchronously wait for GPU assignment */ - atomic_inc(&kbdev->pm.gpu_users_waiting); - mutex_unlock(&arb_vm_state->vm_state_lock); - kbase_pm_unlock(kbdev); - kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); - kbase_pm_lock(kbdev); - mutex_lock(&arb_vm_state->vm_state_lock); - atomic_dec(&kbdev->pm.gpu_users_waiting); + break; } + + /* Need to synchronously wait for GPU assignment */ + atomic_inc(&kbdev->pm.gpu_users_waiting); mutex_unlock(&arb_vm_state->vm_state_lock); + kbase_pm_unlock(kbdev); + kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev); + kbase_pm_lock(kbdev); + mutex_lock(&arb_vm_state->vm_state_lock); + atomic_dec(&kbdev->pm.gpu_users_waiting); } + mutex_unlock(&arb_vm_state->vm_state_lock); + return res; } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c index e47dd440bff2..851e6feafd30 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_clk_rate_trace_mgr.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -50,14 +50,22 @@ static struct kbase_clk_rate_trace_op_conf * get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev) { /* base case */ + const void *arbiter_if_node; struct kbase_clk_rate_trace_op_conf *callbacks = (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS; -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) - const void *arbiter_if_node; + + /* Nothing left to do here if there is no Arbiter/virtualization or if + * CONFIG_OF is not enabled. + */ + if (!IS_ENABLED(CONFIG_OF)) + return callbacks; if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev)) return callbacks; + if (!kbase_has_arbiter(kbdev)) + return callbacks; + arbiter_if_node = of_get_property(kbdev->dev->of_node, "arbiter-if", NULL); if (!arbiter_if_node) arbiter_if_node = of_get_property(kbdev->dev->of_node, "arbiter_if", NULL); @@ -69,8 +77,6 @@ get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev) dev_dbg(kbdev->dev, "Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n"); -#endif - return callbacks; } diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c index 414ad546811a..a9b629ad7ea5 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -48,7 +48,7 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop /* Not a valid register on TMIX */ /* TGOx specific register */ - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_TLS_ALLOC)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_THREAD_TLS_ALLOC)) regdump->thread_tls_alloc = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(THREAD_TLS_ALLOC)); #endif /* !MALI_USE_CSF */ @@ -64,7 +64,7 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, struct kbasep_gpuprop /* AMBA_FEATURES enum is mapped to COHERENCY_FEATURES enum */ regdump->coherency_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(COHERENCY_FEATURES)); - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CORE_FEATURES)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CORE_FEATURES)) regdump->core_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(CORE_FEATURES)); #if MALI_USE_CSF @@ -116,7 +116,7 @@ int kbase_backend_gpuprops_get_curr_config(struct kbase_device *kbdev, int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, struct kbasep_gpuprops_regdump *regdump) { - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) { regdump->l2_features = KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(L2_FEATURES)); regdump->l2_config = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c index cc8a0ff7fa42..b251de4fc23e 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c @@ -98,82 +98,6 @@ static u64 kbase_job_write_affinity(struct kbase_device *kbdev, base_jd_core_req return affinity; } -/** - * select_job_chain() - Select which job chain to submit to the GPU - * @katom: Pointer to the atom about to be submitted to the GPU - * - * Selects one of the fragment job chains attached to the special atom at the - * end of a renderpass, or returns the address of the single job chain attached - * to any other type of atom. - * - * Which job chain is selected depends upon whether the tiling phase of the - * renderpass completed normally or was soft-stopped because it used too - * much memory. It also depends upon whether one of the fragment job chains - * has already been run as part of the same renderpass. - * - * Return: GPU virtual address of the selected job chain - */ -static u64 select_job_chain(struct kbase_jd_atom *katom) -{ - struct kbase_context *const kctx = katom->kctx; - u64 jc = katom->jc; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) - return jc; - - compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[katom->renderpass_id]; - /* We can read a subset of renderpass state without holding - * higher-level locks (but not end_katom, for example). - * If the end-of-renderpass atom is running with as-yet indeterminate - * OOM state then assume that the start atom was not soft-stopped. - */ - switch (rp->state) { - case KBASE_JD_RP_OOM: - /* Tiling ran out of memory. - * Start of incremental rendering, used once. - */ - jc = katom->jc_fragment.norm_read_forced_write; - break; - case KBASE_JD_RP_START: - case KBASE_JD_RP_PEND_OOM: - /* Tiling completed successfully first time. - * Single-iteration rendering, used once. - */ - jc = katom->jc_fragment.norm_read_norm_write; - break; - case KBASE_JD_RP_RETRY_OOM: - /* Tiling ran out of memory again. - * Continuation of incremental rendering, used as - * many times as required. - */ - jc = katom->jc_fragment.forced_read_forced_write; - break; - case KBASE_JD_RP_RETRY: - case KBASE_JD_RP_RETRY_PEND_OOM: - /* Tiling completed successfully this time. - * End of incremental rendering, used once. - */ - jc = katom->jc_fragment.forced_read_norm_write; - break; - default: - WARN_ON(1); - break; - } - - dev_dbg(kctx->kbdev->dev, "Selected job chain 0x%llx for end atom %pK in state %d\n", jc, - (void *)katom, (int)rp->state); - - katom->jc = jc; - return jc; -} - static inline bool kbasep_jm_wait_js_free(struct kbase_device *kbdev, unsigned int js, struct kbase_context *kctx) { @@ -196,7 +120,7 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, { struct kbase_context *kctx; u32 cfg; - u64 const jc_head = select_job_chain(katom); + u64 jc_head = katom->jc; u64 affinity; struct slot_rb *ptr_slot_rb = &kbdev->hwaccess.backend.slot_rb[js]; @@ -220,21 +144,21 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, */ cfg = (u32)kctx->as_nr; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION) && !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION; if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START)) { /* Force a cache maintenance operation if the newly submitted * katom to the slot is from a different kctx. For a JM GPU - * that has the feature BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + * that has the feature KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, * applies a FLUSH_INV_SHADER_OTHER. Otherwise, do a * FLUSH_CLEAN_INVALIDATE. */ u64 tagged_kctx = ptr_slot_rb->last_kctx_tagged; if (tagged_kctx != SLOT_RB_NULL_TAG_VAL && tagged_kctx != SLOT_RB_TAG_KCTX(kctx)) { - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER)) cfg |= JS_CONFIG_START_FLUSH_INV_SHADER_OTHER; else cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE; @@ -246,15 +170,14 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END) && !(kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)) cfg |= JS_CONFIG_END_FLUSH_NO_ACTION; - else if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) + else if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE)) cfg |= JS_CONFIG_END_FLUSH_CLEAN; else cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; cfg |= JS_CONFIG_THREAD_PRI(8); - if ((katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) || - (katom->core_req & BASE_JD_REQ_END_RENDERPASS)) + if (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK; if (!ptr_slot_rb->job_chain_flag) { @@ -268,7 +191,7 @@ int kbase_job_hw_submit(struct kbase_device *kbdev, struct kbase_jd_atom *katom, kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, CONFIG_NEXT), cfg); - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) kbase_reg_write32(kbdev, JOB_SLOT_OFFSET(js, FLUSH_ID_NEXT), katom->flush_id); /* Write an approximate start timestamp. @@ -440,7 +363,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) * jobs to hang. Reset GPU before allowing * any other jobs on the slot to continue. */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3076)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3076)) { if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { if (kbase_prepare_to_reset_gpu_locked( kbdev, RESET_FLAGS_NONE)) @@ -740,66 +663,6 @@ void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, } } -static int softstop_start_rp_nolock(struct kbase_context *kctx, struct kbase_va_region *reg) -{ - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_atom *katom; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - katom = kbase_gpu_inspect(kbdev, 1, 0); - - if (!katom) { - dev_dbg(kctx->kbdev->dev, "No atom on job slot\n"); - return -ESRCH; - } - - if (!(katom->core_req & BASE_JD_REQ_START_RENDERPASS)) { - dev_dbg(kctx->kbdev->dev, "Atom %pK on job slot is not start RP\n", (void *)katom); - return -EPERM; - } - - compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[katom->renderpass_id]; - if (WARN_ON(rp->state != KBASE_JD_RP_START && rp->state != KBASE_JD_RP_RETRY)) - return -EINVAL; - - dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %pK\n", (int)rp->state, (void *)reg); - - if (WARN_ON(katom != rp->start_katom)) - return -EINVAL; - - dev_dbg(kctx->kbdev->dev, "Adding region %pK to list %pK\n", (void *)reg, - (void *)&rp->oom_reg_list); - list_move_tail(®->link, &rp->oom_reg_list); - dev_dbg(kctx->kbdev->dev, "Added region to list\n"); - - rp->state = (rp->state == KBASE_JD_RP_START ? KBASE_JD_RP_PEND_OOM : - KBASE_JD_RP_RETRY_PEND_OOM); - - kbase_job_slot_softstop(kbdev, 1, katom); - - return 0; -} - -int kbase_job_slot_softstop_start_rp(struct kbase_context *const kctx, - struct kbase_va_region *const reg) -{ - struct kbase_device *const kbdev = kctx->kbdev; - int err; - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - err = softstop_start_rp_nolock(kctx, reg); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return err; -} - void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) { struct kbase_device *kbdev = kctx->kbdev; @@ -839,7 +702,7 @@ u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev) { u32 flush_id = 0; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_FLUSH_REDUCTION)) { mutex_lock(&kbdev->pm.lock); if (kbdev->pm.backend.gpu_powered) flush_id = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(LATEST_FLUSH)); @@ -1085,7 +948,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) /* The flush has completed so reset the active indicator */ kbdev->irq_reset_flush = false; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8463)) { u64 val; const u32 timeout_us = kbase_get_timeout_ms(kbdev, KBASE_CLEAN_CACHE_TIMEOUT) * USEC_PER_MSEC; @@ -1268,14 +1131,12 @@ bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, unsigned int { unsigned int i; -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_gpu_lost(kbdev)) { /* GPU access has been removed, reset will be done by * Arbiter instead */ return false; } -#endif if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) kbase_instr_hwcnt_on_unrecoverable_error(kbdev); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c index e1105bf90899..a4a640a0fb92 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -425,7 +425,7 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, struct kbase_jd_a } } - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { if (katom->atom_flags & KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT) { kbase_pm_protected_l2_override(kbdev, false); katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT; @@ -698,7 +698,7 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbas kbase_pm_protected_entry_override_disable(kbdev); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { /* * Power on L2 caches; this will also result in the * correct value written to coherency enable register. @@ -714,13 +714,13 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, struct kbas katom[idx]->protected_state.enter = KBASE_ATOM_ENTER_PROTECTED_FINISHED; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) return -EAGAIN; /* ***TRANSITION TO HIGHER STATE*** */ fallthrough; case KBASE_ATOM_ENTER_PROTECTED_FINISHED: - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TGOX_R1_1234)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TGOX_R1_1234)) { /* * Check that L2 caches are powered and, if so, * enter protected mode. @@ -864,11 +864,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->hwaccess_lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_removed(kbdev)) -#else - if (kbase_reset_gpu_is_active(kbdev)) -#endif + if (kbase_reset_gpu_is_active(kbdev) || (kbase_is_gpu_removed(kbdev))) return; for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { @@ -896,7 +892,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) break; case KBASE_ATOM_GPU_RB_WAITING_BLOCKED: - if (kbase_js_atom_blocked_on_x_dep(katom[idx])) + if (katom[idx]->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) break; katom[idx]->gpu_rb_state = @@ -1236,7 +1232,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp * When a hard-stop is followed close after a soft-stop, the completion * code may be set to STOPPED, even though the job is terminated */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8438)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TMIX_8438)) { if (completion_code == BASE_JD_EVENT_STOPPED && (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_HARD_STOPPED)) { completion_code = BASE_JD_EVENT_TERMINATED; @@ -1331,6 +1327,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp dev_dbg(kbdev->dev, "Update job chain address of atom %pK to resume from 0x%llx\n", (void *)katom, job_tail); + /* Some of the job has been executed, so we update the job chain address to where + * we should resume from + */ katom->jc = job_tail; KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx, katom, job_tail, js); } @@ -1381,6 +1380,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, unsigned int js, u32 comp dev_dbg(kbdev->dev, "Cross-slot dependency %pK has become runnable.\n", (void *)katom); + /* Cross-slot dependency has now become runnable. Try to submit it. */ + /* Check if there are lower priority jobs to soft stop */ kbase_job_slot_ctx_priority_check_locked(kctx, katom); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c index 202671b323d5..99037c25bf08 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -46,7 +46,7 @@ static inline bool timer_callback_should_run(struct kbase_device *kbdev, int nr_ } #endif /* CONFIG_MALI_BIFROST_DEBUG */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_9435)) { /* Timeouts would have to be 4x longer (due to micro- * architectural design) to support OpenCL conformance tests, so * only run the timer when there's: @@ -100,7 +100,7 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) /* The current version of the model doesn't support * Soft-Stop */ - if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { + if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_5736)) { u32 ticks = atom->ticks++; #if !defined(CONFIG_MALI_JOB_DUMP) && !defined(CONFIG_MALI_VECTOR_DUMP) diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c index b034ffef0ceb..0f4a8cd096bb 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_model_dummy.c @@ -1953,7 +1953,8 @@ void midgard_model_read_reg(void *h, u32 addr, u32 *const value) *value = dummy->control_reg_values->gpu_features_lo; } else if (addr == GPU_CONTROL_REG(GPU_FEATURES_HI)) { *value = dummy->control_reg_values->gpu_features_hi; - } else { + } + else { model_error_log( KBASE_CORE, "Dummy model register access: Reading unsupported register 0x%x. Returning 0\n", diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c index 6db242af0578..e1941d50133a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c @@ -36,9 +36,7 @@ #include #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* !CONFIG_MALI_ARBITER_SUPPORT */ #endif /* !MALI_USE_CSF */ #include #include @@ -100,10 +98,8 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev) if (callbacks) callbacks->power_on_callback(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) dev_err(kbdev->dev, "Attempting to power on while GPU lost\n"); -#endif kbdev->pm.backend.gpu_powered = true; } @@ -136,9 +132,7 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work, kbase_pm_gpu_poweroff_wait_wq); kbdev->pm.backend.ca_cores_enabled = ~0ull; -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_pm_set_gpu_lost(kbdev, false); -#endif init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); #if !MALI_USE_CSF @@ -180,15 +174,18 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) - kbdev->pm.backend.gpu_sleep_supported = - kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_GPU_SLEEP) && - !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_1997) && - kbdev->pm.backend.callback_power_runtime_gpu_active && - kbdev->pm.backend.callback_power_runtime_gpu_idle; + kbdev->pm.backend.gpu_sleep_allowed = 0; + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_GPU_SLEEP) && + !kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_1997) && + kbdev->pm.backend.callback_power_runtime_gpu_active && + kbdev->pm.backend.callback_power_runtime_gpu_idle) + set_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); kbdev->pm.backend.apply_hw_issue_TITANHW_2938_wa = - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TITANHW_2938) && - kbdev->pm.backend.gpu_sleep_supported; + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2938) && + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); + + /* FW Sleep-on-Idle is feature is kept disabled */ #endif if (IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED)) @@ -196,14 +193,14 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) /* WA1: L2 always_on for GPUs being affected by GPU2017-1336 */ if (!IS_ENABLED(CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE)) { - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2017_1336)) kbdev->pm.backend.l2_always_on = true; return 0; } /* WA3: Clock slow down for GPUs being affected by GPU2017-1336 */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2017_1336)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2017_1336)) { kbdev->pm.backend.gpu_clock_slow_down_wa = true; kbdev->pm.backend.gpu_clock_slow_down_desired = true; INIT_WORK(&kbdev->pm.backend.gpu_clock_control_work, @@ -348,13 +345,11 @@ static void pm_handle_power_off(struct kbase_device *kbdev) */ wait_for_mmu_fault_handling_in_gpu_poweroff_wait_wq(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* poweron_required may have changed while pm lock * was released. */ if (kbase_pm_is_gpu_lost(kbdev)) backend->poweron_required = false; -#endif /* Turn off clock now that fault have been handled. We * dropped locks so poweron_required may have changed - @@ -948,13 +943,11 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) /* System resume callback has begun */ kbdev->pm.resuming = true; kbdev->pm.suspending = false; -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_gpu_lost(kbdev)) { dev_dbg(kbdev->dev, "%s: GPU lost in progress\n", __func__); kbase_pm_unlock(kbdev); return; } -#endif kbase_pm_do_poweron(kbdev, true); #if !MALI_USE_CSF @@ -964,7 +957,6 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) kbase_pm_unlock(kbdev); } -#ifdef CONFIG_MALI_ARBITER_SUPPORT void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) { unsigned long flags; @@ -975,8 +967,10 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) #endif struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - if (!kbdev->arb.arb_if) + if (!kbase_has_arbiter(kbdev)) { + dev_warn(kbdev->dev, "%s called with no active arbiter!\n", __func__); return; + } mutex_lock(&kbdev->pm.lock); mutex_lock(&arb_vm_state->vm_state_lock); @@ -991,7 +985,8 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) #if MALI_USE_CSF /* Full GPU reset will have been done by hypervisor, so cancel */ - kbase_reset_gpu_prevent_and_wait(kbdev); + if (kbase_reset_gpu_prevent_and_wait(kbdev)) + dev_warn(kbdev->dev, "Failed to prevent GPU reset."); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_csf_scheduler_spin_lock(kbdev, &flags_sched); @@ -1041,7 +1036,6 @@ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) mutex_unlock(&arb_vm_state->vm_state_lock); mutex_unlock(&kbdev->pm.lock); } -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) int kbase_pm_force_mcu_wakeup_after_sleep(struct kbase_device *kbdev) @@ -1253,4 +1247,5 @@ out: return ret; } + #endif diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c index 8daef13388a3..6522e5ca66e9 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2013-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2013-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -55,11 +55,18 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) unsigned long flags; #if MALI_USE_CSF u64 old_core_mask = 0; -#endif + bool mmu_sync_needed = false; + if (!IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) && + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901)) { + mmu_sync_needed = true; + down_write(&kbdev->csf.mmu_sync_sem); + } +#endif spin_lock_irqsave(&kbdev->hwaccess_lock, flags); #if MALI_USE_CSF + if (!(core_mask & kbdev->pm.debug_core_mask)) { dev_err(kbdev->dev, "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", @@ -98,6 +105,9 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) old_core_mask, core_mask); } } + + if (mmu_sync_needed) + up_write(&kbdev->csf.mmu_sync_sem); #endif dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", pm_backend->ca_cores_enabled); @@ -105,6 +115,10 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) return; unlock: spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); +#if MALI_USE_CSF + if (mmu_sync_needed) + up_write(&kbdev->csf.mmu_sync_sem); +#endif } KBASE_EXPORT_TEST_API(kbase_devfreq_set_core_mask); #endif diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h index 34c34df7f82f..a25fe6bdc912 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h @@ -114,6 +114,27 @@ enum kbase_pm_runtime_suspend_abort_reason { ABORT_REASON_NON_IDLE_CGS }; +/* The following indices point to the corresponding bits stored in + * &kbase_pm_backend_data.gpu_sleep_allowed. They denote the conditions that + * would be checked against to determine the level of support for GPU sleep + * and firmware sleep-on-idle. + */ +#define KBASE_GPU_SUPPORTS_GPU_SLEEP ((uint8_t)0) +#define KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE ((uint8_t)1) +#define KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED ((uint8_t)2) +#define KBASE_GPU_IGNORE_IDLE_EVENT ((uint8_t)3) +#define KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE ((uint8_t)4) + +/* FW sleep-on-idle could be enabled if + * &kbase_pm_backend_data.gpu_sleep_allowed is equal to this value. + */ +#define KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED \ + ((uint8_t)((1 << KBASE_GPU_SUPPORTS_GPU_SLEEP) | \ + (1 << KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE) | \ + (0 << KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED) | \ + (0 << KBASE_GPU_IGNORE_IDLE_EVENT) | \ + (0 << KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE))) + /** * struct kbasep_pm_metrics - Metrics data collected for use by the power * management framework. @@ -304,7 +325,7 @@ union kbase_pm_policy_data { * called previously. * See &struct kbase_pm_callback_conf. * @ca_cores_enabled: Cores that are currently available - * @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for BASE_HW_ISSUE_TITANHW_2938 + * @apply_hw_issue_TITANHW_2938_wa: Indicates if the workaround for KBASE_HW_ISSUE_TITANHW_2938 * needs to be applied when unmapping memory from GPU. * @mcu_state: The current state of the micro-control unit, only applicable * to GPUs that have such a component @@ -350,10 +371,9 @@ union kbase_pm_policy_data { * @core_idle_work: Work item used to wait for undesired cores to become inactive. * The work item is enqueued when Host controls the power for * shader cores and down scaling of cores is performed. - * @gpu_sleep_supported: Flag to indicate that if GPU sleep feature can be - * supported by the kernel driver or not. If this - * flag is not set, then HW state is directly saved - * when GPU idle notification is received. + * @gpu_sleep_allowed: Bitmask to indicate the conditions that would be + * used to determine what support for GPU sleep is + * available. * @gpu_sleep_mode_active: Flag to indicate that the GPU needs to be in sleep * mode. It is set when the GPU idle notification is * received and is cleared when HW state has been @@ -497,7 +517,7 @@ struct kbase_pm_backend_data { struct work_struct core_idle_work; #ifdef KBASE_PM_RUNTIME - bool gpu_sleep_supported; + unsigned long gpu_sleep_allowed; bool gpu_sleep_mode_active; bool exit_gpu_sleep_mode; bool gpu_idled; diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c index 9e85cf5589f4..c6b6f3a8668a 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c @@ -47,9 +47,7 @@ #include #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if MALI_USE_CSF #include @@ -615,11 +613,11 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) /* * Skip if it is not supported */ - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) return; #if MALI_USE_CSF - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) { val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG)); kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(L2_CONFIG), L2_CONFIG_PBHA_HWU_SET(val, kbdev->pbha_propagate_bits)); @@ -743,16 +741,8 @@ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state s } #ifdef KBASE_PM_RUNTIME -/** - * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on - * MCU side - * - * @kbdev: Pointer to the device. - * - * This function is called to re-enable the Doorbell notification on MCU side - * when MCU needs to beome active again. - */ -static void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev) + +void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev) { u32 val = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL)); @@ -778,7 +768,7 @@ static void wait_mcu_as_inactive(struct kbase_device *kbdev) kbase_get_timeout_ms(kbdev, KBASE_AS_INACTIVE_TIMEOUT) * USEC_PER_MSEC; lockdep_assert_held(&kbdev->hwaccess_lock); - if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TURSEHW_2716)) + if (!kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TURSEHW_2716)) return; /* Wait for the AS_ACTIVE_INT bit to become 0 for the AS used by MCU FW */ @@ -927,6 +917,18 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) if (kbase_pm_is_mcu_desired(kbdev) && !backend->policy_change_clamp_state_to_off && backend->l2_state == KBASE_L2_ON) { + kbdev->csf.mcu_halted = false; + + /* Ensure that FW would not go to sleep immediately after + * resumption. + */ + kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface, + GLB_REQ, + GLB_REQ_REQ_IDLE_DISABLE, + GLB_REQ_IDLE_DISABLE_MASK); + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, false); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false); + kbase_csf_firmware_trigger_reload(kbdev); backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD; } @@ -1005,7 +1007,6 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) case KBASE_MCU_ON: backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); - if (!kbase_pm_is_mcu_desired(kbdev)) backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; else if (kbdev->csf.firmware_hctl_core_pwr) { @@ -1185,7 +1186,7 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) break; case KBASE_MCU_POWER_DOWN: - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TITANHW_2922)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TITANHW_2922)) { if (!kbdev->csf.firmware_hctl_core_pwr) kbasep_pm_toggle_power_interrupt(kbdev, true); backend->mcu_state = KBASE_MCU_OFF; @@ -1206,7 +1207,20 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) #ifdef KBASE_PM_RUNTIME case KBASE_MCU_ON_SLEEP_INITIATE: if (!kbase_pm_is_mcu_desired(kbdev)) { - kbase_csf_firmware_trigger_mcu_sleep(kbdev); + bool db_notif_disabled = false; + + if (likely(test_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, + &kbdev->pm.backend.gpu_sleep_allowed))) + db_notif_disabled = + kbase_reg_read32(kbdev, + GPU_CONTROL_ENUM(MCU_CONTROL)) & + MCU_CNTRL_DOORBELL_DISABLE_MASK; + + /* If DB notification is enabled on FW side then send a sleep + * request to FW. + */ + if (!db_notif_disabled) + kbase_csf_firmware_trigger_mcu_sleep(kbdev); backend->mcu_state = KBASE_MCU_ON_PEND_SLEEP; } else backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; @@ -1240,6 +1254,16 @@ static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) case KBASE_MCU_IN_SLEEP: if (kbase_pm_is_mcu_desired(kbdev) && backend->l2_state == KBASE_L2_ON) { wait_mcu_as_inactive(kbdev); + /* Ensure that FW would not go to sleep immediately after + * resumption. + */ + kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface, + GLB_REQ, + GLB_REQ_REQ_IDLE_DISABLE, + GLB_REQ_IDLE_DISABLE_MASK); + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, false); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false); + KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_WAKEUP( kbdev, kbase_backend_get_cycle_cnt(kbdev)); kbase_pm_enable_mcu_db_notification(kbdev); @@ -1384,20 +1408,6 @@ static bool need_tiler_control(struct kbase_device *kbdev) #endif } -/** - * hctl_l2_power_down - Initiate power down of L2 cache - * - * @kbdev: The kbase device structure for the device. - * - * This function initiates the power down of L2 cache when Host controls the power - * for Tiler block. The function expects that power down of Tiler to already have - * been initiated and it triggers the L2 power down only after the power down for - * Tiler is complete. - * The function shall be called only if L2 is in ready state. - */ -static void hctl_l2_power_down(struct kbase_device *kbdev) -{ -} /** * hctl_tiler_power_up_done - Check and/or initiate power up of Tiler @@ -1444,7 +1454,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) u64 l2_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2); u64 l2_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2); -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores * are vulnerable to corruption if gpu is lost @@ -1473,7 +1482,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) } break; } -#endif /* mask off ready from trans in case transitions finished * between the register reads @@ -1574,7 +1582,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_RESTORE_CLOCKS: /* We always assume only GPUs being affected by - * BASE_HW_ISSUE_GPU2017_1336 fall into this state + * KBASE_HW_ISSUE_GPU2017_1336 fall into this state */ WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); @@ -1676,7 +1684,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_SLOW_DOWN_CLOCKS: /* We always assume only GPUs being affected by - * BASE_HW_ISSUE_GPU2017_1336 fall into this state + * KBASE_HW_ISSUE_GPU2017_1336 fall into this state */ WARN_ON_ONCE(!kbdev->pm.backend.gpu_clock_slow_down_wa); @@ -1725,11 +1733,6 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) case KBASE_L2_PEND_OFF: if (likely(!backend->l2_always_on)) { - if (need_tiler_control(kbdev) && l2_ready) { - hctl_l2_power_down(kbdev); - break; - } - if (l2_trans || l2_ready) break; } else if (kbdev->cache_clean_in_progress) @@ -1744,11 +1747,10 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) } #endif /* Disabling MCU after L2 cache power down is to address - * BASE_HW_ISSUE_TITANHW_2922 hardware issue. + * KBASE_HW_ISSUE_TITANHW_2922 hardware issue. */ if (backend->l2_force_off_after_mcu_halt) { - kbase_csf_firmware_disable_mcu(kbdev); - kbase_csf_firmware_disable_mcu_wait(kbdev); + kbase_csf_stop_firmware_and_wait(kbdev); WARN_ON_ONCE(backend->mcu_state != KBASE_MCU_OFF); backend->l2_force_off_after_mcu_halt = false; } @@ -1895,12 +1897,7 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores * are vulnerable to corruption if gpu is lost */ - if (kbase_is_gpu_removed(kbdev) -#ifdef CONFIG_MALI_ARBITER_SUPPORT - || kbase_pm_is_gpu_lost(kbdev)) { -#else - ) { -#endif + if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n"); break; @@ -2005,9 +2002,8 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) kbdev, KBASE_PM_POLICY_EVENT_IDLE); if (kbdev->pm.backend.protected_transition_override || -#ifdef CONFIG_MALI_ARBITER_SUPPORT - kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev) || -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + (kbase_has_arbiter(kbdev) && (kbase_pm_is_suspending(kbdev) || + kbase_pm_is_gpu_lost(kbdev))) || !stt->configured_ticks || WARN_ON(stt->cancel_queued)) { backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; @@ -2074,10 +2070,9 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) kbdev, KBASE_PM_POLICY_EVENT_TIMER_MISS); backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; -#ifdef CONFIG_MALI_ARBITER_SUPPORT - } else if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { + } else if (kbase_has_arbiter(kbdev) && + (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev))) { backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ } break; @@ -2096,7 +2091,7 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) if (!backend->partial_shaderoff) shader_poweroff_timer_queue_cancel(kbdev); - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) { + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) { kbase_gpu_start_cache_clean_nolock(kbdev, GPU_COMMAND_CACHE_CLN_INV_L2); backend->shaders_state = KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON; @@ -2446,6 +2441,9 @@ void kbase_pm_reset_complete(struct kbase_device *kbdev) backend->in_reset = false; #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) backend->gpu_wakeup_override = false; + backend->db_mirror_interrupt_enabled = false; + backend->gpu_sleep_mode_active = false; + backend->exit_gpu_sleep_mode = false; #endif kbase_pm_update_state(kbdev); @@ -2670,12 +2668,9 @@ static int pm_wait_for_poweroff_work_complete(struct kbase_device *kbdev, bool k const long timeout = kbase_csf_timeout_in_jiffies( kbase_get_timeout_ms(kbdev, CSF_PM_TIMEOUT) + extra_wait_time_ms); #else -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* Handling of timeout error isn't supported for arbiter builds */ - const long timeout = MAX_SCHEDULE_TIMEOUT; -#else - const long timeout = (long)msecs_to_jiffies(PM_TIMEOUT_MS); -#endif + const long timeout = kbase_has_arbiter(kbdev) ? MAX_SCHEDULE_TIMEOUT : + (long)msecs_to_jiffies(PM_TIMEOUT_MS); #endif int err = 0; @@ -2796,7 +2791,8 @@ static void update_user_reg_page_mapping(struct kbase_device *kbdev) * when the context (user process) needs to access to the page. */ unmap_mapping_range(kbdev->csf.user_reg.filp->f_inode->i_mapping, - kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, 1); + (loff_t)kctx->csf.user_reg.file_offset << PAGE_SHIFT, PAGE_SIZE, + 1); list_del_init(&kctx->csf.user_reg.link); dev_dbg(kbdev->dev, "Updated USER Reg page mapping of ctx %d_%d", kctx->tgid, kctx->id); @@ -2823,12 +2819,10 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) #endif /* !MALI_USE_CSF */ lockdep_assert_held(&kbdev->pm.lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) { dev_err(kbdev->dev, "%s: Cannot power up while GPU lost", __func__); return; } -#endif if (backend->gpu_powered) { #if MALI_USE_CSF && defined(KBASE_PM_RUNTIME) @@ -2876,10 +2870,8 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) * consistent state */ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); - } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - else { - if (kbdev->arb.arb_if) { + } else { + if (kbase_has_arbiter(kbdev)) { struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; /* In the case that the GPU has just been granted by @@ -2895,8 +2887,8 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) * that a repartitioning occurred. In this case the current config * should be read again. */ - kbase_gpuprops_get_curr_config_props(kbdev, &kbdev->gpu_props.curr_config); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + kbase_gpuprops_get_curr_config_props(kbdev, &kbdev->gpu_props.curr_config); mutex_lock(&kbdev->mmu_hw_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -2988,12 +2980,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) } #endif - if (kbase_is_gpu_removed(kbdev) -#ifdef CONFIG_MALI_ARBITER_SUPPORT - || kbase_pm_is_gpu_lost(kbdev)) { -#else - ) { -#endif + if (kbase_is_gpu_removed(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { /* Ensure we unblock any threads that are stuck waiting * for the GPU */ @@ -3011,10 +2998,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) /* GPU is about to be turned off, switch to dummy page */ update_user_reg_page_mapping(kbdev); #endif - -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ if (kbdev->pm.backend.callback_power_off) kbdev->pm.backend.callback_power_off(kbdev); @@ -3068,6 +3052,7 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) return HRTIMER_NORESTART; } + static int kbase_set_gpu_quirks(struct kbase_device *kbdev) { #if MALI_USE_CSF @@ -3097,7 +3082,7 @@ static int kbase_set_gpu_quirks(struct kbase_device *kbdev) kbdev->hw_quirks_gpu = hw_quirks_gpu; #endif /* !MALI_USE_CSF */ - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_IDVS_GROUP_SIZE)) { u32 default_idvs_group_size = 0xF; u32 group_size = 0; @@ -3131,10 +3116,10 @@ static int kbase_set_sc_quirks(struct kbase_device *kbdev) if (kbase_is_gpu_removed(kbdev)) return -EIO; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162)) hw_quirks_sc |= SC_VAR_ALGORITHM; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_TLS_HASHING)) hw_quirks_sc |= SC_TLS_HASH_ENABLE; kbdev->hw_quirks_sc = hw_quirks_sc; @@ -3153,7 +3138,7 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev) return -EIO; /* Set tiler clock gate override if required */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_T76X_3953)) hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE; kbdev->hw_quirks_tiler = hw_quirks_tiler; @@ -3370,9 +3355,8 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) /* The GPU doesn't seem to be responding to the reset so try a hard * reset, but only when NOT in arbitration mode. */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (!kbdev->arb.arb_if) { -#endif + + if (!kbase_has_arbiter(kbdev)) { dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT); @@ -3402,9 +3386,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", RESET_TIMEOUT); -#ifdef CONFIG_MALI_ARBITER_SUPPORT } -#endif return -EINVAL; } @@ -3487,6 +3469,7 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) kbase_amba_set_shareable_cache_support(kbdev); #if MALI_USE_CSF kbase_backend_update_gpu_timestamp_offset(kbdev); + kbdev->csf.compute_progress_timeout_cc = 0; #endif /* Sanity check protected mode was left after reset */ diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h index 033c80a7c6b4..a7fa191b89d1 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -821,6 +821,21 @@ bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev); */ bool kbase_pm_is_mcu_inactive(struct kbase_device *kbdev, enum kbase_mcu_state state); +#ifdef KBASE_PM_RUNTIME + +/** + * kbase_pm_enable_mcu_db_notification - Enable the Doorbell notification on + * MCU side + * + * @kbdev: Pointer to the device. + * + * This function is called to re-enable the Doorbell notification on MCU side + * when MCU needs to beome active again. + */ +void kbase_pm_enable_mcu_db_notification(struct kbase_device *kbdev); + +#endif /* KBASE_PM_RUNTIME */ + /** * kbase_pm_idle_groups_sched_suspendable - Check whether the scheduler can be * suspended to low power state when all @@ -963,11 +978,29 @@ static inline bool kbase_pm_gpu_sleep_allowed(struct kbase_device *kbdev) * A high positive value of autosuspend_delay can be used to keep the * GPU in sleep state for a long time. */ - if (unlikely(!kbdev->dev->power.autosuspend_delay || - (kbdev->dev->power.autosuspend_delay < 0))) + if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) return false; - return kbdev->pm.backend.gpu_sleep_supported; + return test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); +} + +/** + * kbase_pm_fw_sleep_on_idle_allowed - Check if FW sleep-on-idle could be enabled + * + * @kbdev: Device pointer + * + * This function should be called whenever the conditions that impact + * FW sleep-on-idle support change so that it could be enabled/disabled + * accordingly. + * + * Return: true if FW sleep-on-idle is allowed + */ +static inline bool kbase_pm_fw_sleep_on_idle_allowed(struct kbase_device *kbdev) +{ + if (unlikely(kbdev->dev->power.autosuspend_delay <= 0)) + return false; + + return kbdev->pm.backend.gpu_sleep_allowed == KBASE_GPU_FW_SLEEP_ON_IDLE_ALLOWED; } /** diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c index c8e3f40335d6..457e91a0a978 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c @@ -77,7 +77,16 @@ void kbase_pm_policy_init(struct kbase_device *kbdev) spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbdev->pm.backend.pm_current_policy = default_policy; kbdev->pm.backend.csf_pm_sched_flags = default_policy->pm_sched_flags; + +#ifdef KBASE_PM_RUNTIME + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); +#endif /* KBASE_PM_RUNTIME */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + #else CSTD_UNUSED(flags); kbdev->pm.backend.pm_current_policy = default_policy; @@ -400,6 +409,13 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, const struct kbase_pm_polic /* New policy in place, release the clamping on mcu/L2 off state */ kbdev->pm.backend.policy_change_clamp_state_to_off = false; kbase_pm_update_state(kbdev); + +#ifdef KBASE_PM_RUNTIME + if (kbase_pm_idle_groups_sched_suspendable(kbdev)) + clear_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); + else + set_bit(KBASE_GPU_IGNORE_IDLE_EVENT, &kbdev->pm.backend.gpu_sleep_allowed); +#endif /* KBASE_PM_RUNTIME */ #endif spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); diff --git a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c index 331c26c6a310..d3715d97d23c 100644 --- a/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c +++ b/drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c @@ -46,7 +46,6 @@ static struct kbase_timeout_info timeout_info[KBASE_TIMEOUT_SELECTOR_COUNT] = { CSF_FIRMWARE_PING_TIMEOUT_CYCLES) }, [CSF_PM_TIMEOUT] = { "CSF_PM_TIMEOUT", CSF_PM_TIMEOUT_CYCLES }, [CSF_GPU_RESET_TIMEOUT] = { "CSF_GPU_RESET_TIMEOUT", CSF_GPU_RESET_TIMEOUT_CYCLES }, - [CSF_CSG_SUSPEND_TIMEOUT] = { "CSF_CSG_SUSPEND_TIMEOUT", CSF_CSG_SUSPEND_TIMEOUT_CYCLES }, [CSF_CSG_TERM_TIMEOUT] = { "CSF_CSG_TERM_TIMEOUT", CSF_CSG_TERM_TIMEOUT_CYCLES }, [CSF_FIRMWARE_BOOT_TIMEOUT] = { "CSF_FIRMWARE_BOOT_TIMEOUT", CSF_FIRMWARE_BOOT_TIMEOUT_CYCLES }, @@ -236,6 +235,15 @@ void kbase_device_set_timeout_ms(struct kbase_device *kbdev, enum kbase_timeout_ } selector_str = timeout_info[selector].selector_str; +#if MALI_USE_CSF + if (IS_ENABLED(CONFIG_MALI_REAL_HW) && !IS_ENABLED(CONFIG_MALI_IS_FPGA) && + unlikely(timeout_ms >= MAX_TIMEOUT_MS)) { + dev_warn(kbdev->dev, "%s is capped from %dms to %dms\n", + timeout_info[selector].selector_str, timeout_ms, MAX_TIMEOUT_MS); + timeout_ms = MAX_TIMEOUT_MS; + } +#endif + kbdev->backend_time.device_scaled_timeouts[selector] = timeout_ms; dev_dbg(kbdev->dev, "\t%-35s: %ums\n", selector_str, timeout_ms); } diff --git a/drivers/gpu/arm/bifrost/build.bp b/drivers/gpu/arm/bifrost/build.bp index 861282bd4fab..2ae771f5b546 100644 --- a/drivers/gpu/arm/bifrost/build.bp +++ b/drivers/gpu/arm/bifrost/build.bp @@ -113,7 +113,7 @@ bob_defaults { mali_hw_errata_1485982_use_clock_alternative: { kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"], }, - platform_is_fpga: { + mali_is_fpga: { kbuild_options: ["CONFIG_MALI_IS_FPGA=y"], }, mali_coresight: { @@ -148,7 +148,6 @@ bob_defaults { // is an umbrella feature that would be open for inappropriate use // (catch-all for experimental CS code without separating it into // different features). - "MALI_INCREMENTAL_RENDERING_JM={{.incremental_rendering_jm}}", "MALI_BASE_CSF_PERFORMANCE_TESTS={{.base_csf_performance_tests}}", ], } @@ -162,6 +161,9 @@ bob_kernel_module { "*.c", "*.h", "Kbuild", + "arbiter/*.c", + "arbiter/*.h", + "arbiter/Kbuild", "backend/gpu/*.c", "backend/gpu/*.h", "backend/gpu/Kbuild", @@ -257,13 +259,6 @@ bob_kernel_module { "ipa/backend/*_csf.h", ], }, - mali_arbiter_support: { - srcs: [ - "arbiter/*.c", - "arbiter/*.h", - "arbiter/Kbuild", - ], - }, kbuild_options: [ "CONFIG_MALI_BIFROST=m", "CONFIG_MALI_KUTF=n", diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c index f973d39ebb22..fe1dbfaca872 100644 --- a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_csf.c @@ -186,17 +186,15 @@ void kbase_destroy_context(struct kbase_context *kctx) * Customer side that a hang could occur if context termination is * not blocked until the resume of GPU device. */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_inc(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_inc(&kbdev->pm.gpu_users_waiting); while (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); wait_event(kbdev->pm.resume_wait, !kbase_pm_is_suspending(kbdev)); } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_dec(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_dec(&kbdev->pm.gpu_users_waiting); /* Have synchronized against the System suspend and incremented the * pm.active_count. So any subsequent invocation of System suspend diff --git a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c index 06c2ed813de3..ef474f625f63 100644 --- a/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c +++ b/drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c @@ -231,14 +231,13 @@ void kbase_destroy_context(struct kbase_context *kctx) if (WARN_ON(!kbdev)) return; - /* Context termination could happen whilst the system suspend of + /* Context termination could happen whilst the system suspend of * the GPU device is ongoing or has completed. It has been seen on * Customer side that a hang could occur if context termination is * not blocked until the resume of GPU device. */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_inc(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_inc(&kbdev->pm.gpu_users_waiting); while (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { dev_dbg(kbdev->dev, "Suspend in progress when destroying context"); @@ -255,9 +254,8 @@ void kbase_destroy_context(struct kbase_context *kctx) */ wait_event(kbdev->pm.resume_wait, !kbase_pm_is_resuming(kbdev)); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_dec(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kbdev)) + atomic_dec(&kbdev->pm.gpu_users_waiting); kbase_mem_pool_group_mark_dying(&kctx->mem_pools); diff --git a/drivers/gpu/arm/bifrost/csf/Kbuild b/drivers/gpu/arm/bifrost/csf/Kbuild index 5df35864efc7..8159bc9d10e8 100644 --- a/drivers/gpu/arm/bifrost/csf/Kbuild +++ b/drivers/gpu/arm/bifrost/csf/Kbuild @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software @@ -48,8 +48,10 @@ bifrost_kbase-y += \ ifeq ($(CONFIG_MALI_BIFROST_NO_MALI),y) bifrost_kbase-y += csf/mali_kbase_csf_firmware_no_mali.o +bifrost_kbase-y += csf/mali_kbase_csf_fw_io_no_mali.o else bifrost_kbase-y += csf/mali_kbase_csf_firmware.o +bifrost_kbase-y += csf/mali_kbase_csf_fw_io.o endif bifrost_kbase-$(CONFIG_DEBUG_FS) += csf/mali_kbase_debug_csf_fault.o diff --git a/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c b/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c index 61a4be9ccc94..ec47b88fac53 100644 --- a/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c +++ b/drivers/gpu/arm/bifrost/csf/ipa_control/mali_kbase_csf_ipa_control.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -943,6 +943,8 @@ void kbase_ipa_control_protm_entered(struct kbase_device *kbdev) struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; lockdep_assert_held(&kbdev->hwaccess_lock); + + ipa_ctrl->protm_start = ktime_get_raw_ns(); } @@ -955,6 +957,7 @@ void kbase_ipa_control_protm_exited(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->hwaccess_lock); + for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) { struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i]; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c index 9a33169ec554..d3300ea8dcde 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf.c @@ -45,6 +45,9 @@ #define CS_RING_BUFFER_MAX_SIZE ((uint32_t)(1 << 31)) /* 2GiB */ #define CS_RING_BUFFER_MIN_SIZE ((uint32_t)4096) +/* 0.2 second assuming 600 MHz GPU clock, which is double of iterator disabling timeout */ +#define MAX_PROGRESS_TIMEOUT_EVENT_DELAY ((u32)120000000) + #define PROTM_ALLOC_MAX_RETRIES ((u8)5) const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT] = { @@ -1261,6 +1264,7 @@ static int create_queue_group(struct kbase_context *const kctx, INIT_LIST_HEAD(&group->error_fatal.link); INIT_WORK(&group->timer_event_work, timer_event_worker); INIT_LIST_HEAD(&group->protm_event_work); + group->progress_timer_state = 0; atomic_set(&group->pending_protm_event_work, 0); bitmap_zero(group->protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); @@ -1306,12 +1310,6 @@ int kbase_csf_queue_group_create(struct kbase_context *const kctx, const u32 tiler_count = hweight64(create->in.tiler_mask); const u32 fragment_count = hweight64(create->in.fragment_mask); const u32 compute_count = hweight64(create->in.compute_mask); - size_t i; - - for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) - return -EINVAL; - } mutex_lock(&kctx->csf.lock); @@ -2178,19 +2176,74 @@ static void timer_event_worker(struct work_struct *data) } /** - * handle_progress_timer_event() - Progress timer timeout event handler. + * handle_progress_timer_events() - Progress timer timeout events handler. * - * @group: Pointer to GPU queue group for which the timeout event is received. + * @kbdev: Instance of a GPU platform device that implements a CSF interface. + * @slot_mask: Bitmap reflecting the slots on which progress timer timeouts happen. * * Notify a waiting user space client of the timeout. * Enqueue a work item to terminate the group and notify the event notification * thread of progress timeout fault for the GPU command queue group. + * Ignore fragment timeout if it is following a compute timeout. */ -static void handle_progress_timer_event(struct kbase_queue_group *const group) +static void handle_progress_timer_events(struct kbase_device *const kbdev, unsigned long *slot_mask) { - kbase_debug_csf_fault_notify(group->kctx->kbdev, group->kctx, DF_PROGRESS_TIMER_TIMEOUT); + u32 max_csg_slots = kbdev->csf.global_iface.group_num; + u32 csg_nr; + struct kbase_queue_group *group = NULL; + struct kbase_csf_cmd_stream_group_info *ginfo; - queue_work(group->kctx->csf.wq, &group->timer_event_work); + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + if (likely(bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS))) + return; + + /* Log each timeout and Update timestamp of compute progress timeout */ + for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { + group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; + ginfo = &kbdev->csf.global_iface.groups[csg_nr]; + group->progress_timer_state = + kbase_csf_firmware_csg_output(ginfo, CSG_PROGRESS_TIMER_STATE); + + dev_info( + kbdev->dev, + "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u with state %x", + kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, + group->kctx->id, csg_nr, group->progress_timer_state); + + if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_COMPUTE) + kbdev->csf.compute_progress_timeout_cc = kbase_backend_get_cycle_cnt(kbdev); + } + + /* Ignore fragment timeout if it is following a compute timeout. + * Otherwise, terminate the command stream group. + */ + for_each_set_bit(csg_nr, slot_mask, max_csg_slots) { + group = kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; + + /* Check if it is a fragment timeout right after another compute timeout. + * In such case, kill compute CSG and give fragment CSG a second chance + */ + if (CSG_PROGRESS_TIMER_STATE_GET(group->progress_timer_state) == + CSG_PROGRESS_TIMER_STATE_FRAGMENT) { + u64 cycle_counter = kbase_backend_get_cycle_cnt(kbdev); + u64 compute_progress_timeout_cc = kbdev->csf.compute_progress_timeout_cc; + + if (compute_progress_timeout_cc <= cycle_counter && + cycle_counter <= compute_progress_timeout_cc + + MAX_PROGRESS_TIMEOUT_EVENT_DELAY) { + dev_info( + kbdev->dev, + "Ignored Fragment iterator timeout for group %d on slot %d", + group->handle, group->csg_nr); + continue; + } + } + + kbase_debug_csf_fault_notify(group->kctx->kbdev, group->kctx, + DF_PROGRESS_TIMER_TIMEOUT); + queue_work(group->kctx->csf.wq, &group->timer_event_work); + } } /** @@ -2291,14 +2344,14 @@ static void handle_fault_event(struct kbase_queue *const queue, const u32 cs_ack const u8 cs_fault_exception_type = CS_FAULT_EXCEPTION_TYPE_GET(cs_fault); const u32 cs_fault_exception_data = CS_FAULT_EXCEPTION_DATA_GET(cs_fault); const u64 cs_fault_info_exception_data = CS_FAULT_INFO_EXCEPTION_DATA_GET(cs_fault_info); - bool use_old_log_format = true; + bool has_trace_info = false; bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP); kbase_csf_scheduler_spin_lock_assert_held(kbdev); - if (use_old_log_format && !skip_fault_report) + if (!has_trace_info && !skip_fault_report) dev_warn(kbdev->dev, "Ctx %d_%d Group %d CSG %d CSI: %d\n" "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" @@ -2478,13 +2531,13 @@ static void handle_fatal_event(struct kbase_queue *const queue, const u32 cs_fatal_exception_type = CS_FATAL_EXCEPTION_TYPE_GET(cs_fatal); const u32 cs_fatal_exception_data = CS_FATAL_EXCEPTION_DATA_GET(cs_fatal); const u64 cs_fatal_info_exception_data = CS_FATAL_INFO_EXCEPTION_DATA_GET(cs_fatal_info); - bool use_old_log_format = true; + bool has_trace_info = false; bool skip_fault_report = kbase_ctx_flag(queue->kctx, KCTX_PAGE_FAULT_REPORT_SKIP); kbase_csf_scheduler_spin_lock_assert_held(kbdev); - if (use_old_log_format && !skip_fault_report) + if (!has_trace_info && !skip_fault_report) dev_warn(kbdev->dev, "Ctx %d_%d Group %d CSG %d CSI: %d\n" "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" @@ -2649,6 +2702,8 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, * @csg_nr: CSG number. * @track: Pointer that tracks the highest idle CSG and the newly possible viable * protected mode requesting group, in current IRQ context. + * @progress_timeout_slot_mask: slot mask to indicate on which slot progress timeout + * happens. * * Handles interrupts for a CSG and for CSs within it. * @@ -2660,7 +2715,8 @@ static void process_cs_interrupts(struct kbase_queue_group *const group, * See process_cs_interrupts() for details of per-stream interrupt handling. */ static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const csg_nr, - struct irq_idle_and_protm_track *track) + struct irq_idle_and_protm_track *track, + unsigned long *progress_timeout_slot_mask) { struct kbase_csf_cmd_stream_group_info *ginfo; struct kbase_queue_group *group = NULL; @@ -2747,13 +2803,9 @@ static void process_csg_interrupts(struct kbase_device *const kbdev, u32 const c KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_PROGRESS_TIMER_EVENT, group, req ^ ack); - dev_info( - kbdev->dev, - "[%llu] Iterator PROGRESS_TIMER timeout notification received for group %u of ctx %d_%d on slot %u\n", - kbase_backend_get_cycle_cnt(kbdev), group->handle, group->kctx->tgid, - group->kctx->id, csg_nr); - handle_progress_timer_event(group); + set_bit(csg_nr, progress_timeout_slot_mask); + } process_cs_interrupts(group, ginfo, irqreq, irqack, track); @@ -2863,6 +2915,7 @@ static inline void check_protm_enter_req_complete(struct kbase_device *kbdev, u3 dev_dbg(kbdev->dev, "Protected mode entry interrupt received"); kbdev->protected_mode = true; + kbase_ipa_protection_mode_switch_event(kbdev); kbase_ipa_control_protm_entered(kbdev); kbase_hwcnt_backend_csf_protm_entered(&kbdev->hwcnt_gpu_iface); @@ -3047,18 +3100,25 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) struct irq_idle_and_protm_track track = { .protm_grp = NULL, .idle_seq = U32_MAX, .idle_slot = S8_MAX }; + DECLARE_BITMAP(progress_timeout_csgs, MAX_SUPPORTED_CSGS) = { 0 }; kbase_csf_scheduler_spin_lock(kbdev, &flags); - /* Looping through and track the highest idle and protm groups */ + /* Looping through and track the highest idle and protm groups. + * Also track the groups for which progress timer timeout happened. + */ while (csg_interrupts != 0) { u32 const csg_nr = (u32)ffs((int)csg_interrupts) - 1; - process_csg_interrupts(kbdev, csg_nr, &track); + process_csg_interrupts(kbdev, csg_nr, &track, + progress_timeout_csgs); csg_interrupts &= ~(1U << csg_nr); } /* Handle protm from the tracked information */ process_tracked_info_for_protm(kbdev, &track); + /* Handle pending progress timeout(s) */ + handle_progress_timer_events(kbdev, progress_timeout_csgs); + kbase_csf_scheduler_spin_unlock(kbdev, flags); } @@ -3087,11 +3147,28 @@ void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) /* Handle IDLE Hysteresis notification event */ if ((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK) { + u32 const glb_idle_timer_cfg = + kbase_csf_firmware_global_input_read( + global_iface, GLB_IDLE_TIMER_CONFIG); + dev_dbg(kbdev->dev, "Idle-hysteresis event flagged"); kbase_csf_firmware_global_input_mask( global_iface, GLB_REQ, glb_ack, GLB_REQ_IDLE_EVENT_MASK); + if (glb_idle_timer_cfg & + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) { + /* The FW is going to sleep, we shall: + * - Enable fast GPU idle handling to avoid + * confirming CSGs status in gpu_idle_worker(). + * - Enable doorbell mirroring to minimise the + * chance of KBase raising kernel doorbells which + * would cause the FW to be woken up. + */ + kbdev->csf.scheduler.fast_gpu_idle_handling = true; + kbase_pm_enable_db_mirror_interrupt(kbdev); + } + glb_idle_irq_received = true; /* Defer handling this IRQ to account for a race condition * where the idle worker could be executed before we have diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c index c885845bc62e..32f33a58a6f7 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_csg_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -28,6 +28,8 @@ #include #include #include +#include +#include #define MAX_SCHED_STATE_STRING_LEN (16) /** @@ -268,6 +270,87 @@ static const struct file_operations kbasep_csf_debugfs_scheduler_state_fops = { .open = simple_open, .llseek = default_llseek, }; +static int kbasep_csf_debugfs_eviction_timeout_get(void *data, u64 *val) +{ + struct kbase_device *const kbdev = data; + unsigned long flags; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + *val = kbdev->csf.csg_suspend_timeout_ms - CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS; + kbase_csf_scheduler_spin_unlock(kbdev, flags); + + return 0; +} + +static int kbasep_csf_debugfs_eviction_timeout_set(void *data, u64 val) +{ + struct kbase_device *const kbdev = data; + unsigned long flags_schd, flags_hw; + u64 dur_ms = val; + int ret = 0; + + if (unlikely(dur_ms < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN || + dur_ms > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX)) { + dev_err(kbdev->dev, "Invalid CSG suspend timeout input (%llu)", dur_ms); + return -EFAULT; + } + dur_ms = dur_ms + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS; + + /* The 'fw_load_lock' is taken to synchronize against the deferred + * loading of FW, update will take effect after firmware gets loaded. + */ + mutex_lock(&kbdev->fw_load_lock); + if (unlikely(!kbdev->csf.firmware_inited)) { + kbase_csf_scheduler_spin_lock(kbdev, &flags_schd); + kbdev->csf.csg_suspend_timeout_ms = (unsigned int)dur_ms; + kbase_csf_scheduler_spin_unlock(kbdev, flags_schd); + mutex_unlock(&kbdev->fw_load_lock); + dev_info(kbdev->dev, "CSF set csg suspend timeout deferred till fw is loaded"); + goto end; + } + mutex_unlock(&kbdev->fw_load_lock); + + /* Firmware reloading is triggered by silent reset, and then update will take effect. + */ + kbase_csf_scheduler_pm_active(kbdev); + if (kbase_csf_scheduler_killable_wait_mcu_active(kbdev)) { + dev_err(kbdev->dev, + "Unable to activate the MCU, the csg suspend timeout value shall remain unchanged"); + kbase_csf_scheduler_pm_idle(kbdev); + ret = -EFAULT; + goto exit; + } + spin_lock_irqsave(&kbdev->hwaccess_lock, flags_hw); + if (kbase_reset_gpu_silent(kbdev)) { + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags_hw); + dev_err(kbdev->dev, "CSF set csg suspend timeout pending reset, try again"); + kbase_csf_scheduler_pm_idle(kbdev); + ret = -EFAULT; + goto exit; + } + /* GPU reset is placed and it will take place only after hwaccess_lock is released, + * update on host side should be done after GPU reset is placed and before it takes place. + */ + kbase_csf_scheduler_spin_lock(kbdev, &flags_schd); + kbdev->csf.csg_suspend_timeout_ms = (unsigned int)dur_ms; + kbase_csf_scheduler_spin_unlock(kbdev, flags_schd); + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags_hw); + /* Keep PM active until reset finished to allow FW reloading to take place, + * and then update request will be sent to FW during initialization. + */ + kbase_reset_gpu_wait(kbdev); + kbase_csf_scheduler_pm_idle(kbdev); + +end: + dev_info(kbdev->dev, "CSF set csg suspend timeout: %u ms", (unsigned int)dur_ms); + +exit: + return ret; +} + +DEFINE_DEBUGFS_ATTRIBUTE(kbasep_csf_debugfs_eviction_timeout_fops, + &kbasep_csf_debugfs_eviction_timeout_get, + &kbasep_csf_debugfs_eviction_timeout_set, "%llu\n"); void kbase_csf_debugfs_init(struct kbase_device *kbdev) { @@ -280,6 +363,8 @@ void kbase_csf_debugfs_init(struct kbase_device *kbdev) &kbasep_csf_debugfs_scheduling_timer_kick_fops); debugfs_create_file("scheduler_state", 0644, kbdev->mali_debugfs_directory, kbdev, &kbasep_csf_debugfs_scheduler_state_fops); + debugfs_create_file("eviction_timeout_ms", 0644, kbdev->mali_debugfs_directory, kbdev, + &kbasep_csf_debugfs_eviction_timeout_fops); kbase_csf_tl_reader_debugfs_init(kbdev); } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h index 155c20aaa356..38e7cb940d97 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_defs.h @@ -30,6 +30,7 @@ #include "mali_kbase_csf_firmware.h" #include "mali_kbase_csf_event.h" #include +#include "mali_kbase_csf_fw_io.h" #include @@ -267,7 +268,6 @@ enum kbase_queue_group_priority { * @CSF_PM_TIMEOUT: Timeout for GPU Power Management to reach the desired * Shader, L2 and MCU state. * @CSF_GPU_RESET_TIMEOUT: Waiting timeout for GPU reset to complete. - * @CSF_CSG_SUSPEND_TIMEOUT: Timeout given for a CSG to be suspended. * @CSF_CSG_TERM_TIMEOUT: Timeout given for a CSG to be terminated. * @CSF_FIRMWARE_BOOT_TIMEOUT: Maximum time to wait for firmware to boot. * @CSF_FIRMWARE_PING_TIMEOUT: Maximum time to wait for firmware to respond @@ -290,7 +290,6 @@ enum kbase_timeout_selector { CSF_FIRMWARE_TIMEOUT, CSF_PM_TIMEOUT, CSF_GPU_RESET_TIMEOUT, - CSF_CSG_SUSPEND_TIMEOUT, CSF_CSG_TERM_TIMEOUT, CSF_FIRMWARE_BOOT_TIMEOUT, CSF_FIRMWARE_PING_TIMEOUT, @@ -554,6 +553,8 @@ struct kbase_protected_suspend_buffer { * returned to userspace if such an error has occurred. * @timer_event_work: Work item to handle the progress timeout fatal event * for the group. + * @progress_timer_state: Value of CSG_PROGRESS_TIMER_STATE register when progress + * timer timeout is reported for the group. * @deschedule_deferred_cnt: Counter keeping a track of the number of threads * that tried to deschedule the group and had to defer * the descheduling due to the dump on fault. @@ -610,6 +611,7 @@ struct kbase_queue_group { struct kbase_csf_notification error_fatal; struct work_struct timer_event_work; + u32 progress_timer_state; /** * @dvs_buf: Address and size of scratch memory. @@ -931,13 +933,11 @@ struct kbase_csf_reset_gpu { * of CSG slots. * @resident_group: pointer to the queue group that is resident on the CSG slot. * @state: state of the slot as per enum @kbase_csf_csg_slot_state. - * @trigger_jiffies: value of jiffies when change in slot state is recorded. * @priority: dynamic priority assigned to CSG slot. */ struct kbase_csf_csg_slot { struct kbase_queue_group *resident_group; atomic_t state; - unsigned long trigger_jiffies; u8 priority; }; @@ -1137,6 +1137,8 @@ struct kbase_csf_mcu_shared_regions { * @gpuq_kthread: Dedicated thread primarily used to handle * latency-sensitive tasks such as GPU queue * submissions. + * @gpu_idle_timer_enabled: Tracks whether the GPU idle timer is enabled or disabled. + * @fw_soi_enabled: True if FW Sleep-on-Idle is currently enabled. */ struct kbase_csf_scheduler { struct mutex lock; @@ -1214,6 +1216,8 @@ struct kbase_csf_scheduler { */ spinlock_t gpu_metrics_lock; #endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */ + atomic_t gpu_idle_timer_enabled; + atomic_t fw_soi_enabled; }; /* @@ -1677,6 +1681,7 @@ struct kbase_csf_user_reg { * @gpu_idle_dur_count_no_modifier: Update csffw_glb_req_idle_enable to make the shr(10) * modifier conditional on the new flag * in GLB_IDLE_TIMER_CONFIG. + * @csg_suspend_timeout_ms: Timeout given for a CSG to be suspended. * for any request sent to the firmware. * @hwcnt: Contain members required for handling the dump of * HW counters. @@ -1695,8 +1700,21 @@ struct kbase_csf_user_reg { * kbase_queue.pending_kick_link. * @quirks_ext: Pointer to an allocated buffer containing the firmware * workarounds configuration. + * @mmu_sync_sem: RW Semaphore to defer MMU operations till the P.Mode entrance + * or DCS request has been completed. * @pmode_sync_sem: RW Semaphore to prevent MMU operations during P.Mode entrance. - * @gpu_idle_timer_enabled: Tracks whether the GPU idle timer is enabled or disabled. + * @page_fault_cnt_ptr_address: GPU VA of the location in FW data memory, extracted from the + * FW image header, that will store the GPU VA of FW visible + * memory location where the @page_fault_cnt value will be written to. + * @page_fault_cnt_ptr: CPU VA of the FW visible memory location where the @page_fault_cnt + * value will be written to. + * @page_fault_cnt: Counter that is incremented on every GPU page fault, just before the + * MMU is unblocked to retry the memory transaction that caused the GPU + * page fault. The access to counter is serialized appropriately. + * @mcu_halted: Flag to inform MCU FSM that the MCU has already halted. + * @fw_io: Firmware I/O interface. + * @compute_progress_timeout_cc: Value of GPU cycle count register when progress + * timer timeout is reported for the compute iterator. */ struct kbase_csf_device { struct kbase_mmu_table mcu_mmu; @@ -1734,6 +1752,7 @@ struct kbase_csf_device { u64 gpu_idle_hysteresis_ns; u32 gpu_idle_dur_count; u32 gpu_idle_dur_count_no_modifier; + u32 csg_suspend_timeout_ms; struct kbase_csf_hwcnt hwcnt; struct kbase_csf_mcu_fw fw; struct kbase_csf_firmware_log fw_log; @@ -1752,8 +1771,14 @@ struct kbase_csf_device { struct list_head pending_gpuq_kick_queues[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; spinlock_t pending_gpuq_kick_queues_lock; u32 *quirks_ext; + struct rw_semaphore mmu_sync_sem; struct rw_semaphore pmode_sync_sem; - bool gpu_idle_timer_enabled; + u32 page_fault_cnt_ptr_address; + u32 *page_fault_cnt_ptr; + u32 page_fault_cnt; + bool mcu_halted; + struct kbase_csf_fw_io fw_io; + u64 compute_progress_timeout_cc; }; /** diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c index 35f09028098c..2d8f96641181 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.c @@ -40,6 +40,7 @@ #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" #include #include +#include #include #include #include @@ -55,6 +56,7 @@ #include #include +#include #define MALI_MAX_DEFAULT_FIRMWARE_NAME_LEN ((size_t)64) #define DEFAULT_FW_NAME MALI_RELEASE_NAME".mali_csffw.bin" @@ -68,6 +70,7 @@ static unsigned int csf_firmware_boot_timeout_ms; module_param(csf_firmware_boot_timeout_ms, uint, 0444); MODULE_PARM_DESC(csf_firmware_boot_timeout_ms, "Maximum time to wait for firmware to boot."); +static bool kbase_iter_trace_enable; #ifdef CONFIG_MALI_BIFROST_DEBUG /* Makes Driver wait indefinitely for an acknowledgment for the different @@ -97,6 +100,7 @@ MODULE_PARM_DESC(fw_debug, "Enables effective use of a debugger for debugging fi #define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4) #define CSF_FIRMWARE_ENTRY_TYPE_BUILD_INFO_METADATA (6) #define CSF_FIRMWARE_ENTRY_TYPE_FUNC_CALL_LIST (7) +#define CSF_FIRMWARE_ENTRY_TYPE_PAGE_FAULT_CNT (8) #define CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP (9) #define CSF_FIRMWARE_CACHE_MODE_NONE (0ul << 3) @@ -115,7 +119,8 @@ MODULE_PARM_DESC(fw_debug, "Enables effective use of a debugger for debugging fi #define CSF_GLB_REQ_CFG_MASK \ (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK | \ - GLB_REQ_CFG_PWROFF_TIMER_MASK | GLB_REQ_IDLE_ENABLE_MASK) + GLB_REQ_CFG_PWROFF_TIMER_MASK | GLB_REQ_IDLE_ENABLE_MASK | \ + GLB_REQ_CFG_EVICTION_TIMER_MASK | GLB_REQ_ITER_TRACE_ENABLE_MASK) static inline u32 input_page_read(const u32 *const input, const u32 offset) { @@ -179,6 +184,92 @@ struct firmware_timeline_metadata { size_t size; }; +static void reinit_page_fault_cnt_firmware_memory(struct kbase_device *kbdev) +{ + if (!kbdev->csf.page_fault_cnt_ptr) + return; + + /* Store the GPU address of shared memory location, where the page fault counter + * value will be written, inside the FW data memory. + */ + kbase_csf_update_firmware_memory( + kbdev, kbdev->csf.page_fault_cnt_ptr_address, + (u32)((kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + + PAGE_SIZE - sizeof(u32))); + + *kbdev->csf.page_fault_cnt_ptr = kbdev->csf.page_fault_cnt = 0; +} + +static void init_page_fault_cnt_firmware_memory(struct kbase_device *kbdev) +{ + if (!kbdev->csf.page_fault_cnt_ptr_address) + return; + + if (WARN_ON_ONCE(!kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg)) + return; + + /* Save the CPU address of shared memory location where the page fault counter + * value will be written. + * The shared memory location comes from the last 4 bytes of the page that + * is allocated to maintain the extract offset value for different trace + * buffers. Only the first 4 bytes of every cacheline is used for the extract offset + * value. + */ + kbdev->csf.page_fault_cnt_ptr = + (u32 *)((u8 *)kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + PAGE_SIZE - + sizeof(u32)); + reinit_page_fault_cnt_firmware_memory(kbdev); +} + +/** + * set_iterator_trace_enable - Set the value for 'kbase_iter_trace_enable' global variable + * according to the value of GLB_FEATURES.ITER_TRACE_SUPPORTED bit, + * and the corresponding device tree entry. + * @kbdev: Kernel base device pointer + */ +static void set_iterator_trace_enable(struct kbase_device *kbdev) +{ + const struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; + bool dev_support_iter_trace = iface->features & GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK; + const void *dt_iter_trace_param; + unsigned int val; + + if (!dev_support_iter_trace) { + kbase_iter_trace_enable = false; + return; + } + + + /* check device tree for iterator trace enable property and + * fallback to "iter_trace_enable" if not found and try again + */ + dt_iter_trace_param = of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL); + + if (!dt_iter_trace_param) + dt_iter_trace_param = + of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL); + + val = (dt_iter_trace_param) ? be32_to_cpup(dt_iter_trace_param) : 0; + dev_dbg(kbdev->dev, "Iterator trace enable device-tree config value: %u", val); + + kbase_iter_trace_enable = val ? true : false; +} + +static void iterator_trace_reinit(struct kbase_device *kbdev) +{ + if (kbase_iter_trace_enable) { + kbase_csf_firmware_global_input_mask(&kbdev->csf.global_iface, GLB_REQ, + GLB_REQ_ITER_TRACE_ENABLE_MASK, + GLB_REQ_ITER_TRACE_ENABLE_MASK); + } +} + +static void iterator_trace_init(struct kbase_device *kbdev) +{ + set_iterator_trace_enable(kbdev); + iterator_trace_reinit(kbdev); +} + /* The shared interface area, used for communicating with firmware, is managed * like a virtual memory zone. Reserve the virtual space from that zone * corresponding to shared interface entry parsed from the firmware image. @@ -217,7 +308,7 @@ void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_DISABLE); } -static void wait_for_firmware_stop(struct kbase_device *kbdev) +void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) { u32 val; const u32 timeout_us = @@ -232,17 +323,12 @@ static void wait_for_firmware_stop(struct kbase_device *kbdev) KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_OFF(kbdev, kbase_backend_get_cycle_cnt(kbdev)); } -void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) -{ - wait_for_firmware_stop(kbdev); -} - -static void stop_csf_firmware(struct kbase_device *kbdev) +void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev) { /* Stop the MCU firmware */ kbase_csf_firmware_disable_mcu(kbdev); - wait_for_firmware_stop(kbdev); + kbase_csf_firmware_disable_mcu_wait(kbdev); } static void wait_for_firmware_boot(struct kbase_device *kbdev) @@ -261,7 +347,6 @@ static void wait_for_firmware_boot(struct kbase_device *kbdev) */ remaining = wait_event_timeout(kbdev->csf.event_wait, kbdev->csf.interrupt_received == true, wait_timeout); - if (!remaining) dev_err(kbdev->dev, "Timed out waiting for fw boot completion"); @@ -485,6 +570,8 @@ static int reload_fw_image(struct kbase_device *kbdev) kbdev->csf.firmware_full_reload_needed = false; kbase_csf_firmware_reload_trace_buffers_data(kbdev); + reinit_page_fault_cnt_firmware_memory(kbdev); + iterator_trace_reinit(kbdev); out: return ret; } @@ -1043,6 +1130,14 @@ static int load_firmware_entry(struct kbase_device *kbdev, const struct kbase_cs } kbase_csf_firmware_log_parse_logging_call_list_entry(kbdev, entry); return 0; + case CSF_FIRMWARE_ENTRY_TYPE_PAGE_FAULT_CNT: + /* Entry about the location of page fault counter */ + if (size < sizeof(*entry)) { + dev_err(kbdev->dev, "Page fault counter entry too short (size=%u)", size); + return -EINVAL; + } + kbdev->csf.page_fault_cnt_ptr_address = *entry; + return 0; case CSF_FIRMWARE_ENTRY_TYPE_CORE_DUMP: /* Core Dump section */ if (size < CORE_DUMP_ENTRY_START_ADDR_OFFSET + sizeof(*entry)) { @@ -1657,12 +1752,13 @@ static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK); } - kbdev->csf.gpu_idle_timer_enabled = set; + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, set); } static void enable_gpu_idle_timer(struct kbase_device *const kbdev) { struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + bool const fw_soi_allowed = kbase_pm_fw_sleep_on_idle_allowed(kbdev); kbase_csf_scheduler_spin_lock_assert_held(kbdev); @@ -1670,14 +1766,114 @@ static void enable_gpu_idle_timer(struct kbase_device *const kbdev) kbdev->csf.gpu_idle_dur_count); kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG, - kbdev->csf.gpu_idle_dur_count_no_modifier, + kbdev->csf.gpu_idle_dur_count_no_modifier + << GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_SHIFT, GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK); + kbase_csf_firmware_global_input_mask(global_iface, GLB_IDLE_TIMER_CONFIG, + fw_soi_allowed + << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT, + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK); set_gpu_idle_timer_glb_req(kbdev, true); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, fw_soi_allowed); dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x", kbdev->csf.gpu_idle_dur_count); } +/** + * convert_dur_to_suspend_count() - Convert CSG suspend timeout from ms to cycle count + * @kbdev: Instance of a GPU platform device that implements a CSF interface + * @dur_ms: Timeout value in ms + * @no_modifier: Indicate whether bit-shift is applied, 0 when applied, 1 otherwise + * + * Convert CSG suspend timeout from ms to cycle count, then generate a register value + * combining cycle count and timer source + * + * Return: Register value which will be stored into register GLB_EVICTION_TIMER. + */ +static u32 convert_dur_to_suspend_count(struct kbase_device *kbdev, const u64 dur_ms, + u32 *no_modifier) +{ + /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ + u64 freq = kbase_arch_timer_get_cntfrq(kbdev); + u64 dur_val = dur_ms; + u32 cnt_val_u32, reg_val_u32; + const bool src_system_timestamp = freq > 0; + const u8 SUSPEND_VAL_UNIT_SHIFT = 10; + + if (!src_system_timestamp) { + /* Get the cycle_counter source alternative */ + spin_lock(&kbdev->pm.clk_rtm.lock); + if (kbdev->pm.clk_rtm.clks[0]) + freq = kbdev->pm.clk_rtm.clks[0]->clock_val; + else + dev_err(kbdev->dev, "No GPU clock, unexpected intregration issue!"); + spin_unlock(&kbdev->pm.clk_rtm.lock); + + dev_info(kbdev->dev, + "No timestamp frequency, use cycle counter for csg suspend timeout!"); + } + + /* Formula for dur_val = (dur/1e3) * freq_HZ) */ + dur_val = dur_val * freq; + dur_val = div_u64(dur_val, MSEC_PER_SEC); + if (dur_val < S32_MAX) { + *no_modifier = 1; + } else { + dur_val = dur_val >> SUSPEND_VAL_UNIT_SHIFT; + *no_modifier = 0; + } + + /* Interface limits the value field to S32_MAX */ + cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; + + reg_val_u32 = GLB_EVICTION_TIMER_TIMEOUT_SET(0, cnt_val_u32); + /* add the source flag */ + reg_val_u32 = GLB_EVICTION_TIMER_TIMER_SOURCE_SET( + reg_val_u32, + (src_system_timestamp ? GLB_EVICTION_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP : + GLB_EVICTION_TIMER_TIMER_SOURCE_GPU_COUNTER)); + + return reg_val_u32; +} + +/** + * set_csg_suspend_timeout() - Update CSG suspend timeout setting on FW side + * + * @kbdev: Instance of a GPU platform device that implements a CSF interface + */ +static void set_csg_suspend_timeout(struct kbase_device *const kbdev) +{ + u32 dur_ms, dur_val; + u32 no_modifier = 0; + struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; + + kbase_csf_scheduler_spin_lock_assert_held(kbdev); + + dur_ms = kbdev->csf.csg_suspend_timeout_ms; + if (unlikely(dur_ms < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN + + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS || + dur_ms > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX + + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS)) { + dev_err(kbdev->dev, "Unexpected CSG suspend timeout: %ums, default to: %ums", + dur_ms, CSG_SUSPEND_TIMEOUT_MS); + kbdev->csf.csg_suspend_timeout_ms = CSG_SUSPEND_TIMEOUT_MS; + dur_ms = CSG_SUSPEND_TIMEOUT_MS; + } + dur_ms = dur_ms - CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS; + + dur_val = convert_dur_to_suspend_count(kbdev, dur_ms, &no_modifier); + + kbase_csf_firmware_global_input(global_iface, GLB_EVICTION_TIMER, dur_val); + + kbase_csf_firmware_global_input_mask(global_iface, GLB_EVICTION_TIMER_CONFIG, no_modifier, + GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK); + + set_global_request(global_iface, GLB_REQ_CFG_EVICTION_TIMER_MASK); + + dev_dbg(kbdev->dev, "Updating CSG suspend timeout with count-value: 0x%.8x", dur_val); +} + static bool global_debug_request_complete(struct kbase_device *const kbdev, u32 const req_mask) { struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; @@ -1766,7 +1962,8 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK | GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK | GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK | GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK | - GLB_REQ_DEBUG_CSF_REQ_MASK | GLB_ACK_IRQ_MASK_IDLE_ENABLE_MASK; + GLB_REQ_DEBUG_CSF_REQ_MASK | GLB_ACK_IRQ_MASK_IDLE_ENABLE_MASK | + GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK | GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_MASK; const struct kbase_csf_global_iface *const global_iface = &kbdev->csf.global_iface; unsigned long flags; @@ -1781,11 +1978,10 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); - /* The GPU idle timer is always enabled for simplicity. Checks will be - * done before scheduling the GPU idle worker to see if it is - * appropriate for the current power policy. + /* The csg suspend timeout is always enabled so customer has the flexibility to update it + * at any time. */ - enable_gpu_idle_timer(kbdev); + set_csg_suspend_timeout(kbdev); /* Unmask the interrupts */ kbase_csf_firmware_global_input(global_iface, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -1963,6 +2159,7 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) KBASE_KTRACE_ADD(kbdev, CSF_FIRMWARE_REBOOT, NULL, 0u); + /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -2068,23 +2265,34 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, * gets serialized. */ kbase_csf_scheduler_lock(kbdev); + while (atomic_read(&kbdev->csf.scheduler.pending_gpu_idle_work) > 0) { + kbase_csf_scheduler_unlock(kbdev); + kbase_csf_scheduler_wait_for_kthread_pending_work( + kbdev, &kbdev->csf.scheduler.pending_gpu_idle_work); + kbase_csf_scheduler_lock(kbdev); + } kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbdev->csf.gpu_idle_hysteresis_ns = dur_ns; kbdev->csf.gpu_idle_dur_count = hysteresis_val; kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; - if (kbdev->csf.gpu_idle_timer_enabled) { + if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) { /* Timer is already enabled. Disable the timer as FW only reads * the new idle timer value when timer is re-enabled. */ kbase_csf_firmware_disable_gpu_idle_timer(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); /* Ensure that the request has taken effect */ - wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) + dev_err(kbdev->dev, + "Failed to disable GLB_IDLE timer when setting a new idle hysteresis timeout"); kbase_csf_scheduler_spin_lock(kbdev, &flags); kbase_csf_firmware_enable_gpu_idle_timer(kbdev); kbase_csf_scheduler_spin_unlock(kbdev, flags); - wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK)) + dev_err(kbdev->dev, + "Failed to re-enable GLB_IDLE timer when setting a new idle hysteresis timeout"); } else { kbase_csf_scheduler_spin_unlock(kbdev, flags); } @@ -2190,78 +2398,6 @@ u32 kbase_csf_firmware_reset_mcu_core_pwroff_time(struct kbase_device *kbdev) return kbase_csf_firmware_set_mcu_core_pwroff_time(kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_NS); } -/** - * kbase_csf_get_iterator_trace_enable - Parsing the iterator_trace enable firstly from - * the module parameter, and then from device-tree. - * @kbdev: Kernel base device pointer - * - * Return: true on enabled, otherwise false. - */ -static bool kbase_csf_get_iterator_trace_enable(struct kbase_device *kbdev) -{ - const void *dt_iter_trace_param; - unsigned int val; - - - /* check device tree for iterator trace enable property and - * fallback to "iter_trace_enable" if not found and try again - */ - dt_iter_trace_param = of_get_property(kbdev->dev->of_node, "iter-trace-enable", NULL); - - if (!dt_iter_trace_param) - dt_iter_trace_param = - of_get_property(kbdev->dev->of_node, "iter_trace_enable", NULL); - - val = (dt_iter_trace_param) ? be32_to_cpup(dt_iter_trace_param) : 0; - dev_dbg(kbdev->dev, "Iterator trace enable device-tree config value: %u", val); - - return (val != 0); -} - -/** - * kbase_device_csf_iterator_trace_init - Send request to enable iterator - * trace port. - * @kbdev: Kernel base device pointer - * - * Return: 0 on success (or if enable request is not sent), or error - * code -EINVAL on failure of GPU to acknowledge enable request. - */ -static int kbase_device_csf_iterator_trace_init(struct kbase_device *kbdev) -{ - /* Enable the iterator trace port if supported by the GPU and is - * configured to do so. The FW must advertise this feature in GLB_FEATURES. - */ - if (kbdev->pm.backend.gpu_powered) { - const struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; - bool dev_support_iter_trace = iface->features & - GLB_FEATURES_ITER_TRACE_SUPPORTED_MASK; - - dev_dbg(kbdev->dev, "Device supporting iterator trace: %s\n", - dev_support_iter_trace ? "true" : "false"); - if (dev_support_iter_trace && kbase_csf_get_iterator_trace_enable(kbdev)) { - long ack_timeout = kbase_csf_timeout_in_jiffies( - kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT)); - - /* write enable request to global input */ - kbase_csf_firmware_global_input_mask(iface, GLB_REQ, - GLB_REQ_ITER_TRACE_ENABLE_MASK, - GLB_REQ_ITER_TRACE_ENABLE_MASK); - /* Ring global doorbell */ - kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); - - ack_timeout = wait_event_timeout( - kbdev->csf.event_wait, - !((kbase_csf_firmware_global_input_read(iface, GLB_REQ) ^ - kbase_csf_firmware_global_output(iface, GLB_ACK)) & - GLB_REQ_ITER_TRACE_ENABLE_MASK), - ack_timeout); - - return ack_timeout ? 0 : -EINVAL; - } - } - return 0; -} - int kbase_csf_firmware_early_init(struct kbase_device *kbdev) { init_waitqueue_head(&kbdev->csf.event_wait); @@ -2275,9 +2411,7 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->csf.firmware_reload_work, kbase_csf_firmware_reload_worker); INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker); - kbdev->csf.glb_init_request_pending = true; - - init_rwsem(&kbdev->csf.pmode_sync_sem); + init_rwsem(&kbdev->csf.mmu_sync_sem); mutex_init(&kbdev->csf.reg_lock); kbase_csf_pending_gpuq_kick_queues_init(kbdev); @@ -2307,6 +2441,8 @@ int kbase_csf_firmware_late_init(struct kbase_device *kbdev) convert_dur_to_idle_count(kbdev, kbdev->csf.gpu_idle_hysteresis_ns, &no_modifier); kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; + kbdev->csf.csg_suspend_timeout_ms = CSG_SUSPEND_TIMEOUT_MS; + return 0; } @@ -2495,6 +2631,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) goto err_out; } + init_page_fault_cnt_firmware_memory(kbdev); + ret = kbase_csf_firmware_cfg_fw_wa_init(kbdev); if (ret != 0) { dev_err(kbdev->dev, "Failed to initialize firmware workarounds"); @@ -2515,6 +2653,8 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) if (ret != 0) goto err_out; + iterator_trace_init(kbdev); + ret = kbase_csf_doorbell_mapping_init(kbdev); if (ret != 0) goto err_out; @@ -2545,10 +2685,6 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) if (ret != 0) goto err_out; - ret = kbase_device_csf_iterator_trace_init(kbdev); - if (ret != 0) - goto err_out; - if (kbdev->csf.fw_core_dump.available) kbase_csf_firmware_core_dump_init(kbdev); @@ -2598,7 +2734,7 @@ void kbase_csf_firmware_unload_term(struct kbase_device *kbdev) kbdev->csf.firmware_inited = false; if (WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_OFF)) { kbdev->pm.backend.mcu_state = KBASE_MCU_OFF; - stop_csf_firmware(kbdev); + kbase_csf_stop_firmware_and_wait(kbdev); } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); @@ -2804,6 +2940,7 @@ void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev) kbase_csf_scheduler_spin_lock_assert_held(kbdev); set_gpu_idle_timer_glb_req(kbdev, false); + atomic_set(&kbdev->csf.scheduler.fw_soi_enabled, false); dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); @@ -2947,7 +3084,6 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) KBASE_TLSTREAM_TL_KBASE_CSFFW_FW_REQUEST_SLEEP(kbdev, kbase_backend_get_cycle_cnt(kbdev)); kbase_csf_scheduler_spin_lock(kbdev, &flags); - set_gpu_idle_timer_glb_req(kbdev, false); set_global_request(global_iface, GLB_REQ_SLEEP_MASK); dev_dbg(kbdev->dev, "Sending sleep request to MCU"); kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); @@ -2956,11 +3092,23 @@ void kbase_csf_firmware_trigger_mcu_sleep(struct kbase_device *kbdev) bool kbase_csf_firmware_is_mcu_in_sleep(struct kbase_device *kbdev) { + bool db_notif_disabled; + lockdep_assert_held(&kbdev->hwaccess_lock); - return (global_request_complete(kbdev, GLB_REQ_SLEEP_MASK) && - kbase_csf_firmware_mcu_halted(kbdev)); + db_notif_disabled = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL)) & + MCU_CNTRL_DOORBELL_DISABLE_MASK; + + if (!db_notif_disabled || !kbase_csf_firmware_mcu_halted(kbdev)) + return false; + + if (global_request_complete(kbdev, GLB_REQ_SLEEP_MASK)) + return true; + + kbase_pm_enable_mcu_db_notification(kbdev); + dev_dbg(kbdev->dev, "Enabled DB notification"); + return false; } #endif @@ -3222,3 +3370,127 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } + +#ifdef KBASE_PM_RUNTIME + +void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + /* There are 3 possibilities: + * - Sleep-on-Idle allowed + * - Sleep-on-Idle not allowed, GLB_IDLE timer disabled + * - Sleep-on-Idle not allowed, GLB_IDLE timer enabled + */ + if (kbase_pm_fw_sleep_on_idle_allowed(kbdev)) { + if (likely(atomic_read(&kbdev->csf.scheduler.fw_soi_enabled))) + return; + } else { + if (test_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed)) { + if (likely(!atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled))) + return; + } else if (likely(atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled))) { + return; + } + } + + if (kbase_reset_gpu_try_prevent(kbdev)) + return; + + kbase_csf_scheduler_lock(kbdev); + + if (atomic_read(&scheduler->pending_gpu_idle_work) > 0) + goto out_unlock_scheduler_lock; + + if ((scheduler->state == SCHED_SUSPENDED) || (scheduler->state == SCHED_SLEEPING)) + goto out_unlock_scheduler_lock; + + if (kbdev->pm.backend.mcu_state != KBASE_MCU_ON) + goto out_unlock_scheduler_lock; + + /* Ensure that an existing DISABLE request is completed before + * proceeding. They are made without waiting for them to complete such + * as when enabling the MCU. + */ + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) { + dev_err(kbdev->dev, + "Existing GLB_IDLE timer config change failed to complete in time (gpu_sleep_allowed:%lx)", + kbdev->pm.backend.gpu_sleep_allowed); + goto out_unlock_scheduler_lock; + } + + /* Disable the GLB IDLE timer if it's currently enabled */ + if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_csf_firmware_disable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) { + dev_err(kbdev->dev, + "Failed to disable GLB_IDLE timer following FW Sleep-on-Idle config change (gpu_sleep_allowed:%lx)", + kbdev->pm.backend.gpu_sleep_allowed); + goto out_unlock_scheduler_lock; + } + } + + /* The GLB IDLE timer and, consequently, FW Sleep-on-Idle could remain + * disabled in certain cases. Otherwise, we shall re-enable GLB IDLE + * timer with the new FW Sleep-on-Idle configuration. + */ + if (!test_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed)) { + kbase_csf_scheduler_spin_lock(kbdev, &flags); + kbase_csf_firmware_enable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_ENABLE_MASK)) { + dev_err(kbdev->dev, + "Failed to re-enable GLB_IDLE timer following FW Sleep-on-Idle config change (gpu_sleep_allowed:%lx)", + kbdev->pm.backend.gpu_sleep_allowed); + goto out_unlock_scheduler_lock; + } + } + + if (atomic_read(&scheduler->fw_soi_enabled)) { + dev_dbg(kbdev->dev, "FW Sleep-on-Idle was enabled"); + KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, true); + } else { + dev_dbg(kbdev->dev, "FW Sleep-on-Idle was disabled"); + KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, false); + } + +out_unlock_scheduler_lock: + kbase_csf_scheduler_unlock(kbdev); + kbase_reset_gpu_allow(kbdev); +} + +int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev) +{ + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; + unsigned long flags; + + lockdep_assert_held(&scheduler->lock); + + if (WARN_ON_ONCE(scheduler->state != SCHED_INACTIVE)) + return 0; + + if (!atomic_read(&kbdev->csf.scheduler.fw_soi_enabled)) + return 0; + + kbase_csf_scheduler_spin_lock(kbdev, &flags); + if (atomic_read(&scheduler->fw_soi_enabled)) { + kbase_csf_firmware_disable_gpu_idle_timer(kbdev); + kbase_csf_scheduler_spin_unlock(kbdev, flags); + if (wait_for_global_request(kbdev, GLB_REQ_IDLE_DISABLE_MASK)) { + dev_err(kbdev->dev, "Failed to disable Sleep-on-Idle config"); + return -ETIMEDOUT; + } + KBASE_KTRACE_ADD(kbdev, FIRMWARE_SLEEP_ON_IDLE_CHANGED, NULL, false); + } else { + kbase_csf_scheduler_spin_unlock(kbdev, flags); + } + + return 0; +} + +#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h index f7a9c07dd6a9..20cb03991bbe 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware.h @@ -591,13 +591,20 @@ void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev); void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev); /** - * kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled - * status. + * kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled status. * * @kbdev: Instance of a GPU platform device that implements a CSF interface. */ void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev); +/** + * kbase_csf_stop_firmware_and_wait - Disable firmware and wait for the MCU to reach + * disabled status. + * + * @kbdev: Instance of a GPU platform device that implements a CSF interface. + */ +void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev); + #ifdef KBASE_PM_RUNTIME /** * kbase_csf_firmware_trigger_mcu_sleep - Send the command to put MCU in sleep @@ -927,4 +934,27 @@ int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev); */ int kbase_csf_firmware_req_core_dump(struct kbase_device *const kbdev); +#ifdef KBASE_PM_RUNTIME + +/** + * kbase_csf_firmware_soi_update - Update FW Sleep-on-Idle config + * + * @kbdev: Device pointer + * + * This function reconfigures the FW Sleep-on-Idle configuration if necessary. + */ +void kbase_csf_firmware_soi_update(struct kbase_device *kbdev); + +/** + * kbase_csf_firmware_soi_disable_on_scheduler_suspend - Disable FW Sleep-on-Idle config + * on scheduler suspension + * + * @kbdev: Device pointer + * + * Return: 0 on success, otherwise failure + */ +int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev); + +#endif /* KBASE_PM_RUNTIME */ + #endif diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c index 0af560fd4260..a206ed3da210 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_firmware_no_mali.c @@ -684,7 +684,7 @@ static inline void set_gpu_idle_timer_glb_req(struct kbase_device *const kbdev, global_iface, GLB_REQ, GLB_REQ_REQ_IDLE_DISABLE, GLB_REQ_IDLE_DISABLE_MASK); } - kbdev->csf.gpu_idle_timer_enabled = set; + atomic_set(&kbdev->csf.scheduler.gpu_idle_timer_enabled, set); } static void enable_gpu_idle_timer(struct kbase_device *const kbdev) @@ -788,12 +788,6 @@ static void global_init(struct kbase_device *const kbdev, u64 core_mask) set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); - /* The GPU idle timer is always enabled for simplicity. Checks will be - * done before scheduling the GPU idle worker to see if it is - * appropriate for the current power policy. - */ - enable_gpu_idle_timer(kbdev); - /* Unmask the interrupts */ kbase_csf_firmware_global_input(global_iface, GLB_ACK_IRQ_MASK, ack_irq_mask); @@ -901,6 +895,7 @@ void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) kbdev->csf.firmware_reloaded = true; } } +KBASE_EXPORT_TEST_API(kbase_csf_firmware_trigger_reload); void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) { @@ -909,6 +904,7 @@ void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) if (unlikely(!kbdev->csf.firmware_inited)) return; + /* Tell MCU state machine to transit to next state */ kbdev->csf.firmware_reloaded = true; kbase_pm_update_state(kbdev); @@ -1019,7 +1015,7 @@ u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, kbdev->csf.gpu_idle_dur_count = hysteresis_val; kbdev->csf.gpu_idle_dur_count_no_modifier = no_modifier; - if (kbdev->csf.gpu_idle_timer_enabled) { + if (atomic_read(&kbdev->csf.scheduler.gpu_idle_timer_enabled)) { /* Timer is already enabled. Disable the timer as FW only reads * the new idle timer value when timer is re-enabled. */ @@ -1142,7 +1138,7 @@ int kbase_csf_firmware_early_init(struct kbase_device *kbdev) INIT_WORK(&kbdev->csf.firmware_reload_work, kbase_csf_firmware_reload_worker); INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker); - init_rwsem(&kbdev->csf.pmode_sync_sem); + init_rwsem(&kbdev->csf.mmu_sync_sem); mutex_init(&kbdev->csf.reg_lock); kbase_csf_pending_gpuq_kick_queues_init(kbdev); @@ -1210,6 +1206,7 @@ int kbase_csf_firmware_load_init(struct kbase_device *kbdev) /* NO_MALI: Don't load the MMU tables or boot CSF firmware */ + ret = invent_capabilities(kbdev); if (ret != 0) goto error; @@ -1540,6 +1537,12 @@ void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) kbase_reg_write32(kbdev, GPU_CONTROL_ENUM(MCU_CONTROL), MCU_CONTROL_REQ_DISABLE); } +void kbase_csf_stop_firmware_and_wait(struct kbase_device *kbdev) +{ + /* Stop the MCU firmware, no wait required on NO_MALI instance */ + kbase_csf_firmware_disable_mcu(kbdev); +} + void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) { /* NO_MALI: Nothing to do here */ @@ -1662,3 +1665,16 @@ void kbase_csf_firmware_mcu_shared_mapping_term(struct kbase_device *kbdev, vunmap(csf_mapping->cpu_addr); kfree(csf_mapping->phys); } + +#ifdef KBASE_PM_RUNTIME + +void kbase_csf_firmware_soi_update(struct kbase_device *kbdev) +{ +} + +int kbase_csf_firmware_soi_disable_on_scheduler_suspend(struct kbase_device *kbdev) +{ + return 0; +} + +#endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c new file mode 100644 index 000000000000..c65f837a9f72 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_csf_fw_io.h" +#include + +#include + +static inline u32 input_page_read(const u32 *const input, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return input[offset / sizeof(u32)]; +} + +static inline void input_page_write(u32 *const input, const u32 offset, const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = value; +} + +static inline void input_page_partial_write(u32 *const input, const u32 offset, u32 value, u32 mask) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = (input_page_read(input, offset) & ~mask) | (value & mask); +} + +static inline u32 output_page_read(const u32 *const output, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return output[offset / sizeof(u32)]; +} + +void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io) +{ + spin_lock_init(&fw_io->lock); + bitmap_zero(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); + +void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io) +{ + /* Nothing to do. */ +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_term); + +void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, u32 value) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); + input_page_write(iface->input, offset, value); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write); + +void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", offset, value, mask); + input_page_partial_write(iface->input, offset, value, mask); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write_mask); + +u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(iface->input, offset); + dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_input_read); + +u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(iface->output, offset); + dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write); + +void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset, u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", offset, value, mask); + input_page_partial_write(info->input, offset, value, mask); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write_mask); + +u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_input_read); + +u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write); + +void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", offset, value, mask); + input_page_partial_write(info->input, offset, value, mask); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write_mask); + +u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_input_read); + +u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + set_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status); + +void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + clear_bit(status_bit, fw_io->status); +} + +bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + return test_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_test_status); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h new file mode 100644 index 000000000000..a8eb1ab51fbc --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io.h @@ -0,0 +1,362 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_CSF_FW_IO_H_ +#define _KBASE_CSF_FW_IO_H_ + +#include +#include +#include +#include + +/** The wait completed because the GPU was lost. */ +#define KBASE_CSF_FW_IO_WAIT_GPU_LOST 1 + +/** The wait was aborted because of an unexpected event. */ +#define KBASE_CSF_FW_IO_WAIT_UNSUPPORTED 255 + +/** + * enum kbase_csf_fw_io_status_bits - Status bits for firmware I/O interface. + * + * @KBASE_FW_IO_STATUS_GPU_SUSPENDED: The GPU is suspended. + * @KBASE_FW_IO_STATUS_NUM_BITS: Number of bits used to encode the state. + */ +enum kbase_csf_fw_io_status_bits { + KBASE_FW_IO_STATUS_GPU_SUSPENDED = 0, + KBASE_FW_IO_STATUS_NUM_BITS, +}; + +/** + * struct kbase_csf_fw_io - Manager of firmware input/output interface. + * + * @lock: Mutex to serialize access to the interface. + * @status: Internal status of the MCU interface. + */ +struct kbase_csf_fw_io { + spinlock_t lock; + DECLARE_BITMAP(status, KBASE_FW_IO_STATUS_NUM_BITS); +}; + +struct kbase_csf_global_iface; +struct kbase_csf_cmd_stream_group_info; +struct kbase_csf_cmd_stream_info; + +/** + * kbase_csf_fw_io_init() - Initialize manager of firmware input/output interface. + * + * @fw_io: Firmware I/O interface to initialize. + */ +void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io); + +/** + * kbase_csf_fw_io_term() - Terminate manager of firmware input/output interface. + * + * @fw_io: Firmware I/O interface to terminate. + */ +void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io); + +/** + * kbase_csf_fw_io_open() - Start a transaction with the firmware input/output interface. + * + * @fw_io: Firmware I/O interface to open. + * + * Return: 0 on success, otherwise an error code reflecting the status of the + * interface. + */ +static inline int kbase_csf_fw_io_open(struct kbase_csf_fw_io *fw_io) +{ + if (test_bit(KBASE_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status)) + return -KBASE_CSF_FW_IO_WAIT_GPU_LOST; + + spin_lock(&fw_io->lock); + + return 0; +} + +/** + * kbase_csf_fw_io_open_force() - Force a transaction with the firmware input/output interface. + * + * @fw_io: Firmware I/O interface to open. + * + * This function forces the start of a transaction regardless of the status + * of the interface. + */ +static inline void kbase_csf_fw_io_open_force(struct kbase_csf_fw_io *fw_io) +{ + spin_lock(&fw_io->lock); +} + +/** + * kbase_csf_fw_io_close() - End a transaction with the firmware input/output interface. + * + * @fw_io: Firmware I/O interface to close. + */ +static inline void kbase_csf_fw_io_close(struct kbase_csf_fw_io *fw_io) +{ + spin_unlock(&fw_io->lock); +} + +/** + * kbase_csf_fw_io_assert_opened() - Assert if a transaction with the firmware input/output + * interface has started. + * + * @fw_io: Firmware I/O interface. + */ +static inline void kbase_csf_fw_io_assert_opened(struct kbase_csf_fw_io *fw_io) +{ + lockdep_assert_held(&fw_io->lock); +} + +/** + * kbase_csf_fw_io_global_write() - Write a word in the global input page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + */ +void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value); + +/** + * kbase_csf_fw_io_global_write_mask() - Write part of a word in the global input page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value, u32 mask); + +/** + * kbase_csf_fw_io_global_input_read() - Read a word in the global input page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the global input page. + */ +u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset); + +/** + * kbase_csf_fw_io_global_read() - Read a word in the global output page. + * + * @fw_io: Firmware I/O manager. + * @iface: CSF interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the global output page. + */ +u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset); + +/** + * kbase_csf_fw_io_group_write() - Write a word in a CSG's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + */ +void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value); + +/** + * kbase_csf_fw_io_group_write_mask() - Write part of a word in a CSG's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset, u32 value, u32 mask); + +/** + * kbase_csf_fw_io_group_input_read() - Read a word in a CSG's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from a CSG's input page. + */ +u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset); + +/** + * kbase_csf_fw_io_group_read() - Read a word in a CSG's output page. + * + * @fw_io: Firmware I/O manager. + * @info: CSG interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the CSG's output page. + */ +u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset); + +/** + * kbase_csf_fw_io_stream_write() - Write a word in a CS's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + */ +void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value); + +/** + * kbase_csf_fw_io_stream_write_mask() - Write part of a word in a CS's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to write, in bytes. + * @value: Value to be written. + * @mask: Bitmask with the bits to be modified set. + */ +void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value, u32 mask); + +/** + * kbase_csf_fw_io_stream_input_read() - Read a word in a CS's input page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from a CS's input page. + */ +u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset); + +/** + * kbase_csf_fw_io_stream_read() - Read a word in a CS's output page. + * + * @fw_io: Firmware I/O manager. + * @info: CSI interface provided by the firmware. + * @offset: Offset of the word to be read, in bytes. + * + * Return: Value of the word read from the CS's output page. + */ +u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset); + +/** + * kbase_csf_fw_io_set_status() - Set a FW I/O status bit. + * + * @fw_io: Firmware I/O manager. + * @status_bit: Status bit to set. + */ +void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit); + +/** + * kbase_csf_fw_io_clear_status() - Clear a FW I/O status bit. + * + * @fw_io: Firmware I/O manager. + * @status_bit: Status bit to clear. + */ +void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit); + +/** + * kbase_csf_fw_io_test_status() - Test a FW I/O status bit. + * + * @fw_io: Firmware I/O manager. + * @status_bit: Status bit to test. + * + * Return: Value of the tested status bit. + */ +bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit); + +/** + * kbase_csf_fw_io_wait_event_timeout() - Wait until condition gets true, timeout + * occurs or a FW I/O status bit is set. The rest of the functionalities is equal + * to wait_event_timeout(). + * + * @fw_io: Firmware I/O manager. + * @wq_head: The waitqueue to wait on. + * @condition: C expression for the event to wait for + * @timeout: Timeout, in jiffies + * + * Return: Remaining jiffies (at least 1) on success, + * 0 on timeout, + * negative KBASE_CSF_FW_IO_WAIT_* error codes otherwise. + */ +#define kbase_csf_fw_io_wait_event_timeout(fw_io, wq_head, condition, timeout) \ + ({ \ + int __ret; \ + int __wait_remaining = wait_event_timeout( \ + wq_head, condition || kbasep_csf_fw_io_check_status(fw_io), timeout); \ + __ret = kbasep_csf_fw_io_handle_wait_result(fw_io, __wait_remaining); \ + __ret; \ + }) + +/** + * kbasep_csf_fw_io_check_status() - Private function to check if any FW I/O status bit is set. + * + * @fw_io: Firmware I/O manager. + * + * Return: True if any FW I/O status bit is set, false otherwise. + */ +static inline bool kbasep_csf_fw_io_check_status(struct kbase_csf_fw_io *fw_io) +{ + return !bitmap_empty(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS); +} + +/** + * kbasep_csf_fw_io_handle_wait_result() - Private function to handle the wait_event_timeout() + * result. + * + * @fw_io: Firmware I/O manager + * @wait_remaining: Remaining jiffies returned by wait_event_timeout() + * + * Return: Remaining jiffies (at least 1) on success, + * 0 on timeout, + * negative KBASE_CSF_FW_IO_WAIT_* error codes otherwise. + */ +static inline int kbasep_csf_fw_io_handle_wait_result(struct kbase_csf_fw_io *fw_io, + int wait_remaining) +{ + /* Check for any FW IO status bit set */ + if (!bitmap_empty(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS)) + return (test_bit(KBASE_FW_IO_STATUS_GPU_SUSPENDED, fw_io->status)) ? + -KBASE_CSF_FW_IO_WAIT_GPU_LOST : + -KBASE_CSF_FW_IO_WAIT_UNSUPPORTED; + + return wait_remaining; +} +#endif diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c new file mode 100644 index 000000000000..0cffc8475654 --- /dev/null +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_fw_io_no_mali.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#include "mali_kbase.h" +#include "mali_kbase_csf_fw_io.h" +#include + +#include + +static inline u32 input_page_read(const u32 *const input, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return input[offset / sizeof(u32)]; +} + +static inline void input_page_write(u32 *const input, const u32 offset, const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = value; +} + +static inline void input_page_partial_write(u32 *const input, const u32 offset, u32 value, u32 mask) +{ + WARN_ON(offset % sizeof(u32)); + + input[offset / sizeof(u32)] = (input_page_read(input, offset) & ~mask) | (value & mask); +} + +static inline u32 output_page_read(const u32 *const output, const u32 offset) +{ + WARN_ON(offset % sizeof(u32)); + + return output[offset / sizeof(u32)]; +} + +static inline void output_page_write(u32 *const output, const u32 offset, const u32 value) +{ + WARN_ON(offset % sizeof(u32)); + + output[offset / sizeof(u32)] = value; +} + +void kbase_csf_fw_io_init(struct kbase_csf_fw_io *fw_io) +{ + spin_lock_init(&fw_io->lock); + bitmap_zero(fw_io->status, KBASE_FW_IO_STATUS_NUM_BITS); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_init); + +void kbase_csf_fw_io_term(struct kbase_csf_fw_io *fw_io) +{ + /* Nothing to do. */ +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_term); + +void kbase_csf_fw_io_global_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, u32 value) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); + input_page_write(iface->input, offset, value); + + if (offset == GLB_REQ) { + /* NO_MALI: Immediately acknowledge requests - except for PRFCNT_ENABLE + * and PRFCNT_SAMPLE. These will be processed along with the + * corresponding performance counter registers when the global doorbell + * is rung in order to emulate the performance counter sampling behavior + * of the real firmware. + */ + const u32 ack = output_page_read(iface->output, GLB_ACK); + const u32 req_mask = ~(GLB_REQ_PRFCNT_ENABLE_MASK | GLB_REQ_PRFCNT_SAMPLE_MASK); + const u32 toggled = (value ^ ack) & req_mask; + + output_page_write(iface->output, GLB_ACK, ack ^ toggled); + } +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write); + +void kbase_csf_fw_io_global_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = iface->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", offset, value, mask); + + /* NO_MALI: Go through existing function to capture writes */ + kbase_csf_fw_io_global_write(fw_io, iface, offset, + (input_page_read(iface->input, offset) & ~mask) | + (value & mask)); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_write_mask); + +u32 kbase_csf_fw_io_global_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(iface->input, offset); + dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_global_input_read); + +u32 kbase_csf_fw_io_global_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_global_iface *iface, u32 offset) +{ + const struct kbase_device *const kbdev = iface->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(iface->output, offset); + dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_group_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); + + if (offset == CSG_REQ) { + /* NO_MALI: Immediately acknowledge requests */ + output_page_write(info->output, CSG_ACK, value); + } +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write); + +void kbase_csf_fw_io_group_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, + u32 offset, u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", offset, value, mask); + + /* NO_MALI: Go through existing function to capture writes */ + kbase_csf_fw_io_group_write(fw_io, info, offset, + (input_page_read(info->input, offset) & ~mask) | + (value & mask)); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_write_mask); + +u32 kbase_csf_fw_io_group_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_group_input_read); + +u32 kbase_csf_fw_io_group_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_group_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_stream_write(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); + input_page_write(info->input, offset, value); + + if (offset == CS_REQ) { + /* NO_MALI: Immediately acknowledge requests */ + output_page_write(info->output, CS_ACK, value); + } +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write); + +void kbase_csf_fw_io_stream_write_mask(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset, + u32 value, u32 mask) +{ + const struct kbase_device *const kbdev = info->kbdev; + + lockdep_assert_held(&fw_io->lock); + + dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", offset, value, mask); + + /* NO_MALI: Go through existing function to capture writes */ + kbase_csf_fw_io_stream_write(fw_io, info, offset, + (input_page_read(info->input, offset) & ~mask) | + (value & mask)); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_write_mask); + +u32 kbase_csf_fw_io_stream_input_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = input_page_read(info->input, offset); + dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); + + return val; +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_stream_input_read); + +u32 kbase_csf_fw_io_stream_read(struct kbase_csf_fw_io *fw_io, + const struct kbase_csf_cmd_stream_info *info, u32 offset) +{ + const struct kbase_device *const kbdev = info->kbdev; + u32 val; + + lockdep_assert_held(&fw_io->lock); + + val = output_page_read(info->output, offset); + dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); + + return val; +} + +void kbase_csf_fw_io_set_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + set_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_set_status); + +void kbase_csf_fw_io_clear_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + clear_bit(status_bit, fw_io->status); +} + +bool kbase_csf_fw_io_test_status(struct kbase_csf_fw_io *fw_io, + enum kbase_csf_fw_io_status_bits status_bit) +{ + return test_bit(status_bit, fw_io->status); +} +KBASE_EXPORT_TEST_API(kbase_csf_fw_io_test_status); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c index 12a79b4852fb..fb181026719f 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_heap_context_alloc.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -180,8 +180,9 @@ void kbase_csf_heap_context_allocator_term(struct kbase_csf_heap_context_allocat u64 kbase_csf_heap_context_allocator_alloc(struct kbase_csf_heap_context_allocator *const ctx_alloc) { struct kbase_context *const kctx = ctx_alloc->kctx; - u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | BASE_MEM_PROT_CPU_WR | - BASEP_MEM_NO_USER_FREE | BASE_MEM_PROT_CPU_RD; + base_mem_alloc_flags flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | + BASE_MEM_PROT_CPU_RD; u64 nr_pages = PFN_UP(MAX_TILER_HEAPS * ctx_alloc->heap_context_size_aligned); u64 heap_gpu_va = 0; diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c index ba47b7190395..09c92f0bed4e 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_kcpu.c @@ -371,7 +371,7 @@ static int kbase_kcpu_jit_allocate_prepare(struct kbase_kcpu_command_queue *kcpu goto out; } - if (copy_from_user(info, data, sizeof(*info) * count) != 0) { + if (copy_from_user(info, data, size_mul(sizeof(*info), count)) != 0) { ret = -EINVAL; goto out_free; } @@ -563,7 +563,7 @@ static int kbase_kcpu_jit_free_prepare(struct kbase_kcpu_command_queue *kcpu_que goto out_free; } - if (copy_from_user(ids, data, sizeof(*ids) * count)) { + if (copy_from_user(ids, data, size_mul(sizeof(*ids), count))) { ret = -EINVAL; goto out_free; } @@ -852,7 +852,8 @@ static int kbase_kcpu_cqs_wait_prepare(struct kbase_kcpu_command_queue *queue, if (!objs) return -ENOMEM; - if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), nr_objs * sizeof(*objs))) { + if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } @@ -957,7 +958,8 @@ static int kbase_kcpu_cqs_set_prepare(struct kbase_kcpu_command_queue *kcpu_queu if (!objs) return -ENOMEM; - if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), nr_objs * sizeof(*objs))) { + if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } @@ -1115,7 +1117,7 @@ static int kbase_kcpu_cqs_wait_operation_prepare( return -ENOMEM; if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_operation_info->objs), - nr_objs * sizeof(*objs))) { + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } @@ -1280,7 +1282,7 @@ static int kbase_kcpu_cqs_set_operation_prepare( return -ENOMEM; if (copy_from_user(objs, u64_to_user_ptr(cqs_set_operation_info->objs), - nr_objs * sizeof(*objs))) { + size_mul(nr_objs, sizeof(*objs)))) { kfree(objs); return -ENOMEM; } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h index 9a7c6e451f66..7e96a9d01fc7 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_registers.h @@ -172,6 +172,11 @@ #define CSG_STATUS_EP_CURRENT 0x0010 /* () Endpoint allocation status register */ #define CSG_STATUS_EP_REQ 0x0014 /* () Endpoint request status register */ #define CSG_RESOURCE_DEP 0x001C /* () Current resource dependencies */ +/* TODO: GPUCORE-xxxx: Remove after spec alignment, use 0x1C as CSG_RESOURCE_DEP is deprecated*/ +/* CSG_OUTPUT_BLOCK register offsets */ +#ifndef CSG_PROGRESS_TIMER_STATE +#define CSG_PROGRESS_TIMER_STATE 0x001C /* () Current resource status */ +#endif /* GLB_CONTROL_BLOCK register offsets */ #define GLB_VERSION 0x0000 /* () Global interface version */ @@ -1243,6 +1248,21 @@ CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) +/* CSG_PROGRESS_TIMER_STATE register */ +#ifndef CSG_PROGRESS_TIMER_STATE_GET +#define CSG_PROGRESS_TIMER_STATE_SHIFT 0 +#define CSG_PROGRESS_TIMER_STATE_MASK ((u32)0xFFFFFFFF << CSG_PROGRESS_TIMER_STATE_SHIFT) +#define CSG_PROGRESS_TIMER_STATE_GET(reg_val) \ + (((reg_val)&CSG_PROGRESS_TIMER_STATE_MASK) >> CSG_PROGRESS_TIMER_STATE_SHIFT) +#define CSG_PROGRESS_TIMER_STATE_SET(reg_val, value) \ + (((reg_val) & ~CSG_PROGRESS_TIMER_STATE_MASK) | \ + (((value) << CSG_PROGRESS_TIMER_STATE_SHIFT) & CSG_PROGRESS_TIMER_STATE_MASK)) +/* CSG_PROGRESS_TIMER_STATE values */ +#define CSG_PROGRESS_TIMER_STATE_COMPUTE 0x0 +#define CSG_PROGRESS_TIMER_STATE_FRAGMENT 0x1 +#define CSG_PROGRESS_TIMER_STATE_TILER 0x2 +#define CSG_PROGRESS_TIMER_STATE_NEURAL 0x3 +#endif /* End of CSG_OUTPUT_BLOCK register set definitions */ /* STREAM_CONTROL_BLOCK register set definitions */ @@ -1380,6 +1400,13 @@ #define GLB_REQ_SLEEP_SET(reg_val, value) \ (((reg_val) & ~GLB_REQ_SLEEP_MASK) | \ (((value) << GLB_REQ_SLEEP_SHIFT) & GLB_REQ_SLEEP_MASK)) +#define GLB_REQ_CFG_EVICTION_TIMER_SHIFT 16 +#define GLB_REQ_CFG_EVICTION_TIMER_MASK (0x1 << GLB_REQ_CFG_EVICTION_TIMER_SHIFT) +#define GLB_REQ_CFG_EVICTION_TIMER_GET(reg_val) \ + (((reg_val)&GLB_REQ_CFG_EVICTION_TIMER_MASK) >> GLB_REQ_CFG_EVICTION_TIMER_SHIFT) +#define GLB_REQ_CFG_EVICTION_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_REQ_CFG_EVICTION_TIMER_MASK) | \ + (((value) << GLB_REQ_CFG_EVICTION_TIMER_SHIFT) & GLB_REQ_CFG_EVICTION_TIMER_MASK)) #define GLB_REQ_INACTIVE_COMPUTE_SHIFT 20 #define GLB_REQ_INACTIVE_COMPUTE_MASK (0x1 << GLB_REQ_INACTIVE_COMPUTE_SHIFT) #define GLB_REQ_INACTIVE_COMPUTE_GET(reg_val) \ @@ -1524,6 +1551,17 @@ (((reg_val) & ~GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK) | \ (((value) << GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SHIFT) & \ GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK)) +#define GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_SHIFT 11 +#define GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_ITER_TRACE_ENABLE_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT 16 +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_GET(reg_val) \ + (((reg_val)&GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK) >> \ + GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT) +#define GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SET(reg_val, value) \ + (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK) | \ + (((value) << GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_SHIFT) & \ + GLB_ACK_IRQ_MASK_CFG_EVICTION_TIMER_MASK)) #define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT 20 #define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) #define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_GET(reg_val) \ @@ -1635,6 +1673,45 @@ GLB_PWROFF_TIMER_CONFIG_NO_MODIFIER_MASK)) #endif /* End of GLB_PWROFF_TIMER_CONFIG values */ +/* GLB_EVICTION_TIMER register */ +#ifndef GLB_EVICTION_TIMER +#define GLB_EVICTION_TIMER 0x0090 +#define GLB_EVICTION_TIMER_TIMEOUT_SHIFT (0) +#define GLB_EVICTION_TIMER_TIMEOUT_MASK ((0x7FFFFFFF) << GLB_EVICTION_TIMER_TIMEOUT_SHIFT) +#define GLB_EVICTION_TIMER_TIMEOUT_GET(reg_val) \ + (((reg_val)&GLB_EVICTION_TIMER_TIMEOUT_MASK) >> GLB_EVICTION_TIMER_TIMEOUT_SHIFT) +#define GLB_EVICTION_TIMER_TIMEOUT_SET(reg_val, value) \ + (((reg_val) & ~GLB_EVICTION_TIMER_TIMEOUT_MASK) | \ + (((value) << GLB_EVICTION_TIMER_TIMEOUT_SHIFT) & GLB_EVICTION_TIMER_TIMEOUT_MASK)) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT (31) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_MASK ((0x1) << GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_GET(reg_val) \ + (((reg_val)&GLB_EVICTION_TIMER_TIMER_SOURCE_MASK) >> GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT) +#define GLB_EVICTION_TIMER_TIMER_SOURCE_SET(reg_val, value) \ + (((reg_val) & ~GLB_EVICTION_TIMER_TIMER_SOURCE_MASK) | \ + (((value) << GLB_EVICTION_TIMER_TIMER_SOURCE_SHIFT) & \ + GLB_EVICTION_TIMER_TIMER_SOURCE_MASK)) +/* GLB_EVICTION_TIMER_TIMER_SOURCE values */ +#define GLB_EVICTION_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0U +#define GLB_EVICTION_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1U +/* End of GLB_EVICTION_TIMER_TIMER_SOURCE values */ +#endif /* End of GLB_EVICTION_TIMER */ + +/* GLB_EVICTION_TIMER_CONFIG register */ +#ifndef GLB_EVICTION_TIMER_CONFIG +#define GLB_EVICTION_TIMER_CONFIG 0x0094 /* () Configuration fields for GLB_EVICTION_TIMER */ +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT 0 +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK \ + (0x1 << GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT) +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_GET(reg_val) \ + (((reg_val)&GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK) >> \ + GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT) +#define GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SET(reg_val, value) \ + (((reg_val) & ~GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK) | \ + (((value) << GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_SHIFT) & \ + GLB_EVICTION_TIMER_CONFIG_NO_MODIFIER_MASK)) +#endif /* End of GLB_EVICTION_TIMER_CONFIG values */ + /* GLB_ALLOC_EN register */ #define GLB_ALLOC_EN_MASK_SHIFT 0 #define GLB_ALLOC_EN_MASK_MASK (GPU_ULL(0xFFFFFFFFFFFFFFFF) << GLB_ALLOC_EN_MASK_SHIFT) @@ -1717,6 +1794,15 @@ (((reg_val) & ~GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK) | \ (((value) << GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_SHIFT) & \ GLB_IDLE_TIMER_CONFIG_NO_MODIFIER_MASK)) +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT 9 +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK (0x1 << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT) +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_GET(reg_val) \ + (((reg_val)&GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) >> \ + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT) +#define GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SET(reg_val, value) \ + (((reg_val) & ~GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK) | \ + (((value) << GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_SHIFT) & \ + GLB_IDLE_TIMER_CONFIG_SLEEP_ON_IDLE_MASK)) #endif /* End of GLB_IDLE_TIMER_CONFIG values */ /* GLB_INSTR_FEATURES register */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c index b07cc9600a04..ffd27318cba3 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_reset_gpu.c @@ -496,12 +496,10 @@ static void kbase_csf_reset_gpu_worker(struct work_struct *data) bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_gpu_lost(kbdev)) { /* GPU access has been removed, reset will be done by Arbiter instead */ return false; } -#endif if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c index 642531c1033c..cd6abd62f6c5 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.c @@ -42,6 +42,7 @@ #include #endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */ + /* Value to indicate that a queue group is not groups_to_schedule list */ #define KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID (U32_MAX) @@ -93,6 +94,11 @@ static bool queue_group_scheduled_locked(struct kbase_queue_group *group); #define kctx_as_enabled(kctx) (!kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) +bool is_gpu_level_suspend_supported(struct kbase_device *const kbdev) +{ + return false; +} + #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) /** * gpu_metrics_ctx_init() - Take a reference on GPU metrics context if it exists, @@ -1196,13 +1202,20 @@ static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) scheduler_enable_tick_timer_nolock(kbdev); } -static void scheduler_suspend(struct kbase_device *kbdev) +static int scheduler_suspend(struct kbase_device *kbdev) { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; lockdep_assert_held(&scheduler->lock); if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) { +#if KBASE_PM_RUNTIME + int ret; + + ret = kbase_csf_firmware_soi_disable_on_scheduler_suspend(kbdev); + if (ret) + return ret; +#endif /* KBASE_PM_RUNTIME */ dev_dbg(kbdev->dev, "Suspending the Scheduler"); scheduler_pm_idle(kbdev); scheduler->state = SCHED_SUSPENDED; @@ -1211,6 +1224,8 @@ static void scheduler_suspend(struct kbase_device *kbdev) #endif KBASE_KTRACE_ADD(kbdev, SCHED_SUSPENDED, NULL, scheduler->state); } + + return 0; } /** @@ -1462,7 +1477,7 @@ static int sched_halt_stream(struct kbase_queue *queue) long remaining; int slot; int err = 0; - const u32 group_schedule_timeout = kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT); + const u32 group_schedule_timeout = kbdev->csf.csg_suspend_timeout_ms; const u32 fw_timeout_ms = kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT); if (WARN_ON(!group)) @@ -1942,7 +1957,6 @@ static enum kbase_csf_csg_slot_state update_csg_slot_status(struct kbase_device if ((state == CSG_ACK_STATE_START) || (state == CSG_ACK_STATE_RESUME)) { slot_state = CSG_SLOT_RUNNING; atomic_set(&csg_slot->state, slot_state); - csg_slot->trigger_jiffies = jiffies; KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_RUNNING, csg_slot->resident_group, state); dev_dbg(kbdev->dev, "Group %u running on slot %d\n", @@ -1953,7 +1967,6 @@ static enum kbase_csf_csg_slot_state update_csg_slot_status(struct kbase_device if ((state == CSG_ACK_STATE_SUSPEND) || (state == CSG_ACK_STATE_TERMINATE)) { slot_state = CSG_SLOT_STOPPED; atomic_set(&csg_slot->state, slot_state); - csg_slot->trigger_jiffies = jiffies; KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, csg_slot->resident_group, state); dev_dbg(kbdev->dev, "Group %u stopped on slot %d\n", @@ -2052,7 +2065,6 @@ static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) kbase_csf_ring_csg_doorbell(kbdev, slot); spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); atomic_set(&csg_slot[slot].state, CSG_SLOT_DOWN2STOP); - csg_slot[slot].trigger_jiffies = jiffies; KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOP_REQ, group, halt_cmd); KBASE_TLSTREAM_TL_KBASE_DEVICE_HALTING_CSG(kbdev, kbdev->id, (u32)slot, suspend); @@ -2367,11 +2379,6 @@ static void cancel_tock_work(struct kbase_csf_scheduler *const scheduler) atomic_set(&scheduler->pending_tock_work, false); } -static void cancel_gpu_idle_work(struct kbase_csf_scheduler *const scheduler) -{ - atomic_set(&scheduler->pending_gpu_idle_work, false); -} - static void remove_group_from_runnable(struct kbase_csf_scheduler *const scheduler, struct kbase_queue_group *group, enum kbase_csf_group_state run_state) @@ -2776,7 +2783,6 @@ static bool cleanup_csg_slot(struct kbase_queue_group *group) spin_unlock_bh(&kbdev->csf.scheduler.gpu_metrics_lock); #endif - csg_slot->trigger_jiffies = jiffies; atomic_set(&csg_slot->state, CSG_SLOT_READY); KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_CLEANED, group, (u64)slot); @@ -2990,7 +2996,6 @@ static void program_csg_slot(struct kbase_queue_group *group, s8 slot, u8 prio) /* Update status before rings the door-bell, marking ready => run */ atomic_set(&csg_slot->state, CSG_SLOT_READY2RUN); - csg_slot->trigger_jiffies = jiffies; csg_slot->priority = prio; /* Trace the programming of the CSG on the slot */ @@ -3556,8 +3561,7 @@ static void program_suspending_csg_slots(struct kbase_device *kbdev) while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); - long remaining = kbase_csf_timeout_in_jiffies( - kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT)); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.csg_suspend_timeout_ms); bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); @@ -4101,7 +4105,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, * entry to protected mode happens with a memory region being locked and * the same region is then accessed by the GPU in protected mode. */ - down_write(&kbdev->csf.pmode_sync_sem); + down_write(&kbdev->csf.mmu_sync_sem); spin_lock_irqsave(&scheduler->interrupt_lock, flags); /* Check if the previous transition to enter & exit the protected @@ -4167,7 +4171,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); err = kbase_csf_wait_protected_mode_enter(kbdev); - up_write(&kbdev->csf.pmode_sync_sem); + up_write(&kbdev->csf.mmu_sync_sem); if (err) schedule_actions_trigger_df( @@ -4182,7 +4186,7 @@ static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, } spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); - up_write(&kbdev->csf.pmode_sync_sem); + up_write(&kbdev->csf.mmu_sync_sem); } /** @@ -4760,27 +4764,28 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool s { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; - int ret; - ret = suspend_active_queue_groups(kbdev, slot_mask, false); + if (unlikely(suspend_active_queue_groups(kbdev, slot_mask, false))) { + if (!is_gpu_level_suspend_supported(kbdev)) { + const int csg_nr = ffs(slot_mask[0]) - 1; + struct kbase_queue_group *group; + enum dumpfault_error_type error_type = DF_CSG_SUSPEND_TIMEOUT; - if (unlikely(ret)) { - const int csg_nr = ffs(slot_mask[0]) - 1; - struct kbase_queue_group *group = scheduler->csg_slots[csg_nr].resident_group; - enum dumpfault_error_type error_type = DF_CSG_SUSPEND_TIMEOUT; + group = scheduler->csg_slots[csg_nr].resident_group; - /* The suspend of CSGs failed, - * trigger the GPU reset to be in a deterministic state. - */ - dev_warn( - kbdev->dev, - "[%llu] Timeout (%d ms) waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", - kbase_backend_get_cycle_cnt(kbdev), - kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT), - kbdev->csf.global_iface.group_num, slot_mask); - if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) - error_type = DF_PING_REQUEST_TIMEOUT; - schedule_actions_trigger_df(kbdev, group->kctx, error_type); + /* The suspend of CSGs failed, + * trigger the GPU reset to be in a deterministic state. + */ + dev_warn( + kbdev->dev, + "[%llu] Timeout (%d ms) waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", + kbase_backend_get_cycle_cnt(kbdev), + kbase_get_timeout_ms(kbdev, CSF_FIRMWARE_TIMEOUT), + kbdev->csf.global_iface.group_num, slot_mask); + if (kbase_csf_firmware_ping_wait(kbdev, FW_PING_AFTER_ERROR_TIMEOUT_MS)) + error_type = DF_PING_REQUEST_TIMEOUT; + schedule_actions_trigger_df(kbdev, group->kctx, error_type); + } if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) kbase_reset_gpu(kbdev); @@ -4788,6 +4793,8 @@ static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, bool s return -1; } + kbdev->csf.mcu_halted = false; + /* Check if the groups became active whilst the suspend was ongoing, * but only for the case where the system suspend is not in progress */ @@ -4947,9 +4954,13 @@ static bool scheduler_suspend_on_idle(struct kbase_device *kbdev) } dev_dbg(kbdev->dev, "Scheduler to be suspended on GPU becoming idle"); - scheduler_suspend(kbdev); - cancel_tick_work(scheduler); - return true; + ret = scheduler_suspend(kbdev); + if (!ret) { + cancel_tick_work(scheduler); + return true; + } + + return false; } static void gpu_idle_worker(struct kbase_device *kbdev) @@ -5193,8 +5204,7 @@ static int wait_csg_slots_suspend(struct kbase_device *kbdev, unsigned long *slo bitmap_copy(slot_mask_local, slot_mask, MAX_SUPPORTED_CSGS); while (!bitmap_empty(slot_mask_local, MAX_SUPPORTED_CSGS)) { - long remaining = kbase_csf_timeout_in_jiffies( - kbase_get_timeout_ms(kbdev, CSF_CSG_SUSPEND_TIMEOUT)); + long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.csg_suspend_timeout_ms); DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); bitmap_copy(changed, slot_mask_local, MAX_SUPPORTED_CSGS); @@ -5343,20 +5353,6 @@ static void evict_lru_or_blocked_csg(struct kbase_device *kbdev) } } -static void scheduler_enable_gpu_idle_timer(struct kbase_device *kbdev) -{ - struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; - unsigned long flags; - - lockdep_assert_held(&scheduler->lock); - - if (!kbdev->csf.gpu_idle_timer_enabled) { - spin_lock_irqsave(&scheduler->interrupt_lock, flags); - kbase_csf_firmware_enable_gpu_idle_timer(kbdev); - spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); - } -} - static void schedule_actions(struct kbase_device *kbdev, bool is_tick) { struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; @@ -5399,7 +5395,6 @@ static void schedule_actions(struct kbase_device *kbdev, bool is_tick) * in particular, no alterations to on-slot CSGs. */ if (keep_lru_on_slots(kbdev)) { - scheduler_enable_gpu_idle_timer(kbdev); return; } } @@ -5472,7 +5467,6 @@ redo_local_tock: wait_csg_slots_start(kbdev); wait_csg_slots_finish_prio_update(kbdev); - scheduler_enable_gpu_idle_timer(kbdev); if (new_protm_top_grp) { scheduler_group_check_protm_enter(kbdev, scheduler->top_grp); @@ -5495,6 +5489,15 @@ redo_local_tock: } evict_lru_or_blocked_csg(kbdev); + +#ifdef KBASE_PM_RUNTIME + if (atomic_read(&scheduler->non_idle_offslot_grps)) + set_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed); + else + clear_bit(KBASE_GPU_NON_IDLE_OFF_SLOT_GROUPS_AVAILABLE, + &kbdev->pm.backend.gpu_sleep_allowed); +#endif /* KBASE_PM_RUNTIME */ } /** @@ -5653,19 +5656,20 @@ static int suspend_active_queue_groups(struct kbase_device *kbdev, unsigned long { struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; u32 num_groups = kbdev->csf.global_iface.group_num; + struct kbase_queue_group *group; u32 slot_num; int ret; lockdep_assert_held(&scheduler->lock); - for (slot_num = 0; slot_num < num_groups; slot_num++) { - struct kbase_queue_group *group = scheduler->csg_slots[slot_num].resident_group; + for (slot_num = 0; slot_num < num_groups; slot_num++) { + group = scheduler->csg_slots[slot_num].resident_group; - if (group) { + if (group) { suspend_queue_group(group); - set_bit(slot_num, slot_mask); + set_bit(slot_num, slot_mask); + } } - } ret = wait_csg_slots_suspend(kbdev, slot_mask); return ret; @@ -5819,7 +5823,10 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) /* Cancel any potential queued delayed work(s) */ cancel_tick_work(scheduler); cancel_tock_work(scheduler); - cancel_gpu_idle_work(scheduler); + /* gpu_idle_worker() might already be running at this point, which + * could decrement the pending_gpu_idle_worker counter to below 0. + * It'd be safer to let it run if one has already been scheduled. + */ cancel_delayed_work_sync(&scheduler->ping_work); mutex_lock(&scheduler->lock); @@ -5837,10 +5844,22 @@ static void scheduler_inner_reset(struct kbase_device *kbdev) scheduler->top_kctx = NULL; scheduler->top_grp = NULL; + atomic_set(&scheduler->gpu_idle_timer_enabled, false); + atomic_set(&scheduler->fw_soi_enabled, false); + KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, scheduler->num_active_address_spaces | (((u64)scheduler->total_runnable_grps) << 32)); +#ifdef KBASE_PM_RUNTIME + if (scheduler->state == SCHED_SLEEPING) { +#if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) + hrtimer_cancel(&scheduler->gpu_metrics_timer); +#endif + scheduler->state = SCHED_SUSPENDED; + KBASE_KTRACE_ADD(kbdev, SCHED_SUSPENDED, NULL, scheduler->state); + } +#endif mutex_unlock(&scheduler->lock); } @@ -6726,6 +6745,11 @@ static int kbase_csf_scheduler_kthread(void *data) while (atomic_read(&scheduler->pending_gpu_idle_work) > 0) gpu_idle_worker(kbdev); + /* Update GLB_IDLE timer/FW Sleep-on-Idle config (which might + * have been disabled during FW boot et. al.). + */ + kbase_csf_firmware_soi_update(kbdev); + dev_dbg(kbdev->dev, "Waking up for event after a scheduling iteration."); wake_up_all(&kbdev->csf.event_wait); } @@ -6786,6 +6810,9 @@ int kbase_csf_scheduler_init(struct kbase_device *kbdev) scheduler->gpu_metrics_timer.function = gpu_metrics_timer_callback; #endif /* CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD */ + atomic_set(&scheduler->gpu_idle_timer_enabled, false); + atomic_set(&scheduler->fw_soi_enabled, false); + return kbase_csf_mcu_shared_regs_data_init(kbdev); } @@ -6988,8 +7015,9 @@ int kbase_csf_scheduler_pm_suspend_no_lock(struct kbase_device *kbdev) goto exit; } else { dev_dbg(kbdev->dev, "Scheduler PM suspend"); - scheduler_suspend(kbdev); - cancel_tick_work(scheduler); + result = scheduler_suspend(kbdev); + if (!result) + cancel_tick_work(scheduler); } } diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h index e84994600809..915945bb495e 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_scheduler.h @@ -693,4 +693,6 @@ void kbase_csf_scheduler_force_wakeup(struct kbase_device *kbdev); void kbase_csf_scheduler_force_sleep(struct kbase_device *kbdev); #endif +bool is_gpu_level_suspend_supported(struct kbase_device *const kbdev); + #endif /* _KBASE_CSF_SCHEDULER_H_ */ diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c index 51d665f23970..5a5a4c315396 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_tiler_heap.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -260,8 +260,9 @@ static struct kbase_csf_tiler_heap_chunk *alloc_new_chunk(struct kbase_context * u64 chunk_size) { u64 nr_pages = PFN_UP(chunk_size); - u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | BASE_MEM_PROT_CPU_WR | - BASEP_MEM_NO_USER_FREE | BASE_MEM_COHERENT_LOCAL | BASE_MEM_PROT_CPU_RD; + base_mem_alloc_flags flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | + BASE_MEM_COHERENT_LOCAL | BASE_MEM_PROT_CPU_RD; struct kbase_csf_tiler_heap_chunk *chunk = NULL; /* The chunk kernel mapping needs to be large enough to: * - initially zero the CHUNK_HDR_SIZE area @@ -350,13 +351,14 @@ static struct kbase_csf_tiler_heap_chunk *alloc_new_chunk(struct kbase_context * } remove_external_chunk_mappings(kctx, chunk); - kbase_gpu_vm_unlock(kctx); /* If page migration is enabled, we don't want to migrate tiler heap pages. * This does not change if the constituent pages are already marked as isolated. */ if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(chunk->region->gpu_alloc, NOT_MOVABLE); + kbase_set_phy_alloc_page_status(kctx, chunk->region->gpu_alloc, NOT_MOVABLE); + + kbase_gpu_vm_unlock(kctx); return chunk; @@ -640,7 +642,7 @@ static bool kbasep_is_buffer_descriptor_region_suitable(struct kbase_context *co if (!(reg->flags & KBASE_REG_CPU_RD) || kbase_is_region_shrinkable(reg) || (reg->flags & KBASE_REG_PF_GROW)) { - dev_err(kctx->kbdev->dev, "Region has invalid flags: 0x%lX!\n", reg->flags); + dev_err(kctx->kbdev->dev, "Region has invalid flags: 0x%llX!\n", reg->flags); return false; } @@ -737,7 +739,7 @@ int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, u32 const chunk_ KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING); if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(buf_desc_reg->gpu_alloc, NOT_MOVABLE); + kbase_set_phy_alloc_page_status(kctx, buf_desc_reg->gpu_alloc, NOT_MOVABLE); kbase_gpu_vm_unlock(kctx); diff --git a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c index cdab5a17f70c..a9469c5949b4 100644 --- a/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c +++ b/drivers/gpu/arm/bifrost/csf/mali_kbase_csf_trace_buffer.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -469,14 +469,15 @@ unsigned int kbase_csf_firmware_trace_buffer_read_data(struct firmware_trace_buf } else { unsigned int bytes_copied_head, bytes_copied_tail; - bytes_copied_tail = min_t(unsigned int, num_bytes, (buffer_size - extract_offset)); + bytes_copied_tail = + min_t(unsigned int, num_bytes, size_sub(buffer_size, extract_offset)); memcpy(data, &data_cpu_va[extract_offset], bytes_copied_tail); bytes_copied_head = min_t(unsigned int, (num_bytes - bytes_copied_tail), insert_offset); memcpy(&data[bytes_copied_tail], data_cpu_va, bytes_copied_head); - bytes_copied = bytes_copied_head + bytes_copied_tail; + bytes_copied = size_add(bytes_copied_head, bytes_copied_tail); extract_offset += bytes_copied; if (extract_offset >= buffer_size) extract_offset = bytes_copied_head; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c index fe8201f7f7e6..da56d71f473f 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_coresight_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -443,7 +443,7 @@ kbase_debug_coresight_csf_config_create(void *client_data, } config = kzalloc(sizeof(struct kbase_debug_coresight_csf_config), GFP_KERNEL); - if (WARN_ON(!client)) + if (WARN_ON(!config)) return NULL; config->client = client; diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h index 18520db15502..04da9c8b9057 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_csf.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -212,6 +212,9 @@ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_EVICT_CTX_SLOTS_START), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_BUSY), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_INACTIVE), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_SUSPENDED), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_SLEEPING), + /* info_val == true if FW Sleep-on-Idle is enabled, false otherwise */ + KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_SLEEP_ON_IDLE_CHANGED), + /* info_val = mcu state */ #define KBASEP_MCU_STATE(n) KBASE_KTRACE_CODE_MAKE_CODE(PM_MCU_##n), #include "backend/gpu/mali_kbase_pm_mcu_states.h" diff --git a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h index 0b0de2385f85..7c40f472a78b 100644 --- a/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h +++ b/drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_csf.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -69,6 +69,7 @@ DEFINE_MALI_ADD_EVENT(SCHED_BUSY); DEFINE_MALI_ADD_EVENT(SCHED_INACTIVE); DEFINE_MALI_ADD_EVENT(SCHED_SUSPENDED); DEFINE_MALI_ADD_EVENT(SCHED_SLEEPING); +DEFINE_MALI_ADD_EVENT(FIRMWARE_SLEEP_ON_IDLE_CHANGED); #define KBASEP_MCU_STATE(n) DEFINE_MALI_ADD_EVENT(PM_MCU_##n); #include "backend/gpu/mali_kbase_pm_mcu_states.h" #undef KBASEP_MCU_STATE diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h index 991f70fe8540..d40eec013cb5 100644 --- a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -149,13 +149,17 @@ KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY), KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK), /* info_val == kctx->refcount */ KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX), -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* * Arbitration events */ - KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST), KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE), KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_EVT), -#endif + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_GRANTED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STARTED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOP_REQUESTED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_STOPPED), + KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_REQUESTED), #if MALI_USE_CSF #include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" diff --git a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h index 1ebddfa3f44f..acc78eb5b0b2 100644 --- a/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h +++ b/drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -95,13 +95,16 @@ DEFINE_MALI_ADD_EVENT(PM_RUNTIME_RESUME_CALLBACK); #undef KBASEP_L2_STATE DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK); DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX); -#ifdef CONFIG_MALI_ARBITER_SUPPORT -DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST); DEFINE_MALI_ADD_EVENT(ARB_VM_STATE); DEFINE_MALI_ADD_EVENT(ARB_VM_EVT); +DEFINE_MALI_ADD_EVENT(ARB_GPU_GRANTED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST); +DEFINE_MALI_ADD_EVENT(ARB_GPU_STARTED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_STOP_REQUESTED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_STOPPED); +DEFINE_MALI_ADD_EVENT(ARB_GPU_REQUESTED); -#endif #if MALI_USE_CSF #include "backend/mali_kbase_debug_linux_ktrace_csf.h" #else diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c index 6fae88665d43..218022ac3186 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_csf.c @@ -132,11 +132,15 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) fail_update_l2_features: kbase_backend_devfreq_term(kbdev); -fail_devfreq_init: - kbasep_pm_metrics_term(kbdev); -fail_pm_metrics_init: - kbase_ipa_control_term(kbdev); +fail_devfreq_init: + { + kbasep_pm_metrics_term(kbdev); + } +fail_pm_metrics_init: + { + kbase_ipa_control_term(kbdev); + } #ifdef CONFIG_MALI_BIFROST_DEBUG #if IS_ENABLED(CONFIG_MALI_REAL_HW) fail_interrupt_test: diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c index 4706a4f15b65..3b27b87657a5 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_csf.c @@ -32,7 +32,7 @@ bool kbase_is_gpu_removed(struct kbase_device *kbdev) { - if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + if (!kbase_has_arbiter(kbdev)) return false; @@ -88,6 +88,7 @@ static void kbase_gpu_fault_interrupt(struct kbase_device *kbdev) void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) { u32 power_changed_mask = (POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ); + struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); @@ -95,7 +96,6 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) kbase_gpu_fault_interrupt(kbdev); if (val & GPU_PROTECTED_FAULT) { - struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; unsigned long flags; dev_err_ratelimited(kbdev->dev, "GPU fault in protected mode"); @@ -149,10 +149,33 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) unsigned long flags; dev_dbg(kbdev->dev, "Doorbell mirror interrupt received"); + + /* Assume that the doorbell comes from userspace which + * presents new works in order to invalidate a possible GPU + * idle event. + * If the doorbell was raised by KBase then the FW would handle + * the pending doorbell then raise a 2nd GBL_IDLE IRQ which + * would allow us to put the GPU to sleep. + */ + atomic_set(&scheduler->gpu_no_longer_idle, true); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); kbase_pm_disable_db_mirror_interrupt(kbdev); - kbdev->pm.backend.exit_gpu_sleep_mode = true; - kbase_csf_scheduler_invoke_tick(kbdev); + + if (likely(kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP)) { + kbdev->pm.backend.exit_gpu_sleep_mode = true; + kbase_csf_scheduler_invoke_tick(kbdev); + } else if (likely(test_bit(KBASE_GPU_SUPPORTS_FW_SLEEP_ON_IDLE, + &kbdev->pm.backend.gpu_sleep_allowed)) && + (kbdev->pm.backend.mcu_state != KBASE_MCU_ON_PEND_SLEEP)) { + /* The firmware is going to sleep on its own but new + * doorbells were rung before we manage to handle + * the GLB_IDLE IRQ in the bottom half. We shall enable + * DB notification to allow the DB to be handled by FW. + */ + dev_dbg(kbdev->dev, "Re-enabling MCU immediately following DB_MIRROR IRQ"); + kbase_pm_enable_mcu_db_notification(kbdev); + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); } #endif @@ -179,7 +202,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) * cores. */ if (kbdev->pm.backend.l2_always_on || - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) kbase_pm_power_changed(kbdev); } diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c index c0cb835de69c..4dd9a228aa11 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_hw_jm.c @@ -30,7 +30,7 @@ bool kbase_is_gpu_removed(struct kbase_device *kbdev) { - if (!IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + if (!kbase_has_arbiter(kbdev)) return false; return (KBASE_REG_READ(kbdev, GPU_CONTROL_ENUM(GPU_ID)) == 0); @@ -103,7 +103,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) * cores. */ if (kbdev->pm.backend.l2_always_on || - kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) + kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_921)) kbase_pm_power_changed(kbdev); } diff --git a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c index 0fe76918296a..8cdf26e28ac6 100644 --- a/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c +++ b/drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c @@ -32,10 +32,6 @@ #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT -#include -#endif - #include #include #include diff --git a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c index ccb62c2a5cb5..e5bed33d1129 100644 --- a/drivers/gpu/arm/bifrost/device/mali_kbase_device.c +++ b/drivers/gpu/arm/bifrost/device/mali_kbase_device.c @@ -51,10 +51,7 @@ #include "backend/gpu/mali_kbase_irq_internal.h" #include "mali_kbase_regs_history_debugfs.h" #include "mali_kbase_pbha.h" - -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include "arbiter/mali_kbase_arbiter_pm.h" -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) @@ -69,6 +66,22 @@ static DEFINE_MUTEX(kbase_dev_list_lock); static LIST_HEAD(kbase_dev_list); static unsigned int kbase_dev_nr; +static unsigned int mma_wa_id; + +static int set_mma_wa_id(const char *val, const struct kernel_param *kp) +{ + return kbase_param_set_uint_minmax(val, kp, 1, 15); +} + +static const struct kernel_param_ops mma_wa_id_ops = { + .set = set_mma_wa_id, + .get = param_get_uint, +}; + +module_param_cb(mma_wa_id, &mma_wa_id_ops, &mma_wa_id, 0444); +__MODULE_PARM_TYPE(mma_wa_id, "uint"); +MODULE_PARM_DESC(mma_wa_id, "PBHA ID for MMA workaround. Valid range is from 1 to 15."); + struct kbase_device *kbase_device_alloc(void) { return vzalloc(sizeof(struct kbase_device)); @@ -320,6 +333,10 @@ int kbase_device_misc_init(struct kbase_device *const kbdev) if (err) goto dma_set_mask_failed; + /* Set mma_wa_id if it has been passed in as a module parameter */ + if ((kbdev->gpu_props.gpu_id.arch_id >= GPU_ID_ARCH_MAKE(14, 8, 0)) && mma_wa_id != 0) + kbdev->mma_wa_id = mma_wa_id; + err = kbase_pbha_read_dtb(kbdev); if (err) goto term_as; @@ -597,17 +614,12 @@ int kbase_device_early_init(struct kbase_device *kbdev) /* We're done accessing the GPU registers for now. */ kbase_pm_register_access_disable(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { if (kbdev->pm.arb_vm_state) err = kbase_arbiter_pm_install_interrupts(kbdev); } else { err = kbase_install_interrupts(kbdev); } -#else - err = kbase_install_interrupts(kbdev); -#endif - if (err) goto gpuprops_term; @@ -633,15 +645,10 @@ ktrace_term: void kbase_device_early_term(struct kbase_device *kbdev) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) + if (kbase_has_arbiter(kbdev)) kbase_arbiter_pm_release_interrupts(kbdev); else kbase_release_interrupts(kbdev); -#else - kbase_release_interrupts(kbdev); -#endif - kbase_gpuprops_term(kbdev); kbase_device_backend_term(kbdev); kbase_regmap_term(kbdev); diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c index 2cce391606c3..d7dd6200d497 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access.c @@ -31,8 +31,8 @@ #define KBASE_REGMAP_ACCESS_ALWAYS_POWERED (1U << 16) static u32 always_powered_regs[] = { -#if MALI_USE_CSF -#else /* MALI_USE_CSF */ + +#if !MALI_USE_CSF PTM_AW_IRQ_CLEAR, PTM_AW_IRQ_INJECTION, PTM_AW_IRQ_MASK, @@ -44,13 +44,19 @@ static u32 always_powered_regs[] = { PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE1, PTM_AW_MESSAGE__PTM_OUTGOING_MESSAGE_STATUS, PTM_ID, -#endif /* MALI_USE_CSF */ +#endif /* !MALI_USE_CSF */ }; static void kbasep_reg_setup_always_powered_registers(struct kbase_device *kbdev) { u32 i; + +#if !MALI_USE_CSF + if (kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(9, 14, 0)) + return; +#endif /* MALI_USE_CSF */ + for (i = 0; i < ARRAY_SIZE(always_powered_regs); i++) { u32 reg_enum = always_powered_regs[i]; diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h index a8708fafc638..591391c6a8a1 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h index a62d1707ebb7..9392d44f684b 100644 --- a/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h +++ b/drivers/gpu/arm/bifrost/hw_access/mali_kbase_hw_access_regmap_legacy.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h index 56ec5e015d11..e0568d8f8c6d 100644 --- a/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h +++ b/drivers/gpu/arm/bifrost/hw_access/regmap/mali_kbase_regmap_csf_macros.h @@ -185,6 +185,7 @@ */ #define AS_MEMATTR_ATTRIBUTE0_MEMORY_TYPE_SHARED 0x0 + /* CSF_CONFIG register */ #define CSF_CONFIG_FORCE_COHERENCY_FEATURES_SHIFT 2 diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h index cc3ba98ab6fe..a6d418b8e82c 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -78,6 +78,18 @@ typedef int kbase_hwcnt_backend_init_fn(const struct kbase_hwcnt_backend_info *i */ typedef void kbase_hwcnt_backend_term_fn(struct kbase_hwcnt_backend *backend); +/** + * typedef kbase_hwcnt_backend_acquire_fn - Enable counter collection. + * @backend: Non-NULL pointer to backend interface. + */ +typedef void kbase_hwcnt_backend_acquire_fn(const struct kbase_hwcnt_backend *backend); + +/** + * typedef kbase_hwcnt_backend_release_fn - Disable counter collection. + * @backend: Non-NULL pointer to backend interface. + */ +typedef void kbase_hwcnt_backend_release_fn(const struct kbase_hwcnt_backend *backend); + /** * typedef kbase_hwcnt_backend_timestamp_ns_fn - Get the current backend * timestamp. @@ -206,6 +218,10 @@ typedef int kbase_hwcnt_backend_dump_get_fn(struct kbase_hwcnt_backend *backend, * metadata. * @init: Function ptr to initialise an instance of the backend. * @term: Function ptr to terminate an instance of the backend. + * @acquire: Callback to indicate that counter collection has + * been enabled. + * @release: Callback to indicate that counter collection has + * been disabled. * @timestamp_ns: Function ptr to get the current backend timestamp. * @dump_enable: Function ptr to enable dumping. * @dump_enable_nolock: Function ptr to enable dumping while the @@ -222,6 +238,8 @@ struct kbase_hwcnt_backend_interface { kbase_hwcnt_backend_metadata_fn *metadata; kbase_hwcnt_backend_init_fn *init; kbase_hwcnt_backend_term_fn *term; + kbase_hwcnt_backend_acquire_fn *acquire; + kbase_hwcnt_backend_release_fn *release; kbase_hwcnt_backend_timestamp_ns_fn *timestamp_ns; kbase_hwcnt_backend_dump_enable_fn *dump_enable; kbase_hwcnt_backend_dump_enable_nolock_fn *dump_enable_nolock; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c index 8bdfc9a0bfda..b937c047a94a 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf.c @@ -1710,6 +1710,22 @@ static void kbasep_hwcnt_backend_csf_term(struct kbase_hwcnt_backend *backend) kbasep_hwcnt_backend_csf_destroy(backend_csf); } +static void kbasep_hwcnt_backend_csf_acquire(const struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend; + struct kbase_hwcnt_backend_csf_info *csf_info = backend_csf->info; + + csf_info->csf_if->acquire(csf_info->csf_if->ctx); +} + +static void kbasep_hwcnt_backend_csf_release(const struct kbase_hwcnt_backend *backend) +{ + struct kbase_hwcnt_backend_csf *backend_csf = (struct kbase_hwcnt_backend_csf *)backend; + struct kbase_hwcnt_backend_csf_info *csf_info = backend_csf->info; + + csf_info->csf_if->release(csf_info->csf_if->ctx); +} + /** * kbasep_hwcnt_backend_csf_info_destroy() - Destroy a CSF backend info. * @info: Pointer to info to destroy. @@ -2168,6 +2184,8 @@ int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, u3 iface->metadata = kbasep_hwcnt_backend_csf_metadata; iface->init = kbasep_hwcnt_backend_csf_init; iface->term = kbasep_hwcnt_backend_csf_term; + iface->acquire = kbasep_hwcnt_backend_csf_acquire; + iface->release = kbasep_hwcnt_backend_csf_release; iface->timestamp_ns = kbasep_hwcnt_backend_csf_timestamp_ns; iface->dump_enable = kbasep_hwcnt_backend_csf_dump_enable; iface->dump_enable_nolock = kbasep_hwcnt_backend_csf_dump_enable_nolock; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h index c982900cf755..81f809fdc83a 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if.h @@ -114,6 +114,20 @@ typedef void (*kbase_hwcnt_backend_csf_if_lock_fn)(struct kbase_hwcnt_backend_cs typedef void (*kbase_hwcnt_backend_csf_if_unlock_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx, unsigned long flags); +/** + * typedef kbase_hwcnt_backend_csf_if_acquire_fn - Enable counter collection. + * + * @ctx: Non-NULL pointer to a CSF context. + */ +typedef void (*kbase_hwcnt_backend_csf_if_acquire_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx); + +/** + * typedef kbase_hwcnt_backend_csf_if_release_fn - Disable counter collection. + * + * @ctx: Non-NULL pointer to a CSF context. + */ +typedef void (*kbase_hwcnt_backend_csf_if_release_fn)(struct kbase_hwcnt_backend_csf_if_ctx *ctx); + /** * typedef kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn - Get performance * counter information. @@ -272,6 +286,10 @@ typedef void (*kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn)( * @assert_lock_held: Function ptr to assert backend spinlock is held. * @lock: Function ptr to acquire backend spinlock. * @unlock: Function ptr to release backend spinlock. + * @acquire: Callback to indicate that counter collection has + * been enabled. + * @release: Callback to indicate that counter collection has + * been disabled. * @get_prfcnt_info: Function ptr to get performance counter related * information. * @ring_buf_alloc: Function ptr to allocate ring buffer for CSF HWC. @@ -292,6 +310,8 @@ struct kbase_hwcnt_backend_csf_if { kbase_hwcnt_backend_csf_if_assert_lock_held_fn assert_lock_held; kbase_hwcnt_backend_csf_if_lock_fn lock; kbase_hwcnt_backend_csf_if_unlock_fn unlock; + kbase_hwcnt_backend_csf_if_acquire_fn acquire; + kbase_hwcnt_backend_csf_if_release_fn release; kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn get_prfcnt_info; kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn ring_buf_alloc; kbase_hwcnt_backend_csf_if_ring_buf_sync_fn ring_buf_sync; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c index 3cf039f70056..29f8a2a8838d 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_csf_if_fw.c @@ -131,6 +131,26 @@ static void kbasep_hwcnt_backend_csf_if_fw_unlock(struct kbase_hwcnt_backend_csf kbase_csf_scheduler_spin_unlock(kbdev, flags); } +static void kbasep_hwcnt_backend_csf_if_fw_acquire(struct kbase_hwcnt_backend_csf_if_ctx *ctx) +{ + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + /* Mark performance counters collection as enabled */ + set_bit(KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED, + &fw_ctx->kbdev->pm.backend.gpu_sleep_allowed); +} + +static void kbasep_hwcnt_backend_csf_if_fw_release(struct kbase_hwcnt_backend_csf_if_ctx *ctx) +{ + struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = + (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; + + /* Mark performance counters collection as disabled */ + clear_bit(KBASE_GPU_PERF_COUNTERS_COLLECTION_ENABLED, + &fw_ctx->kbdev->pm.backend.gpu_sleep_allowed); +} + /** * kbasep_hwcnt_backend_csf_if_fw_on_freq_change() - On freq change callback * @@ -813,6 +833,8 @@ int kbase_hwcnt_backend_csf_if_fw_create(struct kbase_device *kbdev, if_fw->assert_lock_held = kbasep_hwcnt_backend_csf_if_fw_assert_lock_held; if_fw->lock = kbasep_hwcnt_backend_csf_if_fw_lock; if_fw->unlock = kbasep_hwcnt_backend_csf_if_fw_unlock; + if_fw->acquire = kbasep_hwcnt_backend_csf_if_fw_acquire; + if_fw->release = kbasep_hwcnt_backend_csf_if_fw_release; if_fw->get_prfcnt_info = kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info; if_fw->ring_buf_alloc = kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc; if_fw->ring_buf_sync = kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c index 1ee0e3b823ad..c3f2bcdbf256 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm.c @@ -683,7 +683,7 @@ static int kbasep_hwcnt_backend_jm_dump_alloc(const struct kbase_hwcnt_backend_j struct kbase_context *kctx, u64 *gpu_dump_va) { struct kbase_va_region *reg; - u64 flags; + base_mem_alloc_flags flags; u64 nr_pages; /* Calls to this function are inherently asynchronous, with respect to @@ -851,6 +851,14 @@ static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend) kbasep_hwcnt_backend_jm_destroy((struct kbase_hwcnt_backend_jm *)backend); } +static void kbasep_hwcnt_backend_jm_acquire(const struct kbase_hwcnt_backend *backend) +{ +} + +static void kbasep_hwcnt_backend_jm_release(const struct kbase_hwcnt_backend *backend) +{ +} + /** * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info. * @info: Pointer to info to destroy. @@ -932,6 +940,8 @@ int kbase_hwcnt_backend_jm_create(struct kbase_device *kbdev, iface->metadata = kbasep_hwcnt_backend_jm_metadata; iface->init = kbasep_hwcnt_backend_jm_init; iface->term = kbasep_hwcnt_backend_jm_term; + iface->acquire = kbasep_hwcnt_backend_jm_acquire; + iface->release = kbasep_hwcnt_backend_jm_release; iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable; iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock; diff --git a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c index cf2a2e65bc25..88917e72ac58 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c +++ b/drivers/gpu/arm/bifrost/hwcnt/backend/mali_kbase_hwcnt_backend_jm_watchdog.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -317,6 +317,14 @@ kbasep_hwcnt_backend_jm_watchdog_term_partial(struct kbase_hwcnt_backend_jm_watc kfree(wd_backend); } +static void kbasep_hwcnt_backend_jm_watchdog_acquire(const struct kbase_hwcnt_backend *backend) +{ +} + +static void kbasep_hwcnt_backend_jm_watchdog_release(const struct kbase_hwcnt_backend *backend) +{ +} + /* Job manager watchdog backend, implementation of kbase_hwcnt_backend_term_fn * Calling term does *not* destroy the interface */ @@ -807,6 +815,8 @@ int kbase_hwcnt_backend_jm_watchdog_create(struct kbase_hwcnt_backend_interface .metadata = kbasep_hwcnt_backend_jm_watchdog_metadata, .init = kbasep_hwcnt_backend_jm_watchdog_init, .term = kbasep_hwcnt_backend_jm_watchdog_term, + .acquire = kbasep_hwcnt_backend_jm_watchdog_acquire, + .release = kbasep_hwcnt_backend_jm_watchdog_release, .timestamp_ns = kbasep_hwcnt_backend_jm_watchdog_timestamp_ns, .dump_enable = kbasep_hwcnt_backend_jm_watchdog_dump_enable, .dump_enable_nolock = kbasep_hwcnt_backend_jm_watchdog_dump_enable_nolock, diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c index 8b1de2e1cdaf..8d308f1138a7 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -599,6 +599,9 @@ int kbase_hwcnt_accumulator_acquire(struct kbase_hwcnt_context *hctx, return errcode; } + /* Inform the backend that counter collection has been enabled. */ + hctx->iface->acquire(hctx->accum.backend); + spin_lock_irqsave(&hctx->state_lock, flags); WARN_ON(hctx->disable_count == 0); @@ -646,6 +649,9 @@ void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) mutex_unlock(&hctx->accum_lock); + /* Inform the backend that counter collection has been disabled. */ + hctx->iface->release(hctx->accum.backend); + kbasep_hwcnt_accumulator_term(hctx); mutex_lock(&hctx->accum_lock); diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h index 1f25282d378a..896f1389eb37 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_gpu.h @@ -169,7 +169,7 @@ enum kbase_hwcnt_physical_set { /** * struct kbase_hwcnt_gpu_info - Information about hwcnt blocks on the GPUs. * @l2_count: L2 cache count. - * @sc_core_mask: Shader core mask. May be sparse. + * @sc_core_mask: Shader core mask. May be sparse. * @clk_cnt: Number of clock domains available. * @csg_cnt: Number of CSGs available. * @prfcnt_values_per_block: Total entries (header + counters) of performance diff --git a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h index c7afe173d426..45f67f7c9a1b 100644 --- a/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h +++ b/drivers/gpu/arm/bifrost/hwcnt/mali_kbase_hwcnt_types.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2018-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2018-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -19,65 +19,6 @@ * */ -/* - * Hardware counter types. - * Contains structures for describing the physical layout of hardware counter - * dump buffers and enable maps within a system. - * - * Also contains helper functions for manipulation of these dump buffers and - * enable maps. - * - * Through use of these structures and functions, hardware counters can be - * enabled, copied, accumulated, and generally manipulated in a generic way, - * regardless of the physical counter dump layout. - * - * Terminology: - * - * Hardware Counter System: - * A collection of hardware counter blocks, making a full hardware counter - * system. - * Hardware Counter Block: - * A block of hardware counters (e.g. shader block, tiler block). - * Hardware Counter Block Instance: - * An instance of a Hardware Counter Block (e.g. an MP4 GPU might have - * 4 shader block instances). - * - * Block Header: - * A header value inside a counter block. Headers don't count anything, - * so it is only valid to copy or zero them. Headers are always the first - * values in the block. - * Block Counter: - * A counter value inside a counter block. Counters can be zeroed, copied, - * or accumulated. Counters are always immediately after the headers in the - * block. - * Block Value: - * A catch-all term for block headers and block counters. - * - * Enable Map: - * An array of u64 bitfields, where each bit either enables exactly one - * block value, or is unused (padding). Note that this is derived from - * the client configuration, and is not obtained from the hardware. - * Dump Buffer: - * An array of u64 values, where each u64 corresponds either to one block - * value, or is unused (padding). - * Block State Buffer: - * An array of blk_stt_t values, where each blk_stt_t corresponds to one block - * instance and is used to track the on/off power state transitions, as well has - * hardware resource availability, and whether the block was operating - * in normal or protected mode. - * Availability Mask: - * A bitfield, where each bit corresponds to whether a block instance is - * physically available (e.g. an MP3 GPU may have a sparse core mask of - * 0b1011, meaning it only has 3 cores but for hardware counter dumps has the - * same dump buffer layout as an MP4 GPU with a core mask of 0b1111. In this - * case, the availability mask might be 0b1011111 (the exact layout will - * depend on the specific hardware architecture), with the 3 extra early bits - * corresponding to other block instances in the hardware counter system). - * Metadata: - * Structure describing the physical layout of the enable map and dump buffers - * for a specific hardware counter system. - */ - #ifndef _KBASE_HWCNT_TYPES_H_ #define _KBASE_HWCNT_TYPES_H_ diff --git a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c index e4138580de20..9305747ff472 100644 --- a/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c +++ b/drivers/gpu/arm/bifrost/ipa/mali_kbase_ipa_debugfs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -115,7 +115,7 @@ static ssize_t param_string_set(struct file *file, const char __user *user_buf, goto end; } - buf_size = min(param->size - 1, count); + buf_size = min(size_sub(param->size, 1), count); if (copy_from_user(param->addr.str, user_buf, buf_size)) { ret = -EFAULT; goto end; diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h index 373b9b1b73b3..23e919314333 100644 --- a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -308,11 +308,11 @@ enum kbase_atom_gpu_rb_state { * powered down and GPU shall come out of fully * coherent mode before entering protected mode. * @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change; - * for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on + * for KBASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on * so that coherency register contains correct value when * GPU enters protected mode. * @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for - * BASE_HW_ISSUE_TGOX_R1_1234 check + * KBASE_HW_ISSUE_TGOX_R1_1234 check * that L2 is powered up and switch GPU to protected mode. */ enum kbase_atom_enter_protected_state { @@ -500,10 +500,6 @@ enum kbase_atom_exit_protected_state { * is snapshot of the age_count counter in kbase * context. * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. - * @renderpass_id:Renderpass identifier used to associate an atom that has - * BASE_JD_REQ_START_RENDERPASS set in its core requirements - * with an atom that has BASE_JD_REQ_END_RENDERPASS set. - * @jc_fragment: Set of GPU fragment job chains */ struct kbase_jd_atom { struct work_struct work; @@ -564,8 +560,6 @@ struct kbase_jd_atom { enum base_jd_event_code event_code; base_jd_core_req core_req; u8 jobslot; - u8 renderpass_id; - struct base_jd_fragment jc_fragment; u32 ticks; int sched_priority; @@ -676,71 +670,6 @@ static inline bool kbase_jd_atom_is_earlier(const struct kbase_jd_atom *katom_a, #define KBASE_JD_DEP_QUEUE_SIZE 256 -/** - * enum kbase_jd_renderpass_state - State of a renderpass - * @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to - * START. - * @KBASE_JD_RP_START: Renderpass making a first attempt at tiling. - * Can transition to PEND_OOM or COMPLETE. - * @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much - * memory and has a soft-stop pending. Can transition to - * OOM or COMPLETE. - * @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much - * memory and therefore switched to incremental - * rendering. The fragment job chain is forced to run. - * Can only transition to RETRY. - * @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at - * tiling. Can transition to RETRY_PEND_OOM or COMPLETE. - * @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at - * tiling used too much memory again and has a - * soft-stop pending. Can transition to RETRY_OOM - * or COMPLETE. - * @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at - * tiling used too much memory again. The fragment job - * chain is forced to run. Can only transition to RETRY. - * - * A state machine is used to control incremental rendering. - */ -enum kbase_jd_renderpass_state { - KBASE_JD_RP_COMPLETE, /* COMPLETE => START */ - KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */ - KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */ - KBASE_JD_RP_OOM, /* OOM => RETRY */ - KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or COMPLETE */ - KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or COMPLETE */ - KBASE_JD_RP_RETRY_OOM /* RETRY_OOM => RETRY */ -}; - -/** - * struct kbase_jd_renderpass - Data for a renderpass - * @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then - * all other members are invalid. - * Both the job dispatcher context and hwaccess_lock must be - * locked to modify this so that it can be read with either - * (or both) locked. - * @start_katom: Address of the atom that is the start of a renderpass. - * Both the job dispatcher context and hwaccess_lock must be - * locked to modify this so that it can be read with either - * (or both) locked. - * @end_katom: Address of the atom that is the end of a renderpass, or NULL - * if that atom hasn't been added to the job scheduler yet. - * The job dispatcher context and hwaccess_lock must be - * locked to modify this so that it can be read with either - * (or both) locked. - * @oom_reg_list: A list of region structures which triggered out-of-memory. - * The hwaccess_lock must be locked to access this. - * - * Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS - * are associated with an object of this type, which is created and maintained - * by kbase to keep track of each renderpass. - */ -struct kbase_jd_renderpass { - enum kbase_jd_renderpass_state state; - struct kbase_jd_atom *start_katom; - struct kbase_jd_atom *end_katom; - struct list_head oom_reg_list; -}; - /** * struct kbase_jd_context - per context object encapsulating all the * Job dispatcher related state. @@ -751,9 +680,6 @@ struct kbase_jd_renderpass { * @atoms: Array of the objects representing atoms, * containing the complete state and attributes * of an atom. - * @renderpasses: Array of renderpass state for incremental - * rendering, indexed by user-specified renderpass - * ID. * @job_nr: Tracks the number of atoms being processed by the * kbase. This includes atoms that are not tracked by * scheduler: 'not ready to run' & 'dependency-only' @@ -803,7 +729,6 @@ struct kbase_jd_context { struct mutex lock; struct kbasep_js_kctx_info sched_info; struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT]; - struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT]; struct workqueue_struct *job_done_wq; wait_queue_head_t zero_jobs_wait; diff --git a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h index 333ad2d2b150..65b54c68d8c7 100644 --- a/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h +++ b/drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -567,22 +567,6 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom */ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp); -/** - * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot - * dependency - * @katom: Pointer to an atom in the slot ringbuffer - * - * A cross-slot dependency is ignored if necessary to unblock incremental - * rendering. If the atom at the start of a renderpass used too much memory - * and was soft-stopped then the atom at the end of a renderpass is submitted - * to hardware regardless of its dependency on the start-of-renderpass atom. - * This can happen multiple times for the same pair of atoms. - * - * Return: true to block the atom or false to allow it to be submitted to - * hardware. - */ -bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *katom); - /** * kbase_js_sched - Submit atoms from all available contexts. * @@ -809,8 +793,7 @@ static inline bool kbasep_js_has_atom_finished(const struct kbasep_js_atom_retained_state *katom_retained_state) { return (bool)(katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && - katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT && - katom_retained_state->event_code != BASE_JD_EVENT_END_RP_DONE); + katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT); } /** diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h deleted file mode 100644 index 843094076d69..000000000000 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h +++ /dev/null @@ -1,163 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, - * please update base/tools/hwconfig_generator/hwc_{issues,features}.py - * For more information see base/tools/docs/hwconfig_generator.md - */ - -#ifndef _BASE_HWCONFIG_FEATURES_H_ -#define _BASE_HWCONFIG_FEATURES_H_ - -#include - -enum base_hw_feature { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_L2_SLICE_HASH, - BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - BASE_HW_FEATURE_CORE_FEATURES, - BASE_HW_FEATURE_PBHA_HWU, - BASE_HW_FEATURE_LARGE_PAGE_ALLOC, - BASE_HW_FEATURE_THREAD_TLS_ALLOC, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_generic[] = { - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tMIx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tHEx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tSIx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tDVx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tNOx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tGOx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CORE_FEATURES, - BASE_HW_FEATURE_THREAD_TLS_ALLOC, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tTRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tNAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tBEx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tBAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tODx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tGRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tVAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tTUx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_L2_SLICE_HASH, BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tTIx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_L2_SLICE_HASH, - BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_CORE_FEATURES, - BASE_HW_FEATURE_PBHA_HWU, - BASE_HW_FEATURE_END -}; - -__maybe_unused static const enum base_hw_feature base_hw_features_tKRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_L2_CONFIG, BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_L2_SLICE_HASH, BASE_HW_FEATURE_GPU_SLEEP, - BASE_HW_FEATURE_CORE_FEATURES, BASE_HW_FEATURE_PBHA_HWU, - BASE_HW_FEATURE_LARGE_PAGE_ALLOC, BASE_HW_FEATURE_END -}; - - -#endif /* _BASE_HWCONFIG_FEATURES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h deleted file mode 100644 index 409e2e8bedcf..000000000000 --- a/drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h +++ /dev/null @@ -1,621 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * - * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, - * please update base/tools/hwconfig_generator/hwc_{issues,features}.py - * For more information see base/tools/docs/hwconfig_generator.md - */ - -#ifndef _BASE_HWCONFIG_ISSUES_H_ -#define _BASE_HWCONFIG_ISSUES_H_ - -#include - -enum base_hw_issue { - BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_10682, - BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_T76X_3953, - BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_7940, - BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TMIX_8138, - BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, - BASE_HW_ISSUE_TMIX_8463, - BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TMIX_8438, - BASE_HW_ISSUE_TNOX_1194, - BASE_HW_ISSUE_TGOX_R1_1234, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TSIX_1792, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, - BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_KRAKEHW_2151, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_KRAKEHW_2269, - BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__attribute__(( - unused)) static const enum base_hw_issue base_hw_issues_generic[] = { BASE_HW_ISSUE_END }; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_T76X_3953, BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, - BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, - BASE_HW_ISSUE_TMIX_8343, BASE_HW_ISSUE_TMIX_8463, BASE_HW_ISSUE_TMIX_8456, - BASE_HW_ISSUE_TMIX_8438, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tMIx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_7940, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TMIX_8138, BASE_HW_ISSUE_TMIX_8206, BASE_HW_ISSUE_TMIX_8343, - BASE_HW_ISSUE_TMIX_8456, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_11054, - BASE_HW_ISSUE_TMIX_7891, BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_10682, BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tHEx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_7891, - BASE_HW_ISSUE_TMIX_8042, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TSIX_1792, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_11054, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tSIx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tDVx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNOx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TNOX_1194, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, BASE_HW_ISSUE_TSIX_1116, - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TGOX_R1_1234, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_GPU2017_1336, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGOx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TMIX_8133, - BASE_HW_ISSUE_TSIX_1116, BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTRx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_3076, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_GPU2017_1336, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNAx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBEx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { - BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TTRX_3485, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p1[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p2[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBAx[] = { - BASE_HW_ISSUE_5736, BASE_HW_ISSUE_9435, BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TTRX_3414, BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_TTRX_3470, BASE_HW_ISSUE_TTRX_3464, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tODx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGRx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tVAx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, - BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_1997, - BASE_HW_ISSUE_GPU2019_3878, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTUx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p2[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p3[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_GPU2019_3878, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2019_3901, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTIx[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p0[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, - BASE_HW_ISSUE_GPU2021PRO_290, BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2922, BASE_HW_ISSUE_TITANHW_2952, - BASE_HW_ISSUE_TITANHW_2938, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p1[] = { - BASE_HW_ISSUE_TSIX_2033, BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2021PRO_290, - BASE_HW_ISSUE_TITANHW_2710, BASE_HW_ISSUE_TITANHW_2679, - BASE_HW_ISSUE_GPU2022PRO_148, BASE_HW_ISSUE_TITANHW_2938, - BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p0[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TITANHW_2922, - BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p1[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TURSEHW_2934, BASE_HW_ISSUE_END -}; - -__maybe_unused static const enum base_hw_issue base_hw_issues_model_tKRx[] = { - BASE_HW_ISSUE_TTRX_1337, BASE_HW_ISSUE_TURSEHW_2716, BASE_HW_ISSUE_GPU2022PRO_148, - BASE_HW_ISSUE_KRAKEHW_2151, BASE_HW_ISSUE_KRAKEHW_2269, BASE_HW_ISSUE_TURSEHW_2934, - BASE_HW_ISSUE_END -}; - - -#endif /* _BASE_HWCONFIG_ISSUES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_csffw.bin b/drivers/gpu/arm/bifrost/mali_csffw.bin index a1168ffdc7436dd69b31f643f2a77b9fc993d6e0..4319d6b74e3bdea3eec1abd666bcf9c06ddbdcbe 100644 GIT binary patch delta 35016 zcmZo@5Nv1=oZ!KzGSSnTO+}S~K_YQtmlT^sA|t~OiHWP=tW*CObta!=T+C=NIhQGz z(PZ*LCT&KG$%@RPlbM+17;PpiG8@Y}2s<(`GB7Zt7nH{5rj`_CCKofX3otZ1;dNxt znOw-M&*X7$@*-w^3yW0KZ$n%Qtm2Y5ZRKUQ~3tuxTPEHUM&`;E8U|7x!Q7~V# zfuT~ffuWEE6(7}XV35>mV93^Lm|i%6X|DX_fCdIps|JS9*ail4a4@z3WV$Z%%nC5W zko~QCBg5C6hRKeClA$vt8W}cAHZn-&Ky=O(X<)DwgJ4g|28Q~)28P_M28L4gMuxv? zjSS~C8{rmoW;QU)FKuA>Sl+;3t=EXqx7kowo6$NmtAT-ufdL$~COD?{8q}m2P0j?r&g7o(OW9C^O?EuzO9JHY-hjCer7$V=EI= z#x^DVHVEFg)1K#H69lz`$VK$iU#h!oZ-wz{0Qq!oS1Rz#zK1f#Dfb z14H|g28ImQ28P@<4Ga$&G(_DrDE~xE1H<8^4GaSD5PO6YA+%m1NL_sc!vrv+ zf#CyG;vbZ6lmt=ekOWfLz+jdJp*=((;wd5!eg~A6k%owCNkheJ8yI-kH84DBgwRe+ z5c&v|j_GV**jvB8fniQ(1B2k21_p($1_u2N4Gc;>4Gc@SHZTZHff%@KS_8w=ISmZ^ zp!AmM5b;CPAr?8!fbg|uL)2Z^57EEk5F|u)9D+oN#gPUE$E^(vhmJ$U<4!a%)Pu6o zx)Trsx=umpOQ#?PbDV}K%!AT3ry&L@oPnsDa0ZespPYyAf1QWu+a}$>V7RS;fkg&F zhunnF8=!Q>EePN1HZ=doJcJk&Bm>FcQjehqK81(}$U^vWvJeX{y=h>Wzqx@y?Ja~p z1Eo*BhwzVlgg9L73xuBW10t^U6GBTdG=lZNVyK5O_OUiH{N32VV8;fbLhp|^gT17Mh45B4Gei|jo>(60;PL2 z8^J~8EX_u6iP)yq2rdzq8G_Y;YO8uDh6aWu3JnY)P&y4t7eMJAC_N8KuY=ONp!EG8 zJQtWg^!#Sz{sQ;+pE(+IUw-sw_Bil^;eybI{J)$V81^#k z2Nm-UuNawUvvW)iR$b0ikT&^`st&vMPo4*yA2cTGtBLXm|6Gy5pz-jc@3#{PlLOS0 zOng3-FfuAIs2upjqho2;D;0gg9gKg+6M1$9}B>yDKNbDVqo~l$m#lkL4o7L z>fe+7)NPpNq)zTpmtfkLFnO7}LZbN3{h$5&S7c;nawJGUnE8W&!Qq>PpulhL0G8j3 zY}0-;Ff_T!UR>>F;Ks0@VLzj)ftx0n<@;?1*oh1o3>r%=PWxfN5bFkV7>fqmWiAbt z3k)A*{u#J2d~lkqu3@d<{V6~IVNCG(RKhU1NkhdhLIEVe zVIc6~)&CFAI6lPuy7^r9W2T?oTeTF>zpH=Kf-aihE5q*Vbe@kuW()+g(3)E85ltl3`&e2jDI_TeDEP~@=={|#@NZSy7C}}cDiCr0-Te>bQQQy z{#0;l`C-t&`XOU-qpoN@+<1|MOx_Rj|2-94J~1$|O<FVWBiabIbKf}Ang0? zOY+711D}|bm;Ue(y7Z$#;OGwzA=Zx@CO_9xW_SOm;KutQbF!emH&f2f$#MEJBEA23 zKQR6A|G?mI4OLBrw0 ziUv?dW>9zec7SE_Edx2_;)|>wEPpX-FkfKO$i2w%k&%PZjM1F+n}jHb{0w>uFuP~Z3>EF2N z;l}vEdUCs=to8Xn8(cLmT2u&pVB|Nw;Qi@E3B!kf4E`S&0y}>wFdX>Aq@s|)XrYkc z;l})7)lY>CiQS9}dtZET=lGyJ`M#mF^pZao6$jXPFM5A^0WNJAGZ+mS6F{X3$7E3> zQ_%=ea#m!tX!&8t;8}5!o%sUy2hqv#M*2(_cTJvPBq6N-zvYKPf&wUcD(!ty;K1^s zWAbjJ*-SV7P3~(KkU6>5SW)QBA27dB_vHsi7B^6tIC+Dy z7UR3gw~g%;8yR@r4zLScP;g`UFy~*w#wkA<8K(SLA^`FhgJ#Q*#>v(u`aCB4Km|zm z2Jde?43o=D)S17@Fic)xA}#q%2IOf4hS%K;tP|Y6IT*NnJHY7hO+sk$6%!A}W0Pe~ zJ=qWbP;g`YkT5ydR8sTDZ?2Dwj2xP*7g%R7e8~LYf^&&S|jPnA>cyKoT5Hoqb zsXt@uLW*y;Vv1ya6NH0%V0i1}nj%TiFDudq~5+wmKc=UG04|626i{{tt7f}5qA z6Sx>xs7S2fR1*BK?Dyn-mT?0Al0apYP{m{?rAxlwz9dc7u~K66o$PO=A^R;r&>;hq z8xFo?P{1g`rdUOA2mEDzz_dcq?ceu+$=|Kq8PzA-TkA7@Ntj$<-N*DVVUkTW-a@d< z#*rUd-X3_#AOx~v@-Z6+#+b?Mw)KJnoD7N^Ur(s#yeQ!|(~aSS$>iy_x!TWv|JVtt zzcSsJb;K@dxc&S7__yeT8EzSF%sR}MG~NDv-|?I2fu-BO@9QV)+lewBpX_QUFOm5B z|97U}3<__bGwcUNQtO6BH{Wkx5+>K!#WQ~k5S)C^uG{}xfKbd21_n=%pRtzOpa^HI zaQwi?$XLLtzzRyb{h*@D`haP3w<31`W5VKXyv6e3Wb3D*+i#ik)PI?_{u!uRcW&I!0CwkO1?N?&@Bbcq$-wB~#`9szPX#wdP+&Uq zeE7K=Tws8C|G#r@e($Wz$in}jp(tF+;}t&AVKGF^X>3DB&gH0ZKsoKj-)Re_&-~Oc0!W(A{2?#ls&I zu?ZkKgZ+P>n;0-S7;F~xFlAv(+wARQ&dBs9VRDOa2vdjRADcK&{Bp%CUIe>>#|1&*Iu zY#`~!k%uY6@so?eflrPM39iB!Gc&m_Iev1P{Mla>WQ$aQ0*l~>z#o(C0=(*1|E|vv z|G>mAv!DOdnF4_ii+^wb_}`xS4*#bU1p*)D{H|ayy9!}X`)%OH|6%$c5b5!3Ss+nM0fCuH$|5-Ak;z{0}+iKT#JKcgG}C!PX<4;sHGPYASO>ij?XT%efr-(RdB8dM>YpHMK^&D$9}PVRA3ePsL0QDBjg8*w1Jzzhdsa6K+Q7z zKe;Fwj&^_?+SJ?HSZJ<-ZKG96o6zKx7MkIec1};OGY7 zW&YZjY$`wAF~`jJuiUBMgKC)cKGB6)fe)sFx%nNRR@Ti?=PdA;v9!>S@lp$ zTz_rK`JChMtrN<({biI}oa^vy1(a|4Yg6v$T!(MRp?uw6MtQ|~4&NLUAO@=bD$IBI zxr1RTEQgQKus-~-36Yd;nUID7)NDjgXF zJ}~^c_G5{F!>7+sm;d;A?Z*lMhff+%13vt`_G68J!>3rNWiNhS`>{d5;Zq%y|KR7f zA3GGT{n%sR@NF^FdDnhAeCl`bgofJrpDYSTJ~1;eyzZRr9incq?Pur@7U>8#fe#yh zGMa67dUX*L*{c6sJ~6Nz`NYh|@xb9zC0zIXpOY7b^wdZEG;m{3yx`90=J3e|>Quj< z4xjW2ETPu8|1@ypfT~D^I^OoDft!Hh4RO)_naTg5{Qu-n5f&B@VWtcQ1yCvf z|9i>rhOG=A0)Fs)`1s%9lZ1osw*!fw006ZqKz(kB{futD-;N|qPKs30ll#H&f#HYW zCjkcEZzqx%6q%LJePRGLaFrMn8WRK)4E8f8By9uLaYfs{KA+4Kouk9z#!#PF&uE!g zesU9|Ceux3P$k>XbdgEp=S86hOd6sWm^Bzals2$ve7rb$L9__}l;3LQ3T|L!{$>xE zCNGQ@u5bSB{pmxgTKR!bOsX0cjxr1yngv1^ue@Yr^#1gt2-K8h$Ph^2$ngI3r4ZzB zjs)*dZ=kFxKNJ{1y%7OWZ=?{^`~a!o%3!qON@)3^z`$~mQGxZtwBI0gAYDw#5c%ZK zOiWw}jqL1K8R|d$WnfSQDOPaf`ylW`p@L13|AXakjS5B!fe-)rH45s#w62Fa`tSFi zUCgQs>I!a}ZdTC7FQ^Tr_#4!U0~Z&h_GM6Gs9>DHG-b*Uh6yg; z4lqdmW=Ng-!(aobJp=0I*6;qvz%O=T>JI}44L3!op&*5s6%0y}zZpIl#PEMu{!0VY zM^kG3VNk&If$4{a8`lS)pRGT3R<)N-GS<3l8JgxsLV5VWZ?W=Zg#q zM}9=`9{CZ&+w!ASUfy@LB%M!kOaW-;m%&~PxT*4kz=0qg7>F4;MC2KA@qQ_ zQ~-)ug%9kYw5rIF0C6r8fBhHX^^%}QqJQ%SrU{_v0YwOCER1mi=S8NO6MrP~I(%T9 z92v)3{{d_xV}+LtgCSxjy9n znEqoW1EU7VMGH5s5Bxs`Hn#s*xq#yVNK|7Z`^OcG25w9rZvJ6>(Ds9IfrZ=gmq%7{ zebD$};l`xc{-b{9iuNCM3phV4`^h+?{YS^5fSv6>4y>3CPfzVXc5t-+ShWC@rr5r+ zr?GsK5CJ)?{>!ZO3~Ed>4BeQNI6qYWWi(^f`+kvWhW`f!&I3Op<{tPFGq>SK!<+>_ z4CXfcSTTojM!`nRB;m?`5tbq@?qQn2FlSPN2s6iKrpe+7+G)|h72Ps+HvZt4!1hsq zK~VA7ONN>JA9DXNnhTg{m~rS|yUaA>|M%X%-k)HR2uo>97r>_WGtB_g{(rwu+5zg@ zG;d&>08`CS5wOww(~pA5D-wLfK;3ud3Q$M>z$XR;;opqlVa&-~iE2z7;ggLLJs5i? zS0~_l3~JMUI530qp7{lr1s`S3zT`~}_@U0A_<@JBB(3&ePrNg zyTCn*Yo>sUw4h6o8~=w7KY3@c&*7Up;fDh=qj~EO2WDpdHIqeC9VL(cbZ|3vQ+MHb z&BAc^hs6UcH%73R7EaDj6=z!eb8<(jNqxaq&_D%jjEYf%?;@uvsH6mybD(Hv1P$;g zG!}ryZa6+%-|7A7L)n2(Oln3I0Wu6)1_fLfK-tdw)0<*Yu@4%16Uvaz5C#vAF2T4GM0(2@D@v|0%fr`OfecVlcS!g&NF}ApU`!A>cymD-KpB4b_Y4_5L52PzHpU z6!|_h|84mp)4-@yuTa78L7wrzXD0!sUyL8Uzwt1EqBrP=f`RfYMn`Kb>G~lP016rJPj5{ z89(qdcKuKkV9aUAIrXwj05p7Z?jKlf2IB_-MvzL7*v!ATUNSNw!V4sp`CB1_RWbE5 z6N5qm$Y#C|Oa277{rS%LM**pI)&Csq#(E4#_8~j69Ap3}8bPBru;>LvDeEuBkCM)y zNcH~25eRB-c~meOcobMPZKe?Wc;1pD0r>UWC-keJM0r%x;rTpwSsGpP7~VB$RT ziHXCYYQx5BFBybp{ZM28jmI#4sBzBxJU{Wo4;F(HKR6Ul{9s_Pa8pnM$^HKR^RGhv zPDT?Aa{)6Borjk}t-7;6coz77;NSvRZJ!v_HoHyzVaW}eEE8&Ep27Of!f?V5h6Yfj zDDj(NJNw5K42&O`|1+ytxG938diswg44}qs`;R38>=&=UWMpK>IR0Y>14qX3m+B0l zdKzTbk_JYNu#3}w7&rvFx%q!!;M(f(|;H;FdDS~Fk}D~)}TVcq2d60@C6u;(XF9_(V(Nj>wyJm z2>yfrAEpV{e+Y0+|6!1@;bVh8*Ck#R3vdMlG7aKZUM0>4oCr~s{SaYJB?-6}gnl#4 z0687x5s;&AOxDejthaDuS;_dp=%7C=K%?CtUm=&d zO#2tqGa4)?PpD@!NGNCh%AUIGBO@30HwkIZkBpq4hAzlk9O@uXG0$L8nec;Q1IQZb z-wdf&e{gV4{bA6+G~wzG9!|yyS6_0h1hxM`cI^1b%by~5k!c3w%!XH742`ci89AIG zT?GFR+#DcfkR$;zEVF_e+%`0b;ro#COT!H`7R&g7`6r`2s7rA8BJ&IeRfcaC5)7Zd zgfo2Asbu(A!o~2>hmGN*3isp{xpMUrcQR?PUtrX*bYs21puw}>6V$wX;0T%xVm`pm ze1Um}rW^N%$bXEc%(^#0wM^CBA3PJ7K~=fdtse{wpols3gMr~DN_peq#_&P>=gc1r z0-#AM4mFo=2RIIVVo*J?N#Hj}z=R(L3PzhR{1C8|uHQW2#}b8b83wI}A4>|DE?ju2 z=FS1CTx)DH4*ZzmaOVd@!GV|R<>?OA4s!c1{8&&>?Qr170t2vm1(u79iVPpl{0CXY zXa}0=I(PBJCRRnh53;{J-2Qy8`U!F(qU$&lG*QJ4sw`nsP=Sy+D3%YNzd(`>9~c=x zvsoa`oj(~=6x?_|X#55hPd_gr6=>kDp+`aemj`P)Dh{yQf4Q>;T=4sIez-h&W}bBY z`oEy?`hF3V0zmD4{}0Su3{yZUL8Sr|ZEWfykdEL44L3`00Rd|KGki#EnEJz@03?QJ z{%2M&D~bGO_$~H>n}P2`^v{(a85vrC7%bq{@V;^X1;Yn6E-w)9^%C|n(Qsqkk@3Ul{~B&giX0!xep*6BUO`22e@^{j$WX8w)bRv$31MANkR_n5C*KG4pI|e< zolb@iP5&ewWK=LJd4H2(KJbZ28C2$js&la)+zOzoo%wYW6R7-$RO$5!0w9-6{L3&0 z+-LYEAy)sTc|CMg-v8hC9#9L&zkNf88-pgp1cndNO^BQYn%op_0%cu?PYkRxey#Z6 z^6dbp!?%nuMwJb}8oqt~o7C|JXO%;8V@%*0@j!0_S0WX}R)e!dTjegwENt?d56;K1X=%2HAU*GX z%=$5d!GRq#ZZYe}D)4fM-pL0GB$=ujChHZ6vG9FJ|1tSnfn3(nAG1I!9X>HIf(id?EzN`KSeR%7on$WEu0-S*kjUN~s867r${BO^GhwH0gqQVCT2aW`gEgC;zW;-;_ z{=tyI1S;A&1KdEZcTf>9>j$F%$aE$pz7NVjCf5`y=r8{*^AMyITnk`P%BaNmf#FAh z8^lX(lTQ~)Yrg#t;(>}akYO-q{{Fu6Kd7E;`uFTXfEy?Sot(^HWX`nr|Kx@uDVFyC zvJWPoE0kbb2A&k+}QXVY}l=0CB{pW{}!vVy{P~-U?*#o zq;NED0OdA@36rOkXw@^!W>Af&m#lY^VbP2#m)p-|z%bLJoMG0b9}&Ele#FQ!8aRGr zU_A6Ag7?sm7+HoHnr@6r93Ogr!Ly(_xB%i}1vdu{d}2^%m;tJ-C4O_2f=W|Rh=ayt zc7NpMZ~d`y1H-4vhWjtLKU4{om#eB;)LW>sms_Y>aH(jxMY<`0bF0K}t`C_NTuS;6 zKw@4jL1!v>IP^}HV#|F!-wa9~x@eE=F+Lf6U#w(|X7P@%vIN=aN(f2?HS(Ga_c zo?}5d6_jfwesg`=6@!{*Az3#37o#2fHwp3jFL~=3X8ije{)@p(MZt~xgY|!0r6AKq zP#MT{5j^A84<`LNKRlh>T~^G*5k9Hhg0XM1WqD!!`X8VOId@UPjp;+^Kk(XscHNgB z+(E;LvThx23>p>}5TyXV?&Ak{_74+x?D)vQFR`CNi}`|so1h!R2haa*-yF=nKRHxF zq8k*^=fC~0^xX)a44tf5!OLO%AL6~qb`=Uj@-Pim-U?tXoRbSG)asd`YFPGz*ABG5 z1U2v-L9XO=GuTX|3mw@%$n5}?{v4nM8lbvQO`$^5P2e|20>g)uzaZ7~*NdPI#fGgc zZY+py*<{H|VWu}bChJw|)$jRRun~73n(f2I{i^lMYM`!<5jY7dR4^;@ekl9{8fFD` zZ2y1n-ZkOJ2h$Tu44OwiF*7M#aQ?u;*zn_nsG#D3PyCDrvL+^Yf0JMYwbQvPgg-Fy zGwyf(z{t#9z@R9Y;4t}irL@Hd(Fs2qWEemEWo-D-AagLw|4V_wgalRvt`AFgd4H2& zU{v(}Cc*=1E`mDG3>Av67?`}j2{2C9tGXZr84Q^6!?1z1LcxvwgVLYL0@W%kydTc~ zm~2t)pji2n?LmPYsJHIOso=)=;pR`?-4C)D6dMz~zkMv6@}pt$%<4dqjNb}wT>4BG zIX-;-&-8#vk^O_(@5%qFBbbi-njBH1RPXq0q7i7dPXp6|m&}Tw4w`_&@9%cI72G&J zsQqHtD6m%{k3r-0#gCwvbZ*=L8Kw|*134NrB3N)?%8w-sPBIJ{EkBkhFket|3;HpK zpA$6jz{HSoYdQY|&Rhz0I9JK*?`%U0n?0rpdSK z{F%&uPS&YcVVplXuwGPr-`^=eY8a;c*eTKI#Bvc53>+T{C)d`S%kqAZge==iVE=IL zr-EC+4-ff9Cz<`AAi6#IV!b5e^~ta6EkUc=)EY84ct5OxST(J|g@yM++mFfj8e}+l zKa@d4nHqIGA{1cbIcy)k{{}@aAKYA)3n1GI95_Bm?k(6m<%dB4CrT;>Eg0ebU;{C% ztWi$_VHhacvVGXKd&AZXKLk7{?`f=H5|Es1)?~sY{(o{#Q;A@sK*mPK5C0fiel#*D zxc&Zqdh*96QJzMID=*a;Jsht55RjZK)U3rLz!~vFO<;%fkskt_lRcX47lf}!sI*64k7zMi+mhe+%Ei>!(h4b%8wZem-v!C-%dF4V-ADp zgDXE~7%a+A_~7Hf{K3ar!WksWpm69%1H+*oEest-hh8!;EXtSwrjs(fzkLJ`NKKAv z(PM0w+}Wb~i}XI~WZc8#XJo8Zj~nXig4m z^AUlqaN)S9;P(If#6RHFvaL;xF=6t>dQpiD4!^#;?gb6g3Bt>pE1>YIG|JRg+)PVQ@$ka+tiu|lDOQJL|BJk!C?PJ+z8m_C8)`hD#- zOt1e;{?jhOq_cCfSce+ZsXvqLI^-BNCSR-)7?gS>kf0Orh8!4FlV|>7%$XIawM*_$7A1M;2 z!5GBO;5hvwgJUvBj}p`QUm!oJ&H4rNdg4yqmmi=7h68AVL4iw$F~j(x_cs@&$yq%L zJ}sFCJ~3%}xUqhy{^|YAg~RDn31h1aXy`}t2ZI3TYgRS~x8L7Y|FOFnxH-C6x$%5B z^|wNjOOf*jg8-ue>xUITIUfAeWV-=2@LG=xX#JM>WVv1ig{40f-C7cw-4xQ;KQ#XI z`F7agz^4X=(=VC1BHVs|xBoRcu2+lk>EzDdY|w!8w_X#*p3T~Q!Hi5biIW@p3z*!0 zPyW=e1Ts%(vcv=#zWzTJkhO7*-row6COb`tmi-nW#Ht1w#5sg*LFujuYK)DOA5Kta zY?;hDF`dc!=j6PJauN=|8C5mh3id9@z4GIN1cMT2g6sEpg`blzR7+Sfd?^0m{mny& zNg-Of<;N|KB|mO}MnjdZzGPq&PVoNbD#W4+vQW5!!y@q`10(Z?f6T%K-roX*CV!jg z!#HuW^CVlzJwH~kGhbD3V^U=MuJCZ5-z+KJFAtjRAXcgTE6kaau!nc4mDXLis4vjRf}qs0s- zrJKIr{v}NAnG%Y3-PxZh^%x7Yx~3{IW=>u^wUOoLce5Xpjix0_qb*JI0Cl1jKx?C1 z1txEprokEUn@L09vfu^g$%)fU@TxYLZe+Fb52G8ihQ|#Ch5IiUKe#io$uTN8Jz!99 zcyI+ApQww4y^$@OGu;xe@gJt^Fd6=zEH`5ZWBlanGwK-^PY$11&-6ib@{O4$9Q)E4 zCUSnT|2tV^mIh<jreb7B}Vn23Z@V8%!fWZ39t%Smv5nE%7T z?f-YzA3V^iy8*JwZqn>Cj1MNq&yiyWrJaU3q6F3!Zkp4G5kXFKON7Gzu(`2+{J|io zc=RQM3-^bdzmreQRbl#@T1%{)KQ%EX>|+NR$Egm5bawKC8b`GHf_ zn|Yz&rGhT=J24!0XTAcHU-KXr&qKDWq;yZtwVf{~lH8}o;q zlcg6oF@K8?-n?LO93zYOCkMgFOiT4RygxZecz^N`nQXE28B;{^WQApc?B61Uyg@1} zmPIl$e~S>Y_$R7ugUGJ4OxDEfA(wg zp4A~>@y#M@+!z%Eb}6_qC^0Di{4V&5N%!oPA2=8p6ps8@ARw^U;R6$c!zTtIP*cq569dDMPYg^4J~22* z7=VH*I728y(!uu|L$QDZXwc*EO9saO->3cL*vOzDuoq;a!UrZsMui*CVVe^iD-1U> zfOcRoa%Ko_WK?7$WY8^bdtU>S!ul!b4E7HaFlGpX{Nd;j;r%TIyh@$} z22VYxwbfqrhbL4TTap zi41z6-V8^A9C-BC;S&R=_qUAf$rConFkYLyc7r%mM)u@m8&ud*61=}9Bu@UYL4eN;&8iGXvWN1_kC1os;7?iW|B8VNf{o zQj7uAlwJXDO0#@i!NBps3M{V10MdKq#{vbV56ny)&PRSMP?)@IqZ;GO$!9m}D&PO% z{jCDL356wr;Y03k=T8OG!dF)%PDFeor7GJiNYS$2~l>y$qXDpr$&Hz_hjXiYBL zq{Jz)p8Yr5gx}w9PhPl5ScUUBXl$MdT(UP7Fn!Zkr5fQ&r72B^mx-zTf5~f<{62xvc^9R24fiJTo~G zz{+=R>rhDm+u=~*z|MOGR{Su!1^(dB?{Z?gz+u4gA^hj$wC!pvAj!!Swky>y_`!6* zu|jhrXy%^IS3 z6ciYgmE?Xf2qAFFVp0H&2eLVmz~K`MkMFl{S^nP+lsSA7;P(CYBxQ2XE*ZU&1Zc4iT8yA@=f?_$J3n{|n0Y|; zJ197rCNoZFE&y!|@cs57dGfVgHUejoSTsPTpT&M=@K&_RO1ll1ZX`_(*1JlL~tbh{(j;ggFeFt)ke2Z2N)R?7?eO8trUzL zf3R@xa{6^y;vu67WLW{j2ftsSJ|M^`3>DUonBd}0znnj5FnE8FxjD7yeLyF1k=C z;Mo8F`}ZFV3JhwTADBQ(q^!I@MQHnfVCOvWi9zeYC#H05hfkmlgc*ULRXA+F85tPs z88sa~G&D^3p&$U7O9^pf{P3Bv;fI2N`-3SzZg7AmPO}=O{J6;h3a-=!5mo;W+?=36 zG0?y+Xz&5VhK$jI_Ns6zaemHace44>9ndv|q3+D%gACr3ys3?f<1BXNZ&+qAf zAboA0PbD1ApETG&C2CFPJ1gTDzdqNabFKi;1@#6h+H^_M*0s##_~vjWqH7^#1mw0G2&rn?4yTPHgo4)S)!l@1T)e=U-Tnz?i|L$$arshR_El{@)k9 zzbyo>JQb?A#BO}SC`af6KfnG(?{6oHy}unP2UR=X-}aPxf7@B){cTa<wM%c>ZTlU{hrJ&^+1kkStR}!sLWQD%>rg*iqGRWBlMex&M%~+S^}j zZlH+>BR9?ulYV`W1eb9Rj2{ZYqf#v!*xbCoH6%>Fa7crvB^~5z&@4FTzsbjr1W7yn z+99c+*jVK9NkQ<)H)q2GKiCAg4HVrNKSWI~JS@(~2QtIGWdl3JjA@58xLd%*se>Ei zhh>vb9*z{d^V8v5PT(#Vb|tQlY@kw*k@Lf)pOY<)SSf7z$)&*YgPoy520Rw-&iG-% zUeI1TWH)gm+@vx2=#gND#Geo+xqebmJo?F15aFcCzm^gXE?i0mE`~0rig*mbhB`8S z&;=Rl-wN|+#8E9ILyt`EJsK(X>xaX)n833?7&h#7VOQdYJM`U;Nyl_pI6qwbG1=f) zh$iQURX?~CKvSsTT~GduAKLeTMwXhPX7hlmQU!(zhspm>$ZH`&LjDiOYXLYNHw zOWI_C^Kwj~e|eAhiVWW#m~vFuIR}m{Nz$d2(6IhFo5Jph7UI2A!v{@Coj9I#MF{HIpV4h$Zb&G zsjC4Xp3F6{OFgd1v2c9Y@ndq_1q)Fjzn1$I0JsCJG(+2TF?w?K}Q~ zv`f$VhnxT_AAvj!iWi59hROc-WSLr0Cf~Xx!&LZh@{e2KBq4WO0~FF8x3%~>{;|7x zxEZ=xxv_uH{WrPcHrOX?ZiA!Z)@?b)50ii1_Gdcrce2-=1g7bKC$G5^!IbrPGTYq* zHOs#%Ub8YVeB|T8j2VX4Tx>t$sPVTvDro{ah6gOP2Ss7rl^Xvh`{$MACVAl`@ za6+2waZd&0Qm%V4OgH~b*0?7J3J|DZ-aQ$n<$osk+*1MxZn&ogQhDQ^3{%mc$=~j6 zVrnUwyz0K5QcHpNw-!*}2s~uK`hnvgs9UojBQcXBL13@3}T=b>XA*Aw z8pojS5oE-uC_~V}C_}&jG*a~Yd+GmzouE-7fe%dlFw6h{1&JtpU_sUmpq@0`&(AOEH3`Y_+0(FcxsV zW)PU}?Dd1Oz%gT{vtz=omn!0x&dwJYs{B9vw_s2NEhFc8&B$(3)#Ajszoshw)sd2@ zAB+yF&Ws-g7+Ln$XV^JAXuiGJ`ig^5(4p-W2a}XT;43C(r3<%SsyVm3;$UZd#m>g? zy1|E?>ovQiuUqJki2ZJ_xY_g)nqP6S2|F;oVq?$CV0p#J6!eOj(ft)0dwzn%{{P>r z|3|-KV_>=fZp`$9W&k*xe_Xiv!-b*2{pw`y$I5C4LDhsis3phi#^xphDv{N%zZ7E( zc*V#iv!DHA0R!s?q2H539+#*u`|&FWwC;0)d*mw?2KEb}UOdBx**~muJ=_%CCb%Y$;t z$NSqI(C`zuN2;*d``e-dhKWv}SU?R@P}7f{vBLY?g@Vb~o@xku`Og0XvZi4@Pkc9WrvW=lXJb5@aGJd%BYu85xe(sB`KNvGX z-Nm2Zeg3mN*!7X2o}XnugF-^V?vISzEMM7EK#OB;Uj(i2-1(7%f9FN-ZyAu*e1*YA z$ih;M0*?KEzBl~>^*$LD*go|9hV5%3wJXh_z^KUe|NE-H4xboUIMgQpdZ|>u^9Q3H zC=y;@6aW`ykA8xJ9~Afydo6x}`ZP?6McdrIIan|#xqLeSU9`-k09u;Bm;g^BZVnZo zJ#!ufzTX%?=}+OpO9p|nFBt_G6hLdn9KK})gVv03{$>`It!LELy2$+TL_Ih2#|^9; zYR+zxe;5?7>&sl6{9|W@_y#9Gou~zHwoce;K01#0t(K<7a3=AIWWy+c*V#J8gK>m>i&OU_~*~} zJ^vs*W~mSUU%wZF`ZWHH8~i>s2zY;r&|y*Z{=^YAdDg3tAeV0kxIx}L@QFc>={Iu& zhl*Q0({JVv5ci;PBwE*x4@>)r1_J090M%ynJXZ>JeU{C?o3|&8%7(hcplN(-3u`qqO z`G4}f*D~4+Y7XBrqWwQGax$obX1Z8^b2l*5Gin%I1T7x7aZ_;n@qNdC&@h9*)xn~~GQ&Cv~%Bqe{aGdQq2e0W`~ey=V*9}M3*<}F0>){e$Y6A1s9-eUC;-W_Dzbg} z{?o$^wAPD34HT+kDhxm5;z6O!G-2ish6ah>-0)?64&SaY9QZW3>uq2C&xWbc!J;n> zQ-9o)01Ya=ZvgFIk@(I1(cu#V6DYxFR8+7CAC;ot} z_P^G^3R>I$&BE~K_nJQ%Zmb_Fe@y++q4a6<)gLpIydGTraY5tij}snOf1KID`jsaE zwC3Of1Ir};`Y$(%j=tnzJo=Kwdqv_3g`+<>B#-{!kv#fRNe~nP8ZoUuRxoS;2d1V3 z>&FcYY#%KCu*^96vd3HG^#KOfZyX}t9~l@~KPqq^ec8w8@NEYJSYOoAA3Hc)9<0g@ z+9+J%@`;1x$R}nNDJGWJ%*u-*qB9sDT>W98BqUYOpv?MFfamB-4&|N~>kmT)kBS5Aa{D{L8w5bp&$nMPD1#EGm`bEWfCGoKRXuo*tN#aXZjdoi z;G_uRfripRi`XT8Gkmd#W&2S5qxFZu259*cWDyH0ftXbUeltmcj!^(vj~LKo)L_~_ zGS7ie z4B{Nnkx7mM@r&Flpk>;i#0av6ajs4Uhz(h!2}*j*ifkXse|WgH{$N<}{rjYUjAo$m zKhAZaqzX@_;HBXl1v2}=y`6qA>Hq)x2EpA8Tfcl?@?-Cd1_#iDV1wf1Umvujb@zdm z|1o^<`}biZXu*d<8p8*z|Dg3h>67h0N-{+#Opf{}HM#7gif8+t1-T3w9v46>=@>u! z`~zAqC;gX6&E=B<2WaCT$kE=PIKqy6V&*vTiAjy^HLJMz4?%tw4Ur2>;5Gh67eUdv zX!7Ha%8VN)vwzYS%-O^8(Sw081F{s1@q_VX=TDl95tDO2$#cK^2U-Rr;K2A{`Q+)J zmU4kgpT-T|-yT#>4*I-~ao1$VFTRYcC;NYirp!Qvd^4hJgvBg9|Li74YrZhOImx z4;N0h`YFYyJDHT2KK%c|pwRL|g+TxmFpl6r+4a+fF?RC*pW;jpDkg(C zizmna@?@-^y!@BG*^ggr514ehFEHvdU6{L(K~d;|fTF+y$h7#E?{9uNC~`i4OpAk7 z1TrWwD1Z6>_0;jaeMlfubT|IFk&6v0Kx>joCk ziYL(EZ39DM!jYHEj0$dFzMD?Y|ECRFoH6I$1djMWk`H{p{Y#o`^1lnT1nR*52~7VI zrv@<^NrKJ}VNzgt%_87h!>HS_f5Tn@117~U-?OLAWsG5eIYIf#4-cVfjEpNpjDB4L z1=ff7pP*@=FAbnWkEZWoWZWmj^(*rMw^sT^mTxB*LCX_%{F~mx#3-TB(|P1&Upgot z&%R_}V448Rs{KDf`aneh!)sPC1vlmowg0A{VPce$%KB^afI-3V!Id8;B(D6ppmF8L z6$?-s;P>BY%#0FxRzIa4fVi9fi9ML|=O7lz{X6!OjZp)%j9=;V_cPO{Gc#%`Z2u2RLcYZ=bIWP_hQIQ;=L zqltLo&oj?iR)U(yoBtet4l;sK>GSud>Dnxe(vq`(ZU#&K_Op3x3M!iF?~py{)?Saim5?ix)=we zhX71#O{6`44~r(fV; zY+(A3Fg=Kq@dAji!o}zTqVu>ImkBJ{U9hz=mqF9&0#5~~jGnH-&1k~J`+Is6H{%+{ z`_tKZ7`>%7>@L^~$|fH`{XPYTiWeJQJ}qGM{#KJbJ%@+Uiisn1`U)OK1;$g;Px3HI zGp?Whl!sA~Y3{D+9K4JwAglCv8IzbeLZ(mRWej8D2-^ONmyv~;v2yx*0Y>K-haZ^= z%+r{rW`j0(DS^(8`}Y0DFV+no_Og5|VP|~6^3jK#yVCgs6C0b}=L^i*oEKTX>J&3* zNbG0%^d)%m4+rDh?9VR<_b_Pqe_&v9{?NcI+~WNyBgy;Ij^OD%f{ZGR`=@UdWVC0D zoc>XeQJ#h6L*kF=B0`KtjC#}Eg&4J2SUwp1m|iBt=)@HEW4fO(qvZ6EzGFRxN`b)VMckzrs?b=jI#XuejD^K zvaw$U9~Jpw^K@eoMr|J9i_?B6F-Yv3^2337dWHz2ne67D20a2F1o$~GD0~oLFlYhE z9AIbJ@8S0GJJa-?B8-VbxBh{S-{|_m65#ORVFReOKV3(ZQG$shV7i+q<06qipiwJD zt_sk(aY6DtpyH~0`X5n7HKr%U(^bS6a~SQWPZVPmXEd6=Qj9TP2a$3l_W%99=SKwO z7&8B!2Mjv5F0j4kVrWUf_T0nm%lC$#(@n$~jrbb=g3=MFIQjH_&7bKd;*1JR|MyOx zCeC<;aoY532}VsF(9ttY3LFo-KS_XlY0MA2KkWcdF|xQZJYY~!$XMXS@FBSov@&4& z0SQJquPY5xeykE;G-R}7_;9v?A!C8l&xF{BD!;X(k{OlKY zd}QS3yyy)+;0Du{=^B!Z6$&@Nbwtn)4TV`7y}#8YF@EI&t>;h@s$kIa{`9A4`T14#2Zz5b55O#Ch7WzeAg=6cnDS#K!ztJOUkui3m~wq&mW#0|SS%!v_Thix#cQU^y0T zhEJT+`DGX-wL!fwg9jEFJDeCkd~2x8;7G8@a7;M(PVNg`~z`meEvj@;?i=kKgzFgekf+-B*@T zUv2LH0Jk6C*?)taVRga#6L@F{)HPRzIH>;btREj(r!SCYv|%d$HT|9}qXy%J>Fjch zflQXarbo#!+S-1T0F7!YFjfeDVC2ubFzLq!)gbn?E1*+Z7z-jFfOc(wDl3S5)Wu0Z zToi=Zm9B(>WO$}ukz+Ju{wBdNokN~cp81;u&vY$$Mn~pv0u0mh!%-=XLJ?? zMfU-A$%~)`P@v%>iT#Wp7EG5_VANOt_`?FU?O6#NJ^t2@0^C?0IDK*eH@E@}G?^SC z9Ay|UUY%a7z?jQ8Z~8+8Mn|R&&FLzNj2ZQ35*QR2^g+%G_@U0gapCR{h6hagycZbt zMK5xEWMSre01|QjWCLnaF$v2~aAO6T&CKz&A>zmUF(@YCDMyLn=ec!!b z_vHuw&JFx-j-PD6?K~!Z!wZc1(HB4lwKVvCJCo-8$p*Ym3FHXF4CdEN$_#4klQ||z zx;g(~$!3|ZV#qrA?hl0rY!g}MGkn!4WShC%ubCA6zchp~fVT3=fkvKZXRuA~4dj^Q^vQwo><^X)Wo{g^ zKut`>*Gx)I9~79`C*OU^HvO#kNjEAWkweVwux@v91J-oL017Qf_ec; zZtN4zzGOLf_9c^Z$b-|L*&WztI((B5;r_tSCGeXCJlve%U$fEYQw76x1r4c`tYZ6e_g8)H5>aJHBFYV779e30N)=yu=-L&HVTbo9sn_S5cyj@xx}eSVQ~2BRA& zXMhGd8bMXj7ZpY`!AJiLz=u0ADWCnp(D3j3_375CjFyUf|EU*(+FeZG*7n!$TXr%j zD7rByF)1@lFmU_#ebw}-s*JUauG59p7#*2>|4k25V^m?^@YBHU!*{*ujcSaZj3=g_ zR%6uUEdI%)py~Gi`_<`x)fja-BcR+}(~Z>`4cV>1I(AOaR%cXZTt2;Doza}d=Tim8 z^i%4L2CR4g|NJgA{l7Y+Jfrk>c@4%?M)um>42u80S55EMWc26#{{8=dNKf#?_v-1- zH5p@>xc5(Y(_*yJ1RdGz|ACd8;T5L>%LJ|!90sft*j8{DFi&v)!~)LlHw+jjFnriG zeWMnmWIf{y_6*R%rLD~s><(8d*c?7?Z-gnl@MA%P#0N%xh7S`Oy+7>;Wl-7W#-x7s zC6CM5ms~EOGosdID7Y!Q8M#@5#||{ASu`9zFg1q$;NYL|Q>`ShWvoAS1&c0;p(7LemBO|}i z1yCgmDuY3Z%i%+Ob9)AxiUW8Phceg!Acylk09BlfDr}%#4GQfU3T}?m7iu#~^Gg&k zp$MMVX4E%_cWxLi{NQL{PylUU@Fq-#ltTd6a?HDAF$45P-K|}vfJSm z6T{)@x;l*R+K^>z3>6%^**})Buz&PnWB;ha#h|JRUSh|7fl<={w1Z!hXMe!-`8tf! z^>zP2?bRn2`4t@9*j90^=9*yP_T{_uUy!*17eGD-#U2~jw+Y+1nn2I zJplO+qykj0T`*wGQ2fBc$e7UhXxfhknQ%A34=nt-7Y!=hzBw4Vd^^Al8Z#64&8#r( zM}jVBVZTuQZ)S!GKXzzvXo+5QP+(Bxn!q~az)J=}12?XfU%vbP1sM-An$hjTj|&V6 z9~>AC{b^1bM9 z{Hu-vP)G<|1*HjQ6;O(3%>bnZEjLJ70QGVhl>U7``VZtS2T)s}p5efcD*_g54j&5| zKnaRzdf)|63k{@s!VksxkZDM-bFW!3{d~)+ph)%g9Ju%4O6zS4P`rApIih*E5ir3#+@JQ8TmnO z7QXJE|oEv@prh zO$nOmHh`_&{R`&MD+WyS8x$E6ey}=xjBEf|ZU|0K5*NJ1KJ5qqSq<`=!-tB-mJAkE z2RFy*xAhq%?KPmbuz-iITQW42S z4HUfWV2{H@KqiO(;IIe9Iin&dy)a}*fxSC@j{&1aJ;SGp2GHIfW-HY)Ck7^sf{Pw* z42o86|G#Vh$^xqcMKO~aEV5X^yUbvbwIYK_@$>fyKVey##o+QY77f3P7SQ=ViTw;8 zeuJBt65!$h5_a+zL9P;fz@}j-4YlsyclH0q4<_tpm<+Ps+U?)>dq1W}7&3Zj{{4RM zKkEZf(qvR(RQ~t<&_7U=+kf#}14;|_UtHFJlD_>HtLY~V8P76KnLgc!QJrz(^!-MR zao{GVf-z$ZWBl}5W5#So=IJku8Ra>}e}M*w|9_u5UC@M4o>6bQp$TInqyF^GCXB_5 zz1!7I8EqLE!=|U2F~&0PpMKbkaWdP50`E^Z%C^UtGcqx<-Kg^ZbfIE;iUs2ywhM*c zpKcUSSF>cC!uW3bZcD~mfsTK>Ub8VVUsxe&>Bjh>@!xb0E5>asKA&bVO=q%ZG-2`i zRKqge%9=4p;o{#%(##h$GCoQ&emMR2>PtpOMHfa34u=(>Bj`7L*t>aCWB}vzW7dpW zOkdKc8`&^c3S#U7n!d+|F+hpsH?xAs1JGy;=wNRz(5x$XDFj!-ftL(zprgH~>)A3& zF?~s!?q$n3LvQ9U#P&7F0pi|xu6~#@cFcWar!NL z#u+jomgBb-px&k;hl`6M9yz^4Yz1D_hXj{e|jka{rf2U7#n4Chaq zJ)J&vBgt_z@I09IgRy~ehV!Szo=%_UYi-waVsv8^`ImU%M+Q5ny~$Z|^hXN=Xi#x_ zhcjau3&=^^KRGiBF);`2aGpNVlTm&81W!g0G0^`@Eg8^??Em= z>9%mPtWlNMM3iPE^o$I`ELP25)rufp-pG=VN~L9 z`E~$wuA%sJ10P0Rq)m+ipu_T!Zxoqc;lt=`&>-aI+`+_&2Mi6~U{`BXdqtC%XmceK!r^%QAx=P3Q+mS@yZwGRw@9<+ZV6*%K zT8KRTogbr(*qjvB3I5;ygN7?qkcOxo{2Aq#I{!>h@MjcZ+L1iH%%9PNX->-Y10eps zgz4A(8RaDG{vg&VaclrBOek>o{k9`%I!^$jhs2x|(2yz?vvUF%<(NMIp57C{D8aNN zb^5XZMmLbz??8OW@VK1V(%%TP_dMUQwQGa#w;id|Z2~dXGDZEKzAunbMFe)U!jaF+ z&Xb)qK({L-PyZIk7>|FW;`E{*#t^0rKc=4uV(eh~{9Wt+^pIf2C5&9te+DzU3%&XN z=Re3T8nWEq{xH7z{`LR%pb$niCbrD91_qlQ)9b<*Hwdl&(csnzZhM3JLGQmW|1sSm zoKebe<_{+~Z@2f~r~Y7l@cui?PY93UCyN`O8-tRD_xtaSKbdd%fB0{}=*IqlK}ppc zG`sNb`?de?zsLL(P~m*Qpu=^WONl}G-S^r5x336iJiyGf>+klgXvTR=Od)@^bHy?G zG3xm5Wni$`!Ner6mBo$m!Kd%;e?@*Ua=c5agVG^D?8m<$@9hCJf$*0$1||^x)5gFA!XSCbxFi1HJ`>bYQ0^yL=3``&# zvzmbkghBGyjO3Vp;36aIbYEd6XG@PQ3``&#u!VsMgh6^>IAse16A0&QVPFE`k}V8O zAPkbnZf=AOlZR!(UIr!*&e+Sq1i~QYFkG{jfeD0L_A)SmaL--_CJ+Y6V>dEGmC3`h z;ur%H2sa#KU;<%~au}X+jDZP+=Nw~T0^udc7??m9B#+(53Oyzd%L(Thm_T^Oc?KpB z1}TT(HRlCJ_Gfih&7)Sza?RfiOrOyOBGBm^>^$ zd}d$*;UAwFm_Qh$9EN$mFff6z$QK4C5SIDEzy!h|dF)0ioMU92em0uP*^=Wg0}}`f z{AFMQVUQjeR{6`o1j0Ih8JIxW?7+{+1i~QYFdV|q$OOVM{ESQ>oWjq@ z1i~PB>_+}5WAd;JkYHp2;Rp#vCJ+WGhv6IvMkWw0kziy3;Tj1>CJ+Y6V>gncfyu)% zLy?gQgbNfInLrq%9EMvI8JR%1N0E^Ugr_JnGJ!Bi9=nkeolG8<4O)y$Al#wF$OOV5 z+4sWCG!oVn!wq&M9VO0%4Fmb|Y8ZV)C$zs9|IR z;e;ASCJ+WGhvAYMMkWxhsbORS;g%XkCJ+Y6V>fcgVuT*5UyxtWCCH3av1Ju zWn=>3DXol5AUvm)kqLxB^4N_`02iEZm|QIS7@0cy7?~zOF-QvxFX>}s0^v1%j7%WB zrH_#bghBGyjlJ=e$-{ENbVeo+UNN1K34}q)VR+AUMkWwGGM$kLgwIT8WCCH3Ja!{r z{A2R4+_8X>34{+UU}OSeka8HlvVf5Zgzqe1WCGzQ3mBO|7$lF~$RF&?9+nqYGBSbi zjg^c{APiCt!*5nHGJ)`ym5fXv{AVR269|Lku^Y)D$n0VHViO}12!Ghb$OOV5L!YX?hnLt=) z4_$#l#KhxQfZ4-x!Y4*15T5agkqLxB%3*lTCq^a^-tvi&35552Vq^kgkUVxH zJ0h7qELZ$yWCGz0zZsc87^EDAkNjq20^u{i8JR%%%5O#{5C+L(H*!W2vxnsY7A7VT zKEcAo1i~QYFnouFi3xfa|HnWH24PGWD5PrbR#00`1ep zXJP_jkUVxH8w6Nbr=P85cDCeDU}6Gc0R<)|5C-XiVHE`?CJ@$9U}6Gc69pzF5C+L( zH}^&xvxlXECKD3~YiKetfiOrp4BKcjF@dm)CKD3~`)D#TfiOrOyOA&YnLR8mjF^}} z*ujX234}q)VK~Hyi3x;bjF^}}IK_yG34}rN*p2)#gW1C}z?z8(gd?n(m_Qh$9ENkO znV3Mh#F~i-glnvsm_QgLkKIU)Ma&+S87@prAY91i~G@OiUmQQVzp&e3_U)c!@6)69}*IWnuzhkUVxHHMTH& zSk4GzVglg>VN6UQ3{nomTf&%_KzL6W6B7s@31eacVURp_BQ5qadsuFWXJP{39q~*| zAPiCt!)M}|m_YbSJQEWL--%~p0%4Fmb|Vw^f=bR~%r2HEGMJb^_(BE~69|L!!0?j{ zCMFPmlflFU!e26&m_QgLkKNpe^UNNW4+@!>K=?%=6B7u7l*90!LMA2RCJ<(*W?}+kka{Q&pf{6))LGoCQRCd_N#00_~ z8=06un1K zWH482WN1`tWH_$YIN4E9lKaO-CMFP;-`xPxq04QsiHQk>*Y0kZ>?kM-GC_blVG|P* z2;V#i7UXEy#KZ)`lN|*`Kq^G0E1Y8#nRba$WO~9mMv>{DM#%O80Tvl14xX({Odt$X zDgaU{Fzphf07$6-Sn2cv0T!O=4FW6z)4`dLN0>!NVaHY`CJ=`BlY!yHRwgD8wq|K$ zIIxw83FJCqMh1otTbY51|An_O5n3$$-xX37>AhDf^34~#`X>4a=0%60~4Gap~p-yFFUt^<91F3mhbWa_nk^| delta 35200 zcmZo@5Nv1=oZ!JIGSSnTO+=M}p(TD|mlRt^JR?Jd=)_fU)~SDtGLug-E@o7ioXZr< zs51E=lQyHqWJPAt$xO_0j5?DQnT=%)gdG_e85kJS3rgd2Q%j06lZzSH1sEEh@H#Tc zOfF>BXR^36c@eX|g{84+Vw$;ukx62*Nm5!$vaz9=sbNZTa#CVaim{2Ad8)aIWwNPZ zk^%!m1_uL!12+SM2R8#l#AZ$wd0z2@@(m2F3K)3$$6JhwlM@65^kX#|81^zl6wKCa zU?|mWU}$AQ#RoMT7(}%i7}B*GrYDv&&6TeVY+$IhZeZ|?YhXYJTjLr)rt322R)QIZ z?3rqf3@39NCOZmBhKhWw*-%)U(b_hrv8Tic_7~Y#U zF#I>&yiGKVvp&bLkzs)#$RUmlCBh60AU-J7FfcSQ6j(8W`JlAJz`($CVJj1u&!7Ny zISYdVM16!P3&Vx2OiUW;3=B1fjSLJ9EDQ_^3@i)_ApAQ_4Ge{=8yKE3H84C~(!fx| z+Q4vfO#?#?dqX`#>%s;GD-MW67)Jwx{_+NfD;y1AaRE*UKZX;+m*Q$*=v>smFwLxi z!E!+Z!yYKzVh-V-F^8D*!W^RSfhC0h%M!vb<8ENsIJbdepF;yf`Md^(I>-73hO;Xh z7``|{1caO+24z9%7AP&@0ujFkrA6Ez21mF-4BiFh|AX<}A?9g$K-2|!K-69Fgz#T^ zLil0zUJyoW@D6n`v!?Y-fz%z&dgIP4hf+#3m5Dk%E1?BI7 z^6x-tkr;^jfEb9nE+~IbYy*ShvId3+@nDM>-X%h4t|W-M957wqz;GrBB5?~W(7?c# z4B^WpLlknQL+BV0h*FqpN|TdqV@mjh+Su#%&D@8B-tz{+ZUm;5@g1L1a3F=9&Q!mzV*u zXdaZ`F&iRpZ~&tJ&ml;N@EnFj$+RO440E?OF!UUUh|8RSi07PWV5kS>$2q4U27fsP zF<9v|L}3?{o^cwY!Q>1?-Lf-~?8tTj!k4)K(YH&wfuVO>0|SQ)gf6)Wp)Ww`8Mh$( ztlJR%H4mZrKTHOaudN&&rPqtSR<&pilB7;HW;YHtXI~*n_s41Cve@jVZRA5j!@QF$B z$R{Qijsov*ISvOtF)65JFjO#VFnsvaz@W$Q;c)}k3b$_#rv6_hdQaY_mc*3fFj-1n zO6y?&NUgvRDFy}&h7Yw3-rpVufK5|icky0@=V~+mY6(CGmDAAU^1JQ0Vw<|wDK%W{46X#4?*PX6Kj$pP&63ojWM&%Weg^!{X8=>5r}VDddJBju2P z{13QR2Dq_)nER6pY)lH+n8m-Cq2lW%8*6KVY>(4+m;U5Xbl?+{(xI0Oj36UC6+vER zOz{3>S}^&5wuTUc0ozKD5W{P30f$W14|+c*|I$_yW-x#WHkN>$_aSexv5qmwBN;kQ z;yZsFdC9;Cl3-9`{Gk2Y0px=Zo|6yigfk{jmeiHcL>S1O~30?5Y=KZF_t3>ZJG`U#2&pZ_{8ydMrs=GIfxU+@Fuh8NikiZ7EJ zllz{5;sm1az)J=}P*84q2J%0nQi>bnhp5TXdb*&%>C)5E*AbqWaXI7YOFiLBKPCuV z`Z2}e=t~1(-)~QxFXkWk#H76RhmX*u9}NOWe|QM7empVxv7R!!?>_}M-VgbcdG)=S z7W|kTr7vSQ`5*5Gjz9h%7#xoLn8Wb&Da7S5Zm@`P$UOU!NAT=RErLaPl909me9xng*&;2|xY6EJ)CE1DPxG{KHo7 zPj^CD+`KD;|DwxWl|LiiT)M%b8ZfITae1%Du{Q{$|q#Nr623?;0g>DRr3=b0B zI4e%Fn_isqLqXsqo8c8kb0+`euHuYrJ{;A-``r+v0Y*R_i%s*To85ub=Sue28VEB;qzsZe5W9mh2uo&kBknv#S zK7>wQYwFMVYVu#xWsv$72O1+g1;dVewj7lI5G4Ld<$GJEo0i&CZrhRG$C3fymgLCOf05636Z zx0GbM^=tAj%Q%5|PN1?*sA95{(k0(-Pn;%eSSc~Co$PI;A^XXp&>;g9#|K|BD1fpr zN=Z1uDuUbXFY^PY6^d^EzPnBSY~{`w-wXh0)p5)184r)wHKPS&~5MyOj;T_#%q)Po$Q&)|4p9YB<9Zgf#rurg~PXuK!M+& zIvi2t30!3U;P{*W!P%D#i~(*8AKv@`nQ-6}lRC!(4L9x&@Bgtp`1k$&zW_Jx4<{$T zb&^+D_gBL$(9Ow>|HH1o3>x)crmcSls^Xm+H#C5qHCe}bm8$mNV=oyP9o%?69Qvu? z#t6#E&O9I3_kha)Fz^3&q0PUYl^IzWKluHctmGOL^l~S|eg*|52Nne;2bH%B4+P#a zJYe|1#AIT?;NY==i5WVs0P3hS{kQ{Z-SO-Pwe8-2=0s}WF?`_p3u@n8*t+?c>n}#$ z2^+;c{6DZVGA4+6FiyVeZZFF0;s1e=kud?3at!wWeQsdD;9#;@)5Da7F=KPQk2xdL z8;8j=d_$NT7$?8;HDHXJEbk}HbR~4Mjh_#wZP4YXq80fE)HDJ`GE2hpj~pyZ-7f6C zu$5^&td+y4!1uu&tY-2JKP9$fe?iJ8`TMZV1+yjx`TMaILzv6_?UXMBIDRUz0U3AT zlOqpPhT|s_g9D!&84_HDGiGLTUvm6pGMPO<6=b_!fC7uahl&3u2L*W5zy4L9A^w4h zUuHl5r!xftA0GeO{_(#(^Bw+ACkg~U-1$|(V0IP4zV^$&jsL^--ypK-ojQDX?{UFb8 zaKYhQx`FBkc{Y%FhHe5MY=5zSlxJAyr0_wWh4Yfbw^)k1_2=R1U^jrr|>|>ZI6@e zC5Lb677pJgn}JL_@X2v3IP}ZuU>ZY==*NP<>N=7G^tqy6OOt>-}kzQ=H@QEvp`CN$bx| zIiGVJzI8(RwLgt=i*p^mt$^}Ne{RbCoa^xIIFz6J(r@7pN07jpPZmp z$Nk)x@9^mt)LG#_Kju4pdIIGI{Ji#KhJeGjd~nRy3w-eSdF{sn0f$c2kZU*YGqA3GGT{n%sR z@NF^Fd15~uKJ`0zLPL%JCyT<7Ps|JquRABlhp0<@`w{wsMLNPw;KRosjAq-NUR?x5 zb?V>A^FqX!?*Ev)Eu_b9#t#EG2E_~RjBXB}Y@jad`{D3OufP&&YWoiZHx8(ZRH$2P ze;Bw4DBf^qa&!2`Xuu6sS@dJ_giw+C^dAOp3J?w76u_Y&@FDhxf!hM8SO-)r_=kbp z5~vspRLt{-f!hJ7m?c!q{)d6v5s28Q$xsKF{xEPm0~Om2b++~o1Gfu`7x+Ma`4$D$ ztT@>)Ot@bBhk@H2sPfxT<-9)(+#W#1W=rgn-SRG{2yNYXEfXBTK~m%J*dgM<0CVF{TKfA4&O4sCJ20psQ=^e zi9y8S+f#)jpBPvUd~#$+&QQwW|8VAia*k4t!#5>xQON({$p0%pSPZZHU{G}UmI97q z{tvtUU-{u;c;$zKqQf^fsMv=84&MTxhHL$C{-nU-@ac2`%xuLB{txs2FU)ZGbQ&C5 z{2%IP{C}9?@JSf#Oa2cN|0iZTeA)u#b^U*s>F_B794h=Dn*S$fDP=i)YlK=={r}33 z2VxGNM4 zYkt4!D8r#CbV0%G-*>~w+!69jioYl8M4S-105+B-gLQ#w#>!0o58MAuW{b37x)3ng zAyP!?#}B>_hW{NtNjUg^+u#TabWm#rG(sS;pV7_t+ZKn(WsypH-ai;VF#PcQB*5VN zZHE(sBD3LJq%CRGMC7B>YqP z%3!qOs!w2=#qi-DBa((SD>e>n3~qk_>w;KP4@je`0wjq90aK)mty`|F)dGniEw)FB?Rg7(=#9@+F0)R$2& zcl+jG2I_6sL)rigY77;O6PRZBf0^jSrNU4l@S7=7;5Snj<7*CXw%1&&vws*&&~Q@( z8?^Hy1HZ(D**^>%{68>on!7QmgJitGa$-NY7-s)q6kt%||6ue}1JsdJ>i%I+!1&?F ze+@UT4=aCk|EMcy{jsy6`^SL_j<16C8B>4k;NZ}Z+0UWS`eO&jj;|B_yMJ^Pb^lmZ zu>B&)L40Kl>I@%$urkdMRBR8`aC3dhQqKP&|Brx~hUvDidi9?!GRkn>ZnsMQW4%37mhSH25{xLTEFqHQG1Q~8( ztYEYh{P3Tjv7r9T<#mw2<@gZ17nBx2DM7QsONK#H3zQOOdVhKm3>q=ua`ViXl);%H znIH&C3E>a6^LA9Av!6t%u#S-`0(Wi%U7-fwv2j~ z84O=}3Ju&CCo+7P@C#H3uzj;IV^9Gl4p1?`JPnj4^xRlxfKx@Fz;C8bP^w^k&B@NG zI{Syg0t@J<->#1g{NkW=5#axUfr~)}Bpw7!6`b%?q4~oCmMR<=KkTmmcl$>IgQ*+G zM$Qk%|Ihxhl7V?1qXx%C3pcJ089xLzcK=xE!0`YiuCbB*;|fLtH>M90{xCc!DX?%m z{ql$l=ZB{M7H&+6-9PFa+JEfy;Qa9W2h)u1A018sJG*}z@R$xyUF|=1aCHAz<-qY( zFoW$|1{)}))qlyJu%1DUWrm>}lM?5LtA7~HnDxG2WSId<*^F}z{D_!);782dh93=c z7W^=n+wfz>9Htos8!^*|^F>xrY2(lF!EF!A45v>lQvM&9I2hEAd}88a`1pcNp^9NM z%jErW+G)#wW$a}8D8L}7*z$v8!m*bOGxg^AMS=XH7X|VPmOn4*Z{(d>;9})g=)(Wu`j5$9 zl8oz>88tXAXzaZEl3AJM!QGdv%8U;flo*sHz{5i37hD$Hla-Vel|B2C*C^nJI)mB= zX0{Tk1+Q5c7(U4JEqKkxz~*-LB{u`t1r0YPw+^>5H~tTke>l2rb5nKUc+JAdGn;Fw zh8vgLrWXbB3qY>0{&{iVM+Sbj3wK{~F$uVc3%V4!@qeiL!8?O}4&N+>3Py8*0%rYn zlMg048U_Ava5HvO2N}e0_lLy;CO1pBjZp9G`>4S8<0AVU?Tx%MrQA4NEM53NSpJx7 zl44T7VJl=H2$DGqHgc+hY6(ys0ZI>y;DO4F8;4(ud16F!m{=x7-qP5? zE?#-b$Ox;eaMx1EMUT)$?@xCEyg$7Fl~xL%`ivoiGsD9{;Db0n!-v-m%nBSI^nN^h zzyJ>gg?dN1lbe_{8$P^k0O@1W0O{h*;Htlv!OrkuVFSn!4H;b-9SKdZ7#OEK0P)*0 z`ZE;V`VtsEWc*Wb`}4j155!t<6$-VMBSG{7I|IXq2@UP9*ch2K96ppcFlaJN^t&+S z#|9>W4=fCx7w-OGnBf0`kz?ksm7o^QR|QbIHv|=32R_w9hWS8q7p%V-xdPn;e=Om5 zl3~!`Dqy;B>!q49`-59A8I^@HPW_mnAf0jQrMhvL8-qsJkA`}Vg63CDjOwo#85ut` zq%&U(01YaBc=gMCqd|tkhlT`$ga9|j53B!j{9^r}`HF!PWId>U-0-X6>)wm1^-O9C zZkqLOM&M#lp@K<~@5AfgYUPYd3Ka|=%xa7l3FV-~2Q!yhk?+I)-)iNu4^-=!K~}Y*$m|CXr}RUIRs8>a-@Y5<7{(v3z*Sb~hGsX0dV!6c zpz^QZ`_qeX&}fcD2Ezw-24;Q(e8H8s4P-FlNE-`+nanAfaKQTLDRf0x>g`0vBNc8u2 z%fAXc8BH|I1?tT>be>%1_-0`Ws^CD?^w}Rg8!{?DbAvw{X8&MtaQo%}$~E4gv2i{( zk>A`C7+*7RXu5HHT*1KjVZ%RWbq6;~a3G2N=1ymS&B;Cc#}WomgSh+05&`y$pg3U2 zIQ?S=14qW`m+B0lnj2(#djq3}@5OphtC3O7!A%jGK*fJRS_X_td>?-O?EYbp;Q#G? zF}up`muf9E>14KxOmnABGH!2Higl8N5FQ_mA%!Rx*BI`+57v90o?k z#Lr9&jEbj!%wceFWBjoB2gk=b45ANizf>1uR7m*D#OVJ;FoEkgL%~ERuvb3LVPLxe zilo~=I0SD0n4`dz01CYY44_bjWx^#4%oibU>;3_92W$p|S?$0lhI-Hxh6reC4&+-< z34Zqn&jygkzzxX#3+fpS7L+H{Ga4k6bAA=fWc`-G%Jq?vYvF4<1X#$yZ)-tki%ef*l`Ic=@F- zGRsq45+SKqmlNrU|YqlP7D zno5IbzbB|2`M}YQqv8NN^9AM^nr_@5F8pOQW!Aj~uDh7$FoWx4RyT$WP_$?yfGQkB z0Rbw-?)^}xU{&P%@aYGr@{#_jfmAF^`JupYfE`p1g6E+^6krokEFWxsfh0j~@|GV8 z3?QxAKN(aM+;~1L`^9L&tn>3CQc(czp?MV4f3aWN0ZMsatk;4IOMlJ})st`LO4o;j z9sT{H{|5$cW;I03#RiKENrB%?8#rHcbMQ*c_+hXB6sn-e1~s@DKBP6w_+d~05`)wR zl0Uc^X8d3jU{>P$VEALzM@EK@9|j5B8h$tKzhL;l#^nnF0lvb%W*Tk`9~S&`t$zUO zr!ju``d`D1Ns;4&!3>i9hgDe6S8X(IcEqj<{3`)Eoe*XuXqtx-k zU;)F2-hYx0X8d4eU{vz{#FGr_8G*`9P-Q3igIfVq@iD({Vgi+;li%d=)X)CQ{Z%k~ zH#kvNuZLAO|GuAq&OvmzF=#SOVE7>31WMwN-e$k_j&`I>3}o;?7yffzpAog7?f&NTJksYrtb)JB`EQ&gb!;@Q$?G0#9& zF8dD&ZsrRCZoj`T`)~9B+nNKTfoDO;X~9fM79rK4;oVJ z`UNUTxj(G?hbTxVPbd}R2M@pWgGqnR4^5M|mKKA$v07yojM|fv%L;itet;tA+(iX9 zrVl=oAC^giIcxR5iM|F`_Q%@vq!-D z!Rj|?&=a&a;{W%$T@!w6Fgl^cpn2pIGn2vv=MOB54L>#r2`V1=#LsvjYhr@;CyvC) zuPTg~HV93YtCWzl+~xg=BY{!T`x8qJs3Zn;Y#AyPUokLwe_}|S98!5f2(m6^$`8W^ z)(QnT_76LLPd2MkVd42O?f>MsDhI{9pKK2bKGc?*{*^X-?aRwV*rJ6$lobHY8a;c*eTKI z#Bvc59vmOoC(o=im*x3z_dj?cMgseX_@4@H0Y5zC8=YkKgTktCvS6DiYk|Y>@9C4- z>n(X9u?K2DI((Qk`Cz^5ez6!xU-VF5nK1dmJdMd$ znuLU!1w7e2_7}J_DmMIBB>-BpHTg%AgPk8}MUNwkTSJETw}+uuUaBdB`hLnEe4Lp+ z_&7;9ANesyfkElOj|PPUKUx&Py%VJc8Q{K&Qc}j`o@PD9mdU%D_03E!T=~JF32IJ* zB01wngWHrJ2RJk~PWf?=1JbHI$N`dJ{=?=53cneD*>=v@Ia#koOfCT2g>kGn$u76Q zv4hdDu^}OYA&21uKSRq87S7FKEk=w?4GNRHT75)dYgjlgf|l*f{R57?SFLJ{IgwD*)#0rH9MrFni@=OOmI|(xXV)_KGvOl%iFzx<5*`Qs5>D-RVUhQgZ zv%rQ-elS6bQDySPc2ODSKj0CxL`Ef0LYVnP_vHuw)(y>&rC5jBj(Rv-k(e|oIaH>w#pp%#H6A5gF%4vH7lEg+wbo?|FXM*1_rF$cs|Vg zQz6Nv$oYdofYE^UL(Wf*2mdtLZh#FG>Tvuq>M!hDCI-ApbgBh7}94GJTD`0x| zYqDOyk{)P1*nyWUWd~leMKFBi0Ij-H3eJ#BIGhle(3HURA>fY%WYJus_qPP6$#wnF zvY#A^Sk*w|B!{rAVg1&x#wa;ic7ifv$7II|=}eb@OkOoXPQv9kqpF5m!QKVASAJZO zU{C_hMg9K1XBdH`@!C{g3k%5u0`E@_MU#yt z`YTtCsaKKaKAcIK-JZcK`7A7X!qe&AsQm2Yex!hcNpQOls=-14KAVZ&a~I4-Z- z33lcSpyfeqA3}d3bou{8=<@st(zODr3zYsEH~4;g;(quuyNb&v1|HBj*O5=mJoXiJ za?6|&Zr+`2J!!XI_YY(&#GhAt$2GgX80$1f%gfy;sym?zJjYJyjF z&NL&&z{&Hc$)P*T8`;bo(=73tqBC8G>B7IsQPX!Y=1&%$QO{^OdGd^UP;)|LrU{2m zIm1NG54-Q#Ta;* z-MLw37|SNlo-Jqk$)O0bz>ezy7gJLN!)rzbZ~Q9|-_35sh=sa2B|>k1v$?T<{J|io zc=RQM3-AV%)c2FjNG){m_PVW4qw#7{K=zu^PNR;j4WQ?9117f zEz#rf`sPsK_06MXa>%3y$S`43Icn*pxYfBD-1U>fcA7Sa%Ko_WK=v@P~Z;R*YFd(ui+bbUqd~k z;{WH~-y(cK&1~fPBk79*Tb({J@Ogg=a5?aa!4Krfdp{TqloR~FG&p;I3vr42!3^3P zprV}M{Vm1?B(IzRY9p`Q=>09k+5g-BLeRL(>2Lgu(iw@b7#JlTyuSrFgT$mWW;in_ zANlm5STy(nL@@)X0#H8v?SCPrVhIO_PydULd}3ghZ~#wkDLA}hWbpnL0$%IRk-)K^ z;X#4>hOOS;VjMsYo1C>Sh$+H*^1gL)!2wR75Cl~kj2VI;r#m`CAar_vi-DQ}YQ2DZ z8=z(>!-r`NKfZ_k^8Oa$>HRIh-Qm;w3Q%ndDUul`>#x^mQCE4^5AScFl}k(tEC~!BY=1j{GGJg+Jo1Tw0kkHbNs;-(w8`Zg4OylC zFsN8hUb|6|DS&J8g^fz}lIz)jvrYK@{o_xTU+f=Rou~d<`hh_gG%oh`BG+@!=r!@h&}T+vla#%`oeA;&wj+%B10(Z+PfVb8j*2vsd4l)11!15ZwkFv7+Xhfk z0g_QZ_JiSo_qQcs2PQw=Y-<2>DyVVq{cS2_;;Dd9p}@UiD}w@qBJ&5eA0S_Pe+zK( z{+8k}IcSSdywgv|PcjCMpG*uP%cUF{Ii-GnfBwV!+XSf388rkig4BTI88S3BI(||B zuU%uvV0gfw;Qg%tlzJ685`4d{0hLG!puEF-(f8X5H^)yZlmBi}m1zIN`;mhOG;_-H zk%y1xBMSq=1mADZ941?B%~I%b_5b$2xZI6#Ca9lt_9gqmvoATBR%9|{yh!lv05)!tQfh1L|ulc4T;Fg6w?eFqvsvhe`n0YKIC3cHS$n!jI7{@CS!} zmlM+k4g-!4H-AiCw@r-&BsuxUHYJ0GA4~@vD>OHPX7pLWD%^fBDl$F*6_OxDpk&8b z01B^$t=``f94F7&p2U>kFlmPq>$!hFzOS3?z9Ud9!2{HmzVU;BL0}_f3e)RbL;2)& zJJJ{@O%~m0D2~~aQpo%9y?1i#&WZJq9x$UKV*xDpF=Q}lfabkHTkh^OfQFuRnjJpz zfY&uLW-w$heqdy5baMEh((L?6gVFyBgMq^*79QVk&%FG<9Vm17B*5+a?TX8hPYjHp zHsgU$Oj-v%*|Qz_#J~+ImpMQ?-5fw;8B7X{33rr#Feu#lv4Y{w51s;M9^Y^8z(L3~ znQ=OE0jLS%`|W}AlH@Z!}v&(?#gwtfc-39tNuKr&N6wdrO z11ecmH7Xc296p#dg7TUFhX&Ao4MmO!j)dReh5j?>GknmTymq&^kIX|x6$S8cGQ)?U zUx*^t{t**g-0he1Ck+PgZxJBhfXr68@FPK8p~6vyX%d5mV!@)1ptS}bOp^unh}9EY zx-%*;C^D)seF*#G|Am2pQStxxb3YWItJ@R`IQIYluJMyW0W|r*`9b6-Xqe3VQvkRB z2X;=-K-qy$Oxg~g7#KiX4nPB>T)!C^80#4|9W*{PHB9)SAOKp;#Q5PeBWMww$Ac+9 zZg7AmC3735{J6;h3YUz@XZOhRi2vXKPf~Gykp4OO%N{rN**`&ZJ6C>y>|s>Xdcgjg zg9WtlMPoBaeDQx)H&DaFVsh+W%lf|c53<@4K&y*DD~rxw1WmHBf1T*Q>mvt${gPYi6{pAr}yCyVThU;&lNlMD8lgq;XH^wELI{|k6~ zqT=Dt%*qn`g?;V+NBZ^TkgYLLZp;e_!xL#W~syYU619H9^V{Q4KYzwHS2{Kg4@Zi<{^9T~CvcYwyAsz&Hc)}d$ob*nkIAnNTPYm=!KJ|QgPoy525g=?8luVZVdsA?1<)ibWbGZ} zhc@skt7fPjJfLb%fuX`-^2QVLNZyy5+<8ozh2ulh|H&(kSxIqxNd6BB3GUadZ0t&G zA34Ec!SNyD|KvZ%Y?L@YSo{Zt1}j5`3^;ZBGk#zJ+XMGY)Ny3L%%0qTT$btozsc*4 z+cDk$H~HOhF{aD^CbON;W7_?1ve}6cxe5PJogKRy)JkfGMb?TF8r&ryx2qYrF@E?u z`SyuuMyttCa@1vrcjff4CGh3Jly}QT=uY!lM%cCU=}g4zt$DyU#{5o%%gl;+%!g+TUE@ zKr?VTSO^L}hL0R94sPGSZ~FtPCmh|h+&DhW{{2FdONr|>1EYrvC@XOlaT>sUpS0t{ z*5(bYkc>3%oQe|6?+hRQ{%3PxcLNO(vMU+5Dg0oP7G22kA@%p`!3iq6+o2u z{F*FwQ30gH;-U&uiOb};i#8zMoQqmaC2o^XTvT9r^wXkZvfMSX$xN4YnD+miY;ehp zY4Oj=8JFUiihoW%amjG_Yz4=%eg9sV&{<4TCqj30~- z7(NQHdw(l&0Hq6d=*~q)#$%EOZX6#veoUTsMGut6&s~uKIUwq)5>tumWS*-&NW8kM z0U+Lmt6<0dxhluP{^7*`$#T~mSlB--`#(A9nu8SkhZb;l2boAYpV`P}T_O?IMjX#tB-A-Uy{%3N+od~9? zKa+ReNl^3rv*I-?1H(r?F3i|rc+JIT;Ku&J{Lf^EyJk$%e-IdpSxh^?Yk?( z^z!%QJ8(gT67YxvXd1)s2dkTc3n=ySDuL22%R=@K+ka1XxTnNh0B+ebWC(m|NZ^=U zcuN|T^m^{eFg5+2yz1U2rjn4!argCvUm4G@k;I269hXa2>9gYPViJ2S;0(-$N zlXdrJfQG3mCVM@wU@V>7{J^_@_U`}BH*EF#)=}f|$w3%2esbg!1M{7iYC@NP90|Dm z^D1*wGmuev2fLqbb z7g-b-6#slr{Riraf`%$|3P3#)P+JPJM%ICy?dqTJLYn_1HbRsJ`~&s3L5e_Y&t&$4 z4DWorVK1mA_xjdRHu=&+CB{vYzdqEjZ-%(IP}GfyQ3g~tD|SaPzGe_u?c5#Ul(ER! zDdFNv6=f-Bj{O_F zzI9YFC^D#lH}+_8y=G*$scLaz++R}_|LRCd)DK1nHD|_;0`-h6`|C4oogFmaUQ~L; z#;E*?l_~HQ6SE3vp{%q2hyNBWuQ=ElU$L_>yl!CUdd)8B?-u?eV!zufZZ^Aw=2skS z!VXNY*jVc_SY9zQ1-)WsbbZCfTALuT|Nr;Y|Ix457?>`A8zlYLUotRqIRChC^M?yV zgZuTD49bnKI2b2?dnCg(5mXJhgW777Wgbg0J^D4-;&F-kfgiteKqrh$aF2Y&!oYq3 z)Nf|^u=R&ku7{hV+XQ!p4;?>#e_t{A@?%9YgjnmBhErsG%^+s(7XBlHzcNGV6$69vWY;IDjC_+Y(AZ$NSrv2w107VYBzQX#orqoj$RMgA&G(PfYBL72e+t z1Waywsv+?C`~Cl*zAL03e;{V^zNZy(8^Zj*G#G$dlM$d9B{lDFJ3uXUb(0M5Z)?IP zhdc|DIuUx~2Lnh_RX0P=Ws)RlB@62V?{6~#Cm(rcz*s-|>oe7mZU$I#WUOE`WGpCf zN8&?wym0KVegF}NwA335ia|pxAeF34pj@cBqJZ@y2LtPe^*_8nF<4BldY&Ww?l)+P zS!ThH@0$M@?ODEMggm?m>i4jG*g2W;g<{r}|DYX98U>&|OQ5LX*v}5N1U420Su7vG z0_qfl2Fp00gJmF#EPpJhaQ@U$2X zIJxJAM1A81|1T3=8I)jMLxxG9&LPupMvfUjWEdDV>@Lpup}=tChlZ2D2Nq72iyU8h z3KiVGe~w>l)D+@qtrZ`*}KCY-|RB-$Gz3V^A1CSz?{R|2T1zcYRvsk`muQ1lz>+p$zg+q1nnwLuT1wR;VL9z4tq5!xk$K0cYQD4;&`!xj8U%hQ$UA;uVPYzn^5cdElNQT<#}7P=j0H>& z9%LQ(!Lb3hg6I52MpMwx0Eh08i;$plcm`@cv~TeK6u<%M!Fo74d{W@@|G>e?pbjcA zKrwmX6N6O84<-gig^nK#3@i!44d)5uh&hksmri1~Clm{8y*`FyubKUeEC1P9w`#o&rs`@845?g9Jgzr1gg( zH`8==H%2w_A2RWvg>;Z5stOg1ifkX!euBmq>~I{boe0A$fThMictfHPYg^B-!g(h`VV|!P-1kG{LR84_=8J;<25&XMTOvRW<`f@ z8B7;{=m-h?W)hh2!%~_-li|bn28M4IhO8e~F#P<^{8z(`<->*l(|>d*F=)kKxc*~? zQtks#-e>*FQ=j1fWugx#^K-kcFl3!5`I|*B(&ge0g^G(mR5~PovlueI=4N3~yz=9M z24lem2IfirUv3l~dC9?eODY<7GLTyI(!pg1ce&wj02w- z1Or?gV_JXgkZ|}W$Oz&)#I*j{$-y#%;WZbB+k;)XNgIVLTt0EI9Qnk|BE`h=npt^K zM05s7KjVYzKXxbxNiit1e-z+3{*ptv=LI`x3t7VP9~_p)fAClyf2m~5_M4%B;Twm9 zLcIa&1kfOa-4Bk9tg|E^oc$rt;QxW0L&MDxoC>Eymf$nq{UOl6It!Gb84cQh7&3TN z9AKB*-vM5Of8!+sVDoJ#!Q0uS2N zvwm0$T9PyYwi1O|1tbNE5P{!J;uc7&O+i%#XegX%|BiY_gB|4+^^67;}LmW zqydxu|G!_@-LUn`_r*WcB1RyQzw2>JJ6BWMwbLK?#dga4p5c2fg*z5n{zrlvR`=3uP_^6`M zvu8mrgNDZi(84{&5C8svmggz_Wm0qbq`;x!W(bZ;?@tV7llOd7U|c@=?nh7V4eKmStzug+uqux9eaPfNu>^Hv`FLGvE2 z;Dd@9H+X-$5INc5^E$?(liz>#Wo(#i`z4n1=`SV)M>nPq&nGYbV$XCTaPpHcZah+d zL5&Q9gdg9({+VPcB2?h;`TH91m`~#dJ~!`g7a}IdeC=a2nf&^zDr5U(!EbvxkNsp; z@N{GPaCY+bZ?(cQzZeu+f2c6rcpl-V=*FzX^g(!X`gdo3{|ihyp%PKpG{u=LpJTxZ&2uM_^6?{{o|q)CC5e`Y%%Joj+XwO$0D9zGhHx%y9g` z#ORp72Qm*X>i?x7&?&;{0~3=|03%q2f$7L6W=1Zs7+1jeXB)QifV>+xS?i}1lf>`I zZa;k_{_S8=V*2p=2ZKV(4;2OhP_Q_HgJjiD7e?dB?|+IjU5Jzt%$Z#IXFt;l$H`WI0|a_N{VSCPnG6{Zz;jC=WindKAO0Zq^!7d%*a{nwo$!O< z0E5DY*D>uqhPK-v9u*2(E1YBzvbvyQN z*ehVbr1<4~?9^Vy81^3?LRVfIC{O*sxI(1o*CkM3eTe@F8ZH0Q06H#d`Wi;YeL^X} zG9PejrC((Eb^_cO6#75ChKW%^L#Ok|OTBbZK%RZcz`!&CltcS}g7ksQrq`@u3U16F zR{x#8hlx>2YQ`Uv2Mh{^53c+;ApxpBuKc)Sq2Tu8d+5LEAD9>=^rrljdH~|{{1@<48drXtu>eh0Z2UjnjG56y7UnyMO+J5*y<}t5aAQ3YaGtLWP_h^IQ;@Mqlx&HpJ$%4tOPZiW&a+34l;sK z>GSuJ>C!BW(vtsvZU#&K_^$YK-Se|A87wv2qTRlHm!Iy>!r0GvV)}g+#$-*wKaiax zj4HpryZorEWKdvJV)`)mCxec|w~P?B3sse1`44xeXR|W8Gv-d;%*vRltt0Fh7@TnZ zrJnG_gv$wseoRO>^kd3~LqBR39Qx5PLGS|TR56CvEeh-(PfT}bW3)$#P{$fZO;A5j zz<^Ql^Y@VH^Vt{!nR1+_e_~^dVtNob-H)A7Pm6hPfjg*G*0{m@6N5RZ59<}zqnUErwzmBG_RxEW2D{{Ncp#m%^ek$3ucZbomZmAeb}g7V1& za08v8;>AXnPYXcvDbCYlco?mi7)+#sSF@uK0ewI&Pf~6~Nv%k0?+{2*Z z|AB$c`9lMFb3VwHcRBzb(ut&lolRw=knDztkUt9!56y ztKcIjKWv0d+{)tIgXPnQv6%#n2Zqx^u4iA9O)BMYn7 z4>pCONEWR>7yo}Zo4!ztF`gfh*d+G<{eEjYzc`~blhm*2I^v8*eE0r>QW>bU`TPC( z-|0Exj0#M#`=k1-`|X*-^c@n6a$c7kru7TF={eBbDX|Mim{G~!C|_M zG^3YD>AHzO)-W(yH2zq_03Mw0XV^b|hBTuW55oqb4~(2F`$0>Cr+<)UlyK!}WK_EU zg82jU8gX%E1_m9ciyZ9IS3uj$etsAKVer5rV}}#Nhp!El85{`~8IB3ZUow=x1rG=~ ze0bFW8bM+FkTl&_hEdVJ;fF;JgQCI*4hD;s>o1uS{(k@bk6FRcjp4(u9~Tt-zZ58f z+9iw$tO}rgC?9@U^fG%+XZmd!Mpeep>5Q_Brh+s6S-5@t zzWXOk(e~-?vW)sFfBpx!{rH~t8#FFqdBOD?LlVd#49cfJT~K2D5dC-7j}1)I=g2bJ zFs=SM{gy1F2IJM~%yNu@Ou0X&hsiP8+J52yjWH=ORtSDzBk18AojE?paWDG z3nCwY_J@FyJ48O};-nug3PS8kS3*HDInyu5F&e6(7-4)7v>J!8;DiEbNgOi+V*+UG zi6dvanmnT;^CyOc=~?oO5sWR<56d$;3xk?tk{4maLlXNLKh#c_R$$au|NaBq%vSFy=BYnSNh^(UGZvbGniuV@CZRhr2%* zK!;T5gIo?el!EUg3nRw`CVl=3jQY|SIXTD;!8*A& zkbRQVCkICMS)fKI<7*}*rwl0lWlC!;N)nWCq(rhaW82tRGbvSthd2y!%7q0pr*G$t*KBzbKGzcu^n^q8mYm zft|-bgLNjzaSWfngfdh}X86f4d^ps=FjeXS$86tkdmI_m{68@8fqIV&(*l1e3$Q4& zi%(QgWSQDdibUW~aV*-ntM#A6k%RmbN z5kRgpj1Lmr3f&I;U}(4qn!^70-@g9)1n_BRZmy3nGR)Ng57KEAOh2Q- zXeP+=57Z5{a${0H`-7q3-}h6~g;g0X74QF5F9fyLn81znuitrhF)1jzF(@%9GfXgW z`}ckB^juZOT1NiquT>cxnezWm*HdFuVgL8j!0p3#ndy;gjGl};rY}}w)Z{$)lSx6- z?f>_~)32&A>T)iFa#u{}R%bM1?*!{uKHXWJQJHb-^mKJba~AJ!85z?TsWTd|p8fyx zyW;fg>WuP?eA~aPGo~`K7w%?I{P#U?da@>?Kktw4zCR#c#}D5#r|;KfjAi=0Z@Qco zqm`lerwvx%QKDCzpz&xe)(KoI7(Q}vu&v-Q01ZAcPjLRk0?zt3445V`d}x^7sKqE* z&oqNQ1GM0&y19bg_DTht!^bU+FohR>ELb4*fsvo#Ltmr!rwt}&U-Ile`;u!n6KL;& z%Cd|Iw@5e8>SxfPlR`C%hQkN8Mpd`a9~}G>eryn8{m8-Q2;DsgI^O)_KTyeP;07sI z89ucAV^GkHaAQznRQ~^6>{ragogbO_VOE@d$+7$FOSawW7j}MRMtx&=e~972 z4~_*43ZRYS>=!^AhP#*;6i>WlP!IrB@`|jpz_z|(VmL9KSBKGE8?p?Jp@L&K`^OR% z_K!Yn>>pJ)7*th1FgSuzFQcXbXfwVh&whvL^*W5w^+o>~O__C{T;x{>bYok^wVG>! zh1-|!Wq(2Did_H={eWT(1&8#r(M}jVBM}W|8W`+qr zc4%;D314(j;9AKw!NBdrONQNFzAyX>G8|+uquYfa7Z?;iI4~Uiae=}80ds}JCk9qf z*MsSG6BB3`x$tk67@pUMn5Eo;e!Pgk^n>L==qo-JoePtHa4<4__`}frgM-oe6AN?w zogX@&6vymP$^4o@fl>Luk1YWQe(ZQ~;Kv37#t(lP4!va9?f<1fsqIGt&$J&Km>53% zW@!7df$4(6$1IRjm=ahO{(pD+!TR9acjLbXZj6dwzMuRX|EjA16bhga;Jte52S)%i zI7NWw{Xj_()H`BO`uBZH{XdYO9Kh{_13#_^Sg<*KjBNk~w-`96jiJeE2WY#x1FI@1 zSt+?`gOk;*9~=yyzc2g^GQk$y4!rEfq{gZWng#)tbuAeNZh>xa)u7{zxM8iuhK$eO zTYfuzs)r4SD1g)FWw%3ajH>=0SUJEIfT3iUt)*aIvm9&rA|GTmICQIJ2ufl-wOR0Dx6jGP{>&nOGZ z7$}0x`i#1|@qd{WK&LjR{oldv@G%N(kuHiw{~K=o;Ba6BSu|Z+pHWg&9qcnkaJ~ci zClVAS)1&nnCHVyc7*)Z`f)zkv6*-;XfKhb1p8=y>1j}Cyw-4Xde|Wh4`~LLz4|awR zbHR4Wfb9Zh0|tKGi=dDXcVkoxas#!)6C5BDZ|rVNZm>WJ|G{AoidTjVDR2x;KV`sZ zkrI33{tLzr%-q5;8VpPtVHZ8z7!74U#f!w<^;_dVgi#{(7(w~H2T0Y54Q=%M-VJNpmT2cZ4! zj7p5k|Gq!_2g=y?Upm%+^xJ=_TLUV*?7x&u=QUzH%Xnn^Z6ii?#?{lAjTz&ZKXD{X zPcdeUVQij$(3mlsQFFSk38Orx?=R3|wg2BQPY*R=lxIwtUTVVF$QU{Oy9r}4@ zri`|Xj1ALQnlZ*QzMRf(&N!LvK!EqB6JgsIm@_glvYm+X{&XN>`U(riJ!}U8y+54@ zo}OjNIEB$|`hQEtT7d_DcfDp~V!p6K($bCb!_B|bd#o6@v3P&$NSf|u&1k~n{VgYD zdYv_6j=bjIN7Bp}G%`L)GJa70d-WwFqoNC=1&6~5&{6UmK1`3aWz1%J;y(SDEn}r1 z%FZFsZl37@c8mdbCw?<4h&%udn1D{@_JXc$VN&EuIPj984RkIyC@V2mKw7)33gF7O z5wz^b_uCV<>G$jyXXwTLM(iMi903m6<_78tVcS(VeThAz7o*Pf_x6m+jN7IQJ23i# zdiW_0j16qdet~3NQfGps-?$8Re@-9tqV7Z=dw4ZlErTkmfZQXM`W-~jpT zz$YfvBcGT#KsOR(BrtqvZvZV!|jOf#H6{f~6|#4d%9<7nV{Fzp9p z1LF+mPwyk0K1oY$Kjh5l#wha6@xqS`c2EnEv*PHF76#CO;dB{S#xxd?1GcxhG72#< z+3%b#;LRv9eVsd_oLZkaNy4IVRTmf zazVEQzFMkS7!KQ-Kbe{cRX{eVBCy)Sro zX$0Ow@sR^)`g713=wOiyMngGG#stviJigzyIQo9u;4|GPfYE^M=O578$nmhE}IVQbe0 z-)}2ir(YmSHxtvJ=|RDaDk88;8;*QtcAo5{0lK)sd3skcV?6#%i__Uc7(4H zVeDY}@;&Uw^cNwFOBl~g?+s;i7kc}BANZmn4O#AQe;D6>-~MC!vrtAgCN|sj1_qsN z(}f}!H`M3Y>f+s(ER1(wGAQo?od9|3{rAY9 zPHx_A@4tusWPR}d`w=Mbz)u!8J~sv>4e$5g9e*+30L^+ay0JfCP*U{<%~rhsKJCZ* z?=rsxR5%|n=y2WUQesek|2_H#^K1hnsqK!DjJuhcs{U>N7sEJ@iAm(|_IdG)evAgy z`xqE>wlOgYY-Mp{eDLXe@jsCtjNB}q3>rT!g3<$*!l&<v*)q?UmRgwND6FoE!uS_UQ%2FZai zrnw8BGrHUEXk%ak;R9_9Odt$W3d2|07??o#P8$Og2tR3KU;<&7Jf@K!&NH%1-}`~l z+4e*~0}}{e=x1O8VVE8e{iL6P354JDGcbYhmwpB&5C+MCFs8W|e=@q;-k8C_1i}wy zFff5INGS||nZdvW!hdEkFo7`3Oa>+phRI_ZIYEqtW%^qtCTH6h3mKR|_`^a5CJ=_{ z0nsdr7??nqXAuJv2#YLYU;<%~90+5Y%gDpzZu?^u0}}`{tY%;WV~|=X7Fo@}1i~__ z8JIv=Wibu(deP zzy!h$=NXtl7^D=2L(VfWfpE-u1||?rInTfZ!Z3MEBP%SKJZwE~Fff5|zzqf_5QZrS z(J40=m_Rt^1_Kiam)u}r0%4FG2xA&K!HLPkHsT=z69^|fWMBeekWv^fdC0&7!Zi;W zm_WGYAp;W#!{jlIT;a{+VVm)afeC~QUNJC%Fibg!Zh6JP1j0S97??nK$}0vY5C+MC zFs6|QLYO>kD?T$YfpEiT1||>&DTU!FpBb1yc+O`ACJp&yRl5p zwjF;Mm_T^K9|k56hUo#(Oa3r0f$*9?3``)rAbdfAkqLxh%0cuK1x6+iextz11j1hw7@0sABnQHnMh0{< zdD!02WMl&22bzpbAPiCp!(TKRnLzlDCL3GMkWyEF=k`}VG(0SCJ+Y6fiR|#4Re`1Y=2lYGJ!CI4I>j6gVaK?hz%nX z2+Pi>L#45loC2W4Id>VTW1~8lGBfoNy3kjNdbyMN@3WNf9Fx2-g%bGJ$YQ5hD`_!{jlI)VK>O zIjb3&GO8Jw3ZNLK97MNNGctj2PcF^9xa<(usRkScNH9#>) zDGX0(VPpc~IW3G#AiSi7kqLxh@|Z?`IM2vBJ@!45i%l;hQ%5f&(*!7nX#vqodKsBO zcug-O69{kVWn=D69@}jV`Kthm>v+Va*dG*gmtblGJ&wkHAW^72FZaihD)a#I5K_UZ9C+cm_T@f91{}=!}NgYC2~wm zAiPG7i3xXJP{3J?czMAbdof zi3x;Z@|Z?)Oa>L4224yV449ZUKru`?h(2P##00`;449Ze_=*7&69|LkKp4|Vg*l*t z(~^m4hb0r!0VoD3h2blfOiUnr$C8N&gr8V4F@Z2l9@9vRrJ#b-iHYfi6BE+~D26Eq z(NCP1m_YcA6B82%e{o`B0%4FG2xA%1j0~hEG6dxP~x0>5>#@gGcj?b zGcgH3F-#ALR!L`K0%4tWCMFOzNoQgLVUQdMV>)%iB~Zy(z{Dg`z{I2g#UQ0HY*N6) z1j052OiUo`QozIn!Z3MEBTw80m7JAKOd6F;Oa@R4Qx2kCDw&u-*r$?-34}u`nV3Kr zBnQHnMm~4~Dmj~&m@Jx@m>i%Oq!fljnwXeCIHrk-34~Lcn3zBqCXZ?4kGG(bvzv*@ zqnn8-0E%JCL3Bzt6B7vMbTcu5a7i~469|LkKp4|Vf$yM_a|#nv#1tl`1Ske6h2fGZ zOiUnLGlhu>gj=RCF@Z2l9>d6~8Y~{R8FQJKK)7Ho6B7u-RDm&%^}642%%95$lzSB9czR+vlO<^QiCuiDN+W}* zY9oWXY9qsDl}3iYDvb=zs*RH!1tmFutY=~Z;p*vu!YtyG0|W)QH8wCYfiUNuhRKeC zq98K_xFa?&F@dn_PAmYP$Mv>_aVk`onvDN7l zV2lM&#&!o0mM4rHESs5_Kp19%0LTOZkO>0QKb&V20GSY=uwgS369_|m%fN78GZPaC z7qc`n?AXl21ahG;BLl;W%}h)n3=)3;6K7&z5ZJ=R1i~P3jxA92%nS?`TbP(Y7$j~0 z5tmR%*uumF!Z6DUwlFb)a4t(DLk2{VD9ExITbP(Y7$iOcCeFmba9|4)69|LEcfiD% z85mw{VPXPdkobcwOia@gE;33e2y6w7TQf1iY*X0E#00{DuNxR7wnClC$iU#Sm5B+2 zLE;WDaV7?af~`zUAPf@EfQd6RFwEG>#00`1@d;a*n5H*eWRws%u$74kgimZ`Vgg~v Z==XMmpm.resuming; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* * Check whether a gpu lost is in progress * * @kbdev: The kbase device structure for the device (must be a valid pointer) * * Indicates whether a gpu lost has been received and jobs are no longer - * being scheduled + * being scheduled. * - * Return: false if gpu is lost - * Return: != false otherwise + * Return: false if GPU is already lost or if no Arbiter is present (as GPU will + * always be present in this case), true otherwise. */ static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev) { - return (atomic_read(&kbdev->pm.gpu_lost) == 0 ? false : true); + return (kbdev->arb.arb_if && ((bool)atomic_read(&kbdev->pm.gpu_lost))); } /* @@ -556,7 +538,6 @@ static inline void kbase_pm_set_gpu_lost(struct kbase_device *kbdev, bool gpu_lo if (new_val != cur_val) KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, (u64)new_val); } -#endif /** * kbase_pm_is_active - Determine whether the GPU is active diff --git a/drivers/gpu/arm/bifrost/mali_kbase_caps.h b/drivers/gpu/arm/bifrost/mali_kbase_caps.h index f6bcdd06e1aa..000e30e1ed84 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_caps.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_caps.h @@ -33,13 +33,21 @@ * * @MALI_KBASE_CAP_SYSTEM_MONITOR: System Monitor * @MALI_KBASE_CAP_JIT_PRESSURE_LIMIT: JIT Pressure limit - * @MALI_KBASE_CAP_MEM_DONT_NEED: Not needed physical memory - * @MALI_KBASE_CAP_MEM_GROW_ON_GPF: Memory grow on page fault - * @MALI_KBASE_CAP_MEM_PROTECTED: Protected memory - * @MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP: CPU cache maintenance required when - * imported GPU memory is mapped/unmapped - * @MALI_KBASE_CAP_MEM_KERNEL_SYNC: Kernel side cache sync ops required - * @MALI_KBASE_CAP_MEM_SAME_VA: Same VA on CPU and GPU + * @MALI_KBASE_CAP_QUERY_MEM_DONT_NEED: BASE_MEM_DONT_NEED is queryable + * @MALI_KBASE_CAP_QUERY_MEM_GROW_ON_GPF: BASE_MEM_GROW_ON_GPF is queryable + * @MALI_KBASE_CAP_QUERY_MEM_PROTECTED: BASE_MEM_PROTECTED is queryable + * @MALI_KBASE_CAP_QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP: BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is + * queryable + * @MALI_KBASE_CAP_QUERY_MEM_KERNEL_SYNC: BASE_MEM_KERNEL_SYNC is queryable + * @MALI_KBASE_CAP_QUERY_MEM_SAME_VA: BASE_MEM_SAME_VA is queryable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_DONT_NEED: BASE_MEM_DONT_NEED is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS: BASE_MEM_PROTECTED is not + * allocatable in functions other + * than base_mem_protected + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_8: BASE_MEM_UNUSED_BIT_8 is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_19: BASE_MEM_UNUSED_BIT_19 is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_20: BASE_MEM_UNUSED_BIT_20 is not allocatable + * @MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_27: BASE_MEM_UNUSED_BIT_27 is not allocatable * @MALI_KBASE_NUM_CAPS: Delimiter * * New enumerator must not be negative and smaller than @MALI_KBASE_NUM_CAPS. @@ -47,12 +55,18 @@ enum mali_kbase_cap { MALI_KBASE_CAP_SYSTEM_MONITOR = 0, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT, - MALI_KBASE_CAP_MEM_DONT_NEED, - MALI_KBASE_CAP_MEM_GROW_ON_GPF, - MALI_KBASE_CAP_MEM_PROTECTED, - MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP, - MALI_KBASE_CAP_MEM_KERNEL_SYNC, - MALI_KBASE_CAP_MEM_SAME_VA, + MALI_KBASE_CAP_QUERY_MEM_DONT_NEED, + MALI_KBASE_CAP_QUERY_MEM_GROW_ON_GPF, + MALI_KBASE_CAP_QUERY_MEM_PROTECTED, + MALI_KBASE_CAP_QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP, + MALI_KBASE_CAP_QUERY_MEM_KERNEL_SYNC, + MALI_KBASE_CAP_QUERY_MEM_SAME_VA, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_DONT_NEED, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_8, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_19, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_20, + MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_27, MALI_KBASE_NUM_CAPS }; @@ -68,34 +82,67 @@ static inline bool mali_kbase_supports_jit_pressure_limit(unsigned long api_vers return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT); } -static inline bool mali_kbase_supports_mem_dont_need(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_dont_need(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_DONT_NEED); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_DONT_NEED); } -static inline bool mali_kbase_supports_mem_grow_on_gpf(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_grow_on_gpf(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_GROW_ON_GPF); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_GROW_ON_GPF); } -static inline bool mali_kbase_supports_mem_protected(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_protected(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_PROTECTED); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_PROTECTED); } -static inline bool mali_kbase_supports_mem_import_sync_on_map_unmap(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_import_sync_on_map_unmap(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_IMPORT_SYNC_ON_MAP_UNMAP); + return mali_kbase_supports_cap(api_version, + MALI_KBASE_CAP_QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP); } -static inline bool mali_kbase_supports_mem_kernel_sync(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_kernel_sync(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_KERNEL_SYNC); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_KERNEL_SYNC); } -static inline bool mali_kbase_supports_mem_same_va(unsigned long api_version) +static inline bool mali_kbase_supports_query_mem_same_va(unsigned long api_version) { - return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_SAME_VA); + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_QUERY_MEM_SAME_VA); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_dont_need(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_DONT_NEED); +} + +static inline bool +mali_kbase_supports_reject_alloc_mem_protected_in_unprotected_allocs(unsigned long api_version) +{ + return mali_kbase_supports_cap( + api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_8(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_8); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_19(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_19); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_20(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_20); +} + +static inline bool mali_kbase_supports_reject_alloc_mem_unused_bit_27(unsigned long api_version) +{ + return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_REJECT_ALLOC_MEM_UNUSED_BIT_27); } #endif /* __KBASE_CAPS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h index baca78679f0b..7657c25d565c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h @@ -255,6 +255,42 @@ enum { */ #define DEFAULT_PROGRESS_TIMEOUT_CYCLES (2500000000ull) +/* MIN value of iterators' suspend timeout*/ +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN (200) +#if CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN <= 0 +#error "CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN should be larger than 0" +#endif + +/* MAX value of iterators' suspend timeout*/ +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX (60000) +#if CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX >= (0xFFFFFFFF) +#error "CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX should be less than U32_MAX" +#endif + +/* Firmware iterators' suspend timeout, default 4000ms. Customer can update this by + * using debugfs -- csg_suspend_timeout + */ +#if IS_ENABLED(CONFIG_MALI_REAL_HW) && !IS_ENABLED(CONFIG_MALI_IS_FPGA) +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS (4000) +#else +#define CSG_SUSPEND_TIMEOUT_FIRMWARE_MS (31000) +#endif +#if (CSG_SUSPEND_TIMEOUT_FIRMWARE_MS < CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MIN) || \ + (CSG_SUSPEND_TIMEOUT_FIRMWARE_MS > CSG_SUSPEND_TIMEOUT_FIRMWARE_MS_MAX) +#error "CSG_SUSPEND_TIMEOUT_FIRMWARE_MS is out of range" +#endif + +/* Additional time in milliseconds added to the firmware iterators' suspend timeout, + * default 100ms + */ +#define CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS (100) + +/* Host side CSG suspend timeout */ +#define CSG_SUSPEND_TIMEOUT_MS (CSG_SUSPEND_TIMEOUT_FIRMWARE_MS + CSG_SUSPEND_TIMEOUT_HOST_ADDED_MS) + +/* MAX allowed timeout value(ms) on host side, should be less than ANR timeout */ +#define MAX_TIMEOUT_MS (4500) + #else /* MALI_USE_CSF */ /* A default timeout in clock cycles to be used when an invalid timeout @@ -327,14 +363,6 @@ enum { */ #define DEFAULT_PROGRESS_TIMEOUT ((u64)5 * 500 * 1024 * 1024) -/* Default threshold at which to switch to incremental rendering - * - * Fraction of the maximum size of an allocation that grows on GPU page fault - * that can be used up before the driver switches to incremental rendering, - * in 256ths. 0 means disable incremental rendering. - */ -#define DEFAULT_IR_THRESHOLD (192) - /* Waiting time in clock cycles for the completion of a MMU operation. * * Ideally 1.6M GPU cycles required for the L2 cache (512KiB slice) flush. diff --git a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c index 1e7e823f44c4..9f88d4f3fbc6 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_core_linux.c @@ -62,9 +62,7 @@ #include "csf/mali_kbase_csf_cpu_queue.h" #include "csf/mali_kbase_csf_event.h" #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include "arbiter/mali_kbase_arbiter_pm.h" -#endif #include "mali_kbase_cs_experimental.h" @@ -76,6 +74,7 @@ #if IS_ENABLED(CONFIG_DEBUG_FS) #include "mali_kbase_pbha_debugfs.h" #endif +#include "mali_kbase_ioctl_helpers.h" #include #include @@ -153,21 +152,29 @@ static const struct mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CA #if MALI_USE_CSF { 1, 0 }, /* SYSTEM_MONITOR */ { 1, 0 }, /* JIT_PRESSURE_LIMIT */ - { 1, 22 }, /* MEM_DONT_NEED */ - { 1, 0 }, /* MEM_GROW_ON_GPF */ - { 1, 0 }, /* MEM_PROTECTED */ - { 1, 26 }, /* MEM_IMPORT_SYNC_ON_MAP_UNMAP */ - { 1, 26 }, /* MEM_KERNEL_SYNC */ - { 1, 28 } /* MEM_SAME_VA */ + { 1, 22 }, /* QUERY_MEM_DONT_NEED */ + { 1, 0 }, /* QUERY_MEM_GROW_ON_GPF */ + { 1, 0 }, /* QUERY_MEM_PROTECTED */ + { 1, 26 }, /* QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP */ + { 1, 26 }, /* QUERY_MEM_KERNEL_SYNC */ + { 1, 28 }, /* QUERY_MEM_SAME_VA */ + { 1, 31 }, /* REJECT_ALLOC_MEM_DONT_NEED */ + { 1, 31 }, /* REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS */ + { 1, 31 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_20 */ + { 1, 31 } /* REJECT_ALLOC_MEM_UNUSED_BIT_27 */ #else { 11, 15 }, /* SYSTEM_MONITOR */ { 11, 25 }, /* JIT_PRESSURE_LIMIT */ - { 11, 40 }, /* MEM_DONT_NEED */ - { 11, 2 }, /* MEM_GROW_ON_GPF */ - { 11, 2 }, /* MEM_PROTECTED */ - { 11, 43 }, /* MEM_IMPORT_SYNC_ON_MAP_UNMAP */ - { 11, 43 }, /* MEM_KERNEL_SYNC */ - { 11, 44 } /* MEM_SAME_VA */ + { 11, 40 }, /* QUERY_MEM_DONT_NEED */ + { 11, 2 }, /* QUERY_MEM_GROW_ON_GPF */ + { 11, 2 }, /* QUERY_MEM_PROTECTED */ + { 11, 43 }, /* QUERY_MEM_IMPORT_SYNC_ON_MAP_UNMAP */ + { 11, 43 }, /* QUERY_MEM_KERNEL_SYNC */ + { 11, 44 }, /* QUERY_MEM_SAME_VA */ + { 11, 46 }, /* REJECT_ALLOC_MEM_DONT_NEED */ + { 11, 46 }, /* REJECT_ALLOC_MEM_PROTECTED_IN_UNPROTECTED_ALLOCS */ + { 11, 46 }, /* REJECT_ALLOC_MEM_UNUSED_BIT_8 */ + { 11, 46 } /* REJECT_ALLOC_MEM_UNUSED_BIT_19 */ #endif }; @@ -819,7 +826,7 @@ static int kbase_api_mem_alloc_ex(struct kbase_context *kctx, union kbase_ioctl_mem_alloc_ex *alloc_ex) { struct kbase_va_region *reg; - u64 flags = alloc_ex->in.flags; + base_mem_alloc_flags flags = alloc_ex->in.flags; u64 gpu_va; /* Calls to this function are inherently asynchronous, with respect to @@ -929,7 +936,7 @@ static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem static int kbase_api_mem_alloc(struct kbase_context *kctx, union kbase_ioctl_mem_alloc *alloc) { struct kbase_va_region *reg; - u64 flags = alloc->in.flags; + base_mem_alloc_flags flags = alloc->in.flags; u64 gpu_va; /* Calls to this function are inherently asynchronous, with respect to @@ -1055,16 +1062,6 @@ static int kbase_api_get_ddk_version(struct kbase_context *kctx, static int kbase_api_mem_jit_init(struct kbase_context *kctx, struct kbase_ioctl_mem_jit_init *jit_init) { - size_t i; - - for (i = 0; i < sizeof(jit_init->padding); i++) { - /* Ensure all padding bytes are 0 for potential future - * extension - */ - if (jit_init->padding[i]) - return -EINVAL; - } - return kbase_region_tracker_init_jit(kctx, jit_init->va_pages, jit_init->max_allocations, jit_init->trim_level, jit_init->group_id, jit_init->phys_pages); @@ -1130,7 +1127,7 @@ static int kbase_api_mem_commit(struct kbase_context *kctx, struct kbase_ioctl_m static int kbase_api_mem_alias(struct kbase_context *kctx, union kbase_ioctl_mem_alias *alias) { struct base_mem_aliasing_info *ai; - u64 flags; + base_mem_alloc_flags flags; int err; if (alias->in.nents == 0 || alias->in.nents > BASE_MEM_ALIAS_MAX_ENTS) @@ -1141,7 +1138,7 @@ static int kbase_api_mem_alias(struct kbase_context *kctx, union kbase_ioctl_mem return -ENOMEM; err = copy_from_user(ai, u64_to_user_ptr(alias->in.aliasing_info), - sizeof(*ai) * alias->in.nents); + size_mul(sizeof(*ai), alias->in.nents)); if (err) { vfree(ai); return -EFAULT; @@ -1169,7 +1166,7 @@ static int kbase_api_mem_alias(struct kbase_context *kctx, union kbase_ioctl_mem static int kbase_api_mem_import(struct kbase_context *kctx, union kbase_ioctl_mem_import *import) { int ret; - u64 flags = import->in.flags; + base_mem_alloc_flags flags = import->in.flags; if (flags & BASEP_MEM_FLAGS_KERNEL_ONLY) return -ENOMEM; @@ -1281,7 +1278,8 @@ static int kbase_api_sticky_resource_map(struct kbase_context *kctx, if (!map->count || map->count > BASE_EXT_RES_COUNT_MAX) return -EOVERFLOW; - ret = copy_from_user(gpu_addr, u64_to_user_ptr(map->address), sizeof(u64) * map->count); + ret = copy_from_user(gpu_addr, u64_to_user_ptr(map->address), + size_mul(sizeof(u64), map->count)); if (ret != 0) return -EFAULT; @@ -1320,7 +1318,8 @@ static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, if (!unmap->count || unmap->count > BASE_EXT_RES_COUNT_MAX) return -EOVERFLOW; - ret = copy_from_user(gpu_addr, u64_to_user_ptr(unmap->address), sizeof(u64) * unmap->count); + ret = copy_from_user(gpu_addr, u64_to_user_ptr(unmap->address), + size_mul(sizeof(u64), unmap->count)); if (ret != 0) return -EFAULT; @@ -1402,7 +1401,6 @@ static int kbasep_cs_queue_group_create_1_6(struct kbase_context *kctx, union kbase_ioctl_cs_queue_group_create_1_6 *create) { int ret; - size_t i; union kbase_ioctl_cs_queue_group_create new_create = { .in = { .tiler_mask = create->in.tiler_mask, @@ -1415,13 +1413,7 @@ static int kbasep_cs_queue_group_create_1_6(struct kbase_context *kctx, .compute_max = create->in.compute_max, } }; - for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) - return -EINVAL; - } - ret = kbase_csf_queue_group_create(kctx, &new_create); - create->out.group_handle = new_create.out.group_handle; create->out.group_uid = new_create.out.group_uid; @@ -1432,7 +1424,6 @@ static int kbasep_cs_queue_group_create_1_18(struct kbase_context *kctx, union kbase_ioctl_cs_queue_group_create_1_18 *create) { int ret; - size_t i; union kbase_ioctl_cs_queue_group_create new_create = { .in = { .tiler_mask = create->in.tiler_mask, @@ -1447,13 +1438,7 @@ static int kbasep_cs_queue_group_create_1_18(struct kbase_context *kctx, .dvs_buf = create->in.dvs_buf, } }; - for (i = 0; i < ARRAY_SIZE(create->in.padding); i++) { - if (create->in.padding[i] != 0) - return -EINVAL; - } - ret = kbase_csf_queue_group_create(kctx, &new_create); - create->out.group_handle = new_create.out.group_handle; create->out.group_uid = new_create.out.group_uid; @@ -1580,14 +1565,15 @@ static int kbase_ioctl_cs_get_glb_iface(struct kbase_context *kctx, ¶m->out.prfcnt_size, ¶m->out.instr_features); if (copy_to_user(user_groups, group_data, - MIN(max_group_num, param->out.group_num) * sizeof(*group_data))) + size_mul(MIN(max_group_num, param->out.group_num), + sizeof(*group_data)))) err = -EFAULT; } if (!err) if (copy_to_user(user_streams, stream_data, - MIN(max_total_stream_num, param->out.total_stream_num) * - sizeof(*stream_data))) + size_mul(MIN(max_total_stream_num, param->out.total_stream_num), + sizeof(*stream_data)))) err = -EFAULT; kfree(group_data); @@ -1611,10 +1597,6 @@ static int kbase_ioctl_read_user_page(struct kbase_context *kctx, if (unlikely(user_page->in.offset != LATEST_FLUSH)) return -EINVAL; - /* Validating padding that must be zero */ - if (unlikely(user_page->in.padding != 0)) - return -EINVAL; - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); if (!kbdev->pm.backend.gpu_powered) user_page->out.val_lo = POWER_DOWN_LATEST_FLUSH_VALUE; @@ -1641,65 +1623,6 @@ kbasep_ioctl_context_priority_check(struct kbase_context *kctx, return 0; } -#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ - do { \ - int ret; \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - ret = function(arg); \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - -#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - err = copy_from_user(¶m, uarg, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - ret = function(arg, ¶m); \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - -#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - memset(¶m, 0, sizeof(param)); \ - ret = function(arg, ¶m); \ - err = copy_to_user(uarg, ¶m, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - -#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ - do { \ - type param; \ - int ret, err; \ - dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ - BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE | _IOC_READ)); \ - BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ - err = copy_from_user(¶m, uarg, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - ret = function(arg, ¶m); \ - err = copy_to_user(uarg, ¶m, sizeof(param)); \ - if (err) \ - return -EFAULT; \ - dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ - return ret; \ - } while (0) - static int kbasep_ioctl_set_limited_core_count( struct kbase_context *kctx, struct kbase_ioctl_set_limited_core_count *set_limited_core_count) @@ -2418,30 +2341,41 @@ static int core_mask_parse(struct kbase_device *const kbdev, const char *const b static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *const new_mask) { u64 new_core_mask = new_mask->new_core_mask; - u64 shader_present = kbdev->gpu_props.shader_present; + u64 shader_present; + unsigned long flags; + int ret = 0; - lockdep_assert_held(&kbdev->pm.lock); - lockdep_assert_held(&kbdev->hwaccess_lock); + kbase_csf_scheduler_lock(kbdev); + kbase_pm_lock(kbdev); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + shader_present = kbdev->gpu_props.shader_present; if ((new_core_mask & shader_present) != new_core_mask) { dev_err(kbdev->dev, - "Invalid core mask 0x%llX: Includes non-existent cores (present = 0x%llX)", + "Invalid requested core mask 0x%llX: Includes non-existent cores (present = 0x%llX)", new_core_mask, shader_present); - return -EINVAL; - + ret = -EINVAL; + goto exit; } else if (!(new_core_mask & shader_present & kbdev->pm.backend.ca_cores_enabled)) { dev_err(kbdev->dev, - "Invalid core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", + "Invalid requested core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", new_core_mask, kbdev->gpu_props.shader_present, kbdev->pm.backend.ca_cores_enabled); - return -EINVAL; + ret = -EINVAL; + goto exit; } if (kbdev->pm.debug_core_mask != new_core_mask) kbase_pm_set_debug_core_mask(kbdev, new_core_mask); - return 0; +exit: + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_unlock(kbdev); + kbase_csf_scheduler_unlock(kbdev); + + return ret; } #else struct kbase_core_mask { @@ -2478,15 +2412,23 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con { u64 shader_present = kbdev->gpu_props.shader_present; u64 group_core_mask = kbdev->gpu_props.coherency_info.group.core_mask; - u64 *new_core_mask = &new_mask->new_core_mask[0]; + u64 *new_core_mask; + unsigned long flags; + int ret = 0; size_t i; + kbase_pm_lock(kbdev); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + new_core_mask = &new_mask->new_core_mask[0]; + for (i = 0; i < BASE_JM_MAX_NR_SLOTS; ++i) { if ((new_core_mask[i] & shader_present) != new_core_mask[i]) { dev_err(kbdev->dev, "Invalid core mask 0x%llX for JS %zu: Includes non-existent cores (present = 0x%llX)", new_core_mask[i], i, shader_present); - return -EINVAL; + ret = -EINVAL; + goto exit; } else if (!(new_core_mask[i] & shader_present & kbdev->pm.backend.ca_cores_enabled)) { @@ -2494,17 +2436,20 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con "Invalid core mask 0x%llX for JS %zu: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX)", new_core_mask[i], i, kbdev->gpu_props.shader_present, kbdev->pm.backend.ca_cores_enabled); - return -EINVAL; + ret = -EINVAL; + goto exit; } else if (!(new_core_mask[i] & group_core_mask)) { dev_err(kbdev->dev, "Invalid core mask 0x%llX for JS %zu: No intersection with group 0 core mask 0x%llX", new_core_mask[i], i, group_core_mask); - return -EINVAL; + ret = -EINVAL; + goto exit; } else if (!(new_core_mask[i] & kbdev->gpu_props.curr_config.shader_present)) { dev_err(kbdev->dev, "Invalid core mask 0x%llX for JS %zu: No intersection with current core mask 0x%llX", new_core_mask[i], i, kbdev->gpu_props.curr_config.shader_present); - return -EINVAL; + ret = -EINVAL; + goto exit; } } @@ -2515,7 +2460,11 @@ static int core_mask_set(struct kbase_device *kbdev, struct kbase_core_mask *con } } - return 0; +exit: + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + kbase_pm_unlock(kbdev); + + return ret; } #endif @@ -2539,7 +2488,6 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr struct kbase_core_mask core_mask = {}; int err; - unsigned long flags; CSTD_UNUSED(attr); @@ -2552,14 +2500,8 @@ static ssize_t core_mask_store(struct device *dev, struct device_attribute *attr if (err) return err; - mutex_lock(&kbdev->pm.lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - err = core_mask_set(kbdev, &core_mask); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - mutex_unlock(&kbdev->pm.lock); - if (err) return err; @@ -4385,7 +4327,8 @@ void registers_unmap(struct kbase_device *kbdev) kbase_common_reg_unmap(kbdev); } -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) +#if defined(CONFIG_OF) + static bool kbase_is_pm_enabled(const struct device_node *gpu_node) { const struct device_node *power_model_node; @@ -4424,13 +4367,13 @@ static bool kbase_is_full_coherency_enabled(const struct device_node *gpu_node) } return false; } -#endif /* defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) */ +#endif /* defined(CONFIG_OF) */ int kbase_device_backend_init(struct kbase_device *kbdev) { int err = 0; -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) +#if defined(CONFIG_OF) /* * Attempt to initialize arbitration. * If the platform is not suitable for arbitration, return -EPERM. @@ -4439,13 +4382,13 @@ int kbase_device_backend_init(struct kbase_device *kbdev) */ if (kbase_is_pm_enabled(kbdev->dev->of_node)) { /* Arbitration AND power management invalid */ - dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n"); + dev_dbg(kbdev->dev, "Arbitration not supported with power management"); return -EPERM; } if (kbase_is_full_coherency_enabled(kbdev->dev->of_node)) { /* Arbitration AND full coherency invalid */ - dev_err(kbdev->dev, "Invalid combination of arbitration AND full coherency\n"); + dev_dbg(kbdev->dev, "Arbitration not supported with full coherency"); return -EPERM; } @@ -4466,21 +4409,19 @@ int kbase_device_backend_init(struct kbase_device *kbdev) if (product_model != GPU_ID_PRODUCT_TGOX && product_model != GPU_ID_PRODUCT_TNOX && product_model != GPU_ID_PRODUCT_TBAX) { kbase_arbiter_pm_early_term(kbdev); - dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); + dev_dbg(kbdev->dev, "GPU platform not suitable for arbitration"); return -EPERM; } #endif /* !MALI_USE_CSF */ - dev_info(kbdev->dev, "Arbitration interface enabled\n"); + dev_info(kbdev->dev, "Arbitration interface enabled"); } -#endif /* defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) */ +#endif /* defined(CONFIG_OF) */ return err; } void kbase_device_backend_term(struct kbase_device *kbdev) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_arbiter_pm_early_term(kbdev); -#endif } int power_control_init(struct kbase_device *kbdev) @@ -4898,7 +4839,7 @@ static struct dentry *init_debugfs(struct kbase_device *kbdev) if (IS_ERR_OR_NULL(dentry)) return dentry; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { dentry = debugfs_create_file("protected_debug_mode", 0444, kbdev->mali_debugfs_directory, kbdev, &fops_protected_debug_mode); @@ -5785,11 +5726,11 @@ static int kbase_platform_device_probe(struct platform_device *pdev) #if (KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) mutex_unlock(&kbase_probe_mutex); #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT - mutex_lock(&kbdev->pm.lock); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT); - mutex_unlock(&kbdev->pm.lock); -#endif + if (kbase_has_arbiter(kbdev)) { + mutex_lock(&kbdev->pm.lock); + kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_INITIALIZED_EVT); + mutex_unlock(&kbdev->pm.lock); + } } return err; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h index 0c794e2e90bc..e6222979b72c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,9 +30,6 @@ */ static inline void mali_kbase_print_cs_experimental(void) { -#if MALI_INCREMENTAL_RENDERING_JM - pr_info("mali_kbase: INCREMENTAL_RENDERING_JM (experimental) enabled"); -#endif /* MALI_INCREMENTAL_RENDERING_JM */ } #endif /* _KBASE_CS_EXPERIMENTAL_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c index c92fb9e0957e..4b7f6a186ac0 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_debug_mem_allocs.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -58,7 +58,7 @@ static void debug_zone_mem_allocs_show(struct kbase_reg_zone *zone, struct seq_f for (p = rb_first(rbtree); p; p = rb_next(p)) { reg = rb_entry(p, struct kbase_va_region, rblink); if (!(reg->flags & KBASE_REG_FREE)) { - seq_printf(sfile, "%16llx, %16zx, %16zx, %8lx, %s\n", + seq_printf(sfile, "%16llx, %16zx, %16zx, %8llx, %s\n", reg->start_pfn << PAGE_SHIFT, reg->nr_pages << PAGE_SHIFT, kbase_reg_current_backed_size(reg) << PAGE_SHIFT, reg->flags, type_names[reg->gpu_alloc->type]); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_defs.h b/drivers/gpu/arm/bifrost/mali_kbase_defs.h index 2335e0b8e449..b97df15f7a17 100755 --- a/drivers/gpu/arm/bifrost/mali_kbase_defs.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_defs.h @@ -28,8 +28,8 @@ #define _KBASE_DEFS_H_ #include -#include -#include +#include +#include #include #include #include @@ -52,10 +52,6 @@ #include -#if IS_ENABLED(CONFIG_DEBUG_FS) -#include -#endif /* CONFIG_DEBUG_FS */ - #ifdef CONFIG_MALI_BIFROST_DEVFREQ #include #endif /* CONFIG_MALI_BIFROST_DEVFREQ */ @@ -64,9 +60,7 @@ #include #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #include #include @@ -78,6 +72,7 @@ #include #include #include +#include #include /** Number of milliseconds before we time out on a GPU soft/hard reset */ @@ -297,24 +292,33 @@ struct kbase_fault { #define MAX_PAGES_FOR_FREE_PGDS ((size_t)9) /* Maximum number of pointers to free PGDs */ -#define MAX_FREE_PGDS ((PAGE_SIZE / sizeof(struct page *)) * MAX_PAGES_FOR_FREE_PGDS) +#define MAX_FREE_PGDS ((PAGE_SIZE / sizeof(phys_addr_t)) * MAX_PAGES_FOR_FREE_PGDS) /** * struct kbase_mmu_table - object representing a set of GPU page tables - * @mmu_lock: Lock to serialize the accesses made to multi level GPU - * page tables - * @pgd: Physical address of the page allocated for the top - * level page table of the context, this is used for - * MMU HW programming as the address translation will - * start from the top level page table. - * @group_id: A memory group ID to be passed to a platform-specific - * memory group manager. - * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). - * @kctx: If this set of MMU tables belongs to a context then - * this is a back-reference to the context, otherwise - * it is NULL. - * @scratch_mem: Scratch memory used for MMU operations, which are - * serialized by the @mmu_lock. + * @mmu_lock: Lock to serialize the accesses made to multi level GPU + * page tables + * @pgd: Physical address of the page allocated for the top + * level page table of the context, this is used for + * MMU HW programming as the address translation will + * start from the top level page table. + * @group_id: A memory group ID to be passed to a platform-specific + * memory group manager. + * Valid range is 0..(MEMORY_GROUP_MANAGER_NR_GROUPS-1). + * @kctx: If this set of MMU tables belongs to a context then + * this is a back-reference to the context, otherwise + * it is NULL. + * @scratch_mem: Scratch memory used for MMU operations, which are + * serialized by the @mmu_lock. + * @pgd_pages_list: List head to link all 16K/64K pages allocated for the PGDs of mmut. + * These pages will be used to allocate 4KB PGD pages for + * the GPU page table. + * Linked with &kbase_page_metadata.data.pt_mapped.pgd_link. + * @last_allocated_pgd_page: Pointer to PGD page from where the last sub page + * was allocated for mmut. + * @last_freed_pgd_page: Pointer to PGD page to which the last freed 4K sub page + * was returned for mmut. + * @num_free_pgd_sub_pages: The total number of free 4K PGD pages in the mmut. */ struct kbase_mmu_table { struct mutex mmu_lock; @@ -332,7 +336,7 @@ struct kbase_mmu_table { * @levels: Array of PGD pages, large enough to copy one PGD * for each level of the MMU table. */ - u64 levels[MIDGARD_MMU_BOTTOMLEVEL][PAGE_SIZE / sizeof(u64)]; + u64 levels[MIDGARD_MMU_BOTTOMLEVEL][GPU_PAGE_SIZE / sizeof(u64)]; } teardown_pages; /** * @free_pgds: Scratch memory used for insertion, update and teardown @@ -341,11 +345,18 @@ struct kbase_mmu_table { */ struct { /** @pgds: Array of pointers to PGDs to free. */ - struct page *pgds[MAX_FREE_PGDS]; + phys_addr_t pgds[MAX_FREE_PGDS]; /** @head_index: Index of first free element in the PGDs array. */ size_t head_index; } free_pgds; } scratch_mem; + +#if GPU_PAGES_PER_CPU_PAGE > 1 + struct list_head pgd_pages_list; + struct page *last_allocated_pgd_page; + struct page *last_freed_pgd_page; + u32 num_free_pgd_sub_pages; +#endif }; #if MALI_USE_CSF @@ -371,14 +382,9 @@ static inline int kbase_as_has_page_fault(struct kbase_as *as, struct kbase_faul * * @used_pages: Tracks usage of OS shared memory. Updated when OS memory is * allocated/freed. - * @ir_threshold: Fraction of the maximum size of an allocation that grows - * on GPU page fault that can be used before the driver - * switches to incremental rendering, in 1/256ths. - * 0 means disabled. */ struct kbasep_mem_device { atomic_t used_pages; - atomic_t ir_threshold; }; struct kbase_clk_rate_listener; @@ -483,9 +489,7 @@ struct kbase_pm_device_data { #if MALI_USE_CSF bool runtime_active; #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT atomic_t gpu_lost; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ wait_queue_head_t zero_active_count_wait; wait_queue_head_t resume_wait; @@ -501,10 +505,8 @@ struct kbase_pm_device_data { void (*callback_power_runtime_term)(struct kbase_device *kbdev); u32 dvfs_period; struct kbase_pm_backend_data backend; -#ifdef CONFIG_MALI_ARBITER_SUPPORT struct kbase_arbiter_vm_state *arb_vm_state; atomic_t gpu_users_waiting; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ struct kbase_clk_rate_trace_manager clk_rtm; }; @@ -947,7 +949,7 @@ struct kbase_mem_migrate { * @ipa.last_sample_time: Records the time when counters, used for dynamic * energy estimation, were last sampled. * @previous_frequency: Previous frequency of GPU clock used for - * BASE_HW_ISSUE_GPU2017_1336 workaround, This clock is + * KBASE_HW_ISSUE_GPU2017_1336 workaround, This clock is * restored when L2 is powered on. * @job_fault_debug: Flag to control the dumping of debug data for job faults, * set when the 'job_fault' debugfs file is opened. @@ -1069,7 +1071,8 @@ struct kbase_mem_migrate { * KCPU queue. These structures may outlive kbase module * itself. Therefore, in such a case, a warning should be * be produced. - * @va_region_slab: kmem_cache (slab) for allocated kbase_va_region structures. + * @va_region_slab: kmem_cache (slab) for allocated @kbase_va_region structures. + * @page_metadata_slab: kmem_cache (slab) for allocated @kbase_page_metadata structures. * @fence_signal_timeout_enabled: Global flag for whether fence signal timeout tracking * is enabled. * @pcm_prioritized_process_nb: Notifier block for the Priority Control Manager @@ -1134,8 +1137,8 @@ struct kbase_device { struct kbase_gpu_props gpu_props; - unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; - unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + unsigned long hw_issues_mask[(KBASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; + unsigned long hw_features_mask[(KBASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG]; struct { atomic_t count; @@ -1151,6 +1154,12 @@ struct kbase_device { */ u8 pbha_propagate_bits; + /** + * @mma_wa_id: The PBHA ID to use for the PBHA OVERRIDE based workaround for MMA violation. + * + */ + u32 mma_wa_id; + #if MALI_USE_CSF struct kbase_hwcnt_backend_csf_if hwcnt_backend_csf_if_fw; #else @@ -1242,7 +1251,6 @@ struct kbase_device { atomic_t job_fault_debug; #endif /* !MALI_USE_CSF */ -#if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *mali_debugfs_directory; struct dentry *debugfs_ctx_directory; struct dentry *debugfs_instr_directory; @@ -1264,7 +1272,6 @@ struct kbase_device { u32 reg_offset; } regs_dump_debugfs_data; #endif /* !MALI_CUSTOMER_RELEASE */ -#endif /* CONFIG_DEBUG_FS */ atomic_t ctx_num; @@ -1355,9 +1362,7 @@ struct kbase_device { } dummy_job_wa; bool dummy_job_wa_loaded; -#ifdef CONFIG_MALI_ARBITER_SUPPORT struct kbase_arbiter_device arb; -#endif /* Priority Control Manager device */ struct priority_control_manager_device *pcm_dev; @@ -1382,6 +1387,9 @@ struct kbase_device { atomic_t live_fence_metadata; #endif struct kmem_cache *va_region_slab; +#if GPU_PAGES_PER_CPU_PAGE > 1 + struct kmem_cache *page_metadata_slab; +#endif #if IS_ENABLED(CONFIG_MALI_TRACE_POWER_GPU_WORK_PERIOD) /** @@ -2131,6 +2139,18 @@ static inline u64 kbase_get_lock_region_min_size_log2(struct kbase_gpu_props con return 15; /* 32 kB */ } +/** + * kbase_has_arbiter - Check whether GPU has an arbiter. + * + * @kbdev: KBase device. + * + * Return: True if there is an arbiter, False otherwise. + */ +static inline bool kbase_has_arbiter(struct kbase_device *kbdev) +{ + return (bool)kbdev->arb.arb_if; +} + /* Conversion helpers for setting up high resolution timers */ #define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x)) * 1000000U)) #define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x)) diff --git a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c index 9c39f0e20f76..7b578c81af60 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -218,7 +218,7 @@ static bool wa_blob_load_needed(struct kbase_device *kbdev) if (of_machine_is_compatible("arm,juno")) return false; - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3485)) + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_TTRX_3485)) return true; return false; @@ -311,7 +311,7 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev) while (blob_offset) { const struct wa_blob *blob; size_t nr_pages; - u64 flags; + base_mem_alloc_flags flags; u64 gpu_va; struct kbase_va_region *va_region; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c index 8a2e13f03683..60ad1c272f84 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpu_metrics.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -39,9 +39,6 @@ static inline void validate_tracepoint_data(struct kbase_gpu_metrics_ctx *gpu_me u64 start_time, u64 end_time, u64 total_active) { #if 0 - WARN(total_active > NSEC_PER_SEC, "total_active %llu > 1 second for aid %u active_cnt %u", - total_active, gpu_metrics_ctx->aid, gpu_metrics_ctx->active_cnt); - WARN(start_time >= end_time, "start_time %llu >= end_time %llu for aid %u active_cnt %u", start_time, end_time, gpu_metrics_ctx->aid, gpu_metrics_ctx->active_cnt); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c index 25ee8c1042a2..9719580837cc 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2011-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -437,7 +437,7 @@ int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) { int err = 0; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_L2_CONFIG)) { struct kbasep_gpuprops_regdump *regdump = &PRIV_DATA_REGDUMP(kbdev); /* Check for L2 cache size & hash overrides */ @@ -699,7 +699,7 @@ static void kbase_populate_user_data(struct kbase_device *kbdev, struct gpu_prop data->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; } - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_GROUP_SPLIT)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_THREAD_GROUP_SPLIT)) data->thread_props.max_thread_group_split = 0; /* Raw Register Values */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c index 5e59bf60aa38..99558b82ba7b 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_gwt.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_gwt.c @@ -178,6 +178,10 @@ int kbase_gpu_gwt_dump(struct kbase_context *kctx, union kbase_ioctl_cinstr_gwt_ __user void *user_addr = (__user void *)(uintptr_t)gwt_dump->in.addr_buffer; __user void *user_sizes = (__user void *)(uintptr_t)gwt_dump->in.size_buffer; + /* We don't have any valid user space buffer to copy the write modified addresses. */ + if (!gwt_dump->in.len || !gwt_dump->in.addr_buffer || !gwt_dump->in.size_buffer) + return -EINVAL; + kbase_gpu_vm_lock(kctx); if (!kctx->gwt_enabled) { @@ -186,14 +190,6 @@ int kbase_gpu_gwt_dump(struct kbase_context *kctx, union kbase_ioctl_cinstr_gwt_ return -EPERM; } - if (!gwt_dump->in.len || !gwt_dump->in.addr_buffer || !gwt_dump->in.size_buffer) { - kbase_gpu_vm_unlock(kctx); - /* We don't have any valid user space buffer to copy the - * write modified addresses. - */ - return -EINVAL; - } - if (list_empty(&kctx->gwt_snapshot_list) && !list_empty(&kctx->gwt_current_list)) { list_replace_init(&kctx->gwt_current_list, &kctx->gwt_snapshot_list); @@ -227,14 +223,14 @@ int kbase_gpu_gwt_dump(struct kbase_context *kctx, union kbase_ioctl_cinstr_gwt_ if (count) { err = copy_to_user((user_addr + (ubuf_count * sizeof(u64))), - (void *)addr_buffer, count * sizeof(u64)); + (void *)addr_buffer, size_mul(count, sizeof(u64))); if (err) { dev_err(kctx->kbdev->dev, "Copy to user failure\n"); kbase_gpu_vm_unlock(kctx); return err; } err = copy_to_user((user_sizes + (ubuf_count * sizeof(u64))), - (void *)num_page_buffer, count * sizeof(u64)); + (void *)num_page_buffer, size_mul(count, sizeof(u64))); if (err) { dev_err(kctx->kbdev->dev, "Copy to user failure\n"); kbase_gpu_vm_unlock(kctx); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.c b/drivers/gpu/arm/bifrost/mali_kbase_hw.c index 1fde75b996c4..e04aad2422c7 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hw.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.c @@ -23,8 +23,8 @@ * Run-time work-arounds helpers */ -#include -#include +#include +#include #include #include "mali_kbase.h" #include "mali_kbase_hw.h" @@ -92,7 +92,7 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) break; } - for (; *features != BASE_HW_FEATURE_END; features++) + for (; *features != KBASE_HW_FEATURE_END; features++) set_bit(*features, &kbdev->hw_features_mask[0]); #if defined(CONFIG_MALI_VECTOR_DUMP) @@ -103,8 +103,8 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) * in the implementation of flush reduction optimization due to * unclear or ambiguous ARCH spec. */ - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_CLEAN_ONLY_SAFE)) - clear_bit(BASE_HW_FEATURE_FLUSH_REDUCTION, &kbdev->hw_features_mask[0]); + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE)) + clear_bit(KBASE_HW_FEATURE_FLUSH_REDUCTION, &kbdev->hw_features_mask[0]); #endif } @@ -113,7 +113,7 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) * @kbdev: Device pointer * * Return: pointer to an array of hardware issues, terminated by - * BASE_HW_ISSUE_END. + * KBASE_HW_ISSUE_END. * * In debugging versions of the driver, unknown versions of a known GPU will * be treated as the most recent known version not later than the actual @@ -424,7 +424,7 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) gpu_id->product_major, gpu_id->arch_major, gpu_id->arch_minor, gpu_id->arch_rev, gpu_id->version_major, gpu_id->version_minor, gpu_id->version_status); - for (; *issues != BASE_HW_ISSUE_END; issues++) + for (; *issues != KBASE_HW_ISSUE_END; issues++) set_bit(*issues, &kbdev->hw_issues_mask[0]); return 0; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hw.h b/drivers/gpu/arm/bifrost/mali_kbase_hw.h index 44e1ee4a4a50..f14e5fb6d9ab 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_hw.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2012-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2012-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -23,8 +23,8 @@ * DOC: Run-time work-arounds helpers */ -#ifndef _KBASE_HW_H_ -#define _KBASE_HW_H_ +#ifndef _MALI_KBASE_HW_H_ +#define _MALI_KBASE_HW_H_ #include "mali_kbase_defs.h" @@ -47,7 +47,7 @@ * @kbdev: Device pointer */ #define kbase_hw_has_l2_slice_hash_feature(kbdev) \ - test_bit(BASE_HW_FEATURE_L2_SLICE_HASH, &(kbdev)->hw_features_mask[0]) + test_bit(KBASE_HW_FEATURE_L2_SLICE_HASH, &(kbdev)->hw_features_mask[0]) /** * kbase_hw_set_issues_mask - Set the hardware issues mask based on the GPU ID @@ -73,4 +73,4 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev); */ void kbase_hw_set_features_mask(struct kbase_device *kbdev); -#endif /* _KBASE_HW_H_ */ +#endif /* _MALI_KBASE_HW_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h new file mode 100644 index 000000000000..265cb9585cc6 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_features.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_HWCONFIG_FEATURES_H_ +#define _KBASE_HWCONFIG_FEATURES_H_ + +#include + +enum base_hw_feature { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_TLS_HASHING, + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_L2_SLICE_HASH, + KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + KBASE_HW_FEATURE_CORE_FEATURES, + KBASE_HW_FEATURE_PBHA_HWU, + KBASE_HW_FEATURE_LARGE_PAGE_ALLOC, + KBASE_HW_FEATURE_THREAD_TLS_ALLOC, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_generic[] = { + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tMIx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tHEx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tSIx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tDVx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tNOx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_TLS_HASHING, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tGOx[] = { + KBASE_HW_FEATURE_THREAD_GROUP_SPLIT, KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, KBASE_HW_FEATURE_TLS_HASHING, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_CORE_FEATURES, + KBASE_HW_FEATURE_THREAD_TLS_ALLOC, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tTRx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tNAx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tBEx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tBAx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_IDVS_GROUP_SIZE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_FLUSH_INV_SHADER_OTHER, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tODx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tGRx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tVAx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tTUx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_L2_SLICE_HASH, KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tTIx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, + KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, + KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_L2_SLICE_HASH, + KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_CORE_FEATURES, + KBASE_HW_FEATURE_PBHA_HWU, + KBASE_HW_FEATURE_END +}; + +__maybe_unused static const enum base_hw_feature base_hw_features_tKRx[] = { + KBASE_HW_FEATURE_FLUSH_REDUCTION, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE, + KBASE_HW_FEATURE_L2_CONFIG, KBASE_HW_FEATURE_CLEAN_ONLY_SAFE, + KBASE_HW_FEATURE_L2_SLICE_HASH, KBASE_HW_FEATURE_GPU_SLEEP, + KBASE_HW_FEATURE_CORE_FEATURES, KBASE_HW_FEATURE_PBHA_HWU, + KBASE_HW_FEATURE_LARGE_PAGE_ALLOC, KBASE_HW_FEATURE_END +}; + + +#endif /* _KBASE_HWCONFIG_FEATURES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h new file mode 100644 index 000000000000..b1a3a41b232b --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_hwconfig_issues.h @@ -0,0 +1,609 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_HWCONFIG_ISSUES_H_ +#define _KBASE_HWCONFIG_ISSUES_H_ + +#include + +enum base_hw_issue { + KBASE_HW_ISSUE_5736, + KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_10682, + KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_T76X_3953, + KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_7940, + KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TMIX_8138, + KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, + KBASE_HW_ISSUE_TMIX_8463, + KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TMIX_8438, + KBASE_HW_ISSUE_TNOX_1194, + KBASE_HW_ISSUE_TGOX_R1_1234, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TSIX_1792, + KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, + KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_GPU2019_3212, + KBASE_HW_ISSUE_TURSEHW_1997, + KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2019_3901, + KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TITANHW_2952, + KBASE_HW_ISSUE_KRAKEHW_2151, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2269, + KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_generic[] = { KBASE_HW_ISSUE_END }; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0_05dev0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_T76X_3953, KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, KBASE_HW_ISSUE_TMIX_8463, KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TMIX_8438, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_7940, KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, KBASE_HW_ISSUE_TMIX_8463, KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TMIX_8438, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tMIx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_7940, KBASE_HW_ISSUE_TMIX_8042, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, + KBASE_HW_ISSUE_TMIX_8343, KBASE_HW_ISSUE_TMIX_8463, KBASE_HW_ISSUE_TMIX_8456, + KBASE_HW_ISSUE_TMIX_8438, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tMIx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_7940, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TMIX_8138, KBASE_HW_ISSUE_TMIX_8206, KBASE_HW_ISSUE_TMIX_8343, + KBASE_HW_ISSUE_TMIX_8456, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p2[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_11054, + KBASE_HW_ISSUE_TMIX_7891, KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tHEx_r0p3[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_10682, KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tHEx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_7891, + KBASE_HW_ISSUE_TMIX_8042, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_11054, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TSIX_1792, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_11054, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TSIX_1792, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_11054, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tSIx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tDVx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TNOX_1194, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNOx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TNOX_1194, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TMIX_8133, + KBASE_HW_ISSUE_TSIX_1116, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TGOX_R1_1234, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_GPU2017_1336, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGOx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TMIX_8133, KBASE_HW_ISSUE_TSIX_1116, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = { + KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, + KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTRx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = { + KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_3076, + KBASE_HW_ISSUE_TTRX_921, + KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_GPU2017_1336, + KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tNAx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBEx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TTRX_3485, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p1[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tBAx_r0p2[] = { + KBASE_HW_ISSUE_9435, KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TTRX_2968_TTRX_3162, + KBASE_HW_ISSUE_TTRX_921, KBASE_HW_ISSUE_TTRX_3414, + KBASE_HW_ISSUE_TTRX_3083, KBASE_HW_ISSUE_TTRX_3470, + KBASE_HW_ISSUE_TTRX_3464, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tBAx[] = { + KBASE_HW_ISSUE_5736, KBASE_HW_ISSUE_9435, + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TTRX_3414, KBASE_HW_ISSUE_TTRX_3083, + KBASE_HW_ISSUE_TTRX_3470, KBASE_HW_ISSUE_TTRX_3464, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_GPU2019_3212, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tODx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_GPU2019_3212, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tGRx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tVAx_r0p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tVAx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_1997, + KBASE_HW_ISSUE_GPU2019_3878, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, + KBASE_HW_ISSUE_GPU2021PRO_290, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r0p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_1997, + KBASE_HW_ISSUE_GPU2019_3878, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, + KBASE_HW_ISSUE_GPU2021PRO_290, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_TURSEHW_2934, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTUx[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p0[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p2[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTUx_r1p3[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_GPU2019_3878, + KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2019_3901, KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tTIx[] = { + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TITANHW_2952, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p0[] = { + KBASE_HW_ISSUE_TSIX_2033, + KBASE_HW_ISSUE_TTRX_1337, + KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2021PRO_290, + KBASE_HW_ISSUE_TITANHW_2710, + KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TITANHW_2952, + KBASE_HW_ISSUE_TITANHW_2938, + KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tTIx_r0p1[] = { + KBASE_HW_ISSUE_TSIX_2033, KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, + KBASE_HW_ISSUE_GPU2021PRO_290, KBASE_HW_ISSUE_TITANHW_2710, KBASE_HW_ISSUE_TITANHW_2679, + KBASE_HW_ISSUE_GPU2022PRO_148, KBASE_HW_ISSUE_TITANHW_2938, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p0[] = { + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_KRAKEHW_2151, KBASE_HW_ISSUE_KRAKEHW_2269, KBASE_HW_ISSUE_TITANHW_2922, + KBASE_HW_ISSUE_TURSEHW_2934, KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_tKRx_r0p1[] = { + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_KRAKEHW_2269, KBASE_HW_ISSUE_TURSEHW_2934, KBASE_HW_ISSUE_KRAKEHW_2321, + KBASE_HW_ISSUE_END +}; + +__maybe_unused static const enum base_hw_issue base_hw_issues_model_tKRx[] = { + KBASE_HW_ISSUE_TTRX_1337, KBASE_HW_ISSUE_TURSEHW_2716, KBASE_HW_ISSUE_GPU2022PRO_148, + KBASE_HW_ISSUE_KRAKEHW_2151, KBASE_HW_ISSUE_KRAKEHW_2269, KBASE_HW_ISSUE_TURSEHW_2934, + KBASE_HW_ISSUE_KRAKEHW_2321, KBASE_HW_ISSUE_END +}; + + +#endif /* _KBASE_HWCONFIG_ISSUES_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h b/drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h new file mode 100644 index 000000000000..e87925bab9b0 --- /dev/null +++ b/drivers/gpu/arm/bifrost/mali_kbase_ioctl_helpers.h @@ -0,0 +1,542 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * (C) COPYRIGHT 2024 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef _KBASE_IOCTL_HELPERS_H_ +#define _KBASE_IOCTL_HELPERS_H_ + +#include + +/* Macro for IOCTLs that don't have IOCTL struct */ +#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ + do { \ + int ret; \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + ret = function(arg); \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Macro for IOCTLs that have input IOCTL struct */ +#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + err = check_padding_##cmd(¶m); \ + if (unlikely(err)) \ + return -EINVAL; \ + ret = function(arg, ¶m); \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Macro for IOCTLs that have output IOCTL struct */ +#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + memset(¶m, 0, sizeof(param)); \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Macro for IOCTLs that have input and output IOCTL struct */ +#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ + do { \ + type param; \ + int ret, err; \ + dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ + BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE | _IOC_READ)); \ + BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ + err = copy_from_user(¶m, uarg, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + err = check_padding_##cmd(¶m); \ + if (unlikely(err)) \ + return -EINVAL; \ + ret = function(arg, ¶m); \ + err = copy_to_user(uarg, ¶m, sizeof(param)); \ + if (unlikely(err)) \ + return -EFAULT; \ + dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, #function); \ + return ret; \ + } while (0) + +/* Inline functions to check padding bytes in the input IOCTL struct. + * Return 0 if all padding bytes are zero, non-zero otherwise. + */ +static inline int check_padding_KBASE_IOCTL_VERSION_CHECK(struct kbase_ioctl_version_check *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_VERSION_CHECK_RESERVED(struct kbase_ioctl_version_check *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_SET_FLAGS(struct kbase_ioctl_set_flags *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_GET_GPUPROPS(struct kbase_ioctl_get_gpuprops *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_ALLOC(union kbase_ioctl_mem_alloc *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_QUERY(union kbase_ioctl_mem_query *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_FREE(struct kbase_ioctl_mem_free *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_HWCNT_READER_SETUP(struct kbase_ioctl_hwcnt_reader_setup *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_HWCNT_SET(struct kbase_ioctl_hwcnt_values *p) +{ + return p->padding; +} + +static inline int check_padding_KBASE_IOCTL_GET_DDK_VERSION(struct kbase_ioctl_get_ddk_version *p) +{ + return p->padding; +} + +static inline int check_padding_KBASE_IOCTL_MEM_JIT_INIT(struct kbase_ioctl_mem_jit_init *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_SYNC(struct kbase_ioctl_mem_sync *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_MEM_FIND_CPU_OFFSET(union kbase_ioctl_mem_find_cpu_offset *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_TLSTREAM_ACQUIRE(struct kbase_ioctl_tlstream_acquire *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_COMMIT(struct kbase_ioctl_mem_commit *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_ALIAS(union kbase_ioctl_mem_alias *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_IMPORT(union kbase_ioctl_mem_import *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_FLAGS_CHANGE(struct kbase_ioctl_mem_flags_change *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_STREAM_CREATE(struct kbase_ioctl_stream_create *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_FENCE_VALIDATE(struct kbase_ioctl_fence_validate *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_PROFILE_ADD(struct kbase_ioctl_mem_profile_add *p) +{ + return p->padding; +} + +static inline int +check_padding_KBASE_IOCTL_STICKY_RESOURCE_MAP(struct kbase_ioctl_sticky_resource_map *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_STICKY_RESOURCE_UNMAP(struct kbase_ioctl_sticky_resource_unmap *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET( + union kbase_ioctl_mem_find_gpu_start_and_offset *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CINSTR_GWT_DUMP(union kbase_ioctl_cinstr_gwt_dump *p) +{ + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + p->in.padding = 0; + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_EXEC_INIT(struct kbase_ioctl_mem_exec_init *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_GET_CPU_GPU_TIMEINFO(union kbase_ioctl_get_cpu_gpu_timeinfo *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.paddings); i++) { + if (p->in.paddings[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CONTEXT_PRIORITY_CHECK(struct kbase_ioctl_context_priority_check *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_SET_LIMITED_CORE_COUNT(struct kbase_ioctl_set_limited_core_count *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KINSTR_PRFCNT_ENUM_INFO(struct kbase_ioctl_kinstr_prfcnt_enum_info *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KINSTR_PRFCNT_SETUP(union kbase_ioctl_kinstr_prfcnt_setup *p) +{ + return 0; +} + +#if MALI_UNIT_TEST +#endif /* MALI_UNIT_TEST */ + +#if MALI_USE_CSF + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_REGISTER(struct kbase_ioctl_cs_queue_register *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_QUEUE_KICK(struct kbase_ioctl_cs_queue_kick *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_QUEUE_BIND(union kbase_ioctl_cs_queue_bind *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) + p->in.padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_REGISTER_EX(struct kbase_ioctl_cs_queue_register_ex *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + for (i = 0; i < ARRAY_SIZE(p->ex_padding); i++) { + if (p->ex_padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_TERMINATE(struct kbase_ioctl_cs_queue_terminate *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_CREATE_1_6(union kbase_ioctl_cs_queue_group_create_1_6 *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_CREATE_1_18( + union kbase_ioctl_cs_queue_group_create_1_18 *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_CREATE(union kbase_ioctl_cs_queue_group_create *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE(struct kbase_ioctl_cs_queue_group_term *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->padding); i++) { + if (p->padding[i]) + return -1; + } + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KCPU_QUEUE_DELETE(struct kbase_ioctl_kcpu_queue_delete *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_KCPU_QUEUE_ENQUEUE(struct kbase_ioctl_kcpu_queue_enqueue *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_TILER_HEAP_INIT(union kbase_ioctl_cs_tiler_heap_init *p) +{ + return p->in.padding; +} + +static inline int +check_padding_KBASE_IOCTL_CS_TILER_HEAP_INIT_1_13(union kbase_ioctl_cs_tiler_heap_init_1_13 *p) +{ + return p->in.padding; +} + +static inline int +check_padding_KBASE_IOCTL_CS_TILER_HEAP_TERM(struct kbase_ioctl_cs_tiler_heap_term *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_CS_GET_GLB_IFACE(union kbase_ioctl_cs_get_glb_iface *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_CS_CPU_QUEUE_DUMP(struct kbase_ioctl_cs_cpu_queue_info *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_MEM_ALLOC_EX(union kbase_ioctl_mem_alloc_ex *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.extra); i++) { + if (p->in.extra[i]) + return -1; + } + + return 0; +} + +static inline int check_padding_KBASE_IOCTL_READ_USER_PAGE(union kbase_ioctl_read_user_page *p) +{ + return p->in.padding; +} + +static inline int +check_padding_KBASE_IOCTL_QUEUE_GROUP_CLEAR_FAULTS(struct kbase_ioctl_queue_group_clear_faults *p) +{ + size_t i; + + /* + * Checking p->padding is deferred till the support window for backward-compatibility ends. + * GPUCORE-42000 will add the checking. + * + * To avoid the situation with old version of base which might not set padding bytes as 0, + * padding bytes are set as zero here on behalf on user space. + */ + for (i = 0; i < ARRAY_SIZE(p->padding); i++) + p->padding[i] = 0; + + return 0; +} + +#else /* MALI_USE_CSF */ + +static inline int check_padding_KBASE_IOCTL_JOB_SUBMIT(struct kbase_ioctl_job_submit *p) +{ + return 0; +} + +static inline int +check_padding_KBASE_IOCTL_SOFT_EVENT_UPDATE(struct kbase_ioctl_soft_event_update *p) +{ + return 0; +} + +static inline int check_padding_KBASE_IOCTL_KINSTR_JM_FD(union kbase_kinstr_jm_fd *p) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(p->in.padding); i++) { + if (p->in.padding[i]) + return -1; + } + + return 0; +} + +#endif /* !MALI_USE_CSF */ + +#endif /* _KBASE_IOCTL_HELPERS_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_jd.c b/drivers/gpu/arm/bifrost/mali_kbase_jd.c index 418a1913b241..4da7fa377bd7 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_jd.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_jd.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -39,9 +39,6 @@ #include #include #include - -#include - #include /* Return whether katom will run on the GPU or not. Currently only soft jobs and @@ -209,7 +206,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, } if (copy_from_user(input_extres, get_compat_pointer(katom->kctx, user_atom->extres_list), - sizeof(*input_extres) * katom->nr_extres) != 0) { + size_mul(sizeof(*input_extres), katom->nr_extres)) != 0) { err = -EINVAL; goto failed_input_copy; } @@ -697,7 +694,6 @@ static void jd_trace_atom_submit(struct kbase_context *const kctx, static bool jd_submit_atom(struct kbase_context *const kctx, const struct base_jd_atom *const user_atom, - const struct base_jd_fragment *const user_jc_incr, struct kbase_jd_atom *const katom) { struct kbase_device *kbdev = kctx->kbdev; @@ -755,8 +751,6 @@ static bool jd_submit_atom(struct kbase_context *const kctx, } #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - katom->renderpass_id = user_atom->renderpass_id; - /* Implicitly sets katom->protected_state.enter as well. */ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; @@ -875,20 +869,7 @@ static bool jd_submit_atom(struct kbase_context *const kctx, /* Create a new atom. */ jd_trace_atom_submit(kctx, katom, &katom->sched_priority); -#if !MALI_INCREMENTAL_RENDERING_JM - /* Reject atoms for incremental rendering if not supported */ - if (katom->core_req & (BASE_JD_REQ_START_RENDERPASS | BASE_JD_REQ_END_RENDERPASS)) { - dev_err(kctx->kbdev->dev, "Rejecting atom with unsupported core_req 0x%x\n", - katom->core_req); - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return kbase_jd_done_nolock(katom, true); - } -#endif /* !MALI_INCREMENTAL_RENDERING_JM */ - - if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) { - WARN_ON(katom->jc != 0); - katom->jc_fragment = *user_jc_incr; - } else if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { + if (!katom->jc && (katom->core_req & BASE_JD_REQ_ATOM_TYPE) != BASE_JD_REQ_DEP) { /* Reject atoms with job chain = NULL, as these cause issues * with soft-stop */ @@ -1018,8 +999,7 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a struct kbase_device *kbdev; u32 latest_flush; - bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2) || - stride == offsetof(struct base_jd_atom_v2, renderpass_id)); + bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2)); CSTD_UNUSED(uk6_atom); @@ -1035,10 +1015,7 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a return -EINVAL; } - if (stride != offsetof(struct base_jd_atom_v2, renderpass_id) && - stride != sizeof(struct base_jd_atom_v2) && - stride != offsetof(struct base_jd_atom, renderpass_id) && - stride != sizeof(struct base_jd_atom)) { + if (stride != sizeof(struct base_jd_atom_v2) && stride != sizeof(struct base_jd_atom)) { dev_err(kbdev->dev, "Stride %u passed to job_submit isn't supported by the kernel\n", stride); return -EINVAL; @@ -1057,7 +1034,6 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a struct base_jd_atom user_atom = { .seq_nr = 0, }; - struct base_jd_fragment user_jc_incr; struct kbase_jd_atom *katom; if (unlikely(jd_atom_is_v2)) { @@ -1082,44 +1058,6 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a } } - if (stride == offsetof(struct base_jd_atom_v2, renderpass_id)) { - dev_dbg(kbdev->dev, "No renderpass ID: use 0\n"); - user_atom.renderpass_id = 0; - } else { - /* Ensure all padding bytes are 0 for potential future - * extension - */ - size_t j; - - dev_dbg(kbdev->dev, "Renderpass ID is %d\n", user_atom.renderpass_id); - for (j = 0; j < sizeof(user_atom.padding); j++) { - if (user_atom.padding[j]) { - dev_err(kbdev->dev, "Bad padding byte %zu: %d\n", j, - user_atom.padding[j]); - err = -EINVAL; - break; - } - } - if (err) - break; - } - - /* In this case 'jc' is the CPU address of a struct - * instead of a GPU address of a job chain. - */ - if (user_atom.core_req & BASE_JD_REQ_END_RENDERPASS) { - if (copy_from_user(&user_jc_incr, u64_to_user_ptr(user_atom.jc), - sizeof(user_jc_incr))) { - dev_err(kbdev->dev, - "Invalid jc address 0x%llx passed to job_submit\n", - user_atom.jc); - err = -EFAULT; - break; - } - dev_dbg(kbdev->dev, "Copied IR jobchain addresses\n"); - user_atom.jc = 0; - } - user_addr = (void __user *)((uintptr_t)user_addr + stride); mutex_lock(&jctx->lock); @@ -1172,8 +1110,7 @@ int kbase_jd_submit(struct kbase_context *kctx, void __user *user_addr, u32 nr_a mutex_lock(&jctx->lock); } KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_START(kbdev, katom); - need_to_try_schedule_context |= - jd_submit_atom(kctx, &user_atom, &user_jc_incr, katom); + need_to_try_schedule_context |= jd_submit_atom(kctx, &user_atom, katom); KBASE_TLSTREAM_TL_JD_SUBMIT_ATOM_END(kbdev, katom); /* Register a completed job as a disjoint event when the GPU is in a disjoint state * (ie. being reset). @@ -1579,9 +1516,6 @@ int kbase_jd_init(struct kbase_context *kctx) #endif } - for (i = 0; i < BASE_JD_RP_COUNT; i++) - kctx->jctx.renderpasses[i].state = KBASE_JD_RP_COMPLETE; - mutex_init(&kctx->jctx.lock); init_waitqueue_head(&kctx->jctx.zero_jobs_wait); diff --git a/drivers/gpu/arm/bifrost/mali_kbase_js.c b/drivers/gpu/arm/bifrost/mali_kbase_js.c index 3e6b6b5eb066..d42fde37db2a 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_js.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_js.c @@ -333,19 +333,6 @@ static void jsctx_queue_foreach_prio(struct kbase_context *kctx, unsigned int js rb_erase(node, &queue->runnable_tree); callback(kctx->kbdev, entry); - - /* Runnable end-of-renderpass atoms can also be in the linked - * list of atoms blocked on cross-slot dependencies. Remove them - * to avoid calling the callback twice. - */ - if (entry->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST) { - WARN_ON(!(entry->core_req & BASE_JD_REQ_END_RENDERPASS)); - dev_dbg(kctx->kbdev->dev, "Del runnable atom %pK from X_DEP list\n", - (void *)entry); - - list_del(&entry->queue); - entry->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - } } while (!list_empty(&queue->x_dep_head)) { @@ -1230,7 +1217,7 @@ static bool kbase_js_ctx_pullable(struct kbase_context *kctx, unsigned int js, b dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_ctx_pullable\n", (void *)katom); return false; /* next atom blocked */ } - if (kbase_js_atom_blocked_on_x_dep(katom)) { + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || katom->x_pre_dep->will_fail_event_code) { dev_dbg(kbdev->dev, @@ -1371,9 +1358,6 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, struct kbase_jd_at (dep_atom->status != KBASE_JD_ATOM_STATE_UNUSED)) { katom->atom_flags |= KBASE_KATOM_FLAG_X_DEP_BLOCKED; - dev_dbg(kbdev->dev, "Set X_DEP flag on atom %pK\n", - (void *)katom); - katom->x_pre_dep = dep_atom; dep_atom->x_post_dep = katom; if (kbase_jd_katom_dep_type(&katom->dep[i]) == @@ -1447,110 +1431,12 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx) } KBASE_EXPORT_TEST_API(kbase_js_update_ctx_priority); -/** - * js_add_start_rp() - Add an atom that starts a renderpass to the job scheduler - * @start_katom: Pointer to the atom to be added. - * Return: 0 if successful or a negative value on failure. - */ -static int js_add_start_rp(struct kbase_jd_atom *const start_katom) -{ - struct kbase_context *const kctx = start_katom->kctx; - struct kbase_jd_renderpass *rp; - struct kbase_device *const kbdev = kctx->kbdev; - unsigned long flags; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) - return -EINVAL; - - if (start_katom->core_req & BASE_JD_REQ_END_RENDERPASS) - return -EINVAL; - - compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; - - if (rp->state != KBASE_JD_RP_COMPLETE) - return -EINVAL; - - dev_dbg(kctx->kbdev->dev, "JS add start atom %pK of RP %d\n", (void *)start_katom, - start_katom->renderpass_id); - - /* The following members are read when updating the job slot - * ringbuffer/fifo therefore they require additional locking. - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - rp->state = KBASE_JD_RP_START; - rp->start_katom = start_katom; - rp->end_katom = NULL; - INIT_LIST_HEAD(&rp->oom_reg_list); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - return 0; -} - -/** - * js_add_end_rp() - Add an atom that ends a renderpass to the job scheduler - * @end_katom: Pointer to the atom to be added. - * Return: 0 if successful or a negative value on failure. - */ -static int js_add_end_rp(struct kbase_jd_atom *const end_katom) -{ - struct kbase_context *const kctx = end_katom->kctx; - struct kbase_jd_renderpass *rp; - struct kbase_device *const kbdev = kctx->kbdev; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return -EINVAL; - - if (end_katom->core_req & BASE_JD_REQ_START_RENDERPASS) - return -EINVAL; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - dev_dbg(kbdev->dev, "JS add end atom %pK in state %d of RP %d\n", (void *)end_katom, - (int)rp->state, end_katom->renderpass_id); - - if (rp->state == KBASE_JD_RP_COMPLETE) - return -EINVAL; - - if (rp->end_katom == NULL) { - /* We can't be in a retry state until the fragment job chain - * has completed. - */ - unsigned long flags; - - WARN_ON(rp->state == KBASE_JD_RP_RETRY); - WARN_ON(rp->state == KBASE_JD_RP_RETRY_PEND_OOM); - WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - rp->end_katom = end_katom; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } else - WARN_ON(rp->end_katom != end_katom); - - return 0; -} - bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom) { unsigned long flags; struct kbasep_js_kctx_info *js_kctx_info; struct kbase_device *kbdev; struct kbasep_js_device_data *js_devdata; - int err = 0; bool enqueue_required = false; bool timer_sync = false; @@ -1566,17 +1452,6 @@ bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom) mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - if (atom->core_req & BASE_JD_REQ_START_RENDERPASS) - err = js_add_start_rp(atom); - else if (atom->core_req & BASE_JD_REQ_END_RENDERPASS) - err = js_add_end_rp(atom); - - if (err < 0) { - atom->event_code = BASE_JD_EVENT_JOB_INVALID; - atom->status = KBASE_JD_ATOM_STATE_COMPLETED; - goto out_unlock; - } - /* * Begin Runpool transaction */ @@ -1860,10 +1735,7 @@ kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_ kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, katom_retained_state); if (new_ref_count == 2 && kbase_ctx_flag(kctx, KCTX_PRIVILEGED) && -#ifdef CONFIG_MALI_ARBITER_SUPPORT - !kbase_pm_is_gpu_lost(kbdev) && -#endif - !kbase_pm_is_suspending(kbdev)) { + !kbase_pm_is_gpu_lost(kbdev) && !kbase_pm_is_suspending(kbdev)) { /* Context is kept scheduled into an address space even when * there are no jobs, in this case we have to handle the * situation where all jobs have been evicted from the GPU and @@ -1880,10 +1752,7 @@ kbasep_js_runpool_release_ctx_internal(struct kbase_device *kbdev, struct kbase_ * which was previously acquired by kbasep_js_schedule_ctx(). */ if (new_ref_count == 1 && (!kbasep_js_is_submit_allowed(js_devdata, kctx) || -#ifdef CONFIG_MALI_ARBITER_SUPPORT - kbase_pm_is_gpu_lost(kbdev) || -#endif - kbase_pm_is_suspending(kbdev))) { + kbase_pm_is_gpu_lost(kbdev) || kbase_pm_is_suspending(kbdev))) { int num_slots = kbdev->gpu_props.num_job_slots; unsigned int slot; @@ -2189,11 +2058,7 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, struct kbase_cont * of it being called strictly after the suspend flag is set, and will * wait for this lock to drop) */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { -#else - if (kbase_pm_is_suspending(kbdev)) { -#endif /* Cause it to leave at some later point */ bool retained; CSTD_UNUSED(retained); @@ -2267,7 +2132,6 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_ js_devdata = &kbdev->js_data; js_kctx_info = &kctx->jctx.sched_info; -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* This should only happen in response to a system call * from a user-space thread. * In a non-arbitrated environment this can never happen @@ -2279,18 +2143,10 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, struct kbase_ * the wait event for KCTX_SCHEDULED, since no context * can be scheduled until we have the GPU again. */ - if (kbdev->arb.arb_if == NULL) + if (!kbase_has_arbiter(kbdev)) { if (WARN_ON(kbase_pm_is_suspending(kbdev))) return; -#else - /* This should only happen in response to a system call - * from a user-space thread. - * In a non-arbitrated environment this can never happen - * whilst suspending. - */ - if (WARN_ON(kbase_pm_is_suspending(kbdev))) - return; -#endif + } mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); @@ -2416,63 +2272,63 @@ void kbasep_js_resume(struct kbase_device *kbdev) struct kbase_context *kctx, *n; unsigned long flags; -#ifndef CONFIG_MALI_ARBITER_SUPPORT - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + if (kbase_has_arbiter(kbdev)) { + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - list_for_each_entry_safe(kctx, n, - &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - struct kbasep_js_kctx_info *js_kctx_info; + list_for_each_entry_safe( + kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], + jctx.sched_info.ctx.ctx_list_entry[js]) { + struct kbasep_js_kctx_info *js_kctx_info; + bool timer_sync = false; + + /* Drop lock so we can take kctx mutexes */ + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + js_kctx_info = &kctx->jctx.sched_info; + + mutex_lock(&js_kctx_info->ctx.jsctx_mutex); + mutex_lock(&js_devdata->runpool_mutex); + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync = kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + + if (timer_sync) + kbase_backend_ctx_count_changed(kbdev); + + mutex_unlock(&js_devdata->runpool_mutex); + mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); + + /* Take lock before accessing list again */ + spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + } + spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); + } else { bool timer_sync = false; - /* Drop lock so we can take kctx mutexes */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - js_kctx_info = &kctx->jctx.sched_info; - - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - mutex_lock(&js_devdata->runpool_mutex); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync = kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); + list_for_each_entry_safe( + kctx, n, &kbdev->js_data.ctx_list_unpullable[js][prio], + jctx.sched_info.ctx.ctx_list_entry[js]) { + if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && + kbase_js_ctx_pullable(kctx, js, false)) + timer_sync |= kbase_js_ctx_list_add_pullable_nolock( + kbdev, kctx, js); + } spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - if (timer_sync) + if (timer_sync) { + mutex_lock(&js_devdata->runpool_mutex); kbase_backend_ctx_count_changed(kbdev); - - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - - /* Take lock before accessing list again */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + mutex_unlock(&js_devdata->runpool_mutex); + } } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -#else - bool timer_sync = false; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - list_for_each_entry_safe(kctx, n, - &kbdev->js_data.ctx_list_unpullable[js][prio], - jctx.sched_info.ctx.ctx_list_entry[js]) { - if (!kbase_ctx_flag(kctx, KCTX_SCHEDULED) && - kbase_js_ctx_pullable(kctx, js, false)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( - kbdev, kctx, js); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (timer_sync) { - mutex_lock(&js_devdata->runpool_mutex); - kbase_backend_ctx_count_changed(kbdev); - mutex_unlock(&js_devdata->runpool_mutex); - } -#endif } } mutex_unlock(&js_devdata->queue_mutex); @@ -2515,7 +2371,7 @@ static unsigned int kbase_js_get_slot(struct kbase_device *kbdev, struct kbase_j bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_atom *katom) { - bool enqueue_required, add_required = true; + bool enqueue_required; katom->slot_nr = kbase_js_get_slot(kctx->kbdev, katom); @@ -2525,10 +2381,7 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_at /* If slot will transition from unpullable to pullable then add to * pullable list */ - if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) - enqueue_required = true; - else - enqueue_required = false; + enqueue_required = jsctx_rb_none_to_pull(kctx, katom->slot_nr); if ((katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) || (katom->pre_dep && @@ -2541,15 +2394,9 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_at list_add_tail(&katom->queue, &queue->x_dep_head); katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - if (kbase_js_atom_blocked_on_x_dep(katom)) { - enqueue_required = false; - add_required = false; - } + enqueue_required = false; } else { dev_dbg(kctx->kbdev->dev, "Atom %pK not added to X_DEP list\n", (void *)katom); - } - - if (add_required) { /* Check if there are lower priority jobs to soft stop */ kbase_job_slot_ctx_priority_check_locked(kctx, katom); @@ -2575,30 +2422,22 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, struct kbase_jd_at */ static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) { - struct kbase_context *const kctx = katom->kctx; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); + lockdep_assert_held(&katom->kctx->kbdev->hwaccess_lock); while (katom) { WARN_ON(!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - if (!kbase_js_atom_blocked_on_x_dep(katom)) { - dev_dbg(kctx->kbdev->dev, + if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { + dev_dbg(katom->kctx->kbdev->dev, "Del atom %pK from X_DEP list in js_move_to_tree\n", (void *)katom); list_del(&katom->queue); katom->atom_flags &= ~KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST; - /* For incremental rendering, an end-of-renderpass atom - * may have had its dependency on start-of-renderpass - * ignored and may therefore already be in the tree. - */ - if (!(katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)) { - jsctx_tree_add(kctx, katom); - katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; - } + jsctx_tree_add(katom->kctx, katom); + katom->atom_flags |= KBASE_KATOM_FLAG_JSCTX_IN_TREE; } else { - dev_dbg(kctx->kbdev->dev, "Atom %pK blocked on x-dep in js_move_to_tree\n", - (void *)katom); + dev_dbg(katom->kctx->kbdev->dev, + "Atom %pK blocked on x-dep in js_move_to_tree\n", (void *)katom); break; } @@ -2671,11 +2510,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", (void *)kctx); return NULL; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) -#else - if (kbase_pm_is_suspending(kbdev)) -#endif return NULL; katom = jsctx_rb_peek(kctx, js); @@ -2705,7 +2540,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) return NULL; } - if (kbase_js_atom_blocked_on_x_dep(katom)) { + if (katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED) { if (katom->x_pre_dep->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || katom->x_pre_dep->will_fail_event_code) { dev_dbg(kbdev->dev, @@ -2745,190 +2580,6 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, unsigned int js) return katom; } -/** - * js_return_of_start_rp() - Handle soft-stop of an atom that starts a - * renderpass - * @start_katom: Pointer to the start-of-renderpass atom that was soft-stopped - * - * This function is called to switch to incremental rendering if the tiler job - * chain at the start of a renderpass has used too much memory. It prevents the - * tiler job being pulled for execution in the job scheduler again until the - * next phase of incremental rendering is complete. - * - * If the end-of-renderpass atom is already in the job scheduler (because a - * previous attempt at tiling used too much memory during the same renderpass) - * then it is unblocked; otherwise, it is run by handing it to the scheduler. - */ -static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) -{ - struct kbase_context *const kctx = start_katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - struct kbase_jd_atom *end_katom; - unsigned long flags; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) - return; - - compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; - - if (WARN_ON(rp->start_katom != start_katom)) - return; - - dev_dbg(kctx->kbdev->dev, "JS return start atom %pK in state %d of RP %d\n", - (void *)start_katom, (int)rp->state, start_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) - return; - - /* The tiler job might have been soft-stopped for some reason other - * than running out of memory. - */ - if (rp->state == KBASE_JD_RP_START || rp->state == KBASE_JD_RP_RETRY) { - dev_dbg(kctx->kbdev->dev, "JS return isn't OOM in state %d of RP %d\n", - (int)rp->state, start_katom->renderpass_id); - return; - } - - dev_dbg(kctx->kbdev->dev, "JS return confirm OOM in state %d of RP %d\n", (int)rp->state, - start_katom->renderpass_id); - - if (WARN_ON(rp->state != KBASE_JD_RP_PEND_OOM && rp->state != KBASE_JD_RP_RETRY_PEND_OOM)) - return; - - /* Prevent the tiler job being pulled for execution in the - * job scheduler again. - */ - dev_dbg(kbdev->dev, "Blocking start atom %pK\n", (void *)start_katom); - atomic_inc(&start_katom->blocked); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - rp->state = (rp->state == KBASE_JD_RP_PEND_OOM) ? KBASE_JD_RP_OOM : KBASE_JD_RP_RETRY_OOM; - - /* Was the fragment job chain submitted to kbase yet? */ - end_katom = rp->end_katom; - if (end_katom) { - dev_dbg(kctx->kbdev->dev, "JS return add end atom %pK\n", (void *)end_katom); - - if (rp->state == KBASE_JD_RP_RETRY_OOM) { - /* Allow the end of the renderpass to be pulled for - * execution again to continue incremental rendering. - */ - dev_dbg(kbdev->dev, "Unblocking end atom %pK\n", (void *)end_katom); - atomic_dec(&end_katom->blocked); - WARN_ON(!(end_katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE)); - WARN_ON(end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); - - kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, end_katom->slot_nr); - - /* Expect the fragment job chain to be scheduled without - * further action because this function is called when - * returning an atom to the job scheduler ringbuffer. - */ - end_katom = NULL; - } else { - WARN_ON(end_katom->status != KBASE_JD_ATOM_STATE_QUEUED && - end_katom->status != KBASE_JD_ATOM_STATE_IN_JS); - } - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - if (end_katom) - kbase_jd_dep_clear_locked(end_katom); -} - -/** - * js_return_of_end_rp() - Handle completion of an atom that ends a renderpass - * @end_katom: Pointer to the end-of-renderpass atom that was completed - * - * This function is called to continue incremental rendering if the tiler job - * chain at the start of a renderpass used too much memory. It resets the - * mechanism for detecting excessive memory usage then allows the soft-stopped - * tiler job chain to be pulled for execution again. - * - * The start-of-renderpass atom must already been submitted to kbase. - */ -static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) -{ - struct kbase_context *const kctx = end_katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - struct kbase_jd_atom *start_katom; - unsigned long flags; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - if (WARN_ON(rp->end_katom != end_katom)) - return; - - dev_dbg(kctx->kbdev->dev, "JS return end atom %pK in state %d of RP %d\n", - (void *)end_katom, (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM)) - return; - - /* Reduce the number of mapped pages in the memory regions that - * triggered out-of-memory last time so that we can detect excessive - * memory usage again. - */ - kbase_gpu_vm_lock(kctx); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - while (!list_empty(&rp->oom_reg_list)) { - struct kbase_va_region *reg = - list_first_entry(&rp->oom_reg_list, struct kbase_va_region, link); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - dev_dbg(kbdev->dev, "Reset backing to %zu pages for region %pK\n", - reg->threshold_pages, (void *)reg); - - if (!WARN_ON(reg->flags & KBASE_REG_VA_FREED)) - kbase_mem_shrink(kctx, reg, reg->threshold_pages); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - dev_dbg(kbdev->dev, "Deleting region %pK from list\n", (void *)reg); - list_del_init(®->link); - kbase_va_region_alloc_put(kctx, reg); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - kbase_gpu_vm_unlock(kctx); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - rp->state = KBASE_JD_RP_RETRY; - dev_dbg(kbdev->dev, "Changed state to %d for retry\n", rp->state); - - /* Allow the start of the renderpass to be pulled for execution again - * to begin/continue incremental rendering. - */ - start_katom = rp->start_katom; - if (!WARN_ON(!start_katom)) { - dev_dbg(kbdev->dev, "Unblocking start atom %pK\n", (void *)start_katom); - atomic_dec(&start_katom->blocked); - (void)kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, start_katom->slot_nr); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -} - static void js_return_worker(struct work_struct *data) { struct kbase_jd_atom *katom = container_of(data, struct kbase_jd_atom, work); @@ -2949,9 +2600,7 @@ static void js_return_worker(struct work_struct *data) katom->event_code); KBASE_KTRACE_ADD_JM(kbdev, JS_RETURN_WORKER, kctx, katom, katom->jc, 0); - - if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); + KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_EX(kbdev, katom); kbase_backend_complete_wq(kbdev, katom); @@ -2960,8 +2609,7 @@ static void js_return_worker(struct work_struct *data) mutex_lock(&js_devdata->queue_mutex); mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) - atomic_dec(&katom->blocked); + atomic_dec(&katom->blocked); spin_lock_irqsave(&kbdev->hwaccess_lock, flags); @@ -3026,16 +2674,6 @@ static void js_return_worker(struct work_struct *data) mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); mutex_unlock(&js_devdata->queue_mutex); - if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) { - mutex_lock(&kctx->jctx.lock); - js_return_of_start_rp(katom); - mutex_unlock(&kctx->jctx.lock); - } else if (katom->event_code == BASE_JD_EVENT_END_RP_DONE) { - mutex_lock(&kctx->jctx.lock); - js_return_of_end_rp(katom); - mutex_unlock(&kctx->jctx.lock); - } - dev_dbg(kbdev->dev, "JS: retained state %s finished", kbasep_js_has_atom_finished(&retained_state) ? "has" : "hasn't"); @@ -3071,144 +2709,6 @@ void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) queue_work(kctx->jctx.job_done_wq, &katom->work); } -/** - * js_complete_start_rp() - Handle completion of atom that starts a renderpass - * @kctx: Context pointer - * @start_katom: Pointer to the atom that completed - * - * Put any references to virtual memory regions that might have been added by - * kbase_job_slot_softstop_start_rp() because the tiler job chain completed - * despite any pending soft-stop request. - * - * If the atom that just completed was soft-stopped during a previous attempt to - * run it then there should be a blocked end-of-renderpass atom waiting for it, - * which we must unblock to process the output of the tiler job chain. - * - * Return: true if caller should call kbase_backend_ctx_count_changed() - */ -static bool js_complete_start_rp(struct kbase_context *kctx, - struct kbase_jd_atom *const start_katom) -{ - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - bool timer_sync = false; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(start_katom->core_req & BASE_JD_REQ_START_RENDERPASS))) - return false; - - compiletime_assert((1ull << (sizeof(start_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[start_katom->renderpass_id]; - - if (WARN_ON(rp->start_katom != start_katom)) - return false; - - dev_dbg(kctx->kbdev->dev, "Start atom %pK is done in state %d of RP %d\n", - (void *)start_katom, (int)rp->state, start_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) - return false; - - if (rp->state == KBASE_JD_RP_PEND_OOM || rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { - unsigned long flags; - - dev_dbg(kctx->kbdev->dev, "Start atom %pK completed before soft-stop\n", - (void *)start_katom); - - kbase_gpu_vm_lock(kctx); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - while (!list_empty(&rp->oom_reg_list)) { - struct kbase_va_region *reg = - list_first_entry(&rp->oom_reg_list, struct kbase_va_region, link); - - WARN_ON(reg->flags & KBASE_REG_VA_FREED); - dev_dbg(kctx->kbdev->dev, "Deleting region %pK from list\n", (void *)reg); - list_del_init(®->link); - kbase_va_region_alloc_put(kctx, reg); - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - kbase_gpu_vm_unlock(kctx); - } else { - dev_dbg(kctx->kbdev->dev, "Start atom %pK did not exceed memory threshold\n", - (void *)start_katom); - - WARN_ON(rp->state != KBASE_JD_RP_START && rp->state != KBASE_JD_RP_RETRY); - } - - if (rp->state == KBASE_JD_RP_RETRY || rp->state == KBASE_JD_RP_RETRY_PEND_OOM) { - struct kbase_jd_atom *const end_katom = rp->end_katom; - - if (!WARN_ON(!end_katom)) { - unsigned long flags; - - /* Allow the end of the renderpass to be pulled for - * execution again to continue incremental rendering. - */ - dev_dbg(kbdev->dev, "Unblocking end atom %pK!\n", (void *)end_katom); - atomic_dec(&end_katom->blocked); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - timer_sync = kbase_js_ctx_list_add_pullable_nolock(kbdev, kctx, - end_katom->slot_nr); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - } - - return timer_sync; -} - -/** - * js_complete_end_rp() - Handle final completion of atom that ends a renderpass - * @kctx: Context pointer - * @end_katom: Pointer to the atom that completed for the last time - * - * This function must only be called if the renderpass actually completed - * without the tiler job chain at the start using too much memory; otherwise - * completion of the end-of-renderpass atom is handled similarly to a soft-stop. - */ -static void js_complete_end_rp(struct kbase_context *kctx, struct kbase_jd_atom *const end_katom) -{ - struct kbase_device *const kbdev = kctx->kbdev; - unsigned long flags; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kctx->jctx.lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - if (WARN_ON(rp->end_katom != end_katom)) - return; - - dev_dbg(kbdev->dev, "End atom %pK is done in state %d of RP %d\n", (void *)end_katom, - (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE) || WARN_ON(rp->state == KBASE_JD_RP_OOM) || - WARN_ON(rp->state == KBASE_JD_RP_RETRY_OOM)) - return; - - /* Rendering completed without running out of memory. - */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - WARN_ON(!list_empty(&rp->oom_reg_list)); - rp->state = KBASE_JD_RP_COMPLETE; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - dev_dbg(kbdev->dev, "Renderpass %d is complete\n", end_katom->renderpass_id); -} - bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom *katom) { struct kbasep_js_kctx_info *js_kctx_info; @@ -3225,13 +2725,6 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom dev_dbg(kbdev->dev, "%s for atom %pK (s:%u)\n", __func__, (void *)katom, atom_slot); - /* Update the incremental rendering state machine. - */ - if (katom->core_req & BASE_JD_REQ_START_RENDERPASS) - timer_sync |= js_complete_start_rp(kctx, katom); - else if (katom->core_req & BASE_JD_REQ_END_RENDERPASS) - js_complete_end_rp(kctx, katom); - js_kctx_info = &kctx->jctx.sched_info; js_devdata = &kbdev->js_data; @@ -3320,61 +2813,6 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, struct kbase_jd_atom return context_idle; } -/** - * js_end_rp_is_complete() - Check whether an atom that ends a renderpass has - * completed for the last time. - * - * @end_katom: Pointer to the atom that completed on the hardware. - * - * An atom that ends a renderpass may be run on the hardware several times - * before notifying userspace or allowing dependent atoms to be executed. - * - * This function is used to decide whether or not to allow end-of-renderpass - * atom completion. It only returns false if the atom at the start of the - * renderpass was soft-stopped because it used too much memory during the most - * recent attempt at tiling. - * - * Return: True if the atom completed for the last time. - */ -static bool js_end_rp_is_complete(struct kbase_jd_atom *const end_katom) -{ - struct kbase_context *const kctx = end_katom->kctx; - struct kbase_device *const kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - - if (WARN_ON(!(end_katom->core_req & BASE_JD_REQ_END_RENDERPASS))) - return true; - - compiletime_assert((1ull << (sizeof(end_katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - - if (WARN_ON(rp->end_katom != end_katom)) - return true; - - dev_dbg(kbdev->dev, "JS complete end atom %pK in state %d of RP %d\n", (void *)end_katom, - (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE)) - return true; - - /* Failure of end-of-renderpass atoms must not return to the - * start of the renderpass. - */ - if (end_katom->event_code != BASE_JD_EVENT_DONE) - return true; - - if (rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM) - return true; - - dev_dbg(kbdev->dev, "Suppressing end atom completion\n"); - return false; -} - struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_t *end_timestamp) { struct kbase_device *kbdev; @@ -3387,12 +2825,6 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_ lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - if ((katom->core_req & BASE_JD_REQ_END_RENDERPASS) && !js_end_rp_is_complete(katom)) { - katom->event_code = BASE_JD_EVENT_END_RP_DONE; - kbase_js_unpull(kctx, katom); - return NULL; - } - if (katom->will_fail_event_code) katom->event_code = katom->will_fail_event_code; @@ -3442,70 +2874,6 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, ktime_ return NULL; } -/** - * kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot - * dependency - * @katom: Pointer to an atom in the slot ringbuffer - * - * A cross-slot dependency is ignored if necessary to unblock incremental - * rendering. If the atom at the start of a renderpass used too much memory - * and was soft-stopped then the atom at the end of a renderpass is submitted - * to hardware regardless of its dependency on the start-of-renderpass atom. - * This can happen multiple times for the same pair of atoms. - * - * Return: true to block the atom or false to allow it to be submitted to - * hardware - */ -bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom) -{ - struct kbase_context *const kctx = katom->kctx; - struct kbase_device *kbdev = kctx->kbdev; - struct kbase_jd_renderpass *rp; - - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!(katom->atom_flags & KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { - dev_dbg(kbdev->dev, "Atom %pK is not blocked on a cross-slot dependency", - (void *)katom); - return false; - } - - if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) { - dev_dbg(kbdev->dev, "Atom %pK is blocked on a cross-slot dependency", - (void *)katom); - return true; - } - - compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <= - ARRAY_SIZE(kctx->jctx.renderpasses), - "Should check invalid access to renderpasses"); - - rp = &kctx->jctx.renderpasses[katom->renderpass_id]; - /* We can read a subset of renderpass state without holding - * higher-level locks (but not end_katom, for example). - */ - - WARN_ON(rp->state == KBASE_JD_RP_COMPLETE); - - dev_dbg(kbdev->dev, "End atom has cross-slot dep in state %d\n", (int)rp->state); - - if (rp->state != KBASE_JD_RP_OOM && rp->state != KBASE_JD_RP_RETRY_OOM) - return true; - - /* Tiler ran out of memory so allow the fragment job chain to run - * if it only depends on the tiler job chain. - */ - if (katom->x_pre_dep != rp->start_katom) { - dev_dbg(kbdev->dev, "Dependency is on %pK not start atom %pK\n", - (void *)katom->x_pre_dep, (void *)rp->start_katom); - return true; - } - - dev_dbg(kbdev->dev, "Ignoring cross-slot dep on atom %pK\n", (void *)katom->x_pre_dep); - - return false; -} - void kbase_js_sched(struct kbase_device *kbdev, unsigned int js_mask) { struct kbasep_js_device_data *js_devdata; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c index aae4df83e98d..87085912bd6c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_kinstr_jm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -516,7 +516,8 @@ static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, do { changes_tail = changes->tail; changes_count = reader_changes_count_locked(changes); - read_size = min(changes_count * entry_size, buffer_size & ~(entry_size - 1)); + read_size = + min(size_mul(changes_count, entry_size), buffer_size & ~(entry_size - 1)); if (!read_size) break; @@ -743,7 +744,6 @@ int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr size_t const change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); int status; int fd; - size_t i; if (!ctx || !jm_fd_arg) return -EINVAL; @@ -753,10 +753,6 @@ int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, union kbase_kinstr if (!is_power_of_2(in->count)) return -EINVAL; - for (i = 0; i < sizeof(in->padding); ++i) - if (in->padding[i]) - return -EINVAL; - status = reader_init(&reader, ctx, in->count); if (status < 0) return status; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.c b/drivers/gpu/arm/bifrost/mali_kbase_mem.c index 7470a94a44ad..1436d8290ebc 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,6 @@ #include #include #include -#include #include #include @@ -52,6 +52,11 @@ static DEFINE_STATIC_KEY_FALSE(large_pages_static_key); #define VA_REGION_SLAB_NAME_PREFIX "va-region-slab-" #define VA_REGION_SLAB_NAME_SIZE (DEVNAME_SIZE + sizeof(VA_REGION_SLAB_NAME_PREFIX) + 1) +#if GPU_PAGES_PER_CPU_PAGE > 1 +#define PAGE_METADATA_SLAB_NAME_PREFIX "page-metadata-slab-" +#define PAGE_METADATA_SLAB_NAME_SIZE (DEVNAME_SIZE + sizeof(PAGE_METADATA_SLAB_NAME_PREFIX) + 1) +#endif + #if MALI_JIT_PRESSURE_LIMIT_BASE /* @@ -152,7 +157,7 @@ static void kbasep_mem_page_size_init(struct kbase_device *kbdev) switch (large_page_conf) { case LARGE_PAGE_AUTO: { - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_LARGE_PAGE_ALLOC)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_LARGE_PAGE_ALLOC)) static_branch_inc(&large_pages_static_key); dev_info(kbdev->dev, "Large page allocation set to %s after hardware feature check", static_branch_unlikely(&large_pages_static_key) ? "true" : "false"); @@ -160,7 +165,7 @@ static void kbasep_mem_page_size_init(struct kbase_device *kbdev) } case LARGE_PAGE_ON: { static_branch_inc(&large_pages_static_key); - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_LARGE_PAGE_ALLOC)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_LARGE_PAGE_ALLOC)) dev_warn(kbdev->dev, "Enabling large page allocations on unsupporting GPU!"); else @@ -196,16 +201,16 @@ KBASE_EXPORT_TEST_API(kbase_is_large_pages_enabled); int kbase_mem_init(struct kbase_device *kbdev) { int err = 0; - struct kbasep_mem_device *memdev; char va_region_slab_name[VA_REGION_SLAB_NAME_SIZE]; +#if GPU_PAGES_PER_CPU_PAGE > 1 + char page_metadata_slab_name[PAGE_METADATA_SLAB_NAME_SIZE]; +#endif #if IS_ENABLED(CONFIG_OF) struct device_node *mgm_node = NULL; #endif KBASE_DEBUG_ASSERT(kbdev); - memdev = &kbdev->memdev; - kbasep_mem_page_size_init(kbdev); scnprintf(va_region_slab_name, VA_REGION_SLAB_NAME_SIZE, VA_REGION_SLAB_NAME_PREFIX "%s", @@ -219,6 +224,17 @@ int kbase_mem_init(struct kbase_device *kbdev) return -ENOMEM; } +#if GPU_PAGES_PER_CPU_PAGE > 1 + scnprintf(page_metadata_slab_name, PAGE_METADATA_SLAB_NAME_SIZE, + PAGE_METADATA_SLAB_NAME_PREFIX "%s", kbdev->devname); + kbdev->page_metadata_slab = kmem_cache_create( + page_metadata_slab_name, sizeof(struct kbase_page_metadata), 0, 0, NULL); + if (kbdev->page_metadata_slab == NULL) { + dev_err(kbdev->dev, "Failed to create page_metadata_slab"); + return -ENOMEM; + } +#endif + kbase_mem_migrate_init(kbdev); kbase_mem_pool_group_config_set_max_size(&kbdev->mem_pool_defaults, KBASE_MEM_POOL_MAX_SIZE_KCTX); @@ -228,12 +244,6 @@ int kbase_mem_init(struct kbase_device *kbdev) kbdev->dma_buf_root = RB_ROOT; mutex_init(&kbdev->dma_buf_lock); -#ifdef IR_THRESHOLD - atomic_set(&memdev->ir_threshold, IR_THRESHOLD); -#else - atomic_set(&memdev->ir_threshold, DEFAULT_IR_THRESHOLD); -#endif - kbdev->mgm_dev = &kbase_native_mgm_dev; #if IS_ENABLED(CONFIG_OF) @@ -299,6 +309,10 @@ void kbase_mem_term(struct kbase_device *kbdev) kbase_mem_migrate_term(kbdev); +#if GPU_PAGES_PER_CPU_PAGE > 1 + kmem_cache_destroy(kbdev->page_metadata_slab); + kbdev->page_metadata_slab = NULL; +#endif kmem_cache_destroy(kbdev->va_region_slab); kbdev->va_region_slab = NULL; @@ -1221,6 +1235,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa * to satisfy the memory allocation request. */ size_t nr_pages_to_account = 0; + size_t nr_pages_from_partials = 0; if (WARN_ON(alloc->type != KBASE_MEM_TYPE_NATIVE) || WARN_ON(alloc->imported.native.kctx == NULL) || @@ -1279,6 +1294,7 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pa *tp++ = as_tagged_tag(page_to_phys(sa->page + pidx), FROM_PARTIAL); nr_left--; + nr_pages_from_partials++; if (bitmap_full(sa->sub_pages, NUM_PAGES_IN_2MB_LARGE_PAGE)) { @@ -1386,6 +1402,13 @@ alloc_failed: alloc->nents += nr_pages_to_free; kbase_free_phy_pages_helper(alloc, nr_pages_to_free); + + /* Notice that the sub-pages from "partials" are not subtracted + * from the counter by the free pages helper, because they just go + * back to the "partials" they belong to, therefore they must be + * subtracted from the counter here. + */ + nr_left += nr_pages_from_partials; } /* Undo the preliminary memory accounting that was done early on @@ -1994,11 +2017,13 @@ out_term: } KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages); -void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc, +void kbase_set_phy_alloc_page_status(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc, enum kbase_page_status status) { u32 i = 0; + lockdep_assert_held(&kctx->reg_lock); + for (; i < alloc->nents; i++) { struct tagged_addr phys = alloc->pages[i]; struct kbase_page_metadata *page_md = kbase_page_private(as_page(phys)); @@ -2018,7 +2043,7 @@ void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc, } } -bool kbase_check_alloc_flags(unsigned long flags) +bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags) { /* Only known input flags should be set. */ if (flags & ~BASE_MEM_FLAGS_INPUT_MASK) @@ -2094,6 +2119,36 @@ bool kbase_check_alloc_flags(unsigned long flags) return false; #endif + /* Cannot be set only allocation, only with base_mem_set */ + if ((flags & BASE_MEM_DONT_NEED) && + (mali_kbase_supports_reject_alloc_mem_dont_need(kctx->api_version))) + return false; + + /* Cannot directly allocate protected memory, it is imported instead */ + if ((flags & BASE_MEM_PROTECTED) && + (mali_kbase_supports_reject_alloc_mem_protected_in_unprotected_allocs( + kctx->api_version))) + return false; + +/* No unused bits are valid for allocations */ +#if MALI_USE_CSF + if ((flags & BASE_MEM_UNUSED_BIT_20) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_20(kctx->api_version))) + return false; + + if ((flags & BASE_MEM_UNUSED_BIT_27) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_27(kctx->api_version))) + return false; +#else /* MALI_USE_CSF */ + if ((flags & BASE_MEM_UNUSED_BIT_8) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_8(kctx->api_version))) + return false; + + if ((flags & BASE_MEM_UNUSED_BIT_19) && + (mali_kbase_supports_reject_alloc_mem_unused_bit_19(kctx->api_version))) + return false; +#endif /* MALI_USE_CSF */ + return true; } @@ -2258,7 +2313,7 @@ KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); void kbase_gpu_vm_lock_with_pmode_sync(struct kbase_context *kctx) { #if MALI_USE_CSF - down_read(&kctx->kbdev->csf.pmode_sync_sem); + down_read(&kctx->kbdev->csf.mmu_sync_sem); #endif kbase_gpu_vm_lock(kctx); } @@ -2274,7 +2329,7 @@ void kbase_gpu_vm_unlock_with_pmode_sync(struct kbase_context *kctx) { kbase_gpu_vm_unlock(kctx); #if MALI_USE_CSF - up_read(&kctx->kbdev->csf.pmode_sync_sem); + up_read(&kctx->kbdev->csf.mmu_sync_sem); #endif } @@ -3230,15 +3285,17 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, if (kbase_is_page_migration_enabled()) { kbase_gpu_vm_lock(kctx); mutex_lock(&kctx->jit_evict_lock); - kbase_set_phy_alloc_page_status(reg->gpu_alloc, ALLOCATED_MAPPED); + kbase_set_phy_alloc_page_status(kctx, reg->gpu_alloc, + ALLOCATED_MAPPED); mutex_unlock(&kctx->jit_evict_lock); kbase_gpu_vm_unlock(kctx); } } } else { /* No suitable JIT allocation was found so create a new one */ - u64 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | - BASE_MEM_GROW_ON_GPF | BASE_MEM_COHERENT_LOCAL | BASEP_MEM_NO_USER_FREE; + base_mem_alloc_flags flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_RD | + BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | + BASE_MEM_COHERENT_LOCAL | BASEP_MEM_NO_USER_FREE; u64 gpu_addr; #if !MALI_USE_CSF @@ -3335,6 +3392,7 @@ end: return reg; } +KBASE_EXPORT_TEST_API(kbase_jit_allocate); void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) { @@ -3376,13 +3434,30 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) trace_jit_stats(kctx, reg->jit_bin_id, UINT_MAX); + kbase_gpu_vm_lock_with_pmode_sync(kctx); + if (unlikely(atomic_read(®->cpu_alloc->kernel_mappings))) { + WARN_ON(atomic64_read(®->no_user_free_count) > 1); + kbase_va_region_no_user_free_dec(reg); + mutex_lock(&kctx->jit_evict_lock); + list_del(®->jit_node); + mutex_unlock(&kctx->jit_evict_lock); + kbase_mem_free_region(kctx, reg); + kbase_gpu_vm_unlock_with_pmode_sync(kctx); + return; + } kbase_mem_evictable_mark_reclaim(reg->gpu_alloc); - - kbase_gpu_vm_lock(kctx); reg->flags |= KBASE_REG_DONT_NEED; reg->flags &= ~KBASE_REG_ACTIVE_JIT_ALLOC; kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents); - kbase_gpu_vm_unlock(kctx); + + /* Inactive JIT regions should be freed by the shrinker and not impacted + * by page migration. Once freed, they will enter into the page migration + * state machine via the mempools. + */ + if (kbase_is_page_migration_enabled()) + kbase_set_phy_alloc_page_status(kctx, reg->gpu_alloc, NOT_MOVABLE); + + kbase_gpu_vm_unlock_with_pmode_sync(kctx); /* * Add the allocation to the eviction list and the jit pool, after this @@ -3397,14 +3472,9 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) list_move(®->jit_node, &kctx->jit_pool_head); - /* Inactive JIT regions should be freed by the shrinker and not impacted - * by page migration. Once freed, they will enter into the page migration - * state machine via the mempools. - */ - if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(reg->gpu_alloc, NOT_MOVABLE); mutex_unlock(&kctx->jit_evict_lock); } +KBASE_EXPORT_TEST_API(kbase_jit_free); void kbase_jit_backing_lost(struct kbase_va_region *reg) { diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem.h b/drivers/gpu/arm/bifrost/mali_kbase_mem.h index 19b50f5bf08b..880b8525ae37 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem.h @@ -104,8 +104,8 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int /* Index of chosen MEMATTR for this region (0..7) */ #define KBASE_REG_MEMATTR_MASK (7ul << 16) -#define KBASE_REG_MEMATTR_INDEX(x) (((x)&7) << 16) -#define KBASE_REG_MEMATTR_VALUE(x) (((x)&KBASE_REG_MEMATTR_MASK) >> 16) +#define KBASE_REG_MEMATTR_INDEX(x) (((x) & 7) << 16) +#define KBASE_REG_MEMATTR_VALUE(x) (((x) & KBASE_REG_MEMATTR_MASK) >> 16) /* AS_MEMATTR values from MMU_MEMATTR_STAGE1: */ /* Use GPU implementation-defined caching policy. */ @@ -482,6 +482,26 @@ struct kbase_page_metadata { struct kbase_mmu_table *mmut; /* GPU virtual page frame number info is in GPU_PAGE_SIZE units */ u64 pgd_vpfn_level; +#if GPU_PAGES_PER_CPU_PAGE > 1 + /** + * @pgd_link: Link to the &kbase_mmu_table.pgd_pages_list + */ + struct list_head pgd_link; + /** + * @pgd_page: Back pointer to the PGD page that the metadata is + * associated with + */ + struct page *pgd_page; + /** + * @allocated_sub_pages: Bitmap representing the allocation status + * of sub pages in the @pgd_page + */ + DECLARE_BITMAP(allocated_sub_pages, GPU_PAGES_PER_CPU_PAGE); + /** + * @num_allocated_sub_pages: The number of allocated sub pages in @pgd_page + */ + s8 num_allocated_sub_pages; +#endif } pt_mapped; struct { struct kbase_device *kbdev; @@ -510,6 +530,7 @@ enum kbase_jit_report_flags { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0) }; /** * kbase_set_phy_alloc_page_status - Set the page migration status of the underlying * physical allocation. + * @kctx: Pointer to Kbase context. * @alloc: the physical allocation containing the pages whose metadata is going * to be modified * @status: the status the pages should end up in @@ -518,7 +539,7 @@ enum kbase_jit_report_flags { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE = (1u << 0) }; * proper states are set. Instead, it is only used when we change the allocation * to NOT_MOVABLE or from NOT_MOVABLE to ALLOCATED_MAPPED */ -void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc, +void kbase_set_phy_alloc_page_status(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc, enum kbase_page_status status); static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc) @@ -620,9 +641,6 @@ static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_m * @nr_pages: The size of the region in pages. * @initial_commit: Initial commit, for aligning the start address and * correctly growing KBASE_REG_TILER_ALIGN_TOP regions. - * @threshold_pages: If non-zero and the amount of memory committed to a region - * that can grow on page fault exceeds this number of pages - * then the driver switches to incremental rendering. * @flags: Flags * @extension: Number of pages allocated on page fault. * @cpu_alloc: The physical memory we mmap to the CPU when mapping this region. @@ -659,8 +677,7 @@ struct kbase_va_region { void *user_data; size_t nr_pages; size_t initial_commit; - size_t threshold_pages; - unsigned long flags; + base_mem_alloc_flags flags; size_t extension; struct kbase_mem_phy_alloc *cpu_alloc; struct kbase_mem_phy_alloc *gpu_alloc; @@ -914,10 +931,12 @@ static inline struct kbase_mem_phy_alloc *kbase_alloc_create(struct kbase_contex atomic_set(&alloc->gpu_mappings, 0); atomic_set(&alloc->kernel_mappings, 0); alloc->nents = 0; - alloc->pages = (void *)(alloc + 1); - /* fill pages with invalid address value */ - for (i = 0; i < nr_pages; i++) - alloc->pages[i] = as_tagged(KBASE_INVALID_PHYSICAL_ADDRESS); + if (type != KBASE_MEM_TYPE_ALIAS) { + alloc->pages = (void *)(alloc + 1); + /* fill pages with invalid address value */ + for (i = 0; i < nr_pages; i++) + alloc->pages[i] = as_tagged(KBASE_INVALID_PHYSICAL_ADDRESS); + } INIT_LIST_HEAD(&alloc->mappings); alloc->type = type; alloc->group_id = group_id; @@ -1307,7 +1326,7 @@ struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool, const bool alloc_ */ void kbase_mem_pool_free_page(struct kbase_mem_pool *pool, struct page *p); -bool kbase_check_alloc_flags(unsigned long flags); +bool kbase_check_alloc_flags(struct kbase_context *kctx, unsigned long flags); bool kbase_check_import_flags(unsigned long flags); static inline bool kbase_import_size_is_valid(struct kbase_device *kbdev, u64 va_pages) @@ -1799,8 +1818,8 @@ static inline dma_addr_t kbase_dma_addr_from_tagged(struct tagged_addr tagged_pa phys_addr_t pa = as_phys_addr_t(tagged_pa); struct page *page = pfn_to_page(PFN_DOWN(pa)); dma_addr_t dma_addr = (is_huge(tagged_pa) || is_partial(tagged_pa)) ? - kbase_dma_addr_as_priv(page) : - kbase_dma_addr(page); + kbase_dma_addr_as_priv(page) : + kbase_dma_addr(page); return dma_addr; } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c index 7c319abf381b..9f26c088630c 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.c @@ -83,8 +83,6 @@ #define KBASE_MEM_ION_SYNC_WORKAROUND #endif -#define IR_THRESHOLD_STEPS (256u) - /* * fully_backed_gpf_memory - enable full physical backing of all grow-on-GPU-page-fault * allocations in the kernel. @@ -295,7 +293,7 @@ void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, struct kbase_vmap_s } struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, - u64 extension, u64 *flags, u64 *gpu_va, + u64 extension, base_mem_alloc_flags *flags, u64 *gpu_va, enum kbase_caller_mmu_sync_info mmu_sync_info) { struct kbase_va_region *reg; @@ -320,9 +318,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages else dev_dbg(dev, "Keeping requested GPU VA of 0x%llx\n", (unsigned long long)*gpu_va); - if (!kbase_check_alloc_flags(*flags)) { - dev_warn(dev, "%s called with bad flags (%llx)", __func__, - (unsigned long long)*flags); + if (!kbase_check_alloc_flags(kctx, *flags)) { + dev_warn(dev, "%s called with bad flags (%llx)", __func__, *flags); goto bad_flags; } @@ -335,6 +332,12 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages } #endif + /* Ensure GPU cached if CPU cached */ + if ((*flags & BASE_MEM_CACHED_CPU) != 0) { + dev_warn_once(dev, "Clearing BASE_MEM_UNCACHED_GPU flag to avoid MMA violation\n"); + *flags &= ~BASE_MEM_UNCACHED_GPU; + } + if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ @@ -406,17 +409,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages *flags &= ~BASE_MEM_CACHED_CPU; if (*flags & BASE_MEM_GROW_ON_GPF) { - unsigned int const ir_threshold = - (unsigned int)atomic_read(&kctx->kbdev->memdev.ir_threshold); - - reg->threshold_pages = - ((va_pages * ir_threshold) + (IR_THRESHOLD_STEPS / 2)) / IR_THRESHOLD_STEPS; - } else - reg->threshold_pages = 0; - - if (*flags & BASE_MEM_GROW_ON_GPF) { - /* kbase_check_alloc_sizes() already checks extension is valid for - * assigning to reg->extension + /* kbase_check_alloc_sizes() already checks extension is valid for assigning to + * reg->extension. */ reg->extension = extension; #if !MALI_USE_CSF @@ -597,11 +591,11 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co *out |= BASE_MEM_COHERENT_SYSTEM; if (KBASE_REG_SHARE_IN & reg->flags) *out |= BASE_MEM_COHERENT_LOCAL; - if (mali_kbase_supports_mem_dont_need(kctx->api_version)) { + if (mali_kbase_supports_query_mem_dont_need(kctx->api_version)) { if (KBASE_REG_DONT_NEED & reg->flags) *out |= BASE_MEM_DONT_NEED; } - if (mali_kbase_supports_mem_grow_on_gpf(kctx->api_version)) { + if (mali_kbase_supports_query_mem_grow_on_gpf(kctx->api_version)) { /* Prior to this version, this was known about by * user-side but we did not return them. Returning * it caused certain clients that were not expecting @@ -611,7 +605,7 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co if (KBASE_REG_PF_GROW & reg->flags) *out |= BASE_MEM_GROW_ON_GPF; } - if (mali_kbase_supports_mem_protected(kctx->api_version)) { + if (mali_kbase_supports_query_mem_protected(kctx->api_version)) { /* Prior to this version, this was known about by * user-side but we did not return them. Returning * it caused certain clients that were not expecting @@ -640,17 +634,17 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co #endif /* MALI_USE_CSF */ if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; - if (mali_kbase_supports_mem_import_sync_on_map_unmap(kctx->api_version)) { + if (mali_kbase_supports_query_mem_import_sync_on_map_unmap(kctx->api_version)) { if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { if (reg->gpu_alloc->imported.umm.need_sync) *out |= BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP; } } - if (mali_kbase_supports_mem_kernel_sync(kctx->api_version)) { + if (mali_kbase_supports_query_mem_kernel_sync(kctx->api_version)) { if (unlikely(reg->cpu_alloc != reg->gpu_alloc)) *out |= BASE_MEM_KERNEL_SYNC; } - if (mali_kbase_supports_mem_same_va(kctx->api_version)) { + if (mali_kbase_supports_query_mem_same_va(kctx->api_version)) { if (kbase_bits_to_zone(reg->flags) == SAME_VA_ZONE) { /* Imported memory is an edge case, where declaring it SAME_VA * would be ambiguous. @@ -746,7 +740,7 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s kctx = KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, struct kbase_context, reclaim); #if MALI_USE_CSF - if (!down_read_trylock(&kctx->kbdev->csf.pmode_sync_sem)) { + if (!down_read_trylock(&kctx->kbdev->csf.mmu_sync_sem)) { dev_warn(kctx->kbdev->dev, "Can't shrink GPU memory when P.Mode entrance is in progress"); return 0; @@ -791,7 +785,7 @@ static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s mutex_unlock(&kctx->jit_evict_lock); #if MALI_USE_CSF - up_read(&kctx->kbdev->csf.pmode_sync_sem); + up_read(&kctx->kbdev->csf.mmu_sync_sem); #endif return freed; } @@ -886,7 +880,7 @@ void kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) /* Indicate to page migration that the memory can be reclaimed by the shrinker. */ if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(gpu_alloc, NOT_MOVABLE); + kbase_set_phy_alloc_page_status(kctx, gpu_alloc, NOT_MOVABLE); mutex_unlock(&kctx->jit_evict_lock); kbase_mem_evictable_mark_reclaim(gpu_alloc); @@ -944,7 +938,7 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) * from. */ if (kbase_is_page_migration_enabled()) - kbase_set_phy_alloc_page_status(gpu_alloc, ALLOCATED_MAPPED); + kbase_set_phy_alloc_page_status(kctx, gpu_alloc, ALLOCATED_MAPPED); } } @@ -962,7 +956,8 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) * * Return: 0 on success, error code otherwise. */ -static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, unsigned int flags, +static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, + base_mem_alloc_flags flags, struct kbase_va_region *reg) { unsigned int real_flags = 0; @@ -1045,7 +1040,7 @@ static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, unsig * * Return: 0 on success, error code otherwise. */ -static int kbase_mem_flags_change_native(struct kbase_context *kctx, unsigned int flags, +static int kbase_mem_flags_change_native(struct kbase_context *kctx, base_mem_alloc_flags flags, struct kbase_va_region *reg) { bool kbase_reg_dont_need_flag = (KBASE_REG_DONT_NEED & reg->flags); @@ -1077,8 +1072,8 @@ static int kbase_mem_flags_change_native(struct kbase_context *kctx, unsigned in return ret; } -int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, - unsigned int mask) +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, base_mem_alloc_flags flags, + base_mem_alloc_flags mask) { struct kbase_va_region *reg; int ret = -EINVAL; @@ -1468,7 +1463,7 @@ static int get_umm_memory_group_id(struct kbase_context *kctx, struct dma_buf *d * object that wraps the dma-buf. */ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, - u64 *flags, u32 padding) + base_mem_alloc_flags *flags, u32 padding) { struct kbase_va_region *reg; struct dma_buf *dma_buf; @@ -1614,7 +1609,8 @@ u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev) static struct kbase_va_region *kbase_mem_from_user_buffer(struct kbase_context *kctx, unsigned long address, unsigned long size, - u64 *va_pages, u64 *flags) + u64 *va_pages, + base_mem_alloc_flags *flags) { struct kbase_va_region *reg; enum kbase_memory_zone zone = CUSTOM_VA_ZONE; @@ -1746,7 +1742,7 @@ bad_size: return NULL; } -u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, +u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages) { struct kbase_va_region *reg; @@ -1968,7 +1964,8 @@ bad_flags: } int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, u64 *flags) + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + base_mem_alloc_flags *flags) { struct kbase_va_region *reg; @@ -3107,7 +3104,7 @@ static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_regi return -ENOMEM; } - if (reg->flags & KBASE_REG_DONT_NEED) + if (kbase_is_region_shrinkable(reg)) return -EINVAL; prot = PAGE_KERNEL; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h index 28666037d8c6..a4b3db7fdf89 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_linux.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -51,7 +51,7 @@ struct kbase_hwc_dma_mapping { * Return: 0 on success or error code */ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages, - u64 extension, u64 *flags, u64 *gpu_va, + u64 extension, base_mem_alloc_flags *flags, u64 *gpu_va, enum kbase_caller_mmu_sync_info mmu_sync_info); /** @@ -84,7 +84,8 @@ int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *co * Return: 0 on success or error code */ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, u64 *flags); + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, + base_mem_alloc_flags *flags); /** * kbase_mem_alias - Create a new allocation for GPU, aliasing one or more @@ -99,7 +100,7 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, * * Return: 0 on failure or otherwise the GPU VA for the alias */ -u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, +u64 kbase_mem_alias(struct kbase_context *kctx, base_mem_alloc_flags *flags, u64 stride, u64 nents, struct base_mem_aliasing_info *ai, u64 *num_pages); /** @@ -112,8 +113,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nent * * Return: 0 on success or error code */ -int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, - unsigned int mask); +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, base_mem_alloc_flags flags, + base_mem_alloc_flags mask); /** * kbase_mem_commit - Change the physical backing size of a region diff --git a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c index f0ce0cf4e56c..eecab323f59f 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_mem_migrate.c @@ -68,6 +68,12 @@ bool kbase_alloc_page_metadata(struct kbase_device *kbdev, struct page *p, dma_a if (!IS_ENABLED(CONFIG_PAGE_MIGRATION_SUPPORT)) return false; + /* Composite large-page is excluded from migration, trigger a warn if a development + * wrongly leads to it. + */ + if (is_huge_head(as_tagged(page_to_phys(p))) || is_partial(as_tagged(page_to_phys(p)))) + dev_WARN(kbdev->dev, "%s: migration-metadata attempted on large-page.", __func__); + page_md = kzalloc(sizeof(struct kbase_page_metadata), GFP_KERNEL); if (!page_md) return false; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c index d688509cee03..f9a3788a2ecf 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_native_mgm.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -122,12 +122,16 @@ static vm_fault_t kbase_native_mgm_vmf_insert_pfn_prot(struct memory_group_manag } static u64 kbase_native_mgm_update_gpu_pte(struct memory_group_manager_device *mgm_dev, - unsigned int group_id, int mmu_level, u64 pte) + unsigned int group_id, unsigned int pbha_id, + unsigned int pte_flags, int mmu_level, u64 pte) { if (WARN_ON(group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS)) return pte; - pte |= ((u64)group_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + if ((pte_flags & BIT(MMA_VIOLATION)) && pbha_id) { + pr_warn_once("MMA violation! Applying PBHA override workaround to PTE\n"); + pte |= ((u64)pbha_id << PTE_PBHA_SHIFT) & PTE_PBHA_MASK; + } /* Address could be translated into a different bus address here */ pte |= ((u64)1 << PTE_RES_BIT_MULTI_AS_SHIFT); @@ -150,14 +154,24 @@ static u64 kbase_native_mgm_pte_to_original_pte(struct memory_group_manager_devi return pte; } +static bool kbase_native_mgm_get_import_memory_cached_access_permitted( + struct memory_group_manager_device *mgm_dev, + struct memory_group_manager_import_data *import_data) +{ + CSTD_UNUSED(mgm_dev); + CSTD_UNUSED(import_data); + + return true; +} + struct memory_group_manager_device kbase_native_mgm_dev = { - .ops = { - .mgm_alloc_page = kbase_native_mgm_alloc, - .mgm_free_page = kbase_native_mgm_free, - .mgm_get_import_memory_id = NULL, - .mgm_vmf_insert_pfn_prot = kbase_native_mgm_vmf_insert_pfn_prot, - .mgm_update_gpu_pte = kbase_native_mgm_update_gpu_pte, - .mgm_pte_to_original_pte = kbase_native_mgm_pte_to_original_pte, - }, + .ops = { .mgm_alloc_page = kbase_native_mgm_alloc, + .mgm_free_page = kbase_native_mgm_free, + .mgm_get_import_memory_id = NULL, + .mgm_vmf_insert_pfn_prot = kbase_native_mgm_vmf_insert_pfn_prot, + .mgm_update_gpu_pte = kbase_native_mgm_update_gpu_pte, + .mgm_pte_to_original_pte = kbase_native_mgm_pte_to_original_pte, + .mgm_get_import_memory_cached_access_permitted = + kbase_native_mgm_get_import_memory_cached_access_permitted }, .data = NULL }; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c index c5b6fada2451..ea79811ea293 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2021-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2021-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -220,6 +220,24 @@ void kbase_pbha_write_settings(struct kbase_device *kbdev) for (i = 0; i < GPU_SYSC_ALLOC_COUNT; ++i) kbase_reg_write32(kbdev, GPU_SYSC_ALLOC_OFFSET(i), kbdev->sysc_alloc[i]); } + + if (kbdev->mma_wa_id) { + /* PBHA OVERRIDE register index (0-3) */ + uint reg_index = kbdev->mma_wa_id >> 2; + /* PBHA index within a PBHA OVERRIDE register (0-3) */ + uint pbha_index = kbdev->mma_wa_id & 0x3; + /* 4 bits of read attributes + 4 bits of write attributes for each PBHA */ + uint pbha_shift = pbha_index * 8; + /* Noncacheable read = noncacheable write = b0001*/ + uint pbha_override_rw_noncacheable = 0x01 | 0x10; + + u32 pbha_override_val = + kbase_reg_read32(kbdev, GPU_SYSC_PBHA_OVERRIDE_OFFSET(reg_index)); + pbha_override_val &= ~((u32)0xFF << pbha_shift); + pbha_override_val |= ((u32)pbha_override_rw_noncacheable << pbha_shift); + kbase_reg_write32(kbdev, GPU_SYSC_PBHA_OVERRIDE_OFFSET(reg_index), + pbha_override_val); + } #else CSTD_UNUSED(kbdev); #endif /* MALI_USE_CSF */ @@ -280,7 +298,7 @@ static int kbase_pbha_read_propagate_bits_property(struct kbase_device *kbdev, u8 bits = 0; int err; - if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) + if (!kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) return 0; err = of_property_read_u8(pbha_node, "propagate-bits", &bits); @@ -310,6 +328,43 @@ static int kbase_pbha_read_propagate_bits_property(struct kbase_device *kbdev, kbdev->pbha_propagate_bits = bits; return 0; } + +static int kbase_pbha_read_mma_wa_id_property(struct kbase_device *kbdev, + const struct device_node *pbha_node) +{ + u32 mma_wa_id = 0; + int err; + + /* Skip if kbdev->mma_wa_id has already been set via the module parameter */ + if ((kbdev->gpu_props.gpu_id.arch_id < GPU_ID_ARCH_MAKE(14, 8, 0)) || kbdev->mma_wa_id != 0) + return 0; + + err = of_property_read_u32(pbha_node, "mma-wa-id", &mma_wa_id); + + /* Property does not exist. This is not a mandatory property, ignore this error */ + if (err == -EINVAL) + return 0; + + if (err == -ENODATA) { + dev_err(kbdev->dev, "DTB property mma-wa-id has no value\n"); + return err; + } + + if (err == -EOVERFLOW) { + dev_err(kbdev->dev, "DTB value for mma-wa-id is out of range\n"); + return err; + } + + if (mma_wa_id == 0 || mma_wa_id > 15) { + dev_err(kbdev->dev, + "Invalid DTB value for mma-wa-id: %u. Valid range is between 1 and 15.\n", + mma_wa_id); + return -EINVAL; + } + + kbdev->mma_wa_id = mma_wa_id; + return 0; +} #endif /* MALI_USE_CSF */ int kbase_pbha_read_dtb(struct kbase_device *kbdev) @@ -331,6 +386,12 @@ int kbase_pbha_read_dtb(struct kbase_device *kbdev) return err; err = kbase_pbha_read_propagate_bits_property(kbdev, pbha_node); + + if (err < 0) + return err; + + err = kbase_pbha_read_mma_wa_id_property(kbdev, pbha_node); + return err; #else return 0; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c index 8ab0d1823165..81f2df5ea977 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pbha_debugfs.c @@ -234,7 +234,7 @@ void kbase_pbha_debugfs_init(struct kbase_device *kbdev) debugfs_create_file("int_id_overrides", mode, debugfs_pbha_dir, kbdev, &pbha_int_id_overrides_fops); #if MALI_USE_CSF - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PBHA_HWU)) + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PBHA_HWU)) debugfs_create_file("propagate_bits", mode, debugfs_pbha_dir, kbdev, &pbha_propagate_bits_fops); #endif /* MALI_USE_CSF */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.c b/drivers/gpu/arm/bifrost/mali_kbase_pm.c index 17c34f334aad..6719a120c1f3 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.c @@ -31,9 +31,7 @@ #include #include -#ifdef CONFIG_MALI_ARBITER_SUPPORT #include -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #include @@ -62,10 +60,11 @@ int kbase_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, current->pid); lockdep_assert_held(&kbdev->pm.lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT + /* If there is an Arbiter, wait for Arbiter to grant GPU back to KBase + * so suspend request can be handled. + */ if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, suspend_handler)) return 1; -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ if (kbase_pm_is_suspending(kbdev)) { switch (suspend_handler) { @@ -91,9 +90,7 @@ int kbase_pm_context_active_handle_suspend_locked(struct kbase_device *kbdev, * any cores requested by the policy */ kbase_hwaccess_pm_gpu_active(kbdev); -#ifdef CONFIG_MALI_ARBITER_SUPPORT kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_REF_EVENT); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ kbase_clk_rate_trace_manager_gpu_active(kbdev); } @@ -222,8 +219,7 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) kbdev->pm.suspending = true; mutex_unlock(&kbdev->pm.lock); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { unsigned long flags; #if MALI_USE_CSF @@ -241,7 +237,6 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); #endif } -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ /* From now on, the active count will drop towards zero. Sometimes, * it'll go up briefly before going down again. However, once @@ -287,19 +282,17 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev) */ if (kbase_hwaccess_pm_suspend(kbdev)) { /* No early return yet */ - if (IS_ENABLED(CONFIG_MALI_ARBITER_SUPPORT)) + if (kbase_has_arbiter(kbdev)) WARN_ON_ONCE(1); else goto exit; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) { + if (kbase_has_arbiter(kbdev)) { mutex_lock(&kbdev->pm.arb_vm_state->vm_state_lock); kbase_arbiter_pm_vm_stopped(kbdev); mutex_unlock(&kbdev->pm.arb_vm_state->vm_state_lock); } -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #if MALI_USE_CSF kbase_backend_invalidate_gpu_timestamp_offset(kbdev); @@ -339,14 +332,13 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) kbase_hwaccess_pm_resume(kbdev); /* Initial active call, to power on the GPU/cores if needed */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbase_pm_context_active_handle_suspend( - kbdev, (arb_gpu_start ? KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : - KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE))) - return; -#else - kbase_pm_context_active(kbdev); -#endif + if (kbase_has_arbiter(kbdev)) { + if (kbase_pm_context_active_handle_suspend( + kbdev, (arb_gpu_start ? KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : + KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE))) + return; + } else + kbase_pm_context_active(kbdev); resume_job_scheduling(kbdev); @@ -371,26 +363,18 @@ int kbase_pm_suspend(struct kbase_device *kbdev) { int result = 0; -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) + if (kbase_has_arbiter(kbdev)) kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_SUSPEND_EVENT); else result = kbase_pm_driver_suspend(kbdev); -#else - result = kbase_pm_driver_suspend(kbdev); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ return result; } void kbase_pm_resume(struct kbase_device *kbdev) { -#ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbdev->arb.arb_if) + if (kbase_has_arbiter(kbdev)) kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_OS_RESUME_EVENT); else kbase_pm_driver_resume(kbdev, false); -#else - kbase_pm_driver_resume(kbdev, false); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ } diff --git a/drivers/gpu/arm/bifrost/mali_kbase_pm.h b/drivers/gpu/arm/bifrost/mali_kbase_pm.h index 0c3575bfd54a..25e4732a8d08 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_pm.h +++ b/drivers/gpu/arm/bifrost/mali_kbase_pm.h @@ -120,12 +120,10 @@ enum kbase_pm_suspend_handler { * (e.g. guarantee it's going to be idled very soon after) */ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE, -#ifdef CONFIG_MALI_ARBITER_SUPPORT /** Special case when Arbiter has notified we can use GPU. * Active count should always start at 0 in this case. */ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED, -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ }; /** @@ -234,7 +232,7 @@ void kbase_pm_vsync_callback(int buffer_updated, void *data); * kbase components to complete the suspend. * * Despite kbase_pm_suspend(), it will ignore to update Arbiter - * status if MALI_ARBITER_SUPPORT is enabled. + * status if there is one. * * @note the mechanisms used here rely on all user-space threads being frozen * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up @@ -258,11 +256,10 @@ int kbase_pm_driver_suspend(struct kbase_device *kbdev); * Also called when using VM arbiter, when GPU access has been granted. * * Despite kbase_pm_resume(), it will ignore to update Arbiter - * status if MALI_ARBITER_SUPPORT is enabled. + * status if there is one. */ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start); -#ifdef CONFIG_MALI_ARBITER_SUPPORT /** * kbase_pm_handle_gpu_lost() - Handle GPU Lost for the VM * @kbdev: Device pointer @@ -273,6 +270,5 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start); * Kill any running tasks and put the driver into a GPU powered-off state. */ void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ #endif /* _KBASE_PM_H_ */ diff --git a/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c b/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c index 3128292a9a30..e490a2a3d179 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_reg_track.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2023-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -648,7 +648,7 @@ int kbase_add_va_region_rbtree(struct kbase_device *kbdev, struct kbase_va_regio } else if (!kbase_is_region_free(tmp)) { dev_warn( dev, - "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%lx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", + "!(tmp->flags & KBASE_REG_FREE): tmp->start_pfn=0x%llx tmp->flags=0x%llx tmp->nr_pages=0x%zx gpu_pfn=0x%llx nr_pages=0x%zx\n", tmp->start_pfn, tmp->flags, tmp->nr_pages, gpu_pfn, nr_pages); err = -ENOMEM; goto exit; diff --git a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c index c5c16b497369..bae1630c94a9 100644 --- a/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c +++ b/drivers/gpu/arm/bifrost/mali_kbase_softjobs.c @@ -143,9 +143,8 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) * delay suspend until we process the atom (which may be at the end of a * long chain of dependencies */ -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_inc(&kctx->kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kctx->kbdev)) + atomic_inc(&kctx->kbdev->pm.gpu_users_waiting); pm_active_err = kbase_pm_context_active_handle_suspend( kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); if (pm_active_err) { @@ -163,11 +162,8 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) kbasep_add_waiting_soft_job(katom); return pm_active_err; - } -#ifdef CONFIG_MALI_ARBITER_SUPPORT - else + } else if (kbase_has_arbiter(kctx->kbdev)) atomic_dec(&kctx->kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, &ts); @@ -553,7 +549,7 @@ static int kbase_debug_copy_prepare(struct kbase_jd_atom *katom) goto out_cleanup; } - ret = copy_from_user(user_buffers, user_structs, sizeof(*user_buffers) * nr); + ret = copy_from_user(user_buffers, user_structs, size_mul(sizeof(*user_buffers), nr)); if (ret) { ret = -EFAULT; goto out_cleanup; @@ -1235,7 +1231,7 @@ static int kbase_jit_free_prepare(struct kbase_jd_atom *katom) goto free_info; } - if (copy_from_user(ids, data, sizeof(*ids) * count) != 0) { + if (copy_from_user(ids, data, size_mul(sizeof(*ids), count)) != 0) { ret = -EINVAL; goto free_info; } @@ -1688,9 +1684,8 @@ void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) if (kbase_process_soft_job(katom_iter) == 0) { kbase_finish_soft_job(katom_iter); resched |= kbase_jd_done_nolock(katom_iter, true); -#ifdef CONFIG_MALI_ARBITER_SUPPORT - atomic_dec(&kbdev->pm.gpu_users_waiting); -#endif /* CONFIG_MALI_ARBITER_SUPPORT */ + if (kbase_has_arbiter(kctx->kbdev)) + atomic_dec(&kbdev->pm.gpu_users_waiting); } mutex_unlock(&kctx->jctx.lock); } diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c index bd5f3914b8c7..196d481d6827 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_csf.c @@ -100,7 +100,7 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, struct u32 as_no; /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) { dev_err(kbdev->dev, "Unexpected Page fault in firmware address space at VA 0x%016llX\n" "raw fault status: 0x%X\n" @@ -115,22 +115,6 @@ void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, struct fault_source_id_internal_requester_get(kbdev, source_id), fault_source_id_core_type_description_get(kbdev, source_id), fault_source_id_internal_requester_get_str(kbdev, source_id, access_type)); - } else { - dev_err(kbdev->dev, - "Unexpected Page fault in firmware address space at VA 0x%016llX\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n", - fault->addr, fault->status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(fault->status), source_id, - FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type)); } kbase_debug_csf_fault_notify(kbdev, NULL, DF_GPU_PAGE_FAULT); @@ -163,7 +147,7 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas const uintptr_t fault_addr = fault->addr; /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) { dev_err(kbdev->dev, "GPU bus fault in AS%u at PA %pK\n" "PA_VALID: %s\n" @@ -181,25 +165,6 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas fault_source_id_core_type_description_get(kbdev, source_id), fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), kctx->pid); - } else { - dev_err(kbdev->dev, - "GPU bus fault in AS%u at PA %pK\n" - "PA_VALID: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n" - "pid: %d\n", - as_no, (void *)fault_addr, addr_valid, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(access_type), source_id, - FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), - kctx->pid); } /* AS transaction begin */ @@ -246,7 +211,7 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as unsigned int as_no = as->number; /* terminal fault, print info about the fault */ - if (kbdev->gpu_props.gpu_id.product_model <= GPU_ID_MODEL_MAKE(13, 0)) { + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(14, 0)) { dev_err(kbdev->dev, "Unhandled Page fault in AS%u at VA 0x%016llX\n" "Reason: %s\n" @@ -265,26 +230,6 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), kctx->pid); - } else { - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (type:idx:IR 0x%X:0x%X:0x%X): %s %u, %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(status), source_id, - FAULT_SOURCE_ID_CORE_TYPE_GET(source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - FAULT_SOURCE_ID_CORE_INDEX_GET(source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, - access_type), - kctx->pid); } } @@ -294,6 +239,14 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as * will abort all jobs and stop any hw counter dumping */ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); + /* Update the page fault counter value in firmware visible memory, just before disabling + * the MMU which would in turn unblock the MCU firmware. + */ + if (kbdev->csf.page_fault_cnt_ptr) { + spin_lock(&kbdev->mmu_mask_change); + *kbdev->csf.page_fault_cnt_ptr = ++kbdev->csf.page_fault_cnt; + spin_unlock(&kbdev->mmu_mask_change); + } kbase_mmu_disable(kctx); kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); kbase_debug_csf_fault_notify(kbdev, kctx, DF_GPU_PAGE_FAULT); @@ -487,15 +440,6 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); } -int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, struct kbase_va_region *const reg) -{ - CSTD_UNUSED(kctx); - CSTD_UNUSED(reg); - - /* Can't soft-stop the provoking job */ - return -EPERM; -} - /** * kbase_mmu_gpu_fault_worker() - Process a GPU fault for the device. * diff --git a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c index e8e136117fd8..a7f3f40ef325 100644 --- a/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c +++ b/drivers/gpu/arm/bifrost/mmu/backend/mali_kbase_mmu_jm.c @@ -119,23 +119,37 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as const u32 access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); const u32 source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); /* terminal fault, print info about the fault */ - dev_err(kbdev->dev, - "Unhandled Page fault in AS%u at VA 0x%016llX\n" - "Reason: %s\n" - "raw fault status: 0x%X\n" - "exception type 0x%X: %s\n" - "access type 0x%X: %s\n" - "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" - "pid: %d\n", - as_no, fault->addr, reason_str, status, exception_type, - kbase_gpu_exception_name(exception_type), access_type, - kbase_gpu_access_type_name(status), source_id, - FAULT_SOURCE_ID_CORE_ID_GET(source_id), - FAULT_SOURCE_ID_UTLB_ID_GET(source_id), - fault_source_id_internal_requester_get(kbdev, source_id), - fault_source_id_core_type_description_get(kbdev, source_id), - fault_source_id_internal_requester_get_str(kbdev, source_id, access_type), - kctx->pid); + if (kbdev->gpu_props.gpu_id.product_model < GPU_ID_MODEL_MAKE(9, 0)) { + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "pid: %d\n", + as_no, fault->addr, reason_str, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(status), kctx->pid); + } else { + dev_err(kbdev->dev, + "Unhandled Page fault in AS%u at VA 0x%016llX\n" + "Reason: %s\n" + "raw fault status: 0x%X\n" + "exception type 0x%X: %s\n" + "access type 0x%X: %s\n" + "source id 0x%X (core_id:utlb:IR 0x%X:0x%X:0x%X): %s, %s\n" + "pid: %d\n", + as_no, fault->addr, reason_str, status, exception_type, + kbase_gpu_exception_name(exception_type), access_type, + kbase_gpu_access_type_name(status), source_id, + FAULT_SOURCE_ID_CORE_ID_GET(source_id), + FAULT_SOURCE_ID_UTLB_ID_GET(source_id), + fault_source_id_internal_requester_get(kbdev, source_id), + fault_source_id_core_type_description_get(kbdev, source_id), + fault_source_id_internal_requester_get_str(kbdev, source_id, + access_type), + kctx->pid); + } } /* hardware counters dump fault handling */ @@ -271,7 +285,7 @@ static void validate_protected_page_fault(struct kbase_device *kbdev) */ u32 protected_debug_mode = 0; - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { + if (kbase_hw_has_feature(kbdev, KBASE_HW_FEATURE_PROTECTED_DEBUG_MODE)) { protected_debug_mode = kbase_reg_read32(kbdev, GPU_CONTROL_ENUM(GPU_STATUS)) & GPU_STATUS_GPU_DBG_ENABLED; } @@ -387,13 +401,6 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) dev_dbg(kbdev->dev, "Leaving %s irq_stat %u\n", __func__, irq_stat); } -int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, struct kbase_va_region *const reg) -{ - dev_dbg(kctx->kbdev->dev, "Switching to incremental rendering for region %pK\n", - (void *)reg); - return kbase_job_slot_softstop_start_rp(kctx, reg); -} - int kbase_mmu_as_init(struct kbase_device *kbdev, unsigned int i) { kbdev->as[i].number = i; diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c index 8d52e90f9579..4963d990054f 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include #if !MALI_USE_CSF @@ -58,6 +57,257 @@ /* Macro to convert updated PDGs to flags indicating levels skip in flush */ #define pgd_level_to_skip_flush(dirty_pgds) (~(dirty_pgds)&0xF) +/** + * kmap_pgd() - Map a PGD page and return the address of it + * + * @p: Pointer to the PGD page to be mapped. + * @pgd: The physical address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + * + * Return: The mapped address of the @pgd, adjusted by the offset of @pgd from the start of page. + */ +static inline void *kmap_pgd(struct page *p, phys_addr_t pgd) +{ +#if GPU_PAGES_PER_CPU_PAGE > 1 + return kbase_kmap(p) + (pgd & ~PAGE_MASK); +#else + CSTD_UNUSED(pgd); + return kbase_kmap(p); +#endif +} + +/** + * kmap_atomic_pgd() - Variant of kmap_pgd for atomic mapping + * + * @p: Pointer to the PGD page to be mapped. + * @pgd: The physical address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + * + * Return: The mapped address of the @pgd. + */ +static inline void *kmap_atomic_pgd(struct page *p, phys_addr_t pgd) +{ +#if GPU_PAGES_PER_CPU_PAGE > 1 + return kbase_kmap_atomic(p) + (pgd & ~PAGE_MASK); +#else + CSTD_UNUSED(pgd); + return kbase_kmap_atomic(p); +#endif +} + +/** + * kunmap_pgd() - Unmap a PGD page + * + * @p: Pointer to the PGD page to be unmapped. + * @pgd_address: The address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + */ +static inline void kunmap_pgd(struct page *p, void *pgd_address) +{ + /* It is okay to not align pgd_address to PAGE_SIZE boundary */ + kbase_kunmap(p, pgd_address); +} + +/** + * kunmap_atomic_pgd() - Variant of kunmap_pgd for atomic unmapping + * + * @pgd_address: The address of the PGD. May not be PAGE_SIZE aligned but shall be + * GPU_PAGE_SIZE aligned. + */ +static inline void kunmap_atomic_pgd(void *pgd_address) +{ + /* It is okay to not align pgd_address to PAGE_SIZE boundary */ + kbase_kunmap_atomic(pgd_address); +} + +/** + * pgd_dma_addr() - Return dma addr of a PGD + * + * @p: Pointer to the PGD page. + * @pgd: The physical address of the PGD. + * + * Return: DMA address of the PGD + */ +static inline dma_addr_t pgd_dma_addr(struct page *p, phys_addr_t pgd) +{ +#if GPU_PAGES_PER_CPU_PAGE > 1 + return kbase_page_private(p)->dma_addr + (pgd & ~PAGE_MASK); +#else + CSTD_UNUSED(pgd); + return kbase_dma_addr(p); +#endif +} + +/** + * get_pgd_sub_page_index() - Return the index of a sub PGD page in the PGD page. + * + * @pgd: The physical address of the PGD. + * + * Return: The index value ranging from 0 to (GPU_PAGES_PER_CPU_PAGE - 1) + */ +static inline u32 get_pgd_sub_page_index(phys_addr_t pgd) +{ + return (pgd & ~PAGE_MASK) / GPU_PAGE_SIZE; +} + +#if GPU_PAGES_PER_CPU_PAGE > 1 +/** + * alloc_pgd_page_metadata() - Allocate page metadata for a PGD. + * + * @kbdev: Pointer to the instance of a kbase device. + * @mmut: Structure holding details of the MMU table for a kcontext. + * @p: PGD page. + * + * The PGD page, @p is linked to &kbase_mmu_table.pgd_pages_list for allocating + * sub PGD pages from the list. + * + * Return: True on success. + */ +static bool alloc_pgd_page_metadata(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, + struct page *p) +{ + struct kbase_page_metadata *page_md; + + if (!kbase_is_page_migration_enabled()) { + page_md = kmem_cache_zalloc(kbdev->page_metadata_slab, GFP_KERNEL); + if (!page_md) + return false; + + page_md->dma_addr = kbase_dma_addr_as_priv(p); + set_page_private(p, (unsigned long)page_md); + } else { + page_md = kbase_page_private(p); + } + + page_md->data.pt_mapped.num_allocated_sub_pages = 1; + set_bit(0, page_md->data.pt_mapped.allocated_sub_pages); + page_md->data.pt_mapped.pgd_page = p; + list_add(&page_md->data.pt_mapped.pgd_link, &mmut->pgd_pages_list); + + return true; +} + +/** + * free_pgd_page_metadata() - Free page metadata for a PGD. + * + * @kbdev: Pointer to the instance of a kbase device. + * @p: PGD page where the metadata belongs to. + * + * The PGD page, @p is removed from &kbase_mmu_table.pgd_pages_list. + */ +static void free_pgd_page_metadata(struct kbase_device *kbdev, struct page *p) +{ + struct kbase_page_metadata *page_md = kbase_page_private(p); + + WARN_ON_ONCE(page_md->data.pt_mapped.num_allocated_sub_pages); + page_md->data.pt_mapped.pgd_page = NULL; + list_del_init(&page_md->data.pt_mapped.pgd_link); + + if (kbase_is_page_migration_enabled()) + return; + + set_page_private(p, (unsigned long)page_md->dma_addr); + kmem_cache_free(kbdev->page_metadata_slab, page_md); +} + +/** + * allocate_pgd_sub_page() - Allocate a PGD sub page + * + * @page_md: Page metadata of a PGD page where a sub page is allocated from. + * + * Return: Physical address of allocated PGD sub page on success. + * KBASE_INVALID_PHYSICAL_ADDRESS on failure. + */ +static inline phys_addr_t allocate_pgd_sub_page(struct kbase_page_metadata *page_md) +{ + unsigned long sub_page_index; + + if (page_md->data.pt_mapped.num_allocated_sub_pages == GPU_PAGES_PER_CPU_PAGE) + return KBASE_INVALID_PHYSICAL_ADDRESS; + sub_page_index = find_first_zero_bit(page_md->data.pt_mapped.allocated_sub_pages, + GPU_PAGES_PER_CPU_PAGE); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (WARN_ON_ONCE(sub_page_index >= GPU_PAGES_PER_CPU_PAGE)) + return KBASE_INVALID_PHYSICAL_ADDRESS; + if (WARN_ON_ONCE(page_md->data.pt_mapped.num_allocated_sub_pages > GPU_PAGES_PER_CPU_PAGE)) + return KBASE_INVALID_PHYSICAL_ADDRESS; +#endif + set_bit(sub_page_index, page_md->data.pt_mapped.allocated_sub_pages); + page_md->data.pt_mapped.num_allocated_sub_pages++; + + return (page_to_phys(page_md->data.pt_mapped.pgd_page) + (sub_page_index * GPU_PAGE_SIZE)); +} + +/** + * free_pgd_sub_page() - Free a PGD sub page + * + * @pgd: Sub PGD to be freed. + * + * Return: The number of remaining allocated sub pages in the PGD. + */ +static int free_pgd_sub_page(phys_addr_t pgd) +{ + struct page *p = pfn_to_page(PFN_DOWN(pgd)); + struct kbase_page_metadata *page_md = kbase_page_private(p); + const u32 sub_page_index = get_pgd_sub_page_index(pgd); + +#ifdef CONFIG_MALI_BIFROST_DEBUG + if (WARN_ON_ONCE(!test_bit(sub_page_index, page_md->data.pt_mapped.allocated_sub_pages))) + return page_md->data.pt_mapped.num_allocated_sub_pages; +#endif + clear_bit(sub_page_index, page_md->data.pt_mapped.allocated_sub_pages); + if (!WARN_ON_ONCE(page_md->data.pt_mapped.num_allocated_sub_pages <= 0)) + page_md->data.pt_mapped.num_allocated_sub_pages--; + + return page_md->data.pt_mapped.num_allocated_sub_pages; +} + +/** + * allocate_from_pgd_pages_list() - Allocate a PGD from the PGD pages list + * + * @mmut: Structure holding details of the MMU table for a kcontext. + * + * Return: Physical address of the allocated PGD. + */ +static inline phys_addr_t allocate_from_pgd_pages_list(struct kbase_mmu_table *mmut) +{ + struct list_head *entry; + phys_addr_t pgd; + + lockdep_assert_held(&mmut->mmu_lock); + + if (unlikely(!mmut->num_free_pgd_sub_pages)) + return KBASE_INVALID_PHYSICAL_ADDRESS; + + if (mmut->last_allocated_pgd_page) { + pgd = allocate_pgd_sub_page(kbase_page_private(mmut->last_allocated_pgd_page)); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + goto success; + } + + if (mmut->last_freed_pgd_page) { + pgd = allocate_pgd_sub_page(kbase_page_private(mmut->last_freed_pgd_page)); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + goto success; + } + + list_for_each(entry, &mmut->pgd_pages_list) { + struct kbase_page_metadata *page_md = + list_entry(entry, struct kbase_page_metadata, data.pt_mapped.pgd_link); + + pgd = allocate_pgd_sub_page(page_md); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + goto success; + } + + return KBASE_INVALID_PHYSICAL_ADDRESS; +success: + mmut->num_free_pgd_sub_pages--; + return pgd; +} +#endif + static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, const u64 start_vpfn, struct tagged_addr *phys, size_t nr, unsigned long flags, int const group_id, u64 *dirty_pgds, @@ -417,7 +667,7 @@ static bool kbase_mmu_handle_isolated_pgd_page(struct kbase_device *kbdev, * PGD page, which is done inside kbase_mmu_free_pgd() for the * PGD page that did not get isolated. */ - dma_sync_single_for_device(kbdev->dev, kbase_dma_addr(p), PAGE_SIZE, + dma_sync_single_for_device(kbdev->dev, pgd_dma_addr(p, page_to_phys(p)), PAGE_SIZE, DMA_BIDIRECTIONAL); kbase_mmu_account_freed_pgd(kbdev, mmut); } @@ -444,6 +694,20 @@ static void kbase_mmu_free_pgd(struct kbase_device *kbdev, struct kbase_mmu_tabl lockdep_assert_held(&mmut->mmu_lock); p = pfn_to_page(PFN_DOWN(pgd)); +#if GPU_PAGES_PER_CPU_PAGE > 1 + if (free_pgd_sub_page(pgd)) { + mmut->num_free_pgd_sub_pages++; + mmut->last_freed_pgd_page = p; + return; + } + + mmut->num_free_pgd_sub_pages -= (GPU_PAGES_PER_CPU_PAGE - 1); + if (p == mmut->last_freed_pgd_page) + mmut->last_freed_pgd_page = NULL; + if (p == mmut->last_allocated_pgd_page) + mmut->last_allocated_pgd_page = NULL; + free_pgd_page_metadata(kbdev, p); +#endif page_is_isolated = kbase_mmu_handle_isolated_pgd_page(kbdev, mmut, p); if (likely(!page_is_isolated)) { @@ -473,19 +737,19 @@ static void kbase_mmu_free_pgds_list(struct kbase_device *kbdev, struct kbase_mm lockdep_assert_held(&mmut->mmu_lock); for (i = 0; i < mmut->scratch_mem.free_pgds.head_index; i++) - kbase_mmu_free_pgd(kbdev, mmut, page_to_phys(mmut->scratch_mem.free_pgds.pgds[i])); + kbase_mmu_free_pgd(kbdev, mmut, mmut->scratch_mem.free_pgds.pgds[i]); mmut->scratch_mem.free_pgds.head_index = 0; } -static void kbase_mmu_add_to_free_pgds_list(struct kbase_mmu_table *mmut, struct page *p) +static void kbase_mmu_add_to_free_pgds_list(struct kbase_mmu_table *mmut, phys_addr_t pgd) { lockdep_assert_held(&mmut->mmu_lock); if (WARN_ON_ONCE(mmut->scratch_mem.free_pgds.head_index > (MAX_FREE_PGDS - 1))) return; - mmut->scratch_mem.free_pgds.pgds[mmut->scratch_mem.free_pgds.head_index++] = p; + mmut->scratch_mem.free_pgds.pgds[mmut->scratch_mem.free_pgds.head_index++] = pgd; } static inline void kbase_mmu_reset_free_pgds_list(struct kbase_mmu_table *mmut) @@ -666,6 +930,14 @@ static void kbase_gpu_mmu_handle_write_fault(struct kbase_context *kctx, return; } + if (unlikely(region->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill( + kctx, faulting_as, "Unexpected write permission fault on an alias region", + &faulting_as->pf_data); + return; + } + pfn_offset = fault_pfn - region->start_pfn; fault_phys_addr = &kbase_get_gpu_phy_pages(region)[pfn_offset]; @@ -781,6 +1053,7 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si * either small or 2 MiB pages, depending on the number of pages requested. * @grow_2mb_pool: Pointer to variable to store which pool needs to grow - true for 2 MiB, false for * pool of small pages. + * @fallback_to_small: Whether fallback to small pages or not * @prealloc_sas: Pointer to kbase_sub_alloc structures * * This function will try to allocate as many pages as possible from the context pool, then if @@ -798,7 +1071,7 @@ static size_t estimate_pool_space_required(struct kbase_mem_pool *pool, const si */ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_region *region, size_t new_pages, size_t *pages_to_grow, bool *grow_2mb_pool, - struct kbase_sub_alloc **prealloc_sas) + bool fallback_to_small, struct kbase_sub_alloc **prealloc_sas) { size_t total_gpu_pages_alloced = 0; size_t total_cpu_pages_alloced = 0; @@ -816,7 +1089,8 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg return false; } - if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) { + if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE && + !fallback_to_small) { root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id]; *grow_2mb_pool = true; } else { @@ -964,6 +1238,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) bool grown = false; size_t pages_to_grow; bool grow_2mb_pool = false; + bool fallback_to_small = false; struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; int i; size_t current_backed_size; @@ -1004,13 +1279,11 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) #endif #endif -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* check if we still have GPU */ if (unlikely(kbase_is_gpu_removed(kbdev))) { dev_dbg(kbdev->dev, "%s: GPU has been removed", __func__); goto fault_done; } -#endif if (unlikely(fault->protected_mode)) { kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Protected mode fault", fault); @@ -1133,7 +1406,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data) } page_fault_retry: - if (kbase_is_large_pages_enabled()) { + if (kbase_is_large_pages_enabled() && !fallback_to_small) { /* Preallocate (or re-allocate) memory for the sub-allocation structs if necessary */ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { if (!prealloc_sas[i]) { @@ -1170,6 +1443,14 @@ page_fault_retry: goto fault_done; } + if (unlikely(region->gpu_alloc->type == KBASE_MEM_TYPE_ALIAS)) { + kbase_gpu_vm_unlock(kctx); + kbase_mmu_report_fault_and_kill(kctx, faulting_as, + "Unexpected page fault on an alias region", + &faulting_as->pf_data); + goto fault_done; + } + if (region->gpu_alloc->group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) { kbase_gpu_vm_unlock(kctx); kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Bad physical memory group ID", @@ -1297,7 +1578,7 @@ page_fault_retry: spin_lock(&kctx->mem_partials_lock); grown = page_fault_try_alloc(kctx, region, new_pages, &pages_to_grow, &grow_2mb_pool, - prealloc_sas); + fallback_to_small, prealloc_sas); spin_unlock(&kctx->mem_partials_lock); if (grown) { @@ -1341,22 +1622,6 @@ page_fault_retry: else trace_mali_mmu_page_fault_grow(region, fault, new_pages); -#if MALI_INCREMENTAL_RENDERING_JM - /* Switch to incremental rendering if we have nearly run out of - * memory in a JIT memory allocation. - */ - if (region->threshold_pages && - kbase_reg_current_backed_size(region) > region->threshold_pages) { - dev_dbg(kctx->kbdev->dev, "%zu pages exceeded IR threshold %zu", - new_pages + current_backed_size, region->threshold_pages); - - if (kbase_mmu_switch_to_ir(kctx, region) >= 0) { - dev_dbg(kctx->kbdev->dev, "Get region %pK for IR", (void *)region); - kbase_va_region_alloc_get(kctx, region); - } - } -#endif - /* AS transaction begin */ /* clear MMU interrupt - this needs to be done after updating @@ -1440,6 +1705,15 @@ page_fault_retry: lp_mem_pool->order; ret = kbase_mem_pool_grow(lp_mem_pool, pages_to_grow, kctx->task); + /* Retry handling the fault with small pages if required + * number of 2MB pages couldn't be allocated. + */ + if (ret < 0) { + fallback_to_small = true; + dev_dbg(kbdev->dev, + "No room for 2MB pages, fallback to small pages"); + goto page_fault_retry; + } } else { struct kbase_mem_pool *const mem_pool = &kctx->mem_pools.small[group_id]; @@ -1484,12 +1758,32 @@ fault_done: dev_dbg(kbdev->dev, "Leaving page_fault_worker %pK", (void *)data); } +/** + * kbase_mmu_alloc_pgd() - Allocate a PGD + * + * @kbdev: Pointer to the instance of a kbase device. + * @mmut: Structure holding details of the MMU table for a kcontext. + * + * A 4KB sized PGD page is allocated for the PGD from the memory pool if PAGE_SIZE is 4KB. + * Otherwise PGD is sub-allocated from a page that is allocated from the memory pool or + * from one of the pages earlier allocated for the PGD of @mmut. + * + * Return: Physical address of the allocated PGD. + */ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_mmu_table *mmut) { u64 *page; struct page *p; phys_addr_t pgd; + lockdep_assert_held(&mmut->mmu_lock); + +#if GPU_PAGES_PER_CPU_PAGE > 1 + pgd = allocate_from_pgd_pages_list(mmut); + if (pgd != KBASE_INVALID_PHYSICAL_ADDRESS) + return pgd; +#endif + p = kbase_mem_pool_alloc(&kbdev->mem_pools.small[mmut->group_id]); if (!p) return KBASE_INVALID_PHYSICAL_ADDRESS; @@ -1499,6 +1793,15 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ if (page == NULL) goto alloc_free; +#if GPU_PAGES_PER_CPU_PAGE > 1 + if (!alloc_pgd_page_metadata(kbdev, mmut, p)) { + kbase_kunmap(p, page); + goto alloc_free; + } + mmut->num_free_pgd_sub_pages += (GPU_PAGES_PER_CPU_PAGE - 1); + mmut->last_allocated_pgd_page = p; +#endif + pgd = page_to_phys(p); /* If the MMU tables belong to a context then account the memory usage @@ -1517,12 +1820,12 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, struct kbase_ kbase_trace_gpu_mem_usage_inc(kbdev, mmut->kctx, 1); - kbdev->mmu_mode->entries_invalidate(page, KBASE_MMU_PAGE_ENTRIES); + kbdev->mmu_mode->entries_invalidate(page, KBASE_MMU_PAGE_ENTRIES * GPU_PAGES_PER_CPU_PAGE); /* As this page is newly created, therefore there is no content to * clean or invalidate in the GPU caches. */ - kbase_mmu_sync_pgd_cpu(kbdev, kbase_dma_addr(p), PAGE_SIZE); + kbase_mmu_sync_pgd_cpu(kbdev, pgd_dma_addr(p, pgd), PAGE_SIZE); kbase_kunmap(p, page); return pgd; @@ -1564,7 +1867,7 @@ static int mmu_get_next_pgd(struct kbase_device *kbdev, struct kbase_mmu_table * vpfn &= 0x1FF; p = pfn_to_page(PFN_DOWN(*pgd)); - page = kbase_kmap(p); + page = kmap_pgd(p, *pgd); if (page == NULL) { dev_err(kbdev->dev, "%s: kmap failure", __func__); return -EINVAL; @@ -1573,7 +1876,7 @@ static int mmu_get_next_pgd(struct kbase_device *kbdev, struct kbase_mmu_table * if (!kbdev->mmu_mode->pte_is_valid(page[vpfn], level)) { dev_dbg(kbdev->dev, "%s: invalid PTE at level %d vpfn 0x%llx", __func__, level, vpfn); - kbase_kunmap(p, page); + kunmap_pgd(p, page); return -EFAULT; } else { target_pgd = kbdev->mmu_mode->pte_to_phy_addr( @@ -1581,7 +1884,7 @@ static int mmu_get_next_pgd(struct kbase_device *kbdev, struct kbase_mmu_table * kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, page[vpfn])); } - kbase_kunmap(p, page); + kunmap_pgd(p, page); *pgd = target_pgd; return 0; @@ -1713,10 +2016,10 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, for (level = MIDGARD_MMU_TOPLEVEL; level <= MIDGARD_MMU_BOTTOMLEVEL; level++) { idx = (vpfn >> ((3 - level) * 9)) & 0x1FF; pgds[level] = pgd; - page = kbase_kmap(p); + page = kmap_pgd(p, pgd); if (mmu_mode->ate_is_valid(page[idx], level)) break; /* keep the mapping */ - kbase_kunmap(p, page); + kunmap_pgd(p, page); pgd = mmu_mode->pte_to_phy_addr(kbdev->mgm_dev->ops.mgm_pte_to_original_pte( kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, page[idx])); p = phys_to_page(pgd); @@ -1751,7 +2054,7 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, if (!num_of_valid_entries) { mmu_mode->set_num_valid_entries(page, 0); - kbase_kunmap(p, page); + kunmap_pgd(p, page); kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level - 1, KBASE_MMU_OP_NONE, dirty_pgds, 0); @@ -1762,7 +2065,7 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, * returned to the memory pool. */ - kbase_mmu_add_to_free_pgds_list(mmut, p); + kbase_mmu_add_to_free_pgds_list(mmut, pgd); vpfn += count; continue; @@ -1774,9 +2077,9 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev, * going to be done by the caller */ kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (idx * sizeof(u64)), - kbase_dma_addr(p) + sizeof(u64) * idx, sizeof(u64) * pcount, + pgd_dma_addr(p, pgd) + sizeof(u64) * idx, sizeof(u64) * pcount, KBASE_MMU_OP_NONE); - kbase_kunmap(p, page); + kunmap_pgd(p, page); next: vpfn += count; } @@ -1895,7 +2198,7 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table goto failure_recovery; } - parent_page_va = kbase_kmap(parent_page); + parent_page_va = kmap_pgd(parent_page, parent_pgd); if (unlikely(parent_page_va == NULL)) { dev_err(kbdev->dev, "%s: kmap failure", __func__); @@ -1907,15 +2210,17 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table kbdev->mmu_mode->entry_set_pte(&pte, target_pgd); parent_page_va[parent_vpfn] = kbdev->mgm_dev->ops.mgm_update_gpu_pte( - kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, parent_index, pte); + kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, PBHA_ID_DEFAULT, PTE_FLAGS_NONE, + parent_index, pte); kbdev->mmu_mode->set_num_valid_entries(parent_page_va, current_valid_entries + 1); - kbase_kunmap(parent_page, parent_page_va); + kunmap_pgd(parent_page, parent_page_va); if (parent_index != insert_level) { /* Newly allocated PGDs */ - kbase_mmu_sync_pgd_cpu( - kbdev, kbase_dma_addr(parent_page) + (parent_vpfn * sizeof(u64)), - sizeof(u64)); + kbase_mmu_sync_pgd_cpu(kbdev, + pgd_dma_addr(parent_page, parent_pgd) + + (parent_vpfn * sizeof(u64)), + sizeof(u64)); } else { /* A new valid entry is added to an existing PGD. Perform the * invalidate operation for GPU cache as it could be having a @@ -1923,7 +2228,7 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table */ kbase_mmu_sync_pgd( kbdev, mmut->kctx, parent_pgd + (parent_vpfn * sizeof(u64)), - kbase_dma_addr(parent_page) + (parent_vpfn * sizeof(u64)), + pgd_dma_addr(parent_page, parent_pgd) + (parent_vpfn * sizeof(u64)), sizeof(u64), KBASE_MMU_OP_FLUSH_PT); } @@ -1934,6 +2239,9 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table spin_lock(&page_md->migrate_lock); +#if GPU_PAGES_PER_CPU_PAGE > 1 + page_md->status = PAGE_STATUS_SET(page_md->status, NOT_MOVABLE); +#else WARN_ON_ONCE(PAGE_STATUS_GET(page_md->status) != ALLOCATE_IN_PROGRESS || IS_PAGE_ISOLATED(page_md->status)); @@ -1945,6 +2253,7 @@ static int update_parent_pgds(struct kbase_device *kbdev, struct kbase_mmu_table } else { page_md->status = PAGE_STATUS_SET(page_md->status, NOT_MOVABLE); } +#endif spin_unlock(&page_md->migrate_lock); } @@ -1957,11 +2266,11 @@ failure_recovery: for (; pgd_index < cur_level; pgd_index++) { phys_addr_t pgd = pgds_to_insert[pgd_index]; struct page *pgd_page = pfn_to_page(PFN_DOWN(pgd)); - u64 *pgd_page_va = kbase_kmap(pgd_page); + u64 *pgd_page_va = kmap_pgd(pgd_page, pgd); u64 vpfn = (insert_vpfn >> ((3 - pgd_index) * 9)) & 0x1FF; kbdev->mmu_mode->entries_invalidate(&pgd_page_va[vpfn], 1); - kbase_kunmap(pgd_page, pgd_page_va); + kunmap_pgd(pgd_page, pgd_page_va); } return err; @@ -1977,6 +2286,8 @@ failure_recovery: * @level_high: The higher bound for the levels for which the PGD allocs are required * @new_pgds: Ptr to an array (size MIDGARD_MMU_BOTTOMLEVEL+1) to write the * newly allocated PGD addresses to. + * @pool_grown: True if new PGDs required the memory pool to grow to allocate more pages, + * or false otherwise * * Numerically, level_low < level_high, not to be confused with top level and * bottom level concepts for MMU PGDs. They are only used as low and high bounds @@ -1987,19 +2298,22 @@ failure_recovery: * * -ENOMEM - allocation failed for a PGD. */ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, - phys_addr_t *new_pgds, int level_low, int level_high) + phys_addr_t *new_pgds, int level_low, int level_high, + bool *pool_grown) { int err = 0; int i; lockdep_assert_held(&mmut->mmu_lock); + *pool_grown = false; for (i = level_low; i <= level_high; i++) { + if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) + continue; do { new_pgds[i] = kbase_mmu_alloc_pgd(kbdev, mmut); if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) break; - mutex_unlock(&mmut->mmu_lock); err = kbase_mem_pool_grow(&kbdev->mem_pools.small[mmut->group_id], (size_t)level_high, NULL); @@ -2007,17 +2321,9 @@ static int mmu_insert_alloc_pgds(struct kbase_device *kbdev, struct kbase_mmu_ta if (err) { dev_err(kbdev->dev, "%s: kbase_mem_pool_grow() returned error %d", __func__, err); - - /* Free all PGDs allocated in previous successful iterations - * from (i-1) to level_low - */ - for (i = (i - 1); i >= level_low; i--) { - if (new_pgds[i] != KBASE_INVALID_PHYSICAL_ADDRESS) - kbase_mmu_free_pgd(kbdev, mmut, new_pgds[i]); - } - return err; } + *pool_grown = true; } while (1); } @@ -2047,6 +2353,8 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp if (WARN_ON(kctx == NULL)) return -EINVAL; + lockdep_assert_held(&kctx->reg_lock); + /* 64-bit address range is the max */ KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); @@ -2082,6 +2390,7 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp struct page *p; register unsigned int num_of_valid_entries; bool newly_created_pgd = false; + bool pool_grown; if (count > remain) count = remain; @@ -2089,6 +2398,10 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp cur_level = MIDGARD_MMU_BOTTOMLEVEL; insert_level = cur_level; + for (l = MIDGARD_MMU_TOPLEVEL + 1; l <= cur_level; l++) + new_pgds[l] = KBASE_INVALID_PHYSICAL_ADDRESS; + +repeat_page_table_walk: /* * Repeatedly calling mmu_get_lowest_valid_pgd() is clearly * suboptimal. We don't have to re-parse the whole tree @@ -2103,7 +2416,7 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp if (err) { dev_err(kbdev->dev, "%s: mmu_get_lowest_valid_pgd() returned error %d", __func__, err); - goto fail_unlock; + goto fail_unlock_free_pgds; } /* No valid pgd at cur_level */ @@ -2112,9 +2425,12 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp * down to the lowest valid pgd at insert_level */ err = mmu_insert_alloc_pgds(kbdev, mmut, new_pgds, (insert_level + 1), - cur_level); + cur_level, &pool_grown); if (err) - goto fail_unlock; + goto fail_unlock_free_pgds; + + if (pool_grown) + goto repeat_page_table_walk; newly_created_pgd = true; @@ -2129,7 +2445,7 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp p = pfn_to_page(PFN_DOWN(pgd)); - pgd_page = kbase_kmap(p); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_err(kbdev->dev, "%s: kmap failure", __func__); err = -ENOMEM; @@ -2168,8 +2484,8 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp flush_op = newly_created_pgd ? KBASE_MMU_OP_NONE : KBASE_MMU_OP_FLUSH_PT; kbase_mmu_sync_pgd(kbdev, kctx, pgd + (vindex * sizeof(u64)), - kbase_dma_addr(p) + (vindex * sizeof(u64)), count * sizeof(u64), - flush_op); + pgd_dma_addr(p, pgd) + (vindex * sizeof(u64)), + count * sizeof(u64), flush_op); if (newly_created_pgd) { err = update_parent_pgds(kbdev, mmut, cur_level, insert_level, insert_vpfn, @@ -2180,14 +2496,14 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp kbdev->mmu_mode->entries_invalidate(&pgd_page[vindex], count); - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); goto fail_unlock_free_pgds; } } insert_vpfn += count; remain -= count; - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); } mutex_unlock(&mmut->mmu_lock); @@ -2200,9 +2516,9 @@ static int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 start_vp fail_unlock_free_pgds: /* Free the pgds allocated by us from insert_level+1 to bottom level */ for (l = cur_level; l > insert_level; l--) - kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); + if (new_pgds[l] != KBASE_INVALID_PHYSICAL_ADDRESS) + kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); -fail_unlock: if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) { /* Invalidate the pages we have partially completed */ mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE, @@ -2326,10 +2642,15 @@ u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, struct tagged_addr co unsigned long const flags, int const level, int const group_id) { u64 entry; + unsigned int pte_flags = 0; kbdev->mmu_mode->entry_set_ate(&entry, phy, flags, level); - return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, (unsigned int)group_id, level, - entry); + + if ((flags & KBASE_REG_GPU_CACHED) && !(flags & KBASE_REG_CPU_CACHED)) + pte_flags |= BIT(MMA_VIOLATION); + + return kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, (unsigned int)group_id, + kbdev->mma_wa_id, pte_flags, level, entry); } static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, @@ -2348,6 +2669,9 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm int l, cur_level, insert_level; struct tagged_addr *start_phys = phys; + if (mmut->kctx) + lockdep_assert_held(&mmut->kctx->reg_lock); + /* Note that 0 is a valid start_vpfn */ /* 64-bit address range is the max */ KBASE_DEBUG_ASSERT(start_vpfn <= (U64_MAX / PAGE_SIZE)); @@ -2370,6 +2694,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm register unsigned int num_of_valid_entries; bool newly_created_pgd = false; enum kbase_mmu_op_type flush_op; + bool pool_grown; if (count > remain) count = remain; @@ -2389,6 +2714,10 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm insert_level = cur_level; + for (l = MIDGARD_MMU_TOPLEVEL + 1; l <= cur_level; l++) + new_pgds[l] = KBASE_INVALID_PHYSICAL_ADDRESS; + +repeat_page_table_walk: /* * Repeatedly calling mmu_get_lowest_valid_pgd() is clearly * suboptimal. We don't have to re-parse the whole tree @@ -2403,7 +2732,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm if (err) { dev_err(kbdev->dev, "%s: mmu_get_lowest_valid_pgd() returned error %d", __func__, err); - goto fail_unlock; + goto fail_unlock_free_pgds; } /* No valid pgd at cur_level */ @@ -2412,9 +2741,12 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm * down to the lowest valid pgd at insert_level */ err = mmu_insert_alloc_pgds(kbdev, mmut, new_pgds, (insert_level + 1), - cur_level); + cur_level, &pool_grown); if (err) - goto fail_unlock; + goto fail_unlock_free_pgds; + + if (pool_grown) + goto repeat_page_table_walk; newly_created_pgd = true; @@ -2428,7 +2760,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm } p = pfn_to_page(PFN_DOWN(pgd)); - pgd_page = kbase_kmap(p); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_err(kbdev->dev, "%s: kmap failure", __func__); @@ -2498,8 +2830,8 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm flush_op = newly_created_pgd ? KBASE_MMU_OP_NONE : KBASE_MMU_OP_FLUSH_PT; kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (vindex * sizeof(u64)), - kbase_dma_addr(p) + (vindex * sizeof(u64)), count * sizeof(u64), - flush_op); + pgd_dma_addr(p, pgd) + (vindex * sizeof(u64)), + count * sizeof(u64), flush_op); if (newly_created_pgd) { err = update_parent_pgds(kbdev, mmut, cur_level, insert_level, insert_vpfn, @@ -2510,7 +2842,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm kbdev->mmu_mode->entries_invalidate(&pgd_page[vindex], count); - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); goto fail_unlock_free_pgds; } } @@ -2518,7 +2850,7 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm phys += (count / GPU_PAGES_PER_CPU_PAGE); insert_vpfn += count; remain -= count; - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); } mutex_unlock(&mmut->mmu_lock); @@ -2528,9 +2860,9 @@ static int mmu_insert_pages_no_flush(struct kbase_device *kbdev, struct kbase_mm fail_unlock_free_pgds: /* Free the pgds allocated by us from insert_level+1 to bottom level */ for (l = cur_level; l > insert_level; l--) - kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); + if (new_pgds[l] != KBASE_INVALID_PHYSICAL_ADDRESS) + kbase_mmu_free_pgd(kbdev, mmut, new_pgds[l]); -fail_unlock: if (insert_vpfn != (start_vpfn * GPU_PAGES_PER_CPU_PAGE)) { /* Invalidate the pages we have partially completed */ mmu_insert_pages_failure_recovery(kbdev, mmut, start_vpfn * GPU_PAGES_PER_CPU_PAGE, @@ -2773,7 +3105,7 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, { phys_addr_t current_pgd = pgds[level]; struct page *p = phys_to_page(current_pgd); - u64 *current_page = kbase_kmap(p); + u64 *current_page = kmap_pgd(p, current_pgd); unsigned int current_valid_entries = kbdev->mmu_mode->get_num_valid_entries(current_page); unsigned int index = (vpfn >> ((3 - level) * 9)) & 0x1FFU; @@ -2787,7 +3119,7 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, if (current_valid_entries == 1 && level != MIDGARD_MMU_LEVEL(0)) { kbdev->mmu_mode->set_num_valid_entries(current_page, 0); - kbase_kunmap(p, current_page); + kunmap_pgd(p, current_page); kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level - 1, flush_op, dirty_pgds, as_nr); @@ -2795,7 +3127,8 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, /* Check if fine grained GPU cache maintenance is being used */ if (flush_op == KBASE_MMU_OP_FLUSH_PT) { /* Ensure the invalidated PTE is visible in memory right away */ - kbase_mmu_sync_pgd_cpu(kbdev, kbase_dma_addr(p) + (index * sizeof(u64)), + kbase_mmu_sync_pgd_cpu(kbdev, + pgd_dma_addr(p, current_pgd) + (index * sizeof(u64)), sizeof(u64)); /* Invalidate the GPU cache for the whole PGD page and not just for * the cacheline containing the invalidated PTE, as the PGD page is @@ -2807,17 +3140,17 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev, KBASE_MMU_OP_FLUSH_PT); } - kbase_mmu_add_to_free_pgds_list(mmut, p); + kbase_mmu_add_to_free_pgds_list(mmut, current_pgd); } else { current_valid_entries--; kbdev->mmu_mode->set_num_valid_entries(current_page, current_valid_entries); - kbase_kunmap(p, current_page); + kunmap_pgd(p, current_page); kbase_mmu_sync_pgd(kbdev, mmut->kctx, current_pgd + (index * sizeof(u64)), - kbase_dma_addr(p) + (index * sizeof(u64)), sizeof(u64), - flush_op); + pgd_dma_addr(p, current_pgd) + (index * sizeof(u64)), + sizeof(u64), flush_op); /* When fine grained GPU cache maintenance is used then invalidate the MMU caches * now as the top most level PGD entry, affected by the teardown operation, has @@ -2921,7 +3254,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase phys_addr_t next_pgd; index = (vpfn >> ((3 - level) * 9)) & 0x1FF; - page = kbase_kmap(p); + page = kmap_pgd(p, pgd); if (mmu_mode->ate_is_valid(page[index], level)) break; /* keep the mapping */ else if (!mmu_mode->pte_is_valid(page[index], level)) { @@ -2936,7 +3269,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase next_pgd = mmu_mode->pte_to_phy_addr( kbdev->mgm_dev->ops.mgm_pte_to_original_pte( kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, page[index])); - kbase_kunmap(p, page); + kunmap_pgd(p, page); pgds[level] = pgd; pgd = next_pgd; p = phys_to_page(pgd); @@ -2947,7 +3280,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase case MIDGARD_MMU_LEVEL(1): dev_warn(kbdev->dev, "%s: No support for ATEs at level %d", __func__, level); - kbase_kunmap(p, page); + kunmap_pgd(p, page); goto out; case MIDGARD_MMU_LEVEL(2): /* can only teardown if count >= 512 */ @@ -2987,7 +3320,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (!num_of_valid_entries) { mmu_mode->set_num_valid_entries(page, 0); - kbase_kunmap(p, page); + kunmap_pgd(p, page); /* To avoid the invalid ATEs from the PGD page (that is going to be freed) * from getting reloaded into the GPU L2 cache whilst the teardown is @@ -3002,7 +3335,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (flush_op == KBASE_MMU_OP_FLUSH_PT) { /* Ensure the invalidated ATEs are visible in memory right away */ kbase_mmu_sync_pgd_cpu(kbdev, - kbase_dma_addr(p) + (index * sizeof(u64)), + pgd_dma_addr(p, pgd) + (index * sizeof(u64)), pcount * sizeof(u64)); /* Invalidate the GPU cache for the whole PGD page and not just for * the cachelines containing the invalidated ATEs, as the PGD page @@ -3014,7 +3347,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase KBASE_MMU_OP_FLUSH_PT); } - kbase_mmu_add_to_free_pgds_list(mmut, p); + kbase_mmu_add_to_free_pgds_list(mmut, pgd); vpfn += count; nr -= count; @@ -3024,8 +3357,8 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase mmu_mode->set_num_valid_entries(page, num_of_valid_entries); kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - kbase_dma_addr(p) + (index * sizeof(u64)), pcount * sizeof(u64), - flush_op); + pgd_dma_addr(p, pgd) + (index * sizeof(u64)), + pcount * sizeof(u64), flush_op); /* When fine grained GPU cache maintenance is used then invalidation of MMU cache * is done inline for every bottom level PGD touched in the teardown. @@ -3033,7 +3366,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase if (flush_op == KBASE_MMU_OP_FLUSH_PT) mmu_invalidate_on_teardown(kbdev, mmut->kctx, vpfn, pcount, level, as_nr); next: - kbase_kunmap(p, page); + kunmap_pgd(p, page); vpfn += count; nr -= count; } @@ -3238,7 +3571,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb goto fail_unlock; p = pfn_to_page(PFN_DOWN(pgd)); - pgd_page = kbase_kmap(p); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_warn(kbdev->dev, "kmap failure on update_pages"); err = -ENOMEM; @@ -3258,7 +3591,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb pgd_page[level_index] = kbase_mmu_create_ate( kbdev, *target_phys, flags, MIDGARD_MMU_LEVEL(2), group_id); kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (level_index * sizeof(u64)), - kbase_dma_addr(p) + (level_index * sizeof(u64)), + pgd_dma_addr(p, pgd) + (level_index * sizeof(u64)), sizeof(u64), KBASE_MMU_OP_NONE); } else { for (i = 0; i < count; i += GPU_PAGES_PER_CPU_PAGE) { @@ -3283,7 +3616,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb * will be done by the caller. */ kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - kbase_dma_addr(p) + (index * sizeof(u64)), + pgd_dma_addr(p, pgd) + (index * sizeof(u64)), count * sizeof(u64), KBASE_MMU_OP_NONE); } @@ -3296,7 +3629,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_device *kbdev, struct kb vpfn += count; nr -= count; - kbase_kunmap(p, pgd_page); + kunmap_pgd(p, pgd_page); } mutex_unlock(&mmut->mmu_lock); @@ -3532,7 +3865,7 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p goto get_pgd_at_level_error; } - pgd_page = kbase_kmap(phys_to_page(pgd)); + pgd_page = kmap_pgd(phys_to_page(pgd), pgd); if (!pgd_page) { dev_warn(kbdev->dev, "%s: kmap failure for PGD page.", __func__); ret = -EINVAL; @@ -3641,8 +3974,10 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, pgd_page[index]))); #endif kbdev->mmu_mode->entry_set_pte(&managed_pte, as_phys_addr_t(new_phys)); - *target = kbdev->mgm_dev->ops.mgm_update_gpu_pte( - kbdev->mgm_dev, MGM_DEFAULT_PTE_GROUP, level, managed_pte); + *target = kbdev->mgm_dev->ops.mgm_update_gpu_pte(kbdev->mgm_dev, + MGM_DEFAULT_PTE_GROUP, + PBHA_ID_DEFAULT, PTE_FLAGS_NONE, + level, managed_pte); } kbdev->mmu_mode->set_num_valid_entries(pgd_page, num_of_valid_entries); @@ -3653,29 +3988,35 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p * maintenance is necessary. */ kbase_mmu_sync_pgd(kbdev, mmut->kctx, pgd + (index * sizeof(u64)), - kbase_dma_addr(phys_to_page(pgd)) + (index * sizeof(u64)), + pgd_dma_addr(phys_to_page(pgd), pgd) + (index * sizeof(u64)), pgd_entries_to_sync * sizeof(u64), KBASE_MMU_OP_FLUSH_PT); /* Unlock MMU region. * - * Notice that GPUs which don't issue flush commands via GPU control - * still need an additional GPU cache flush here, this time only - * for the page table, because the function call above to sync PGDs - * won't have any effect on them. + * For GPUs without FLUSH_PA_RANGE support, the GPU caches were completely + * cleaned and invalidated after locking the virtual address range affected + * by the migration. As long as the lock is in place, GPU access to the + * locked range would remain blocked. So there is no need to clean and + * invalidate the GPU caches again after the copying the page contents + * of old page and updating the page table entry to point to new page. + * + * For GPUs with FLUSH_PA_RANGE support, the contents of old page would + * have been evicted from the GPU caches after locking the virtual address + * range. The page table entry contents also would have been invalidated + * from the GPU's L2 cache by kbase_mmu_sync_pgd() after the page table + * update. + * + * If kbase_mmu_hw_do_unlock_no_addr() fails, GPU reset will be triggered which + * would remove the MMU lock and so there is no need to rollback page migration + * and the failure can be ignored. */ spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags); if (kbdev->pm.backend.gpu_ready && mmut->kctx->as_nr >= 0) { int as_nr = mmut->kctx->as_nr; struct kbase_as *as = &kbdev->as[as_nr]; + int local_ret = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, &op_param); - if (mmu_flush_cache_on_gpu_ctrl(kbdev)) { - ret = kbase_mmu_hw_do_unlock(kbdev, as, &op_param); - } else { - ret = kbase_gpu_cache_flush_and_busy_wait(kbdev, - GPU_COMMAND_CACHE_CLN_INV_L2); - if (!ret) - ret = kbase_mmu_hw_do_unlock_no_addr(kbdev, as, &op_param); - } + CSTD_UNUSED(local_ret); } /* Release the transition prevention in L2 by ending the transaction */ @@ -3684,22 +4025,27 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p /* Releasing locks before checking the migration transaction error state */ mutex_unlock(&kbdev->mmu_hw_mutex); - /* Checking the final migration transaction error state */ - if (ret < 0) { - dev_err(kbdev->dev, "%s: failed to unlock MMU region.", __func__); - goto undo_mappings; - } - /* Undertaking metadata transfer, while we are holding the mmu_lock */ spin_lock(&page_md->migrate_lock); if (level == MIDGARD_MMU_BOTTOMLEVEL) { - size_t page_array_index = (page_md->data.mapped.vpfn / GPU_PAGES_PER_CPU_PAGE) - - page_md->data.mapped.reg->start_pfn; + enum kbase_page_status page_status = PAGE_STATUS_GET(page_md->status); - WARN_ON(PAGE_STATUS_GET(page_md->status) != ALLOCATED_MAPPED); + if (page_status == ALLOCATED_MAPPED) { + /* Replace page in array of pages of the physical allocation. */ + size_t page_array_index = + div_u64(page_md->data.mapped.vpfn, GPU_PAGES_PER_CPU_PAGE) - + page_md->data.mapped.reg->start_pfn; - /* Replace page in array of pages of the physical allocation. */ - page_md->data.mapped.reg->gpu_alloc->pages[page_array_index] = new_phys; + page_md->data.mapped.reg->gpu_alloc->pages[page_array_index] = new_phys; + } else if (page_status == NOT_MOVABLE) { + dev_dbg(kbdev->dev, + "%s: migration completed and page has become NOT_MOVABLE.", + __func__); + } else { + dev_WARN(kbdev->dev, + "%s: migration completed but page has moved to status %d.", + __func__, page_status); + } } /* Update the new page dma_addr with the transferred metadata from the old_page */ page_md->dma_addr = new_dma_addr; @@ -3710,7 +4056,7 @@ int kbase_mmu_migrate_page(struct tagged_addr old_phys, struct tagged_addr new_p set_page_private(as_page(old_phys), 0); l2_state_defer_out: - kbase_kunmap(phys_to_page(pgd), pgd_page); + kunmap_pgd(phys_to_page(pgd), pgd_page); pgd_page_map_error: get_pgd_at_level_error: page_state_change_out: @@ -3725,7 +4071,7 @@ old_page_map_error: undo_mappings: /* Unlock the MMU table and undo mappings. */ mutex_unlock(&mmut->mmu_lock); - kbase_kunmap(phys_to_page(pgd), pgd_page); + kunmap_pgd(phys_to_page(pgd), pgd_page); kbase_kunmap(as_page(new_phys), new_page); kbase_kunmap(as_page(old_phys), old_page); @@ -3744,7 +4090,7 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl lockdep_assert_held(&mmut->mmu_lock); - pgd_page = kbase_kmap_atomic(p); + pgd_page = kmap_atomic_pgd(p, pgd); /* kmap_atomic should NEVER fail. */ if (WARN_ON_ONCE(pgd_page == NULL)) return; @@ -3753,7 +4099,7 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl * kmap_atomic usage */ pgd_page_buffer = mmut->scratch_mem.teardown_pages.levels[level]; - memcpy(pgd_page_buffer, pgd_page, PAGE_SIZE); + memcpy(pgd_page_buffer, pgd_page, GPU_PAGE_SIZE); } /* When page migration is enabled, kbase_region_tracker_term() would ensure @@ -3764,7 +4110,7 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl WARN_ON_ONCE(kbdev->mmu_mode->get_num_valid_entries(pgd_page)); /* Invalidate page after copying */ mmu_mode->entries_invalidate(pgd_page, KBASE_MMU_PAGE_ENTRIES); - kbase_kunmap_atomic(pgd_page); + kunmap_atomic_pgd(pgd_page); pgd_page = pgd_page_buffer; if (level < MIDGARD_MMU_BOTTOMLEVEL) { @@ -3783,13 +4129,20 @@ static void mmu_teardown_level(struct kbase_device *kbdev, struct kbase_mmu_tabl kbase_mmu_free_pgd(kbdev, mmut, pgd); } -static void kbase_mmu_mark_non_movable(struct page *page) +static void kbase_mmu_mark_non_movable(struct kbase_device *const kbdev, struct page *page) { struct kbase_page_metadata *page_md; if (!kbase_is_page_migration_enabled()) return; + /* Composite large-page is excluded from migration, trigger a warn if a development + * wrongly leads to it. + */ + if (is_huge_head(as_tagged(page_to_phys(page))) || + is_partial(as_tagged(page_to_phys(page)))) + dev_WARN(kbdev->dev, "%s: migration on large-page attempted.", __func__); + page_md = kbase_page_private(page); spin_lock(&page_md->migrate_lock); @@ -3817,6 +4170,10 @@ int kbase_mmu_init(struct kbase_device *const kbdev, struct kbase_mmu_table *con mmut->kctx = kctx; mmut->pgd = KBASE_INVALID_PHYSICAL_ADDRESS; +#if GPU_PAGES_PER_CPU_PAGE > 1 + INIT_LIST_HEAD(&mmut->pgd_pages_list); +#endif + /* We allocate pages into the kbdev memory pool, then * kbase_mmu_alloc_pgd will allocate out of that pool. This is done to * avoid allocations from the kernel happening with the lock held. @@ -3831,10 +4188,12 @@ int kbase_mmu_init(struct kbase_device *const kbdev, struct kbase_mmu_table *con return -ENOMEM; } + mutex_lock(&mmut->mmu_lock); mmut->pgd = kbase_mmu_alloc_pgd(kbdev, mmut); + mutex_unlock(&mmut->mmu_lock); } - kbase_mmu_mark_non_movable(pfn_to_page(PFN_DOWN(mmut->pgd))); + kbase_mmu_mark_non_movable(kbdev, pfn_to_page(PFN_DOWN(mmut->pgd))); return 0; } @@ -3892,6 +4251,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, size_t dump_size; struct kbase_device *kbdev; struct kbase_mmu_mode const *mmu_mode; + struct page *p; if (WARN_ON(kctx == NULL)) return 0; @@ -3900,7 +4260,8 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, kbdev = kctx->kbdev; mmu_mode = kbdev->mmu_mode; - pgd_page = kbase_kmap(pfn_to_page(PFN_DOWN(pgd))); + p = pfn_to_page(PFN_DOWN(pgd)); + pgd_page = kmap_pgd(p, pgd); if (!pgd_page) { dev_warn(kbdev->dev, "%s: kmap failure", __func__); return 0; @@ -3934,7 +4295,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, dump_size = kbasep_mmu_dump_level(kctx, target_pgd, level + 1, buffer, size_left); if (!dump_size) { - kbase_kunmap(pfn_to_page(PFN_DOWN(pgd)), pgd_page); + kunmap_pgd(p, pgd_page); return 0; } size += dump_size; @@ -3942,7 +4303,7 @@ static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, } } - kbase_kunmap(pfn_to_page(PFN_DOWN(pgd)), pgd_page); + kunmap_pgd(p, pgd_page); return size; } @@ -4048,7 +4409,6 @@ void kbase_mmu_bus_fault_worker(struct work_struct *data) return; } -#ifdef CONFIG_MALI_ARBITER_SUPPORT /* check if we still have GPU */ if (unlikely(kbase_is_gpu_removed(kbdev))) { dev_dbg(kbdev->dev, "%s: GPU has been removed", __func__); @@ -4056,7 +4416,6 @@ void kbase_mmu_bus_fault_worker(struct work_struct *data) atomic_dec(&kbdev->faults_pending); return; } -#endif if (unlikely(fault->protected_mode)) { kbase_mmu_report_fault_and_kill(kctx, faulting_as, "Permission failure", fault); diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c index e1b72ed24321..548d88cf216e 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_faults_decoder.c @@ -65,7 +65,7 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde if (older_source_id_fmt) utlb_id = FAULT_SOURCE_ID_UTLB_ID_GET(source_id); - if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "shader") == 0) { + if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "shader")) { if (utlb_id == 0) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_shader_r_t( @@ -75,7 +75,7 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde ir, kbdev->gpu_props.gpu_id.arch_id); } else return "Load/store cache"; - } else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "tiler")) { + } else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "tiler")) { #if MALI_USE_CSF if (utlb_id == 0) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) @@ -92,14 +92,14 @@ const char *fault_source_id_internal_requester_get_str(struct kbase_device *kbde #endif } #if MALI_USE_CSF - else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { + else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "csf")) { if (access_type == AS_FAULTSTATUS_ACCESS_TYPE_READ) return decode_fault_source_csf_r_t(ir, kbdev->gpu_props.gpu_id.arch_id); else return decode_fault_source_csf_w_t(ir, kbdev->gpu_props.gpu_id.arch_id); } #else - else if (strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "jm")) + else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "jm")) return decode_fault_source_jm_t(ir, kbdev->gpu_props.gpu_id.arch_id); #endif else if (!strcmp(source_id_enc_core_type_get_str(kbdev, source_id), "I2c") || diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h index c2b377de54a9..560baceafe8a 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -42,7 +42,7 @@ struct kbase_context; * enum kbase_mmu_fault_type - MMU fault type descriptor. * @KBASE_MMU_FAULT_TYPE_UNKNOWN: unknown fault * @KBASE_MMU_FAULT_TYPE_PAGE: page fault - * @KBASE_MMU_FAULT_TYPE_BUS: nus fault + * @KBASE_MMU_FAULT_TYPE_BUS: bus fault * @KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED: page_unexpected fault * @KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED: bus_unexpected fault */ diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c index ba67ae0e01e9..46c04f2b1fc1 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_hw_direct.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -529,8 +529,8 @@ int kbase_mmu_hw_do_flush(struct kbase_device *kbdev, struct kbase_as *as, return ret; #if MALI_USE_CSF && !IS_ENABLED(CONFIG_MALI_BIFROST_NO_MALI) - /* WA for the BASE_HW_ISSUE_GPU2019_3901. */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_GPU2019_3901) && + /* WA for the KBASE_HW_ISSUE_GPU2019_3901. */ + if (kbase_hw_has_issue(kbdev, KBASE_HW_ISSUE_GPU2019_3901) && mmu_cmd == AS_COMMAND_COMMAND_FLUSH_MEM) { ret = apply_hw_issue_GPU2019_3901_wa(kbdev, &mmu_cmd, as->number); if (ret) { @@ -635,6 +635,15 @@ void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, #endif kbase_reg_write32(kbdev, MMU_CONTROL_ENUM(IRQ_CLEAR), pf_bf_mask); +#if MALI_USE_CSF + /* For valid page faults, this function is called just before unblocking the MMU (which + * would in turn unblock the MCU firmware) and so this is an opportune location to + * update the page fault counter value in firmware visible memory. + */ + if (likely(type == KBASE_MMU_FAULT_TYPE_PAGE) && kbdev->csf.page_fault_cnt_ptr) + *kbdev->csf.page_fault_cnt_ptr = ++kbdev->csf.page_fault_cnt; +#endif + unlock: spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); } diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h index 4c2c1a64ca41..8b68791e4c77 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_internal.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -38,19 +38,6 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, struct kbas void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, struct kbase_as *as, const char *reason_str, struct kbase_fault *fault); -/** - * kbase_mmu_switch_to_ir() - Switch to incremental rendering if possible - * @kctx: kbase_context for the faulting address space. - * @reg: of a growable GPU memory region in the same context. - * Takes ownership of the reference if successful. - * - * Used to switch to incremental rendering if we have nearly run out of - * virtual address space in a growable memory region. - * - * Return: 0 if successful, otherwise a negative error code. - */ -int kbase_mmu_switch_to_ir(struct kbase_context *kctx, struct kbase_va_region *reg); - /** * kbase_mmu_page_fault_worker() - Process a page fault. * diff --git a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c index e3ad78daed59..7aace473011f 100644 --- a/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c +++ b/drivers/gpu/arm/bifrost/mmu/mali_kbase_mmu_mode_aarch64.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -179,7 +179,7 @@ static void set_num_valid_entries(u64 *pgd, unsigned int num_of_valid_entries) static void entry_set_pte(u64 *entry, phys_addr_t phy) { - page_table_entry_set(entry, (phy & PAGE_MASK) | ENTRY_ACCESS_BIT | ENTRY_IS_PTE); + page_table_entry_set(entry, (phy & GPU_PAGE_MASK) | ENTRY_ACCESS_BIT | ENTRY_IS_PTE); } static void entries_invalidate(u64 *entry, u32 count) diff --git a/drivers/gpu/arm/bifrost/tests/Kconfig b/drivers/gpu/arm/bifrost/tests/Kconfig index aa011bac8990..88a4194c5cd7 100644 --- a/drivers/gpu/arm/bifrost/tests/Kconfig +++ b/drivers/gpu/arm/bifrost/tests/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note # -# (C) COPYRIGHT 2017, 2020-2023 ARM Limited. All rights reserved. +# (C) COPYRIGHT 2017, 2020-2024 ARM Limited. All rights reserved. # # This program is free software and is provided to you under the terms of the # GNU General Public License version 2 as published by the Free Software diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c index 6c343cf9f73b..0598d4397e2a 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2020-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -425,7 +425,7 @@ static const char *kutf_clk_trace_do_get_platform(struct kutf_context *context, const void *arbiter_if_node = NULL; const void *power_node = NULL; const char *platform = "GPU"; -#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) +#if defined(CONFIG_OF) struct kutf_clk_rate_trace_fixture_data *data = context->fixture; arbiter_if_node = of_get_property(data->kbdev->dev->of_node, "arbiter-if", NULL); diff --git a/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c b/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c index 8937d69f182f..f341a411324e 100644 --- a/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c +++ b/drivers/gpu/arm/bifrost/tests/mali_kutf_mgm_integration_test/mali_kutf_mgm_integration_test_main.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -95,7 +95,9 @@ static void mali_kutf_mgm_pte_translation_test(struct kutf_context *context) data->group_id, mmu_level, original_pte); translated_pte = mgm_dev->ops.mgm_update_gpu_pte(mgm_dev, data->group_id, - mmu_level, original_pte); + PBHA_ID_DEFAULT, + PTE_FLAGS_NONE, mmu_level, + original_pte); if (translated_pte == original_pte) { snprintf( msg_buf, sizeof(msg_buf), diff --git a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c index a91278dd3bef..f254aa84dc20 100644 --- a/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c +++ b/drivers/gpu/arm/bifrost/tl/backend/mali_kbase_timeline_csf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -35,7 +35,7 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) u32 const num_sb_entries = kbdev->gpu_props.gpu_id.arch_major >= 11 ? 16 : 8; u32 const supports_gpu_sleep = #ifdef KBASE_PM_RUNTIME - kbdev->pm.backend.gpu_sleep_supported; + test_bit(KBASE_GPU_SUPPORTS_GPU_SLEEP, &kbdev->pm.backend.gpu_sleep_allowed); #else false; #endif /* KBASE_PM_RUNTIME */ diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c index d98e22880419..719e26124409 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_timeline_io.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -29,6 +29,7 @@ #include #include #include +#include /* Explicitly include epoll header for old kernels. Not required from 4.16. */ #if KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE @@ -169,7 +170,7 @@ static inline int copy_stream_header(char __user *buffer, size_t size, ssize_t * const char *hdr, size_t hdr_size, size_t *hdr_btc) { const size_t offset = hdr_size - *hdr_btc; - const size_t copy_size = MIN((size_t)((ssize_t)size - *copy_len), *hdr_btc); + const size_t copy_size = MIN(size_sub((ssize_t)size, *copy_len), *hdr_btc); if (!*hdr_btc) return 0; diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c index 34cabbd6e535..d4465c44addb 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.c @@ -62,6 +62,7 @@ enum tl_msg_id_obj { KBASE_TL_EVENT_ATOM_SOFTJOB_START, KBASE_TL_EVENT_ATOM_SOFTJOB_END, KBASE_TL_ARBITER_GRANTED, + KBASE_TL_ARBITER_LOST, KBASE_TL_ARBITER_STARTED, KBASE_TL_ARBITER_STOP_REQUESTED, KBASE_TL_ARBITER_STOPPED, @@ -272,6 +273,10 @@ enum tl_msg_id_obj { "Arbiter has granted gpu access", \ "@p", \ "gpu") \ + TRACEPOINT_DESC(KBASE_TL_ARBITER_LOST, \ + "Received a gpu lost event from the arbiter", \ + "@p", \ + "gpu") \ TRACEPOINT_DESC(KBASE_TL_ARBITER_STARTED, \ "Driver is running again and able to process jobs", \ "@p", \ @@ -1546,6 +1551,29 @@ void __kbase_tlstream_tl_arbiter_granted( kbase_tlstream_msgbuf_release(stream, acq_flags); } +void __kbase_tlstream_tl_arbiter_lost( + struct kbase_tlstream *stream, + const void *gpu +) +{ + const u32 msg_id = KBASE_TL_ARBITER_LOST; + const size_t msg_size = sizeof(msg_id) + sizeof(u64) + + sizeof(gpu) + ; + char *buffer; + unsigned long acq_flags; + size_t pos = 0; + + buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); + + pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); + pos = kbasep_serialize_timestamp(buffer, pos); + pos = kbasep_serialize_bytes(buffer, + pos, &gpu, sizeof(gpu)); + + kbase_tlstream_msgbuf_release(stream, acq_flags); +} + void __kbase_tlstream_tl_arbiter_started( struct kbase_tlstream *stream, const void *gpu diff --git a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h index 987d4d3dbf27..6dd4b44ea6b2 100644 --- a/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h +++ b/drivers/gpu/arm/bifrost/tl/mali_kbase_tracepoints.h @@ -271,6 +271,11 @@ void __kbase_tlstream_tl_arbiter_granted( const void *gpu ); +void __kbase_tlstream_tl_arbiter_lost( + struct kbase_tlstream *stream, + const void *gpu +); + void __kbase_tlstream_tl_arbiter_started( struct kbase_tlstream *stream, const void *gpu @@ -1550,6 +1555,25 @@ struct kbase_tlstream; ); \ } while (0) +/** + * KBASE_TLSTREAM_TL_ARBITER_LOST - Received a gpu lost event from the arbiter + * + * @kbdev: Kbase device + * @gpu: Name of the GPU object + */ +#define KBASE_TLSTREAM_TL_ARBITER_LOST( \ + kbdev, \ + gpu \ + ) \ + do { \ + u32 enabled = (u32)atomic_read(&kbdev->timeline_flags); \ + if (enabled & TLSTREAM_ENABLED) \ + __kbase_tlstream_tl_arbiter_lost( \ + __TL_DISPATCH_STREAM(kbdev, obj), \ + gpu \ + ); \ + } while (0) + /** * KBASE_TLSTREAM_TL_ARBITER_STARTED - Driver is running again and able to process jobs * diff --git a/drivers/hwtracing/coresight/mali/build.bp b/drivers/hwtracing/coresight/mali/build.bp index 33dcd22fa364..d69148c8cb70 100644 --- a/drivers/hwtracing/coresight/mali/build.bp +++ b/drivers/hwtracing/coresight/mali/build.bp @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software diff --git a/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c b/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c index 59d5cd314c2f..727e5c7a552a 100644 --- a/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c +++ b/drivers/hwtracing/coresight/mali/sources/itm/coresight_mali_source_itm_core.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -85,14 +85,14 @@ static struct kbase_debug_coresight_csf_op dwt_itm_enable_ops[] = { }; static struct kbase_debug_coresight_csf_op dwt_itm_disable_ops[] = { - // Disable ITM/DWT functionality via DEMCR register - WRITE_IMM_OP(CS_SCS_BASE_ADDR + SCS_DEMCR, 0x00000000), // Unlock ITM configuration WRITE_IMM_OP(CS_ITM_BASE_ADDR + CORESIGHT_LAR, CS_MALI_UNLOCK_COMPONENT), // Check ITM is disabled POLL_OP(CS_ITM_BASE_ADDR + ITM_TCR, ITM_TCR_BUSY_BIT, 0x0), // Lock WRITE_IMM_OP(CS_ITM_BASE_ADDR + CORESIGHT_LAR, 0x00000000), + // Disable ITM/DWT functionality via DEMCR register + WRITE_IMM_OP(CS_SCS_BASE_ADDR + SCS_DEMCR, 0x00000000), // Set enabled bit off at the end of sequence BIT_AND_OP(&itm_state.enabled, 0x0), }; diff --git a/include/linux/mali_hw_access.h b/include/linux/mali_hw_access.h index 4ed9da994e57..106393fc3372 100644 --- a/include/linux/mali_hw_access.h +++ b/include/linux/mali_hw_access.h @@ -30,6 +30,13 @@ #define mali_writel(val, addr) writel(val, addr) +#define mali_ioremap(addr, size) ioremap(addr, size) + +#define mali_iounmap(addr) iounmap(addr) + +#define mali_arch_timer_get_cntfrq() arch_timer_get_cntfrq() + + #define mali_readq(addr) ((u64)mali_readl(addr) | ((u64)mali_readl(addr + 4) << 32)) static inline u64 mali_readq_coherent(const void __iomem *addr) @@ -45,17 +52,11 @@ static inline u64 mali_readq_coherent(const void __iomem *addr) return lo | (((u64)hi1) << 32); } -#define mali_writeq(val, addr) \ - do { \ - mali_writel(val & 0xFFFFFFFF, addr); \ - mali_writel(val >> 32, addr + 4); \ +#define mali_writeq(val, addr) \ + do { \ + u64 __val = (u64)val; \ + mali_writel((u32)(__val & 0xFFFFFFFF), addr); \ + mali_writel((u32)(__val >> 32), addr + 4); \ } while (0) -#define mali_ioremap(addr, size) ioremap(addr, size) - -#define mali_iounmap(addr) iounmap(addr) - -#define mali_arch_timer_get_cntfrq() arch_timer_get_cntfrq() - - #endif /* _MALI_HW_ACCESS_H_ */ diff --git a/include/linux/memory_group_manager.h b/include/linux/memory_group_manager.h index e92d3dea0178..ec55d74f56ad 100644 --- a/include/linux/memory_group_manager.h +++ b/include/linux/memory_group_manager.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -35,10 +35,16 @@ typedef int vm_fault_t; #define PTE_PBHA_SHIFT (59) #define PTE_PBHA_MASK ((uint64_t)0xf << PTE_PBHA_SHIFT) #define PTE_RES_BIT_MULTI_AS_SHIFT (63) +#define PTE_FLAGS_NONE (0) +#define PBHA_ID_DEFAULT (0) struct memory_group_manager_device; struct memory_group_manager_import_data; +enum mgm_pte_flags { + MMA_VIOLATION = 0, +}; + /** * struct memory_group_manager_ops - Callbacks for memory group manager * operations @@ -50,6 +56,8 @@ struct memory_group_manager_import_data; * @mgm_pte_to_original_pte: Callback to get the original PTE entry as given * to mgm_update_gpu_pte * @mgm_vmf_insert_pfn_prot: Callback to map a physical memory page for the CPU + * @mgm_get_import_memory_cached_access_permitted: Callback to query if a given imported + * memory is allowed to be accessed as cached or not by the GPU */ struct memory_group_manager_ops { /* @@ -119,6 +127,11 @@ struct memory_group_manager_ops { * @group_id: A physical memory group ID. The meaning of this is * defined by the systems integrator. Its valid range is * 0 .. MEMORY_GROUP_MANAGER_NR_GROUPS-1. + * + * @pbha_id: PBHA Overrride ID to encode into the PTE + * @pte_flags: PTE related flags, defined in enum mgm_pte_flags + * + * * @mmu_level: The level of the page table entry in @ate. * @pte: The page table entry to modify, in LPAE or AArch64 format * (depending on the driver's configuration). This should be @@ -128,13 +141,14 @@ struct memory_group_manager_ops { * This function allows the memory group manager to modify a GPU page * table entry before it is stored by the kbase module (controller * driver). It may set certain bits in the page table entry attributes - * or modify the physical address, based on the physical memory group ID - * and/or additional data in struct memory_group_manager_device. + * or modify the physical address, based on the physical memory group ID, + * PBHA ID, PTE flags and/or additional data in struct memory_group_manager_device. * * Return: A modified GPU page table entry to be stored in a page table. */ u64 (*mgm_update_gpu_pte)(struct memory_group_manager_device *mgm_dev, - unsigned int group_id, int mmu_level, u64 pte); + unsigned int group_id, unsigned int pbha_id, + unsigned int pte_flags, int mmu_level, u64 pte); /* * mgm_pte_to_original_pte - Undo any modification done during mgm_update_gpu_pte() @@ -182,6 +196,20 @@ struct memory_group_manager_ops { unsigned int group_id, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot); + + /* + * mgm_get_import_memory_cached_access_permitted - Check if a given imported memory + * is allowed to be accessed as cached or not by the GPU + * + * @mgm_dev: The memory group manager through which the request + * is being made. + * @import_data: Pointer to the data which describes imported memory. + * + * Return: true if cached access is permitted, false otherwise + */ + bool (*mgm_get_import_memory_cached_access_permitted)( + struct memory_group_manager_device *mgm_dev, + struct memory_group_manager_import_data *import_data); }; /** diff --git a/include/linux/version_compat_defs.h b/include/linux/version_compat_defs.h index 46f227e108f6..d94a39ebbadc 100644 --- a/include/linux/version_compat_defs.h +++ b/include/linux/version_compat_defs.h @@ -26,6 +26,9 @@ #include #include #include +#include +#include +#include #if (KERNEL_VERSION(4, 4, 267) < LINUX_VERSION_CODE) #include @@ -176,6 +179,7 @@ static inline void kbase_kunmap_atomic(void *address) */ #define check_mul_overflow(a, b, d) __builtin_mul_overflow(a, b, d) #define check_add_overflow(a, b, d) __builtin_add_overflow(a, b, d) +#define check_sub_overflow(a, b, d) __builtin_sub_overflow(a, b, d) #endif /* @@ -356,8 +360,10 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #if KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE #define KBASE_REGISTER_SHRINKER(reclaim, name, priv_data) register_shrinker(reclaim) +/* clang-format off */ #elif ((KERNEL_VERSION(6, 7, 0) > LINUX_VERSION_CODE) && \ !(defined(__ANDROID_COMMON_KERNEL__) && (KERNEL_VERSION(6, 6, 0) == LINUX_VERSION_CODE))) +/* clang-format on */ #define KBASE_REGISTER_SHRINKER(reclaim, name, priv_data) register_shrinker(reclaim, name) #else @@ -368,9 +374,10 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m } while (0) #endif /* KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE */ - +/* clang-format off */ #if ((KERNEL_VERSION(6, 7, 0) > LINUX_VERSION_CODE) && \ !(defined(__ANDROID_COMMON_KERNEL__) && (KERNEL_VERSION(6, 6, 0) == LINUX_VERSION_CODE))) +/* clang-format on */ #define KBASE_UNREGISTER_SHRINKER(reclaim) unregister_shrinker(&reclaim) #define KBASE_GET_KBASE_DATA_FROM_SHRINKER(s, type, var) container_of(s, type, var) #define DEFINE_KBASE_SHRINKER struct shrinker @@ -387,6 +394,31 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #endif +static inline int kbase_param_set_uint_minmax(const char *val, const struct kernel_param *kp, + unsigned int min, unsigned int max) +{ +#if (KERNEL_VERSION(5, 15, 0) > LINUX_VERSION_CODE) + uint uint_val; + int ret; + + if (!val) + return -EINVAL; + + ret = kstrtouint(val, 0, &uint_val); + + if (ret == 0) { + if (uint_val < min || uint_val > max) + return -EINVAL; + + *((uint *)kp->arg) = uint_val; + } + + return ret; +#else + return param_set_uint_minmax(val, kp, min, max); +#endif +} + #if (KERNEL_VERSION(4, 20, 0) <= LINUX_VERSION_CODE) #include #endif @@ -394,4 +426,99 @@ static inline long kbase_pin_user_pages_remote(struct task_struct *tsk, struct m #define __maybe_unused __attribute__((unused)) #endif +#if KERNEL_VERSION(5, 4, 103) <= LINUX_VERSION_CODE +#define mali_sysfs_emit(buf, fmt, ...) sysfs_emit(buf, fmt, __VA_ARGS__) +#else +#define mali_sysfs_emit(buf, fmt, ...) scnprintf(buf, PAGE_SIZE, fmt, __VA_ARGS__) +#endif + +#if KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE +#include +#include + +static inline struct devfreq *devfreq_get_devfreq_by_node(struct device_node *node) +{ + struct platform_device *pdev = of_find_device_by_node(node); + + if (!pdev || !node) + return NULL; + + return devfreq_get_devfreq_by_phandle(&pdev->dev, 0); +} +#endif + +#if (KERNEL_VERSION(5, 16, 0) <= LINUX_VERSION_CODE && \ + KERNEL_VERSION(5, 18, 0) > LINUX_VERSION_CODE) || \ + (KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE && \ + KERNEL_VERSION(5, 15, 85) >= LINUX_VERSION_CODE) || \ + (KERNEL_VERSION(5, 10, 200) >= LINUX_VERSION_CODE) +/* + * Kernel revisions + * - up to 5.10.200 + * - between 5.11.0 and 5.15.85 inclusive + * - between 5.16.0 and 5.17.15 inclusive + * do not provide an implementation of + * size_add, size_sub and size_mul. + * The implementations below provides + * backward compatibility implementations of these functions. + */ + +static inline size_t __must_check size_mul(size_t factor1, size_t factor2) +{ + size_t ret_val; + + if (check_mul_overflow(factor1, factor2, &ret_val)) + return SIZE_MAX; + return ret_val; +} + +static inline size_t __must_check size_add(size_t addend1, size_t addend2) +{ + size_t ret_val; + + if (check_add_overflow(addend1, addend2, &ret_val)) + return SIZE_MAX; + return ret_val; +} + +static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) +{ + size_t ret_val; + + if (minuend == SIZE_MAX || subtrahend == SIZE_MAX || + check_sub_overflow(minuend, subtrahend, &ret_val)) + return SIZE_MAX; + return ret_val; +} +#endif + +#if KERNEL_VERSION(5, 5, 0) > LINUX_VERSION_CODE +static inline unsigned long bitmap_get_value8(const unsigned long *map, unsigned long start) +{ + const size_t index = BIT_WORD(start); + const unsigned long offset = start % BITS_PER_LONG; + + return (map[index] >> offset) & 0xFF; +} + +static inline unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + offset = find_next_bit(addr, size, offset); + if (offset == size) + return size; + + offset = round_down(offset, 8); + *clump = bitmap_get_value8(addr, offset); + + return offset; +} + +#define find_first_clump8(clump, bits, size) find_next_clump8((clump), (bits), (size), 0) + +#define for_each_set_clump8(start, clump, bits, size) \ + for ((start) = find_first_clump8(&(clump), (bits), (size)); (start) < (size); \ + (start) = find_next_clump8(&(clump), (bits), (size), (start) + 8)) +#endif + #endif /* _VERSION_COMPAT_DEFS_H_ */ diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h index b4b38f242bd2..2b2fd1dd7bc8 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_base_csf_kernel.h @@ -61,13 +61,15 @@ * must be less than BASE_MEM_FLAGS_NR_BITS !!! */ -/* A mask of all the flags which are only valid for allocations within kbase, - * and may not be passed from user space. +/* A mask of all the flags which are only valid within kbase, + * and may not be passed to/from user space. */ #define BASEP_MEM_FLAGS_KERNEL_ONLY (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE) -/* A mask of all flags that should not be queried */ -#define BASE_MEM_DONT_QUERY (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) +/* A mask of flags that, when provied, cause other flags to be + * enabled but are not enabled themselves + */ +#define BASE_MEM_FLAGS_ACTION_MODIFIERS (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) /* A mask of all currently reserved flags */ #define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) diff --git a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h index 28e7db49f676..2b5b8b25fc2c 100644 --- a/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/csf/mali_kbase_csf_ioctl.h @@ -116,10 +116,14 @@ * from the parent process. * 1.30: * - Implement support for setting GPU Timestamp Offset register. + * 1.31: + * - Reject non-protected allocations containing the BASE_MEM_PROTECTED memory flag. + * - Reject allocations containing the BASE_MEM_DONT_NEED memory flag (it is only settable). + * - Reject allocations containing the BASE_MEM_UNUSED_BIT_xx memory flags. */ #define BASE_UK_VERSION_MAJOR 1 -#define BASE_UK_VERSION_MINOR 30 +#define BASE_UK_VERSION_MINOR 31 /** * struct kbase_ioctl_version_check - Check version compatibility between diff --git a/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h b/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h index b5351c50aa30..fad61299b1c1 100644 --- a/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h +++ b/include/uapi/gpu/arm/bifrost/jm/mali_base_jm_kernel.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2019-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2019-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -53,15 +53,17 @@ * must be less than BASE_MEM_FLAGS_NR_BITS !!! */ -/* A mask of all the flags which are only valid for allocations within kbase, - * and may not be passed from user space. +/* A mask of all the flags which are only valid within kbase, + * and may not be passed to/from user space. */ #define BASEP_MEM_FLAGS_KERNEL_ONLY \ (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE | BASE_MEM_FLAG_MAP_FIXED | \ BASEP_MEM_PERFORM_JIT_TRIM) -/* A mask of all flags that should not be queried */ -#define BASE_MEM_DONT_QUERY (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) +/* A mask of flags that, when provied, cause other flags to be + * enabled but are not enabled themselves + */ +#define BASE_MEM_FLAGS_ACTION_MODIFIERS (BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_IMPORT_SHARED) /* A mask of all currently reserved flags */ #define BASE_MEM_FLAGS_RESERVED ((base_mem_alloc_flags)0) @@ -120,10 +122,6 @@ */ #define BASE_JD_ATOM_COUNT 256 -/* Maximum number of concurrent render passes. - */ -#define BASE_JD_RP_COUNT (256) - /* Set/reset values for a software event */ #define BASE_JD_SOFT_EVENT_SET ((unsigned char)1) #define BASE_JD_SOFT_EVENT_RESET ((unsigned char)0) @@ -363,40 +361,6 @@ typedef __u32 base_jd_core_req; */ #define BASE_JD_REQ_JOB_SLOT ((base_jd_core_req)1 << 17) -/* SW-only requirement: The atom is the start of a renderpass. - * - * If this bit is set then the job chain will be soft-stopped if it causes the - * GPU to write beyond the end of the physical pages backing the tiler heap, and - * committing more memory to the heap would exceed an internal threshold. It may - * be resumed after running one of the job chains attached to an atom with - * BASE_JD_REQ_END_RENDERPASS set and the same renderpass ID. It may be - * resumed multiple times until it completes without memory usage exceeding the - * threshold. - * - * Usually used with BASE_JD_REQ_T. - */ -#define BASE_JD_REQ_START_RENDERPASS ((base_jd_core_req)1 << 18) - -/* SW-only requirement: The atom is the end of a renderpass. - * - * If this bit is set then the atom incorporates the CPU address of a - * base_jd_fragment object instead of the GPU address of a job chain. - * - * Which job chain is run depends upon whether the atom with the same renderpass - * ID and the BASE_JD_REQ_START_RENDERPASS bit set completed normally or - * was soft-stopped when it exceeded an upper threshold for tiler heap memory - * usage. - * - * It also depends upon whether one of the job chains attached to the atom has - * already been run as part of the same renderpass (in which case it would have - * written unresolved multisampled and otherwise-discarded output to temporary - * buffers that need to be read back). The job chain for doing a forced read and - * forced write (from/to temporary buffers) is run as many times as necessary. - * - * Usually used with BASE_JD_REQ_FS. - */ -#define BASE_JD_REQ_END_RENDERPASS ((base_jd_core_req)1 << 19) - /* SW-only requirement: The atom needs to run on a limited core mask affinity. * * If this bit is set then the kbase_context.limited_core_mask will be applied @@ -412,7 +376,6 @@ typedef __u32 base_jd_core_req; BASE_JD_REQ_EVENT_COALESCE | BASE_JD_REQ_COHERENT_GROUP | \ BASE_JD_REQ_SPECIFIC_COHERENT_GROUP | BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END | BASE_JD_REQ_JOB_SLOT | \ - BASE_JD_REQ_START_RENDERPASS | BASE_JD_REQ_END_RENDERPASS | \ BASE_JD_REQ_LIMITED_CORE_MASK)) /* Mask of all bits in base_jd_core_req that control the type of the atom. @@ -471,62 +434,6 @@ struct base_dependency { base_jd_dep_type dependency_type; }; -/** - * struct base_jd_fragment - Set of GPU fragment job chains used for rendering. - * - * @norm_read_norm_write: Job chain for full rendering. - * GPU address of a fragment job chain to render in the - * circumstance where the tiler job chain did not exceed - * its memory usage threshold and no fragment job chain - * was previously run for the same renderpass. - * It is used no more than once per renderpass. - * @norm_read_forced_write: Job chain for starting incremental - * rendering. - * GPU address of a fragment job chain to render in - * the circumstance where the tiler job chain exceeded - * its memory usage threshold for the first time and - * no fragment job chain was previously run for the - * same renderpass. - * Writes unresolved multisampled and normally- - * discarded output to temporary buffers that must be - * read back by a subsequent forced_read job chain - * before the renderpass is complete. - * It is used no more than once per renderpass. - * @forced_read_forced_write: Job chain for continuing incremental - * rendering. - * GPU address of a fragment job chain to render in - * the circumstance where the tiler job chain - * exceeded its memory usage threshold again - * and a fragment job chain was previously run for - * the same renderpass. - * Reads unresolved multisampled and - * normally-discarded output from temporary buffers - * written by a previous forced_write job chain and - * writes the same to temporary buffers again. - * It is used as many times as required until - * rendering completes. - * @forced_read_norm_write: Job chain for ending incremental rendering. - * GPU address of a fragment job chain to render in the - * circumstance where the tiler job chain did not - * exceed its memory usage threshold this time and a - * fragment job chain was previously run for the same - * renderpass. - * Reads unresolved multisampled and normally-discarded - * output from temporary buffers written by a previous - * forced_write job chain in order to complete a - * renderpass. - * It is used no more than once per renderpass. - * - * This structure is referenced by the main atom structure if - * BASE_JD_REQ_END_RENDERPASS is set in the base_jd_core_req. - */ -struct base_jd_fragment { - __u64 norm_read_norm_write; - __u64 norm_read_forced_write; - __u64 forced_read_forced_write; - __u64 forced_read_norm_write; -}; - /** * typedef base_jd_prio - Base Atom priority. * @@ -591,9 +498,7 @@ typedef __u8 base_jd_prio; * struct base_jd_atom_v2 - Node of a dependency graph used to submit a * GPU job chain or soft-job to the kernel driver. * - * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS - * is set in the base_jd_core_req) the CPU address of a - * base_jd_fragment object. + * @jc: GPU address of a job chain. * @udata: User data. * @extres_list: List of external resources. * @nr_extres: Number of external resources or JIT allocations. @@ -612,9 +517,6 @@ typedef __u8 base_jd_prio; * specified. * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. * @core_req: Core requirements. - * @renderpass_id: Renderpass identifier used to associate an atom that has - * BASE_JD_REQ_START_RENDERPASS set in its core requirements - * with an atom that has BASE_JD_REQ_END_RENDERPASS set. * @padding: Unused. Must be zero. * * This structure has changed since UK 10.2 for which base_jd_core_req was a @@ -642,8 +544,7 @@ struct base_jd_atom_v2 { __u8 device_nr; __u8 jobslot; base_jd_core_req core_req; - __u8 renderpass_id; - __u8 padding[7]; + __u8 padding[8]; }; /** @@ -651,9 +552,7 @@ struct base_jd_atom_v2 { * at the beginning. * * @seq_nr: Sequence number of logical grouping of atoms. - * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS - * is set in the base_jd_core_req) the CPU address of a - * base_jd_fragment object. + * @jc: GPU address of a job chain. * @udata: User data. * @extres_list: List of external resources. * @nr_extres: Number of external resources or JIT allocations. @@ -835,11 +734,6 @@ enum { * @BASE_JD_EVENT_REMOVED_FROM_NEXT: raised when an atom that was configured in * the GPU has to be retried (but it has not * started) due to e.g., GPU reset - * @BASE_JD_EVENT_END_RP_DONE: this is used for incremental rendering to signal - * the completion of a renderpass. This value - * shouldn't be returned to userspace but I haven't - * seen where it is reset back to JD_EVENT_DONE. - * * HW and low-level SW events are represented by event codes. * The status of jobs which succeeded are also represented by * an event code (see @BASE_JD_EVENT_DONE). @@ -938,8 +832,6 @@ enum base_jd_event_code { BASE_JD_EVENT_RANGE_KERNEL_ONLY_START = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | 0x000, BASE_JD_EVENT_REMOVED_FROM_NEXT = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_JOB | 0x000, - BASE_JD_EVENT_END_RP_DONE = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | - BASE_JD_SW_EVENT_JOB | 0x001, BASE_JD_EVENT_RANGE_KERNEL_ONLY_END = BASE_JD_SW_EVENT | BASE_JD_SW_EVENT_KERNEL | BASE_JD_SW_EVENT_RESERVED | 0x3FF diff --git a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h index 1c115adb8172..34da87860ee2 100644 --- a/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h @@ -164,10 +164,16 @@ * - Re-allow child process to do supported file operations (like mmap, ioctl * read, poll) on the file descriptor of mali device that was inherited * from the parent process. - */ + * 11.46: + * - Remove renderpass_id from base_jd_atom_v2 to deprecate support for JM Incremental Rendering + * 11.47: + * - Reject non-protected allocations containing the BASE_MEM_PROTECTED memory flag. + * - Reject allocations containing the BASE_MEM_DONT_NEED memory flag (it is only settable). + * - Reject allocations containing the BASE_MEM_UNUSED_BIT_xx memory flags. + */ #define BASE_UK_VERSION_MAJOR 11 -#define BASE_UK_VERSION_MINOR 45 +#define BASE_UK_VERSION_MINOR 47 /** * struct kbase_ioctl_version_check - Check version compatibility between diff --git a/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h b/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h index c009d5ddd494..bbbee900415e 100644 --- a/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h +++ b/include/uapi/gpu/arm/bifrost/mali_base_common_kernel.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2022-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2022-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -146,7 +146,6 @@ struct base_mem_handle { */ #define BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP ((base_mem_alloc_flags)1 << 26) -/* OUT */ /* Kernel side cache sync ops required */ #define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) @@ -156,12 +155,10 @@ struct base_mem_handle { */ #define BASE_MEM_FLAGS_NR_BITS 30 -/* A mask for all output bits, excluding IN/OUT bits. - */ +/* A mask for all bits that are output from kbase, but never input. */ #define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP -/* A mask for all input bits, including IN/OUT bits. - */ +/* A mask for all bits that can be input to kbase. */ #define BASE_MEM_FLAGS_INPUT_MASK \ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) diff --git a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h index 198d6b8a3942..9e7294970efb 100644 --- a/include/uapi/gpu/arm/bifrost/mali_base_kernel.h +++ b/include/uapi/gpu/arm/bifrost/mali_base_kernel.h @@ -74,7 +74,7 @@ * More flags can be added to this list, as long as they don't clash * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). */ -typedef __u32 base_mem_alloc_flags; +typedef __u64 base_mem_alloc_flags; #define BASE_MEM_FLAGS_MODIFIABLE_NATIVE (BASE_MEM_DONT_NEED) @@ -89,9 +89,10 @@ typedef __u32 base_mem_alloc_flags; /* A mask of all the flags that can be returned via the base_mem_get_flags() * interface. */ -#define BASE_MEM_FLAGS_QUERYABLE \ - (BASE_MEM_FLAGS_INPUT_MASK & ~(BASE_MEM_DONT_QUERY | BASE_MEM_FLAGS_RESERVED | \ - BASE_MEM_FLAGS_UNUSED | BASEP_MEM_FLAGS_KERNEL_ONLY)) +#define BASE_MEM_FLAGS_QUERYABLE \ + (BASE_MEM_FLAGS_INPUT_MASK & \ + ~(BASE_MEM_FLAGS_RESERVED | BASE_MEM_FLAGS_UNUSED | BASE_MEM_FLAGS_ACTION_MODIFIERS | \ + BASEP_MEM_FLAGS_KERNEL_ONLY)) /** * enum base_mem_import_type - Memory types supported by @a base_mem_import diff --git a/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h b/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h index d60745f564b0..163637c62297 100644 --- a/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h +++ b/include/uapi/gpu/arm/bifrost/mali_kbase_ioctl.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2017-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2017-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -169,7 +169,7 @@ struct kbase_ioctl_hwcnt_reader_setup { * struct kbase_ioctl_hwcnt_values - Values to set dummy the dummy counters to. * @data: Counter samples for the dummy model. * @size: Size of the counter sample data. - * @padding: Padding. + * @padding: Currently unused, must be zero */ struct kbase_ioctl_hwcnt_values { __u64 data; @@ -193,7 +193,7 @@ struct kbase_ioctl_disjoint_query { * struct kbase_ioctl_get_ddk_version - Query the kernel version * @version_buffer: Buffer to receive the kernel version string * @size: Size of the buffer - * @padding: Padding + * @padding: Currently unused, must be zero * * The ioctl will return the number of bytes written into version_buffer * (which includes a NULL byte) or a negative error code diff --git a/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h b/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h index 11c51d9c2993..648c166b1e3d 100644 --- a/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h +++ b/include/uapi/gpu/arm/bifrost/mali_kbase_mem_profile_debugfs_buf_size.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * - * (C) COPYRIGHT 2014-2023 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2024 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -30,6 +30,6 @@ * KBASE_MEM_PROFILE_MAX_BUF_SIZE - The size of the buffer to accumulate the histogram report text * in @see @ref CCTXP_HIST_BUF_SIZE_MAX_LENGTH_REPORT */ -#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t)(64 + ((80 + (56 * 64)) * 57) + 56)) +#define KBASE_MEM_PROFILE_MAX_BUF_SIZE ((size_t)(64 + ((80 + (56 * 64)) * 69) + 56)) #endif /*_UAPI_KBASE_MEM_PROFILE_DEBUGFS_BUF_SIZE_H_*/