Quellcode-Bibliothek gfx_v11_0.c Sprache: unbekannt
/* * Copyright 2021 Advanced Micro Devices , Inc . * * Permission is hereby granted , free of charge , to any person obtaining a * copy of this software and associated documentation files ( the " Software " ) , * to deal in the Software without restriction , including without limitation * the rights to use , copy , modify , merge , publish , distribute , sublicense , * and / or sell copies of the Software , and to permit persons to whom the * Software is furnished to do so , subject to the following conditions : * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software . * * THE SOFTWARE IS PROVIDED " AS IS " , WITHOUT WARRANTY OF ANY KIND , EXPRESS OR * IMPLIED , INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY , * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT . IN NO EVENT SHALL * THE COPYRIGHT HOLDER ( S ) OR AUTHOR ( S ) BE LIABLE FOR ANY CLAIM , DAMAGES OR * OTHER LIABILITY , WHETHER IN AN ACTION OF CONTRACT , TORT OR OTHERWISE , * ARISING FROM , OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE . * #include <linux/delay.h>#include <linux/kernel.h>#include <linux/firmware.h>#include <linux/module.h>#include <linux/pci.h>#include "amdgpu.h" #include "amdgpu_gfx.h" #include "amdgpu_psp.h" #include "amdgpu_smu.h" #include "imu_v11_0.h" #include "soc21.h" #include "nvd.h" #include "gc/gc_11_0_0_offset.h" #include "gc/gc_11_0_0_sh_mask.h" #include "smuio/smuio_13_0_6_offset.h" #include "smuio/smuio_13_0_6_sh_mask.h" #include "navi10_enum.h" #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h" #include "soc15.h" #include "clearstate_gfx11.h" #include "v11_structs.h" #include "gfx_v11_0.h" #include "gfx_v11_0_cleaner_shader.h" #include "gfx_v11_0_3.h" #include "nbio_v4_3.h" #include "mes_v11_0.h" #include "mes_userqueue.h" #include "amdgpu_userq_fence.h" #define GFX11_NUM_GFX_RINGS 1 #define GFX11_MEC_HPD_SIZE 2048 #define RLCG_UCODE_LOADING_START_ADDRESS 0 x00002000L#define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0 x1388#define regCGTT_WD_CLK_CTRL 0 x5086#define regCGTT_WD_CLK_CTRL_BASE_IDX 1 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0 x4e7e#define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1 #define regPC_CONFIG_CNTL_1 0 x194d#define regPC_CONFIG_CNTL_1_BASE_IDX 1 #define regCP_GFX_MQD_CONTROL_DEFAULT 0 x00000100#define regCP_GFX_HQD_VMID_DEFAULT 0 x00000000#define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0 x00000000#define regCP_GFX_HQD_QUANTUM_DEFAULT 0 x00000a01#define regCP_GFX_HQD_CNTL_DEFAULT 0 x00a00000#define regCP_RB_DOORBELL_CONTROL_DEFAULT 0 x00000000#define regCP_GFX_HQD_RPTR_DEFAULT 0 x00000000#define regCP_HQD_EOP_CONTROL_DEFAULT 0 x00000006#define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0 x00000000#define regCP_MQD_CONTROL_DEFAULT 0 x00000100#define regCP_HQD_PQ_CONTROL_DEFAULT 0 x00308509#define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0 x00000000#define regCP_HQD_PQ_RPTR_DEFAULT 0 x00000000#define regCP_HQD_PERSISTENT_STATE_DEFAULT 0 x0be05501#define regCP_HQD_IB_CONTROL_DEFAULT 0 x00300000"amdgpu/gc_11_0_0_pfp.bin" );"amdgpu/gc_11_0_0_me.bin" );"amdgpu/gc_11_0_0_mec.bin" );"amdgpu/gc_11_0_0_rlc.bin" );"amdgpu/gc_11_0_0_rlc_kicker.bin" );"amdgpu/gc_11_0_0_rlc_1.bin" );"amdgpu/gc_11_0_0_toc.bin" );"amdgpu/gc_11_0_1_pfp.bin" );"amdgpu/gc_11_0_1_me.bin" );"amdgpu/gc_11_0_1_mec.bin" );"amdgpu/gc_11_0_1_rlc.bin" );"amdgpu/gc_11_0_2_pfp.bin" );"amdgpu/gc_11_0_2_me.bin" );"amdgpu/gc_11_0_2_mec.bin" );"amdgpu/gc_11_0_2_rlc.bin" );"amdgpu/gc_11_0_3_pfp.bin" );"amdgpu/gc_11_0_3_me.bin" );"amdgpu/gc_11_0_3_mec.bin" );"amdgpu/gc_11_0_3_rlc.bin" );"amdgpu/gc_11_0_4_pfp.bin" );"amdgpu/gc_11_0_4_me.bin" );"amdgpu/gc_11_0_4_mec.bin" );"amdgpu/gc_11_0_4_rlc.bin" );"amdgpu/gc_11_5_0_pfp.bin" );"amdgpu/gc_11_5_0_me.bin" );"amdgpu/gc_11_5_0_mec.bin" );"amdgpu/gc_11_5_0_rlc.bin" );"amdgpu/gc_11_5_1_pfp.bin" );"amdgpu/gc_11_5_1_me.bin" );"amdgpu/gc_11_5_1_mec.bin" );"amdgpu/gc_11_5_1_rlc.bin" );"amdgpu/gc_11_5_2_pfp.bin" );"amdgpu/gc_11_5_2_me.bin" );"amdgpu/gc_11_5_2_mec.bin" );"amdgpu/gc_11_5_2_rlc.bin" );"amdgpu/gc_11_5_3_pfp.bin" );"amdgpu/gc_11_5_3_me.bin" );"amdgpu/gc_11_5_3_mec.bin" );"amdgpu/gc_11_5_3_rlc.bin" );static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = {0 , regGRBM_STATUS),0 , regGRBM_STATUS2),0 , regGRBM_STATUS3),0 , regCP_STALLED_STAT1),0 , regCP_STALLED_STAT2),0 , regCP_STALLED_STAT3),0 , regCP_CPC_STALLED_STAT1),0 , regCP_CPF_STALLED_STAT1),0 , regCP_BUSY_STAT),0 , regCP_CPC_BUSY_STAT),0 , regCP_CPF_BUSY_STAT),0 , regCP_CPC_BUSY_STAT2),0 , regCP_CPF_BUSY_STAT2),0 , regCP_CPF_STATUS),0 , regCP_GFX_ERROR),0 , regCP_GFX_HPD_STATUS0),0 , regCP_RB_BASE),0 , regCP_RB_RPTR),0 , regCP_RB_WPTR),0 , regCP_RB0_BASE),0 , regCP_RB0_RPTR),0 , regCP_RB0_WPTR),0 , regCP_RB1_BASE),0 , regCP_RB1_RPTR),0 , regCP_RB1_WPTR),0 , regCP_IB1_CMD_BUFSZ),0 , regCP_IB2_CMD_BUFSZ),0 , regCP_IB1_BASE_LO),0 , regCP_IB1_BASE_HI),0 , regCP_IB1_BUFSZ),0 , regCP_IB2_BASE_LO),0 , regCP_IB2_BASE_HI),0 , regCP_IB2_BUFSZ),0 , regCPF_UTCL1_STATUS),0 , regCPC_UTCL1_STATUS),0 , regCPG_UTCL1_STATUS),0 , regGDS_PROTECTION_FAULT),0 , regGDS_VM_PROTECTION_FAULT),0 , regIA_UTCL1_STATUS),0 , regIA_UTCL1_STATUS_2),0 , regPA_CL_CNTL_STATUS),0 , regRLC_UTCL1_STATUS),0 , regRMI_UTCL1_STATUS),0 , regSQC_CACHES),0 , regSQG_STATUS),0 , regWD_UTCL1_STATUS),0 , regGCVM_L2_PROTECTION_FAULT_CNTL),0 , regGCVM_L2_PROTECTION_FAULT_STATUS),0 , regCP_DEBUG),0 , regCP_MEC_CNTL),0 , regCP_MES_CNTL),0 , regCP_MEC1_INSTR_PNTR),0 , regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),0 , regCP_MES_INSTR_PNTR),0 , regCP_ME_INSTR_PNTR),0 , regCP_PFP_INSTR_PNTR),0 , regCP_CPC_STATUS),/* cp header registers */ 0 , regCP_MES_HEADER_DUMP),0 , regCP_MES_HEADER_DUMP),0 , regCP_MES_HEADER_DUMP),0 , regCP_MES_HEADER_DUMP),0 , regCP_MES_HEADER_DUMP),0 , regCP_MES_HEADER_DUMP),0 , regCP_MES_HEADER_DUMP),0 , regCP_MES_HEADER_DUMP),/* SE status registers */ 0 , regGRBM_STATUS_SE0),0 , regGRBM_STATUS_SE1),0 , regGRBM_STATUS_SE2),0 , regGRBM_STATUS_SE3),0 , regGRBM_STATUS_SE4),0 , regGRBM_STATUS_SE5)static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = {/* compute registers */ 0 , regCP_HQD_VMID),0 , regCP_HQD_PERSISTENT_STATE),0 , regCP_HQD_PIPE_PRIORITY),0 , regCP_HQD_QUEUE_PRIORITY),0 , regCP_HQD_QUANTUM),0 , regCP_HQD_PQ_BASE),0 , regCP_HQD_PQ_BASE_HI),0 , regCP_HQD_PQ_RPTR),0 , regCP_HQD_PQ_WPTR_POLL_ADDR),0 , regCP_HQD_PQ_WPTR_POLL_ADDR_HI),0 , regCP_HQD_PQ_DOORBELL_CONTROL),0 , regCP_HQD_PQ_CONTROL),0 , regCP_HQD_IB_BASE_ADDR),0 , regCP_HQD_IB_BASE_ADDR_HI),0 , regCP_HQD_IB_RPTR),0 , regCP_HQD_IB_CONTROL),0 , regCP_HQD_DEQUEUE_REQUEST),0 , regCP_HQD_EOP_BASE_ADDR),0 , regCP_HQD_EOP_BASE_ADDR_HI),0 , regCP_HQD_EOP_CONTROL),0 , regCP_HQD_EOP_RPTR),0 , regCP_HQD_EOP_WPTR),0 , regCP_HQD_EOP_EVENTS),0 , regCP_HQD_CTX_SAVE_BASE_ADDR_LO),0 , regCP_HQD_CTX_SAVE_BASE_ADDR_HI),0 , regCP_HQD_CTX_SAVE_CONTROL),0 , regCP_HQD_CNTL_STACK_OFFSET),0 , regCP_HQD_CNTL_STACK_SIZE),0 , regCP_HQD_WG_STATE_OFFSET),0 , regCP_HQD_CTX_SAVE_SIZE),0 , regCP_HQD_GDS_RESOURCE_STATE),0 , regCP_HQD_ERROR),0 , regCP_HQD_EOP_WPTR_MEM),0 , regCP_HQD_PQ_WPTR_LO),0 , regCP_HQD_PQ_WPTR_HI),0 , regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),0 , regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),0 , regCP_HQD_SUSPEND_WG_STATE_OFFSET),0 , regCP_HQD_DEQUEUE_STATUS),/* cp header registers */ 0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = {/* gfx queue registers */ 0 , regCP_GFX_HQD_ACTIVE),0 , regCP_GFX_HQD_VMID),0 , regCP_GFX_HQD_QUEUE_PRIORITY),0 , regCP_GFX_HQD_QUANTUM),0 , regCP_GFX_HQD_BASE),0 , regCP_GFX_HQD_BASE_HI),0 , regCP_GFX_HQD_OFFSET),0 , regCP_GFX_HQD_CNTL),0 , regCP_GFX_HQD_CSMD_RPTR),0 , regCP_GFX_HQD_WPTR),0 , regCP_GFX_HQD_WPTR_HI),0 , regCP_GFX_HQD_DEQUEUE_REQUEST),0 , regCP_GFX_HQD_MAPPED),0 , regCP_GFX_HQD_QUE_MGR_CONTROL),0 , regCP_GFX_HQD_HQ_CONTROL0),0 , regCP_GFX_HQD_HQ_STATUS0),0 , regCP_GFX_MQD_BASE_ADDR),0 , regCP_GFX_MQD_BASE_ADDR_HI),0 , regCP_RB_WPTR_POLL_ADDR_LO),0 , regCP_RB_WPTR_POLL_ADDR_HI),0 , regCP_RB_RPTR),0 , regCP_IB1_BASE_LO),0 , regCP_IB1_BASE_HI),0 , regCP_IB1_CMD_BUFSZ),0 , regCP_IB1_BUFSZ),/* cp header registers */ 0 , regCP_PFP_HEADER_DUMP),0 , regCP_PFP_HEADER_DUMP),0 , regCP_PFP_HEADER_DUMP),0 , regCP_PFP_HEADER_DUMP),0 , regCP_PFP_HEADER_DUMP),0 , regCP_PFP_HEADER_DUMP),0 , regCP_PFP_HEADER_DUMP),0 , regCP_PFP_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),0 , regCP_ME_HEADER_DUMP),static const struct soc15_reg_golden golden_settings_gc_11_0[] = {0 , regTCP_CNTL, 0 x20000000, 0 x20000000)static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =0 , regCGTT_GS_NGG_CLK_CTRL, 0 x9fff8fff, 0 x00000010),0 , regCGTT_WD_CLK_CTRL, 0 xffff8fff, 0 x00000010),0 , regCPF_GCR_CNTL, 0 x0007ffff, 0 x0000c200),0 , regGL2C_CTRL3, 0 xffff001b, 0 x00f01988),0 , regPA_CL_ENHANCE, 0 xf0ffffff, 0 x00880007),0 , regPA_SC_ENHANCE_3, 0 xfffffffd, 0 x00000008),0 , regPA_SC_VRS_SURFACE_CNTL_1, 0 xfff891ff, 0 x55480100),0 , regTA_CNTL_AUX, 0 xf7f7ffff, 0 x01030000),0 , regTCP_CNTL2, 0 xfcffffff, 0 x0000000a)#define DEFAULT_SH_MEM_CONFIG \3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,struct amdgpu_cu_info *cu_info);static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,int xcc_id);static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,bool all_hub, uint8_t dst_sel);static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,bool enable);static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)struct amdgpu_device *adev = kiq_ring->adev;/* Cleaner shader MC address */ 8 ;6 ));0 ) |0 xa) | /* unmap_latency: 0xa (~ 1s) */ 0 )); /* vmid_mask:0 queue_type:0 (KIQ) */ /* queue mask lo */ /* queue mask hi */ /* cleaner shader addr lo */ /* cleaner shader addr hi */ 0 ); /* oac mask */ 0 ); /* gds heap base:0, gds heap size:0 */ static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring)0 , eng_sel = 0 ;switch (ring->funcs->type) {case AMDGPU_RING_TYPE_COMPUTE:1 ;0 ;break ;case AMDGPU_RING_TYPE_GFX:0 ;4 ;break ;case AMDGPU_RING_TYPE_MES:2 ;5 ;break ;default :1 );5 ));/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 0 ) | /* Queue_Sel */ 0 ) | /* VMID */ 0 ) | /*queue_type: normal compute queue */ 0 ) | /* alloc format: all_on_one_pipe */ 1 )); /* num_queues: must be 1 */ x));static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring,enum amdgpu_unmap_queues_action action,struct amdgpu_device *adev = kiq_ring->adev;4 : 0 ;if (adev->enable_mes && !adev->gfx.kiq[0 ].ring.sched.ready) {return ;4 ));/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 0 ) |1 ));if (action == PREEMPT_QUEUES_NO_UNMAP) {else {0 );0 );0 );static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,struct amdgpu_ring *ring,4 : 0 ;5 ));0 ) |0 ) |2 ));/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,bool all_hub)1 );static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {8 ,7 ,6 ,7 ,2 ,static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)0 ].pmf = &gfx_v11_0_kiq_pm4_funcs;static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)if (amdgpu_sriov_vf(adev))return ;switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 4 ):const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));break ;default :break ;const u32)ARRAY_SIZE(golden_settings_gc_11_0));static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,bool wc, uint32_t reg, uint32_t val)3 ));0 ) | (wc ? WR_CONFIRM : 0 ));0 );static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,int mem_space, int opt, uint32_t addr0,5 ));/* memory (1) or register (0) */ /* wait */ 3 ) | /* equal */ if (mem_space)0 x3); /* Dword align */ /* poll interval */ static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)/* Header itself is a NOP packet */ if (num_nop == 1 ) {return ;/* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 2 , 0 x3ffe)));/* Header is at index 0, followed by num_nops - 1 NOP packet's */ 1 );static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;0 , regSCRATCH_REG0);0 ;unsigned i;int r;0 xCAFEDEAD);5 );if (r) {"amdgpu: cp failed to lock ring %d (%d).\n" ,return r;if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {0 xDEADBEEF);else {1 ));0 xDEADBEEF);for (i = 0 ; i < adev->usec_timeout; i++) {if (tmp == 0 xDEADBEEF)break ;if (amdgpu_emu_mode == 1 )1 );else 1 );if (i >= adev->usec_timeout)return r;static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)struct amdgpu_device *adev = ring->adev;struct amdgpu_ib ib;struct dma_fence *f = NULL;unsigned index;volatile uint32_t *cpu_ptr;long r;/* MES KIQ fw hasn't indirect buffer support for now */ if (adev->enable_mes_kiq &&return 0 ;0 , sizeof (ib));if (r)return r;4 );0 xCAFEDEAD);20 , AMDGPU_IB_POOL_DIRECT, &ib);if (r) {"amdgpu: failed to get ib (%ld).\n" , r);goto err1;0 ] = PACKET3(PACKET3_WRITE_DATA, 3 );1 ] = WRITE_DATA_DST_SEL(5 ) | WR_CONFIRM;2 ] = lower_32_bits(gpu_addr);3 ] = upper_32_bits(gpu_addr);4 ] = 0 xDEADBEEF;5 ;1 , &ib, NULL, &f);if (r)goto err2;false , timeout);if (r == 0 ) {goto err2;else if (r < 0 ) {goto err2;if (le32_to_cpu(*cpu_ptr) == 0 xDEADBEEF)0 ;else return r;static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)const struct psp_firmware_header_v1_0 *toc_hdr;int err = 0 ;"amdgpu/%s_toc.bin" , ucode_prefix);if (err)goto out;const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;return 0 ;return err;static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 2 ):case IP_VERSION(11 , 0 , 3 ):if ((adev->gfx.me_fw_version >= 1505 ) &&1600 ) &&512 )) {if (amdgpu_sriov_vf(adev))true ;else false ;break ;default :false ;break ;static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)char ucode_prefix[25 ];int err;const struct rlc_firmware_header_v2_0 *rlc_hdr;"\n" );sizeof (ucode_prefix));"amdgpu/%s_pfp.bin" , ucode_prefix);if (err)goto out;/* check pfp fw hdr version to decide if enable rs64 for gfx11.*/ union amdgpu_firmware_header *)2 , 0 );if (adev->gfx.rs64_enable) {"CP RS64 enable\n" );else {"amdgpu/%s_me.bin" , ucode_prefix);if (err)goto out;if (adev->gfx.rs64_enable) {else {if (!amdgpu_sriov_vf(adev)) {if (amdgpu_ip_version(adev, GC_HWIP, 0 ) == IP_VERSION(11 , 0 , 0 ) &&0 xCE)"amdgpu/gc_11_0_0_rlc_1.bin" );else if (amdgpu_is_kicker_fw(adev))"amdgpu/%s_rlc_kicker.bin" , ucode_prefix);else "amdgpu/%s_rlc.bin" , ucode_prefix);if (err)goto out;const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;if (err)goto out;"amdgpu/%s_mec.bin" , ucode_prefix);if (err)goto out;if (adev->gfx.rs64_enable) {else {if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)/* only one MEC for gfx 11.0.0. */ if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) {if (err)"Failed to init imu firmware!\n" );return err;if (err) {return err;static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)0 ;const struct cs_section_def *sect = NULL;const struct cs_extent_def *ext = NULL;/* begin clear state */ 2 ;/* context control state */ 3 ;for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {for (ext = sect->section; ext->extent != NULL; ++ext) {if (sect->id == SECT_CONTEXT)2 + ext->reg_count;else return 0 ;/* set PA_SC_TILE_STEERING_OVERRIDE */ 3 ;/* end clear state */ 2 ;/* clear state */ 2 ;return count;static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,volatile u32 *buffer)0 ;int ctx_reg_offset;if (adev->gfx.rlc.cs_data == NULL)return ;if (buffer == NULL)return ;0 , regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;1 ));static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)/* clear state block */ void **)&adev->gfx.rlc.cs_ptr);/* jump table block */ void **)&adev->gfx.rlc.cp_table_ptr);static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;0 ];0 , regSCRATCH_REG0);0 , regSCRATCH_REG1);0 , regSCRATCH_REG2);0 , regSCRATCH_REG3);0 , regGRBM_GFX_CNTL);0 , regGRBM_GFX_INDEX);0 , regRLC_SPARE_INT_0);true ;static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)const struct cs_section_def *cs_data;int r;if (cs_data) {/* init clear state block */ if (r)return r;/* init spm vmid with 0xf */ if (adev->gfx.rlc.funcs->update_spm_vmid)0 xf);return 0 ;static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)static void gfx_v11_0_me_init(struct amdgpu_device *adev)static int gfx_v11_0_mec_init(struct amdgpu_device *adev)int r;0 ].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);/* take ownership of the relevant compute queues */ if (mec_hpd_size) {void **)&hpd);if (r) {"(%d) create HDP EOP bo failed\n" , r);return r;0 , mec_hpd_size);return 0 ;static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)0 , regSQ_IND_INDEX,return RREG32_SOC15(GC, 0 , regSQ_IND_DATA);static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,0 , regSQ_IND_INDEX,while (num--)0 , regSQ_IND_DATA);static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)/* in gfx11 the SIMD_ID is specified as part of the INSTANCE * field when performing a select_se_sh so it should be 0 );/* type 3 wave data */ 3 ;static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,0 );0 , start + SQIND_WAVE_SGPRS_OFFSET, size,static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,/* all sizes are in bytes */ #define MQD_SHADOW_BASE_SIZE 73728 #define MQD_SHADOW_BASE_ALIGNMENT 256 #define MQD_FWWORKAREA_SIZE 484 #define MQD_FWWORKAREA_ALIGNMENT 256 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev,struct amdgpu_gfx_shadow_info *shadow_info)static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,struct amdgpu_gfx_shadow_info *shadow_info,bool skip_check)if (adev->gfx.cp_gfx_shadow || skip_check) {return 0 ;else {0 , sizeof (struct amdgpu_gfx_shadow_info));return -ENOTSUPP;static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 2 ):8 ;0 x20;0 x100;0 ;0 x4C0;break ;case IP_VERSION(11 , 0 , 3 ):8 ;0 x20;0 x100;0 ;0 x4C0;break ;case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 4 ):case IP_VERSION(11 , 5 , 0 ):case IP_VERSION(11 , 5 , 1 ):case IP_VERSION(11 , 5 , 2 ):case IP_VERSION(11 , 5 , 3 ):8 ;0 x20;0 x100;0 x80;0 x300;break ;default :break ;return 0 ;static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,int me, int pipe, int queue)struct amdgpu_ring *ring;unsigned int irq_type;unsigned int hw_prio;true ;if (adev->gfx.disable_kq) {true ;true ;if (!ring_id)1 ;else 1 ;0 );"gfx_%d.%d.%d" , ring->me, ring->pipe, ring->queue);return amdgpu_ring_init(adev, ring, 1024 , &adev->gfx.eop_irq, irq_type,static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,int mec, int pipe, int queue)int r;unsigned irq_type;struct amdgpu_ring *ring;unsigned int hw_prio;/* mec0 is me1 */ 1 ;true ;1 ;0 );"comp_%d.%d.%d" , ring->me, ring->pipe, ring->queue);1 ) * adev->gfx.mec.num_pipe_per_mec)/* type-2 packets are deprecated on MEC, use type-3 instead */ 1024 , &adev->gfx.eop_irq, irq_type,if (r)return r;return 0 ;static struct {unsigned int offset;unsigned int size;static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&4 ;4 ;static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)0 ;for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)/* In case the offset in rlc toc ucode is aligned */ if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1 ].offset)1 ].offset +1 ].size;return total_size;static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)int r;64 * 1024 ,void **)&adev->gfx.rlc.rlc_autoload_ptr);if (r) {"(%d) failed to create fw autoload bo\n" , r);return r;return 0 ;static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,const void *fw_data,char *ptr = adev->gfx.rlc.rlc_autoload_ptr;if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)return ;if (fw_size == 0 )if (fw_size > toc_fw_size)if (fw_size < toc_fw_size)0 , toc_fw_size - fw_size);if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))1 ULL << id;static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,void *data;0 x1;"rlc autoload enabled fw: 0x%llx\n" , *(uint64_t *)fw_autoload_mask);8 - 1 ;static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,const __le32 *fw_data;const struct gfx_firmware_header_v1_0 *cp_hdr;const struct gfx_firmware_header_v2_0 *cpv2_hdr;const struct rlc_firmware_header_v2_0 *rlc_hdr;const struct rlc_firmware_header_v2_2 *rlcv22_hdr;if (adev->gfx.rs64_enable) {/* pfp ucode */ const struct gfx_firmware_header_v2_0 *)/* instruction */ const __le32 *)(adev->gfx.pfp_fw->data +/* data */ const __le32 *)(adev->gfx.pfp_fw->data +ACK,ACK,/* me ucode */ const struct gfx_firmware_header_v2_0 *)/* instruction */ const __le32 *)(adev->gfx.me_fw->data +/* data */ const __le32 *)(adev->gfx.me_fw->data +CK,CK,/* mec ucode */ const struct gfx_firmware_header_v2_0 *)/* instruction */ const __le32 *) (adev->gfx.mec_fw->data +/* data */ const __le32 *) (adev->gfx.mec_fw->data +ACK,ACK,ACK,ACK,else {/* pfp ucode */ const struct gfx_firmware_header_v1_0 *)const __le32 *)(adev->gfx.pfp_fw->data +/* me ucode */ const struct gfx_firmware_header_v1_0 *)const __le32 *)(adev->gfx.me_fw->data +/* mec ucode */ const struct gfx_firmware_header_v1_0 *)const __le32 *) (adev->gfx.mec_fw->data +4 ;/* rlc ucode */ const struct rlc_firmware_header_v2_0 *)const __le32 *)(adev->gfx.rlc_fw->data +if (version_major == 2 ) {if (version_minor >= 2 ) {const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;const __le32 *)(adev->gfx.rlc_fw->data +const __le32 *)(adev->gfx.rlc_fw->data +,static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,const __le32 *fw_data;const struct sdma_firmware_header_v2_0 *sdma_hdr;const struct sdma_firmware_header_v2_0 *)0 ].fw->data;const __le32 *) (adev->sdma.instance[0 ].fw->data +const __le32 *) (adev->sdma.instance[0 ].fw->data +static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,const __le32 *fw_data;unsigned fw_size;const struct mes_firmware_header_v1_0 *mes_hdr;int pipe, ucode_id, data_id;for (pipe = 0 ; pipe < 2 ; pipe++) {if (pipe==0 ) {else {const struct mes_firmware_header_v1_0 *)const __le32 *)(adev->mes.fw[pipe]->data +const __le32 *)(adev->mes.fw[pipe]->data +static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)2 ];0 , sizeof (uint32_t) * 2 );/* RLC autoload sequence 2: copy ucode */ 0 , regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));0 , regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));0 , regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);/* RLC autoload sequence 3: load IMU fw */ if (adev->gfx.imu.funcs->load_microcode)/* RLC autoload sequence 4 init IMU fw */ if (adev->gfx.imu.funcs->setup_imu)if (adev->gfx.imu.funcs->start_imu)/* RLC autoload sequence 5 disable gpa mode */ return 0 ;static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)sizeof (uint32_t), GFP_KERNEL);if (!ptr) {"Failed to allocate memory for GFX IP Dump\n" );else {/* Allocate memory for compute queue registers for all the instances */ sizeof (uint32_t), GFP_KERNEL);if (!ptr) {"Failed to allocate memory for Compute Queues IP Dump\n" );else {/* Allocate memory for gfx queue registers for all the instances */ sizeof (uint32_t), GFP_KERNEL);if (!ptr) {"Failed to allocate memory for GFX Queues IP Dump\n" );else {static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)int i, j, k, r, ring_id;int xcc_id = 0 ;struct amdgpu_device *adev = ip_block->adev;int num_queue_per_pipe = 1 ; /* we only enable 1 KGQ per pipe */ switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 2 ):case IP_VERSION(11 , 0 , 3 ):case IP_VERSION(11 , 0 , 4 ):case IP_VERSION(11 , 5 , 0 ):case IP_VERSION(11 , 5 , 1 ):case IP_VERSION(11 , 5 , 2 ):case IP_VERSION(11 , 5 , 3 ):1 ;1 ;2 ;1 ;4 ;4 ;break ;default :1 ;1 ;1 ;1 ;4 ;8 ;break ;switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 2 ):case IP_VERSION(11 , 0 , 3 ):if (!adev->gfx.disable_uq &&2420 &&2580 &&2650 &&0 ] >= 120 ) {break ;case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 4 ):case IP_VERSION(11 , 5 , 0 ):case IP_VERSION(11 , 5 , 1 ):case IP_VERSION(11 , 5 , 2 ):case IP_VERSION(11 , 5 , 3 ):/* add firmware version checks here */ if (0 && !adev->gfx.disable_uq) {break ;default :break ;switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 2 ):case IP_VERSION(11 , 0 , 3 ):sizeof (gfx_11_0_3_cleaner_shader_hex);if (adev->gfx.me_fw_version >= 2280 &&2370 &&2450 &&0 ] >= 99 ) {true ;if (r) {false ;"Failed to initialize cleaner shader\n" );break ;case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 4 ):sizeof (gfx_11_0_3_cleaner_shader_hex);if (adev->gfx.pfp_fw_version >= 102 &&66 &&0 ] >= 128 ) {true ;if (r) {false ;"Failed to initialize cleaner shader\n" );break ;case IP_VERSION(11 , 5 , 0 ):case IP_VERSION(11 , 5 , 1 ):sizeof (gfx_11_0_3_cleaner_shader_hex);if (adev->gfx.mec_fw_version >= 26 &&0 ] >= 114 ) {true ;if (r) {false ;"Failed to initialize cleaner shader\n" );break ;case IP_VERSION(11 , 5 , 2 ):sizeof (gfx_11_0_3_cleaner_shader_hex);if (adev->gfx.me_fw_version >= 12 &&15 &&15 ) {true ;if (r) {false ;"Failed to initialize cleaner shader\n" );break ;case IP_VERSION(11 , 5 , 3 ):sizeof (gfx_11_0_3_cleaner_shader_hex);if (adev->gfx.me_fw_version >= 7 &&8 &&8 ) {true ;if (r) {false ;"Failed to initialize cleaner shader\n" );break ;default :false ;break ;/* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */ if (amdgpu_ip_version(adev, GC_HWIP, 0 ) == IP_VERSION(11 , 0 , 3 ) &&/* EOP Event */ if (r)return r;/* Bad opcode Event */ if (r)return r;/* Privileged reg */ if (r)return r;/* Privileged inst */ if (r)return r;/* FED error */ if (r)return r;if (r) {"Failed to init rlc BOs!\n" );return r;if (r) {"Failed to init MEC BOs!\n" );return r;if (adev->gfx.num_gfx_rings) {0 ;/* set up the gfx ring */ for (i = 0 ; i < adev->gfx.me.num_me; i++) {for (j = 0 ; j < num_queue_per_pipe; j++) {for (k = 0 ; k < adev->gfx.me.num_pipe_per_me; k++) {if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))continue ;if (r)return r;if (adev->gfx.num_compute_rings) {0 ;/* set up the compute queues - allocate horizontally across pipes */ for (i = 0 ; i < adev->gfx.mec.num_mec; ++i) {for (j = 0 ; j < adev->gfx.mec.num_queue_per_pipe; j++) {for (k = 0 ; k < adev->gfx.mec.num_pipe_per_mec; k++) {if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0 , i,continue ;if (r)return r;0 ]);0 ]);switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 2 ):case IP_VERSION(11 , 0 , 3 ):if ((adev->gfx.me_fw_version >= 2280 ) &&2410 ) &&break ;default :if (!amdgpu_sriov_vf(adev)) {break ;if (!adev->enable_mes_kiq) {0 );if (r) {"Failed to init KIQ BOs!\n" );return r;if (r)return r;sizeof (struct v11_compute_mqd), 0 );if (r)return r;/* allocate visible FB for rlc auto-loading fw */ if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {if (r)return r;if (r)return r;if (amdgpu_gfx_ras_sw_init(adev)) {"Failed to initialize gfx ras block!\n" );return -EINVAL;if (r)return r;return 0 ;static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)void **)&adev->gfx.pfp.pfp_fw_ptr);void **)&adev->gfx.pfp.pfp_fw_data_ptr);static void gfx_v11_0_me_fini(struct amdgpu_device *adev)void **)&adev->gfx.me.me_fw_ptr);void **)&adev->gfx.me.me_fw_data_ptr);static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)void **)&adev->gfx.rlc.rlc_autoload_ptr);static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block)int i;struct amdgpu_device *adev = ip_block->adev;for (i = 0 ; i < adev->gfx.num_gfx_rings; i++)for (i = 0 ; i < adev->gfx.num_compute_rings; i++)0 );if (!adev->enable_mes_kiq) {0 ].ring);0 );if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)return 0 ;static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,int xcc_id)if (instance == 0 xffffffff)0 , GRBM_GFX_INDEX,1 );else 0 , GRBM_GFX_INDEX, INSTANCE_INDEX,if (se_num == 0 xffffffff)1 );else if (sh_num == 0 xffffffff)1 );else 0 , regGRBM_GFX_INDEX, data);static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)0 , regCC_GC_SA_UNIT_DISABLE);0 , regGC_USER_SA_UNIT_DISABLE);return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)0 , regCC_RB_BACKEND_DISABLE);0 , regGC_USER_RB_BACKEND_DISABLE);return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)0 ;/* query sa bitmap from SA_UNIT_DISABLE registers */ /* query rb bitmap from RB_BACKEND_DISABLE registers */ /* generate active rb bitmap according to active sa bitmap */ for (i = 0 ; i < max_sa; i++) {if (active_sa_bitmap & (1 << i))#define DEFAULT_SH_MEM_BASES (0 x6000)#define LDS_APP_BASE 0 x1#define SCRATCH_APP_BASE 0 x2static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)int i;/* * Configure apertures : * LDS : 0 x60000000 ' 00000000 - 0 x60000001 ' 00000000 ( 4 GB ) * Scratch : 0 x60000001 ' 00000000 - 0 x60000002 ' 00000000 ( 4 GB ) * GPUVM : 0 x60010000 ' 00000000 - 0 x60020000 ' 00000000 ( 1 TB ) for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {0 , 0 , 0 , i);/* CP and shaders */ 0 , regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);0 , regSH_MEM_BASES, sh_mem_bases);/* Enable trap for each kfd vmid. */ 0 , regSPI_GDBG_PER_VMID_CNTL);1 );0 , regSPI_GDBG_PER_VMID_CNTL, data);0 , 0 , 0 , 0 );/* * Initialize all compute VMIDs to have no GDS , GWS , or OA * access . These should be enabled by FW for target VMIDs . for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {0 , regGDS_VMID0_BASE, 2 * i, 0 );0 , regGDS_VMID0_SIZE, 2 * i, 0 );0 , regGDS_GWS_VMID0, i, 0 );0 , regGDS_OA_VMID0, i, 0 );static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)int vmid;/* * Initialize all compute and user - gfx VMIDs to have no GDS , GWS , or OA * access . Compute VMIDs should be enabled by FW for target VMIDs , * the driver can enable them for graphics . VMID0 should maintain * access so that HWS firmware can save / restore entries . for (vmid = 1 ; vmid < 16 ; vmid++) {0 , regGDS_VMID0_BASE, 2 * vmid, 0 );0 , regGDS_VMID0_SIZE, 2 * vmid, 0 );0 , regGDS_GWS_VMID0, vmid, 0 );0 , regGDS_OA_VMID0, vmid, 0 );static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)/* TODO: harvest feature to be added later. */ static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)/* TCCs are global (not instanced). */ 0 , regCGTS_TCC_DISABLE) |0 , regCGTS_USER_TCC_DISABLE);16 );static void gfx_v11_0_constants_init(struct amdgpu_device *adev)int i;if (!amdgpu_sriov_vf(adev))0 , GRBM_CNTL, READ_TIMEOUT, 0 xff);0 ;/* Set whether texture coordinate truncation is conformant. */ 0 , regTA_CNTL2);/* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ for (i = 0 ; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0 )].num_ids; i++) {0 , 0 , 0 , i);/* CP and shaders */ 0 , regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);if (i != 0 ) {0 , SH_MEM_BASES, PRIVATE_BASE,48 ));48 ));0 , regSH_MEM_BASES, tmp);0 , 0 , 0 , 0 );static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev,int me, int pipe)if (me != 0 )return 0 ;switch (pipe) {case 0 :return SOC15_REG_OFFSET(GC, 0 , regCP_INT_CNTL_RING0);case 1 :return SOC15_REG_OFFSET(GC, 0 , regCP_INT_CNTL_RING1);default :return 0 ;static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev,int me, int pipe)/* * amdgpu controls only the first MEC . That ' s why this function only * handles the setting of interrupts for this specific MEC . All other * pipes ' interrupts are set by amdkfd . if (me != 1 )return 0 ;switch (pipe) {case 0 :return SOC15_REG_OFFSET(GC, 0 , regCP_ME1_PIPE0_INT_CNTL);case 1 :return SOC15_REG_OFFSET(GC, 0 , regCP_ME1_PIPE1_INT_CNTL);case 2 :return SOC15_REG_OFFSET(GC, 0 , regCP_ME1_PIPE2_INT_CNTL);case 3 :return SOC15_REG_OFFSET(GC, 0 , regCP_ME1_PIPE3_INT_CNTL);default :return 0 ;static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,bool enable)int i, j;if (amdgpu_sriov_vf(adev))return ;for (i = 0 ; i < adev->gfx.me.num_me; i++) {for (j = 0 ; j < adev->gfx.me.num_pipe_per_me; j++) {if (cp_int_cntl_reg) {1 : 0 );1 : 0 );1 : 0 );1 : 0 );static int gfx_v11_0_init_csb(struct amdgpu_device *adev)0 , regRLC_CSIB_ADDR_HI,32 );0 , regRLC_CSIB_ADDR_LO,0 xfffffffc);0 , regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);return 0 ;static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)0 , regRLC_CNTL);0 );0 , regRLC_CNTL, tmp);static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)0 , GRBM_SOFT_RESET, SOFT_RESET_RLC, 1 );50 );0 , GRBM_SOFT_RESET, SOFT_RESET_RLC, 0 );50 );static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,bool enable)0 , regRLC_PG_CNTL);if (!enable) {/* RLC_PG_CNTL[23] = 0 (default) * RLC will wait for handshake acks with SMU * GFXOFF will be enabled * RLC_PG_CNTL [ 23 ] = 1 * RLC will not issue any message to SMU * hence no handshake between SMU & RLC * GFXOFF will be disabled else 0 , regRLC_PG_CNTL, rlc_pg_cntl);static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)/* TODO: enable rlc & smu handshake until smu if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))false );0 , RLC_CNTL, RLC_ENABLE_F32, 1 );50 );static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)/* enable Save Restore Machine */ 0 , regRLC_SRM_CNTL));0 , regRLC_SRM_CNTL), tmp);static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)const struct rlc_firmware_header_v2_0 *hdr;const __le32 *fw_data;unsigned i, fw_size;const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;const __le32 *)(adev->gfx.rlc_fw->data +4 ;0 , regRLC_GPM_UCODE_ADDR,for (i = 0 ; i < fw_size; i++)0 , regRLC_GPM_UCODE_DATA,0 , regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)const struct rlc_firmware_header_v2_2 *hdr;const __le32 *fw_data;unsigned i, fw_size;const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;const __le32 *)(adev->gfx.rlc_fw->data +4 ;0 , regRLC_LX6_IRAM_ADDR, 0 );for (i = 0 ; i < fw_size; i++) {if ((amdgpu_emu_mode == 1 ) && (i % 100 == 99 ))1 );0 , regRLC_LX6_IRAM_DATA,0 , regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);const __le32 *)(adev->gfx.rlc_fw->data +4 ;0 , regRLC_LX6_DRAM_ADDR, 0 );for (i = 0 ; i < fw_size; i++) {if ((amdgpu_emu_mode == 1 ) && (i % 100 == 99 ))1 );0 , regRLC_LX6_DRAM_DATA,0 , regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);0 , regRLC_LX6_CNTL);1 );0 );0 , regRLC_LX6_CNTL, tmp);static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)const struct rlc_firmware_header_v2_3 *hdr;const __le32 *fw_data;unsigned i, fw_size;const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;const __le32 *)(adev->gfx.rlc_fw->data +4 ;0 , regRLC_PACE_UCODE_ADDR, 0 );for (i = 0 ; i < fw_size; i++) {if ((amdgpu_emu_mode == 1 ) && (i % 100 == 99 ))1 );0 , regRLC_PACE_UCODE_DATA,0 , regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);0 , regRLC_GPM_THREAD_ENABLE);1 );0 , regRLC_GPM_THREAD_ENABLE, tmp);const __le32 *)(adev->gfx.rlc_fw->data +4 ;0 , regRLC_GPU_IOV_UCODE_ADDR, 0 );for (i = 0 ; i < fw_size; i++) {if ((amdgpu_emu_mode == 1 ) && (i % 100 == 99 ))1 );0 , regRLC_GPU_IOV_UCODE_DATA,0 , regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);0 , regRLC_GPU_IOV_F32_CNTL);1 );0 , regRLC_GPU_IOV_F32_CNTL, tmp);static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)const struct rlc_firmware_header_v2_0 *hdr;if (!adev->gfx.rlc_fw)return -EINVAL;const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;if (version_major == 2 ) {if (amdgpu_dpm == 1 ) {if (version_minor >= 2 )if (version_minor == 3 )return 0 ;return -EINVAL;static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)int r;if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */ else {if (amdgpu_sriov_vf(adev)) {return 0 ;/* disable CG */ 0 , regRLC_CGCG_CGLS_CTRL, 0 );/* disable PG */ 0 , regRLC_PG_CNTL, 0 );if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {/* legacy rlc firmware loading */ if (r)return r;return 0 ;static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)50000 ; /* wait for 50ms */ int i;/* Trigger an invalidation of the L1 instruction caches */ 0 , regCP_ME_IC_OP_CNTL);1 );0 , regCP_ME_IC_OP_CNTL, tmp);/* Wait for invalidation complete */ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_ME_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;if (amdgpu_emu_mode == 1 )0 , regCP_ME_IC_BASE_CNTL);0 );0 );0 );1 );0 , regCP_ME_IC_BASE_CNTL, tmp);/* Program me ucode address into intruction cache address register */ 0 , regCP_ME_IC_BASE_LO,0 xFFFFF000);0 , regCP_ME_IC_BASE_HI,return 0 ;static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)50000 ; /* wait for 50ms */ int i;/* Trigger an invalidation of the L1 instruction caches */ 0 , regCP_PFP_IC_OP_CNTL);1 );0 , regCP_PFP_IC_OP_CNTL, tmp);/* Wait for invalidation complete */ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_PFP_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;if (amdgpu_emu_mode == 1 )0 , regCP_PFP_IC_BASE_CNTL);0 );0 );0 );1 );0 , regCP_PFP_IC_BASE_CNTL, tmp);/* Program pfp ucode address into intruction cache address register */ 0 , regCP_PFP_IC_BASE_LO,0 xFFFFF000);0 , regCP_PFP_IC_BASE_HI,return 0 ;static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)50000 ; /* wait for 50ms */ int i;/* Trigger an invalidation of the L1 instruction caches */ 0 , regCP_CPC_IC_OP_CNTL);1 );0 , regCP_CPC_IC_OP_CNTL, tmp);/* Wait for invalidation complete */ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_CPC_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;if (amdgpu_emu_mode == 1 )0 , regCP_CPC_IC_BASE_CNTL);0 );0 );1 );0 , regCP_CPC_IC_BASE_CNTL, tmp);/* Program mec1 ucode address into intruction cache address register */ 0 , regCP_CPC_IC_BASE_LO,0 xFFFFF000);0 , regCP_CPC_IC_BASE_HI,return 0 ;static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)50000 ; /* wait for 50ms */ unsigned i, pipe_id;const struct gfx_firmware_header_v2_0 *pfp_hdr;const struct gfx_firmware_header_v2_0 *)0 , regCP_PFP_IC_BASE_LO,0 , regCP_PFP_IC_BASE_HI,0 , regCP_PFP_IC_BASE_CNTL);0 );0 );0 );0 , regCP_PFP_IC_BASE_CNTL, tmp);/* * Programming any of the CP_PFP_IC_BASE registers * forces invalidation of the ME L1 I $ . Wait for the * invalidation complete for (i = 0 ; i < usec_timeout; i++) {0 , regCP_PFP_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;/* Prime the L1 instruction caches */ 0 , regCP_PFP_IC_OP_CNTL);1 );0 , regCP_PFP_IC_OP_CNTL, tmp);/* Waiting for cache primed*/ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_PFP_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to prime instruction cache\n" );return -EINVAL;for (pipe_id = 0 ; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {0 , pipe_id, 0 , 0 );0 , regCP_PFP_PRGRM_CNTR_START,30 ) |2 ));0 , regCP_PFP_PRGRM_CNTR_START_HI,2 );/* * Program CP_ME_CNTL to reset given PIPE to take * effect of CP_PFP_PRGRM_CNTR_START . 0 , regCP_ME_CNTL);if (pipe_id == 0 )1 );else 1 );0 , regCP_ME_CNTL, tmp);/* Clear pfp pipe0 reset bit. */ if (pipe_id == 0 )0 );else 0 );0 , regCP_ME_CNTL, tmp);0 , regCP_GFX_RS64_DC_BASE0_LO,0 , regCP_GFX_RS64_DC_BASE0_HI,0 , 0 , 0 , 0 );0 , regCP_GFX_RS64_DC_BASE_CNTL);0 );0 );0 , regCP_GFX_RS64_DC_BASE_CNTL, tmp);/* Invalidate the data caches */ 0 , regCP_GFX_RS64_DC_OP_CNTL);1 );0 , regCP_GFX_RS64_DC_OP_CNTL, tmp);for (i = 0 ; i < usec_timeout; i++) {0 , regCP_GFX_RS64_DC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate RS64 data cache\n" );return -EINVAL;return 0 ;static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)50000 ; /* wait for 50ms */ unsigned i, pipe_id;const struct gfx_firmware_header_v2_0 *me_hdr;const struct gfx_firmware_header_v2_0 *)0 , regCP_ME_IC_BASE_LO,0 , regCP_ME_IC_BASE_HI,0 , regCP_ME_IC_BASE_CNTL);0 );0 );0 );0 , regCP_ME_IC_BASE_CNTL, tmp);/* * Programming any of the CP_ME_IC_BASE registers * forces invalidation of the ME L1 I $ . Wait for the * invalidation complete for (i = 0 ; i < usec_timeout; i++) {0 , regCP_ME_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;/* Prime the instruction caches */ 0 , regCP_ME_IC_OP_CNTL);1 );0 , regCP_ME_IC_OP_CNTL, tmp);/* Waiting for instruction cache primed*/ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_ME_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to prime instruction cache\n" );return -EINVAL;for (pipe_id = 0 ; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {0 , pipe_id, 0 , 0 );0 , regCP_ME_PRGRM_CNTR_START,30 ) |2 ) );0 , regCP_ME_PRGRM_CNTR_START_HI,2 );/* * Program CP_ME_CNTL to reset given PIPE to take * effect of CP_PFP_PRGRM_CNTR_START . 0 , regCP_ME_CNTL);if (pipe_id == 0 )1 );else 1 );0 , regCP_ME_CNTL, tmp);/* Clear pfp pipe0 reset bit. */ if (pipe_id == 0 )0 );else 0 );0 , regCP_ME_CNTL, tmp);0 , regCP_GFX_RS64_DC_BASE1_LO,0 , regCP_GFX_RS64_DC_BASE1_HI,0 , 0 , 0 , 0 );0 , regCP_GFX_RS64_DC_BASE_CNTL);0 );0 );0 , regCP_GFX_RS64_DC_BASE_CNTL, tmp);/* Invalidate the data caches */ 0 , regCP_GFX_RS64_DC_OP_CNTL);1 );0 , regCP_GFX_RS64_DC_OP_CNTL, tmp);for (i = 0 ; i < usec_timeout; i++) {0 , regCP_GFX_RS64_DC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate RS64 data cache\n" );return -EINVAL;return 0 ;static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)50000 ; /* wait for 50ms */ unsigned i;const struct gfx_firmware_header_v2_0 *mec_hdr;const struct gfx_firmware_header_v2_0 *)0 , regCP_CPC_IC_BASE_CNTL);0 );0 );0 );0 , regCP_CPC_IC_BASE_CNTL, tmp);0 , regCP_MEC_DC_BASE_CNTL);0 );0 );0 , regCP_MEC_DC_BASE_CNTL, tmp);for (i = 0 ; i < adev->gfx.mec.num_pipe_per_mec; i++) {1 , i, 0 , 0 );0 , regCP_MEC_MDBASE_LO, addr2);0 , regCP_MEC_MDBASE_HI,0 , regCP_MEC_RS64_PRGRM_CNTR_START,2 |30 );0 , regCP_MEC_RS64_PRGRM_CNTR_START_HI,2 );0 , regCP_CPC_IC_BASE_LO, addr);0 , regCP_CPC_IC_BASE_HI,0 , 0 , 0 , 0 );/* Trigger an invalidation of the L1 instruction caches */ 0 , regCP_MEC_DC_OP_CNTL);1 );0 , regCP_MEC_DC_OP_CNTL, tmp);/* Wait for invalidation complete */ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_MEC_DC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;/* Trigger an invalidation of the L1 instruction caches */ 0 , regCP_CPC_IC_OP_CNTL);1 );0 , regCP_CPC_IC_OP_CNTL, tmp);/* Wait for invalidation complete */ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_CPC_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;return 0 ;static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)const struct gfx_firmware_header_v2_0 *pfp_hdr;const struct gfx_firmware_header_v2_0 *me_hdr;const struct gfx_firmware_header_v2_0 *mec_hdr;const struct gfx_firmware_header_v2_0 *)const struct gfx_firmware_header_v2_0 *)const struct gfx_firmware_header_v2_0 *)/* config pfp program start addr */ for (pipe_id = 0 ; pipe_id < 2 ; pipe_id++) {0 , pipe_id, 0 , 0 );0 , regCP_PFP_PRGRM_CNTR_START,30 ) |2 ));0 , regCP_PFP_PRGRM_CNTR_START_HI,2 );0 , 0 , 0 , 0 );/* reset pfp pipe */ 0 , regCP_ME_CNTL);1 );1 );0 , regCP_ME_CNTL, tmp);/* clear pfp pipe reset */ 0 );0 );0 , regCP_ME_CNTL, tmp);/* config me program start addr */ for (pipe_id = 0 ; pipe_id < 2 ; pipe_id++) {0 , pipe_id, 0 , 0 );0 , regCP_ME_PRGRM_CNTR_START,30 ) |2 ) );0 , regCP_ME_PRGRM_CNTR_START_HI,2 );0 , 0 , 0 , 0 );/* reset me pipe */ 0 , regCP_ME_CNTL);1 );1 );0 , regCP_ME_CNTL, tmp);/* clear me pipe reset */ 0 );0 );0 , regCP_ME_CNTL, tmp);/* config mec program start addr */ for (pipe_id = 0 ; pipe_id < 4 ; pipe_id++) {1 , pipe_id, 0 , 0 );0 , regCP_MEC_RS64_PRGRM_CNTR_START,2 |30 );0 , regCP_MEC_RS64_PRGRM_CNTR_START_HI,2 );0 , 0 , 0 , 0 );/* reset mec pipe */ 0 , regCP_MEC_RS64_CNTL);1 );1 );1 );1 );0 , regCP_MEC_RS64_CNTL, tmp);/* clear mec pipe reset */ 0 );0 );0 );0 );0 , regCP_MEC_RS64_CNTL, tmp);static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)int i, r;for (i = 0 ; i < adev->usec_timeout; i++) {0 , regCP_STAT);if (amdgpu_ip_version(adev, GC_HWIP, 0 ) ==11 , 0 , 1 ) ||0 ) ==11 , 0 , 4 ) ||0 ) == IP_VERSION(11 , 5 , 0 ) ||0 ) == IP_VERSION(11 , 5 , 1 ) ||0 ) == IP_VERSION(11 , 5 , 2 ) ||0 ) == IP_VERSION(11 , 5 , 3 ))0 ,else 0 , regRLC_RLCS_BOOTLOAD_STATUS);if ((cp_status == 0 ) &&1 )) {break ;1 );if (i >= adev->usec_timeout) {"rlc autoload: gc ucode autoload timeout\n" );return -ETIMEDOUT;if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {if (adev->gfx.rs64_enable) {if (r)return r;if (r)return r;if (r)return r;else {if (r)return r;if (r)return r;if (r)return r;return 0 ;static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)int i;0 , regCP_ME_CNTL);0 : 1 );0 : 1 );0 , regCP_ME_CNTL, tmp);for (i = 0 ; i < adev->usec_timeout; i++) {if (RREG32_SOC15(GC, 0 , regCP_STAT) == 0 )break ;1 );if (i >= adev->usec_timeout)"failed to %s cp gfx\n" , enable ? "unhalt" : "halt" );return 0 ;static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)int r;const struct gfx_firmware_header_v1_0 *pfp_hdr;const __le32 *fw_data;unsigned i, fw_size;const struct gfx_firmware_header_v1_0 *)const __le32 *)(adev->gfx.pfp_fw->data +void **)&adev->gfx.pfp.pfp_fw_ptr);if (r) {"(%d) failed to create pfp fw bo\n" , r);return r;0 , regCP_HYP_PFP_UCODE_ADDR, 0 );for (i = 0 ; i < pfp_hdr->jt_size; i++)0 , regCP_HYP_PFP_UCODE_DATA,0 , regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);return 0 ;static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)int r;const struct gfx_firmware_header_v2_0 *pfp_hdr;const __le32 *fw_ucode, *fw_data;unsigned i, pipe_id, fw_ucode_size, fw_data_size;50000 ; /* wait for 50ms */ const struct gfx_firmware_header_v2_0 *)/* instruction */ const __le32 *)(adev->gfx.pfp_fw->data +/* data */ const __le32 *)(adev->gfx.pfp_fw->data +/* 64kb align */ 64 * 1024 ,void **)&adev->gfx.pfp.pfp_fw_ptr);if (r) {"(%d) failed to create pfp ucode fw bo\n" , r);return r;64 * 1024 ,void **)&adev->gfx.pfp.pfp_fw_data_ptr);if (r) {"(%d) failed to create pfp data fw bo\n" , r);return r;if (amdgpu_emu_mode == 1 )0 , regCP_PFP_IC_BASE_LO,0 , regCP_PFP_IC_BASE_HI,0 , regCP_PFP_IC_BASE_CNTL);0 );0 );0 );0 , regCP_PFP_IC_BASE_CNTL, tmp);/* * Programming any of the CP_PFP_IC_BASE registers * forces invalidation of the ME L1 I $ . Wait for the * invalidation complete for (i = 0 ; i < usec_timeout; i++) {0 , regCP_PFP_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;/* Prime the L1 instruction caches */ 0 , regCP_PFP_IC_OP_CNTL);1 );0 , regCP_PFP_IC_OP_CNTL, tmp);/* Waiting for cache primed*/ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_PFP_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to prime instruction cache\n" );return -EINVAL;for (pipe_id = 0 ; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {0 , pipe_id, 0 , 0 );0 , regCP_PFP_PRGRM_CNTR_START,30 ) |2 ) );0 , regCP_PFP_PRGRM_CNTR_START_HI,2 );/* * Program CP_ME_CNTL to reset given PIPE to take * effect of CP_PFP_PRGRM_CNTR_START . 0 , regCP_ME_CNTL);if (pipe_id == 0 )1 );else 1 );0 , regCP_ME_CNTL, tmp);/* Clear pfp pipe0 reset bit. */ if (pipe_id == 0 )0 );else 0 );0 , regCP_ME_CNTL, tmp);0 , regCP_GFX_RS64_DC_BASE0_LO,0 , regCP_GFX_RS64_DC_BASE0_HI,0 , 0 , 0 , 0 );0 , regCP_GFX_RS64_DC_BASE_CNTL);0 );0 );0 , regCP_GFX_RS64_DC_BASE_CNTL, tmp);/* Invalidate the data caches */ 0 , regCP_GFX_RS64_DC_OP_CNTL);1 );0 , regCP_GFX_RS64_DC_OP_CNTL, tmp);for (i = 0 ; i < usec_timeout; i++) {0 , regCP_GFX_RS64_DC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate RS64 data cache\n" );return -EINVAL;return 0 ;static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)int r;const struct gfx_firmware_header_v1_0 *me_hdr;const __le32 *fw_data;unsigned i, fw_size;const struct gfx_firmware_header_v1_0 *)const __le32 *)(adev->gfx.me_fw->data +void **)&adev->gfx.me.me_fw_ptr);if (r) {"(%d) failed to create me fw bo\n" , r);return r;0 , regCP_HYP_ME_UCODE_ADDR, 0 );for (i = 0 ; i < me_hdr->jt_size; i++)0 , regCP_HYP_ME_UCODE_DATA,0 , regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);return 0 ;static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)int r;const struct gfx_firmware_header_v2_0 *me_hdr;const __le32 *fw_ucode, *fw_data;unsigned i, pipe_id, fw_ucode_size, fw_data_size;50000 ; /* wait for 50ms */ const struct gfx_firmware_header_v2_0 *)/* instruction */ const __le32 *)(adev->gfx.me_fw->data +/* data */ const __le32 *)(adev->gfx.me_fw->data +/* 64kb align*/ 64 * 1024 ,void **)&adev->gfx.me.me_fw_ptr);if (r) {"(%d) failed to create me ucode bo\n" , r);return r;64 * 1024 ,void **)&adev->gfx.me.me_fw_data_ptr);if (r) {"(%d) failed to create me data bo\n" , r);return r;if (amdgpu_emu_mode == 1 )0 , regCP_ME_IC_BASE_LO,0 , regCP_ME_IC_BASE_HI,0 , regCP_ME_IC_BASE_CNTL);0 );0 );0 );0 , regCP_ME_IC_BASE_CNTL, tmp);/* * Programming any of the CP_ME_IC_BASE registers * forces invalidation of the ME L1 I $ . Wait for the * invalidation complete for (i = 0 ; i < usec_timeout; i++) {0 , regCP_ME_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;/* Prime the instruction caches */ 0 , regCP_ME_IC_OP_CNTL);1 );0 , regCP_ME_IC_OP_CNTL, tmp);/* Waiting for instruction cache primed*/ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_ME_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to prime instruction cache\n" );return -EINVAL;for (pipe_id = 0 ; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {0 , pipe_id, 0 , 0 );0 , regCP_ME_PRGRM_CNTR_START,30 ) |2 ) );0 , regCP_ME_PRGRM_CNTR_START_HI,2 );/* * Program CP_ME_CNTL to reset given PIPE to take * effect of CP_PFP_PRGRM_CNTR_START . 0 , regCP_ME_CNTL);if (pipe_id == 0 )1 );else 1 );0 , regCP_ME_CNTL, tmp);/* Clear pfp pipe0 reset bit. */ if (pipe_id == 0 )0 );else 0 );0 , regCP_ME_CNTL, tmp);0 , regCP_GFX_RS64_DC_BASE1_LO,0 , regCP_GFX_RS64_DC_BASE1_HI,0 , 0 , 0 , 0 );0 , regCP_GFX_RS64_DC_BASE_CNTL);0 );0 );0 , regCP_GFX_RS64_DC_BASE_CNTL, tmp);/* Invalidate the data caches */ 0 , regCP_GFX_RS64_DC_OP_CNTL);1 );0 , regCP_GFX_RS64_DC_OP_CNTL, tmp);for (i = 0 ; i < usec_timeout; i++) {0 , regCP_GFX_RS64_DC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate RS64 data cache\n" );return -EINVAL;return 0 ;static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)int r;if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)return -EINVAL;false );if (adev->gfx.rs64_enable)else if (r) {"(%d) failed to load pfp fw\n" , r);return r;if (adev->gfx.rs64_enable)else if (r) {"(%d) failed to load me fw\n" , r);return r;return 0 ;static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)struct amdgpu_ring *ring;const struct cs_section_def *sect = NULL;const struct cs_extent_def *ext = NULL;int r, i;int ctx_reg_offset;/* init the CP */ 0 , regCP_MAX_CONTEXT,1 );0 , regCP_DEVICE_ID, 1 );if (!amdgpu_async_gfx_ring)true );0 ];if (r) {"amdgpu: cp failed to lock ring (%d).\n" , r);return r;0 ));1 ));0 x80000000);0 x80000000);for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {for (ext = sect->section; ext->extent != NULL; ++ext) {if (sect->id == SECT_CONTEXT) {for (i = 0 ; i < ext->reg_count; i++)0 , regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;1 ));0 ));0 ));0 );/* submit cs packet to copy state 0 to next available state */ if (adev->gfx.num_gfx_rings > 1 ) {/* maximum supported gfx ring is 2 */ 1 ];2 );if (r) {"amdgpu: cp failed to lock ring (%d).\n" , r);return r;0 ));0 );return 0 ;static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,0 , regGRBM_GFX_CNTL);0 , regGRBM_GFX_CNTL, tmp);static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,struct amdgpu_ring *ring)0 , regCP_RB_DOORBELL_CONTROL);if (ring->use_doorbell) {1 );else {0 );0 , regCP_RB_DOORBELL_CONTROL, tmp);0 , CP_RB_DOORBELL_RANGE_LOWER,0 , regCP_RB_DOORBELL_RANGE_LOWER, tmp);0 , regCP_RB_DOORBELL_RANGE_UPPER,static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)struct amdgpu_ring *ring;/* Set the write pointer delay */ 0 , regCP_RB_WPTR_DELAY, 0 );/* set the RB to use vmid 0 */ 0 , regCP_RB_VMID, 0 );/* Init gfx ring 0 for pipe 0 */ /* Set ring buffer size */ 0 ];8 );0 , CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);2 );0 , regCP_RB0_CNTL, tmp);/* Initialize the ring buffer's write pointers */ 0 ;0 , regCP_RB0_WPTR, lower_32_bits(ring->wptr));0 , regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));/* set the wb address whether it's enabled or not */ 0 , regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));0 , regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &0 , regCP_RB_WPTR_POLL_ADDR_LO,0 , regCP_RB_WPTR_POLL_ADDR_HI,1 );0 , regCP_RB0_CNTL, tmp);8 ;0 , regCP_RB0_BASE, rb_addr);0 , regCP_RB0_BASE_HI, upper_32_bits(rb_addr));0 , regCP_RB_ACTIVE, 1 );/* Init gfx ring 1 for pipe 1 */ if (adev->gfx.num_gfx_rings > 1 ) {/* maximum supported gfx ring is 2 */ 1 ];8 );0 , CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);2 );0 , regCP_RB1_CNTL, tmp);/* Initialize the ring buffer's write pointers */ 0 ;0 , regCP_RB1_WPTR, lower_32_bits(ring->wptr));0 , regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));/* Set the wb address whether it's enabled or not */ 0 , regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));0 , regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &0 , regCP_RB_WPTR_POLL_ADDR_LO,0 , regCP_RB_WPTR_POLL_ADDR_HI,1 );0 , regCP_RB1_CNTL, tmp);8 ;0 , regCP_RB1_BASE, rb_addr);0 , regCP_RB1_BASE_HI, upper_32_bits(rb_addr));0 , regCP_RB1_ACTIVE, 1 );/* Switch to pipe 0 */ /* start the ring */ return 0 ;static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)if (adev->gfx.rs64_enable) {0 , regCP_MEC_RS64_CNTL);0 : 1 );0 : 1 );0 : 1 );0 : 1 );0 : 1 );1 : 0 );1 : 0 );1 : 0 );1 : 0 );0 : 1 );0 , regCP_MEC_RS64_CNTL, data);else {0 , regCP_MEC_CNTL);if (enable) {0 );if (!adev->enable_mes_kiq)0 );else {1 );1 );0 , regCP_MEC_CNTL, data);50 );static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)const struct gfx_firmware_header_v1_0 *mec_hdr;const __le32 *fw_data;unsigned i, fw_size;int r;if (!adev->gfx.mec_fw)return -EINVAL;false );const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;const __le32 *)void **)&fw);if (r) {"(%d) failed to create mec fw bo\n" , r);return r;/* MEC1 */ 0 , regCP_MEC_ME1_UCODE_ADDR, 0 );for (i = 0 ; i < mec_hdr->jt_size; i++)0 , regCP_MEC_ME1_UCODE_DATA,0 , regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);return 0 ;static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)const struct gfx_firmware_header_v2_0 *mec_hdr;const __le32 *fw_ucode, *fw_data;50000 ; /* Wait for 50 ms */ int r;if (!adev->gfx.mec_fw)return -EINVAL;false );const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;const __le32 *) (adev->gfx.mec_fw->data +const __le32 *) (adev->gfx.mec_fw->data +64 * 1024 ,void **)&fw_ucode_ptr);if (r) {"(%d) failed to create mec fw ucode bo\n" , r);return r;64 * 1024 ,void **)&fw_data_ptr);if (r) {"(%d) failed to create mec fw ucode bo\n" , r);return r;0 , regCP_CPC_IC_BASE_CNTL);0 );0 );0 );0 , regCP_CPC_IC_BASE_CNTL, tmp);0 , regCP_MEC_DC_BASE_CNTL);0 );0 );0 , regCP_MEC_DC_BASE_CNTL, tmp);for (i = 0 ; i < adev->gfx.mec.num_pipe_per_mec; i++) {1 , i, 0 , 0 );0 , regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);0 , regCP_MEC_MDBASE_HI,0 , regCP_MEC_RS64_PRGRM_CNTR_START,2 |30 );0 , regCP_MEC_RS64_PRGRM_CNTR_START_HI,2 );0 , regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);0 , regCP_CPC_IC_BASE_HI,0 , 0 , 0 , 0 );/* Trigger an invalidation of the L1 instruction caches */ 0 , regCP_MEC_DC_OP_CNTL);1 );0 , regCP_MEC_DC_OP_CNTL, tmp);/* Wait for invalidation complete */ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_MEC_DC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;/* Trigger an invalidation of the L1 instruction caches */ 0 , regCP_CPC_IC_OP_CNTL);1 );0 , regCP_CPC_IC_OP_CNTL, tmp);/* Wait for invalidation complete */ for (i = 0 ; i < usec_timeout; i++) {0 , regCP_CPC_IC_OP_CNTL);if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,break ;1 );if (i >= usec_timeout) {"failed to invalidate instruction cache\n" );return -EINVAL;return 0 ;static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;/* tell RLC which is KIQ queue */ 0 , regRLC_CP_SCHEDULERS);0 xffffff00;5 ) | (ring->pipe << 3 ) | (ring->queue);0 , regRLC_CP_SCHEDULERS, tmp | 0 x80);static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)/* set graphics engine doorbell range */ 0 , regCP_RB_DOORBELL_RANGE_LOWER,2 ) << 2 );0 , regCP_RB_DOORBELL_RANGE_UPPER,2 ) << 2 );/* set compute engine doorbell range */ 0 , regCP_MEC_DOORBELL_RANGE_LOWER,2 ) << 2 );0 , regCP_MEC_DOORBELL_RANGE_UPPER,2 ) << 2 );static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev,struct v11_gfx_mqd *mqd,struct amdgpu_mqd_prop *prop)bool priority = 0 ;/* set up default queue priority level * 0 x0 = low priority , 0 x1 = high priority if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH)1 ;static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,struct amdgpu_mqd_prop *prop)struct v11_gfx_mqd *mqd = m;/* set up gfx hqd wptr */ 0 ;0 ;/* set the pointer to the MQD */ 0 xfffffffc;/* set up mqd control */ 0 );1 );0 );/* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */ 0 );0 ;/* set up gfx queue priority */ /* set up time quantum */ 1 );/* set up gfx hqd base. this is similar as CP_RB_BASE */ 8 ;/* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */ 0 xfffffffc;0 xffff;/* set up rb_wptr_poll addr */ 0 xfffffffc;0 xffff;/* set up the gfx_hqd_control, similar as CP_RB0_CNTL */ 4 ) - 1 ;2 );#ifdef __BIG_ENDIAN1 );#endif if (prop->tmz_queue)1 );if (!prop->kernel_queue)1 );/* set up cp_doorbell_control */ if (prop->use_doorbell) {1 );else 0 );/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ /* active the queue */ 1 ;/* set gfx UQ items */ return 0 ;static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset)struct amdgpu_device *adev = ring->adev;struct v11_gfx_mqd *mqd = ring->mqd_ptr;int mqd_idx = ring - &adev->gfx.gfx_ring[0 ];if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {void *)mqd, 0 , sizeof (*mqd));0 );0 , 0 , 0 , 0 );if (adev->gfx.me.mqd_backup[mqd_idx])sizeof (*mqd));else {/* restore mqd with the backup copy */ if (adev->gfx.me.mqd_backup[mqd_idx])sizeof (*mqd));/* reset the ring */ 0 ;0 ;return 0 ;static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)int r, i;for (i = 0 ; i < adev->gfx.num_gfx_rings; i++) {false );if (r)return r;0 );if (r)return r;return gfx_v11_0_cp_gfx_start(adev);static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,struct amdgpu_mqd_prop *prop)struct v11_compute_mqd *mqd = m;0 xC0310800;0 x00000001;0 xffffffff;0 xffffffff;0 xffffffff;0 xffffffff;0 x00000007;8 ;/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4 ) - 1 ));/* enable doorbell? */ if (prop->use_doorbell) {1 );0 );0 );else {0 );/* disable the queue if it's active */ 0 ;0 ;0 ;0 ;/* set the pointer to the MQD */ 0 xfffffffc;/* set MQD vmid to 0 */ 0 );/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 8 ;/* set up the HQD, this is similar to CP_RB0_CNTL */ 4 ) - 1 ));4 ) - 1 ));1 );if (prop->kernel_queue) {1 );1 );if (prop->tmz_queue)1 );/* set the wb address whether it's enabled or not */ 0 xfffffffc;0 xffff;/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 0 xfffffffc;0 xffff;0 ;/* enable the doorbell if requested */ if (prop->use_doorbell) {1 );0 );0 );/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ /* set the vmid for the queue */ 0 ;0 x55);/* set MIN_IB_AVAIL_SIZE */ 3 );/* set static priority for a compute queue/ring */ /* set UQ fenceaddress */ return 0 ;static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;struct v11_compute_mqd *mqd = ring->mqd_ptr;int j;/* inactivate the queue */ if (amdgpu_sriov_vf(adev))0 , regCP_HQD_ACTIVE, 0 );/* disable wptr polling */ 0 , CP_PQ_WPTR_POLL_CNTL, EN, 0 );/* write the EOP addr */ 0 , regCP_HQD_EOP_BASE_ADDR,0 , regCP_HQD_EOP_BASE_ADDR_HI,/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 0 , regCP_HQD_EOP_CONTROL,/* enable doorbell? */ 0 , regCP_HQD_PQ_DOORBELL_CONTROL,/* disable the queue if it's active */ if (RREG32_SOC15(GC, 0 , regCP_HQD_ACTIVE) & 1 ) {0 , regCP_HQD_DEQUEUE_REQUEST, 1 );for (j = 0 ; j < adev->usec_timeout; j++) {if (!(RREG32_SOC15(GC, 0 , regCP_HQD_ACTIVE) & 1 ))break ;1 );0 , regCP_HQD_DEQUEUE_REQUEST,0 , regCP_HQD_PQ_RPTR,0 , regCP_HQD_PQ_WPTR_LO,0 , regCP_HQD_PQ_WPTR_HI,/* set the pointer to the MQD */ 0 , regCP_MQD_BASE_ADDR,0 , regCP_MQD_BASE_ADDR_HI,/* set MQD vmid to 0 */ 0 , regCP_MQD_CONTROL,/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 0 , regCP_HQD_PQ_BASE,0 , regCP_HQD_PQ_BASE_HI,/* set up the HQD, this is similar to CP_RB0_CNTL */ 0 , regCP_HQD_PQ_CONTROL,/* set the wb address whether it's enabled or not */ 0 , regCP_HQD_PQ_RPTR_REPORT_ADDR,0 , regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 0 , regCP_HQD_PQ_WPTR_POLL_ADDR,0 , regCP_HQD_PQ_WPTR_POLL_ADDR_HI,/* enable the doorbell if requested */ if (ring->use_doorbell) {0 , regCP_MEC_DOORBELL_RANGE_LOWER,2 ) << 2 );0 , regCP_MEC_DOORBELL_RANGE_UPPER,2 ) << 2 );0 , regCP_HQD_PQ_DOORBELL_CONTROL,/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 0 , regCP_HQD_PQ_WPTR_LO,0 , regCP_HQD_PQ_WPTR_HI,/* set the vmid for the queue */ 0 , regCP_HQD_VMID, mqd->cp_hqd_vmid);0 , regCP_HQD_PERSISTENT_STATE,/* activate the queue */ 0 , regCP_HQD_ACTIVE,if (ring->use_doorbell)0 , CP_PQ_STATUS, DOORBELL_ENABLE, 1 );return 0 ;static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;struct v11_compute_mqd *mqd = ring->mqd_ptr;if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ /* reset MQD to a clean status */ if (adev->gfx.kiq[0 ].mqd_backup)0 ].mqd_backup, sizeof (*mqd));/* reset ring buffer */ 0 ;0 );0 , 0 , 0 , 0 );else {void *)mqd, 0 , sizeof (*mqd));if (amdgpu_sriov_vf(adev) && adev->in_suspend)0 );0 , 0 , 0 , 0 );if (adev->gfx.kiq[0 ].mqd_backup)0 ].mqd_backup, mqd, sizeof (*mqd));return 0 ;static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset)struct amdgpu_device *adev = ring->adev;struct v11_compute_mqd *mqd = ring->mqd_ptr;int mqd_idx = ring - &adev->gfx.compute_ring[0 ];if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {void *)mqd, 0 , sizeof (*mqd));0 );0 , 0 , 0 , 0 );if (adev->gfx.mec.mqd_backup[mqd_idx])sizeof (*mqd));else {/* restore MQD to a clean status */ if (adev->gfx.mec.mqd_backup[mqd_idx])sizeof (*mqd));/* reset ring buffer */ 0 ;0 );return 0 ;static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)0 ].ring);return 0 ;static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)int i, r;if (!amdgpu_async_gfx_ring)true );for (i = 0 ; i < adev->gfx.num_compute_rings; i++) {false );if (r)return r;return amdgpu_gfx_enable_kcq(adev, 0 );static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)int r, i;struct amdgpu_ring *ring;if (!(adev->flags & AMD_IS_APU))false );if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {/* legacy firmware loading */ if (r)return r;if (adev->gfx.rs64_enable)else if (r)return r;if (amdgpu_async_gfx_ring) {true );true );if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)else if (r)return r;if (r)return r;if (!amdgpu_async_gfx_ring) {if (r)return r;else {if (r)return r;if (adev->gfx.disable_kq) {for (i = 0 ; i < adev->gfx.num_gfx_rings; i++) {/* we don't want to set ring->ready */ if (r)return r;if (amdgpu_async_gfx_ring)0 );else {for (i = 0 ; i < adev->gfx.num_gfx_rings; i++) {if (r)return r;for (i = 0 ; i < adev->gfx.num_compute_rings; i++) {if (r)return r;return 0 ;static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)int r;bool value;if (r)return r;false : true ;/* TODO investigate why this and the hdp flush above is needed, 0 , AMDGPU_GFXHUB(0 ), 0 );return 0 ;static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)/* select RS64 */ if (adev->gfx.rs64_enable) {0 , regCP_GFX_CNTL);1 );0 , regCP_GFX_CNTL, tmp);0 , regCP_MEC_ISA_CNTL);1 );0 , regCP_MEC_ISA_CNTL, tmp);if (amdgpu_emu_mode == 1 )100 );static int get_gb_addr_config(struct amdgpu_device * adev)0 , regGB_ADDR_CONFIG);if (gb_addr_config == 0 )return -EINVAL;1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);1 <<1 <<1 <<1 <<1 << (8 +return 0 ;static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)0 , regCPC_PSP_DEBUG);0 , regCPC_PSP_DEBUG, data);0 , regCPG_PSP_DEBUG);0 , regCPG_PSP_DEBUG, data);static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block)int r;struct amdgpu_device *adev = ip_block->adev;if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {if (adev->gfx.imu.funcs) {/* RLC autoload sequence 1: Program rlc ram */ if (adev->gfx.imu.funcs->program_rlc_ram)/* rlc autoload firmware */ if (r)return r;else {if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {if (adev->gfx.imu.funcs && (amdgpu_dpm > 0 )) {if (adev->gfx.imu.funcs->load_microcode)if (adev->gfx.imu.funcs->setup_imu)if (adev->gfx.imu.funcs->start_imu)/* disable gpa mode in backdoor loading */ if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||if (r) {"(%d) failed to wait rlc autoload complete\n" , r);return r;true ;if (get_gb_addr_config(adev))"Invalid gb_addr_config !\n" );if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&if (r)return r;if (!amdgpu_emu_mode)if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||1 )) {/** * For gfx 11 , rlc firmware loading relies on smu firmware is * loaded firstly , so in direct type , it has to load smc ucode * here before rlc . if (r)return r;if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)if (adev->nbio.funcs->gc_doorbell_init)if (r)return r;/* * init golden registers and rlc resume may override some registers , * reconfig them here if (r)return r;/* get IMU version from HW if it's not set */ if (!adev->gfx.imu_fw_version)0 , regGFX_IMU_SCRATCH_0);return r;static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev,bool enable)unsigned int irq_type;int m, p, r;if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) {for (m = 0 ; m < adev->gfx.me.num_me; m++) {for (p = 0 ; p < adev->gfx.me.num_pipe_per_me; p++) {if (enable)else if (r)return r;if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) {for (m = 0 ; m < adev->gfx.mec.num_mec; ++m) {for (p = 0 ; p < adev->gfx.mec.num_pipe_per_mec; p++) {if (enable)else if (r)return r;return 0 ;static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)struct amdgpu_device *adev = ip_block->adev;0 );0 );0 );false );if (!adev->no_hw_access) {if (amdgpu_async_gfx_ring &&if (amdgpu_gfx_disable_kgq(adev, 0 ))"KGQ disable failed\n" );if (amdgpu_gfx_disable_kcq(adev, 0 ))"KCQ disable failed\n" );if (amdgpu_sriov_vf(adev))/* Remove the steps disabling CPG and clearing KIQ position, * so that CP could perform IDLE - SAVE during switch . Those * steps are necessary to avoid a DMAR error in gfx9 but it is * not reproduced on gfx11 . return 0 ;false );false );false ;return 0 ;static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block)return gfx_v11_0_hw_fini(ip_block);static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block)return gfx_v11_0_hw_init(ip_block);static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block)struct amdgpu_device *adev = ip_block->adev;if (REG_GET_FIELD(RREG32_SOC15(GC, 0 , regGRBM_STATUS),return false ;else return true ;static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)unsigned i;struct amdgpu_device *adev = ip_block->adev;for (i = 0 ; i < adev->usec_timeout; i++) {/* read MC_STATUS */ 0 , regGRBM_STATUS) &if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))return 0 ;1 );return -ETIMEDOUT;int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev,bool req)for (i = 0 ; i < adev->usec_timeout; i++) {/* Request with MeId=2, PipeId=0 */ 0 , CP_GFX_INDEX_MUTEX, REQUEST, req);4 );0 , regCP_GFX_INDEX_MUTEX, tmp);0 , regCP_GFX_INDEX_MUTEX);if (req) {if (val == tmp)break ;else {1 );/* unlocked or locked by firmware */ if (val != tmp)break ;1 );if (i >= adev->usec_timeout)return -EINVAL;return 0 ;static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block)0 ;int r, i, j, k;struct amdgpu_device *adev = ip_block->adev;0 );0 , regCP_INT_CNTL);0 );0 );0 );0 );0 , regCP_INT_CNTL, tmp);for (i = 0 ; i < adev->gfx.mec.num_mec; ++i) {for (j = 0 ; j < adev->gfx.mec.num_queue_per_pipe; j++) {for (k = 0 ; k < adev->gfx.mec.num_pipe_per_mec; k++) {0 );0 , regCP_HQD_DEQUEUE_REQUEST, 0 x2);0 , regSPI_COMPUTE_QUEUE_RESET, 0 x1);for (i = 0 ; i < adev->gfx.me.num_me; ++i) {for (j = 0 ; j < adev->gfx.me.num_queue_per_pipe; j++) {for (k = 0 ; k < adev->gfx.me.num_pipe_per_me; k++) {0 );0 , regCP_GFX_HQD_DEQUEUE_REQUEST, 0 x1);0 , 0 , 0 , 0 );/* Try to acquire the gfx mutex before access to CP_VMID_RESET */ true );if (r) {"Failed to acquire the gfx mutex during soft reset\n" );return r;0 , regCP_VMID_RESET, 0 xfffffffe);// Read CP_VMID_RESET register three times. // to get sufficient time for GFX_HQD_ACTIVE reach 0 0 , regCP_VMID_RESET);0 , regCP_VMID_RESET);0 , regCP_VMID_RESET);/* release the gfx mutex */ false );if (r) {"Failed to release the gfx mutex during soft reset\n" );return r;for (i = 0 ; i < adev->usec_timeout; i++) {if (!RREG32_SOC15(GC, 0 , regCP_HQD_ACTIVE) &&0 , regCP_GFX_HQD_ACTIVE))break ;1 );if (i >= adev->usec_timeout) {"Failed to wait all pipes clean\n" );return -EINVAL;/********** trigger soft reset ***********/ 0 , regGRBM_SOFT_RESET);1 );1 );1 );1 );1 );0 , regGRBM_SOFT_RESET, grbm_soft_reset);/********** exit soft reset ***********/ 0 , regGRBM_SOFT_RESET);0 );0 );0 );0 );0 );0 , regGRBM_SOFT_RESET, grbm_soft_reset);0 , regCP_SOFT_RESET_CNTL);0 x1);0 , regCP_SOFT_RESET_CNTL, tmp);0 , regCP_ME_CNTL, 0 x0);0 , regCP_MEC_RS64_CNTL, 0 x0);for (i = 0 ; i < adev->usec_timeout; i++) {if (!RREG32_SOC15(GC, 0 , regCP_VMID_RESET))break ;1 );if (i >= adev->usec_timeout) {"Failed to wait CP_VMID_RESET to 0\n" );return -EINVAL;0 , regCP_INT_CNTL);1 );1 );1 );1 );0 , regCP_INT_CNTL, tmp);0 );return gfx_v11_0_cp_resume(adev);static bool gfx_v11_0_check_soft_reset(struct amdgpu_ip_block *ip_block)int i, r;struct amdgpu_device *adev = ip_block->adev;struct amdgpu_ring *ring;long tmo = msecs_to_jiffies(1000 );for (i = 0 ; i < adev->gfx.num_gfx_rings; i++) {if (r)return true ;for (i = 0 ; i < adev->gfx.num_compute_rings; i++) {if (r)return true ;return false ;static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block)struct amdgpu_device *adev = ip_block->adev;/** * GFX soft reset will impact MES , need resume MES when do GFX soft reset return amdgpu_mes_resume(adev);static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)if (amdgpu_sriov_vf(adev)) {false );0 , regCP_MES_MTIME_HI);0 , regCP_MES_MTIME_LO);0 , regCP_MES_MTIME_HI);if (clock_counter_hi_pre != clock_counter_hi_after)0 , regCP_MES_MTIME_LO);true );else {0 , regGOLDEN_TSC_COUNT_UPPER);0 , regGOLDEN_TSC_COUNT_LOWER);0 , regGOLDEN_TSC_COUNT_UPPER);if (clock_counter_hi_pre != clock_counter_hi_after)0 , regGOLDEN_TSC_COUNT_LOWER);32 ULL);return clock;static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,struct amdgpu_device *adev = ring->adev;/* GDS Base */ 0 , false ,0 , regGDS_VMID0_BASE) + 2 * vmid,/* GDS Size */ 0 , false ,0 , regGDS_VMID0_SIZE) + 2 * vmid,/* GWS */ 0 , false ,0 , regGDS_GWS_VMID0) + vmid,/* OA */ 0 , false ,0 , regGDS_OA_VMID0) + vmid,1 << (oa_size + oa_base)) - (1 << oa_base));static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)struct amdgpu_device *adev = ip_block->adev;switch (amdgpu_user_queue) {case -1 :case 0 :default :false ;true ;break ;case 1 :false ;false ;break ;case 2 :true ;false ;break ;if (adev->gfx.disable_kq) {/* We need one GFX ring temporarily to set up * the clear state . 1 ;0 ;else {return gfx_v11_0_init_microcode(adev);static int gfx_v11_0_late_init(struct amdgpu_ip_block *ip_block)struct amdgpu_device *adev = ip_block->adev;int r;0 );if (r)return r;0 );if (r)return r;0 );if (r)return r;true );if (r)return r;return 0 ;static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)/* if RLC is not enabled, do nothing */ 0 , regRLC_CNTL);return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false ;static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)unsigned i;1 << RLC_SAFE_MODE__MESSAGE__SHIFT);0 , regRLC_SAFE_MODE, data);/* wait for RLC_SAFE_MODE */ for (i = 0 ; i < adev->usec_timeout; i++) {if (!REG_GET_FIELD(RREG32_SOC15(GC, 0 , regRLC_SAFE_MODE),break ;1 );static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)0 , regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,bool enable)if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))return ;0 , regRLC_CGTT_MGCG_OVERRIDE);if (enable)else if (def != data)0 , regRLC_CGTT_MGCG_OVERRIDE, data);static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,bool enable)if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))return ;0 , regRLC_CGTT_MGCG_OVERRIDE);if (enable)else if (def != data)0 , regRLC_CGTT_MGCG_OVERRIDE, data);static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,bool enable)if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))return ;0 , regRLC_CGTT_MGCG_OVERRIDE);if (enable)else if (def != data)0 , regRLC_CGTT_MGCG_OVERRIDE, data);static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,bool enable)if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))return ;/* It is disabled by HW by default */ if (enable) {if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {/* 1 - RLC_CGTT_MGCG_OVERRIDE */ 0 , regRLC_CGTT_MGCG_OVERRIDE);if (def != data)0 , regRLC_CGTT_MGCG_OVERRIDE, data);else {if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {0 , regRLC_CGTT_MGCG_OVERRIDE);if (def != data)0 , regRLC_CGTT_MGCG_OVERRIDE, data);static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,bool enable)if (!(adev->cg_flags &return ;if (enable) {0 , regRLC_CGTT_MGCG_OVERRIDE);/* unset CGCG override */ if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||/* update CGCG override bits */ if (def != data)0 , regRLC_CGTT_MGCG_OVERRIDE, data);/* enable cgcg FSM(0x0000363F) */ 0 , regRLC_CGCG_CGLS_CTRL);if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {0 x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {0 x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |if (def != data)0 , regRLC_CGCG_CGLS_CTRL, data);/* Program RLC_CGCG_CGLS_CTRL_3D */ 0 , regRLC_CGCG_CGLS_CTRL_3D);if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {0 x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {0 xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |if (def != data)0 , regRLC_CGCG_CGLS_CTRL_3D, data);/* set IDLE_POLL_COUNT(0x00900100) */ 0 , regCP_RB_WPTR_POLL_CNTL);POLL_COUNT_MASK);0 x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |0 x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);if (def != data)0 , regCP_RB_WPTR_POLL_CNTL, data);0 , regCP_INT_CNTL);1 );1 );1 );1 );0 , regCP_INT_CNTL, data);0 , regSDMA0_RLC_CGCG_CTRL);1 );0 , regSDMA0_RLC_CGCG_CTRL, data);/* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ if (adev->sdma.num_instances > 1 ) {0 , regSDMA1_RLC_CGCG_CTRL);1 );0 , regSDMA1_RLC_CGCG_CTRL, data);else {/* Program RLC_CGCG_CGLS_CTRL */ 0 , regRLC_CGCG_CGLS_CTRL);if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)if (def != data)0 , regRLC_CGCG_CGLS_CTRL, data);/* Program RLC_CGCG_CGLS_CTRL_3D */ 0 , regRLC_CGCG_CGLS_CTRL_3D);if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)if (def != data)0 , regRLC_CGCG_CGLS_CTRL_3D, data);0 , regSDMA0_RLC_CGCG_CTRL);0 , regSDMA0_RLC_CGCG_CTRL, data);/* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ if (adev->sdma.num_instances > 1 ) {0 , regSDMA1_RLC_CGCG_CTRL);0 , regSDMA1_RLC_CGCG_CTRL, data);static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,bool enable)0 );if (adev->cg_flags &0 );return 0 ;static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)false );0 , regRLC_SPM_MC_CNTL);if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))else __SHIFT;if (pre_data != data) {if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {0 , regRLC_SPM_MC_CNTL, data);else 0 , regRLC_SPM_MC_CNTL, data);true );if (ringstatic const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)0 , regRLC_PG_CNTL);if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))else 0 , regRLC_PG_CNTL, data);// Program RLC_PG_DELAY3 for CGPG hysteresis if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 4 ):case IP_VERSION(11 , 5 , 0 ):case IP_VERSION(11 , 5 , 1 ):case IP_VERSION(11 , 5 , 2 ):case IP_VERSION(11 , 5 , 3 ):0 , regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);break ;default :break ;static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)0 );0 );static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block,enum amd_powergating_state state)struct amdgpu_device *adev = ip_block->adev;bool enable = (state == AMD_PG_STATE_GATE);if (amdgpu_sriov_vf(adev))return 0 ;switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 2 ):case IP_VERSION(11 , 0 , 3 ):break ;case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 4 ):case IP_VERSION(11 , 5 , 0 ):case IP_VERSION(11 , 5 , 1 ):case IP_VERSION(11 , 5 , 2 ):case IP_VERSION(11 , 5 , 3 ):if (!enable)false );if (enable)true );break ;default :break ;return 0 ;static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,enum amd_clockgating_state state)struct amdgpu_device *adev = ip_block->adev;if (amdgpu_sriov_vf(adev))return 0 ;switch (amdgpu_ip_version(adev, GC_HWIP, 0 )) {case IP_VERSION(11 , 0 , 0 ):case IP_VERSION(11 , 0 , 1 ):case IP_VERSION(11 , 0 , 2 ):case IP_VERSION(11 , 0 , 3 ):case IP_VERSION(11 , 0 , 4 ):case IP_VERSION(11 , 5 , 0 ):case IP_VERSION(11 , 5 , 1 ):case IP_VERSION(11 , 5 , 2 ):case IP_VERSION(11 , 5 , 3 ):break ;default :break ;return 0 ;static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)struct amdgpu_device *adev = ip_block->adev;int data;/* AMD_CG_SUPPORT_GFX_MGCG */ 0 , regRLC_CGTT_MGCG_OVERRIDE);if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))/* AMD_CG_SUPPORT_REPEATER_FGCG */ if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))/* AMD_CG_SUPPORT_GFX_FGCG */ if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))/* AMD_CG_SUPPORT_GFX_PERF_CLK */ if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))/* AMD_CG_SUPPORT_GFX_CGCG */ 0 , regRLC_CGCG_CGLS_CTRL);if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)/* AMD_CG_SUPPORT_GFX_CGLS */ if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)/* AMD_CG_SUPPORT_GFX_3D_CGCG */ 0 , regRLC_CGCG_CGLS_CTRL_3D);if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)/* AMD_CG_SUPPORT_GFX_3D_CGLS */ if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)/* gfx11 is 32bit rptr*/ return *(uint32_t *)ring->rptr_cpu_addr;static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;/* XXX check if swapping is necessary on BE */ if (ring->use_doorbell) {else {0 , regCP_RB0_WPTR);0 , regCP_RB0_WPTR_HI) << 32 ;return wptr;static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;if (ring->use_doorbell) {/* XXX check if swapping is necessary on BE */ else {0 , regCP_RB0_WPTR,0 , regCP_RB0_WPTR_HI,static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)/* gfx11 hardware is 32bit rptr */ return *(uint32_t *)ring->rptr_cpu_addr;static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)/* XXX check if swapping is necessary on BE */ if (ring->use_doorbell)else return wptr;static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;/* XXX check if swapping is necessary on BE */ if (ring->use_doorbell) {else {/* only DOORBELL method supported on gfx11 now */ static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {switch (ring->me) {case 1 :break ;case 2 :break ;default :return ;0 ;else {1 ; /* pfp */ 0 , 1 ,0 x20);static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,struct amdgpu_job *job,struct amdgpu_ib *ib,unsigned vmid = AMDGPU_JOB_GET_VMID(job);0 ;2 );24 );if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {1 );if (flags & AMDGPU_IB_PREEMPTED)1 );if (vmid)true : false );0 x3); /* Dword align */ #ifdef __BIG_ENDIAN2 << 0 ) |#endif static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,struct amdgpu_job *job,struct amdgpu_ib *ib,unsigned vmid = AMDGPU_JOB_GET_VMID(job);24 );/* Currently, there is a high possibility to get wave ID mismatch * between ME and GDS , leading to a hw deadlock , because ME generates * different wave IDs than the GDS expects . This situation happens * randomly when at least 5 compute pipes use GDS ordered append . * The wave IDs generated by ME are also wrong after suspend / resume . * Those are probably bugs somewhere else in the kernel driver . * * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and * GDS to 0 for this ring ( me / pipe ) . if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {1 ));2 ));0 x3); /* Dword align */ #ifdef __BIG_ENDIAN2 << 0 ) |#endif static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,unsigned flags)bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;/* RELEASE_MEM - flush caches, send int */ 6 ));/* must be set with GLM_WB */ 3 ) |5 )));2 : 1 ) |2 : 0 )));/* * the address should be Qword aligned if 64 bit write , Dword * aligned if only send 32 bit data low ( discard data high ) if (write64bit)0 x7);else 0 x3);0 );static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);1 , 0 , lower_32_bits(addr),0 xffffffff, 4 );static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,bool all_hub, uint8_t dst_sel)0 ));static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,unsigned vmid, uint64_t pd_addr)/* compute doesn't have PFP */ if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {/* sync PFP to ME, otherwise we might get invalid PFP reads */ 0 ));0 x0);/* Make sure that we can't skip the SET_Q_MODE packets when the VM * changed in any way . 0 ;static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,unsigned int flags)struct amdgpu_device *adev = ring->adev;/* we only allocate 32bit for each seq wb address */ /* write fence seq to the "addr" */ 3 ));0 ) |5 ) | WR_CONFIRM));if (flags & AMDGPU_FENCE_FLAG_INT) {/* set register to trigger INT */ 3 ));0 ) |0 ) | WR_CONFIRM));0 , regCPC_INT_STATUS));0 );0 x20000000); /* src_id is 178 */ static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,0 ;0 x80000000; /* set load_enable otherwise this package is just NOPs */ if (flags & AMDGPU_HAVE_CTX_SWITCH) {/* set load_global_config & load_global_uconfig */ 0 x8001;/* set load_cs_sh_regs */ 0 x01000000;/* set load_per_context_state & load_gfx_sh_regs for GFX */ 0 x10002;1 ));0 );static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,unsigned ret;3 ));/* discard following DWs if *cond_exec_gpu_addr==0 */ 0 );/* patch dummy value later */ 0 );return ret;static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,bool init_shadow,int vmid)struct amdgpu_device *adev = ring->adev;unsigned int offs, end;if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj)return ;/* * The logic here isn ' t easy to understand because we need to keep state * accross multiple executions of the function as well as between the * CPU and GPU . The general idea is that the newly written GPU command * has a condition on the previous one and only executed if really * necessary . /* * The dw in the NOP controls if the next SET_Q_MODE packet should be * executed or not . Reserve 64 bits just to be on the save side . 1 ));/* * We start with skipping the prefix SET_Q_MODE and always executing * the postfix SET_Q_MODE packet . This is changed below with a * WRITE_DATA command when the postfix executed . 1 : 0 );0 );if (ring->set_q_mode_offs) {2 ;/* * When the postfix SET_Q_MODE packet executes we need to make sure that the * next prefix SET_Q_MODE packet executes as well . if (!shadow_va) {2 ;3 ));5 ) | WR_CONFIRM);0 x1);7 ));0 );0 );if (ring->set_q_mode_offs)if (shadow_va) {/* * If the tokens match try to skip the last postfix SET_Q_MODE * packet to avoid saving / restoring the state all the time . if (ring->set_q_mode_ptr && ring->set_q_mode_token == token)0 ;else {static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)int i, r = 0 ;struct amdgpu_device *adev = ring->adev;struct amdgpu_kiq *kiq = &adev->gfx.kiq[0 ];struct amdgpu_ring *kiq_ring = &kiq->ring;unsigned long flags;if (adev->enable_mes)return -EINVAL;if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)return -EINVAL;if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {return -ENOMEM;/* assert preemption condition */ false );/* assert IB preemption, emit the trailing fence */ /* poll the trailing fence */ for (i = 0 ; i < adev->usec_timeout; i++) {if (ring->trail_seq ==break ;1 );if (i >= adev->usec_timeout) {"ring %d failed to preempt ib\n" , ring->idx);/* deassert preemption condition */ true );return r;static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)struct amdgpu_device *adev = ring->adev;struct v10_de_ib_state de_payload = {0 };void *de_payload_cpu_addr;int cnt;struct v10_gfx_meta_data, de_payload);sizeof (de_payload) >> 2 ) + 4 - 2 ;1 ) |8 ) |0 ));if (resume)sizeof (de_payload) >> 2 );else void *)&de_payload,sizeof (de_payload) >> 2 );static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,bool secure)0 ;0 ));0 : 1 ));static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,struct amdgpu_device *adev = ring->adev;4 ));0 | /* src: register*/ 5 << 8 ) | /* dst: memory */ 1 << 20 )); /* write confirm */ 0 );4 ));4 ));static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,0 ;switch (ring->funcs->type) {case AMDGPU_RING_TYPE_GFX:1 ) | WR_CONFIRM;break ;case AMDGPU_RING_TYPE_KIQ:1 << 16 ); /* no inc addr */ break ;default :break ;3 ));0 );static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,0 , 0 , 0 , reg, 0 , val, mask, 0 x20);static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);0 , 1 , reg0, reg1,0 x20);static void struct amdgpu_device *adev,enum amdgpu_interrupt_state state)if (!me) {switch (pipe) {case 0 :0 , regCP_INT_CNTL_RING0);break ;case 1 :0 , regCP_INT_CNTL_RING1);break ;default :"invalid pipe %d\n" , pipe);return ;else {"invalid me %d\n" , me);return ;switch (state) {case AMDGPU_IRQ_STATE_DISABLE:0 );0 );break ;case AMDGPU_IRQ_STATE_ENABLE:1 );1 );break ;default :break ;static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,int me, int pipe,enum amdgpu_interrupt_state state)/* * amdgpu controls only the first MEC . That ' s why this function only * handles the setting of interrupts for this specific MEC . All other * pipes ' interrupts are set by amdkfd . if (me == 1 ) {switch (pipe) {case 0 :0 , regCP_ME1_PIPE0_INT_CNTL);break ;case 1 :0 , regCP_ME1_PIPE1_INT_CNTL);break ;case 2 :0 , regCP_ME1_PIPE2_INT_CNTL);break ;case 3 :0 , regCP_ME1_PIPE3_INT_CNTL);break ;default :"invalid pipe %d\n" , pipe);return ;else {"invalid me %d\n" , me);return ;switch (state) {case AMDGPU_IRQ_STATE_DISABLE:0 );0 );break ;case AMDGPU_IRQ_STATE_ENABLE:1 );1 );break ;default :break ;static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,struct amdgpu_irq_src *src,unsigned type,enum amdgpu_interrupt_state state)switch (type) {case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:0 , 0 , state);break ;case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:0 , 1 , state);break ;case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:1 , 0 , state);break ;case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:1 , 1 , state);break ;case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:1 , 2 , state);break ;case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:1 , 3 , state);break ;default :break ;return 0 ;static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,struct amdgpu_irq_src *source,struct amdgpu_iv_entry *entry)0 ];struct amdgpu_ring *ring;int i;"IH: CP EOP\n" );if (adev->enable_mes && doorbell_offset) {struct amdgpu_userq_fence_driver *fence_drv = NULL;struct xarray *xa = &adev->userq_xa;unsigned long flags;if (fence_drv)else {0 x0c) >> 2 ;0 x03) >> 0 ;0 x70) >> 4 ;switch (me_id) {case 0 :if (pipe_id == 0 )0 ]);else 1 ]);break ;case 1 :case 2 :for (i = 0 ; i < adev->gfx.num_compute_rings; i++) {/* Per-queue interrupt is supported for MEC starting from VI. * The interrupt can only be enabled / disabled per pipe instead * of per queue . if ((ring->me == me_id) &&break ;return 0 ;static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,struct amdgpu_irq_src *source,unsigned int type,enum amdgpu_interrupt_state state)int i, j;switch (state) {case AMDGPU_IRQ_STATE_DISABLE:case AMDGPU_IRQ_STATE_ENABLE:for (i = 0 ; i < adev->gfx.me.num_me; i++) {for (j = 0 ; j < adev->gfx.me.num_pipe_per_me; j++) {if (cp_int_cntl_reg) {1 : 0 );for (i = 0 ; i < adev->gfx.mec.num_mec; i++) {for (j = 0 ; j < adev->gfx.mec.num_pipe_per_mec; j++) {/* MECs start at 1 */ 1 , j);if (cp_int_cntl_reg) {1 : 0 );break ;default :break ;return 0 ;static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev,struct amdgpu_irq_src *source,unsigned type,enum amdgpu_interrupt_state state)int i, j;switch (state) {case AMDGPU_IRQ_STATE_DISABLE:case AMDGPU_IRQ_STATE_ENABLE:for (i = 0 ; i < adev->gfx.me.num_me; i++) {for (j = 0 ; j < adev->gfx.me.num_pipe_per_me; j++) {if (cp_int_cntl_reg) {1 : 0 );for (i = 0 ; i < adev->gfx.mec.num_mec; i++) {for (j = 0 ; j < adev->gfx.mec.num_pipe_per_mec; j++) {/* MECs start at 1 */ 1 , j);if (cp_int_cntl_reg) {1 : 0 );break ;default :break ;return 0 ;static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,struct amdgpu_irq_src *source,unsigned int type,enum amdgpu_interrupt_state state)int i, j;switch (state) {case AMDGPU_IRQ_STATE_DISABLE:case AMDGPU_IRQ_STATE_ENABLE:for (i = 0 ; i < adev->gfx.me.num_me; i++) {for (j = 0 ; j < adev->gfx.me.num_pipe_per_me; j++) {if (cp_int_cntl_reg) {1 : 0 );break ;default :break ;return 0 ;static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,struct amdgpu_iv_entry *entry)struct amdgpu_ring *ring;int i;0 x0c) >> 2 ;0 x03) >> 0 ;0 x70) >> 4 ;if (!adev->gfx.disable_kq) {switch (me_id) {case 0 :for (i = 0 ; i < adev->gfx.num_gfx_rings; i++) {if (ring->me == me_id && ring->pipe == pipe_id &&break ;case 1 :case 2 :for (i = 0 ; i < adev->gfx.num_compute_rings; i++) {if (ring->me == me_id && ring->pipe == pipe_id &&break ;default :break ;static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,struct amdgpu_irq_src *source,struct amdgpu_iv_entry *entry)"Illegal register access in command stream\n" );return 0 ;static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev,struct amdgpu_irq_src *source,struct amdgpu_iv_entry *entry)"Illegal opcode in command stream \n" );return 0 ;static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,struct amdgpu_irq_src *source,struct amdgpu_iv_entry *entry)"Illegal instruction in command stream\n" );return 0 ;static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,struct amdgpu_irq_src *source,struct amdgpu_iv_entry *entry)if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);return 0 ;#if 0 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,struct amdgpu_irq_src *src,unsigned int type,enum amdgpu_interrupt_state state)struct amdgpu_ring *ring = &(adev->gfx.kiq[0 ].ring);0 , regCP_ME1_PIPE0_INT_CNTL);switch (type) {case AMDGPU_CP_KIQ_IRQ_DRIVER0:if (state == AMDGPU_IRQ_STATE_DISABLE) {0 , regCPC_INT_CNTL);0 );0 , regCPC_INT_CNTL, tmp);0 );else {0 , regCPC_INT_CNTL);1 );0 , regCPC_INT_CNTL, tmp);1 );break ;default :/* kiq only support GENERIC2_INT now */ break ;return 0 ;#endif static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)const unsigned int gcr_cntl =1 ) |1 ) |1 ) |1 ) |1 ) |1 ) |1 ) |1 );/* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ 6 ));0 ); /* CP_COHER_CNTL */ 0 xffffffff); /* CP_COHER_SIZE */ 0 xffffff); /* CP_COHER_SIZE_HI */ 0 ); /* CP_COHER_BASE */ 0 ); /* CP_COHER_BASE_HI */ 0 x0000000A); /* POLL_INTERVAL */ /* GCR_CNTL */ static bool gfx_v11_pipe_reset_support(struct amdgpu_device *adev)/* Disable the pipe reset until the CPFW fully support it.*/ "The CPFW hasn't support pipe reset yet.\n" );return false ;static int gfx_v11_reset_gfx_pipe(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;0 , clean_pipe = 0 ;int r;if (!gfx_v11_pipe_reset_support(adev))return -EOPNOTSUPP;0 );0 );switch (ring->pipe) {case 0 :1 );1 );0 );0 );break ;case 1 :1 );1 );0 );0 );break ;default :break ;0 , regCP_ME_CNTL, reset_pipe);0 , regCP_ME_CNTL, clean_pipe);0 , regCP_GFX_RS64_INSTR_PNTR1)) << 2 ) -0 , 0 , 0 , 0 );0 );"The ring %s pipe reset to the ME firmware start PC: %s\n" , ring->name,0 ? "successfully" : "failed" );/* FIXME: Sometimes driver can't cache the ME firmware start PC correctly, * so the pipe reset status relies on the later gfx ring test result . return 0 ;static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring,unsigned int vmid,struct amdgpu_fence *timedout_fence)struct amdgpu_device *adev = ring->adev;int r;false );if (r) {"reset via MES failed and try pipe reset %d\n" , r);if (r)return r;true );if (r) {"failed to init kgq\n" );return r;if (r) {"failed to remap kgq\n" );return r;return amdgpu_ring_reset_helper_end(ring, timedout_fence);static int gfx_v11_0_reset_compute_pipe(struct amdgpu_ring *ring)struct amdgpu_device *adev = ring->adev;0 , clean_pipe = 0 ;int r;if (!gfx_v11_pipe_reset_support(adev))return -EOPNOTSUPP;0 );0 );0 , regCP_MEC_RS64_CNTL);if (adev->gfx.rs64_enable) {switch (ring->pipe) {case 0 :1 );0 );break ;case 1 :1 );0 );break ;case 2 :1 );0 );break ;case 3 :1 );0 );break ;default :break ;0 , regCP_MEC_RS64_CNTL, reset_pipe);0 , regCP_MEC_RS64_CNTL, clean_pipe);0 , regCP_MEC_RS64_INSTR_PNTR) << 2 ) -else {if (ring->me == 1 ) {switch (ring->pipe) {case 0 :1 );0 );break ;case 1 :1 );0 );break ;case 2 :1 );0 );break ;case 3 :1 );0 );break ;default :break ;/* mec1 fw pc: CP_MEC1_INSTR_PNTR */ else {switch (ring->pipe) {case 0 :1 );0 );break ;case 1 :1 );0 );break ;case 2 :1 );0 );break ;case 3 :1 );0 );break ;default :break ;/* mec2 fw pc: CP:CP_MEC2_INSTR_PNTR */ 0 , regCP_MEC_CNTL, reset_pipe);0 , regCP_MEC_CNTL, clean_pipe);0 , regCP_MEC1_INSTR_PNTR));0 , 0 , 0 , 0 );0 );"The ring %s pipe resets to MEC FW start PC: %s\n" , ring->name,0 ? "successfully" : "failed" );/*FIXME:Sometimes driver can't cache the MEC firmware start PC correctly, so the pipe * reset status relies on the compute ring test result . return 0 ;static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,unsigned int vmid,struct amdgpu_fence *timedout_fence)struct amdgpu_device *adev = ring->adev;int r = 0 ;true );if (r) {"fail(%d) to reset kcq and try pipe reset\n" , r);if (r)return r;true );if (r) {"fail to init kcq\n" );return r;if (r) {"failed to remap kcq\n" );return r;return amdgpu_ring_reset_helper_end(ring, timedout_fence);static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)struct amdgpu_device *adev = ip_block->adev;0 ;if (!adev->gfx.ip_dump_core)return ;for (i = 0 ; i < reg_count; i++)"%-50s \t 0x%08x\n" ,/* print compute queue registers for all instances */ if (!adev->gfx.ip_dump_compute_queues)return ;"\nnum_mec: %d num_pipe: %d num_queue: %d\n" ,for (i = 0 ; i < adev->gfx.mec.num_mec; i++) {for (j = 0 ; j < adev->gfx.mec.num_pipe_per_mec; j++) {for (k = 0 ; k < adev->gfx.mec.num_queue_per_pipe; k++) {"\nmec %d, pipe %d, queue %d\n" , i, j, k);for (reg = 0 ; reg < reg_count; reg++) {if (i && gc_cp_reg_list_11[reg].reg_offset == regCP_MEC_ME1_HEADER_DUMP)"%-50s \t 0x%08x\n" ,"regCP_MEC_ME2_HEADER_DUMP" ,else "%-50s \t 0x%08x\n" ,/* print gfx queue registers for all instances */ if (!adev->gfx.ip_dump_gfx_queues)return ;0 ;"\nnum_me: %d num_pipe: %d num_queue: %d\n" ,for (i = 0 ; i < adev->gfx.me.num_me; i++) {for (j = 0 ; j < adev->gfx.me.num_pipe_per_me; j++) {for (k = 0 ; k < adev->gfx.me.num_queue_per_pipe; k++) {"\nme %d, pipe %d, queue %d\n" , i, j, k);for (reg = 0 ; reg < reg_count; reg++) {"%-50s \t 0x%08x\n" ,static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block)struct amdgpu_device *adev = ip_block->adev;0 ;if (!adev->gfx.ip_dump_core)return ;false );for (i = 0 ; i < reg_count; i++)true );/* dump compute queue registers for all instances */ if (!adev->gfx.ip_dump_compute_queues)return ;false );for (i = 0 ; i < adev->gfx.mec.num_mec; i++) {for (j = 0 ; j < adev->gfx.mec.num_pipe_per_mec; j++) {for (k = 0 ; k < adev->gfx.mec.num_queue_per_pipe; k++) {/* ME0 is for GFX so start from 1 for CP */ 0 );for (reg = 0 ; reg < reg_count; reg++) {if (i &&0 ,else 0 , 0 , 0 , 0 );true );/* dump gfx queue registers for all instances */ if (!adev->gfx.ip_dump_gfx_queues)return ;0 ;false );for (i = 0 ; i < adev->gfx.me.num_me; i++) {for (j = 0 ; j < adev->gfx.me.num_pipe_per_me; j++) {for (k = 0 ; k < adev->gfx.me.num_queue_per_pipe; k++) {0 );for (reg = 0 ; reg < reg_count; reg++) {0 , 0 , 0 , 0 );true );static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)/* Emit the cleaner shader */ 0 ));0 ); /* RESERVED field, programmed to zero */ static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring)static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring)static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {"gfx_v11_0" ,static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {0 xff,0 x3FFF),true ,true ,/* totally 247 maximum if 16 IBs */ 5 + /* update_spm_vmid */ 5 + /* COND_EXEC */ 22 + /* SET_Q_PREEMPTION_MODE */ 7 + /* PIPELINE_SYNC */ 5 +7 +4 + /* VM_FLUSH */ 8 + /* FENCE for VM_FLUSH */ 20 + /* GDS switch */ 5 + /* COND_EXEC */ 7 + /* HDP_flush */ 4 + /* VGT_flush */ 31 + /* DE_META */ 3 + /* CNTX_CTRL */ 5 + /* HDP_INVL */ 22 + /* SET_Q_PREEMPTION_MODE */ 8 + 8 + /* FENCE x2 */ 8 + /* gfx_v11_0_emit_mem_sync */ 2 , /* gfx_v11_0_ring_emit_cleaner_shader */ 4 , /* gfx_v11_0_ring_emit_ib_gfx */ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {0 xff,0 x3FFF),true ,5 + /* update_spm_vmid */ 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 5 + /* hdp invalidate */ 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 5 +7 +2 + /* gfx_v11_0_ring_emit_vm_flush */ 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */ 8 + /* gfx_v11_0_emit_mem_sync */ 2 , /* gfx_v11_0_ring_emit_cleaner_shader */ 7 , /* gfx_v11_0_ring_emit_ib_compute */ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {0 xff,0 x3FFF),true ,20 + /* gfx_v11_0_ring_emit_gds_switch */ 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 5 + /*hdp invalidate */ 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 5 +7 +8 + 8 + 8 , /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7 , /* gfx_v11_0_ring_emit_ib_compute */ static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)int i;0 ].ring.funcs = &gfx_v11_0_ring_funcs_kiq;for (i = 0 ; i < adev->gfx.num_gfx_rings; i++)for (i = 0 ; i < adev->gfx.num_compute_rings; i++)static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = {static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)1 ;1 ;1 ;1 ; /* 0x80 FED error */ static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)if (adev->flags & AMD_IS_APU)else static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)unsigned total_cu = adev->gfx.config.max_cu_per_sh *0 x1000;32 - 1 ;64 ;16 ;static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)/* set gfx eng mqd */ sizeof (struct v11_gfx_mqd);/* set compute eng mqd */ sizeof (struct v11_compute_mqd);static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,if (!bitmap)return ;0 , regGC_USER_SHADER_ARRAY_CONFIG, data);static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)0 , regCC_GC_SHADER_ARRAY_CONFIG);0 , regGC_USER_SHADER_ARRAY_CONFIG);1 );return (~data) & wgp_bitmask;static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)0 ;for (wgp_idx = 0 ; wgp_idx < 16 ; wgp_idx++) {/* if there is one WGP enabled, it means 2 CUs will be enabled */ 3 << (2 * wgp_idx);if (wgp_active_bitmap & (1 << wgp_idx))return cu_active_bitmap;static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,struct amdgpu_cu_info *cu_info)int i, j, k, counter, active_cu_number = 0 ;unsigned disable_masks[8 * 2 ];if (!adev || !cu_info)return -EINVAL;8 , 2 );for (i = 0 ; i < adev->gfx.config.max_shader_engines; i++) {for (j = 0 ; j < adev->gfx.config.max_sh_per_se; j++) {if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1 ))continue ;1 ;0 ;0 xffffffff, 0 );if (i < 8 && j < 2 )2 + j]);/** * GFX11 could support more than 4 SEs , while the bitmap * in cu_info struct is 4 x4 and ioctl interface struct * drm_amdgpu_info_device should keep stable . * So we use last two columns of bitmap to store cu mask for * SEs 4 to 7 , the layout of the bitmap is as below : * SE0 : { SH0 , SH1 } - - > { bitmap [ 0 ] [ 0 ] , bitmap [ 0 ] [ 1 ] } * SE1 : { SH0 , SH1 } - - > { bitmap [ 1 ] [ 0 ] , bitmap [ 1 ] [ 1 ] } * SE2 : { SH0 , SH1 } - - > { bitmap [ 2 ] [ 0 ] , bitmap [ 2 ] [ 1 ] } * SE3 : { SH0 , SH1 } - - > { bitmap [ 3 ] [ 0 ] , bitmap [ 3 ] [ 1 ] } * SE4 : { SH0 , SH1 } - - > { bitmap [ 0 ] [ 2 ] , bitmap [ 0 ] [ 3 ] } * SE5 : { SH0 , SH1 } - - > { bitmap [ 1 ] [ 2 ] , bitmap [ 1 ] [ 3 ] } * SE6 : { SH0 , SH1 } - - > { bitmap [ 2 ] [ 2 ] , bitmap [ 2 ] [ 3 ] } * SE7 : { SH0 , SH1 } - - > { bitmap [ 3 ] [ 2 ] , bitmap [ 3 ] [ 3 ] } 0 ][i % 4 ][j + (i / 4 ) * 2 ] = bitmap;for (k = 0 ; k < adev->gfx.config.max_cu_per_sh; k++) {if (bitmap & mask)1 ;0 xffffffff, 0 xffffffff, 0 xffffffff, 0 );return 0 ;const struct amdgpu_ip_block_version gfx_v11_0_ip_block =11 ,0 ,0 ,
Messung V0.5 in Prozent C=95 H=90 G=92
[zur Elbe Produktseite wechseln0.348QuellennavigatorsAnalyse erneut starten2026-10-02]
2026-10-10
Neuigkeiten Aktuelles Motto des Tages Open Source Software
Jenseits des Üblichen ....
Besucherstatistik
Statistik #Sources=1126864 #Domains=1897691
