// SPDX-License-Identifier: GPL-2.0-only /* * Per core / cpu state * * Used to coordinate shared registers between HT threads or * among events on a single PMU . #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt#include <linux/stddef.h>#include <linux/types.h>#include <linux/init.h>#include <linux/slab.h>#include <linux/export.h>#include <linux/nmi.h>#include <linux/kvm_host.h>#include <asm /cpufeature.h>#include <asm /debugreg.h>#include <asm /hardirq.h>#include <asm /intel-family.h>#include <asm /intel_pt.h>#include <asm /apic.h>#include <asm /cpu_device_id.h>#include <asm /msr.h>#include "../perf_event.h" /* * Intel PerfMon , used on Core and later . static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =0 x003c,0 x00c0,0 x4f2e,0 x412e,0 x00c4,0 x00c5,0 x013c,0 x0300, /* pseudo-encoding */ static struct event_constraint intel_core_event_constraints[] __read_mostly =0 x11, 0 x2), /* FP_ASSIST */ 0 x12, 0 x2), /* MUL */ 0 x13, 0 x2), /* DIV */ 0 x14, 0 x1), /* CYCLES_DIV_BUSY */ 0 x19, 0 x2), /* DELAYED_BYPASS */ 0 xc1, 0 x1), /* FP_COMP_INSTR_RET */ static struct event_constraint intel_core2_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x10, 0 x1), /* FP_COMP_OPS_EXE */ 0 x11, 0 x2), /* FP_ASSIST */ 0 x12, 0 x2), /* MUL */ 0 x13, 0 x2), /* DIV */ 0 x14, 0 x1), /* CYCLES_DIV_BUSY */ 0 x18, 0 x1), /* IDLE_DURING_DIV */ 0 x19, 0 x2), /* DELAYED_BYPASS */ 0 xa1, 0 x1), /* RS_UOPS_DISPATCH_CYCLES */ 0 xc9, 0 x1), /* ITLB_MISS_RETIRED (T30-9) */ 0 xcb, 0 x1), /* MEM_LOAD_RETIRED */ static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x40, 0 x3), /* L1D_CACHE_LD */ 0 x41, 0 x3), /* L1D_CACHE_ST */ 0 x42, 0 x3), /* L1D_CACHE_LOCK */ 0 x43, 0 x3), /* L1D_ALL_REF */ 0 x48, 0 x3), /* L1D_PEND_MISS */ 0 x4e, 0 x3), /* L1D_PREFETCH */ 0 x51, 0 x3), /* L1D */ 0 x63, 0 x3), /* CACHE_LOCK_CYCLES */ static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 xffff, RSP_0),0 x100b),static struct event_constraint intel_westmere_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x51, 0 x3), /* L1D */ 0 x60, 0 x1), /* OFFCORE_REQUESTS_OUTSTANDING */ 0 x63, 0 x3), /* CACHE_LOCK_CYCLES */ 0 xb3, 0 x1), /* SNOOPQ_REQUEST_OUTSTANDING */ static struct event_constraint intel_snb_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x04a3, 0 xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */ 0 x05a3, 0 xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */ 0 x02a3, 0 x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ 0 x06a3, 0 x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */ 0 x48, 0 x4), /* L1D_PEND_MISS.PENDING */ 0 x01c0, 0 x2), /* INST_RETIRED.PREC_DIST */ 0 xcd, 0 x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ 0 x04a3, 0 xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */ 0 x02a3, 0 x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ /* * When HT is off these events can only run on the bottom 4 counters * When HT is on , they are impacted by the HT bug and require EXCL access 0 xd0, 0 xf), /* MEM_UOPS_RETIRED.* */ 0 xd1, 0 xf), /* MEM_LOAD_UOPS_RETIRED.* */ 0 xd2, 0 xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ 0 xd3, 0 xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */ static struct event_constraint intel_ivb_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x0148, 0 x4), /* L1D_PEND_MISS.PENDING */ 0 x0279, 0 xf), /* IDQ.EMPTY */ 0 x019c, 0 xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */ 0 x02a3, 0 xf), /* CYCLE_ACTIVITY.CYCLES_LDM_PENDING */ 0 x04a3, 0 xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */ 0 x05a3, 0 xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */ 0 x06a3, 0 xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */ 0 x08a3, 0 x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ 0 x0ca3, 0 x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */ 0 x01c0, 0 x2), /* INST_RETIRED.PREC_DIST */ /* * When HT is off these events can only run on the bottom 4 counters * When HT is on , they are impacted by the HT bug and require EXCL access 0 xd0, 0 xf), /* MEM_UOPS_RETIRED.* */ 0 xd1, 0 xf), /* MEM_LOAD_UOPS_RETIRED.* */ 0 xd2, 0 xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ 0 xd3, 0 xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */ static struct extra_reg intel_westmere_extra_regs[] __read_mostly =/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 xffff, RSP_0),0 x01bb, MSR_OFFCORE_RSP_1, 0 xffff, RSP_1),0 x100b),static struct event_constraint intel_v1_event_constraints[] __read_mostly =static struct event_constraint intel_gen_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ static struct event_constraint intel_v5_gen_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x0400, 3 ), /* SLOTS */ 0 x0500, 4 ),0 x0600, 5 ),0 x0700, 6 ),0 x0800, 7 ),0 x0900, 8 ),0 x0a00, 9 ),0 x0b00, 10 ),0 x0c00, 11 ),0 x0d00, 12 ),0 x0e00, 13 ),0 x0f00, 14 ),0 x1000, 15 ),static struct event_constraint intel_slm_event_constraints[] __read_mostly =0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* pseudo CPU_CLK_UNHALTED.REF */ static struct event_constraint intel_grt_event_constraints[] __read_mostly = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* pseudo CPU_CLK_UNHALTED.REF */ 0 x013c, 2 ), /* CPU_CLK_UNHALTED.REF_TSC_P */ static struct event_constraint intel_skt_event_constraints[] __read_mostly = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* pseudo CPU_CLK_UNHALTED.REF */ 0 x013c, 2 ), /* CPU_CLK_UNHALTED.REF_TSC_P */ 0 x0073, 4 ), /* TOPDOWN_BAD_SPECULATION.ALL */ 0 x019c, 5 ), /* TOPDOWN_FE_BOUND.ALL */ 0 x02c2, 6 ), /* TOPDOWN_RETIRING.ALL */ static struct event_constraint intel_skl_event_constraints[] = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x1c0, 0 x2), /* INST_RETIRED.PREC_DIST */ /* * when HT is off , these can only run on the bottom 4 counters 0 xd0, 0 xf), /* MEM_INST_RETIRED.* */ 0 xd1, 0 xf), /* MEM_LOAD_RETIRED.* */ 0 xd2, 0 xf), /* MEM_LOAD_L3_HIT_RETIRED.* */ 0 xcd, 0 xf), /* MEM_TRANS_RETIRED.* */ 0 xc6, 0 xf), /* FRONTEND_RETIRED.* */ static struct extra_reg intel_knl_extra_regs[] __read_mostly = {0 x01b7, MSR_OFFCORE_RSP_0, 0 x799ffbb6e7ull, RSP_0),0 x02b7, MSR_OFFCORE_RSP_1, 0 x399ffbffe7ull, RSP_1),static struct extra_reg intel_snb_extra_regs[] __read_mostly = {/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 x3f807f8fffull, RSP_0),0 x01bb, MSR_OFFCORE_RSP_1, 0 x3f807f8fffull, RSP_1),0 x01cd),static struct extra_reg intel_snbep_extra_regs[] __read_mostly = {/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 x3fffff8fffull, RSP_0),0 x01bb, MSR_OFFCORE_RSP_1, 0 x3fffff8fffull, RSP_1),0 x01cd),static struct extra_reg intel_skl_extra_regs[] __read_mostly = {0 x01b7, MSR_OFFCORE_RSP_0, 0 x3fffff8fffull, RSP_0),0 x01bb, MSR_OFFCORE_RSP_1, 0 x3fffff8fffull, RSP_1),0 x01cd),/* * Note the low 8 bits eventsel code is not a continuous field , containing * some # GPing bits . These are masked out . 0 x01c6, MSR_PEBS_FRONTEND, 0 x7fff17, FE),static struct event_constraint intel_icl_event_constraints[] = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x01c0, 0 ), /* old INST_RETIRED.PREC_DIST */ 0 x0100, 0 ), /* INST_RETIRED.PREC_DIST */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x0400, 3 ), /* SLOTS */ 0 ),1 ),2 ),3 ),0 x03, 0 x0a, 0 xf),0 x1f, 0 x28, 0 xf),0 x32, 0 xf), /* SW_PREFETCH_ACCESS.* */ 0 x48, 0 x56, 0 xf),0 x60, 0 x8b, 0 xf),0 x04a3, 0 xff), /* CYCLE_ACTIVITY.STALLS_TOTAL */ 0 x10a3, 0 xff), /* CYCLE_ACTIVITY.CYCLES_MEM_ANY */ 0 x14a3, 0 xff), /* CYCLE_ACTIVITY.STALLS_MEM_ANY */ 0 xa3, 0 xf), /* CYCLE_ACTIVITY.* */ 0 xa8, 0 xb0, 0 xf),0 xb7, 0 xbd, 0 xf),0 xd0, 0 xe6, 0 xf),0 xef, 0 xf),0 xf0, 0 xf4, 0 xf),static struct extra_reg intel_icl_extra_regs[] __read_mostly = {0 x01b7, MSR_OFFCORE_RSP_0, 0 x3fffffbfffull, RSP_0),0 x01bb, MSR_OFFCORE_RSP_1, 0 x3fffffbfffull, RSP_1),0 x01cd),0 x01c6, MSR_PEBS_FRONTEND, 0 x7fff17, FE),static struct extra_reg intel_glc_extra_regs[] __read_mostly = {0 x012a, MSR_OFFCORE_RSP_0, 0 x3fffffffffull, RSP_0),0 x012b, MSR_OFFCORE_RSP_1, 0 x3fffffffffull, RSP_1),0 x01cd),0 x01c6, MSR_PEBS_FRONTEND, 0 x7fff1f, FE),0 x40ad, MSR_PEBS_FRONTEND, 0 x7, FE),0 x04c2, MSR_PEBS_FRONTEND, 0 x8, FE),static struct event_constraint intel_glc_event_constraints[] = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x0100, 0 ), /* INST_RETIRED.PREC_DIST */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x013c, 2 ), /* CPU_CLK_UNHALTED.REF_TSC_P */ 0 x0400, 3 ), /* SLOTS */ 0 ),1 ),2 ),3 ),4 ),5 ),6 ),7 ),0 x2e, 0 xff),0 x3c, 0 xff),/* * Generally event codes < 0 x90 are restricted to counters 0 - 3 . * The 0 x2E and 0 x3C are exception , which has no restriction . 0 x01, 0 x8f, 0 xf),0 x01a3, 0 xf),0 x02a3, 0 xf),0 x08a3, 0 xf),0 x04a4, 0 x1),0 x08a4, 0 x1),0 x02cd, 0 x1),0 xce, 0 x1),0 xd0, 0 xdf, 0 xf),/* * Generally event codes > = 0 x90 are likely to have no restrictions . * The exception are defined as above . 0 x90, 0 xfe, 0 xff),static struct extra_reg intel_rwc_extra_regs[] __read_mostly = {0 x012a, MSR_OFFCORE_RSP_0, 0 x3fffffffffull, RSP_0),0 x012b, MSR_OFFCORE_RSP_1, 0 x3fffffffffull, RSP_1),0 x01cd),0 x02c6, MSR_PEBS_FRONTEND, 0 x9, FE),0 x03c6, MSR_PEBS_FRONTEND, 0 x7fff1f, FE),0 x40ad, MSR_PEBS_FRONTEND, 0 x7, FE),0 x04c2, MSR_PEBS_FRONTEND, 0 x8, FE),static struct event_constraint intel_lnc_event_constraints[] = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x0100, 0 ), /* INST_RETIRED.PREC_DIST */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x013c, 2 ), /* CPU_CLK_UNHALTED.REF_TSC_P */ 0 x0400, 3 ), /* SLOTS */ 0 ),1 ),2 ),3 ),4 ),5 ),6 ),7 ),0 x20, 0 xf),0 x012a, 0 xf),0 x012b, 0 xf),0 x0148, 0 x4),0 x0175, 0 x4),0 x2e, 0 x3ff),0 x3c, 0 x3ff),0 x08a3, 0 x4),0 x0ca3, 0 x4),0 x04a4, 0 x1),0 x08a4, 0 x1),0 x10a4, 0 x1),0 x01b1, 0 x8),0 x01cd, 0 x3fc),0 x02cd, 0 x3),0 xd0, 0 xdf, 0 xf),0 x00e0, 0 xf),static struct extra_reg intel_lnc_extra_regs[] __read_mostly = {0 x012a, MSR_OFFCORE_RSP_0, 0 xfffffffffffull, RSP_0),0 x012b, MSR_OFFCORE_RSP_1, 0 xfffffffffffull, RSP_1),0 x01cd),0 x02c6, MSR_PEBS_FRONTEND, 0 x9, FE),0 x03c6, MSR_PEBS_FRONTEND, 0 x7fff1f, FE),0 x40ad, MSR_PEBS_FRONTEND, 0 xf, FE),0 x04c2, MSR_PEBS_FRONTEND, 0 x8, FE),"event=0x0b,umask=0x10,ldlat=3" );"event=0xcd,umask=0x1,ldlat=3" );"event=0xcd,umask=0x2" );static struct attribute *nhm_mem_events_attrs[] = {/* * topdown events for Intel Core CPUs . * * The events are all in slots , which is a free slot in a 4 wide * pipeline . Some events are already reported in slots , for cycle * events we multiply by the pipeline width ( 4 ) . * * With Hyper Threading on , topdown metrics are either summed or averaged * between the threads of a core : ( count_t0 + count_t1 ) . * * For the average case the metric is always scaled to pipeline width , * so we use factor 2 ( ( count_t0 + count_t1 ) / 2 * 4 ) "event=0x3c,umask=0x0" , /* cpu_clk_unhalted.thread */ "event=0x3c,umask=0x0,any=1" ); /* cpu_clk_unhalted.thread_any */ "4" , "2" );"event=0xe,umask=0x1" ); /* uops_issued.any */ "event=0xc2,umask=0x2" ); /* uops_retired.retire_slots */ "event=0x9c,umask=0x1" ); /* idq_uops_not_delivered_core */ "event=0xd,umask=0x3,cmask=1" , /* int_misc.recovery_cycles */ "event=0xd,umask=0x3,cmask=1,any=1" ); /* int_misc.recovery_cycles_any */ "4" , "2" );"event=0x00,umask=0x4" );"event=0x00,umask=0x80" );"event=0x00,umask=0x81" );"event=0x00,umask=0x82" );"event=0x00,umask=0x83" );"event=0x00,umask=0x84" );"event=0x00,umask=0x85" );"event=0x00,umask=0x86" );"event=0x00,umask=0x87" );static struct attribute *snb_events_attrs[] = {static struct attribute *snb_mem_events_attrs[] = {static struct event_constraint intel_hsw_event_constraints[] = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x148, 0 x4), /* L1D_PEND_MISS.PENDING */ 0 x01c0, 0 x2), /* INST_RETIRED.PREC_DIST */ 0 xcd, 0 x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ 0 x08a3, 0 x4),/* CYCLE_ACTIVITY.STALLS_L1D_PENDING */ 0 x0ca3, 0 x4),/* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */ 0 x04a3, 0 xf),/* * When HT is off these events can only run on the bottom 4 counters * When HT is on , they are impacted by the HT bug and require EXCL access 0 xd0, 0 xf), /* MEM_UOPS_RETIRED.* */ 0 xd1, 0 xf), /* MEM_LOAD_UOPS_RETIRED.* */ 0 xd2, 0 xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ 0 xd3, 0 xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */ static struct event_constraint intel_bdw_event_constraints[] = {0 x00c0, 0 ), /* INST_RETIRED.ANY */ 0 x003c, 1 ), /* CPU_CLK_UNHALTED.CORE */ 0 x0300, 2 ), /* CPU_CLK_UNHALTED.REF */ 0 x148, 0 x4), /* L1D_PEND_MISS.PENDING */ 0 x8a3, 0 x4), /* CYCLE_ACTIVITY.CYCLES_L1D_MISS */ /* * when HT is off , these can only run on the bottom 4 counters 0 xd0, 0 xf), /* MEM_INST_RETIRED.* */ 0 xd1, 0 xf), /* MEM_LOAD_RETIRED.* */ 0 xd2, 0 xf), /* MEM_LOAD_L3_HIT_RETIRED.* */ 0 xcd, 0 xf), /* MEM_TRANS_RETIRED.* */ static u64 intel_pmu_event_map(int hw_event)return intel_perfmon_event_map[hw_event];static __initconst const u64 glc_hw_cache_event_ids0 x81d0,0 xe124,0 x82d0,0 xe424,1 ,1 ,0 x12a,0 x12a,0 x12a,0 x12a,0 x81d0,0 xe12,0 x82d0,0 xe13,1 ,0 xe11,1 ,1 ,1 ,1 ,0 x4c4,0 x4c5,1 ,1 ,1 ,1 ,0 x12a,0 x12a,static __initconst const u64 glc_hw_cache_extra_regs0 x10001,0 x3fbfc00001,0 x3f3ffc0002,0 x3f3fc00002,0 x10c000001,0 x3fb3000001,/* * Notes on the events : * - data reads do not include code reads ( comparable to earlier tables ) * - data counts include speculative execution ( except L1 write , dtlb , bpu ) * - remote node access includes remote memory , remote cache , remote mmio . * - prefetches are not included in the counts . * - icache miss does not include decoded icache #define SKL_DEMAND_DATA_RD BIT_ULL(0 )#define SKL_DEMAND_RFO BIT_ULL(1 )#define SKL_ANY_RESPONSE BIT_ULL(16 )#define SKL_SUPPLIER_NONE BIT_ULL(17 )#define SKL_L3_MISS_LOCAL_DRAM BIT_ULL(26 )#define SKL_L3_MISS_REMOTE_HOP0_DRAM BIT_ULL(27 )#define SKL_L3_MISS_REMOTE_HOP1_DRAM BIT_ULL(28 )#define SKL_L3_MISS_REMOTE_HOP2P_DRAM BIT_ULL(29 )#define SKL_L3_MISS (SKL_L3_MISS_LOCAL_DRAM| \#define SKL_SPL_HIT BIT_ULL(30 )#define SKL_SNOOP_NONE BIT_ULL(31 )#define SKL_SNOOP_NOT_NEEDED BIT_ULL(32 )#define SKL_SNOOP_MISS BIT_ULL(33 )#define SKL_SNOOP_HIT_NO_FWD BIT_ULL(34 )#define SKL_SNOOP_HIT_WITH_FWD BIT_ULL(35 )#define SKL_SNOOP_HITM BIT_ULL(36 )#define SKL_SNOOP_NON_DRAM BIT_ULL(37 )#define SKL_ANY_SNOOP (SKL_SPL_HIT|SKL_SNOOP_NONE| \#define SKL_DEMAND_READ SKL_DEMAND_DATA_RD#define SKL_SNOOP_DRAM (SKL_SNOOP_NONE| \#define SKL_DEMAND_WRITE SKL_DEMAND_RFO#define SKL_LLC_ACCESS SKL_ANY_RESPONSE#define SKL_L3_MISS_REMOTE (SKL_L3_MISS_REMOTE_HOP0_DRAM| \static __initconst const u64 skl_hw_cache_event_ids0 x81d0, /* MEM_INST_RETIRED.ALL_LOADS */ 0 x151, /* L1D.REPLACEMENT */ 0 x82d0, /* MEM_INST_RETIRED.ALL_STORES */ 0 x0,0 x0,0 x0,0 x0,0 x283, /* ICACHE_64B.MISS */ 1 ,1 ,0 x0,0 x0,0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x0,0 x0,0 x81d0, /* MEM_INST_RETIRED.ALL_LOADS */ 0 xe08, /* DTLB_LOAD_MISSES.WALK_COMPLETED */ 0 x82d0, /* MEM_INST_RETIRED.ALL_STORES */ 0 xe49, /* DTLB_STORE_MISSES.WALK_COMPLETED */ 0 x0,0 x0,0 x2085, /* ITLB_MISSES.STLB_HIT */ 0 xe85, /* ITLB_MISSES.WALK_COMPLETED */ 1 ,1 ,1 ,1 ,0 xc4, /* BR_INST_RETIRED.ALL_BRANCHES */ 0 xc5, /* BR_MISP_RETIRED.ALL_BRANCHES */ 1 ,1 ,1 ,1 ,0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x0,0 x0,static __initconst const u64 skl_hw_cache_extra_regs0 x0,0 x0,0 x0,0 x0,#define SNB_DMND_DATA_RD (1 ULL << 0 )#define SNB_DMND_RFO (1 ULL << 1 )#define SNB_DMND_IFETCH (1 ULL << 2 )#define SNB_DMND_WB (1 ULL << 3 )#define SNB_PF_DATA_RD (1 ULL << 4 )#define SNB_PF_RFO (1 ULL << 5 )#define SNB_PF_IFETCH (1 ULL << 6 )#define SNB_LLC_DATA_RD (1 ULL << 7 )#define SNB_LLC_RFO (1 ULL << 8 )#define SNB_LLC_IFETCH (1 ULL << 9 )#define SNB_BUS_LOCKS (1 ULL << 10 )#define SNB_STRM_ST (1 ULL << 11 )#define SNB_OTHER (1 ULL << 15 )#define SNB_RESP_ANY (1 ULL << 16 )#define SNB_NO_SUPP (1 ULL << 17 )#define SNB_LLC_HITM (1 ULL << 18 )#define SNB_LLC_HITE (1 ULL << 19 )#define SNB_LLC_HITS (1 ULL << 20 )#define SNB_LLC_HITF (1 ULL << 21 )#define SNB_LOCAL (1 ULL << 22 )#define SNB_REMOTE (0 xffULL << 23 )#define SNB_SNP_NONE (1 ULL << 31 )#define SNB_SNP_NOT_NEEDED (1 ULL << 32 )#define SNB_SNP_MISS (1 ULL << 33 )#define SNB_NO_FWD (1 ULL << 34 )#define SNB_SNP_FWD (1 ULL << 35 )#define SNB_HITM (1 ULL << 36 )#define SNB_NON_DRAM (1 ULL << 37 )#define SNB_DMND_READ (SNB_DMND_DATA_RD|SNB_LLC_DATA_RD)#define SNB_DMND_WRITE (SNB_DMND_RFO|SNB_LLC_RFO)#define SNB_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO)#define SNB_SNP_ANY (SNB_SNP_NONE|SNB_SNP_NOT_NEEDED| \#define SNB_DRAM_ANY (SNB_LOCAL|SNB_REMOTE|SNB_SNP_ANY)#define SNB_DRAM_REMOTE (SNB_REMOTE|SNB_SNP_ANY)#define SNB_L3_ACCESS SNB_RESP_ANY#define SNB_L3_MISS (SNB_DRAM_ANY|SNB_NON_DRAM)static __initconst const u64 snb_hw_cache_extra_regsstatic __initconst const u64 snb_hw_cache_event_ids0 xf1d0, /* MEM_UOP_RETIRED.LOADS */ 0 x0151, /* L1D.REPLACEMENT */ 0 xf2d0, /* MEM_UOP_RETIRED.STORES */ 0 x0851, /* L1D.ALL_M_REPLACEMENT */ 0 x0,0 x024e, /* HW_PRE_REQ.DL1_MISS */ 0 x0,0 x0280, /* ICACHE.MISSES */ 1 ,1 ,0 x0,0 x0,/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ 0 x01b7,0 x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */ 0 x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */ 0 x82d0, /* MEM_UOP_RETIRED.ALL_STORES */ 0 x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */ 0 x0,0 x0,0 x1085, /* ITLB_MISSES.STLB_HIT */ 0 x0185, /* ITLB_MISSES.CAUSES_A_WALK */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ 0 x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */ 1 ,1 ,1 ,1 ,0 x01b7,0 x01b7,0 x01b7,0 x01b7,0 x01b7,0 x01b7,/* * Notes on the events : * - data reads do not include code reads ( comparable to earlier tables ) * - data counts include speculative execution ( except L1 write , dtlb , bpu ) * - remote node access includes remote memory , remote cache , remote mmio . * - prefetches are not included in the counts because they are not * reliably counted . #define HSW_DEMAND_DATA_RD BIT_ULL(0 )#define HSW_DEMAND_RFO BIT_ULL(1 )#define HSW_ANY_RESPONSE BIT_ULL(16 )#define HSW_SUPPLIER_NONE BIT_ULL(17 )#define HSW_L3_MISS_LOCAL_DRAM BIT_ULL(22 )#define HSW_L3_MISS_REMOTE_HOP0 BIT_ULL(27 )#define HSW_L3_MISS_REMOTE_HOP1 BIT_ULL(28 )#define HSW_L3_MISS_REMOTE_HOP2P BIT_ULL(29 )#define HSW_L3_MISS (HSW_L3_MISS_LOCAL_DRAM| \#define HSW_SNOOP_NONE BIT_ULL(31 )#define HSW_SNOOP_NOT_NEEDED BIT_ULL(32 )#define HSW_SNOOP_MISS BIT_ULL(33 )#define HSW_SNOOP_HIT_NO_FWD BIT_ULL(34 )#define HSW_SNOOP_HIT_WITH_FWD BIT_ULL(35 )#define HSW_SNOOP_HITM BIT_ULL(36 )#define HSW_SNOOP_NON_DRAM BIT_ULL(37 )#define HSW_ANY_SNOOP (HSW_SNOOP_NONE| \#define HSW_SNOOP_DRAM (HSW_ANY_SNOOP & ~HSW_SNOOP_NON_DRAM)#define HSW_DEMAND_READ HSW_DEMAND_DATA_RD#define HSW_DEMAND_WRITE HSW_DEMAND_RFO#define HSW_L3_MISS_REMOTE (HSW_L3_MISS_REMOTE_HOP0|\#define HSW_LLC_ACCESS HSW_ANY_RESPONSE#define BDW_L3_MISS_LOCAL BIT(26 )#define BDW_L3_MISS (BDW_L3_MISS_LOCAL| \static __initconst const u64 hsw_hw_cache_event_ids0 x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ 0 x151, /* L1D.REPLACEMENT */ 0 x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ 0 x0,0 x0,0 x0,0 x0,0 x280, /* ICACHE.MISSES */ 1 ,1 ,0 x0,0 x0,0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x0,0 x0,0 x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ 0 x108, /* DTLB_LOAD_MISSES.MISS_CAUSES_A_WALK */ 0 x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ 0 x149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */ 0 x0,0 x0,0 x6085, /* ITLB_MISSES.STLB_HIT */ 0 x185, /* ITLB_MISSES.MISS_CAUSES_A_WALK */ 1 ,1 ,1 ,1 ,0 xc4, /* BR_INST_RETIRED.ALL_BRANCHES */ 0 xc5, /* BR_MISP_RETIRED.ALL_BRANCHES */ 1 ,1 ,1 ,1 ,0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x0,0 x0,static __initconst const u64 hsw_hw_cache_extra_regs0 x0,0 x0,0 x0,0 x0,static __initconst const u64 westmere_hw_cache_event_ids0 x010b, /* MEM_INST_RETIRED.LOADS */ 0 x0151, /* L1D.REPL */ 0 x020b, /* MEM_INST_RETURED.STORES */ 0 x0251, /* L1D.M_REPL */ 0 x014e, /* L1D_PREFETCH.REQUESTS */ 0 x024e, /* L1D_PREFETCH.MISS */ 0 x0380, /* L1I.READS */ 0 x0280, /* L1I.MISSES */ 1 ,1 ,0 x0,0 x0,/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */ 0 x01b7,/* * Use RFO , not WRITEBACK , because a write miss would typically occur * on RFO . /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ 0 x01b7,0 x010b, /* MEM_INST_RETIRED.LOADS */ 0 x0108, /* DTLB_LOAD_MISSES.ANY */ 0 x020b, /* MEM_INST_RETURED.STORES */ 0 x010c, /* MEM_STORE_RETIRED.DTLB_MISS */ 0 x0,0 x0,0 x01c0, /* INST_RETIRED.ANY_P */ 0 x0185, /* ITLB_MISSES.ANY */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ 0 x03e8, /* BPU_CLEARS.ANY */ 1 ,1 ,1 ,1 ,0 x01b7,0 x01b7,0 x01b7,0 x01b7,0 x01b7,0 x01b7,/* * Nehalem / Westmere MSR_OFFCORE_RESPONSE bits ; * See IA32 SDM Vol 3 B 30 . 6 . 1 . 3 #define NHM_DMND_DATA_RD (1 << 0 )#define NHM_DMND_RFO (1 << 1 )#define NHM_DMND_IFETCH (1 << 2 )#define NHM_DMND_WB (1 << 3 )#define NHM_PF_DATA_RD (1 << 4 )#define NHM_PF_DATA_RFO (1 << 5 )#define NHM_PF_IFETCH (1 << 6 )#define NHM_OFFCORE_OTHER (1 << 7 )#define NHM_UNCORE_HIT (1 << 8 )#define NHM_OTHER_CORE_HIT_SNP (1 << 9 )#define NHM_OTHER_CORE_HITM (1 << 10 )/* reserved */ #define NHM_REMOTE_CACHE_FWD (1 << 12 )#define NHM_REMOTE_DRAM (1 << 13 )#define NHM_LOCAL_DRAM (1 << 14 )#define NHM_NON_DRAM (1 << 15 )#define NHM_LOCAL (NHM_LOCAL_DRAM|NHM_REMOTE_CACHE_FWD)#define NHM_REMOTE (NHM_REMOTE_DRAM)#define NHM_DMND_READ (NHM_DMND_DATA_RD)#define NHM_DMND_WRITE (NHM_DMND_RFO|NHM_DMND_WB)#define NHM_DMND_PREFETCH (NHM_PF_DATA_RD|NHM_PF_DATA_RFO)#define NHM_L3_HIT (NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM)#define NHM_L3_MISS (NHM_NON_DRAM|NHM_LOCAL_DRAM|NHM_REMOTE_DRAM|NHM_REMOTE_CACHE_FWD)#define NHM_L3_ACCESS (NHM_L3_HIT|NHM_L3_MISS)static __initconst const u64 nehalem_hw_cache_extra_regsstatic __initconst const u64 nehalem_hw_cache_event_ids0 x010b, /* MEM_INST_RETIRED.LOADS */ 0 x0151, /* L1D.REPL */ 0 x020b, /* MEM_INST_RETURED.STORES */ 0 x0251, /* L1D.M_REPL */ 0 x014e, /* L1D_PREFETCH.REQUESTS */ 0 x024e, /* L1D_PREFETCH.MISS */ 0 x0380, /* L1I.READS */ 0 x0280, /* L1I.MISSES */ 1 ,1 ,0 x0,0 x0,/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */ 0 x01b7,/* * Use RFO , not WRITEBACK , because a write miss would typically occur * on RFO . /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ 0 x01b7,0 x0f40, /* L1D_CACHE_LD.MESI (alias) */ 0 x0108, /* DTLB_LOAD_MISSES.ANY */ 0 x0f41, /* L1D_CACHE_ST.MESI (alias) */ 0 x010c, /* MEM_STORE_RETIRED.DTLB_MISS */ 0 x0,0 x0,0 x01c0, /* INST_RETIRED.ANY_P */ 0 x20c8, /* ITLB_MISS_RETIRED */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ 0 x03e8, /* BPU_CLEARS.ANY */ 1 ,1 ,1 ,1 ,0 x01b7,0 x01b7,0 x01b7,0 x01b7,0 x01b7,0 x01b7,static __initconst const u64 core2_hw_cache_event_ids0 x0f40, /* L1D_CACHE_LD.MESI */ 0 x0140, /* L1D_CACHE_LD.I_STATE */ 0 x0f41, /* L1D_CACHE_ST.MESI */ 0 x0141, /* L1D_CACHE_ST.I_STATE */ 0 x104e, /* L1D_PREFETCH.REQUESTS */ 0 ,0 x0080, /* L1I.READS */ 0 x0081, /* L1I.MISSES */ 1 ,1 ,0 ,0 ,0 x4f29, /* L2_LD.MESI */ 0 x4129, /* L2_LD.ISTATE */ 0 x4f2A, /* L2_ST.MESI */ 0 x412A, /* L2_ST.ISTATE */ 0 ,0 ,0 x0f40, /* L1D_CACHE_LD.MESI (alias) */ 0 x0208, /* DTLB_MISSES.MISS_LD */ 0 x0f41, /* L1D_CACHE_ST.MESI (alias) */ 0 x0808, /* DTLB_MISSES.MISS_ST */ 0 ,0 ,0 x00c0, /* INST_RETIRED.ANY_P */ 0 x1282, /* ITLBMISSES */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ANY */ 0 x00c5, /* BP_INST_RETIRED.MISPRED */ 1 ,1 ,1 ,1 ,static __initconst const u64 atom_hw_cache_event_ids0 x2140, /* L1D_CACHE.LD */ 0 ,0 x2240, /* L1D_CACHE.ST */ 0 ,0 x0,0 ,0 x0380, /* L1I.READS */ 0 x0280, /* L1I.MISSES */ 1 ,1 ,0 ,0 ,0 x4f29, /* L2_LD.MESI */ 0 x4129, /* L2_LD.ISTATE */ 0 x4f2A, /* L2_ST.MESI */ 0 x412A, /* L2_ST.ISTATE */ 0 ,0 ,0 x2140, /* L1D_CACHE_LD.MESI (alias) */ 0 x0508, /* DTLB_MISSES.MISS_LD */ 0 x2240, /* L1D_CACHE_ST.MESI (alias) */ 0 x0608, /* DTLB_MISSES.MISS_ST */ 0 ,0 ,0 x00c0, /* INST_RETIRED.ANY_P */ 0 x0282, /* ITLB.MISSES */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ANY */ 0 x00c5, /* BP_INST_RETIRED.MISPRED */ 1 ,1 ,1 ,1 ,"event=0x3c" );"2" );/* no_alloc_cycles.not_delivered */ "event=0xca,umask=0x50" );"2" );/* uops_retired.all */ "event=0xc2,umask=0x10" );/* uops_retired.all */ "event=0xc2,umask=0x10" );static struct attribute *slm_events_attrs[] = {static struct extra_reg intel_slm_extra_regs[] __read_mostly =/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 x768005ffffull, RSP_0),0 x02b7, MSR_OFFCORE_RSP_1, 0 x368005ffffull, RSP_1),#define SLM_DMND_READ SNB_DMND_DATA_RD#define SLM_DMND_WRITE SNB_DMND_RFO#define SLM_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO)#define SLM_SNP_ANY (SNB_SNP_NONE|SNB_SNP_MISS|SNB_NO_FWD|SNB_HITM)#define SLM_LLC_ACCESS SNB_RESP_ANY#define SLM_LLC_MISS (SLM_SNP_ANY|SNB_NON_DRAM)static __initconst const u64 slm_hw_cache_extra_regs0 ,static __initconst const u64 slm_hw_cache_event_ids0 ,0 x0104, /* LD_DCU_MISS */ 0 ,0 ,0 ,0 ,0 x0380, /* ICACHE.ACCESSES */ 0 x0280, /* ICACGE.MISSES */ 1 ,1 ,0 ,0 ,/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ 0 x01b7,0 ,/* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ 0 x01b7,/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ 0 x01b7,0 ,0 x0804, /* LD_DTLB_MISS */ 0 ,0 ,0 ,0 ,0 x00c0, /* INST_RETIRED.ANY_P */ 0 x40205, /* PAGE_WALKS.I_SIDE_WALKS */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ANY */ 0 x00c5, /* BP_INST_RETIRED.MISPRED */ 1 ,1 ,1 ,1 ,"event=0x3c" );"3" );/* UOPS_NOT_DELIVERED.ANY */ "event=0x9c" );/* ISSUE_SLOTS_NOT_CONSUMED.RECOVERY */ "event=0xca,umask=0x02" );/* UOPS_RETIRED.ANY */ "event=0xc2" );/* UOPS_ISSUED.ANY */ "event=0x0e" );static struct attribute *glm_events_attrs[] = {static struct extra_reg intel_glm_extra_regs[] __read_mostly = {/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 x760005ffbfull, RSP_0),0 x02b7, MSR_OFFCORE_RSP_1, 0 x360005ffbfull, RSP_1),#define GLM_DEMAND_DATA_RD BIT_ULL(0 )#define GLM_DEMAND_RFO BIT_ULL(1 )#define GLM_ANY_RESPONSE BIT_ULL(16 )#define GLM_SNP_NONE_OR_MISS BIT_ULL(33 )#define GLM_DEMAND_READ GLM_DEMAND_DATA_RD#define GLM_DEMAND_WRITE GLM_DEMAND_RFO#define GLM_DEMAND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO)#define GLM_LLC_ACCESS GLM_ANY_RESPONSE#define GLM_SNP_ANY (GLM_SNP_NONE_OR_MISS|SNB_NO_FWD|SNB_HITM)#define GLM_LLC_MISS (GLM_SNP_ANY|SNB_NON_DRAM)static __initconst const u64 glm_hw_cache_event_ids0 x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ 0 x0,0 x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ 0 x0,0 x0,0 x0,0 x0380, /* ICACHE.ACCESSES */ 0 x0280, /* ICACHE.MISSES */ 1 ,1 ,0 x0,0 x0,0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ 0 x0,0 x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ 0 x0,0 x0,0 x0,0 x00c0, /* INST_RETIRED.ANY_P */ 0 x0481, /* ITLB.MISS */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ 0 x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */ 1 ,1 ,1 ,1 ,static __initconst const u64 glm_hw_cache_extra_regsstatic __initconst const u64 glp_hw_cache_event_ids0 x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ 0 x0,0 x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ 0 x0,0 x0,0 x0,0 x0380, /* ICACHE.ACCESSES */ 0 x0280, /* ICACHE.MISSES */ 1 ,1 ,0 x0,0 x0,0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x1b7, /* OFFCORE_RESPONSE */ 0 x0,0 x0,0 x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ 0 xe08, /* DTLB_LOAD_MISSES.WALK_COMPLETED */ 0 x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ 0 xe49, /* DTLB_STORE_MISSES.WALK_COMPLETED */ 0 x0,0 x0,0 x00c0, /* INST_RETIRED.ANY_P */ 0 x0481, /* ITLB.MISS */ 1 ,1 ,1 ,1 ,0 x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ 0 x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */ 1 ,1 ,1 ,1 ,static __initconst const u64 glp_hw_cache_extra_regs0 x0,0 x0,#define TNT_LOCAL_DRAM BIT_ULL(26 )#define TNT_DEMAND_READ GLM_DEMAND_DATA_RD#define TNT_DEMAND_WRITE GLM_DEMAND_RFO#define TNT_LLC_ACCESS GLM_ANY_RESPONSE#define TNT_SNP_ANY (SNB_SNP_NOT_NEEDED|SNB_SNP_MISS| \#define TNT_LLC_MISS (TNT_SNP_ANY|SNB_NON_DRAM|TNT_LOCAL_DRAM)static __initconst const u64 tnt_hw_cache_extra_regs0 x0,0 x0,"event=0x71,umask=0x0" );"event=0xc2,umask=0x0" );"event=0x73,umask=0x6" );"event=0x74,umask=0x0" );static struct attribute *tnt_events_attrs[] = {static struct extra_reg intel_tnt_extra_regs[] __read_mostly = {/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 x800ff0ffffff9fffull, RSP_0),0 x02b7, MSR_OFFCORE_RSP_1, 0 xff0ffffff9fffull, RSP_1),"event=0xd0,umask=0x5,ldlat=3" );"event=0xd0,umask=0x6" );static struct attribute *grt_mem_attrs[] = {static struct extra_reg intel_grt_extra_regs[] __read_mostly = {/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 x3fffffffffull, RSP_0),0 x02b7, MSR_OFFCORE_RSP_1, 0 x3fffffffffull, RSP_1),0 x5d0),"event=0x72,umask=0x0" );"event=0x73,umask=0x0" );static struct attribute *cmt_events_attrs[] = {static struct extra_reg intel_cmt_extra_regs[] __read_mostly = {/* must define OFFCORE_RSP_X first, see intel_fixup_er() */ 0 x01b7, MSR_OFFCORE_RSP_0, 0 x800ff3ffffffffffull, RSP_0),0 x02b7, MSR_OFFCORE_RSP_1, 0 xff3ffffffffffull, RSP_1),0 x5d0),0 x0127, MSR_SNOOP_RSP_0, 0 xffffffffffffffffull, SNOOP_0),0 x0227, MSR_SNOOP_RSP_1, 0 xffffffffffffffffull, SNOOP_1),"event=0x9c,umask=0x01" );"event=0xc2,umask=0x02" );"event=0xa4,umask=0x02" );static struct attribute *skt_events_attrs[] = {#define KNL_OT_L2_HITE BIT_ULL(19 ) /* Other Tile L2 Hit */ #define KNL_OT_L2_HITF BIT_ULL(20 ) /* Other Tile L2 Hit */ #define KNL_MCDRAM_LOCAL BIT_ULL(21 )#define KNL_MCDRAM_FAR BIT_ULL(22 )#define KNL_DDR_LOCAL BIT_ULL(23 )#define KNL_DDR_FAR BIT_ULL(24 )#define KNL_DRAM_ANY (KNL_MCDRAM_LOCAL | KNL_MCDRAM_FAR | \#define KNL_L2_READ SLM_DMND_READ#define KNL_L2_WRITE SLM_DMND_WRITE#define KNL_L2_PREFETCH SLM_DMND_PREFETCH#define KNL_L2_ACCESS SLM_LLC_ACCESS#define KNL_L2_MISS (KNL_OT_L2_HITE | KNL_OT_L2_HITF | \static __initconst const u64 knl_hw_cache_extra_regs0 ,/* * Used from PMIs where the LBRs are already disabled . * * This function could be called consecutively . It is required to remain in * disabled state if called consecutively . * * During consecutive calls , the same disable value will be written to related * registers , so the PMU state remains unchanged . * * intel_bts events don ' t coexist with intel PMU ' s BTS events because of * x86_add_exclusive ( x86_lbr_exclusive_lbr ) ; there ' s no need to keep them * disabled around intel PMU ' s event batching etc , only inside the PMI handler . * * Avoid PEBS_ENABLE MSR access in PMIs . * The GLOBAL_CTRL has been disabled . All the counters do not count anymore . * It doesn ' t matter if the PEBS is enabled or not . * Usually , the PEBS status are not changed in PMIs . It ' s unnecessary to * access PEBS_ENABLE MSR in disable_all ( ) / enable_all ( ) . * However , there are some cases which may change PEBS status , e . g . PMI * throttle . The PEBS_ENABLE should be updated where the status changes . static __always_inline void __intel_pmu_disable_all(bool bts)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);0 );if (bts && test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))static __always_inline void intel_pmu_disable_all(void )true );static void __intel_pmu_enable_all(int added, bool pmi)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);if (cpuc->fixed_ctrl_val != cpuc->active_fixed_ctrl_val) {if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {struct perf_event *event =if (WARN_ON_ONCE(!event))return ;static void intel_pmu_enable_all(int added)false );static noinline int struct perf_branch_entry *entries,unsigned int cnt, unsigned long flags)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);unsigned int , cnt, x86_pmu.lbr_nr);sizeof (struct perf_branch_entry) * cnt);0 );return cnt;static int struct perf_branch_entry *entries, unsigned int cnt)unsigned long flags;/* must not have branches... */ false ); /* we don't care about BTS */ /* ... until here */ return __intel_pmu_snapshot_branch_stack(entries, cnt, flags);static int struct perf_branch_entry *entries, unsigned int cnt)unsigned long flags;/* must not have branches... */ false ); /* we don't care about BTS */ /* ... until here */ return __intel_pmu_snapshot_branch_stack(entries, cnt, flags);/* * Workaround for : * Intel Errata AAK100 ( model 26 ) * Intel Errata AAP53 ( model 30 ) * Intel Errata BD53 ( model 44 ) * * The official story : * These chips need to be ' reset ' when adding counters by programming the * magic three ( non - counting ) events 0 x4300B5 , 0 x4300D2 , and 0 x4300B1 either * in sequence on the same PMC or on different PMCs . * * In practice it appears some of these events do in fact count , and * we need to program all 4 events . static void intel_pmu_nhm_workaround(void )struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);static const unsigned long nhm_magic[4 ] = {0 x4300B5,0 x4300D2,0 x4300B1,0 x4300B1struct perf_event *event;int i;/* * The Errata requires below steps : * 1 ) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL ; * 2 ) Configure 4 PERFEVTSELx with the magic events and clear * the corresponding PMCx ; * 3 ) set bit0 ~ bit3 of MSR_CORE_PERF_GLOBAL_CTRL ; * 4 ) Clear MSR_CORE_PERF_GLOBAL_CTRL ; * 5 ) Clear 4 pairs of ERFEVTSELx and PMCx ; /* * The real steps we choose are a little different from above . * A ) To reduce MSR operations , we don ' t run step 1 ) as they * are already cleared before this function is called ; * B ) Call x86_perf_event_update to save PMCx before configuring * PERFEVTSELx with magic number ; * C ) With step 5 ) , we do clear only when the PERFEVTSELx is * not used currently . * D ) Call x86_perf_event_set_period to restore PMCx ; /* We always operate 4 pairs of PERF Counters */ for (i = 0 ; i < 4 ; i++) {if (event)for (i = 0 ; i < 4 ; i++) {0 x0);0 xf);0 x0);for (i = 0 ; i < 4 ; i++) {if (event) {else 0 x0);static void intel_pmu_nhm_enable_all(int added)if (added)static void intel_set_tfa(struct cpu_hw_events *cpuc, bool on)0 ;if (cpuc->tfa_shadow != val) {static void intel_tfa_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)/* * We ' re going to use PMC3 , make sure TFA is set before we touch it . if (cntr == 3 )true );static void intel_tfa_pmu_enable_all(int added)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);/* * If we find PMC3 is no longer used when we enable the PMU , we can * clear TFA . if (!test_bit(3 , cpuc->active_mask))false );static inline u64 intel_pmu_get_status(void )return status;static inline void intel_pmu_ack_status(u64 ack)static inline bool event_is_checkpointed(struct perf_event *event)return unlikely(event->hw.config & HSW_IN_TX_CHECKPOINTED) != 0 ;static inline void intel_set_masks(struct perf_event *event, int idx)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);if (event->attr.exclude_host)unsigned long *)&cpuc->intel_ctrl_guest_mask);if (event->attr.exclude_guest)unsigned long *)&cpuc->intel_ctrl_host_mask);if (event_is_checkpointed(event))unsigned long *)&cpuc->intel_cp_status);static inline void intel_clear_masks(struct perf_event *event, int idx)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);unsigned long *)&cpuc->intel_ctrl_guest_mask);unsigned long *)&cpuc->intel_ctrl_host_mask);unsigned long *)&cpuc->intel_cp_status);static void intel_pmu_disable_fixed(struct perf_event *event)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;if (is_topdown_idx(idx)) {struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);/* * When there are other active TopDown events , * don ' t disable the fixed counter 3 . if (*(u64 *)cpuc->active_mask & INTEL_PMC_OTHER_TOPDOWN_BITS(idx))return ;static void intel_pmu_disable_event(struct perf_event *event)struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;switch (idx) {case 0 ... INTEL_PMC_IDX_FIXED - 1 :break ;case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1 :case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:break ;case INTEL_PMC_IDX_FIXED_BTS:return ;case INTEL_PMC_IDX_FIXED_VLBR:break ;default :"Failed to disable the event with invalid index %d\n" ,return ;/* * Needs to be called after x86_pmu_disable_event , * so we don ' t trigger the event without PEBS bit set . if (unlikely(event->attr.precise_ip))static void intel_pmu_assign_event(struct perf_event *event, int idx)if (is_pebs_pt(event))static __always_inline bool intel_pmu_needs_branch_stack(struct perf_event *event)return event->hw.flags & PERF_X86_EVENT_NEEDS_BRANCH_STACK;static void intel_pmu_del_event(struct perf_event *event)if (intel_pmu_needs_branch_stack(event))if (event->attr.precise_ip)if (is_pebs_counter_event_group(event) ||static int icl_set_topdown_event_period(struct perf_event *event)struct hw_perf_event *hwc = &event->hw;/* * The values in PERF_METRICS MSR are derived from fixed counter 3 . * Software should start both registers , PERF_METRICS and fixed * counter 3 , from zero . * Clear PERF_METRICS and Fixed counter 3 in initialization . * After that , both MSRs will be cleared for each read . * Don ' t need to clear them again . if (left == x86_pmu.max_period) {0 );0 );0 ;0 ;if ((hwc->saved_slots) && is_slots_event(event)) {return 0 ;;static inline u64 icl_get_metrics_event_value(u64 metric, u64 slots, int idx)/* * The metric is reported as an 8 bit integer fraction * summing up to 0 xff . * slots - in - metric = ( Metric / 0 xff ) * slots 8 )) & 0 xff;return mul_u64_u32_div(slots, val, 0 xff);static u64 icl_get_topdown_value(struct perf_event *event,int idx = event->hw.idx;if (is_metric_idx(idx))else return delta;static void __icl_update_topdown_event(struct perf_event *event,0 ;if (last_slots)/* * The 8 bit integer fraction of metric may be not accurate , * especially when the changes is very small . * For example , if only a few bad_spec happens , the fraction * may be reduced from 1 to 0 . If so , the bad_spec event value * will be 0 which is definitely less than the last value . * Avoid update event - > count for this case . if (delta > last) {static void update_saved_topdown_regs(struct perf_event *event, u64 slots,int metric_end)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);struct perf_event *other;int idx;1 ) {if (!is_topdown_idx(idx))continue ;/* * Update all active Topdown events . * * The PERF_METRICS and Fixed counter 3 are read separately . The values may be * modify by a NMI . PMU has to be disabled before calling this function . static u64 intel_update_topdown_event(struct perf_event *event, int metric_end, u64 *val)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);struct perf_event *other;bool reset = true ;int idx;if (!val) {/* read Fixed counter 3 */ 3 | INTEL_PMC_FIXED_RDPMC_BASE);if (!slots)return 0 ;/* read PERF_METRICS */ else {0 ];1 ];/* * Don ' t reset the PERF_METRICS and Fixed counter 3 * for each PEBS record read . Utilize the RDPMC metrics * clear mode . false ;1 ) {if (!is_topdown_idx(idx))continue ;0 ,0 );/* * Check and update this event , which may have been cleared * in active_mask e . g . x86_pmu_stop ( ) if (event && !test_bit(event->hw.idx, cpuc->active_mask)) {/* * In x86_pmu_stop ( ) , the event is cleared in active_mask first , * then drain the delta , which indicates context switch for * counting . * Save metric and slots for context switch . * Don ' t need to reset the PERF_METRICS and Fixed counter 3 . * Because the values will be restored in next schedule in . false ;if (reset) {/* The fixed counter 3 has to be written before the PERF_METRICS. */ 0 );0 );if (event)0 , 0 , metric_end);return slots;static u64 icl_update_topdown_event(struct perf_event *event, u64 *val)return intel_update_topdown_event(event, INTEL_PMC_IDX_METRIC_BASE +1 ,);static void intel_pmu_read_event(struct perf_event *event)if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD | PERF_X86_EVENT_TOPDOWN) ||struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);bool pmu_enabled = cpuc->enabled;/* Only need to call update_topdown_event() once for group read. */ if (is_metric_event(event) && (cpuc->txn_flags & PERF_PMU_TXN_READ))return ;0 ;if (pmu_enabled)/* * If the PEBS counters snapshotting is enabled , * the topdown event is available in PEBS records . if (is_topdown_count(event) && !is_pebs_counter_event_group(event))else if (pmu_enabled)0 );return ;static void intel_pmu_enable_fixed(struct perf_event *event)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;0 ;if (is_topdown_idx(idx)) {struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);/* * When there are other active TopDown events , * don ' t enable the fixed counter 3 again . if (*(u64 *)cpuc->active_mask & INTEL_PMC_OTHER_TOPDOWN_BITS(idx))return ;if (event->attr.config1 & INTEL_TD_CFG_METRIC_CLEAR)/* * Enable IRQ generation ( 0 x8 ) , if not PEBS , * and enable ring - 3 counting ( 0 x2 ) and ring - 0 counting ( 0 x1 ) * if requested : if (!event->attr.precise_ip)if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)/* * ANY bit is supported in v3 and up if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip)static void intel_pmu_config_acr(int idx, u64 mask, u32 reload)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);int msr_b, msr_c;int msr_offset;if (!mask && !cpuc->acr_cfg_b[idx])return ;if (idx < INTEL_PMC_IDX_FIXED) {false );else {false );if (cpuc->acr_cfg_b[idx] != mask) {/* Only need to update the reload value when there is a valid config value. */ if (mask && cpuc->acr_cfg_c[idx] != reload) {static void intel_pmu_enable_acr(struct perf_event *event)struct hw_perf_event *hwc = &event->hw;if (!is_acr_event_group(event) || !event->attr.config2) {/* * The disable doesn ' t clear the ACR CFG register . * Check and clear the ACR CFG register . 0 , 0 );return ;static void intel_pmu_enable_event(struct perf_event *event)struct hw_perf_event *hwc = &event->hw;int idx = hwc->idx;if (unlikely(event->attr.precise_ip))switch (idx) {case 0 ... INTEL_PMC_IDX_FIXED - 1 :if (branch_sample_counters(event))break ;case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1 :case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:break ;case INTEL_PMC_IDX_FIXED_BTS:if (!__this_cpu_read(cpu_hw_events.enabled))return ;break ;case INTEL_PMC_IDX_FIXED_VLBR:break ;default :"Failed to enable the event with invalid index %d\n" ,static void intel_pmu_acr_late_setup(struct cpu_hw_events *cpuc)struct perf_event *event, *leader;int i, j, idx;for (i = 0 ; i < cpuc->n_events; i++) {if (!is_acr_event_group(leader))continue ;/* The ACR events must be contiguous. */ for (j = i; j < cpuc->n_events; j++) {if (event->group_leader != leader->group_leader)break ;unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) {if (i + idx >= cpuc->n_events ||return ;unsigned long *)&event->hw.config1);1 ;void intel_pmu_late_setup(void )struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);if (!cpuc->n_late_setup)return ;static void intel_pmu_add_event(struct perf_event *event)if (event->attr.precise_ip)if (intel_pmu_needs_branch_stack(event))if (is_pebs_counter_event_group(event) ||/* * Save and restart an expired event . Called by NMI contexts , * so it has to be careful about preempting normal event ops : int intel_pmu_save_and_restart(struct perf_event *event)/* * For a checkpointed counter always reset back to 0 . This * avoids a situation where the counter overflows , aborts the * transaction and is then set back to shortly before the * overflow , and overflows and aborts again . if (unlikely(event_is_checkpointed(event))) {/* No race with NMIs because the counter should not be armed */ 0 );0 );return static_call(x86_pmu_set_period)(event);static int intel_pmu_set_period(struct perf_event *event)if (unlikely(is_topdown_count(event)))return static_call(intel_pmu_set_topdown_event_period)(event);return x86_perf_event_set_period(event);static u64 intel_pmu_update(struct perf_event *event)if (unlikely(is_topdown_count(event)))return static_call(intel_pmu_update_topdown_event)(event, NULL);return x86_perf_event_update(event);static void intel_pmu_reset(void )struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);unsigned long *cntr_mask = hybrid(cpuc->pmu, cntr_mask);unsigned long *fixed_cntr_mask = hybrid(cpuc->pmu, fixed_cntr_mask);unsigned long flags;int idx;if (!*(u64 *)cntr_mask)return ;"clearing PMU state on CPU#%d\n" , smp_processor_id());0 ull);0 ull);if (fixed_counter_disabled(idx, cpuc->pmu))continue ;0 ull);if (ds)/* Ack all overflows and disable fixed counters */ if (x86_pmu.version >= 2 ) {0 );/* Reset LBRs and LBR freezing */ if (x86_pmu.lbr_nr) {/* * We may be running with guest PEBS events created by KVM , and the * PEBS records are logged into the guest ' s DS and invisible to host . * * In the case of guest PEBS overflow , we only trigger a fake event * to emulate the PEBS overflow PMI for guest PEBS counters in KVM . * The guest will then vm - entry and check the guest DS area to read * the guest PEBS records . * * The contents and other behavior of the guest event do not matter . static void x86_pmu_handle_guest_pebs(struct pt_regs *regs,struct perf_sample_data *data)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);struct perf_event *event = NULL;int bit;if (!unlikely(perf_guest_state()))return ;if (!x86_pmu.pebs_ept || !x86_pmu.pebs_active ||return ;unsigned long *)&guest_pebs_idxs, X86_PMC_IDX_MAX) {if (!event->attr.precise_ip)continue ;0 , event->hw.last_period);/* Inject one fake event is enough. */ break ;static int handle_pmi_common(struct pt_regs *regs, u64 status)struct perf_sample_data data;struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);int bit;int handled = 0 ;/* * Ignore a range of extra bits in status that do not indicate * overflow by themselves . if (!status)return 0 ;/* * In case multiple PEBS events are sampled at the same time , * it is possible to have GLOBAL_STATUS bit 62 set indicating * PEBS buffer overflow and also seeing at most 3 PEBS counters * having their bits set in the status register . This is a sign * that there was at least one PEBS record pending at the time * of the PMU interrupt . PEBS counters must only be processed * via the drain_pebs ( ) calls and not via the regular sample * processing loop coming after that the function , otherwise * phony regular samples may be generated in the sampling buffer * not marked with the EXACT tag . Another possibility is to have * one PEBS event and at least one non - PEBS event which overflows * while PEBS has armed . In this case , bit 62 of GLOBAL_STATUS will * not be set , yet the overflow status bit for the PEBS counter will * be on Skylake . * * To avoid this problem , we systematically ignore the PEBS - enabled * counters from the GLOBAL_STATUS mask and we always process PEBS * events via drain_pebs ( ) . /* * PEBS overflow sets bit 62 in the global status register if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) {/* * PMI throttle may be triggered , which stops the PEBS event . * Although cpuc - > pebs_enabled is updated accordingly , the * MSR_IA32_PEBS_ENABLE is not updated . Because the * cpuc - > enabled has been forced to 0 in PMI . * Update the MSR if pebs_enabled is changed . if (pebs_enabled != cpuc->pebs_enabled)/* * Above PEBS handler ( PEBS counters snapshotting ) has updated fixed * counter 3 and perf metrics counts if they are in counter group , * unnecessary to update again . if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] &&/* * Intel PT if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) {if (!perf_guest_handle_intel_pt_intr())/* * Intel Perf metrics if (__test_and_clear_bit(GLOBAL_STATUS_PERF_METRICS_OVF_BIT, (unsigned long *)&status)) {/* * Checkpointed counters can lead to ' spurious ' PMIs because the * rollback caused by the PMI will have cleared the overflow status * bit . Therefore always force probe these counters . unsigned long *)&status, X86_PMC_IDX_MAX) {struct perf_event *event = cpuc->events[bit];if (!test_bit(bit, cpuc->active_mask))continue ;/* * There may be unprocessed PEBS records in the PEBS buffer , * which still stores the previous values . * Process those records first before handling the latest value . * For example , * A is a regular counter * B is a PEBS event which reads A * C is a PEBS event * * The following can happen : * B - assist A = 1 * C A = 2 * B - assist A = 3 * A - overflow - PMI A = 4 * C - assist - PMI ( PEBS buffer ) A = 5 * * The PEBS buffer has to be drained before handling the A - PMI if (is_pebs_counter_event_group(event))if (!intel_pmu_save_and_restart(event))continue ;0 , last_period);if (has_branch_stack(event))return handled;/* * This handler is triggered by the local APIC , so the APIC IRQ handling * rules apply : static int intel_pmu_handle_irq(struct pt_regs *regs)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);bool late_ack = hybrid_bit(cpuc->pmu, late_ack);bool mid_ack = hybrid_bit(cpuc->pmu, mid_ack);int loops;int handled;int pmu_enabled;/* * Save the PMU state . * It needs to be restored when leaving the handler . /* * In general , the early ACK is only applied for old platforms . * For the big core starts from Haswell , the late ACK should be * applied . * For the small core after Tremont , we have to do the ACK right * before re - enabling counters , which is in the middle of the * NMI handler . if (!late_ack && !mid_ack)0 ;true );if (!status)goto done;0 ;if (++loops > 100 ) {static bool warned;if (!warned) {1 , "perfevents: irq loop stuck!\n" );true ;goto done;/* * Repeat if there is more work to be done : if (status)goto again;if (mid_ack)/* Only restore PMU state when it's active. See x86_pmu_disable(). */ if (pmu_enabled)0 , true );/* * Only unmask the NMI after the overflow counters * have been reset . This avoids spurious NMIs on * Haswell CPUs . if (late_ack)return handled;static struct event_constraint *struct perf_event *event)if (unlikely(intel_pmu_has_bts(event)))return &bts_constraint;return NULL;/* * Note : matches a fake event , like Fixed2 . static struct event_constraint *struct perf_event *event)struct event_constraint *c = &vlbr_constraint;if (unlikely(constraint_match(c, event->hw.config))) {return c;return NULL;static int intel_alt_er(struct cpu_hw_events *cpuc,int idx, u64 config)struct extra_reg *extra_regs = hybrid(cpuc->pmu, extra_regs);int alt_idx = idx;if (!(x86_pmu.flags & PMU_FL_HAS_RSP_1))return idx;if (idx == EXTRA_REG_RSP_0)if (idx == EXTRA_REG_RSP_1)if (config & ~extra_regs[alt_idx].valid_mask)return idx;return alt_idx;static void intel_fixup_er(struct perf_event *event, int idx)struct extra_reg *extra_regs = hybrid(event->pmu, extra_regs);if (idx == EXTRA_REG_RSP_0) {else if (idx == EXTRA_REG_RSP_1) {/* * manage allocation of shared extra msr for certain events * * sharing can be : * per - cpu : to be shared between the various events on a single PMU * per - core : per - cpu + shared by HT threads static struct event_constraint *struct cpu_hw_events *cpuc,struct perf_event *event,struct hw_perf_event_extra *reg)struct event_constraint *c = &emptyconstraint;struct er_account *era;unsigned long flags;int idx = reg->idx;/* * reg - > alloc can be set due to existing state , so for fake cpuc we * need to ignore this , otherwise we might fail to allocate proper fake * state for this extra reg constraint . Also see the comment below . if (reg->alloc && !cpuc->is_fake)return NULL; /* call x86_get_event_constraint() */ /* * we use spin_lock_irqsave ( ) to avoid lockdep issues when * passing a fake cpuc if (!atomic_read(&era->ref) || era->config == reg->config) {/* * If its a fake cpuc - - as per validate_ { group , event } ( ) we * shouldn ' t touch event state and we can avoid doing so * since both will only call get_event_constraints ( ) once * on each event , this avoids the need for reg - > alloc . * * Not doing the ER fixup will only result in era - > reg being * wrong , but since we won ' t actually try and program hardware * this isn ' t a problem either . if (!cpuc->is_fake) {if (idx != reg->idx)/* * x86_schedule_events ( ) can call get_event_constraints ( ) * multiple times on events in the case of incremental * scheduling ( ) . reg - > alloc ensures we only do the ER * allocation once . 1 ;/* lock in msr value */ /* one more user */ /* * need to call x86_get_event_constraint ( ) * to check if associated event has constraints else {if (idx != reg->idx) {goto again;return c;static void struct cpu_hw_events *cpuc,struct hw_perf_event_extra *reg)struct er_account *era;/* * Only put constraint if extra reg was actually allocated . Also takes * care of event which do not use an extra shared reg . * * Also , if this is a fake cpuc we shouldn ' t touch any event state * ( reg - > alloc ) and we don ' t care about leaving inconsistent cpuc state * either since it ' ll be thrown out . if (!reg->alloc || cpuc->is_fake)return ;/* one fewer user */ /* allocate again next time */ 0 ;static struct event_constraint *struct cpu_hw_events *cpuc,struct perf_event *event)struct event_constraint *c = NULL, *d;struct hw_perf_event_extra *xreg, *breg;if (xreg->idx != EXTRA_REG_NONE) {if (c == &emptyconstraint)return c;if (breg->idx != EXTRA_REG_NONE) {if (d == &emptyconstraint) {return c;struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *event_constraints = hybrid(cpuc->pmu, event_constraints);struct event_constraint *c;if (event_constraints) {if (constraint_match(c, event->hw.config)) {return c;return &hybrid_var(cpuc->pmu, unconstrained);static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c;if (c)return c;if (c)return c;if (c)return c;if (c)return c;return x86_get_event_constraints(cpuc, idx, event);static void struct cpu_hw_events *cpuc)struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;struct intel_excl_states *xl;int tid = cpuc->excl_thread_id;/* * nothing needed if in group validation mode if (cpuc->is_fake || !is_ht_workaround_enabled())return ;/* * no exclusion needed if (WARN_ON_ONCE(!excl_cntrs))return ;true ;/* * lock shared state until we are done scheduling * in stop_event_scheduling ( ) * makes scheduling appear as a transaction static void intel_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;struct event_constraint *c = cpuc->event_constraint[idx];struct intel_excl_states *xl;int tid = cpuc->excl_thread_id;if (cpuc->is_fake || !is_ht_workaround_enabled())return ;if (WARN_ON_ONCE(!excl_cntrs))return ;if (!(c->flags & PERF_X86_EVENT_DYNAMIC))return ;if (c->flags & PERF_X86_EVENT_EXCL)else static void struct cpu_hw_events *cpuc)struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;struct intel_excl_states *xl;int tid = cpuc->excl_thread_id;/* * nothing needed if in group validation mode if (cpuc->is_fake || !is_ht_workaround_enabled())return ;/* * no exclusion needed if (WARN_ON_ONCE(!excl_cntrs))return ;false ;/* * release shared state lock ( acquired in intel_start_scheduling ( ) ) static struct event_constraint *struct cpu_hw_events *cpuc, struct event_constraint *c, int idx)if (!(c->flags & PERF_X86_EVENT_DYNAMIC)) {struct event_constraint *cx;/* * grab pre - allocated constraint entry /* * initialize dynamic constraint * with static constraint /* * mark constraint as dynamic return c;static struct event_constraint *struct cpu_hw_events *cpuc, struct perf_event *event,int idx, struct event_constraint *c)struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;struct intel_excl_states *xlo;int tid = cpuc->excl_thread_id;int is_excl, i, w;/* * validating a group does not require * enforcing cross - thread exclusion if (cpuc->is_fake || !is_ht_workaround_enabled())return c;/* * no exclusion needed if (WARN_ON_ONCE(!excl_cntrs))return c;/* * because we modify the constraint , we need * to make a copy . Static constraints come * from static const tables . * * only needed when constraint has not yet * been cloned ( marked dynamic ) /* * From here on , the constraint is dynamic . * Either it was just allocated above , or it * was allocated during a earlier invocation * of this function /* * state of sibling HT 1 ];/* * event requires exclusive counter access * across HT threads if (is_excl && !(event->hw.flags & PERF_X86_EVENT_EXCL_ACCT)) {if (!cpuc->n_excl++)1 );/* * Modify static constraint with current dynamic * state of thread * * EXCLUSIVE : sibling counter measuring exclusive event * SHARED : sibling counter measuring non - exclusive event * UNUSED : sibling counter unused /* * exclusive event in sibling counter * our corresponding counter cannot be used * regardless of our event if (xlo->state[i] == INTEL_EXCL_EXCLUSIVE) {continue ;/* * if measuring an exclusive event , sibling * measuring non - exclusive , then counter cannot * be used if (is_excl && xlo->state[i] == INTEL_EXCL_SHARED) {continue ;/* * if we return an empty mask , then switch * back to static empty constraint to avoid * the cost of freeing later on if (!w)return c;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c1, *c2;/* * first time only * - static constraint : no change across incremental scheduling calls * - dynamic constraint : handled by intel_get_excl_constraints ( ) if (c1) {if (cpuc->excl_cntrs)return intel_get_excl_constraints(cpuc, event, idx, c2);if (event->hw.dyn_constraint != ~0 ULL) {return c2;static void intel_put_excl_constraints(struct cpu_hw_events *cpuc,struct perf_event *event)struct hw_perf_event *hwc = &event->hw;struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;int tid = cpuc->excl_thread_id;struct intel_excl_states *xl;/* * nothing needed if in group validation mode if (cpuc->is_fake)return ;if (WARN_ON_ONCE(!excl_cntrs))return ;if (hwc->flags & PERF_X86_EVENT_EXCL_ACCT) {if (!--cpuc->n_excl)0 );/* * If event was actually assigned , then mark the counter state as * unused now . if (hwc->idx >= 0 ) {/* * put_constraint may be called from x86_schedule_events ( ) * which already has the lock held so here make locking * conditional . if (!xl->sched_started)if (!xl->sched_started)static void struct cpu_hw_events *cpuc,struct perf_event *event)struct hw_perf_event_extra *reg;if (reg->idx != EXTRA_REG_NONE)if (reg->idx != EXTRA_REG_NONE)static void intel_put_event_constraints(struct cpu_hw_events *cpuc,struct perf_event *event)/* * is PMU has exclusive counter restrictions , then * all events are subject to and must call the * put_excl_constraints ( ) routine if (cpuc->excl_cntrs)static void intel_pebs_aliases_core2(struct perf_event *event)if ((event->hw.config & X86_RAW_EVENT_MASK) == 0 x003c) {/* * Use an alternative encoding for CPU_CLK_UNHALTED . THREAD_P * ( 0 x003c ) so that we can use it with PEBS . * * The regular CPU_CLK_UNHALTED . THREAD_P event ( 0 x003c ) isn ' t * PEBS capable . However we can use INST_RETIRED . ANY_P * ( 0 x00c0 ) , which is a PEBS capable event , to get the same * count . * * INST_RETIRED . ANY_P counts the number of cycles that retires * CNTMASK instructions . By setting CNTMASK to a value ( 16 ) * larger than the maximum number of instructions that can be * retired per cycle ( 4 ) and then inverting the condition , we * count all cycles that retire 16 or less instructions , which * is every cycle . * * Thereby we gain a PEBS capable cycle counter . 0 xc0, .inv=1 , .cmask=16 );static void intel_pebs_aliases_snb(struct perf_event *event)if ((event->hw.config & X86_RAW_EVENT_MASK) == 0 x003c) {/* * Use an alternative encoding for CPU_CLK_UNHALTED . THREAD_P * ( 0 x003c ) so that we can use it with PEBS . * * The regular CPU_CLK_UNHALTED . THREAD_P event ( 0 x003c ) isn ' t * PEBS capable . However we can use UOPS_RETIRED . ALL * ( 0 x01c2 ) , which is a PEBS capable event , to get the same * count . * * UOPS_RETIRED . ALL counts the number of cycles that retires * CNTMASK micro - ops . By setting CNTMASK to a value ( 16 ) * larger than the maximum number of micro - ops that can be * retired per cycle ( 4 ) and then inverting the condition , we * count all cycles that retire 16 or less micro - ops , which * is every cycle . * * Thereby we gain a PEBS capable cycle counter . 0 xc2, .umask=0 x01, .inv=1 , .cmask=16 );static void intel_pebs_aliases_precdist(struct perf_event *event)if ((event->hw.config & X86_RAW_EVENT_MASK) == 0 x003c) {/* * Use an alternative encoding for CPU_CLK_UNHALTED . THREAD_P * ( 0 x003c ) so that we can use it with PEBS . * * The regular CPU_CLK_UNHALTED . THREAD_P event ( 0 x003c ) isn ' t * PEBS capable . However we can use INST_RETIRED . PREC_DIST * ( 0 x01c0 ) , which is a PEBS capable event , to get the same * count . * * The PREC_DIST event has special support to minimize sample * shadowing effects . One drawback is that it can be * only programmed on counter 1 , but that seems like an * acceptable trade off . 0 xc0, .umask=0 x01, .inv=1 , .cmask=16 );static void intel_pebs_aliases_ivb(struct perf_event *event)if (event->attr.precise_ip < 3 )return intel_pebs_aliases_snb(event);return intel_pebs_aliases_precdist(event);static void intel_pebs_aliases_skl(struct perf_event *event)if (event->attr.precise_ip < 3 )return intel_pebs_aliases_core2(event);return intel_pebs_aliases_precdist(event);static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event)unsigned long flags = x86_pmu.large_pebs_flags;if (event->attr.use_clockid)if (!event->attr.exclude_kernel)if (event->attr.sample_regs_user & ~PEBS_GP_REGS)return flags;static int intel_pmu_bts_config(struct perf_event *event)struct perf_event_attr *attr = &event->attr;if (unlikely(intel_pmu_has_bts(event))) {/* BTS is not supported by this architecture. */ if (!x86_pmu.bts_active)return -EOPNOTSUPP;/* BTS is currently only allowed for user-mode. */ if (!attr->exclude_kernel)return -EOPNOTSUPP;/* BTS is not allowed for precise events. */ if (attr->precise_ip)return -EOPNOTSUPP;/* disallow bts if conflicting events are present */ if (x86_add_exclusive(x86_lbr_exclusive_lbr))return -EBUSY;return 0 ;static int core_pmu_hw_config(struct perf_event *event)int ret = x86_pmu_hw_config(event);if (ret)return ret;return intel_pmu_bts_config(event);#define INTEL_TD_METRIC_AVAILABLE_MAX (INTEL_TD_METRIC_RETIRING + \1 ) << 8 ))static bool is_available_metric_event(struct perf_event *event)return is_metric_event(event) &&static inline bool is_mem_loads_event(struct perf_event *event)return (event->attr.config & INTEL_ARCH_EVENT_MASK) == X86_CONFIG(.event=0 xcd, .umask=0 x01);static inline bool is_mem_loads_aux_event(struct perf_event *event)return (event->attr.config & INTEL_ARCH_EVENT_MASK) == X86_CONFIG(.event=0 x03, .umask=0 x82);static inline bool require_mem_loads_aux_event(struct perf_event *event)if (!(x86_pmu.flags & PMU_FL_MEM_LOADS_AUX))return false ;if (is_hybrid())return hybrid_pmu(event->pmu)->pmu_type == hybrid_big;return true ;static inline bool intel_pmu_has_cap(struct perf_event *event, int idx)union perf_capabilities *intel_cap = &hybrid(event->pmu, intel_cap);return test_bit(idx, (unsigned long *)&intel_cap->capabilities);static u64 intel_pmu_freq_start_period(struct perf_event *event)int type = event->attr.type;/* * The 127 is the lowest possible recommended SAV ( sample after value ) * for a 4000 freq ( default freq ) , according to the event list JSON file . * Also , assume the workload is idle 50 % time . 64 * 4000 ;if (type != PERF_TYPE_HARDWARE && type != PERF_TYPE_HW_CACHE)goto end;/* * The estimation of the start period in the freq mode is * based on the below assumption . * * For a cycles or an instructions event , 1 GHZ of the * underlying platform , 1 IPC . The workload is idle 50 % time . * The start period = 1 , 000 , 000 , 000 * 1 / freq / 2 . * = 500 , 000 , 000 / freq * * Usually , the branch - related events occur less than the * instructions event . According to the Intel event list JSON * file , the SAV ( sample after value ) of a branch - related event * is usually 1 / 4 of an instruction event . * The start period of branch - related events = 125 , 000 , 000 / freq . * * The cache - related events occurs even less . The SAV is usually * 1 / 20 of an instruction event . * The start period of cache - related events = 25 , 000 , 000 / freq . if (type == PERF_TYPE_HARDWARE) {switch (config) {case PERF_COUNT_HW_CPU_CYCLES:case PERF_COUNT_HW_INSTRUCTIONS:case PERF_COUNT_HW_BUS_CYCLES:case PERF_COUNT_HW_STALLED_CYCLES_FRONTEND:case PERF_COUNT_HW_STALLED_CYCLES_BACKEND:case PERF_COUNT_HW_REF_CPU_CYCLES:500000000 ;break ;case PERF_COUNT_HW_BRANCH_INSTRUCTIONS:case PERF_COUNT_HW_BRANCH_MISSES:125000000 ;break ;case PERF_COUNT_HW_CACHE_REFERENCES:case PERF_COUNT_HW_CACHE_MISSES:25000000 ;break ;default :goto end;if (type == PERF_TYPE_HW_CACHE)25000000 ;/* * Usually , a prime or a number with less factors ( close to prime ) * is chosen as an SAV , which makes it less likely that the sampling * period synchronizes with some periodic event in the workload . * Minus 1 to make it at least avoiding values near power of twos * for the default freq . 1 ;if (start > x86_pmu.max_period)if (x86_pmu.limit_period)return start;static inline bool intel_pmu_has_acr(struct pmu *pmu)return !!hybrid(pmu, acr_cause_mask64);static bool intel_pmu_is_acr_group(struct perf_event *event)/* The group leader has the ACR flag set */ if (is_acr_event_group(event))return true ;/* The acr_mask is set */ if (event->attr.config2)return true ;return false ;static inline void intel_pmu_set_acr_cntr_constr(struct perf_event *event,int *num)1 ;static inline void intel_pmu_set_acr_caused_constr(struct perf_event *event,int idx, u64 cause_mask)if (test_bit(idx, (unsigned long *)&cause_mask))static int intel_pmu_hw_config(struct perf_event *event)int ret = x86_pmu_hw_config(event);if (ret)return ret;if (ret)return ret;if (event->attr.freq && event->attr.sample_freq) {if (event->attr.precise_ip) {if ((event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_FIXED_VLBR_EVENT)return -EINVAL;if (!(event->attr.freq || (event->attr.wakeup_events && !event->attr.watermark))) {if (!(event->attr.sample_type & ~intel_pmu_large_pebs_flags(event)) &&if (x86_pmu.pebs_aliases)if (needs_branch_stack(event)) {/* Avoid branch stack setup for counting events in SAMPLE READ */ if (is_sampling_event(event) ||if (branch_sample_counters(event)) {struct perf_event *leader, *sibling;int num = 0 ;if (!(x86_pmu.flags & PMU_FL_BR_CNTR) ||return -EINVAL;/* * The branch counter logging is not supported in the call stack * mode yet , since we cannot simply flush the LBR during e . g . , * multiplexing . Also , there is no obvious usage with the call * stack mode . Simply forbids it for now . * * If any events in the group enable the branch counter logging * feature , the group is treated as a branch counter logging * group , which requires the extra space to store the counters . if (branch_sample_call_stack(leader))return -EINVAL;if (branch_sample_counters(leader)) {if (branch_sample_call_stack(sibling))return -EINVAL;if (branch_sample_counters(sibling)) {if (num > fls(x86_pmu.lbr_counters))return -EINVAL;/* * Only applying the PERF_SAMPLE_BRANCH_COUNTERS doesn ' t * require any branch stack setup . * Clear the bit to avoid unnecessary branch stack setup . if (0 == (event->attr.branch_sample_type &/* * Force the leader to be a LBR event . So LBRs can be reset * with the leader event . See intel_pmu_lbr_del ( ) for details . if (!intel_pmu_needs_branch_stack(leader))return -EINVAL;if (intel_pmu_needs_branch_stack(event)) {if (ret)return ret;/* * BTS is set up earlier in this path , so don ' t account twice if (!unlikely(intel_pmu_has_bts(event))) {/* disallow lbr if conflicting events are present */ if (x86_add_exclusive(x86_lbr_exclusive_lbr))return -EBUSY;if (event->attr.aux_output) {if (!event->attr.precise_ip)return -EINVAL;if ((event->attr.sample_type & PERF_SAMPLE_READ) &&6 ) &&if (intel_pmu_has_acr(event->pmu) && intel_pmu_is_acr_group(event)) {struct perf_event *sibling, *leader = event->group_leader;struct pmu *pmu = event->pmu;bool has_sw_event = false ;int num = 0 , idx = 0 ;0 ;/* Not support perf metrics */ if (is_metric_event(event))return -EINVAL;/* Not support freq mode */ if (event->attr.freq)return -EINVAL;/* PDist is not supported */ if (event->attr.config2 && event->attr.precise_ip > 2 )return -EINVAL;/* The reload value cannot exceeds the max period */ if (event->attr.sample_period > x86_pmu.max_period)return -EINVAL;/* * The counter - constraints of each event cannot be finalized * unless the whole group is scanned . However , it ' s hard * to know whether the event is the last one of the group . * Recalculate the counter - constraints for each event when * adding a new event . * * The group is traversed twice , which may be optimized later . * In the first round , * - Find all events which do reload when other events * overflow and set the corresponding counter - constraints * - Add all events , which can cause other events reload , * in the cause_mask * - Error out if the number of events exceeds the HW limit * - The ACR events must be contiguous . * Error out if there are non - X86 events between ACR events . * This is not a HW limit , but a SW limit . * With the assumption , the intel_pmu_acr_late_setup ( ) can * easily convert the event idx to counter idx without * traversing the whole event list . if (!is_x86_event(leader))return -EINVAL;if (leader->attr.config2)if (leader->nr_siblings) {if (!is_x86_event(sibling)) {true ;continue ;if (!sibling->attr.config2)continue ;if (has_sw_event)return -EINVAL;if (leader != event && event->attr.config2) {if (has_sw_event)return -EINVAL;if (hweight64(cause_mask) > hweight64(hybrid(pmu, acr_cause_mask64)) ||return -EINVAL;/* * In the second round , apply the counter - constraints for * the events which can cause other events reload . if (leader->nr_siblings) {if (leader != event)if ((event->attr.type == PERF_TYPE_HARDWARE) ||return 0 ;/* * Config Topdown slots and metric events * * The slots event on Fixed Counter 3 can support sampling , * which will be handled normally in x86_perf_event_update ( ) . * * Metric events don ' t support sampling and require being paired * with a slots event as group leader . When the slots event * is used in a metrics group , it too cannot support sampling . if (intel_pmu_has_cap(event, PERF_CAP_METRICS_IDX) && is_topdown_event(event)) {/* The metrics_clear can only be set for the slots event */ if (event->attr.config1 &&return -EINVAL;if (event->attr.config2)return -EINVAL;/* * The TopDown metrics events and slots event don ' t * support any filters . if (event->attr.config & X86_ALL_EVENT_FLAGS)return -EINVAL;if (is_available_metric_event(event)) {struct perf_event *leader = event->group_leader;/* The metric events don't support sampling. */ if (is_sampling_event(event))return -EINVAL;/* The metric events require a slots group leader. */ if (!is_slots_event(leader))return -EINVAL;/* * The leader / SLOTS must not be a sampling event for * metric use ; hardware requires it starts at 0 when used * in conjunction with MSR_PERF_METRICS . if (is_sampling_event(leader))return -EINVAL;/* * Only once we have a METRICs sibling do we * need TopDown magic . /* * The load latency event X86_CONFIG ( . event = 0 xcd , . umask = 0 x01 ) on SPR * doesn ' t function quite right . As a work - around it needs to always be * co - scheduled with a auxiliary event X86_CONFIG ( . event = 0 x03 , . umask = 0 x82 ) . * The actual count of this second event is irrelevant it just needs * to be active to make the first event function correctly . * * In a group , the auxiliary event must be in front of the load latency * event . The rule is to simplify the implementation of the check . * That ' s because perf cannot have a complete group at the moment . if (require_mem_loads_aux_event(event) &&struct perf_event *leader = event->group_leader;struct perf_event *sibling = NULL;/* * When this memload event is also the first event ( no group * exists yet ) , then there is no aux event before it . if (leader == event)return -ENODATA;if (!is_mem_loads_aux_event(leader)) {if (is_mem_loads_aux_event(sibling))break ;if (list_entry_is_head(sibling, &leader->sibling_list, sibling_list))return -ENODATA;if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))return 0 ;if (x86_pmu.version < 3 )return -EINVAL;if (ret)return ret;return 0 ;/* * Currently , the only caller of this function is the atomic_switch_perf_msrs ( ) . * The host perf context helps to prepare the values of the real hardware for * a set of msrs that need to be switched atomically in a vmx transaction . * * For example , the pseudocode needed to add a new msr should look like : * * arr [ ( * nr ) + + ] = ( struct perf_guest_switch_msr ) { * . msr = the hardware msr address , * . host = the value the hardware has when it doesn ' t run a guest , * . guest = the value the hardware has when it runs a guest , * } ; * * These values have nothing to do with the emulated values the guest sees * when it uses { RD , WR } MSR , which should be handled by the KVM context , * specifically in the intel_pmu_ { get , set } _ msr ( ) . static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data;int global_ctrl, pebs_enable;/* * In addition to obeying exclude_guest / exclude_host , remove bits being * used for PEBS when running a guest , because PEBS writes to virtual * addresses ( not physical addresses ) . 0 ;struct perf_guest_switch_msr){if (!x86_pmu.ds_pebs)return arr;/* * If PMU counter has PEBS enabled it is not enough to * disable counter on a guest entry since PEBS memory * write can overshoot guest entry and corrupt guest * memory . Disabling PEBS solves the problem . * * Don ' t do this if the CPU already enforces it . if (x86_pmu.pebs_no_isolation) {struct perf_guest_switch_msr){0 ,return arr;if (!kvm_pmu || !x86_pmu.pebs_ept)return arr;struct perf_guest_switch_msr){unsigned long )cpuc->ds,if (x86_pmu.intel_cap.pebs_baseline) {struct perf_guest_switch_msr){struct perf_guest_switch_msr){if (arr[pebs_enable].host) {/* Disable guest PEBS if host PEBS is enabled. */ 0 ;else {/* Disable guest PEBS thoroughly for cross-mapped PEBS counters. */ /* Set hw GLOBAL_CTRL bits for PEBS counter when it runs for guest */ return arr;static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr, void *data)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;int idx;struct perf_event *event = cpuc->events[idx];0 ;if (!test_bit(idx, cpuc->active_mask))continue ;if (event->attr.exclude_host)else if (event->attr.exclude_guest)return arr;static void core_pmu_enable_event(struct perf_event *event)if (!event->attr.exclude_host)static void core_pmu_enable_all(int added)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);int idx;struct hw_perf_event *hwc = &cpuc->events[idx]->hw;if (!test_bit(idx, cpuc->active_mask) ||continue ;static int hsw_hw_config(struct perf_event *event)int ret = intel_pmu_hw_config(event);if (ret)return ret;if (!boot_cpu_has(X86_FEATURE_RTM) && !boot_cpu_has(X86_FEATURE_HLE))return 0 ;/* * IN_TX / IN_TX - CP filters are not supported by the Haswell PMU with * PEBS or in ANY thread mode . Since the results are non - sensical forbid * this combination . if ((event->hw.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)) &&0 ))return -EOPNOTSUPP;if (event_is_checkpointed(event)) {/* * Sampling of checkpointed events can cause situations where * the CPU constantly aborts because of a overflow , which is * then checkpointed back and ignored . Forbid checkpointing * for sampling . * * But still allow a long sampling period , so that perf stat * from KVM works . if (event->attr.sample_period > 0 &&0 x7fffffff)return -EOPNOTSUPP;return 0 ;static struct event_constraint counter0_constraint =0 , 0 x1);static struct event_constraint counter1_constraint =0 , 0 x2);static struct event_constraint counter0_1_constraint =0 , 0 x3);static struct event_constraint counter2_constraint =0 , 0 x4, 0 );static struct event_constraint fixed0_constraint =0 x00c0, 0 );static struct event_constraint fixed0_counter0_constraint =0 , 0 x100000001ULL);static struct event_constraint fixed0_counter0_1_constraint =0 , 0 x100000003ULL);static struct event_constraint counters_1_7_constraint =0 , 0 xfeULL);static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c;/* Handle special quirk on in_tx_checkpointed only in counter 2 */ if (event->hw.config & HSW_IN_TX_CHECKPOINTED) {if (c->idxmsk64 & (1 U << 2 ))return &counter2_constraint;return &emptyconstraint;return c;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)/* * Fixed counter 0 has less skid . * Force instruction : ppp in Fixed counter 0 if ((event->attr.precise_ip == 3 ) &&return &fixed0_constraint;return hsw_get_event_constraints(cpuc, idx, event);static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c;/* * The : ppp indicates the Precise Distribution ( PDist ) facility , which * is only supported on the GP counter 0 . If a : ppp event which is not * available on the GP counter 0 , error out . * Exception : Instruction PDIR is only available on the fixed counter 0 . if ((event->attr.precise_ip == 3 ) &&if (c->idxmsk64 & BIT_ULL(0 ))return &counter0_constraint;return &emptyconstraint;return c;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c;/* :ppp means to do reduced skid PEBS which is PMC0 only. */ if (event->attr.precise_ip == 3 )return &counter0_constraint;return c;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c;/* * : ppp means to do reduced skid PEBS , * which is available on PMC0 and fixed counter 0 . if (event->attr.precise_ip == 3 ) {/* Force instruction:ppp on PMC0 and Fixed counter 0 */ if (constraint_match(&fixed0_constraint, event->hw.config))return &fixed0_counter0_constraint;return &counter0_constraint;return c;static bool allow_tsx_force_abort = true ;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c = hsw_get_event_constraints(cpuc, idx, event);/* * Without TFA we must not use PMC3 . if (!allow_tsx_force_abort && test_bit(3 , c->idxmsk)) {1 ULL << 3 );return c;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);if (pmu->pmu_type == hybrid_big)return glc_get_event_constraints(cpuc, idx, event);else if (pmu->pmu_type == hybrid_small)return tnt_get_event_constraints(cpuc, idx, event);1 );return &emptyconstraint;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c;/* * The : ppp indicates the Precise Distribution ( PDist ) facility , which * is only supported on the GP counter 0 & 1 and Fixed counter 0 . * If a : ppp event which is not available on the above eligible counters , * error out . if (event->attr.precise_ip == 3 ) {/* Force instruction:ppp on PMC0, 1 and Fixed counter 0 */ if (constraint_match(&fixed0_constraint, event->hw.config)) {/* The fixed counter 0 doesn't support LBR event logging. */ if (branch_sample_counters(event))return &counter0_1_constraint;else return &fixed0_counter0_1_constraint;switch (c->idxmsk64 & 0 x3ull) {case 0 x1:return &counter0_constraint;case 0 x2:return &counter1_constraint;case 0 x3:return &counter0_1_constraint;return &emptyconstraint;return c;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct event_constraint *c;/* The Retire Latency is not supported by the fixed counter 0. */ if (event->attr.precise_ip &&/* * The Instruction PDIR is only available * on the fixed counter 0 . Error out for this case . if (event->attr.precise_ip == 3 )return &emptyconstraint;return &counters_1_7_constraint;return c;static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);if (pmu->pmu_type == hybrid_big)return rwc_get_event_constraints(cpuc, idx, event);if (pmu->pmu_type == hybrid_small)return cmt_get_event_constraints(cpuc, idx, event);1 );return &emptyconstraint;static int adl_hw_config(struct perf_event *event)struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);if (pmu->pmu_type == hybrid_big)return hsw_hw_config(event);else if (pmu->pmu_type == hybrid_small)return intel_pmu_hw_config(event);1 );return -EOPNOTSUPP;static enum intel_cpu_type adl_get_hybrid_cpu_type(void )return INTEL_CPU_TYPE_CORE;static inline bool erratum_hsw11(struct perf_event *event)return (event->hw.config & INTEL_ARCH_EVENT_MASK) ==0 xc0, .umask=0 x01);static struct event_constraint *struct cpu_hw_events *cpuc, int idx,struct perf_event *event)struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);if (pmu->pmu_type == hybrid_tiny)return cmt_get_event_constraints(cpuc, idx, event);return mtl_get_event_constraints(cpuc, idx, event);static int arl_h_hw_config(struct perf_event *event)struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);if (pmu->pmu_type == hybrid_tiny)return intel_pmu_hw_config(event);return adl_hw_config(event);/* * The HSW11 requires a period larger than 100 which is the same as the BDM11 . * A minimum period of 128 is enforced as well for the INST_RETIRED . ALL . * * The message ' interrupt took too long ' can be observed on any counter which * was armed with a period < 32 and two events expired in the same NMI . * A minimum period of 32 is enforced for the rest of the events . static void hsw_limit_period(struct perf_event *event, s64 *left)128 : 32 );/* * Broadwell : * * The INST_RETIRED . ALL period always needs to have lowest 6 bits cleared * ( BDM55 ) and it must not use a period smaller than 100 ( BDM11 ) . We combine * the two to enforce a minimum period of 128 ( the smallest value that has bits * 0 - 5 cleared and > = 100 ) . * * Because of how the code in x86_perf_event_set_period ( ) works , the truncation * of the lower 6 bits is ' harmless ' as we ' ll occasionally add a longer period * to make up for the ' lost ' events due to carrying the ' error ' in period_left . * * Therefore the effective ( average ) period matches the requested period , * despite coarser hardware granularity . static void bdw_limit_period(struct perf_event *event, s64 *left)if (erratum_hsw11(event)) {if (*left < 128 )128 ;0 x3fULL;static void nhm_limit_period(struct perf_event *event, s64 *left)32 LL);static void glc_limit_period(struct perf_event *event, s64 *left)if (event->attr.precise_ip == 3 )128 LL);"config:0-7" );"config:8-15" );"config:18" );"config:19" );"config:21" ); /* v3 + */ "config:23" );"config:24-31" );"config:32" );"config:33" );"config:36" ); /* v6 + */ "config1:0" ); /* PERF_CAPABILITIES.RDPMC_METRICS_CLEAR */ static ssize_t umask2_show(struct device *dev,struct device_attribute *attr,char *page)2;if (mask == ARCH_PERFMON_EVENTSEL_UMASK2)return sprintf(page, "config:8-15,40-47\n" );/* Roll back to the old format if umask2 is not supported. */ return sprintf(page, "config:8-15\n" );static struct device_attribute format_attr_umask2 =0444 , umask2_show, NULL);static struct attribute *format_evtsel_ext_attrs[] = {static umode_tstruct kobject *kobj, struct attribute *attr, int i)struct device *dev = kobj_to_dev(kobj);/* * The umask and umask2 have different formats but share the * same attr name . In update mode , the previous value of the * umask is unconditionally removed before is_visible . If * umask2 format is not enumerated , it ' s impossible to roll * back to the old format . * Does the check in umask2_show rather than is_visible . if (i == 0 )return attr->mode;if (i == 1 )return (mask & ARCH_PERFMON_EVENTSEL_EQ) ? attr->mode : 0 ;/* PERF_CAPABILITIES.RDPMC_METRICS_CLEAR */ if (i == 2 ) {union perf_capabilities intel_cap = hybrid(dev_get_drvdata(dev), intel_cap);return intel_cap.rdpmc_metrics_clear ? attr->mode : 0 ;return 0 ;static struct attribute *intel_arch_formats_attr[] = {char *page, u64 config)return x86_event_sysfs_show(page, config, event);static struct intel_shared_regs *allocate_shared_regs(int cpu)struct intel_shared_regs *regs;int i;sizeof (struct intel_shared_regs),if (regs) {/* * initialize the locks to keep lockdep happy for (i = 0 ; i < EXTRA_REG_MAX; i++)1 ;return regs;static struct intel_excl_cntrs *allocate_excl_cntrs(int cpu)struct intel_excl_cntrs *c;sizeof (struct intel_excl_cntrs),if (c) {1 ;return c;int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)if (is_hybrid() || x86_pmu.extra_regs || x86_pmu.lbr_sel_map) {if (!cpuc->shared_regs)goto err;if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA | PMU_FL_DYN_CONSTRAINT)) {sizeof (struct event_constraint);if (!cpuc->constraint_list)goto err_shared_regs;if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {if (!cpuc->excl_cntrs)goto err_constraint_list;0 ;return 0 ;return -ENOMEM;static int intel_pmu_cpu_prepare(int cpu)return intel_cpuc_prepare(&per_cpu(cpu_hw_events, cpu), cpu);static void flip_smm_bit(void *data)unsigned long set = *(unsigned long *)data;if (set > 0 ) {else {static void intel_pmu_check_counters_mask(u64 *cntr_mask,unsigned int bit;if (bit > INTEL_PMC_MAX_GENERIC) {1 , KERN_ERR "hw perf events %d > max(%d), clipping!" ,1 , 0 );if (bit > INTEL_PMC_MAX_FIXED) {1 , KERN_ERR "hw perf events fixed %d > max(%d), clipping!" ,1 , 0 );static void intel_pmu_check_event_constraints(struct event_constraint *event_constraints,static void intel_pmu_check_extra_regs(struct extra_reg *extra_regs);static inline bool intel_pmu_broken_perf_cap(void )/* The Perf Metric (Bit 15) is always cleared */ if (boot_cpu_data.x86_vfm == INTEL_METEORLAKE ||return true ;return false ;static void update_pmu_cap(struct pmu *pmu)unsigned int cntr, fixed_cntr, ecx, edx;union cpuid35_eax eax;union cpuid35_ebx ebx;if (ebx.split.umask2)if (ebx.split.eq)if (eax.split.cntr_subleaf) {if (eax.split.acr_subleaf) {/* The mask of the counters which can be reloaded */ /* The mask of the counters which can cause a reload of reloadable counters */ if (!intel_pmu_broken_perf_cap()) {/* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */ static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu)struct event_constraint)0 , pmu->cntr_mask64,0 , x86_pmu_num_counters(&pmu->pmu), 0 , 0 );if (pmu->intel_cap.perf_metrics)1 ULL << GLOBAL_CTRL_EN_PERF_METRICS;else 1 ULL << GLOBAL_CTRL_EN_PERF_METRICS);static struct x86_hybrid_pmu *find_hybrid_pmu_for_cpu(void )struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());enum intel_cpu_type cpu_type = c->topo.intel_type;int i;/* * This is running on a CPU model that is known to have hybrid * configurations . But the CPU told us it is not hybrid , shame * on it . There should be a fixup function provided for these * troublesome CPUs ( - > get_hybrid_cpu_type ) . if (cpu_type == INTEL_CPU_TYPE_UNKNOWN) {if (x86_pmu.get_hybrid_cpu_type)else return NULL;/* * This essentially just maps between the ' hybrid_cpu_type ' * and ' hybrid_pmu_type ' enums except for ARL - H processor * which needs to compare atom uarch native id since ARL - H * contains two different atom uarchs . for (i = 0 ; i < x86_pmu.num_hybrid_pmus; i++) {enum hybrid_pmu_type pmu_type = x86_pmu.hybrid_pmu[i].pmu_type;if (cpu_type == INTEL_CPU_TYPE_CORE && pmu_type == hybrid_big)return &x86_pmu.hybrid_pmu[i];if (cpu_type == INTEL_CPU_TYPE_ATOM) {if (x86_pmu.num_hybrid_pmus == 2 && pmu_type == hybrid_small)return &x86_pmu.hybrid_pmu[i];if (native_id == INTEL_ATOM_SKT_NATIVE_ID && pmu_type == hybrid_small)return &x86_pmu.hybrid_pmu[i];if (native_id == INTEL_ATOM_CMT_NATIVE_ID && pmu_type == hybrid_tiny)return &x86_pmu.hybrid_pmu[i];return NULL;static bool init_hybrid_pmu(int cpu)struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);struct x86_hybrid_pmu *pmu = find_hybrid_pmu_for_cpu();if (WARN_ON_ONCE(!pmu || (pmu->pmu.type == -1 ))) {return false ;/* Only check and dump the PMU information for the first CPU */ if (!cpumask_empty(&pmu->supported_cpus))goto end;if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))if (!check_hw_exists(&pmu->pmu, pmu->cntr_mask, pmu->fixed_cntr_mask))return false ;"%s PMU driver: " , pmu->name);"\n" );return true ;static void intel_pmu_cpu_starting(int cpu)struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);int core_id = topology_core_id(cpu);int i;if (is_hybrid() && !init_hybrid_pmu(cpu))return ;/* * Deal with CPUs that don ' t clear their LBRs on power - up , and that may * even boot with LBRs enabled . if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && x86_pmu.lbr_nr)if (x86_pmu.flags & PMU_FL_TFA) {0 ULL;false );if (x86_pmu.version > 1 )/* * Disable perf metrics if any added CPU doesn ' t support it . * * Turn off the check for a hybrid architecture , because the * architecture MSR , MSR_IA32_PERF_CAPABILITIES , only indicate * the architecture features . The perf metrics is a model - specific * feature for now . The corresponding bit should always be 0 on * a hybrid platform , e . g . , Alder Lake . if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics) {union perf_capabilities perf_cap;if (!perf_cap.perf_metrics) {0 ;1 ULL << GLOBAL_CTRL_EN_PERF_METRICS);if (!cpuc->shared_regs)return ;if (!(x86_pmu.flags & PMU_FL_NO_HT_SHARING)) {struct intel_shared_regs *pc;if (pc && pc->core_id == core_id) {0 ] = cpuc->shared_regs;break ;if (x86_pmu.lbr_sel_map)if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {struct cpu_hw_events *sibling;struct intel_excl_cntrs *c;if (c && c->core_id == core_id) {1 ] = cpuc->excl_cntrs;if (!sibling->excl_thread_id)1 ;break ;static void free_excl_cntrs(struct cpu_hw_events *cpuc)struct intel_excl_cntrs *c;if (c) {if (c->core_id == -1 || --c->refcnt == 0 )static void intel_pmu_cpu_dying(int cpu)void intel_cpuc_finish(struct cpu_hw_events *cpuc)struct intel_shared_regs *pc;if (pc) {if (pc->core_id == -1 || --pc->refcnt == 0 )static void intel_pmu_cpu_dead(int cpu)struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);if (is_hybrid() && cpuc->pmu)static void intel_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx,struct task_struct *task, bool sched_in)static int intel_pmu_check_period(struct perf_event *event, u64 value)return intel_pmu_has_bts_period(event, value) ? -EINVAL : 0 ;static void intel_aux_output_init(void )/* Refer also intel_pmu_aux_output_match() */ if (x86_pmu.intel_cap.pebs_output_pt_available)static int intel_pmu_aux_output_match(struct perf_event *event)/* intel_pmu_assign_event() is needed, refer intel_aux_output_init() */ if (!x86_pmu.intel_cap.pebs_output_pt_available)return 0 ;return is_intel_pt_event(event);static void intel_pmu_filter(struct pmu *pmu, int cpu, bool *ret)struct x86_hybrid_pmu *hpmu = hybrid_pmu(pmu);"config1:0-63" );"config1:0-15" );"config1:0-23" );"config1:0-63" );static struct attribute *intel_arch3_formats_attr[] = {static struct attribute *hsw_format_attr[] = {static struct attribute *nhm_format_attr[] = {static struct attribute *slm_format_attr[] = {static struct attribute *cmt_format_attr[] = {static struct attribute *skl_format_attr[] = {static __initconst const struct x86_pmu core_pmu = {"core" ,1 ,/* * Intel PMCs cannot be accessed sanely above 32 - bit width , * so we install an artificial 1 < < 31 period regardless of * the generic event period : 1 ULL<<31 ) - 1 ,/* * Virtual ( or funny metal ) CPU can define x86_pmu . extra_regs * together with PMU version 1 and thus be using core_pmu with * shared_regs . We need following callbacks here to allocate * it properly . static __initconst const struct x86_pmu intel_pmu = {"Intel" ,1 ,/* * Intel PMCs cannot be accessed sanely above 32 bit width , * so we install an artificial 1 < < 31 period regardless of * the generic event period : 1 ULL << 31 ) - 1 ,/* * SMM has access to all 4 rings and while traditionally SMM code only * ran in CPL0 , 2021 - era firmware is starting to make use of CPL3 in SMM . * * Since the EVENTSEL . { USR , OS } CPL filtering makes no distinction * between SMM or not , this results in what should be pure userspace * counters including SMM data . * * This is a clear privilege issue , therefore globally disable * counting SMM by default . 1 ,static __init void intel_clovertown_quirk(void )/* * PEBS is unreliable due to : * * AJ67 - PEBS may experience CPL leaks * AJ68 - PEBS PMI may be delayed by one event * AJ69 - GLOBAL_STATUS [ 62 ] will only be set when DEBUGCTL [ 12 ] * AJ106 - FREEZE_LBRS_ON_PMI doesn ' t work in combination with PEBS * * AJ67 could be worked around by restricting the OS / USR flags . * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI . * * AJ106 could possibly be worked around by not allowing LBR * usage from PEBS , including the fixup . * AJ68 could possibly be worked around by always programming * a pebs_event_reset [ 0 ] value and coping with the lost events . * * But taken together it might just make sense to not enable PEBS on * these chips . "PEBS disabled due to CPU errata\n" );0 ;static const struct x86_cpu_id isolation_ucodes[] = {3 , 3 , 0 x0000001f),1 , 1 , 0 x0000001e),1 , 1 , 0 x00000015),2 , 2 , 0 x00000037),4 , 4 , 0 x0000000a),4 , 4 , 0 x00000023),1 , 1 , 0 x00000014),2 , 2 , 0 x00000010),3 , 3 , 0 x07000009),4 , 4 , 0 x0f000009),5 , 5 , 0 x0e000002),1 , 1 , 0 x0b000014),3 , 3 , 0 x00000021),4 , 7 , 0 x00000000),11 , 11 , 0 x00000000),3 , 3 , 0 x0000007c),3 , 3 , 0 x0000007c),9 , 13 , 0 x0000004e),9 , 12 , 0 x0000004e),static void intel_check_pebs_isolation(void )static __init void intel_pebs_isolation_quirk(void )static const struct x86_cpu_id pebs_ucodes[] = {7 , 7 , 0 x00000028),6 , 6 , 0 x00000618),7 , 7 , 0 x0000070c),static bool intel_snb_pebs_broken(void )return !x86_match_min_microcode_rev(pebs_ucodes);static void intel_snb_check_microcode(void )if (intel_snb_pebs_broken() == x86_pmu.pebs_broken)return ;/* * Serialized by the microcode lock . . if (x86_pmu.pebs_broken) {"PEBS enabled due to microcode update\n" );0 ;else {"PEBS disabled due to CPU errata, please upgrade microcode\n" );1 ;static bool is_lbr_from(unsigned long msr)unsigned long lbr_from_nr = x86_pmu.lbr_from + x86_pmu.lbr_nr;return x86_pmu.lbr_from <= msr && msr < lbr_from_nr;/* * Under certain circumstances , access certain MSR may cause # GP . * The function tests if the input MSR can be safely accessed . static bool check_msr(unsigned long msr, u64 mask)/* * Disable the check for real HW , so we don ' t * mess with potentially enabled registers : if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))return true ;/* * Read the current value , change it and read it back to see if it * matches , this is needed to detect certain hardware emulators * ( qemu / kvm ) that don ' t trap on the MSR access and always return 0 s . if (rdmsrq_safe(msr, &val_old))return false ;/* * Only change the bits which can be updated by wrmsrq . if (is_lbr_from(msr))if (wrmsrq_safe(msr, val_tmp) ||return false ;/* * Quirk only affects validation in wrmsr ( ) , so wrmsrq ( ) ' s value * should equal rdmsrq ( ) ' s even with the quirk . if (val_new != val_tmp)return false ;if (is_lbr_from(msr))/* Here it's sure that the MSR can be safely accessed. * Restore the old value and return . return true ;static __init void intel_sandybridge_quirk(void )static const struct { int id; char *name; } intel_arch_events_map[] __initconst = {"cpu cycles" },"instructions" },"bus cycles" },"cache references" },"cache misses" },"branch instructions" },"branch misses" },static __init void intel_arch_events_quirk(void )int bit;/* disable event that reported as not present by cpuid */ 0 ;"CPUID marked event: \'%s\' unavailable\n" ,static __init void intel_nehalem_quirk(void )union cpuid10_ebx ebx;if (ebx.split.no_branch_misses_retired) {/* * Erratum AAJ80 detected , we work it around by using * the BR_MISP_EXEC . ANY event . This will over - count * branch - misses , but it ' s still much better than the * architectural event which is often completely bogus : 0 x7f89;0 ;"CPU erratum AAJ80 worked around\n" );/* * enable software workaround for errata : * SNB : BJ122 * IVB : BV98 * HSW : HSD29 * * Only needed when HT is enabled . However detecting * if HT is enabled is difficult ( model specific ) . So instead , * we enable the workaround in the early boot , and verify if * it is needed in a later initcall phase once we have valid * topology information to check if HT is actually enabled static __init void intel_ht_bug(void )"event=0xcd,umask=0x1,ldlat=3" );"event=0xd0,umask=0x82" )/* Haswell special events */ "event=0xc9,umask=0x1" );"event=0xc9,umask=0x2" );"event=0xc9,umask=0x4" );"event=0x54,umask=0x2" );"event=0x54,umask=0x1" );"event=0xc8,umask=0x1" );"event=0xc8,umask=0x2" );"event=0xc8,umask=0x4" );"event=0x54,umask=0x2" );"event=0x54,umask=0x1" );"event=0x3c,in_tx=1" );"event=0x3c,in_tx=1,in_tx_cp=1" );static struct attribute *hsw_events_attrs[] = {static struct attribute *hsw_mem_events_attrs[] = {static struct attribute *hsw_tsx_events_attrs[] = {"event=0x54,umask=0x80" );"event=0x54,umask=0x2" );"event=0x54,umask=0x80" );"event=0x54,umask=0x2" );static struct attribute *icl_events_attrs[] = {static struct attribute *icl_td_events_attrs[] = {static struct attribute *icl_tsx_events_attrs[] = {"event=0xcd,umask=0x2" );"event=0x03,umask=0x82" );static struct attribute *glc_events_attrs[] = {static struct attribute *glc_td_events_attrs[] = {static struct attribute *glc_tsx_events_attrs[] = {static ssize_t freeze_on_smi_show(struct device *cdev,struct device_attribute *attr,char *buf)return sprintf(buf, "%lu\n" , x86_pmu.attr_freeze_on_smi);static DEFINE_MUTEX(freeze_on_smi_mutex);static ssize_t freeze_on_smi_store(struct device *cdev,struct device_attribute *attr,const char *buf, size_t count)unsigned long val;0 , &val);if (ret)return ret;if (val > 1 )return -EINVAL;if (x86_pmu.attr_freeze_on_smi == val)goto done;1 );return count;static void update_tfa_sched(void *ignored)struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);/* * check if PMC3 is used * and if so force schedule out for all event types all contexts if (test_bit(3 , cpuc->active_mask))static ssize_t show_sysctl_tfa(struct device *cdev,struct device_attribute *attr,char *buf)return snprintf(buf, 40 , "%d\n" , allow_tsx_force_abort);static ssize_t set_sysctl_tfa(struct device *cdev,struct device_attribute *attr,const char *buf, size_t count)bool val;if (ret)return ret;/* no change */ if (val == allow_tsx_force_abort)return count;1 );return count;static DEVICE_ATTR_RW(freeze_on_smi);static ssize_t branches_show(struct device *cdev,struct device_attribute *attr,char *buf)return snprintf(buf, PAGE_SIZE, "%d\n" , x86_pmu.lbr_nr);static DEVICE_ATTR_RO(branches);static ssize_t branch_counter_nr_show(struct device *cdev,struct device_attribute *attr,char *buf)return snprintf(buf, PAGE_SIZE, "%d\n" , fls(x86_pmu.lbr_counters));static DEVICE_ATTR_RO(branch_counter_nr);static ssize_t branch_counter_width_show(struct device *cdev,struct device_attribute *attr,char *buf)return snprintf(buf, PAGE_SIZE, "%d\n" , LBR_INFO_BR_CNTR_BITS);static DEVICE_ATTR_RO(branch_counter_width);static struct attribute *lbr_attrs[] = {static umode_tstruct kobject *kobj, struct attribute *attr, int i)/* branches */ if (i == 0 )return x86_pmu.lbr_nr ? attr->mode : 0 ;return (x86_pmu.flags & PMU_FL_BR_CNTR) ? attr->mode : 0 ;static char pmu_name_str[30 ];static DEVICE_STRING_ATTR_RO(pmu_name, 0444 , pmu_name_str);static struct attribute *intel_pmu_caps_attrs[] = {static DEVICE_ATTR(allow_tsx_force_abort, 0644 ,static struct attribute *intel_pmu_attrs[] = {static umode_tstruct kobject *kobj, struct attribute *attr, int i)if (attr == &dev_attr_allow_tsx_force_abort.attr)return x86_pmu.flags & PMU_FL_TFA ? attr->mode : 0 ;return attr->mode;static umode_tstruct kobject *kobj, struct attribute *attr, int i)return boot_cpu_has(X86_FEATURE_RTM) ? attr->mode : 0 ;static umode_tstruct kobject *kobj, struct attribute *attr, int i)return x86_pmu.ds_pebs ? attr->mode : 0 ;static umode_tstruct kobject *kobj, struct attribute *attr, int i)if (attr == &event_attr_mem_ld_aux.attr.attr)return x86_pmu.flags & PMU_FL_MEM_LOADS_AUX ? attr->mode : 0 ;return pebs_is_visible(kobj, attr, i);static umode_tstruct kobject *kobj, struct attribute *attr, int i)return x86_pmu.version >= 2 ? attr->mode : 0 ;static umode_tstruct kobject *kobj, struct attribute *attr, int i)/* * Hide the perf metrics topdown events * if the feature is not enumerated . if (x86_pmu.num_topdown_events)return x86_pmu.intel_cap.perf_metrics ? attr->mode : 0 ;return attr->mode;"config2:0-63" );static struct attribute *format_acr_attrs[] = {static umode_tstruct kobject *kobj, struct attribute *attr, int i)struct device *dev = kobj_to_dev(kobj);return intel_pmu_has_acr(dev_get_drvdata(dev)) ? attr->mode : 0 ;static struct attribute_group group_events_td = {"events" ,static struct attribute_group group_events_mem = {"events" ,static struct attribute_group group_events_tsx = {"events" ,static struct attribute_group group_caps_gen = {"caps" ,static struct attribute_group group_caps_lbr = {"caps" ,static struct attribute_group group_format_extra = {"format" ,static struct attribute_group group_format_extra_skl = {"format" ,static struct attribute_group group_format_evtsel_ext = {"format" ,static struct attribute_group group_format_acr = {"format" ,static struct attribute_group group_default = {static const struct attribute_group *attr_update[] = {"event=0x00,umask=0x4" , hybrid_big);"event=0xc2,umask=0x0;event=0x00,umask=0x80" , hybrid_big_small);"event=0x73,umask=0x0;event=0x00,umask=0x81" , hybrid_big_small);"event=0x71,umask=0x0;event=0x00,umask=0x82" , hybrid_big_small);"event=0x74,umask=0x0;event=0x00,umask=0x83" , hybrid_big_small);"event=0x00,umask=0x84" , hybrid_big);"event=0x00,umask=0x85" , hybrid_big);"event=0x00,umask=0x86" , hybrid_big);"event=0x00,umask=0x87" , hybrid_big);static struct attribute *adl_hybrid_events_attrs[] = {"event=0xc2,umask=0x02;event=0x00,umask=0x80" , hybrid_big_small);"event=0x9c,umask=0x01;event=0x00,umask=0x82" , hybrid_big_small);"event=0xa4,umask=0x02;event=0x00,umask=0x83" , hybrid_big_small);static struct attribute *lnl_hybrid_events_attrs[] = {/* The event string must be in PMU IDX order. */ "event=0xc2,umask=0x02;event=0x00,umask=0x80;event=0xc2,umask=0x0" ,"event=0x73,umask=0x0;event=0x00,umask=0x81;event=0x73,umask=0x0" ,"event=0x9c,umask=0x01;event=0x00,umask=0x82;event=0x71,umask=0x0" ,"event=0xa4,umask=0x02;event=0x00,umask=0x83;event=0x74,umask=0x0" ,static struct attribute *arl_h_hybrid_events_attrs[] = {/* Must be in IDX order */ "event=0xd0,umask=0x5,ldlat=3;event=0xcd,umask=0x1,ldlat=3" , hybrid_big_small);"event=0xd0,umask=0x6;event=0xcd,umask=0x2" , hybrid_big_small);"event=0x03,umask=0x82" , hybrid_big);static struct attribute *adl_hybrid_mem_attrs[] = {static struct attribute *mtl_hybrid_mem_attrs[] = {"event=0xd0,umask=0x5,ldlat=3;event=0xcd,umask=0x1,ldlat=3;event=0xd0,umask=0x5,ldlat=3" ,"event=0xd0,umask=0x6;event=0xcd,umask=0x2;event=0xd0,umask=0x6" ,static struct attribute *arl_h_hybrid_mem_attrs[] = {"event=0xc9,umask=0x1" , hybrid_big);"event=0xc9,umask=0x2" , hybrid_big);"event=0xc9,umask=0x4" , hybrid_big);"event=0x54,umask=0x1" , hybrid_big);"event=0x3c,in_tx=1" , hybrid_big);"event=0x3c,in_tx=1,in_tx_cp=1" , hybrid_big);"event=0x54,umask=0x80" , hybrid_big);"event=0x54,umask=0x2" , hybrid_big);static struct attribute *adl_hybrid_tsx_attrs[] = {#define ADL_HYBRID_RTM_FORMAT_ATTR \#define ADL_HYBRID_FORMAT_ATTR \static struct attribute *adl_hybrid_extra_attr_rtm[] = {static struct attribute *adl_hybrid_extra_attr[] = {static struct attribute *mtl_hybrid_extra_attr_rtm[] = {static struct attribute *mtl_hybrid_extra_attr[] = {static bool is_attr_for_this_pmu(struct kobject *kobj, struct attribute *attr)struct device *dev = kobj_to_dev(kobj);struct x86_hybrid_pmu *pmu =struct x86_hybrid_pmu, pmu);struct perf_pmu_events_hybrid_attr *pmu_attr =struct perf_pmu_events_hybrid_attr, attr.attr);return pmu->pmu_type & pmu_attr->pmu_type;static umode_t hybrid_events_is_visible(struct kobject *kobj,struct attribute *attr, int i)return is_attr_for_this_pmu(kobj, attr) ? attr->mode : 0 ;static inline int hybrid_find_supported_cpu(struct x86_hybrid_pmu *pmu)int cpu = cpumask_first(&pmu->supported_cpus);return (cpu >= nr_cpu_ids) ? -1 : cpu;static umode_t hybrid_tsx_is_visible(struct kobject *kobj,struct attribute *attr, int i)struct device *dev = kobj_to_dev(kobj);struct x86_hybrid_pmu *pmu =struct x86_hybrid_pmu, pmu);int cpu = hybrid_find_supported_cpu(pmu);return (cpu >= 0 ) && is_attr_for_this_pmu(kobj, attr) && cpu_has(&cpu_data(cpu), X86_FEATURE_RTM) ? attr->mode : 0 ;static umode_t hybrid_format_is_visible(struct kobject *kobj,struct attribute *attr, int i)struct device *dev = kobj_to_dev(kobj);struct x86_hybrid_pmu *pmu =struct x86_hybrid_pmu, pmu);struct perf_pmu_format_hybrid_attr *pmu_attr =struct perf_pmu_format_hybrid_attr, attr.attr);int cpu = hybrid_find_supported_cpu(pmu);return (cpu >= 0 ) && (pmu->pmu_type & pmu_attr->pmu_type) ? attr->mode : 0 ;static umode_t hybrid_td_is_visible(struct kobject *kobj,struct attribute *attr, int i)struct device *dev = kobj_to_dev(kobj);struct x86_hybrid_pmu *pmu =struct x86_hybrid_pmu, pmu);if (!is_attr_for_this_pmu(kobj, attr))return 0 ;/* Only the big core supports perf metrics */ if (pmu->pmu_type == hybrid_big)return pmu->intel_cap.perf_metrics ? attr->mode : 0 ;return attr->mode;static struct attribute_group hybrid_group_events_td = {"events" ,static struct attribute_group hybrid_group_events_mem = {"events" ,static struct attribute_group hybrid_group_events_tsx = {"events" ,static struct attribute_group hybrid_group_format_extra = {"format" ,static ssize_t intel_hybrid_get_attr_cpus(struct device *dev,struct device_attribute *attr,char *buf)struct x86_hybrid_pmu *pmu =struct x86_hybrid_pmu, pmu);return cpumap_print_to_pagebuf(true , buf, &pmu->supported_cpus);static DEVICE_ATTR(cpus, S_IRUGO, intel_hybrid_get_attr_cpus, NULL);static struct attribute *intel_hybrid_cpus_attrs[] = {static struct attribute_group hybrid_group_cpus = {static const struct attribute_group *hybrid_attr_update[] = {static struct attribute *empty_attrs;static void intel_pmu_check_event_constraints(struct event_constraint *event_constraints,struct event_constraint *c;if (!event_constraints)return ;/* * event on fixed counter2 ( REF_CYCLES ) only works on this * counter , so do not extend mask to generic counters /* * Don ' t extend the topdown slots and metrics * events to the generic counters . if (c->idxmsk64 & INTEL_PMC_MSK_TOPDOWN) {/* * Disable topdown slots and metrics events , * if slots event is not in CPUID . if (!(INTEL_PMC_MSK_FIXED_SLOTS & intel_ctrl))0 ;continue ;if (c->cmask == FIXED_EVENT_FLAGS) {/* Disabled fixed counters which are not in CPUID */ /* * Don ' t extend the pseudo - encoding to the * generic counters if (!use_fixed_pseudo_encoding(c->code))static void intel_pmu_check_extra_regs(struct extra_reg *extra_regs)struct extra_reg *er;/* * Access extra MSR may cause # GP under certain circumstances . * E . g . KVM doesn ' t support offcore event * Check all extra_regs here . if (!extra_regs)return ;for (er = extra_regs; er->msr; er++) {0 x11UL);/* Disable LBR select mapping */ if ((er->idx == EXTRA_REG_LBR) && !er->extra_msr_access)static inline int intel_pmu_v6_addr_offset(int index, bool eventsel)return MSR_IA32_PMC_V6_STEP * index;static const struct { enum hybrid_pmu_type id; char *name; } intel_hybrid_pmu_type_map[] __initconst = {"cpu_atom" },"cpu_core" },"cpu_lowpower" },static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus)unsigned long pmus_mask = pmus;struct x86_hybrid_pmu *pmu;int idx = 0 , bit;sizeof (struct x86_hybrid_pmu),if (!x86_pmu.hybrid_pmu)return -ENOMEM;struct event_constraint)0 , pmu->cntr_mask64,0 , x86_pmu_num_counters(&pmu->pmu), 0 , 0 );if (pmu->pmu_type & hybrid_small_tiny) {0 ;true ;else if (pmu->pmu_type & hybrid_big) {1 ;true ;return 0 ;static __always_inline void intel_pmu_ref_cycles_ext(void )if (!(x86_pmu.events_maskl & (INTEL_PMC_MSK_FIXED_REF_CYCLES >> INTEL_PMC_IDX_FIXED)))0 x013c;static __always_inline void intel_pmu_init_glc(struct pmu *pmu)true ;true ;true ;0 xc9, .umask=0 x04);true ;8 ;sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));static __always_inline void intel_pmu_init_grt(struct pmu *pmu)true ;true ;true ;true ;sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));1 ;static __always_inline void intel_pmu_init_lnc(struct pmu *pmu)static __always_inline void intel_pmu_init_skt(struct pmu *pmu)int intel_pmu_init(void )struct attribute **extra_skl_attr = &empty_attrs;struct attribute **extra_attr = &empty_attrs;struct attribute **td_attr = &empty_attrs;struct attribute **mem_attr = &empty_attrs;struct attribute **tsx_attr = &empty_attrs;union cpuid10_edx edx;union cpuid10_eax eax;union cpuid10_ebx ebx;unsigned int fixed_mask;bool pmem = false ;int version, i;char *name;struct x86_hybrid_pmu *pmu;/* Architectural Perfmon was introduced starting with Core "Yonah" */ if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {switch (boot_cpu_data.x86) {case 6 :if (boot_cpu_data.x86_vfm < INTEL_CORE_YONAH)return p6_pmu_init();break ;case 11 :return knc_pmu_init();case 15 :return p4_pmu_init();"unsupported CPU family %d model %d " ,return -ENODEV;/* * Check whether the Architectural PerfMon supports * Branch Misses Retired hw_event or not . 10 , &eax.full, &ebx.full, &fixed_mask, &edx.full);if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT)return -ENODEV;if (version < 2 )else 1 , 0 );1 ULL << eax.split.bit_width) - 1 ;/* * Quirk : v2 perfmon does not report fixed - purpose events , so * assume at least 3 events , when not running in a hypervisor : if (version > 1 && version < 5 ) {int assume = 3 * !boot_cpu_has(X86_FEATURE_HYPERVISOR);int )edx.split.num_counters_fixed, assume) - 1 , 0 );else if (version >= 5 )if (boot_cpu_has(X86_FEATURE_PDCM)) {if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) {if (boot_cpu_has(X86_FEATURE_ARCH_LBR))/* Install first, so it runs last */ if (version >= 5 ) {if (x86_pmu.intel_cap.anythread_deprecated)" AnyThread deprecated, " );/* * Many features on and after V6 require dynamic constraint , * e . g . , Arch PEBS , ACR . if (version >= 6 )/* * Install the hw - cache - events table : switch (boot_cpu_data.x86_vfm) {case INTEL_CORE_YONAH:"Core events, " );"core" ;break ;case INTEL_CORE2_MEROM:case INTEL_CORE2_MEROM_L:case INTEL_CORE2_PENRYN:case INTEL_CORE2_DUNNINGTON:sizeof (hw_cache_event_ids));"Core2 events, " );"core2" ;break ;case INTEL_NEHALEM:case INTEL_NEHALEM_EP:case INTEL_NEHALEM_EX:sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));/* UOPS_ISSUED.STALLED_CYCLES */ 0 x0e, .umask=0 x01, .inv=1 , .cmask=1 );/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ 0 xb1, .umask=0 x3f, .inv=1 , .cmask=1 );1 ;"Nehalem events, " );"nehalem" ;break ;case INTEL_ATOM_BONNELL:case INTEL_ATOM_BONNELL_MID:case INTEL_ATOM_SALTWELL:case INTEL_ATOM_SALTWELL_MID:case INTEL_ATOM_SALTWELL_TABLET:sizeof (hw_cache_event_ids));"Atom events, " );"bonnell" ;break ;case INTEL_ATOM_SILVERMONT:case INTEL_ATOM_SILVERMONT_D:case INTEL_ATOM_SILVERMONT_MID:case INTEL_ATOM_AIRMONT:case INTEL_ATOM_SILVERMONT_MID2:sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));"Silvermont events, " );"silvermont" ;break ;case INTEL_ATOM_GOLDMONT:case INTEL_ATOM_GOLDMONT_D:sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));/* * It ' s recommended to use CPU_CLK_UNHALTED . CORE_P + NPEBS * for precise cycles . * : pp is identical to : ppp true ;true ;"Goldmont events, " );"goldmont" ;break ;case INTEL_ATOM_GOLDMONT_PLUS:sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));/* * It ' s recommended to use CPU_CLK_UNHALTED . CORE_P + NPEBS * for precise cycles . true ;true ;0 ULL;/* Goldmont Plus has 4-wide pipeline */ "4" ;"Goldmont plus events, " );"goldmont_plus" ;break ;case INTEL_ATOM_TREMONT_D:case INTEL_ATOM_TREMONT:case INTEL_ATOM_TREMONT_L:true ;sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));1 ;/* * It ' s recommended to use CPU_CLK_UNHALTED . CORE_P + NPEBS * for precise cycles . true ;true ;"Tremont events, " );"Tremont" ;break ;case INTEL_ATOM_GRACEMONT:"Gracemont events, " );"gracemont" ;break ;case INTEL_ATOM_CRESTMONT:case INTEL_ATOM_CRESTMONT_X:"Crestmont events, " );"crestmont" ;break ;case INTEL_ATOM_DARKMONT_X:"Darkmont events, " );"darkmont" ;break ;case INTEL_WESTMERE:case INTEL_WESTMERE_EP:case INTEL_WESTMERE_EX:sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));/* UOPS_ISSUED.STALLED_CYCLES */ 0 x0e, .umask=0 x01, .inv=1 , .cmask=1 );/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ 0 xb1, .umask=0 x3f, .inv=1 , .cmask=1 );"Westmere events, " );"westmere" ;break ;case INTEL_SANDYBRIDGE:case INTEL_SANDYBRIDGE_X:sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));if (boot_cpu_data.x86_vfm == INTEL_SANDYBRIDGE_X)else /* all extra regs are per-cpu when HT is on */ /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ 0 x0e, .umask=0 x01, .inv=1 , .cmask=1 );/* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/ 0 xb1, .umask=0 x01, .inv=1 , .cmask=1 );"SandyBridge events, " );"sandybridge" ;break ;case INTEL_IVYBRIDGE:case INTEL_IVYBRIDGE_X:sizeof (hw_cache_event_ids));/* dTLB-load-misses on IVB is different than SNB */ 0 x8108; /* DTLB_LOAD_MISSES.DEMAND_LD_MISS_CAUSES_A_WALK */ sizeof (hw_cache_extra_regs));true ;if (boot_cpu_data.x86_vfm == INTEL_IVYBRIDGE_X)else /* all extra regs are per-cpu when HT is on */ /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ 0 x0e, .umask=0 x01, .inv=1 , .cmask=1 );"IvyBridge events, " );"ivybridge" ;break ;case INTEL_HASWELL:case INTEL_HASWELL_X:case INTEL_HASWELL_L:case INTEL_HASWELL_G:true ;sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));true ;/* all extra regs are per-cpu when HT is on */ true ;"Haswell events, " );"haswell" ;break ;case INTEL_BROADWELL:case INTEL_BROADWELL_D:case INTEL_BROADWELL_G:case INTEL_BROADWELL_X:true ;sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));/* L3_MISS_LOCAL_DRAM is BIT(26) in Broadwell */ MISS|true ;/* all extra regs are per-cpu when HT is on */ "Broadwell events, " );"broadwell" ;break ;case INTEL_XEON_PHI_KNL:case INTEL_XEON_PHI_KNM:sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));/* all extra regs are per-cpu when HT is on */ "Knights Landing/Mill events, " );"knights-landing" ;break ;case INTEL_SKYLAKE_X:true ;case INTEL_SKYLAKE_L:case INTEL_SKYLAKE:case INTEL_KABYLAKE_L:case INTEL_KABYLAKE:case INTEL_COMETLAKE_L:case INTEL_COMETLAKE:true ;sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));/* INT_MISC.RECOVERY_CYCLES has umask 1 in Skylake */ "event=0xd,umask=0x1,cmask=1" ;"event=0xd,umask=0x1,cmask=1,any=1" ;true ;/* all extra regs are per-cpu when HT is on */ /* * Processors with CPUID . RTM_ALWAYS_ABORT have TSX deprecated by default . * TSX force abort hooks are not required on these systems . Only deploy * workaround when microcode has not enabled X86_FEATURE_RTM_ALWAYS_ABORT . if (boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT) &&"Skylake events, " );"skylake" ;break ;case INTEL_ICELAKE_X:case INTEL_ICELAKE_D:1 ;true ;case INTEL_ICELAKE_L:case INTEL_ICELAKE:case INTEL_TIGERLAKE_L:case INTEL_TIGERLAKE:case INTEL_ROCKETLAKE:true ;sizeof (hw_cache_event_ids));sizeof (hw_cache_extra_regs));1 ;true ;0 xc9, .umask=0 x04);true ;4 ;"Icelake events, " );"icelake" ;break ;case INTEL_SAPPHIRERAPIDS_X:case INTEL_EMERALDRAPIDS_X:"Sapphire Rapids events, " );"sapphire_rapids" ;goto glc_common;case INTEL_GRANITERAPIDS_X:case INTEL_GRANITERAPIDS_D:"Granite Rapids events, " );"granite_rapids" ;1 ;true );break ;case INTEL_ALDERLAKE:case INTEL_ALDERLAKE_L:case INTEL_RAPTORLAKE:case INTEL_RAPTORLAKE_P:case INTEL_RAPTORLAKE_S:/* * Alder Lake has 2 types of CPU , core and atom . * * Initialize the common PerfMon capabilities here . /* Initialize big core specific PerfMon capabilities.*/ if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) {2 ;0 x3;1 ;0 x1;else {/* * Quirk : For some Alder Lake machine , when all E - cores are disabled in * a BIOS , the leaf 0 xA will enumerate all counters of P - cores . However , * the X86_FEATURE_HYBRID_CPU is still set . The above codes will * mistakenly add extra counters for P - cores . Correct the number of * counters here . if ((x86_pmu_num_counters(&pmu->pmu) > 8 ) || (x86_pmu_num_counters_fixed(&pmu->pmu) > 4 )) {struct event_constraint)0 , pmu->cntr_mask64,0 , x86_pmu_num_counters(&pmu->pmu), 0 , 0 );/* Initialize Atom core specific PerfMon capabilities.*/ "Alderlake Hybrid events, " );"alderlake_hybrid" ;break ;case INTEL_METEORLAKE:case INTEL_METEORLAKE_L:case INTEL_ARROWLAKE_U:/* Initialize big core specific PerfMon capabilities.*/ /* Initialize Atom core specific PerfMon capabilities.*/ "Meteorlake Hybrid events, " );"meteorlake_hybrid" ;break ;case INTEL_PANTHERLAKE_L:"Pantherlake Hybrid events, " );"pantherlake_hybrid" ;goto lnl_common;case INTEL_LUNARLAKE_M:case INTEL_ARROWLAKE:"Lunarlake Hybrid events, " );"lunarlake_hybrid" ;/* Initialize big core specific PerfMon capabilities.*/ /* Initialize Atom core specific PerfMon capabilities.*/ break ;case INTEL_ARROWLAKE_H:/* Initialize big core specific PerfMon capabilities. */ /* Initialize Atom core specific PerfMon capabilities. */ /* Initialize Lower Power Atom specific PerfMon capabilities. */ "ArrowLake-H Hybrid events, " );"arrowlake_h_hybrid" ;break ;default :switch (x86_pmu.version) {case 1 :"generic architected perfmon v1, " );"generic_arch_v1" ;break ;case 2 :case 3 :case 4 :/* * default constraints for v2 and up "generic architected perfmon, " );"generic_arch_v2+" ;break ;default :/* * The default constraints for v5 and up can support up to * 16 fixed counters . For the fixed counters 4 and later , * the pseudo - encoding is applied . * The constraints may be cut according to the CPUID enumeration * by inserting the EVENT_CONSTRAINT_END . if (fls64(x86_pmu.fixed_cntr_mask64) > INTEL_PMC_MAX_FIXED)1 , 0 );1 ;"generic architected perfmon, " );"generic_arch_v5+" ;break ;sizeof (pmu_name_str), "%s" , name);if (!is_hybrid()) {else {/* * The archPerfmonExt ( 0 x23 ) includes an enhanced enumeration of * PMU architectural features with a per - core view . For non - hybrid , * each core has the same PMU capabilities . It ' s good enough to * update the x86_pmu from the booting CPU . For hybrid , the x86_pmu * is used to keep the common capabilities . Still keep the values * from the leaf 0 xa . The core specific update will be done later * when a new type is online . if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))/* AnyThread may be deprecated on arch perfmon v5 or later */ if (x86_pmu.intel_cap.anythread_deprecated)/* * Access LBR MSR may cause # GP under certain circumstances . * Check all LBR MSR here . * Disable LBR access if any LBR MSRs can not be accessed . if (x86_pmu.lbr_tos && !check_msr(x86_pmu.lbr_tos, 0 x3UL))0 ;for (i = 0 ; i < x86_pmu.lbr_nr; i++) {if (!(check_msr(x86_pmu.lbr_from + i, 0 xffffUL) &&0 xffffUL)))0 ;if (x86_pmu.lbr_nr) {"%d-deep LBR, " , x86_pmu.lbr_nr);/* only support branch_stack snapshot for perfmon >= v2 */ if (x86_pmu.disable_all == intel_pmu_disable_all) {if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) {else {/* Support full width counters using alternative MSR range */ if (x86_pmu.intel_cap.full_width_write) {1 ;"full-width counters, " );/* Support V6+ MSR Aliasing */ if (x86_pmu.version >= 6 ) {if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics)1 ULL << GLOBAL_CTRL_EN_PERF_METRICS;if (x86_pmu.intel_cap.pebs_timing_info)return 0 ;/* * HT bug : phase 2 init * Called once we have valid topology information to check * whether or not HT is enabled * If HT is off , then we disable the workaround static __init int fixup_ht_bug(void )int c;/* * problem not present on this CPU model , nothing to do if (!(x86_pmu.flags & PMU_FL_EXCL_ENABLED))return 0 ;if (topology_max_smt_threads() > 1 ) {"PMU erratum BJ122, BV98, HSD29 worked around, HT is on\n" );return 0 ;"PMU erratum BJ122, BV98, HSD29 workaround disabled, HT off\n" );return 0 ;
Messung V0.5 in Prozent C=93 H=98 G=95
¤ Diese beiden folgenden Angebotsgruppen bietet das Unternehmen0.405Angebot
(Wie Sie bei der Firma Beratungs- und Dienstleistungen beauftragen können 2026-10-01)
¤ *Eine klare Vorstellung vom Zielzustand
