arch/x86/kernel/cpu/perf_event_intel_lbr.c

   1 #ifdef CONFIG_CPU_SUP_INTEL
   2
   3 enum {
   4         LBR_FORMAT_32           = 0x00,
   5         LBR_FORMAT_LIP          = 0x01,
   6         LBR_FORMAT_EIP          = 0x02,
   7         LBR_FORMAT_EIP_FLAGS    = 0x03,
   8 };
   9
  10 /*
  11  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  12  * otherwise it becomes near impossible to get a reliable stack.
  13  */
  14
  15 #define X86_DEBUGCTL_LBR                        (1 << 0)
  16 #define X86_DEBUGCTL_FREEZE_LBRS_ON_PMI         (1 << 11)
  17
  18 static void __intel_pmu_lbr_enable(void)
  19 {
  20         u64 debugctl;
  21
  22         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  23         debugctl |= (X86_DEBUGCTL_LBR | X86_DEBUGCTL_FREEZE_LBRS_ON_PMI);
  24         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  25 }
  26
  27 static void __intel_pmu_lbr_disable(void)
  28 {
  29         u64 debugctl;
  30
  31         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  32         debugctl &= ~(X86_DEBUGCTL_LBR | X86_DEBUGCTL_FREEZE_LBRS_ON_PMI);
  33         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
  34 }
  35
  36 static void intel_pmu_lbr_reset_32(void)
  37 {
  38         int i;
  39
  40         for (i = 0; i < x86_pmu.lbr_nr; i++)
  41                 wrmsrl(x86_pmu.lbr_from + i, 0);
  42 }
  43
  44 static void intel_pmu_lbr_reset_64(void)
  45 {
  46         int i;
  47
  48         for (i = 0; i < x86_pmu.lbr_nr; i++) {
  49                 wrmsrl(x86_pmu.lbr_from + i, 0);
  50                 wrmsrl(x86_pmu.lbr_to   + i, 0);
  51         }
  52 }
  53
  54 static void intel_pmu_lbr_reset(void)
  55 {
  56         if (!x86_pmu.lbr_nr)
  57                 return;
  58
  59         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
  60                 intel_pmu_lbr_reset_32();
  61         else
  62                 intel_pmu_lbr_reset_64();
  63 }
  64
  65 static void intel_pmu_lbr_enable(struct perf_event *event)
  66 {
  67         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  68
  69         if (!x86_pmu.lbr_nr)
  70                 return;
  71
  72         WARN_ON_ONCE(cpuc->enabled);
  73
  74         /*
  75          * Reset the LBR stack if this is the first LBR user or
  76          * we changed task context so as to avoid data leaks.
  77          */
  78
  79         if (!cpuc->lbr_users ||
  80             (event->ctx->task && cpuc->lbr_context != event->ctx)) {
  81                 intel_pmu_lbr_reset();
  82                 cpuc->lbr_context = event->ctx;
  83         }
  84
  85         cpuc->lbr_users++;
  86 }
  87
  88 static void intel_pmu_lbr_disable(struct perf_event *event)
  89 {
  90         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
  91
  92         if (!x86_pmu.lbr_nr)
  93                 return;
  94
  95         cpuc->lbr_users--;
  96         BUG_ON(cpuc->lbr_users < 0);
  97
  98         if (cpuc->enabled && !cpuc->lbr_users)
  99                 __intel_pmu_lbr_disable();
 100 }
 101
 102 static void intel_pmu_lbr_enable_all(void)
 103 {
 104         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 105
 106         if (cpuc->lbr_users)
 107                 __intel_pmu_lbr_enable();
 108 }
 109
 110 static void intel_pmu_lbr_disable_all(void)
 111 {
 112         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 113
 114         if (cpuc->lbr_users)
 115                 __intel_pmu_lbr_disable();
 116 }
 117
 118 static inline u64 intel_pmu_lbr_tos(void)
 119 {
 120         u64 tos;
 121
 122         rdmsrl(x86_pmu.lbr_tos, tos);
 123
 124         return tos;
 125 }
 126
 127 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 128 {
 129         unsigned long mask = x86_pmu.lbr_nr - 1;
 130         u64 tos = intel_pmu_lbr_tos();
 131         int i;
 132
 133         for (i = 0; i < x86_pmu.lbr_nr; i++, tos--) {
 134                 unsigned long lbr_idx = (tos - i) & mask;
 135                 union {
 136                         struct {
 137                                 u32 from;
 138                                 u32 to;
 139                         };
 140                         u64     lbr;
 141                 } msr_lastbranch;
 142
 143                 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 144
 145                 cpuc->lbr_entries[i].from  = msr_lastbranch.from;
 146                 cpuc->lbr_entries[i].to    = msr_lastbranch.to;
 147                 cpuc->lbr_entries[i].flags = 0;
 148         }
 149         cpuc->lbr_stack.nr = i;
 150 }
 151
 152 #define LBR_FROM_FLAG_MISPRED  (1ULL << 63)
 153
 154 /*
 155  * Due to lack of segmentation in Linux the effective address (offset)
 156  * is the same as the linear address, allowing us to merge the LIP and EIP
 157  * LBR formats.
 158  */
 159 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 160 {
 161         unsigned long mask = x86_pmu.lbr_nr - 1;
 162         int lbr_format = x86_pmu.intel_cap.lbr_format;
 163         u64 tos = intel_pmu_lbr_tos();
 164         int i;
 165
 166         for (i = 0; i < x86_pmu.lbr_nr; i++, tos--) {
 167                 unsigned long lbr_idx = (tos - i) & mask;
 168                 u64 from, to, flags = 0;
 169
 170                 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
 171                 rdmsrl(x86_pmu.lbr_to   + lbr_idx, to);
 172
 173                 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
 174                         flags = !!(from & LBR_FROM_FLAG_MISPRED);
 175                         from = (u64)((((s64)from) << 1) >> 1);
 176                 }
 177
 178                 cpuc->lbr_entries[i].from  = from;
 179                 cpuc->lbr_entries[i].to    = to;
 180                 cpuc->lbr_entries[i].flags = flags;
 181         }
 182         cpuc->lbr_stack.nr = i;
 183 }
 184
 185 static void intel_pmu_lbr_read(void)
 186 {
 187         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 188
 189         if (!cpuc->lbr_users)
 190                 return;
 191
 192         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
 193                 intel_pmu_lbr_read_32(cpuc);
 194         else
 195                 intel_pmu_lbr_read_64(cpuc);
 196 }
 197
 198 static void intel_pmu_lbr_init_core(void)
 199 {
 200         x86_pmu.lbr_nr     = 4;
 201         x86_pmu.lbr_tos    = 0x01c9;
 202         x86_pmu.lbr_from   = 0x40;
 203         x86_pmu.lbr_to     = 0x60;
 204 }
 205
 206 static void intel_pmu_lbr_init_nhm(void)
 207 {
 208         x86_pmu.lbr_nr     = 16;
 209         x86_pmu.lbr_tos    = 0x01c9;
 210         x86_pmu.lbr_from   = 0x680;
 211         x86_pmu.lbr_to     = 0x6c0;
 212 }
 213
 214 static void intel_pmu_lbr_init_atom(void)
 215 {
 216         x86_pmu.lbr_nr     = 8;
 217         x86_pmu.lbr_tos    = 0x01c9;
 218         x86_pmu.lbr_from   = 0x40;
 219         x86_pmu.lbr_to     = 0x60;
 220 }
 221
 222 #endif /* CONFIG_CPU_SUP_INTEL */