drtracing.org Git - deliverable/linux.git/blob

1 /*

2 * @file op_model_athlon.c

3 * athlon / K7 / K8 / Family 10h model-specific MSR operations

4 *

6 * @remark Read the file COPYING

7 *

8 * @author John Levon

9 * @author Philippe Elie

10 * @author Graydon Hoare

11 * @author Robert Richter <robert.richter@amd.com>

12 * @author Barry Kasindorf

13 */

15 #include <linux/oprofile.h>

16 #include <linux/device.h>

17 #include <linux/pci.h>

19 #include <asm/ptrace.h>

20 #include <asm/msr.h>

21 #include <asm/nmi.h>

23 #include "op_x86_model.h"

24 #include "op_counter.h"

26 #define NUM_COUNTERS 4

27 #define NUM_CONTROLS 4

29 #define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)

30 #define CTR_READ(l, h, msrs, c) do {rdmsr(msrs->counters[(c)].addr, (l), (h)); } while (0)

31 #define CTR_WRITE(l, msrs, c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1); } while (0)

32 #define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))

34 #define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)

35 #define CTRL_READ(l, h, msrs, c) do {rdmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)

36 #define CTRL_WRITE(l, h, msrs, c) do {wrmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)

37 #define CTRL_SET_ACTIVE(n) (n |= (1<<22))

38 #define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))

39 #define CTRL_CLEAR_LO(x) (x &= (1<<21))

40 #define CTRL_CLEAR_HI(x) (x &= 0xfffffcf0)

41 #define CTRL_SET_ENABLE(val) (val |= 1<<20)

42 #define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))

43 #define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))

44 #define CTRL_SET_UM(val, m) (val |= (m << 8))

45 #define CTRL_SET_EVENT_LOW(val, e) (val |= (e & 0xff))

46 #define CTRL_SET_EVENT_HIGH(val, e) (val |= ((e >> 8) & 0xf))

47 #define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 1) << 9))

48 #define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 1) << 8))

50 #define IBS_FETCH_CTL_HIGH_MASK 0xFFFFFFFF

51 /* high dword bit IbsFetchCtl[bit 49] */

52 #define IBS_FETCH_VALID_BIT (1UL << 17)

53 /* high dword bit IbsFetchCtl[bit 52] */

54 #define IBS_FETCH_PHY_ADDR_VALID_BIT (1UL << 20)

55 /* high dword bit IbsFetchCtl[bit 48] */

56 #define IBS_FETCH_ENABLE (1UL << 16)

58 #define IBS_FETCH_CTL_CNT_MASK 0x00000000FFFF0000UL

59 #define IBS_FETCH_CTL_MAX_CNT_MASK 0x000000000000FFFFUL

61 /*IbsOpCtl masks/bits */

62 #define IBS_OP_VALID_BIT (1ULL<<18) /* IbsOpCtl[bit18] */

63 #define IBS_OP_ENABLE (1ULL<<17) /* IBS_OP_ENABLE[bit17]*/

65 /* Codes used in cpu_buffer.c */

66 #define IBS_FETCH_BEGIN 3

67 #define IBS_OP_BEGIN 4

69 /*IbsOpData3 masks */

70 #define IBS_CTL_LVT_OFFSET_VALID_BIT (1ULL<<8)

72 /*PCI Extended Configuration Constants */

73 /* MSR to set the IBS control register APIC LVT offset */

74 #define IBS_LVT_OFFSET_PCI 0x1CC

76 /* The function interface needs to be fixed, something like add

77 data. Should then be added to linux/oprofile.h. */

78 extern void oprofile_add_ibs_sample(struct pt_regs *const regs,

79 unsigned int * const ibs_sample, u8 code);

81 struct ibs_fetch_sample {

82 /* MSRC001_1031 IBS Fetch Linear Address Register */

83 unsigned int ibs_fetch_lin_addr_low;

84 unsigned int ibs_fetch_lin_addr_high;

85 /* MSRC001_1030 IBS Fetch Control Register */

86 unsigned int ibs_fetch_ctl_low;

87 unsigned int ibs_fetch_ctl_high;

88 /* MSRC001_1032 IBS Fetch Physical Address Register */

89 unsigned int ibs_fetch_phys_addr_low;

90 unsigned int ibs_fetch_phys_addr_high;

91 };

93 struct ibs_op_sample {

94 /* MSRC001_1034 IBS Op Logical Address Register (IbsRIP) */

95 unsigned int ibs_op_rip_low;

96 unsigned int ibs_op_rip_high;

97 /* MSRC001_1035 IBS Op Data Register */

98 unsigned int ibs_op_data1_low;

99 unsigned int ibs_op_data1_high;

100 /* MSRC001_1036 IBS Op Data 2 Register */

101 unsigned int ibs_op_data2_low;

102 unsigned int ibs_op_data2_high;

103 /* MSRC001_1037 IBS Op Data 3 Register */

104 unsigned int ibs_op_data3_low;

105 unsigned int ibs_op_data3_high;

106 /* MSRC001_1038 IBS DC Linear Address Register (IbsDcLinAd) */

107 unsigned int ibs_dc_linear_low;

108 unsigned int ibs_dc_linear_high;

109 /* MSRC001_1039 IBS DC Physical Address Register (IbsDcPhysAd) */

110 unsigned int ibs_dc_phys_low;

111 unsigned int ibs_dc_phys_high;

112 };

113

114 /*

115 * unitialize the APIC for the IBS interrupts if needed on AMD Family10h+

116 */

117 static void clear_ibs_nmi(void);

118

119 static unsigned long reset_value[NUM_COUNTERS];

120 static int ibs_allowed; /* AMD Family10h and later */

121

122 struct op_ibs_config {

123 unsigned long op_enabled;

124 unsigned long fetch_enabled;

125 unsigned long max_cnt_fetch;

126 unsigned long max_cnt_op;

127 unsigned long rand_en;

128 unsigned long dispatched_ops;

129 };

130

131 static struct op_ibs_config ibs_config;

132

133 /* functions for op_amd_spec */

134

135 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)

136 {

137 int i;

138

139 for (i = 0; i < NUM_COUNTERS; i++) {

140 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))

141 msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;

142 else

143 msrs->counters[i].addr = 0;

144 }

145

146 for (i = 0; i < NUM_CONTROLS; i++) {

147 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i))

148 msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;

149 else

150 msrs->controls[i].addr = 0;

151 }

152 }

153

154

155 static void op_amd_setup_ctrs(struct op_msrs const * const msrs)

156 {

157 unsigned int low, high;

158 int i;

159

160 /* clear all counters */

161 for (i = 0 ; i < NUM_CONTROLS; ++i) {

162 if (unlikely(!CTRL_IS_RESERVED(msrs, i)))

163 continue;

164 CTRL_READ(low, high, msrs, i);

165 CTRL_CLEAR_LO(low);

166 CTRL_CLEAR_HI(high);

167 CTRL_WRITE(low, high, msrs, i);

168 }

169

170 /* avoid a false detection of ctr overflows in NMI handler */

171 for (i = 0; i < NUM_COUNTERS; ++i) {

172 if (unlikely(!CTR_IS_RESERVED(msrs, i)))

173 continue;

174 CTR_WRITE(1, msrs, i);

175 }

176

177 /* enable active counters */

178 for (i = 0; i < NUM_COUNTERS; ++i) {

179 if ((counter_config[i].enabled) && (CTR_IS_RESERVED(msrs, i))) {

180 reset_value[i] = counter_config[i].count;

181

182 CTR_WRITE(counter_config[i].count, msrs, i);

183

184 CTRL_READ(low, high, msrs, i);

185 CTRL_CLEAR_LO(low);

186 CTRL_CLEAR_HI(high);

187 CTRL_SET_ENABLE(low);

188 CTRL_SET_USR(low, counter_config[i].user);

189 CTRL_SET_KERN(low, counter_config[i].kernel);

190 CTRL_SET_UM(low, counter_config[i].unit_mask);

191 CTRL_SET_EVENT_LOW(low, counter_config[i].event);

192 CTRL_SET_EVENT_HIGH(high, counter_config[i].event);

193 CTRL_SET_HOST_ONLY(high, 0);

194 CTRL_SET_GUEST_ONLY(high, 0);

195

196 CTRL_WRITE(low, high, msrs, i);

197 } else {

198 reset_value[i] = 0;

199 }

200 }

201 }

202

203 static inline int

204 op_amd_handle_ibs(struct pt_regs * const regs,

205 struct op_msrs const * const msrs)

206 {

207 unsigned int low, high;

208 struct ibs_fetch_sample ibs_fetch;

209 struct ibs_op_sample ibs_op;

210

211 if (!ibs_allowed)

212 return 1;

213

214 if (ibs_config.fetch_enabled) {

215 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);

216 if (high & IBS_FETCH_VALID_BIT) {

217 ibs_fetch.ibs_fetch_ctl_high = high;

218 ibs_fetch.ibs_fetch_ctl_low = low;

219 rdmsr(MSR_AMD64_IBSFETCHLINAD, low, high);

220 ibs_fetch.ibs_fetch_lin_addr_high = high;

221 ibs_fetch.ibs_fetch_lin_addr_low = low;

222 rdmsr(MSR_AMD64_IBSFETCHPHYSAD, low, high);

223 ibs_fetch.ibs_fetch_phys_addr_high = high;

224 ibs_fetch.ibs_fetch_phys_addr_low = low;

225

226 oprofile_add_ibs_sample(regs,

227 (unsigned int *)&ibs_fetch,

228 IBS_FETCH_BEGIN);

229

230 /*reenable the IRQ */

231 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);

232 high &= ~(IBS_FETCH_VALID_BIT);

233 high |= IBS_FETCH_ENABLE;

234 low &= IBS_FETCH_CTL_MAX_CNT_MASK;

235 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);

236 }

237 }

238

239 if (ibs_config.op_enabled) {

240 rdmsr(MSR_AMD64_IBSOPCTL, low, high);

241 if (low & IBS_OP_VALID_BIT) {

242 rdmsr(MSR_AMD64_IBSOPRIP, low, high);

243 ibs_op.ibs_op_rip_low = low;

244 ibs_op.ibs_op_rip_high = high;

245 rdmsr(MSR_AMD64_IBSOPDATA, low, high);

246 ibs_op.ibs_op_data1_low = low;

247 ibs_op.ibs_op_data1_high = high;

248 rdmsr(MSR_AMD64_IBSOPDATA2, low, high);

249 ibs_op.ibs_op_data2_low = low;

250 ibs_op.ibs_op_data2_high = high;

251 rdmsr(MSR_AMD64_IBSOPDATA3, low, high);

252 ibs_op.ibs_op_data3_low = low;

253 ibs_op.ibs_op_data3_high = high;

254 rdmsr(MSR_AMD64_IBSDCLINAD, low, high);

255 ibs_op.ibs_dc_linear_low = low;

256 ibs_op.ibs_dc_linear_high = high;

257 rdmsr(MSR_AMD64_IBSDCPHYSAD, low, high);

258 ibs_op.ibs_dc_phys_low = low;

259 ibs_op.ibs_dc_phys_high = high;

260

261 /* reenable the IRQ */

262 oprofile_add_ibs_sample(regs,

263 (unsigned int *)&ibs_op,

264 IBS_OP_BEGIN);

265 rdmsr(MSR_AMD64_IBSOPCTL, low, high);

266 low &= ~(IBS_OP_VALID_BIT);

267 low |= IBS_OP_ENABLE;

268 wrmsr(MSR_AMD64_IBSOPCTL, low, high);

269 }

270 }

271

272 return 1;

273 }

274

275 static int op_amd_check_ctrs(struct pt_regs * const regs,

276 struct op_msrs const * const msrs)

277 {

278 unsigned int low, high;

279 int i;

280

281 for (i = 0 ; i < NUM_COUNTERS; ++i) {

282 if (!reset_value[i])

283 continue;

284 CTR_READ(low, high, msrs, i);

285 if (CTR_OVERFLOWED(low)) {

286 oprofile_add_sample(regs, i);

287 CTR_WRITE(reset_value[i], msrs, i);

288 }

289 }

290

291 op_amd_handle_ibs(regs, msrs);

292

293 /* See op_model_ppro.c */

294 return 1;

295 }

296

297 static void op_amd_start(struct op_msrs const * const msrs)

298 {

299 unsigned int low, high;

300 int i;

301 for (i = 0 ; i < NUM_COUNTERS ; ++i) {

302 if (reset_value[i]) {

303 CTRL_READ(low, high, msrs, i);

304 CTRL_SET_ACTIVE(low);

305 CTRL_WRITE(low, high, msrs, i);

306 }

307 }

308 if (ibs_allowed && ibs_config.fetch_enabled) {

309 low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;

310 high = IBS_FETCH_ENABLE;

311 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);

312 }

313

314 if (ibs_allowed && ibs_config.op_enabled) {

315 low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF) + IBS_OP_ENABLE;

316 high = 0;

317 wrmsr(MSR_AMD64_IBSOPCTL, low, high);

318 }

319 }

320

321

322 static void op_amd_stop(struct op_msrs const * const msrs)

323 {

324 unsigned int low, high;

325 int i;

326

327 /* Subtle: stop on all counters to avoid race with

328 * setting our pm callback */

329 for (i = 0 ; i < NUM_COUNTERS ; ++i) {

330 if (!reset_value[i])

331 continue;

332 CTRL_READ(low, high, msrs, i);

333 CTRL_SET_INACTIVE(low);

334 CTRL_WRITE(low, high, msrs, i);

335 }

336

337 if (ibs_allowed && ibs_config.fetch_enabled) {

338 low = 0; /* clear max count and enable */

339 high = 0;

340 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);

341 }

342

343 if (ibs_allowed && ibs_config.op_enabled) {

344 low = 0; /* clear max count and enable */

345 high = 0;

346 wrmsr(MSR_AMD64_IBSOPCTL, low, high);

347 }

348 }

349

350 static void op_amd_shutdown(struct op_msrs const * const msrs)

351 {

352 int i;

353

354 for (i = 0 ; i < NUM_COUNTERS ; ++i) {

355 if (CTR_IS_RESERVED(msrs, i))

356 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);

357 }

358 for (i = 0 ; i < NUM_CONTROLS ; ++i) {

359 if (CTRL_IS_RESERVED(msrs, i))

360 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);

361 }

362 }

363

364 #ifndef CONFIG_SMP

365

366 /* no IBS support */

367

368 static void setup_ibs(void)

369 {

370 ibs_allowed = 0;

371 }

372

373 static void clear_ibs_nmi(void) {}

374

375 static int op_amd_init(struct oprofile_operations *ops)

376 {

377 return 0;

378 }

379

380 static void op_amd_exit(void) {}

381

382 #else

383

384 static u8 ibs_eilvt_off;

385

386 static inline void apic_init_ibs_nmi_per_cpu(void *arg)

387 {

388 ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);

389 }

390

391 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)

392 {

393 setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);

394 }

395

396 static int pfm_amd64_setup_eilvt(void)

397 {

398 #define IBSCTL_LVTOFFSETVAL (1 << 8)

399 #define IBSCTL 0x1cc

400 struct pci_dev *cpu_cfg;

401 int nodes;

402 u32 value = 0;

403

404 /* per CPU setup */

405 on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);

406

407 nodes = 0;

408 cpu_cfg = NULL;

409 do {

410 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,

411 PCI_DEVICE_ID_AMD_10H_NB_MISC,

412 cpu_cfg);

413 if (!cpu_cfg)

414 break;

415 ++nodes;

416 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off

417 | IBSCTL_LVTOFFSETVAL);

418 pci_read_config_dword(cpu_cfg, IBSCTL, &value);

419 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {

420 printk(KERN_DEBUG "Failed to setup IBS LVT offset, "

421 "IBSCTL = 0x%08x", value);

422 return 1;

423 }

424 } while (1);

425

426 if (!nodes) {

427 printk(KERN_DEBUG "No CPU node configured for IBS");

428 return 1;

429 }

430

431 #ifdef CONFIG_NUMA

432 /* Sanity check */

433 /* Works only for 64bit with proper numa implementation. */

434 if (nodes != num_possible_nodes()) {

435 printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "

436 "found: %d, expected %d",

437 nodes, num_possible_nodes());

438 return 1;

439 }

440 #endif

441 return 0;

442 }

443

444 /*

445 * initialize the APIC for the IBS interrupts

446 * if available (AMD Family10h rev B0 and later)

447 */

448 static void setup_ibs(void)

449 {

450 ibs_allowed = boot_cpu_has(X86_FEATURE_IBS);

451

452 if (!ibs_allowed)

453 return;

454

455 if (pfm_amd64_setup_eilvt())

456 ibs_allowed = 0;

457 }

458

459

460 /*

461 * unitialize the APIC for the IBS interrupts if needed on AMD Family10h

462 * rev B0 and later */

463 static void clear_ibs_nmi(void)

464 {

465 if (ibs_allowed)

466 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);

467 }

468

469 static int (*create_arch_files)(struct super_block * sb, struct dentry * root);

470

471 static int setup_ibs_files(struct super_block * sb, struct dentry * root)

472 {

473 char buf[12];

474 struct dentry *dir;

475 int ret = 0;

476

477 /* architecture specific files */

478 if (create_arch_files)

479 ret = create_arch_files(sb, root);

480

481 if (ret)

482 return ret;

483

484 if (!ibs_allowed)

485 return ret;

486

487 /* model specific files */

488

489 /* setup some reasonable defaults */

490 ibs_config.max_cnt_fetch = 250000;

491 ibs_config.fetch_enabled = 0;

492 ibs_config.max_cnt_op = 250000;

493 ibs_config.op_enabled = 0;

494 ibs_config.dispatched_ops = 1;

495 snprintf(buf, sizeof(buf), "ibs_fetch");

496 dir = oprofilefs_mkdir(sb, root, buf);

497 oprofilefs_create_ulong(sb, dir, "rand_enable",

498 &ibs_config.rand_en);

499 oprofilefs_create_ulong(sb, dir, "enable",

500 &ibs_config.fetch_enabled);

501 oprofilefs_create_ulong(sb, dir, "max_count",

502 &ibs_config.max_cnt_fetch);

503 snprintf(buf, sizeof(buf), "ibs_uops");

504 dir = oprofilefs_mkdir(sb, root, buf);

505 oprofilefs_create_ulong(sb, dir, "enable",

506 &ibs_config.op_enabled);

507 oprofilefs_create_ulong(sb, dir, "max_count",

508 &ibs_config.max_cnt_op);

509 oprofilefs_create_ulong(sb, dir, "dispatched_ops",

510 &ibs_config.dispatched_ops);

511

512 return 0;

513 }

514

515 static int op_amd_init(struct oprofile_operations *ops)

516 {

517 setup_ibs();

518 create_arch_files = ops->create_files;

519 ops->create_files = setup_ibs_files;

520 return 0;

521 }

522

523 static void op_amd_exit(void)

524 {

525 clear_ibs_nmi();

526 }

527

528 #endif

529

530 struct op_x86_model_spec const op_amd_spec = {

531 .init = op_amd_init,

532 .exit = op_amd_exit,

533 .num_counters = NUM_COUNTERS,

534 .num_controls = NUM_CONTROLS,

535 .fill_in_addresses = &op_amd_fill_in_addresses,

536 .setup_ctrs = &op_amd_setup_ctrs,

537 .check_ctrs = &op_amd_check_ctrs,

538 .start = &op_amd_start,

539 .stop = &op_amd_stop,

540 .shutdown = &op_amd_shutdown

541 };