arch/x86/oprofile/nmi_int.c

   1 /**
   2  * @file nmi_int.c
   3  *
   4  * @remark Copyright 2002-2009 OProfile authors
   5  * @remark Read the file COPYING
   6  *
   7  * @author John Levon <levon@movementarian.org>
   8  * @author Robert Richter <robert.richter@amd.com>
   9  * @author Barry Kasindorf <barry.kasindorf@amd.com>
  10  * @author Jason Yeh <jason.yeh@amd.com>
  11  * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  12  */
  13
  14 #include <linux/init.h>
  15 #include <linux/notifier.h>
  16 #include <linux/smp.h>
  17 #include <linux/oprofile.h>
  18 #include <linux/sysdev.h>
  19 #include <linux/slab.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kdebug.h>
  22 #include <linux/cpu.h>
  23 #include <asm/nmi.h>
  24 #include <asm/msr.h>
  25 #include <asm/apic.h>
  26
  27 #include "op_counter.h"
  28 #include "op_x86_model.h"
  29
  30 static struct op_x86_model_spec *model;
  31 static DEFINE_PER_CPU(struct op_msrs, cpu_msrs);
  32 static DEFINE_PER_CPU(unsigned long, saved_lvtpc);
  33
  34 /* 0 == registered but off, 1 == registered and on */
  35 static int nmi_enabled = 0;
  36
  37
  38 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  39 extern atomic_t multiplex_counter;
  40 #endif
  41
  42 struct op_counter_config counter_config[OP_MAX_COUNTER];
  43
  44 /* common functions */
  45
  46 u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
  47                     struct op_counter_config *counter_config)
  48 {
  49         u64 val = 0;
  50         u16 event = (u16)counter_config->event;
  51
  52         val |= ARCH_PERFMON_EVENTSEL_INT;
  53         val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0;
  54         val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0;
  55         val |= (counter_config->unit_mask & 0xFF) << 8;
  56         event &= model->event_mask ? model->event_mask : 0xFF;
  57         val |= event & 0xFF;
  58         val |= (event & 0x0F00) << 24;
  59
  60         return val;
  61 }
  62
  63
  64 static int profile_exceptions_notify(struct notifier_block *self,
  65                                      unsigned long val, void *data)
  66 {
  67         struct die_args *args = (struct die_args *)data;
  68         int ret = NOTIFY_DONE;
  69         int cpu = smp_processor_id();
  70
  71         switch (val) {
  72         case DIE_NMI:
  73         case DIE_NMI_IPI:
  74                 model->check_ctrs(args->regs, &per_cpu(cpu_msrs, cpu));
  75                 ret = NOTIFY_STOP;
  76                 break;
  77         default:
  78                 break;
  79         }
  80         return ret;
  81 }
  82
  83 static void nmi_cpu_save_registers(struct op_msrs *msrs)
  84 {
  85         struct op_msr *counters = msrs->counters;
  86         struct op_msr *controls = msrs->controls;
  87         unsigned int i;
  88
  89         for (i = 0; i < model->num_counters; ++i) {
  90                 if (counters[i].addr)
  91                         rdmsrl(counters[i].addr, counters[i].saved);
  92         }
  93
  94         for (i = 0; i < model->num_controls; ++i) {
  95                 if (controls[i].addr)
  96                         rdmsrl(controls[i].addr, controls[i].saved);
  97         }
  98 }
  99
 100 static void nmi_cpu_start(void *dummy)
 101 {
 102         struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
 103         model->start(msrs);
 104 }
 105
 106 static int nmi_start(void)
 107 {
 108         on_each_cpu(nmi_cpu_start, NULL, 1);
 109         return 0;
 110 }
 111
 112 static void nmi_cpu_stop(void *dummy)
 113 {
 114         struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
 115         model->stop(msrs);
 116 }
 117
 118 static void nmi_stop(void)
 119 {
 120         on_each_cpu(nmi_cpu_stop, NULL, 1);
 121 }
 122
 123 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 124
 125 static DEFINE_PER_CPU(int, switch_index);
 126
 127 inline int op_x86_phys_to_virt(int phys)
 128 {
 129         return __get_cpu_var(switch_index) + phys;
 130 }
 131
 132 static void nmi_shutdown_mux(void)
 133 {
 134         int i;
 135         for_each_possible_cpu(i) {
 136                 kfree(per_cpu(cpu_msrs, i).multiplex);
 137                 per_cpu(cpu_msrs, i).multiplex = NULL;
 138                 per_cpu(switch_index, i) = 0;
 139         }
 140 }
 141
 142 static int nmi_setup_mux(void)
 143 {
 144         size_t multiplex_size =
 145                 sizeof(struct op_msr) * model->num_virt_counters;
 146         int i;
 147         for_each_possible_cpu(i) {
 148                 per_cpu(cpu_msrs, i).multiplex =
 149                         kmalloc(multiplex_size, GFP_KERNEL);
 150                 if (!per_cpu(cpu_msrs, i).multiplex)
 151                         return 0;
 152         }
 153         return 1;
 154 }
 155
 156 static void nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs)
 157 {
 158         int i;
 159         struct op_msr *multiplex = msrs->multiplex;
 160
 161         for (i = 0; i < model->num_virt_counters; ++i) {
 162                 if (counter_config[i].enabled) {
 163                         multiplex[i].saved = -(u64)counter_config[i].count;
 164                 } else {
 165                         multiplex[i].addr  = 0;
 166                         multiplex[i].saved = 0;
 167                 }
 168         }
 169
 170         per_cpu(switch_index, cpu) = 0;
 171 }
 172
 173 static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs)
 174 {
 175         struct op_msr *multiplex = msrs->multiplex;
 176         int i;
 177
 178         for (i = 0; i < model->num_counters; ++i) {
 179                 int virt = op_x86_phys_to_virt(i);
 180                 if (multiplex[virt].addr)
 181                         rdmsrl(multiplex[virt].addr, multiplex[virt].saved);
 182         }
 183 }
 184
 185 static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs)
 186 {
 187         struct op_msr *multiplex = msrs->multiplex;
 188         int i;
 189
 190         for (i = 0; i < model->num_counters; ++i) {
 191                 int virt = op_x86_phys_to_virt(i);
 192                 if (multiplex[virt].addr)
 193                         wrmsrl(multiplex[virt].addr, multiplex[virt].saved);
 194         }
 195 }
 196
 197 static void nmi_cpu_switch(void *dummy)
 198 {
 199         int cpu = smp_processor_id();
 200         int si = per_cpu(switch_index, cpu);
 201         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 202
 203         nmi_cpu_stop(NULL);
 204         nmi_cpu_save_mpx_registers(msrs);
 205
 206         /* move to next set */
 207         si += model->num_counters;
 208         if ((si > model->num_virt_counters) || (counter_config[si].count == 0))
 209                 per_cpu(switch_index, cpu) = 0;
 210         else
 211                 per_cpu(switch_index, cpu) = si;
 212
 213         model->switch_ctrl(model, msrs);
 214         nmi_cpu_restore_mpx_registers(msrs);
 215
 216         nmi_cpu_start(NULL);
 217 }
 218
 219
 220 /*
 221  * Quick check to see if multiplexing is necessary.
 222  * The check should be sufficient since counters are used
 223  * in ordre.
 224  */
 225 static int nmi_multiplex_on(void)
 226 {
 227         return counter_config[model->num_counters].count ? 0 : -EINVAL;
 228 }
 229
 230 static int nmi_switch_event(void)
 231 {
 232         if (!model->switch_ctrl)
 233                 return -ENOSYS;         /* not implemented */
 234         if (nmi_multiplex_on() < 0)
 235                 return -EINVAL;         /* not necessary */
 236
 237         on_each_cpu(nmi_cpu_switch, NULL, 1);
 238
 239         atomic_inc(&multiplex_counter);
 240
 241         return 0;
 242 }
 243
 244 #else
 245
 246 inline int op_x86_phys_to_virt(int phys) { return phys; }
 247 static inline void nmi_shutdown_mux(void) { }
 248 static inline int nmi_setup_mux(void) { return 1; }
 249 static inline void
 250 nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs) { }
 251
 252 #endif
 253
 254 static void free_msrs(void)
 255 {
 256         int i;
 257         for_each_possible_cpu(i) {
 258                 kfree(per_cpu(cpu_msrs, i).counters);
 259                 per_cpu(cpu_msrs, i).counters = NULL;
 260                 kfree(per_cpu(cpu_msrs, i).controls);
 261                 per_cpu(cpu_msrs, i).controls = NULL;
 262         }
 263 }
 264
 265 static int allocate_msrs(void)
 266 {
 267         size_t controls_size = sizeof(struct op_msr) * model->num_controls;
 268         size_t counters_size = sizeof(struct op_msr) * model->num_counters;
 269
 270         int i;
 271         for_each_possible_cpu(i) {
 272                 per_cpu(cpu_msrs, i).counters = kmalloc(counters_size,
 273                                                         GFP_KERNEL);
 274                 if (!per_cpu(cpu_msrs, i).counters)
 275                         return 0;
 276                 per_cpu(cpu_msrs, i).controls = kmalloc(controls_size,
 277                                                         GFP_KERNEL);
 278                 if (!per_cpu(cpu_msrs, i).controls)
 279                         return 0;
 280         }
 281
 282         return 1;
 283 }
 284
 285 static void nmi_cpu_setup(void *dummy)
 286 {
 287         int cpu = smp_processor_id();
 288         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 289         nmi_cpu_save_registers(msrs);
 290         spin_lock(&oprofilefs_lock);
 291         model->setup_ctrs(model, msrs);
 292         nmi_cpu_setup_mux(cpu, msrs);
 293         spin_unlock(&oprofilefs_lock);
 294         per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC);
 295         apic_write(APIC_LVTPC, APIC_DM_NMI);
 296 }
 297
 298 static struct notifier_block profile_exceptions_nb = {
 299         .notifier_call = profile_exceptions_notify,
 300         .next = NULL,
 301         .priority = 2
 302 };
 303
 304 static int nmi_setup(void)
 305 {
 306         int err = 0;
 307         int cpu;
 308
 309         if (!allocate_msrs())
 310                 err = -ENOMEM;
 311         else if (!nmi_setup_mux())
 312                 err = -ENOMEM;
 313         else
 314                 err = register_die_notifier(&profile_exceptions_nb);
 315
 316         if (err) {
 317                 free_msrs();
 318                 nmi_shutdown_mux();
 319                 return err;
 320         }
 321
 322         /* We need to serialize save and setup for HT because the subset
 323          * of msrs are distinct for save and setup operations
 324          */
 325
 326         /* Assume saved/restored counters are the same on all CPUs */
 327         model->fill_in_addresses(&per_cpu(cpu_msrs, 0));
 328         for_each_possible_cpu(cpu) {
 329                 if (cpu != 0) {
 330                         memcpy(per_cpu(cpu_msrs, cpu).counters,
 331                                 per_cpu(cpu_msrs, 0).counters,
 332                                 sizeof(struct op_msr) * model->num_counters);
 333
 334                         memcpy(per_cpu(cpu_msrs, cpu).controls,
 335                                 per_cpu(cpu_msrs, 0).controls,
 336                                 sizeof(struct op_msr) * model->num_controls);
 337 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 338                         memcpy(per_cpu(cpu_msrs, cpu).multiplex,
 339                                 per_cpu(cpu_msrs, 0).multiplex,
 340                                 sizeof(struct op_msr) * model->num_virt_counters);
 341 #endif
 342                 }
 343         }
 344         on_each_cpu(nmi_cpu_setup, NULL, 1);
 345         nmi_enabled = 1;
 346         return 0;
 347 }
 348
 349 static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 350 {
 351         struct op_msr *counters = msrs->counters;
 352         struct op_msr *controls = msrs->controls;
 353         unsigned int i;
 354
 355         for (i = 0; i < model->num_controls; ++i) {
 356                 if (controls[i].addr)
 357                         wrmsrl(controls[i].addr, controls[i].saved);
 358         }
 359
 360         for (i = 0; i < model->num_counters; ++i) {
 361                 if (counters[i].addr)
 362                         wrmsrl(counters[i].addr, counters[i].saved);
 363         }
 364 }
 365
 366 static void nmi_cpu_shutdown(void *dummy)
 367 {
 368         unsigned int v;
 369         int cpu = smp_processor_id();
 370         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 371
 372         /* restoring APIC_LVTPC can trigger an apic error because the delivery
 373          * mode and vector nr combination can be illegal. That's by design: on
 374          * power on apic lvt contain a zero vector nr which are legal only for
 375          * NMI delivery mode. So inhibit apic err before restoring lvtpc
 376          */
 377         v = apic_read(APIC_LVTERR);
 378         apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
 379         apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu));
 380         apic_write(APIC_LVTERR, v);
 381         nmi_cpu_restore_registers(msrs);
 382 }
 383
 384 static void nmi_shutdown(void)
 385 {
 386         struct op_msrs *msrs;
 387
 388         nmi_enabled = 0;
 389         on_each_cpu(nmi_cpu_shutdown, NULL, 1);
 390         unregister_die_notifier(&profile_exceptions_nb);
 391         nmi_shutdown_mux();
 392         msrs = &get_cpu_var(cpu_msrs);
 393         model->shutdown(msrs);
 394         free_msrs();
 395         put_cpu_var(cpu_msrs);
 396 }
 397
 398 static int nmi_create_files(struct super_block *sb, struct dentry *root)
 399 {
 400         unsigned int i;
 401
 402         for (i = 0; i < model->num_virt_counters; ++i) {
 403                 struct dentry *dir;
 404                 char buf[4];
 405
 406 #ifndef CONFIG_OPROFILE_EVENT_MULTIPLEX
 407                 /* quick little hack to _not_ expose a counter if it is not
 408                  * available for use.  This should protect userspace app.
 409                  * NOTE:  assumes 1:1 mapping here (that counters are organized
 410                  *        sequentially in their struct assignment).
 411                  */
 412                 if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i)))
 413                         continue;
 414 #endif /* CONFIG_OPROFILE_EVENT_MULTIPLEX */
 415
 416                 snprintf(buf,  sizeof(buf), "%d", i);
 417                 dir = oprofilefs_mkdir(sb, root, buf);
 418                 oprofilefs_create_ulong(sb, dir, "enabled", &counter_config[i].enabled);
 419                 oprofilefs_create_ulong(sb, dir, "event", &counter_config[i].event);
 420                 oprofilefs_create_ulong(sb, dir, "count", &counter_config[i].count);
 421                 oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask);
 422                 oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel);
 423                 oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user);
 424         }
 425
 426         return 0;
 427 }
 428
 429 #ifdef CONFIG_SMP
 430 static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
 431                                  void *data)
 432 {
 433         int cpu = (unsigned long)data;
 434         switch (action) {
 435         case CPU_DOWN_FAILED:
 436         case CPU_ONLINE:
 437                 smp_call_function_single(cpu, nmi_cpu_start, NULL, 0);
 438                 break;
 439         case CPU_DOWN_PREPARE:
 440                 smp_call_function_single(cpu, nmi_cpu_stop, NULL, 1);
 441                 break;
 442         }
 443         return NOTIFY_DONE;
 444 }
 445
 446 static struct notifier_block oprofile_cpu_nb = {
 447         .notifier_call = oprofile_cpu_notifier
 448 };
 449 #endif
 450
 451 #ifdef CONFIG_PM
 452
 453 static int nmi_suspend(struct sys_device *dev, pm_message_t state)
 454 {
 455         /* Only one CPU left, just stop that one */
 456         if (nmi_enabled == 1)
 457                 nmi_cpu_stop(NULL);
 458         return 0;
 459 }
 460
 461 static int nmi_resume(struct sys_device *dev)
 462 {
 463         if (nmi_enabled == 1)
 464                 nmi_cpu_start(NULL);
 465         return 0;
 466 }
 467
 468 static struct sysdev_class oprofile_sysclass = {
 469         .name           = "oprofile",
 470         .resume         = nmi_resume,
 471         .suspend        = nmi_suspend,
 472 };
 473
 474 static struct sys_device device_oprofile = {
 475         .id     = 0,
 476         .cls    = &oprofile_sysclass,
 477 };
 478
 479 static int __init init_sysfs(void)
 480 {
 481         int error;
 482
 483         error = sysdev_class_register(&oprofile_sysclass);
 484         if (!error)
 485                 error = sysdev_register(&device_oprofile);
 486         return error;
 487 }
 488
 489 static void exit_sysfs(void)
 490 {
 491         sysdev_unregister(&device_oprofile);
 492         sysdev_class_unregister(&oprofile_sysclass);
 493 }
 494
 495 #else
 496 #define init_sysfs() do { } while (0)
 497 #define exit_sysfs() do { } while (0)
 498 #endif /* CONFIG_PM */
 499
 500 static int __init p4_init(char **cpu_type)
 501 {
 502         __u8 cpu_model = boot_cpu_data.x86_model;
 503
 504         if (cpu_model > 6 || cpu_model == 5)
 505                 return 0;
 506
 507 #ifndef CONFIG_SMP
 508         *cpu_type = "i386/p4";
 509         model = &op_p4_spec;
 510         return 1;
 511 #else
 512         switch (smp_num_siblings) {
 513         case 1:
 514                 *cpu_type = "i386/p4";
 515                 model = &op_p4_spec;
 516                 return 1;
 517
 518         case 2:
 519                 *cpu_type = "i386/p4-ht";
 520                 model = &op_p4_ht2_spec;
 521                 return 1;
 522         }
 523 #endif
 524
 525         printk(KERN_INFO "oprofile: P4 HyperThreading detected with > 2 threads\n");
 526         printk(KERN_INFO "oprofile: Reverting to timer mode.\n");
 527         return 0;
 528 }
 529
 530 static int force_arch_perfmon;
 531 static int force_cpu_type(const char *str, struct kernel_param *kp)
 532 {
 533         if (!strcmp(str, "arch_perfmon")) {
 534                 force_arch_perfmon = 1;
 535                 printk(KERN_INFO "oprofile: forcing architectural perfmon\n");
 536         }
 537
 538         return 0;
 539 }
 540 module_param_call(cpu_type, force_cpu_type, NULL, NULL, 0);
 541
 542 static int __init ppro_init(char **cpu_type)
 543 {
 544         __u8 cpu_model = boot_cpu_data.x86_model;
 545         struct op_x86_model_spec *spec = &op_ppro_spec; /* default */
 546
 547         if (force_arch_perfmon && cpu_has_arch_perfmon)
 548                 return 0;
 549
 550         switch (cpu_model) {
 551         case 0 ... 2:
 552                 *cpu_type = "i386/ppro";
 553                 break;
 554         case 3 ... 5:
 555                 *cpu_type = "i386/pii";
 556                 break;
 557         case 6 ... 8:
 558         case 10 ... 11:
 559                 *cpu_type = "i386/piii";
 560                 break;
 561         case 9:
 562         case 13:
 563                 *cpu_type = "i386/p6_mobile";
 564                 break;
 565         case 14:
 566                 *cpu_type = "i386/core";
 567                 break;
 568         case 15: case 23:
 569                 *cpu_type = "i386/core_2";
 570                 break;
 571         case 26:
 572                 spec = &op_arch_perfmon_spec;
 573                 *cpu_type = "i386/core_i7";
 574                 break;
 575         case 28:
 576                 *cpu_type = "i386/atom";
 577                 break;
 578         default:
 579                 /* Unknown */
 580                 return 0;
 581         }
 582
 583         model = spec;
 584         return 1;
 585 }
 586
 587 /* in order to get sysfs right */
 588 static int using_nmi;
 589
 590 int __init op_nmi_init(struct oprofile_operations *ops)
 591 {
 592         __u8 vendor = boot_cpu_data.x86_vendor;
 593         __u8 family = boot_cpu_data.x86;
 594         char *cpu_type = NULL;
 595         int ret = 0;
 596
 597         if (!cpu_has_apic)
 598                 return -ENODEV;
 599
 600         switch (vendor) {
 601         case X86_VENDOR_AMD:
 602                 /* Needs to be at least an Athlon (or hammer in 32bit mode) */
 603
 604                 switch (family) {
 605                 case 6:
 606                         cpu_type = "i386/athlon";
 607                         break;
 608                 case 0xf:
 609                         /*
 610                          * Actually it could be i386/hammer too, but
 611                          * give user space an consistent name.
 612                          */
 613                         cpu_type = "x86-64/hammer";
 614                         break;
 615                 case 0x10:
 616                         cpu_type = "x86-64/family10";
 617                         break;
 618                 case 0x11:
 619                         cpu_type = "x86-64/family11h";
 620                         break;
 621                 default:
 622                         return -ENODEV;
 623                 }
 624                 model = &op_amd_spec;
 625                 break;
 626
 627         case X86_VENDOR_INTEL:
 628                 switch (family) {
 629                         /* Pentium IV */
 630                 case 0xf:
 631                         p4_init(&cpu_type);
 632                         break;
 633
 634                         /* A P6-class processor */
 635                 case 6:
 636                         ppro_init(&cpu_type);
 637                         break;
 638
 639                 default:
 640                         break;
 641                 }
 642
 643                 if (cpu_type)
 644                         break;
 645
 646                 if (!cpu_has_arch_perfmon)
 647                         return -ENODEV;
 648
 649                 /* use arch perfmon as fallback */
 650                 cpu_type = "i386/arch_perfmon";
 651                 model = &op_arch_perfmon_spec;
 652                 break;
 653
 654         default:
 655                 return -ENODEV;
 656         }
 657
 658 #ifdef CONFIG_SMP
 659         register_cpu_notifier(&oprofile_cpu_nb);
 660 #endif
 661         /* default values, can be overwritten by model */
 662         ops->create_files       = nmi_create_files;
 663         ops->setup              = nmi_setup;
 664         ops->shutdown           = nmi_shutdown;
 665         ops->start              = nmi_start;
 666         ops->stop               = nmi_stop;
 667         ops->cpu_type           = cpu_type;
 668 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 669         ops->switch_events      = nmi_switch_event;
 670 #endif
 671
 672         if (model->init)
 673                 ret = model->init(ops);
 674         if (ret)
 675                 return ret;
 676
 677         init_sysfs();
 678         using_nmi = 1;
 679         printk(KERN_INFO "oprofile: using NMI interrupt.\n");
 680         return 0;
 681 }
 682
 683 void op_nmi_exit(void)
 684 {
 685         if (using_nmi) {
 686                 exit_sysfs();
 687 #ifdef CONFIG_SMP
 688                 unregister_cpu_notifier(&oprofile_cpu_nb);
 689 #endif
 690         }
 691         if (model->exit)
 692                 model->exit();
 693 }