arch/arm/kernel/perf_event.c

   1 #undef DEBUG
   2
   3 /*
   4  * ARM performance counter support.
   5  *
   6  * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
   7  * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
   8  *
   9  * This code is based on the sparc64 perf event code, which is in turn based
  10  * on the x86 code. Callchain code is based on the ARM OProfile backtrace
  11  * code.
  12  */
  13 #define pr_fmt(fmt) "hw perfevents: " fmt
  14
  15 #include <linux/kernel.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/pm_runtime.h>
  18 #include <linux/uaccess.h>
  19
  20 #include <asm/irq_regs.h>
  21 #include <asm/pmu.h>
  22 #include <asm/stacktrace.h>
  23
  24 static int
  25 armpmu_map_cache_event(const unsigned (*cache_map)
  26                                       [PERF_COUNT_HW_CACHE_MAX]
  27                                       [PERF_COUNT_HW_CACHE_OP_MAX]
  28                                       [PERF_COUNT_HW_CACHE_RESULT_MAX],
  29                        u64 config)
  30 {
  31         unsigned int cache_type, cache_op, cache_result, ret;
  32
  33         cache_type = (config >>  0) & 0xff;
  34         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
  35                 return -EINVAL;
  36
  37         cache_op = (config >>  8) & 0xff;
  38         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
  39                 return -EINVAL;
  40
  41         cache_result = (config >> 16) & 0xff;
  42         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
  43                 return -EINVAL;
  44
  45         ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
  46
  47         if (ret == CACHE_OP_UNSUPPORTED)
  48                 return -ENOENT;
  49
  50         return ret;
  51 }
  52
  53 static int
  54 armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
  55 {
  56         int mapping;
  57
  58         if (config >= PERF_COUNT_HW_MAX)
  59                 return -EINVAL;
  60
  61         mapping = (*event_map)[config];
  62         return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
  63 }
  64
  65 static int
  66 armpmu_map_raw_event(u32 raw_event_mask, u64 config)
  67 {
  68         return (int)(config & raw_event_mask);
  69 }
  70
  71 int
  72 armpmu_map_event(struct perf_event *event,
  73                  const unsigned (*event_map)[PERF_COUNT_HW_MAX],
  74                  const unsigned (*cache_map)
  75                                 [PERF_COUNT_HW_CACHE_MAX]
  76                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  77                                 [PERF_COUNT_HW_CACHE_RESULT_MAX],
  78                  u32 raw_event_mask)
  79 {
  80         u64 config = event->attr.config;
  81
  82         switch (event->attr.type) {
  83         case PERF_TYPE_HARDWARE:
  84                 return armpmu_map_hw_event(event_map, config);
  85         case PERF_TYPE_HW_CACHE:
  86                 return armpmu_map_cache_event(cache_map, config);
  87         case PERF_TYPE_RAW:
  88                 return armpmu_map_raw_event(raw_event_mask, config);
  89         }
  90
  91         return -ENOENT;
  92 }
  93
  94 int armpmu_event_set_period(struct perf_event *event)
  95 {
  96         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
  97         struct hw_perf_event *hwc = &event->hw;
  98         s64 left = local64_read(&hwc->period_left);
  99         s64 period = hwc->sample_period;
 100         int ret = 0;
 101
 102         if (unlikely(left <= -period)) {
 103                 left = period;
 104                 local64_set(&hwc->period_left, left);
 105                 hwc->last_period = period;
 106                 ret = 1;
 107         }
 108
 109         if (unlikely(left <= 0)) {
 110                 left += period;
 111                 local64_set(&hwc->period_left, left);
 112                 hwc->last_period = period;
 113                 ret = 1;
 114         }
 115
 116         if (left > (s64)armpmu->max_period)
 117                 left = armpmu->max_period;
 118
 119         local64_set(&hwc->prev_count, (u64)-left);
 120
 121         armpmu->write_counter(event, (u64)(-left) & 0xffffffff);
 122
 123         perf_event_update_userpage(event);
 124
 125         return ret;
 126 }
 127
 128 u64 armpmu_event_update(struct perf_event *event)
 129 {
 130         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 131         struct hw_perf_event *hwc = &event->hw;
 132         u64 delta, prev_raw_count, new_raw_count;
 133
 134 again:
 135         prev_raw_count = local64_read(&hwc->prev_count);
 136         new_raw_count = armpmu->read_counter(event);
 137
 138         if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 139                              new_raw_count) != prev_raw_count)
 140                 goto again;
 141
 142         delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
 143
 144         local64_add(delta, &event->count);
 145         local64_sub(delta, &hwc->period_left);
 146
 147         return new_raw_count;
 148 }
 149
 150 static void
 151 armpmu_read(struct perf_event *event)
 152 {
 153         armpmu_event_update(event);
 154 }
 155
 156 static void
 157 armpmu_stop(struct perf_event *event, int flags)
 158 {
 159         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 160         struct hw_perf_event *hwc = &event->hw;
 161
 162         /*
 163          * ARM pmu always has to update the counter, so ignore
 164          * PERF_EF_UPDATE, see comments in armpmu_start().
 165          */
 166         if (!(hwc->state & PERF_HES_STOPPED)) {
 167                 armpmu->disable(event);
 168                 armpmu_event_update(event);
 169                 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 170         }
 171 }
 172
 173 static void armpmu_start(struct perf_event *event, int flags)
 174 {
 175         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 176         struct hw_perf_event *hwc = &event->hw;
 177
 178         /*
 179          * ARM pmu always has to reprogram the period, so ignore
 180          * PERF_EF_RELOAD, see the comment below.
 181          */
 182         if (flags & PERF_EF_RELOAD)
 183                 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 184
 185         hwc->state = 0;
 186         /*
 187          * Set the period again. Some counters can't be stopped, so when we
 188          * were stopped we simply disabled the IRQ source and the counter
 189          * may have been left counting. If we don't do this step then we may
 190          * get an interrupt too soon or *way* too late if the overflow has
 191          * happened since disabling.
 192          */
 193         armpmu_event_set_period(event);
 194         armpmu->enable(event);
 195 }
 196
 197 static void
 198 armpmu_del(struct perf_event *event, int flags)
 199 {
 200         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 201         struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 202         struct hw_perf_event *hwc = &event->hw;
 203         int idx = hwc->idx;
 204
 205         armpmu_stop(event, PERF_EF_UPDATE);
 206         hw_events->events[idx] = NULL;
 207         clear_bit(idx, hw_events->used_mask);
 208
 209         perf_event_update_userpage(event);
 210 }
 211
 212 static int
 213 armpmu_add(struct perf_event *event, int flags)
 214 {
 215         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 216         struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 217         struct hw_perf_event *hwc = &event->hw;
 218         int idx;
 219         int err = 0;
 220
 221         perf_pmu_disable(event->pmu);
 222
 223         /* If we don't have a space for the counter then finish early. */
 224         idx = armpmu->get_event_idx(hw_events, event);
 225         if (idx < 0) {
 226                 err = idx;
 227                 goto out;
 228         }
 229
 230         /*
 231          * If there is an event in the counter we are going to use then make
 232          * sure it is disabled.
 233          */
 234         event->hw.idx = idx;
 235         armpmu->disable(event);
 236         hw_events->events[idx] = event;
 237
 238         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 239         if (flags & PERF_EF_START)
 240                 armpmu_start(event, PERF_EF_RELOAD);
 241
 242         /* Propagate our changes to the userspace mapping. */
 243         perf_event_update_userpage(event);
 244
 245 out:
 246         perf_pmu_enable(event->pmu);
 247         return err;
 248 }
 249
 250 static int
 251 validate_event(struct pmu_hw_events *hw_events,
 252                struct perf_event *event)
 253 {
 254         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 255
 256         if (is_software_event(event))
 257                 return 1;
 258
 259         if (event->state < PERF_EVENT_STATE_OFF)
 260                 return 1;
 261
 262         if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
 263                 return 1;
 264
 265         return armpmu->get_event_idx(hw_events, event) >= 0;
 266 }
 267
 268 static int
 269 validate_group(struct perf_event *event)
 270 {
 271         struct perf_event *sibling, *leader = event->group_leader;
 272         struct pmu_hw_events fake_pmu;
 273         DECLARE_BITMAP(fake_used_mask, ARMPMU_MAX_HWEVENTS);
 274
 275         /*
 276          * Initialise the fake PMU. We only need to populate the
 277          * used_mask for the purposes of validation.
 278          */
 279         memset(fake_used_mask, 0, sizeof(fake_used_mask));
 280         fake_pmu.used_mask = fake_used_mask;
 281
 282         if (!validate_event(&fake_pmu, leader))
 283                 return -EINVAL;
 284
 285         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
 286                 if (!validate_event(&fake_pmu, sibling))
 287                         return -EINVAL;
 288         }
 289
 290         if (!validate_event(&fake_pmu, event))
 291                 return -EINVAL;
 292
 293         return 0;
 294 }
 295
 296 static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
 297 {
 298         struct arm_pmu *armpmu = (struct arm_pmu *) dev;
 299         struct platform_device *plat_device = armpmu->plat_device;
 300         struct arm_pmu_platdata *plat = dev_get_platdata(&plat_device->dev);
 301
 302         if (plat && plat->handle_irq)
 303                 return plat->handle_irq(irq, dev, armpmu->handle_irq);
 304         else
 305                 return armpmu->handle_irq(irq, dev);
 306 }
 307
 308 static void
 309 armpmu_release_hardware(struct arm_pmu *armpmu)
 310 {
 311         armpmu->free_irq(armpmu);
 312         pm_runtime_put_sync(&armpmu->plat_device->dev);
 313 }
 314
 315 static int
 316 armpmu_reserve_hardware(struct arm_pmu *armpmu)
 317 {
 318         int err;
 319         struct platform_device *pmu_device = armpmu->plat_device;
 320
 321         if (!pmu_device)
 322                 return -ENODEV;
 323
 324         pm_runtime_get_sync(&pmu_device->dev);
 325         err = armpmu->request_irq(armpmu, armpmu_dispatch_irq);
 326         if (err) {
 327                 armpmu_release_hardware(armpmu);
 328                 return err;
 329         }
 330
 331         return 0;
 332 }
 333
 334 static void
 335 hw_perf_event_destroy(struct perf_event *event)
 336 {
 337         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 338         atomic_t *active_events  = &armpmu->active_events;
 339         struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;
 340
 341         if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
 342                 armpmu_release_hardware(armpmu);
 343                 mutex_unlock(pmu_reserve_mutex);
 344         }
 345 }
 346
 347 static int
 348 event_requires_mode_exclusion(struct perf_event_attr *attr)
 349 {
 350         return attr->exclude_idle || attr->exclude_user ||
 351                attr->exclude_kernel || attr->exclude_hv;
 352 }
 353
 354 static int
 355 __hw_perf_event_init(struct perf_event *event)
 356 {
 357         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 358         struct hw_perf_event *hwc = &event->hw;
 359         int mapping;
 360
 361         mapping = armpmu->map_event(event);
 362
 363         if (mapping < 0) {
 364                 pr_debug("event %x:%llx not supported\n", event->attr.type,
 365                          event->attr.config);
 366                 return mapping;
 367         }
 368
 369         /*
 370          * We don't assign an index until we actually place the event onto
 371          * hardware. Use -1 to signify that we haven't decided where to put it
 372          * yet. For SMP systems, each core has it's own PMU so we can't do any
 373          * clever allocation or constraints checking at this point.
 374          */
 375         hwc->idx                = -1;
 376         hwc->config_base        = 0;
 377         hwc->config             = 0;
 378         hwc->event_base         = 0;
 379
 380         /*
 381          * Check whether we need to exclude the counter from certain modes.
 382          */
 383         if ((!armpmu->set_event_filter ||
 384              armpmu->set_event_filter(hwc, &event->attr)) &&
 385              event_requires_mode_exclusion(&event->attr)) {
 386                 pr_debug("ARM performance counters do not support "
 387                          "mode exclusion\n");
 388                 return -EOPNOTSUPP;
 389         }
 390
 391         /*
 392          * Store the event encoding into the config_base field.
 393          */
 394         hwc->config_base            |= (unsigned long)mapping;
 395
 396         if (!hwc->sample_period) {
 397                 /*
 398                  * For non-sampling runs, limit the sample_period to half
 399                  * of the counter width. That way, the new counter value
 400                  * is far less likely to overtake the previous one unless
 401                  * you have some serious IRQ latency issues.
 402                  */
 403                 hwc->sample_period  = armpmu->max_period >> 1;
 404                 hwc->last_period    = hwc->sample_period;
 405                 local64_set(&hwc->period_left, hwc->sample_period);
 406         }
 407
 408         if (event->group_leader != event) {
 409                 if (validate_group(event) != 0)
 410                         return -EINVAL;
 411         }
 412
 413         return 0;
 414 }
 415
 416 static int armpmu_event_init(struct perf_event *event)
 417 {
 418         struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 419         int err = 0;
 420         atomic_t *active_events = &armpmu->active_events;
 421
 422         /* does not support taken branch sampling */
 423         if (has_branch_stack(event))
 424                 return -EOPNOTSUPP;
 425
 426         if (armpmu->map_event(event) == -ENOENT)
 427                 return -ENOENT;
 428
 429         event->destroy = hw_perf_event_destroy;
 430
 431         if (!atomic_inc_not_zero(active_events)) {
 432                 mutex_lock(&armpmu->reserve_mutex);
 433                 if (atomic_read(active_events) == 0)
 434                         err = armpmu_reserve_hardware(armpmu);
 435
 436                 if (!err)
 437                         atomic_inc(active_events);
 438                 mutex_unlock(&armpmu->reserve_mutex);
 439         }
 440
 441         if (err)
 442                 return err;
 443
 444         err = __hw_perf_event_init(event);
 445         if (err)
 446                 hw_perf_event_destroy(event);
 447
 448         return err;
 449 }
 450
 451 static void armpmu_enable(struct pmu *pmu)
 452 {
 453         struct arm_pmu *armpmu = to_arm_pmu(pmu);
 454         struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 455         int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);
 456
 457         if (enabled)
 458                 armpmu->start(armpmu);
 459 }
 460
 461 static void armpmu_disable(struct pmu *pmu)
 462 {
 463         struct arm_pmu *armpmu = to_arm_pmu(pmu);
 464         armpmu->stop(armpmu);
 465 }
 466
 467 #ifdef CONFIG_PM_RUNTIME
 468 static int armpmu_runtime_resume(struct device *dev)
 469 {
 470         struct arm_pmu_platdata *plat = dev_get_platdata(dev);
 471
 472         if (plat && plat->runtime_resume)
 473                 return plat->runtime_resume(dev);
 474
 475         return 0;
 476 }
 477
 478 static int armpmu_runtime_suspend(struct device *dev)
 479 {
 480         struct arm_pmu_platdata *plat = dev_get_platdata(dev);
 481
 482         if (plat && plat->runtime_suspend)
 483                 return plat->runtime_suspend(dev);
 484
 485         return 0;
 486 }
 487 #endif
 488
 489 const struct dev_pm_ops armpmu_dev_pm_ops = {
 490         SET_RUNTIME_PM_OPS(armpmu_runtime_suspend, armpmu_runtime_resume, NULL)
 491 };
 492
 493 static void armpmu_init(struct arm_pmu *armpmu)
 494 {
 495         atomic_set(&armpmu->active_events, 0);
 496         mutex_init(&armpmu->reserve_mutex);
 497
 498         armpmu->pmu = (struct pmu) {
 499                 .pmu_enable     = armpmu_enable,
 500                 .pmu_disable    = armpmu_disable,
 501                 .event_init     = armpmu_event_init,
 502                 .add            = armpmu_add,
 503                 .del            = armpmu_del,
 504                 .start          = armpmu_start,
 505                 .stop           = armpmu_stop,
 506                 .read           = armpmu_read,
 507         };
 508 }
 509
 510 int armpmu_register(struct arm_pmu *armpmu, int type)
 511 {
 512         armpmu_init(armpmu);
 513         pm_runtime_enable(&armpmu->plat_device->dev);
 514         pr_info("enabled with %s PMU driver, %d counters available\n",
 515                         armpmu->name, armpmu->num_events);
 516         return perf_pmu_register(&armpmu->pmu, armpmu->name, type);
 517 }
 518
 519 /*
 520  * Callchain handling code.
 521  */
 522
 523 /*
 524  * The registers we're interested in are at the end of the variable
 525  * length saved register structure. The fp points at the end of this
 526  * structure so the address of this struct is:
 527  * (struct frame_tail *)(xxx->fp)-1
 528  *
 529  * This code has been adapted from the ARM OProfile support.
 530  */
 531 struct frame_tail {
 532         struct frame_tail __user *fp;
 533         unsigned long sp;
 534         unsigned long lr;
 535 } __attribute__((packed));
 536
 537 /*
 538  * Get the return address for a single stackframe and return a pointer to the
 539  * next frame tail.
 540  */
 541 static struct frame_tail __user *
 542 user_backtrace(struct frame_tail __user *tail,
 543                struct perf_callchain_entry *entry)
 544 {
 545         struct frame_tail buftail;
 546
 547         /* Also check accessibility of one struct frame_tail beyond */
 548         if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
 549                 return NULL;
 550         if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail)))
 551                 return NULL;
 552
 553         perf_callchain_store(entry, buftail.lr);
 554
 555         /*
 556          * Frame pointers should strictly progress back up the stack
 557          * (towards higher addresses).
 558          */
 559         if (tail + 1 >= buftail.fp)
 560                 return NULL;
 561
 562         return buftail.fp - 1;
 563 }
 564
 565 void
 566 perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
 567 {
 568         struct frame_tail __user *tail;
 569
 570         if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
 571                 /* We don't support guest os callchain now */
 572                 return;
 573         }
 574
 575         perf_callchain_store(entry, regs->ARM_pc);
 576         tail = (struct frame_tail __user *)regs->ARM_fp - 1;
 577
 578         while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
 579                tail && !((unsigned long)tail & 0x3))
 580                 tail = user_backtrace(tail, entry);
 581 }
 582
 583 /*
 584  * Gets called by walk_stackframe() for every stackframe. This will be called
 585  * whist unwinding the stackframe and is like a subroutine return so we use
 586  * the PC.
 587  */
 588 static int
 589 callchain_trace(struct stackframe *fr,
 590                 void *data)
 591 {
 592         struct perf_callchain_entry *entry = data;
 593         perf_callchain_store(entry, fr->pc);
 594         return 0;
 595 }
 596
 597 void
 598 perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
 599 {
 600         struct stackframe fr;
 601
 602         if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
 603                 /* We don't support guest os callchain now */
 604                 return;
 605         }
 606
 607         fr.fp = regs->ARM_fp;
 608         fr.sp = regs->ARM_sp;
 609         fr.lr = regs->ARM_lr;
 610         fr.pc = regs->ARM_pc;
 611         walk_stackframe(&fr, callchain_trace, entry);
 612 }
 613
 614 unsigned long perf_instruction_pointer(struct pt_regs *regs)
 615 {
 616         if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
 617                 return perf_guest_cbs->get_guest_ip();
 618
 619         return instruction_pointer(regs);
 620 }
 621
 622 unsigned long perf_misc_flags(struct pt_regs *regs)
 623 {
 624         int misc = 0;
 625
 626         if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
 627                 if (perf_guest_cbs->is_user_mode())
 628                         misc |= PERF_RECORD_MISC_GUEST_USER;
 629                 else
 630                         misc |= PERF_RECORD_MISC_GUEST_KERNEL;
 631         } else {
 632                 if (user_mode(regs))
 633                         misc |= PERF_RECORD_MISC_USER;
 634                 else
 635                         misc |= PERF_RECORD_MISC_KERNEL;
 636         }
 637
 638         return misc;
 639 }