static void perf_log_throttle(struct perf_counter *counter, int enable);
static void perf_log_period(struct perf_counter *counter, u64 period);
-static void perf_adjust_freq(struct perf_counter_context *ctx)
+static void perf_adjust_period(struct perf_counter *counter, u64 events)
{
- struct perf_counter *counter;
- u64 interrupts, sample_period;
- u64 events, period;
+ struct hw_perf_counter *hwc = &counter->hw;
+ u64 period, sample_period;
s64 delta;
+ events *= hwc->sample_period;
+ period = div64_u64(events, counter->attr.sample_freq);
+
+ delta = (s64)(period - hwc->sample_period);
+ delta = (delta + 7) / 8; /* low pass filter */
+
+ sample_period = hwc->sample_period + delta;
+
+ if (!sample_period)
+ sample_period = 1;
+
+ perf_log_period(counter, sample_period);
+
+ hwc->sample_period = sample_period;
+}
+
+static void perf_ctx_adjust_freq(struct perf_counter_context *ctx)
+{
+ struct perf_counter *counter;
+ struct hw_perf_counter *hwc;
+ u64 interrupts, freq;
+
spin_lock(&ctx->lock);
list_for_each_entry(counter, &ctx->counter_list, list_entry) {
if (counter->state != PERF_COUNTER_STATE_ACTIVE)
continue;
- interrupts = counter->hw.interrupts;
- counter->hw.interrupts = 0;
+ hwc = &counter->hw;
+
+ interrupts = hwc->interrupts;
+ hwc->interrupts = 0;
+ /*
+ * unthrottle counters on the tick
+ */
if (interrupts == MAX_INTERRUPTS) {
perf_log_throttle(counter, 1);
counter->pmu->unthrottle(counter);
if (!counter->attr.freq || !counter->attr.sample_freq)
continue;
- events = HZ * interrupts * counter->hw.sample_period;
- period = div64_u64(events, counter->attr.sample_freq);
+ /*
+ * if the specified freq < HZ then we need to skip ticks
+ */
+ if (counter->attr.sample_freq < HZ) {
+ freq = counter->attr.sample_freq;
- delta = (s64)(1 + period - counter->hw.sample_period);
- delta >>= 1;
+ hwc->freq_count += freq;
+ hwc->freq_interrupts += interrupts;
- sample_period = counter->hw.sample_period + delta;
+ if (hwc->freq_count < HZ)
+ continue;
- if (!sample_period)
- sample_period = 1;
+ interrupts = hwc->freq_interrupts;
+ hwc->freq_interrupts = 0;
+ hwc->freq_count -= HZ;
+ } else
+ freq = HZ;
- perf_log_period(counter, sample_period);
+ perf_adjust_period(counter, freq * interrupts);
- counter->hw.sample_period = sample_period;
+ /*
+ * In order to avoid being stalled by an (accidental) huge
+ * sample period, force reset the sample period if we didn't
+ * get any events in this freq period.
+ */
+ if (!interrupts) {
+ perf_disable();
+ counter->pmu->disable(counter);
+ atomic_set(&hwc->period_left, 0);
+ counter->pmu->enable(counter);
+ perf_enable();
+ }
}
spin_unlock(&ctx->lock);
}
cpuctx = &per_cpu(perf_cpu_context, cpu);
ctx = curr->perf_counter_ctxp;
- perf_adjust_freq(&cpuctx->ctx);
+ perf_ctx_adjust_freq(&cpuctx->ctx);
if (ctx)
- perf_adjust_freq(ctx);
+ perf_ctx_adjust_freq(ctx);
perf_counter_cpu_sched_out(cpuctx);
if (ctx)
counter->attr.sample_freq = value;
} else {
+ perf_log_period(counter, value);
+
counter->attr.sample_period = value;
counter->hw.sample_period = value;
-
- perf_log_period(counter, value);
}
unlock:
spin_unlock_irq(&ctx->lock);
cpu_entry.cpu = raw_smp_processor_id();
}
+ if (sample_type & PERF_SAMPLE_PERIOD) {
+ header.type |= PERF_SAMPLE_PERIOD;
+ header.size += sizeof(u64);
+ }
+
if (sample_type & PERF_SAMPLE_GROUP) {
header.type |= PERF_SAMPLE_GROUP;
header.size += sizeof(u64) +
if (sample_type & PERF_SAMPLE_CPU)
perf_output_put(&handle, cpu_entry);
+ if (sample_type & PERF_SAMPLE_PERIOD)
+ perf_output_put(&handle, counter->hw.sample_period);
+
/*
* XXX PERF_SAMPLE_GROUP vs inherited counters seems difficult.
*/
* event flow.
*/
+struct freq_event {
+ struct perf_event_header header;
+ u64 time;
+ u64 id;
+ u64 period;
+};
+
static void perf_log_period(struct perf_counter *counter, u64 period)
{
struct perf_output_handle handle;
+ struct freq_event event;
int ret;
- struct {
- struct perf_event_header header;
- u64 time;
- u64 period;
- } freq_event = {
+ if (counter->hw.sample_period == period)
+ return;
+
+ if (counter->attr.sample_type & PERF_SAMPLE_PERIOD)
+ return;
+
+ event = (struct freq_event) {
.header = {
.type = PERF_EVENT_PERIOD,
.misc = 0,
- .size = sizeof(freq_event),
+ .size = sizeof(event),
},
.time = sched_clock(),
+ .id = counter->id,
.period = period,
};
- if (counter->hw.sample_period == period)
- return;
-
- ret = perf_output_begin(&handle, counter, sizeof(freq_event), 0, 0);
+ ret = perf_output_begin(&handle, counter, sizeof(event), 1, 0);
if (ret)
return;
- perf_output_put(&handle, freq_event);
+ perf_output_put(&handle, event);
perf_output_end(&handle);
}
{
int events = atomic_read(&counter->event_limit);
int throttle = counter->pmu->unthrottle != NULL;
+ struct hw_perf_counter *hwc = &counter->hw;
int ret = 0;
if (!throttle) {
- counter->hw.interrupts++;
+ hwc->interrupts++;
} else {
- if (counter->hw.interrupts != MAX_INTERRUPTS) {
- counter->hw.interrupts++;
- if (HZ*counter->hw.interrupts > (u64)sysctl_perf_counter_limit) {
- counter->hw.interrupts = MAX_INTERRUPTS;
+ if (hwc->interrupts != MAX_INTERRUPTS) {
+ hwc->interrupts++;
+ if (HZ * hwc->interrupts > (u64)sysctl_perf_counter_limit) {
+ hwc->interrupts = MAX_INTERRUPTS;
perf_log_throttle(counter, 0);
ret = 1;
}
}
}
+ if (counter->attr.freq) {
+ u64 now = sched_clock();
+ s64 delta = now - hwc->freq_stamp;
+
+ hwc->freq_stamp = now;
+
+ if (delta > 0 && delta < TICK_NSEC)
+ perf_adjust_period(counter, NSEC_PER_SEC / (int)delta);
+ }
+
/*
* XXX event_limit might not quite work as expected on inherited
* counters
enum perf_event_types type,
u32 event, struct pt_regs *regs)
{
- u64 event_config;
-
- event_config = ((u64) type << PERF_COUNTER_TYPE_SHIFT) | event;
-
if (!perf_swcounter_is_counting(counter))
return 0;
- if (counter->attr.config != event_config)
+ if (counter->attr.type != type)
+ return 0;
+ if (counter->attr.config != event)
return 0;
if (regs) {
return NULL;
counter->destroy = tp_perf_counter_destroy;
- counter->hw.sample_period = counter->attr.sample_period;
return &perf_ops_generic;
}
* to be kernel events, and page faults are never hypervisor
* events.
*/
- switch (perf_event_id(&counter->attr)) {
+ switch (counter->attr.config) {
case PERF_COUNT_CPU_CLOCK:
pmu = &perf_ops_cpu_clock;
pmu = NULL;
hwc = &counter->hw;
+ hwc->sample_period = attr->sample_period;
if (attr->freq && attr->sample_freq)
- hwc->sample_period = div64_u64(TICK_NSEC, attr->sample_freq);
- else
- hwc->sample_period = attr->sample_period;
+ hwc->sample_period = 1;
+
+ atomic64_set(&hwc->period_left, hwc->sample_period);
/*
* we currently do not support PERF_SAMPLE_GROUP on inherited counters
if (attr->inherit && (attr->sample_type & PERF_SAMPLE_GROUP))
goto done;
- if (perf_event_raw(attr)) {
+ if (attr->type == PERF_TYPE_RAW) {
pmu = hw_perf_counter_init(counter);
goto done;
}
- switch (perf_event_type(attr)) {
+ switch (attr->type) {
case PERF_TYPE_HARDWARE:
+ case PERF_TYPE_HW_CACHE:
pmu = hw_perf_counter_init(counter);
break;
else
child_counter->state = PERF_COUNTER_STATE_OFF;
+ if (parent_counter->attr.freq)
+ child_counter->hw.sample_period = parent_counter->hw.sample_period;
+
/*
* Link it up in the child's context:
*/
projects / deliverable / linux.git / blobdiff