[deliverable/linux.git] / kernel / trace / trace_hw_branches.c

/*
 * h/w branch tracer for x86 based on BTS
 *
 * Copyright (C) 2008-2009 Intel Corporation.
 * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009
 */
#include <linux/kallsyms.h>
#include <linux/debugfs.h>
#include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/fs.h>

#include <asm/ds.h>

#include "trace_output.h"
#include "trace.h"


#define BTS_BUFFER_SIZE (1 << 13)

static DEFINE_PER_CPU(struct bts_tracer *, tracer);
static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], buffer);

#define this_tracer per_cpu(tracer, smp_processor_id())

static int trace_hw_branches_enabled __read_mostly;
static int trace_hw_branches_suspended __read_mostly;
static struct trace_array *hw_branch_trace __read_mostly;


static void bts_trace_init_cpu(int cpu)
{
	per_cpu(tracer, cpu) =
		ds_request_bts_cpu(cpu, per_cpu(buffer, cpu), BTS_BUFFER_SIZE,
				   NULL, (size_t)-1, BTS_KERNEL);

	if (IS_ERR(per_cpu(tracer, cpu)))
		per_cpu(tracer, cpu) = NULL;
}

static int bts_trace_init(struct trace_array *tr)
{
	int cpu;

	hw_branch_trace = tr;
	trace_hw_branches_enabled = 0;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		bts_trace_init_cpu(cpu);

		if (likely(per_cpu(tracer, cpu)))
			trace_hw_branches_enabled = 1;
	}
	trace_hw_branches_suspended = 0;
	put_online_cpus();

	/* If we could not enable tracing on a single cpu, we fail. */
	return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP;
}

static void bts_trace_reset(struct trace_array *tr)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		if (likely(per_cpu(tracer, cpu))) {
			ds_release_bts(per_cpu(tracer, cpu));
			per_cpu(tracer, cpu) = NULL;
		}
	}
	trace_hw_branches_enabled = 0;
	trace_hw_branches_suspended = 0;
	put_online_cpus();
}

static void bts_trace_start(struct trace_array *tr)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_resume_bts(per_cpu(tracer, cpu));
	trace_hw_branches_suspended = 0;
	put_online_cpus();
}

static void bts_trace_stop(struct trace_array *tr)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_suspend_bts(per_cpu(tracer, cpu));
	trace_hw_branches_suspended = 1;
	put_online_cpus();
}

static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb,
				     unsigned long action, void *hcpu)
{
	int cpu = (long)hcpu;

	switch (action) {
	case CPU_ONLINE:
	case CPU_DOWN_FAILED:
		/* The notification is sent with interrupts enabled. */
		if (trace_hw_branches_enabled) {
			bts_trace_init_cpu(cpu);

			if (trace_hw_branches_suspended &&
			    likely(per_cpu(tracer, cpu)))
				ds_suspend_bts(per_cpu(tracer, cpu));
		}
		break;

	case CPU_DOWN_PREPARE:
		/* The notification is sent with interrupts enabled. */
		if (likely(per_cpu(tracer, cpu))) {
			ds_release_bts(per_cpu(tracer, cpu));
			per_cpu(tracer, cpu) = NULL;
		}
	}

	return NOTIFY_DONE;
}

static struct notifier_block bts_hotcpu_notifier __cpuinitdata = {
	.notifier_call = bts_hotcpu_handler
};

static void bts_trace_print_header(struct seq_file *m)
{
	seq_puts(m, "# CPU#        TO  <-  FROM\n");
}

static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
{
	unsigned long symflags = TRACE_ITER_SYM_OFFSET;
	struct trace_entry *entry = iter->ent;
	struct trace_seq *seq = &iter->seq;
	struct hw_branch_entry *it;

	trace_assign_type(it, entry);

	if (entry->type == TRACE_HW_BRANCHES) {
		if (trace_seq_printf(seq, "%4d  ", iter->cpu) &&
		    seq_print_ip_sym(seq, it->to, symflags) &&
		    trace_seq_printf(seq, "\t  <-  ") &&
		    seq_print_ip_sym(seq, it->from, symflags) &&
		    trace_seq_printf(seq, "\n"))
			return TRACE_TYPE_HANDLED;
		return TRACE_TYPE_PARTIAL_LINE;
	}
	return TRACE_TYPE_UNHANDLED;
}

void trace_hw_branch(u64 from, u64 to)
{
	struct ftrace_event_call *call = &event_hw_branch;
	struct trace_array *tr = hw_branch_trace;
	struct ring_buffer_event *event;
	struct ring_buffer *buf;
	struct hw_branch_entry *entry;
	unsigned long irq1;
	int cpu;

	if (unlikely(!tr))
		return;

	if (unlikely(!trace_hw_branches_enabled))
		return;

	local_irq_save(irq1);
	cpu = raw_smp_processor_id();
	if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
		goto out;

	buf = tr->buffer;
	event = trace_buffer_lock_reserve(buf, TRACE_HW_BRANCHES,
					  sizeof(*entry), 0, 0);
	if (!event)
		goto out;
	entry	= ring_buffer_event_data(event);
	tracing_generic_entry_update(&entry->ent, 0, from);
	entry->ent.type = TRACE_HW_BRANCHES;
	entry->from = from;
	entry->to   = to;
	if (!filter_check_discard(call, entry, buf, event))
		trace_buffer_unlock_commit(buf, event, 0, 0);

 out:
	atomic_dec(&tr->data[cpu]->disabled);
	local_irq_restore(irq1);
}

static void trace_bts_at(const struct bts_trace *trace, void *at)
{
	struct bts_struct bts;
	int err = 0;

	WARN_ON_ONCE(!trace->read);
	if (!trace->read)
		return;

	err = trace->read(this_tracer, at, &bts);
	if (err < 0)
		return;

	switch (bts.qualifier) {
	case BTS_BRANCH:
		trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to);
		break;
	}
}

/*
 * Collect the trace on the current cpu and write it into the ftrace buffer.
 *
 * pre: tracing must be suspended on the current cpu
 */
static void trace_bts_cpu(void *arg)
{
	struct trace_array *tr = (struct trace_array *)arg;
	const struct bts_trace *trace;
	unsigned char *at;

	if (unlikely(!tr))
		return;

	if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled)))
		return;

	if (unlikely(!this_tracer))
		return;

	trace = ds_read_bts(this_tracer);
	if (!trace)
		return;

	for (at = trace->ds.top; (void *)at < trace->ds.end;
	     at += trace->ds.size)
		trace_bts_at(trace, at);

	for (at = trace->ds.begin; (void *)at < trace->ds.top;
	     at += trace->ds.size)
		trace_bts_at(trace, at);
}

static void trace_bts_prepare(struct trace_iterator *iter)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_suspend_bts(per_cpu(tracer, cpu));
	/*
	 * We need to collect the trace on the respective cpu since ftrace
	 * implicitly adds the record for the current cpu.
	 * Once that is more flexible, we could collect the data from any cpu.
	 */
	on_each_cpu(trace_bts_cpu, iter->tr, 1);

	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_resume_bts(per_cpu(tracer, cpu));
	put_online_cpus();
}

static void trace_bts_close(struct trace_iterator *iter)
{
	tracing_reset_online_cpus(iter->tr);
}

void trace_hw_branch_oops(void)
{
	if (this_tracer) {
		ds_suspend_bts_noirq(this_tracer);
		trace_bts_cpu(hw_branch_trace);
		ds_resume_bts_noirq(this_tracer);
	}
}

struct tracer bts_tracer __read_mostly =
{
	.name		= "hw-branch-tracer",
	.init		= bts_trace_init,
	.reset		= bts_trace_reset,
	.print_header	= bts_trace_print_header,
	.print_line	= bts_trace_print_line,
	.start		= bts_trace_start,
	.stop		= bts_trace_stop,
	.open		= trace_bts_prepare,
	.close		= trace_bts_close,
#ifdef CONFIG_FTRACE_SELFTEST
	.selftest	= trace_selftest_startup_hw_branches,
#endif /* CONFIG_FTRACE_SELFTEST */
};

__init static int init_bts_trace(void)
{
	register_hotcpu_notifier(&bts_hotcpu_notifier);
	return register_tracer(&bts_tracer);
}
device_initcall(init_bts_trace);
Commit	Line	Data
1e9b51c2	1	/*
e9a22d1f	2	* h/w branch tracer for x86 based on BTS
1e9b51c2	3	*
5c5317de MM	4	* Copyright (C) 2008-2009 Intel Corporation.
5c5317de MM	5	* Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009
1e9b51c2	6	*/
2d542cf3	7	#include <linux/kallsyms.h>
1e9b51c2 MM	8	#include <linux/debugfs.h>
1e9b51c2 MM	9	#include <linux/ftrace.h>
2d542cf3	10	#include <linux/module.h>
5c5317de MM	11	#include <linux/cpu.h>
5c5317de MM	12	#include <linux/smp.h>
2d542cf3	13	#include <linux/fs.h>
1e9b51c2 MM	14
	15	#include <asm/ds.h>
	16
f0868d1e	17	#include "trace_output.h"
e9a22d1f	18	#include "trace.h"
1e9b51c2 MM	19
1e9b51c2 MM	20
ba9372a8	21	#define BTS_BUFFER_SIZE (1 << 13)
1e9b51c2 MM	22
1e9b51c2 MM	23	static DEFINE_PER_CPU(struct bts_tracer *, tracer);
ba9372a8	24	static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], buffer);
1e9b51c2 MM	25
1e9b51c2 MM	26	#define this_tracer per_cpu(tracer, smp_processor_id())
1e9b51c2	27
ba9372a8 MM	28	static int trace_hw_branches_enabled __read_mostly;
ba9372a8 MM	29	static int trace_hw_branches_suspended __read_mostly;
b1818748	30	static struct trace_array *hw_branch_trace __read_mostly;
1e9b51c2	31
5c5317de	32
de79f54f	33	static void bts_trace_init_cpu(int cpu)
1e9b51c2	34	{
de79f54f MM	35	per_cpu(tracer, cpu) =
	36	ds_request_bts_cpu(cpu, per_cpu(buffer, cpu), BTS_BUFFER_SIZE,
	37	NULL, (size_t)-1, BTS_KERNEL);
a93751ca	38
de79f54f MM	39	if (IS_ERR(per_cpu(tracer, cpu)))
de79f54f MM	40	per_cpu(tracer, cpu) = NULL;
1e9b51c2 MM	41	}
1e9b51c2 MM	42
ba9372a8	43	static int bts_trace_init(struct trace_array *tr)
1e9b51c2	44	{
de79f54f	45	int cpu;
1e9b51c2	46
ba9372a8	47	hw_branch_trace = tr;
de79f54f	48	trace_hw_branches_enabled = 0;
1e9b51c2	49
de79f54f MM	50	get_online_cpus();
	51	for_each_online_cpu(cpu) {
	52	bts_trace_init_cpu(cpu);
1e9b51c2	53
de79f54f MM	54	if (likely(per_cpu(tracer, cpu)))
	55	trace_hw_branches_enabled = 1;
	56	}
ba9372a8	57	trace_hw_branches_suspended = 0;
de79f54f	58	put_online_cpus();
ba9372a8 MM	59
ba9372a8 MM	60	/* If we could not enable tracing on a single cpu, we fail. */
de79f54f	61	return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP;
1e9b51c2 MM	62	}
1e9b51c2 MM	63
ba9372a8	64	static void bts_trace_reset(struct trace_array *tr)
1e9b51c2	65	{
de79f54f	66	int cpu;
5c5317de	67
de79f54f MM	68	get_online_cpus();
	69	for_each_online_cpu(cpu) {
	70	if (likely(per_cpu(tracer, cpu))) {
	71	ds_release_bts(per_cpu(tracer, cpu));
	72	per_cpu(tracer, cpu) = NULL;
	73	}
1e9b51c2	74	}
5c5317de	75	trace_hw_branches_enabled = 0;
ba9372a8	76	trace_hw_branches_suspended = 0;
de79f54f	77	put_online_cpus();
1e9b51c2 MM	78	}
1e9b51c2 MM	79
ba9372a8	80	static void bts_trace_start(struct trace_array *tr)
1e9b51c2	81	{
de79f54f	82	int cpu;
5c5317de	83
de79f54f MM	84	get_online_cpus();
	85	for_each_online_cpu(cpu)
	86	if (likely(per_cpu(tracer, cpu)))
	87	ds_resume_bts(per_cpu(tracer, cpu));
ba9372a8	88	trace_hw_branches_suspended = 0;
de79f54f	89	put_online_cpus();
ba9372a8	90	}
1e9b51c2	91
ba9372a8 MM	92	static void bts_trace_stop(struct trace_array *tr)
ba9372a8 MM	93	{
de79f54f	94	int cpu;
ba9372a8	95
de79f54f MM	96	get_online_cpus();
	97	for_each_online_cpu(cpu)
	98	if (likely(per_cpu(tracer, cpu)))
	99	ds_suspend_bts(per_cpu(tracer, cpu));
ba9372a8	100	trace_hw_branches_suspended = 1;
de79f54f	101	put_online_cpus();
5c5317de MM	102	}
	103
	104	static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb,
	105	unsigned long action, void *hcpu)
	106	{
de79f54f	107	int cpu = (long)hcpu;
5c5317de MM	108
	109	switch (action) {
	110	case CPU_ONLINE:
	111	case CPU_DOWN_FAILED:
de79f54f MM	112	/* The notification is sent with interrupts enabled. */
	113	if (trace_hw_branches_enabled) {
	114	bts_trace_init_cpu(cpu);
	115
	116	if (trace_hw_branches_suspended &&
	117	likely(per_cpu(tracer, cpu)))
	118	ds_suspend_bts(per_cpu(tracer, cpu));
	119	}
5c5317de	120	break;
de79f54f	121
5c5317de	122	case CPU_DOWN_PREPARE:
de79f54f MM	123	/* The notification is sent with interrupts enabled. */
	124	if (likely(per_cpu(tracer, cpu))) {
	125	ds_release_bts(per_cpu(tracer, cpu));
	126	per_cpu(tracer, cpu) = NULL;
	127	}
5c5317de MM	128	}
5c5317de MM	129
5c5317de	130	return NOTIFY_DONE;
1e9b51c2 MM	131	}
1e9b51c2 MM	132
5c5317de MM	133	static struct notifier_block bts_hotcpu_notifier __cpuinitdata = {
	134	.notifier_call = bts_hotcpu_handler
	135	};
	136
1e9b51c2 MM	137	static void bts_trace_print_header(struct seq_file *m)
1e9b51c2 MM	138	{
11edda06	139	seq_puts(m, "# CPU# TO <- FROM\n");
1e9b51c2 MM	140	}
	141
	142	static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
	143	{
e9a22d1f	144	unsigned long symflags = TRACE_ITER_SYM_OFFSET;
1e9b51c2 MM	145	struct trace_entry *entry = iter->ent;
1e9b51c2 MM	146	struct trace_seq *seq = &iter->seq;
a93751ca	147	struct hw_branch_entry *it;
1e9b51c2 MM	148
	149	trace_assign_type(it, entry);
	150
a93751ca	151	if (entry->type == TRACE_HW_BRANCHES) {
1830b52d	152	if (trace_seq_printf(seq, "%4d ", iter->cpu) &&
11edda06 MM	153	seq_print_ip_sym(seq, it->to, symflags) &&
	154	trace_seq_printf(seq, "\t <- ") &&
	155	seq_print_ip_sym(seq, it->from, symflags) &&
a93751ca MM	156	trace_seq_printf(seq, "\n"))
a93751ca MM	157	return TRACE_TYPE_HANDLED;
a419aef8	158	return TRACE_TYPE_PARTIAL_LINE;
1e9b51c2 MM	159	}
	160	return TRACE_TYPE_UNHANDLED;
	161	}
	162
b1818748	163	void trace_hw_branch(u64 from, u64 to)
1e9b51c2	164	{
e1112b4d	165	struct ftrace_event_call *call = &event_hw_branch;
b1818748	166	struct trace_array *tr = hw_branch_trace;
1e9b51c2	167	struct ring_buffer_event *event;
c8647b28	168	struct ring_buffer *buf;
a93751ca	169	struct hw_branch_entry *entry;
0a987751	170	unsigned long irq1;
5c5317de	171	int cpu;
1e9b51c2	172
5c5317de MM	173	if (unlikely(!tr))
	174	return;
	175
	176	if (unlikely(!trace_hw_branches_enabled))
1e9b51c2	177	return;
5c5317de MM	178
	179	local_irq_save(irq1);
	180	cpu = raw_smp_processor_id();
	181	if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
	182	goto out;
	183
c8647b28 ZX	184	buf = tr->buffer;
c8647b28 ZX	185	event = trace_buffer_lock_reserve(buf, TRACE_HW_BRANCHES,
51a763dd	186	sizeof(*entry), 0, 0);
5c5317de MM	187	if (!event)
5c5317de MM	188	goto out;
1e9b51c2 MM	189	entry = ring_buffer_event_data(event);
1e9b51c2 MM	190	tracing_generic_entry_update(&entry->ent, 0, from);
a93751ca	191	entry->ent.type = TRACE_HW_BRANCHES;
1e9b51c2 MM	192	entry->from = from;
1e9b51c2 MM	193	entry->to = to;
c8647b28 ZX	194	if (!filter_check_discard(call, entry, buf, event))
c8647b28 ZX	195	trace_buffer_unlock_commit(buf, event, 0, 0);
5c5317de MM	196
	197	out:
	198	atomic_dec(&tr->data[cpu]->disabled);
	199	local_irq_restore(irq1);
1e9b51c2 MM	200	}
1e9b51c2 MM	201
b1818748	202	static void trace_bts_at(const struct bts_trace trace, void at)
1e9b51c2	203	{
a93751ca MM	204	struct bts_struct bts;
a93751ca MM	205	int err = 0;
1e9b51c2	206
a93751ca MM	207	WARN_ON_ONCE(!trace->read);
a93751ca MM	208	if (!trace->read)
1e9b51c2 MM	209	return;
1e9b51c2 MM	210
a93751ca MM	211	err = trace->read(this_tracer, at, &bts);
	212	if (err < 0)
	213	return;
1e9b51c2	214
a93751ca MM	215	switch (bts.qualifier) {
a93751ca MM	216	case BTS_BRANCH:
b1818748	217	trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to);
a93751ca MM	218	break;
a93751ca MM	219	}
1e9b51c2 MM	220	}
1e9b51c2 MM	221
5c5317de MM	222	/*
	223	* Collect the trace on the current cpu and write it into the ftrace buffer.
	224	*
de79f54f	225	* pre: tracing must be suspended on the current cpu
5c5317de	226	*/
1e9b51c2 MM	227	static void trace_bts_cpu(void *arg)
1e9b51c2 MM	228	{
ba9372a8	229	struct trace_array tr = (struct trace_array )arg;
a93751ca MM	230	const struct bts_trace *trace;
a93751ca MM	231	unsigned char *at;
1e9b51c2	232
b1818748	233	if (unlikely(!tr))
1e9b51c2 MM	234	return;
1e9b51c2 MM	235
5c5317de MM	236	if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled)))
	237	return;
	238
b1818748 MM	239	if (unlikely(!this_tracer))
	240	return;
	241
a93751ca MM	242	trace = ds_read_bts(this_tracer);
a93751ca MM	243	if (!trace)
de79f54f	244	return;
1e9b51c2	245
a93751ca MM	246	for (at = trace->ds.top; (void *)at < trace->ds.end;
a93751ca MM	247	at += trace->ds.size)
b1818748	248	trace_bts_at(trace, at);
1e9b51c2	249
a93751ca MM	250	for (at = trace->ds.begin; (void *)at < trace->ds.top;
a93751ca MM	251	at += trace->ds.size)
b1818748	252	trace_bts_at(trace, at);
1e9b51c2 MM	253	}
	254
	255	static void trace_bts_prepare(struct trace_iterator *iter)
	256	{
de79f54f	257	int cpu;
5c5317de	258
de79f54f MM	259	get_online_cpus();
	260	for_each_online_cpu(cpu)
	261	if (likely(per_cpu(tracer, cpu)))
	262	ds_suspend_bts(per_cpu(tracer, cpu));
	263	/*
	264	* We need to collect the trace on the respective cpu since ftrace
	265	* implicitly adds the record for the current cpu.
	266	* Once that is more flexible, we could collect the data from any cpu.
	267	*/
5c5317de	268	on_each_cpu(trace_bts_cpu, iter->tr, 1);
1e9b51c2	269
de79f54f MM	270	for_each_online_cpu(cpu)
	271	if (likely(per_cpu(tracer, cpu)))
	272	ds_resume_bts(per_cpu(tracer, cpu));
	273	put_online_cpus();
1e9b51c2 MM	274	}
1e9b51c2 MM	275
e23b8ad8 MM	276	static void trace_bts_close(struct trace_iterator *iter)
	277	{
	278	tracing_reset_online_cpus(iter->tr);
	279	}
	280
b1818748 MM	281	void trace_hw_branch_oops(void)
b1818748 MM	282	{
de79f54f MM	283	if (this_tracer) {
de79f54f MM	284	ds_suspend_bts_noirq(this_tracer);
ba9372a8	285	trace_bts_cpu(hw_branch_trace);
de79f54f MM	286	ds_resume_bts_noirq(this_tracer);
de79f54f MM	287	}
b1818748 MM	288	}
b1818748 MM	289
1e9b51c2 MM	290	struct tracer bts_tracer __read_mostly =
1e9b51c2 MM	291	{
a93751ca	292	.name = "hw-branch-tracer",
1e9b51c2	293	.init = bts_trace_init,
5c5317de	294	.reset = bts_trace_reset,
1e9b51c2 MM	295	.print_header = bts_trace_print_header,
	296	.print_line = bts_trace_print_line,
	297	.start = bts_trace_start,
	298	.stop = bts_trace_stop,
e23b8ad8	299	.open = trace_bts_prepare,
321bb5e1 MM	300	.close = trace_bts_close,
	301	#ifdef CONFIG_FTRACE_SELFTEST
	302	.selftest = trace_selftest_startup_hw_branches,
	303	#endif /* CONFIG_FTRACE_SELFTEST */
1e9b51c2 MM	304	};
	305
	306	__init static int init_bts_trace(void)
	307	{
5e01cb69	308	register_hotcpu_notifier(&bts_hotcpu_notifier);
1e9b51c2 MM	309	return register_tracer(&bts_tracer);
	310	}
	311	device_initcall(init_bts_trace);