[deliverable/linux.git] / kernel / trace / trace_hw_branches.c

/*
 * h/w branch tracer for x86 based on BTS
 *
 * Copyright (C) 2008-2009 Intel Corporation.
 * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009
 */
#include <linux/kallsyms.h>
#include <linux/debugfs.h>
#include <linux/ftrace.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/fs.h>

#include <asm/ds.h>

#include "trace_output.h"
#include "trace.h"


#define BTS_BUFFER_SIZE (1 << 13)

static DEFINE_PER_CPU(struct bts_tracer *, tracer);
static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], buffer);

#define this_tracer per_cpu(tracer, smp_processor_id())

static int trace_hw_branches_enabled __read_mostly;
static int trace_hw_branches_suspended __read_mostly;
static struct trace_array *hw_branch_trace __read_mostly;


static void bts_trace_init_cpu(int cpu)
{
	per_cpu(tracer, cpu) =
		ds_request_bts_cpu(cpu, per_cpu(buffer, cpu), BTS_BUFFER_SIZE,
				   NULL, (size_t)-1, BTS_KERNEL);

	if (IS_ERR(per_cpu(tracer, cpu)))
		per_cpu(tracer, cpu) = NULL;
}

static int bts_trace_init(struct trace_array *tr)
{
	int cpu;

	hw_branch_trace = tr;
	trace_hw_branches_enabled = 0;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		bts_trace_init_cpu(cpu);

		if (likely(per_cpu(tracer, cpu)))
			trace_hw_branches_enabled = 1;
	}
	trace_hw_branches_suspended = 0;
	put_online_cpus();

	/* If we could not enable tracing on a single cpu, we fail. */
	return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP;
}

static void bts_trace_reset(struct trace_array *tr)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu) {
		if (likely(per_cpu(tracer, cpu))) {
			ds_release_bts(per_cpu(tracer, cpu));
			per_cpu(tracer, cpu) = NULL;
		}
	}
	trace_hw_branches_enabled = 0;
	trace_hw_branches_suspended = 0;
	put_online_cpus();
}

static void bts_trace_start(struct trace_array *tr)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_resume_bts(per_cpu(tracer, cpu));
	trace_hw_branches_suspended = 0;
	put_online_cpus();
}

static void bts_trace_stop(struct trace_array *tr)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_suspend_bts(per_cpu(tracer, cpu));
	trace_hw_branches_suspended = 1;
	put_online_cpus();
}

static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb,
				     unsigned long action, void *hcpu)
{
	int cpu = (long)hcpu;

	switch (action) {
	case CPU_ONLINE:
	case CPU_DOWN_FAILED:
		/* The notification is sent with interrupts enabled. */
		if (trace_hw_branches_enabled) {
			bts_trace_init_cpu(cpu);

			if (trace_hw_branches_suspended &&
			    likely(per_cpu(tracer, cpu)))
				ds_suspend_bts(per_cpu(tracer, cpu));
		}
		break;

	case CPU_DOWN_PREPARE:
		/* The notification is sent with interrupts enabled. */
		if (likely(per_cpu(tracer, cpu))) {
			ds_release_bts(per_cpu(tracer, cpu));
			per_cpu(tracer, cpu) = NULL;
		}
	}

	return NOTIFY_DONE;
}

static struct notifier_block bts_hotcpu_notifier __cpuinitdata = {
	.notifier_call = bts_hotcpu_handler
};

static void bts_trace_print_header(struct seq_file *m)
{
	seq_puts(m, "# CPU#        TO  <-  FROM\n");
}

static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
{
	unsigned long symflags = TRACE_ITER_SYM_OFFSET;
	struct trace_entry *entry = iter->ent;
	struct trace_seq *seq = &iter->seq;
	struct hw_branch_entry *it;

	trace_assign_type(it, entry);

	if (entry->type == TRACE_HW_BRANCHES) {
		if (trace_seq_printf(seq, "%4d  ", iter->cpu) &&
		    seq_print_ip_sym(seq, it->to, symflags) &&
		    trace_seq_printf(seq, "\t  <-  ") &&
		    seq_print_ip_sym(seq, it->from, symflags) &&
		    trace_seq_printf(seq, "\n"))
			return TRACE_TYPE_HANDLED;
		return TRACE_TYPE_PARTIAL_LINE;;
	}
	return TRACE_TYPE_UNHANDLED;
}

void trace_hw_branch(u64 from, u64 to)
{
	struct ftrace_event_call *call = &event_hw_branch;
	struct trace_array *tr = hw_branch_trace;
	struct ring_buffer_event *event;
	struct hw_branch_entry *entry;
	unsigned long irq1;
	int cpu;

	if (unlikely(!tr))
		return;

	if (unlikely(!trace_hw_branches_enabled))
		return;

	local_irq_save(irq1);
	cpu = raw_smp_processor_id();
	if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
		goto out;

	event = trace_buffer_lock_reserve(tr, TRACE_HW_BRANCHES,
					  sizeof(*entry), 0, 0);
	if (!event)
		goto out;
	entry	= ring_buffer_event_data(event);
	tracing_generic_entry_update(&entry->ent, 0, from);
	entry->ent.type = TRACE_HW_BRANCHES;
	entry->from = from;
	entry->to   = to;
	if (!filter_check_discard(call, entry, tr->buffer, event))
		trace_buffer_unlock_commit(tr, event, 0, 0);

 out:
	atomic_dec(&tr->data[cpu]->disabled);
	local_irq_restore(irq1);
}

static void trace_bts_at(const struct bts_trace *trace, void *at)
{
	struct bts_struct bts;
	int err = 0;

	WARN_ON_ONCE(!trace->read);
	if (!trace->read)
		return;

	err = trace->read(this_tracer, at, &bts);
	if (err < 0)
		return;

	switch (bts.qualifier) {
	case BTS_BRANCH:
		trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to);
		break;
	}
}

/*
 * Collect the trace on the current cpu and write it into the ftrace buffer.
 *
 * pre: tracing must be suspended on the current cpu
 */
static void trace_bts_cpu(void *arg)
{
	struct trace_array *tr = (struct trace_array *)arg;
	const struct bts_trace *trace;
	unsigned char *at;

	if (unlikely(!tr))
		return;

	if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled)))
		return;

	if (unlikely(!this_tracer))
		return;

	trace = ds_read_bts(this_tracer);
	if (!trace)
		return;

	for (at = trace->ds.top; (void *)at < trace->ds.end;
	     at += trace->ds.size)
		trace_bts_at(trace, at);

	for (at = trace->ds.begin; (void *)at < trace->ds.top;
	     at += trace->ds.size)
		trace_bts_at(trace, at);
}

static void trace_bts_prepare(struct trace_iterator *iter)
{
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_suspend_bts(per_cpu(tracer, cpu));
	/*
	 * We need to collect the trace on the respective cpu since ftrace
	 * implicitly adds the record for the current cpu.
	 * Once that is more flexible, we could collect the data from any cpu.
	 */
	on_each_cpu(trace_bts_cpu, iter->tr, 1);

	for_each_online_cpu(cpu)
		if (likely(per_cpu(tracer, cpu)))
			ds_resume_bts(per_cpu(tracer, cpu));
	put_online_cpus();
}

static void trace_bts_close(struct trace_iterator *iter)
{
	tracing_reset_online_cpus(iter->tr);
}

void trace_hw_branch_oops(void)
{
	if (this_tracer) {
		ds_suspend_bts_noirq(this_tracer);
		trace_bts_cpu(hw_branch_trace);
		ds_resume_bts_noirq(this_tracer);
	}
}

struct tracer bts_tracer __read_mostly =
{
	.name		= "hw-branch-tracer",
	.init		= bts_trace_init,
	.reset		= bts_trace_reset,
	.print_header	= bts_trace_print_header,
	.print_line	= bts_trace_print_line,
	.start		= bts_trace_start,
	.stop		= bts_trace_stop,
	.open		= trace_bts_prepare,
	.close		= trace_bts_close,
#ifdef CONFIG_FTRACE_SELFTEST
	.selftest	= trace_selftest_startup_hw_branches,
#endif /* CONFIG_FTRACE_SELFTEST */
};

__init static int init_bts_trace(void)
{
	register_hotcpu_notifier(&bts_hotcpu_notifier);
	return register_tracer(&bts_tracer);
}
device_initcall(init_bts_trace);
Commit	Line	Data
1e9b51c2	1	/*
e9a22d1f	2	* h/w branch tracer for x86 based on BTS
1e9b51c2	3	*
5c5317de MM	4	* Copyright (C) 2008-2009 Intel Corporation.
5c5317de MM	5	* Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009
1e9b51c2	6	*/
2d542cf3	7	#include <linux/kallsyms.h>
1e9b51c2 MM	8	#include <linux/debugfs.h>
1e9b51c2 MM	9	#include <linux/ftrace.h>
2d542cf3	10	#include <linux/module.h>
5c5317de MM	11	#include <linux/cpu.h>
5c5317de MM	12	#include <linux/smp.h>
2d542cf3	13	#include <linux/fs.h>
1e9b51c2 MM	14
	15	#include <asm/ds.h>
	16
f0868d1e	17	#include "trace_output.h"
e9a22d1f	18	#include "trace.h"
1e9b51c2 MM	19
1e9b51c2 MM	20
ba9372a8	21	#define BTS_BUFFER_SIZE (1 << 13)
1e9b51c2 MM	22
1e9b51c2 MM	23	static DEFINE_PER_CPU(struct bts_tracer *, tracer);
ba9372a8	24	static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], buffer);
1e9b51c2 MM	25
1e9b51c2 MM	26	#define this_tracer per_cpu(tracer, smp_processor_id())
1e9b51c2	27
ba9372a8 MM	28	static int trace_hw_branches_enabled __read_mostly;
ba9372a8 MM	29	static int trace_hw_branches_suspended __read_mostly;
b1818748	30	static struct trace_array *hw_branch_trace __read_mostly;
1e9b51c2	31
5c5317de	32
de79f54f	33	static void bts_trace_init_cpu(int cpu)
1e9b51c2	34	{
de79f54f MM	35	per_cpu(tracer, cpu) =
	36	ds_request_bts_cpu(cpu, per_cpu(buffer, cpu), BTS_BUFFER_SIZE,
	37	NULL, (size_t)-1, BTS_KERNEL);
a93751ca	38
de79f54f MM	39	if (IS_ERR(per_cpu(tracer, cpu)))
de79f54f MM	40	per_cpu(tracer, cpu) = NULL;
1e9b51c2 MM	41	}
1e9b51c2 MM	42
ba9372a8	43	static int bts_trace_init(struct trace_array *tr)
1e9b51c2	44	{
de79f54f	45	int cpu;
1e9b51c2	46
ba9372a8	47	hw_branch_trace = tr;
de79f54f	48	trace_hw_branches_enabled = 0;
1e9b51c2	49
de79f54f MM	50	get_online_cpus();
	51	for_each_online_cpu(cpu) {
	52	bts_trace_init_cpu(cpu);
1e9b51c2	53
de79f54f MM	54	if (likely(per_cpu(tracer, cpu)))
	55	trace_hw_branches_enabled = 1;
	56	}
ba9372a8	57	trace_hw_branches_suspended = 0;
de79f54f	58	put_online_cpus();
ba9372a8 MM	59
ba9372a8 MM	60	/* If we could not enable tracing on a single cpu, we fail. */
de79f54f	61	return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP;
1e9b51c2 MM	62	}
1e9b51c2 MM	63
ba9372a8	64	static void bts_trace_reset(struct trace_array *tr)
1e9b51c2	65	{
de79f54f	66	int cpu;
5c5317de	67
de79f54f MM	68	get_online_cpus();
	69	for_each_online_cpu(cpu) {
	70	if (likely(per_cpu(tracer, cpu))) {
	71	ds_release_bts(per_cpu(tracer, cpu));
	72	per_cpu(tracer, cpu) = NULL;
	73	}
1e9b51c2	74	}
5c5317de	75	trace_hw_branches_enabled = 0;
ba9372a8	76	trace_hw_branches_suspended = 0;
de79f54f	77	put_online_cpus();
1e9b51c2 MM	78	}
1e9b51c2 MM	79
ba9372a8	80	static void bts_trace_start(struct trace_array *tr)
1e9b51c2	81	{
de79f54f	82	int cpu;
5c5317de	83
de79f54f MM	84	get_online_cpus();
	85	for_each_online_cpu(cpu)
	86	if (likely(per_cpu(tracer, cpu)))
	87	ds_resume_bts(per_cpu(tracer, cpu));
ba9372a8	88	trace_hw_branches_suspended = 0;
de79f54f	89	put_online_cpus();
ba9372a8	90	}
1e9b51c2	91
ba9372a8 MM	92	static void bts_trace_stop(struct trace_array *tr)
ba9372a8 MM	93	{
de79f54f	94	int cpu;
ba9372a8	95
de79f54f MM	96	get_online_cpus();
	97	for_each_online_cpu(cpu)
	98	if (likely(per_cpu(tracer, cpu)))
	99	ds_suspend_bts(per_cpu(tracer, cpu));
ba9372a8	100	trace_hw_branches_suspended = 1;
de79f54f	101	put_online_cpus();
5c5317de MM	102	}
	103
	104	static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb,
	105	unsigned long action, void *hcpu)
	106	{
de79f54f	107	int cpu = (long)hcpu;
5c5317de MM	108
	109	switch (action) {
	110	case CPU_ONLINE:
	111	case CPU_DOWN_FAILED:
de79f54f MM	112	/* The notification is sent with interrupts enabled. */
	113	if (trace_hw_branches_enabled) {
	114	bts_trace_init_cpu(cpu);
	115
	116	if (trace_hw_branches_suspended &&
	117	likely(per_cpu(tracer, cpu)))
	118	ds_suspend_bts(per_cpu(tracer, cpu));
	119	}
5c5317de	120	break;
de79f54f	121
5c5317de	122	case CPU_DOWN_PREPARE:
de79f54f MM	123	/* The notification is sent with interrupts enabled. */
	124	if (likely(per_cpu(tracer, cpu))) {
	125	ds_release_bts(per_cpu(tracer, cpu));
	126	per_cpu(tracer, cpu) = NULL;
	127	}
5c5317de MM	128	}
5c5317de MM	129
5c5317de	130	return NOTIFY_DONE;
1e9b51c2 MM	131	}
1e9b51c2 MM	132
5c5317de MM	133	static struct notifier_block bts_hotcpu_notifier __cpuinitdata = {
	134	.notifier_call = bts_hotcpu_handler
	135	};
	136
1e9b51c2 MM	137	static void bts_trace_print_header(struct seq_file *m)
1e9b51c2 MM	138	{
11edda06	139	seq_puts(m, "# CPU# TO <- FROM\n");
1e9b51c2 MM	140	}
	141
	142	static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
	143	{
e9a22d1f	144	unsigned long symflags = TRACE_ITER_SYM_OFFSET;
1e9b51c2 MM	145	struct trace_entry *entry = iter->ent;
1e9b51c2 MM	146	struct trace_seq *seq = &iter->seq;
a93751ca	147	struct hw_branch_entry *it;
1e9b51c2 MM	148
	149	trace_assign_type(it, entry);
	150
a93751ca	151	if (entry->type == TRACE_HW_BRANCHES) {
1830b52d	152	if (trace_seq_printf(seq, "%4d ", iter->cpu) &&
11edda06 MM	153	seq_print_ip_sym(seq, it->to, symflags) &&
	154	trace_seq_printf(seq, "\t <- ") &&
	155	seq_print_ip_sym(seq, it->from, symflags) &&
a93751ca MM	156	trace_seq_printf(seq, "\n"))
	157	return TRACE_TYPE_HANDLED;
	158	return TRACE_TYPE_PARTIAL_LINE;;
1e9b51c2 MM	159	}
	160	return TRACE_TYPE_UNHANDLED;
	161	}
	162
b1818748	163	void trace_hw_branch(u64 from, u64 to)
1e9b51c2	164	{
e1112b4d	165	struct ftrace_event_call *call = &event_hw_branch;
b1818748	166	struct trace_array *tr = hw_branch_trace;
1e9b51c2	167	struct ring_buffer_event *event;
a93751ca	168	struct hw_branch_entry *entry;
0a987751	169	unsigned long irq1;
5c5317de	170	int cpu;
1e9b51c2	171
5c5317de MM	172	if (unlikely(!tr))
	173	return;
	174
	175	if (unlikely(!trace_hw_branches_enabled))
1e9b51c2	176	return;
5c5317de MM	177
	178	local_irq_save(irq1);
	179	cpu = raw_smp_processor_id();
	180	if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
	181	goto out;
	182
51a763dd ACM	183	event = trace_buffer_lock_reserve(tr, TRACE_HW_BRANCHES,
51a763dd ACM	184	sizeof(*entry), 0, 0);
5c5317de MM	185	if (!event)
5c5317de MM	186	goto out;
1e9b51c2 MM	187	entry = ring_buffer_event_data(event);
1e9b51c2 MM	188	tracing_generic_entry_update(&entry->ent, 0, from);
a93751ca	189	entry->ent.type = TRACE_HW_BRANCHES;
1e9b51c2 MM	190	entry->from = from;
1e9b51c2 MM	191	entry->to = to;
eb02ce01 TZ	192	if (!filter_check_discard(call, entry, tr->buffer, event))
eb02ce01 TZ	193	trace_buffer_unlock_commit(tr, event, 0, 0);
5c5317de MM	194
	195	out:
	196	atomic_dec(&tr->data[cpu]->disabled);
	197	local_irq_restore(irq1);
1e9b51c2 MM	198	}
1e9b51c2 MM	199
b1818748	200	static void trace_bts_at(const struct bts_trace trace, void at)
1e9b51c2	201	{
a93751ca MM	202	struct bts_struct bts;
a93751ca MM	203	int err = 0;
1e9b51c2	204
a93751ca MM	205	WARN_ON_ONCE(!trace->read);
a93751ca MM	206	if (!trace->read)
1e9b51c2 MM	207	return;
1e9b51c2 MM	208
a93751ca MM	209	err = trace->read(this_tracer, at, &bts);
	210	if (err < 0)
	211	return;
1e9b51c2	212
a93751ca MM	213	switch (bts.qualifier) {
a93751ca MM	214	case BTS_BRANCH:
b1818748	215	trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to);
a93751ca MM	216	break;
a93751ca MM	217	}
1e9b51c2 MM	218	}
1e9b51c2 MM	219
5c5317de MM	220	/*
	221	* Collect the trace on the current cpu and write it into the ftrace buffer.
	222	*
de79f54f	223	* pre: tracing must be suspended on the current cpu
5c5317de	224	*/
1e9b51c2 MM	225	static void trace_bts_cpu(void *arg)
1e9b51c2 MM	226	{
ba9372a8	227	struct trace_array tr = (struct trace_array )arg;
a93751ca MM	228	const struct bts_trace *trace;
a93751ca MM	229	unsigned char *at;
1e9b51c2	230
b1818748	231	if (unlikely(!tr))
1e9b51c2 MM	232	return;
1e9b51c2 MM	233
5c5317de MM	234	if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled)))
	235	return;
	236
b1818748 MM	237	if (unlikely(!this_tracer))
	238	return;
	239
a93751ca MM	240	trace = ds_read_bts(this_tracer);
a93751ca MM	241	if (!trace)
de79f54f	242	return;
1e9b51c2	243
a93751ca MM	244	for (at = trace->ds.top; (void *)at < trace->ds.end;
a93751ca MM	245	at += trace->ds.size)
b1818748	246	trace_bts_at(trace, at);
1e9b51c2	247
a93751ca MM	248	for (at = trace->ds.begin; (void *)at < trace->ds.top;
a93751ca MM	249	at += trace->ds.size)
b1818748	250	trace_bts_at(trace, at);
1e9b51c2 MM	251	}
	252
	253	static void trace_bts_prepare(struct trace_iterator *iter)
	254	{
de79f54f	255	int cpu;
5c5317de	256
de79f54f MM	257	get_online_cpus();
	258	for_each_online_cpu(cpu)
	259	if (likely(per_cpu(tracer, cpu)))
	260	ds_suspend_bts(per_cpu(tracer, cpu));
	261	/*
	262	* We need to collect the trace on the respective cpu since ftrace
	263	* implicitly adds the record for the current cpu.
	264	* Once that is more flexible, we could collect the data from any cpu.
	265	*/
5c5317de	266	on_each_cpu(trace_bts_cpu, iter->tr, 1);
1e9b51c2	267
de79f54f MM	268	for_each_online_cpu(cpu)
	269	if (likely(per_cpu(tracer, cpu)))
	270	ds_resume_bts(per_cpu(tracer, cpu));
	271	put_online_cpus();
1e9b51c2 MM	272	}
1e9b51c2 MM	273
e23b8ad8 MM	274	static void trace_bts_close(struct trace_iterator *iter)
	275	{
	276	tracing_reset_online_cpus(iter->tr);
	277	}
	278
b1818748 MM	279	void trace_hw_branch_oops(void)
b1818748 MM	280	{
de79f54f MM	281	if (this_tracer) {
de79f54f MM	282	ds_suspend_bts_noirq(this_tracer);
ba9372a8	283	trace_bts_cpu(hw_branch_trace);
de79f54f MM	284	ds_resume_bts_noirq(this_tracer);
de79f54f MM	285	}
b1818748 MM	286	}
b1818748 MM	287
1e9b51c2 MM	288	struct tracer bts_tracer __read_mostly =
1e9b51c2 MM	289	{
a93751ca	290	.name = "hw-branch-tracer",
1e9b51c2	291	.init = bts_trace_init,
5c5317de	292	.reset = bts_trace_reset,
1e9b51c2 MM	293	.print_header = bts_trace_print_header,
	294	.print_line = bts_trace_print_line,
	295	.start = bts_trace_start,
	296	.stop = bts_trace_stop,
e23b8ad8	297	.open = trace_bts_prepare,
321bb5e1 MM	298	.close = trace_bts_close,
	299	#ifdef CONFIG_FTRACE_SELFTEST
	300	.selftest = trace_selftest_startup_hw_branches,
	301	#endif /* CONFIG_FTRACE_SELFTEST */
1e9b51c2 MM	302	};
	303
	304	__init static int init_bts_trace(void)
	305	{
5e01cb69	306	register_hotcpu_notifier(&bts_hotcpu_notifier);
1e9b51c2 MM	307	return register_tracer(&bts_tracer);
	308	}
	309	device_initcall(init_bts_trace);