Commit | Line | Data |
---|---|---|
1e9b51c2 | 1 | /* |
a93751ca | 2 | * h/w branch tracer for x86 based on bts |
1e9b51c2 | 3 | * |
5c5317de MM |
4 | * Copyright (C) 2008-2009 Intel Corporation. |
5 | * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009 | |
1e9b51c2 MM |
6 | * |
7 | */ | |
8 | ||
9 | #include <linux/module.h> | |
10 | #include <linux/fs.h> | |
11 | #include <linux/debugfs.h> | |
12 | #include <linux/ftrace.h> | |
13 | #include <linux/kallsyms.h> | |
5c5317de MM |
14 | #include <linux/mutex.h> |
15 | #include <linux/cpu.h> | |
16 | #include <linux/smp.h> | |
1e9b51c2 MM |
17 | |
18 | #include <asm/ds.h> | |
19 | ||
20 | #include "trace.h" | |
f0868d1e | 21 | #include "trace_output.h" |
1e9b51c2 MM |
22 | |
23 | ||
24 | #define SIZEOF_BTS (1 << 13) | |
25 | ||
5c5317de MM |
26 | /* The tracer mutex protects the below per-cpu tracer array. |
27 | It needs to be held to: | |
28 | - start tracing on all cpus | |
29 | - stop tracing on all cpus | |
30 | - start tracing on a single hotplug cpu | |
31 | - stop tracing on a single hotplug cpu | |
32 | - read the trace from all cpus | |
33 | - read the trace from a single cpu | |
34 | */ | |
35 | static DEFINE_MUTEX(bts_tracer_mutex); | |
1e9b51c2 MM |
36 | static DEFINE_PER_CPU(struct bts_tracer *, tracer); |
37 | static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer); | |
38 | ||
39 | #define this_tracer per_cpu(tracer, smp_processor_id()) | |
40 | #define this_buffer per_cpu(buffer, smp_processor_id()) | |
41 | ||
5c5317de | 42 | static int __read_mostly trace_hw_branches_enabled; |
b1818748 | 43 | static struct trace_array *hw_branch_trace __read_mostly; |
1e9b51c2 | 44 | |
5c5317de MM |
45 | |
46 | /* | |
47 | * Start tracing on the current cpu. | |
48 | * The argument is ignored. | |
49 | * | |
50 | * pre: bts_tracer_mutex must be locked. | |
51 | */ | |
1e9b51c2 MM |
52 | static void bts_trace_start_cpu(void *arg) |
53 | { | |
a93751ca MM |
54 | if (this_tracer) |
55 | ds_release_bts(this_tracer); | |
56 | ||
1e9b51c2 MM |
57 | this_tracer = |
58 | ds_request_bts(/* task = */ NULL, this_buffer, SIZEOF_BTS, | |
a93751ca MM |
59 | /* ovfl = */ NULL, /* th = */ (size_t)-1, |
60 | BTS_KERNEL); | |
1e9b51c2 MM |
61 | if (IS_ERR(this_tracer)) { |
62 | this_tracer = NULL; | |
63 | return; | |
64 | } | |
1e9b51c2 MM |
65 | } |
66 | ||
67 | static void bts_trace_start(struct trace_array *tr) | |
68 | { | |
5c5317de | 69 | mutex_lock(&bts_tracer_mutex); |
1e9b51c2 | 70 | |
5c5317de MM |
71 | on_each_cpu(bts_trace_start_cpu, NULL, 1); |
72 | trace_hw_branches_enabled = 1; | |
1e9b51c2 | 73 | |
5c5317de | 74 | mutex_unlock(&bts_tracer_mutex); |
1e9b51c2 MM |
75 | } |
76 | ||
5c5317de MM |
77 | /* |
78 | * Start tracing on the current cpu. | |
79 | * The argument is ignored. | |
80 | * | |
81 | * pre: bts_tracer_mutex must be locked. | |
82 | */ | |
1e9b51c2 MM |
83 | static void bts_trace_stop_cpu(void *arg) |
84 | { | |
85 | if (this_tracer) { | |
1e9b51c2 MM |
86 | ds_release_bts(this_tracer); |
87 | this_tracer = NULL; | |
88 | } | |
89 | } | |
90 | ||
91 | static void bts_trace_stop(struct trace_array *tr) | |
92 | { | |
5c5317de MM |
93 | mutex_lock(&bts_tracer_mutex); |
94 | ||
95 | trace_hw_branches_enabled = 0; | |
96 | on_each_cpu(bts_trace_stop_cpu, NULL, 1); | |
1e9b51c2 | 97 | |
5c5317de MM |
98 | mutex_unlock(&bts_tracer_mutex); |
99 | } | |
100 | ||
101 | static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb, | |
102 | unsigned long action, void *hcpu) | |
103 | { | |
104 | unsigned int cpu = (unsigned long)hcpu; | |
105 | ||
106 | mutex_lock(&bts_tracer_mutex); | |
107 | ||
108 | if (!trace_hw_branches_enabled) | |
109 | goto out; | |
110 | ||
111 | switch (action) { | |
112 | case CPU_ONLINE: | |
113 | case CPU_DOWN_FAILED: | |
114 | smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1); | |
115 | break; | |
116 | case CPU_DOWN_PREPARE: | |
1e9b51c2 | 117 | smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1); |
5c5317de MM |
118 | break; |
119 | } | |
120 | ||
121 | out: | |
122 | mutex_unlock(&bts_tracer_mutex); | |
123 | return NOTIFY_DONE; | |
1e9b51c2 MM |
124 | } |
125 | ||
5c5317de MM |
126 | static struct notifier_block bts_hotcpu_notifier __cpuinitdata = { |
127 | .notifier_call = bts_hotcpu_handler | |
128 | }; | |
129 | ||
1e9b51c2 MM |
130 | static int bts_trace_init(struct trace_array *tr) |
131 | { | |
b1818748 MM |
132 | hw_branch_trace = tr; |
133 | ||
5c5317de | 134 | register_hotcpu_notifier(&bts_hotcpu_notifier); |
213cc060 | 135 | tracing_reset_online_cpus(tr); |
1e9b51c2 MM |
136 | bts_trace_start(tr); |
137 | ||
138 | return 0; | |
139 | } | |
140 | ||
5c5317de MM |
141 | static void bts_trace_reset(struct trace_array *tr) |
142 | { | |
143 | bts_trace_stop(tr); | |
144 | unregister_hotcpu_notifier(&bts_hotcpu_notifier); | |
145 | } | |
146 | ||
1e9b51c2 MM |
147 | static void bts_trace_print_header(struct seq_file *m) |
148 | { | |
1e9b51c2 MM |
149 | seq_puts(m, |
150 | "# CPU# FROM TO FUNCTION\n"); | |
151 | seq_puts(m, | |
152 | "# | | | |\n"); | |
1e9b51c2 MM |
153 | } |
154 | ||
155 | static enum print_line_t bts_trace_print_line(struct trace_iterator *iter) | |
156 | { | |
157 | struct trace_entry *entry = iter->ent; | |
158 | struct trace_seq *seq = &iter->seq; | |
a93751ca | 159 | struct hw_branch_entry *it; |
1e9b51c2 MM |
160 | |
161 | trace_assign_type(it, entry); | |
162 | ||
a93751ca MM |
163 | if (entry->type == TRACE_HW_BRANCHES) { |
164 | if (trace_seq_printf(seq, "%4d ", entry->cpu) && | |
165 | trace_seq_printf(seq, "0x%016llx -> 0x%016llx ", | |
166 | it->from, it->to) && | |
167 | (!it->from || | |
168 | seq_print_ip_sym(seq, it->from, /* sym_flags = */ 0)) && | |
169 | trace_seq_printf(seq, "\n")) | |
170 | return TRACE_TYPE_HANDLED; | |
171 | return TRACE_TYPE_PARTIAL_LINE;; | |
1e9b51c2 MM |
172 | } |
173 | return TRACE_TYPE_UNHANDLED; | |
174 | } | |
175 | ||
b1818748 | 176 | void trace_hw_branch(u64 from, u64 to) |
1e9b51c2 | 177 | { |
b1818748 | 178 | struct trace_array *tr = hw_branch_trace; |
1e9b51c2 | 179 | struct ring_buffer_event *event; |
a93751ca | 180 | struct hw_branch_entry *entry; |
5c5317de MM |
181 | unsigned long irq1, irq2; |
182 | int cpu; | |
1e9b51c2 | 183 | |
5c5317de MM |
184 | if (unlikely(!tr)) |
185 | return; | |
186 | ||
187 | if (unlikely(!trace_hw_branches_enabled)) | |
1e9b51c2 | 188 | return; |
5c5317de MM |
189 | |
190 | local_irq_save(irq1); | |
191 | cpu = raw_smp_processor_id(); | |
192 | if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) | |
193 | goto out; | |
194 | ||
195 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), &irq2); | |
196 | if (!event) | |
197 | goto out; | |
1e9b51c2 MM |
198 | entry = ring_buffer_event_data(event); |
199 | tracing_generic_entry_update(&entry->ent, 0, from); | |
a93751ca | 200 | entry->ent.type = TRACE_HW_BRANCHES; |
5c5317de | 201 | entry->ent.cpu = cpu; |
1e9b51c2 MM |
202 | entry->from = from; |
203 | entry->to = to; | |
5c5317de MM |
204 | ring_buffer_unlock_commit(tr->buffer, event, irq2); |
205 | ||
206 | out: | |
207 | atomic_dec(&tr->data[cpu]->disabled); | |
208 | local_irq_restore(irq1); | |
1e9b51c2 MM |
209 | } |
210 | ||
b1818748 | 211 | static void trace_bts_at(const struct bts_trace *trace, void *at) |
1e9b51c2 | 212 | { |
a93751ca MM |
213 | struct bts_struct bts; |
214 | int err = 0; | |
1e9b51c2 | 215 | |
a93751ca MM |
216 | WARN_ON_ONCE(!trace->read); |
217 | if (!trace->read) | |
1e9b51c2 MM |
218 | return; |
219 | ||
a93751ca MM |
220 | err = trace->read(this_tracer, at, &bts); |
221 | if (err < 0) | |
222 | return; | |
1e9b51c2 | 223 | |
a93751ca MM |
224 | switch (bts.qualifier) { |
225 | case BTS_BRANCH: | |
b1818748 | 226 | trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to); |
a93751ca MM |
227 | break; |
228 | } | |
1e9b51c2 MM |
229 | } |
230 | ||
5c5317de MM |
231 | /* |
232 | * Collect the trace on the current cpu and write it into the ftrace buffer. | |
233 | * | |
234 | * pre: bts_tracer_mutex must be locked | |
235 | */ | |
1e9b51c2 MM |
236 | static void trace_bts_cpu(void *arg) |
237 | { | |
238 | struct trace_array *tr = (struct trace_array *) arg; | |
a93751ca MM |
239 | const struct bts_trace *trace; |
240 | unsigned char *at; | |
1e9b51c2 | 241 | |
b1818748 | 242 | if (unlikely(!tr)) |
1e9b51c2 MM |
243 | return; |
244 | ||
5c5317de MM |
245 | if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled))) |
246 | return; | |
247 | ||
b1818748 MM |
248 | if (unlikely(!this_tracer)) |
249 | return; | |
250 | ||
a93751ca MM |
251 | ds_suspend_bts(this_tracer); |
252 | trace = ds_read_bts(this_tracer); | |
253 | if (!trace) | |
1e9b51c2 MM |
254 | goto out; |
255 | ||
a93751ca MM |
256 | for (at = trace->ds.top; (void *)at < trace->ds.end; |
257 | at += trace->ds.size) | |
b1818748 | 258 | trace_bts_at(trace, at); |
1e9b51c2 | 259 | |
a93751ca MM |
260 | for (at = trace->ds.begin; (void *)at < trace->ds.top; |
261 | at += trace->ds.size) | |
b1818748 | 262 | trace_bts_at(trace, at); |
1e9b51c2 MM |
263 | |
264 | out: | |
a93751ca | 265 | ds_resume_bts(this_tracer); |
1e9b51c2 MM |
266 | } |
267 | ||
268 | static void trace_bts_prepare(struct trace_iterator *iter) | |
269 | { | |
5c5317de MM |
270 | mutex_lock(&bts_tracer_mutex); |
271 | ||
272 | on_each_cpu(trace_bts_cpu, iter->tr, 1); | |
1e9b51c2 | 273 | |
5c5317de | 274 | mutex_unlock(&bts_tracer_mutex); |
1e9b51c2 MM |
275 | } |
276 | ||
b1818748 MM |
277 | void trace_hw_branch_oops(void) |
278 | { | |
279 | mutex_lock(&bts_tracer_mutex); | |
280 | ||
281 | trace_bts_cpu(hw_branch_trace); | |
282 | ||
283 | mutex_unlock(&bts_tracer_mutex); | |
284 | } | |
285 | ||
1e9b51c2 MM |
286 | struct tracer bts_tracer __read_mostly = |
287 | { | |
a93751ca | 288 | .name = "hw-branch-tracer", |
1e9b51c2 | 289 | .init = bts_trace_init, |
5c5317de | 290 | .reset = bts_trace_reset, |
1e9b51c2 MM |
291 | .print_header = bts_trace_print_header, |
292 | .print_line = bts_trace_print_line, | |
293 | .start = bts_trace_start, | |
294 | .stop = bts_trace_stop, | |
295 | .open = trace_bts_prepare | |
296 | }; | |
297 | ||
298 | __init static int init_bts_trace(void) | |
299 | { | |
300 | return register_tracer(&bts_tracer); | |
301 | } | |
302 | device_initcall(init_bts_trace); |