Commit | Line | Data |
---|---|---|
1e9b51c2 | 1 | /* |
a93751ca | 2 | * h/w branch tracer for x86 based on bts |
1e9b51c2 | 3 | * |
5c5317de MM |
4 | * Copyright (C) 2008-2009 Intel Corporation. |
5 | * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009 | |
1e9b51c2 | 6 | */ |
2d542cf3 IM |
7 | #include <linux/spinlock.h> |
8 | #include <linux/kallsyms.h> | |
1e9b51c2 MM |
9 | #include <linux/debugfs.h> |
10 | #include <linux/ftrace.h> | |
2d542cf3 | 11 | #include <linux/module.h> |
5c5317de MM |
12 | #include <linux/cpu.h> |
13 | #include <linux/smp.h> | |
2d542cf3 | 14 | #include <linux/fs.h> |
1e9b51c2 MM |
15 | |
16 | #include <asm/ds.h> | |
17 | ||
18 | #include "trace.h" | |
f0868d1e | 19 | #include "trace_output.h" |
1e9b51c2 MM |
20 | |
21 | ||
ba9372a8 | 22 | #define BTS_BUFFER_SIZE (1 << 13) |
1e9b51c2 | 23 | |
2d542cf3 IM |
24 | /* |
25 | * The tracer lock protects the below per-cpu tracer array. | |
26 | * It needs to be held to: | |
27 | * - start tracing on all cpus | |
28 | * - stop tracing on all cpus | |
29 | * - start tracing on a single hotplug cpu | |
30 | * - stop tracing on a single hotplug cpu | |
31 | * - read the trace from all cpus | |
32 | * - read the trace from a single cpu | |
33 | */ | |
34 | static DEFINE_SPINLOCK(bts_tracer_lock); | |
1e9b51c2 | 35 | static DEFINE_PER_CPU(struct bts_tracer *, tracer); |
ba9372a8 | 36 | static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], buffer); |
1e9b51c2 MM |
37 | |
38 | #define this_tracer per_cpu(tracer, smp_processor_id()) | |
39 | #define this_buffer per_cpu(buffer, smp_processor_id()) | |
40 | ||
ba9372a8 MM |
41 | static int trace_hw_branches_enabled __read_mostly; |
42 | static int trace_hw_branches_suspended __read_mostly; | |
b1818748 | 43 | static struct trace_array *hw_branch_trace __read_mostly; |
1e9b51c2 | 44 | |
5c5317de MM |
45 | |
46 | /* | |
ba9372a8 | 47 | * Initialize the tracer for the current cpu. |
5c5317de MM |
48 | * The argument is ignored. |
49 | * | |
2d542cf3 | 50 | * pre: bts_tracer_lock must be locked. |
5c5317de | 51 | */ |
ba9372a8 | 52 | static void bts_trace_init_cpu(void *arg) |
1e9b51c2 | 53 | { |
a93751ca MM |
54 | if (this_tracer) |
55 | ds_release_bts(this_tracer); | |
56 | ||
ba9372a8 MM |
57 | this_tracer = ds_request_bts(NULL, this_buffer, BTS_BUFFER_SIZE, |
58 | NULL, (size_t)-1, BTS_KERNEL); | |
1e9b51c2 MM |
59 | if (IS_ERR(this_tracer)) { |
60 | this_tracer = NULL; | |
61 | return; | |
62 | } | |
1e9b51c2 MM |
63 | } |
64 | ||
ba9372a8 | 65 | static int bts_trace_init(struct trace_array *tr) |
1e9b51c2 | 66 | { |
ba9372a8 MM |
67 | int cpu, avail; |
68 | ||
2d542cf3 | 69 | spin_lock(&bts_tracer_lock); |
1e9b51c2 | 70 | |
ba9372a8 MM |
71 | hw_branch_trace = tr; |
72 | ||
73 | on_each_cpu(bts_trace_init_cpu, NULL, 1); | |
74 | ||
75 | /* Check on how many cpus we could enable tracing */ | |
76 | avail = 0; | |
77 | for_each_online_cpu(cpu) | |
78 | if (per_cpu(tracer, cpu)) | |
79 | avail++; | |
80 | ||
81 | trace_hw_branches_enabled = (avail ? 1 : 0); | |
82 | trace_hw_branches_suspended = 0; | |
1e9b51c2 | 83 | |
2d542cf3 | 84 | spin_unlock(&bts_tracer_lock); |
ba9372a8 MM |
85 | |
86 | ||
87 | /* If we could not enable tracing on a single cpu, we fail. */ | |
88 | return avail ? 0 : -EOPNOTSUPP; | |
1e9b51c2 MM |
89 | } |
90 | ||
5c5317de | 91 | /* |
ba9372a8 | 92 | * Release the tracer for the current cpu. |
5c5317de MM |
93 | * The argument is ignored. |
94 | * | |
2d542cf3 | 95 | * pre: bts_tracer_lock must be locked. |
5c5317de | 96 | */ |
ba9372a8 | 97 | static void bts_trace_release_cpu(void *arg) |
1e9b51c2 MM |
98 | { |
99 | if (this_tracer) { | |
1e9b51c2 MM |
100 | ds_release_bts(this_tracer); |
101 | this_tracer = NULL; | |
102 | } | |
103 | } | |
104 | ||
ba9372a8 | 105 | static void bts_trace_reset(struct trace_array *tr) |
1e9b51c2 | 106 | { |
2d542cf3 | 107 | spin_lock(&bts_tracer_lock); |
5c5317de | 108 | |
ba9372a8 | 109 | on_each_cpu(bts_trace_release_cpu, NULL, 1); |
5c5317de | 110 | trace_hw_branches_enabled = 0; |
ba9372a8 MM |
111 | trace_hw_branches_suspended = 0; |
112 | ||
113 | spin_unlock(&bts_tracer_lock); | |
114 | } | |
115 | ||
116 | /* | |
117 | * Resume tracing on the current cpu. | |
118 | * The argument is ignored. | |
119 | * | |
120 | * pre: bts_tracer_lock must be locked. | |
121 | */ | |
122 | static void bts_trace_resume_cpu(void *arg) | |
123 | { | |
124 | if (this_tracer) | |
125 | ds_resume_bts(this_tracer); | |
126 | } | |
127 | ||
128 | static void bts_trace_start(struct trace_array *tr) | |
129 | { | |
130 | spin_lock(&bts_tracer_lock); | |
131 | ||
132 | on_each_cpu(bts_trace_resume_cpu, NULL, 1); | |
133 | trace_hw_branches_suspended = 0; | |
134 | ||
135 | spin_unlock(&bts_tracer_lock); | |
136 | } | |
137 | ||
138 | /* | |
139 | * Suspend tracing on the current cpu. | |
140 | * The argument is ignored. | |
141 | * | |
142 | * pre: bts_tracer_lock must be locked. | |
143 | */ | |
144 | static void bts_trace_suspend_cpu(void *arg) | |
145 | { | |
146 | if (this_tracer) | |
147 | ds_suspend_bts(this_tracer); | |
148 | } | |
149 | ||
150 | static void bts_trace_stop(struct trace_array *tr) | |
151 | { | |
152 | spin_lock(&bts_tracer_lock); | |
153 | ||
154 | on_each_cpu(bts_trace_suspend_cpu, NULL, 1); | |
155 | trace_hw_branches_suspended = 1; | |
1e9b51c2 | 156 | |
2d542cf3 | 157 | spin_unlock(&bts_tracer_lock); |
5c5317de MM |
158 | } |
159 | ||
160 | static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb, | |
161 | unsigned long action, void *hcpu) | |
162 | { | |
163 | unsigned int cpu = (unsigned long)hcpu; | |
164 | ||
2d542cf3 | 165 | spin_lock(&bts_tracer_lock); |
5c5317de MM |
166 | |
167 | if (!trace_hw_branches_enabled) | |
168 | goto out; | |
169 | ||
170 | switch (action) { | |
171 | case CPU_ONLINE: | |
172 | case CPU_DOWN_FAILED: | |
ba9372a8 MM |
173 | smp_call_function_single(cpu, bts_trace_init_cpu, NULL, 1); |
174 | ||
175 | if (trace_hw_branches_suspended) | |
176 | smp_call_function_single(cpu, bts_trace_suspend_cpu, | |
177 | NULL, 1); | |
5c5317de MM |
178 | break; |
179 | case CPU_DOWN_PREPARE: | |
ba9372a8 | 180 | smp_call_function_single(cpu, bts_trace_release_cpu, NULL, 1); |
5c5317de MM |
181 | break; |
182 | } | |
183 | ||
184 | out: | |
2d542cf3 | 185 | spin_unlock(&bts_tracer_lock); |
5c5317de | 186 | return NOTIFY_DONE; |
1e9b51c2 MM |
187 | } |
188 | ||
5c5317de MM |
189 | static struct notifier_block bts_hotcpu_notifier __cpuinitdata = { |
190 | .notifier_call = bts_hotcpu_handler | |
191 | }; | |
192 | ||
1e9b51c2 MM |
193 | static void bts_trace_print_header(struct seq_file *m) |
194 | { | |
11edda06 | 195 | seq_puts(m, "# CPU# TO <- FROM\n"); |
1e9b51c2 MM |
196 | } |
197 | ||
198 | static enum print_line_t bts_trace_print_line(struct trace_iterator *iter) | |
199 | { | |
200 | struct trace_entry *entry = iter->ent; | |
201 | struct trace_seq *seq = &iter->seq; | |
a93751ca | 202 | struct hw_branch_entry *it; |
11edda06 | 203 | unsigned long symflags = TRACE_ITER_SYM_OFFSET; |
1e9b51c2 MM |
204 | |
205 | trace_assign_type(it, entry); | |
206 | ||
a93751ca | 207 | if (entry->type == TRACE_HW_BRANCHES) { |
1830b52d | 208 | if (trace_seq_printf(seq, "%4d ", iter->cpu) && |
11edda06 MM |
209 | seq_print_ip_sym(seq, it->to, symflags) && |
210 | trace_seq_printf(seq, "\t <- ") && | |
211 | seq_print_ip_sym(seq, it->from, symflags) && | |
a93751ca MM |
212 | trace_seq_printf(seq, "\n")) |
213 | return TRACE_TYPE_HANDLED; | |
214 | return TRACE_TYPE_PARTIAL_LINE;; | |
1e9b51c2 MM |
215 | } |
216 | return TRACE_TYPE_UNHANDLED; | |
217 | } | |
218 | ||
b1818748 | 219 | void trace_hw_branch(u64 from, u64 to) |
1e9b51c2 | 220 | { |
b1818748 | 221 | struct trace_array *tr = hw_branch_trace; |
1e9b51c2 | 222 | struct ring_buffer_event *event; |
a93751ca | 223 | struct hw_branch_entry *entry; |
0a987751 | 224 | unsigned long irq1; |
5c5317de | 225 | int cpu; |
1e9b51c2 | 226 | |
5c5317de MM |
227 | if (unlikely(!tr)) |
228 | return; | |
229 | ||
230 | if (unlikely(!trace_hw_branches_enabled)) | |
1e9b51c2 | 231 | return; |
5c5317de MM |
232 | |
233 | local_irq_save(irq1); | |
234 | cpu = raw_smp_processor_id(); | |
235 | if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) | |
236 | goto out; | |
237 | ||
51a763dd ACM |
238 | event = trace_buffer_lock_reserve(tr, TRACE_HW_BRANCHES, |
239 | sizeof(*entry), 0, 0); | |
5c5317de MM |
240 | if (!event) |
241 | goto out; | |
1e9b51c2 MM |
242 | entry = ring_buffer_event_data(event); |
243 | tracing_generic_entry_update(&entry->ent, 0, from); | |
a93751ca | 244 | entry->ent.type = TRACE_HW_BRANCHES; |
1e9b51c2 MM |
245 | entry->from = from; |
246 | entry->to = to; | |
51a763dd | 247 | trace_buffer_unlock_commit(tr, event, 0, 0); |
5c5317de MM |
248 | |
249 | out: | |
250 | atomic_dec(&tr->data[cpu]->disabled); | |
251 | local_irq_restore(irq1); | |
1e9b51c2 MM |
252 | } |
253 | ||
b1818748 | 254 | static void trace_bts_at(const struct bts_trace *trace, void *at) |
1e9b51c2 | 255 | { |
a93751ca MM |
256 | struct bts_struct bts; |
257 | int err = 0; | |
1e9b51c2 | 258 | |
a93751ca MM |
259 | WARN_ON_ONCE(!trace->read); |
260 | if (!trace->read) | |
1e9b51c2 MM |
261 | return; |
262 | ||
a93751ca MM |
263 | err = trace->read(this_tracer, at, &bts); |
264 | if (err < 0) | |
265 | return; | |
1e9b51c2 | 266 | |
a93751ca MM |
267 | switch (bts.qualifier) { |
268 | case BTS_BRANCH: | |
b1818748 | 269 | trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to); |
a93751ca MM |
270 | break; |
271 | } | |
1e9b51c2 MM |
272 | } |
273 | ||
5c5317de MM |
274 | /* |
275 | * Collect the trace on the current cpu and write it into the ftrace buffer. | |
276 | * | |
2d542cf3 | 277 | * pre: bts_tracer_lock must be locked |
5c5317de | 278 | */ |
1e9b51c2 MM |
279 | static void trace_bts_cpu(void *arg) |
280 | { | |
ba9372a8 | 281 | struct trace_array *tr = (struct trace_array *)arg; |
a93751ca MM |
282 | const struct bts_trace *trace; |
283 | unsigned char *at; | |
1e9b51c2 | 284 | |
b1818748 | 285 | if (unlikely(!tr)) |
1e9b51c2 MM |
286 | return; |
287 | ||
5c5317de MM |
288 | if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled))) |
289 | return; | |
290 | ||
b1818748 MM |
291 | if (unlikely(!this_tracer)) |
292 | return; | |
293 | ||
a93751ca MM |
294 | ds_suspend_bts(this_tracer); |
295 | trace = ds_read_bts(this_tracer); | |
296 | if (!trace) | |
1e9b51c2 MM |
297 | goto out; |
298 | ||
a93751ca MM |
299 | for (at = trace->ds.top; (void *)at < trace->ds.end; |
300 | at += trace->ds.size) | |
b1818748 | 301 | trace_bts_at(trace, at); |
1e9b51c2 | 302 | |
a93751ca MM |
303 | for (at = trace->ds.begin; (void *)at < trace->ds.top; |
304 | at += trace->ds.size) | |
b1818748 | 305 | trace_bts_at(trace, at); |
1e9b51c2 MM |
306 | |
307 | out: | |
a93751ca | 308 | ds_resume_bts(this_tracer); |
1e9b51c2 MM |
309 | } |
310 | ||
311 | static void trace_bts_prepare(struct trace_iterator *iter) | |
312 | { | |
2d542cf3 | 313 | spin_lock(&bts_tracer_lock); |
5c5317de MM |
314 | |
315 | on_each_cpu(trace_bts_cpu, iter->tr, 1); | |
1e9b51c2 | 316 | |
2d542cf3 | 317 | spin_unlock(&bts_tracer_lock); |
1e9b51c2 MM |
318 | } |
319 | ||
e23b8ad8 MM |
320 | static void trace_bts_close(struct trace_iterator *iter) |
321 | { | |
322 | tracing_reset_online_cpus(iter->tr); | |
323 | } | |
324 | ||
b1818748 MM |
325 | void trace_hw_branch_oops(void) |
326 | { | |
2d542cf3 | 327 | spin_lock(&bts_tracer_lock); |
b1818748 | 328 | |
ba9372a8 MM |
329 | if (trace_hw_branches_enabled) |
330 | trace_bts_cpu(hw_branch_trace); | |
b1818748 | 331 | |
2d542cf3 | 332 | spin_unlock(&bts_tracer_lock); |
b1818748 MM |
333 | } |
334 | ||
1e9b51c2 MM |
335 | struct tracer bts_tracer __read_mostly = |
336 | { | |
a93751ca | 337 | .name = "hw-branch-tracer", |
1e9b51c2 | 338 | .init = bts_trace_init, |
5c5317de | 339 | .reset = bts_trace_reset, |
1e9b51c2 MM |
340 | .print_header = bts_trace_print_header, |
341 | .print_line = bts_trace_print_line, | |
342 | .start = bts_trace_start, | |
343 | .stop = bts_trace_stop, | |
e23b8ad8 MM |
344 | .open = trace_bts_prepare, |
345 | .close = trace_bts_close | |
1e9b51c2 MM |
346 | }; |
347 | ||
348 | __init static int init_bts_trace(void) | |
349 | { | |
5e01cb69 | 350 | register_hotcpu_notifier(&bts_hotcpu_notifier); |
1e9b51c2 MM |
351 | return register_tracer(&bts_tracer); |
352 | } | |
353 | device_initcall(init_bts_trace); |