Commit | Line | Data |
---|---|---|
0067f129 P |
1 | /* |
2 | * This program is free software; you can redistribute it and/or modify | |
3 | * it under the terms of the GNU General Public License as published by | |
4 | * the Free Software Foundation; either version 2 of the License, or | |
5 | * (at your option) any later version. | |
6 | * | |
7 | * This program is distributed in the hope that it will be useful, | |
8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
10 | * GNU General Public License for more details. | |
11 | * | |
12 | * You should have received a copy of the GNU General Public License | |
13 | * along with this program; if not, write to the Free Software | |
14 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
15 | * | |
16 | * Copyright (C) 2007 Alan Stern | |
17 | * Copyright (C) 2009 IBM Corporation | |
24f1e32c | 18 | * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com> |
ba6909b7 P |
19 | * |
20 | * Authors: Alan Stern <stern@rowland.harvard.edu> | |
21 | * K.Prasad <prasad@linux.vnet.ibm.com> | |
22 | * Frederic Weisbecker <fweisbec@gmail.com> | |
0067f129 P |
23 | */ |
24 | ||
25 | /* | |
26 | * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, | |
27 | * using the CPU's debug registers. | |
28 | */ | |
29 | ||
24f1e32c FW |
30 | #include <linux/perf_event.h> |
31 | #include <linux/hw_breakpoint.h> | |
0067f129 P |
32 | #include <linux/irqflags.h> |
33 | #include <linux/notifier.h> | |
34 | #include <linux/kallsyms.h> | |
0067f129 P |
35 | #include <linux/percpu.h> |
36 | #include <linux/kdebug.h> | |
37 | #include <linux/kernel.h> | |
38 | #include <linux/module.h> | |
39 | #include <linux/sched.h> | |
0067f129 P |
40 | #include <linux/smp.h> |
41 | ||
42 | #include <asm/hw_breakpoint.h> | |
43 | #include <asm/processor.h> | |
44 | #include <asm/debugreg.h> | |
45 | ||
24f1e32c | 46 | /* Per cpu debug control register value */ |
28b4e0d8 TH |
47 | DEFINE_PER_CPU(unsigned long, cpu_dr7); |
48 | EXPORT_PER_CPU_SYMBOL(cpu_dr7); | |
24f1e32c FW |
49 | |
50 | /* Per cpu debug address registers values */ | |
51 | static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]); | |
0067f129 P |
52 | |
53 | /* | |
24f1e32c FW |
54 | * Stores the breakpoints currently in use on each breakpoint address |
55 | * register for each cpus | |
0067f129 | 56 | */ |
24f1e32c | 57 | static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]); |
0067f129 P |
58 | |
59 | ||
2c31b795 FW |
60 | static inline unsigned long |
61 | __encode_dr7(int drnum, unsigned int len, unsigned int type) | |
0067f129 P |
62 | { |
63 | unsigned long bp_info; | |
64 | ||
65 | bp_info = (len | type) & 0xf; | |
66 | bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE); | |
2c31b795 FW |
67 | bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)); |
68 | ||
0067f129 P |
69 | return bp_info; |
70 | } | |
71 | ||
2c31b795 FW |
72 | /* |
73 | * Encode the length, type, Exact, and Enable bits for a particular breakpoint | |
74 | * as stored in debug register 7. | |
75 | */ | |
76 | unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type) | |
77 | { | |
78 | return __encode_dr7(drnum, len, type) | DR_GLOBAL_SLOWDOWN; | |
79 | } | |
80 | ||
24f1e32c FW |
81 | /* |
82 | * Decode the length and type bits for a particular breakpoint as | |
83 | * stored in debug register 7. Return the "enabled" status. | |
84 | */ | |
85 | int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type) | |
0067f129 | 86 | { |
24f1e32c | 87 | int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE); |
0067f129 | 88 | |
24f1e32c FW |
89 | *len = (bp_info & 0xc) | 0x40; |
90 | *type = (bp_info & 0x3) | 0x80; | |
0067f129 | 91 | |
24f1e32c | 92 | return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3; |
0067f129 P |
93 | } |
94 | ||
95 | /* | |
24f1e32c FW |
96 | * Install a perf counter breakpoint. |
97 | * | |
98 | * We seek a free debug address register and use it for this | |
99 | * breakpoint. Eventually we enable it in the debug control register. | |
100 | * | |
101 | * Atomic: we hold the counter->ctx->lock and we only handle variables | |
102 | * and registers local to this cpu. | |
0067f129 | 103 | */ |
24f1e32c | 104 | int arch_install_hw_breakpoint(struct perf_event *bp) |
0067f129 | 105 | { |
24f1e32c FW |
106 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
107 | unsigned long *dr7; | |
108 | int i; | |
109 | ||
110 | for (i = 0; i < HBP_NUM; i++) { | |
111 | struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]); | |
112 | ||
113 | if (!*slot) { | |
114 | *slot = bp; | |
115 | break; | |
116 | } | |
0067f129 P |
117 | } |
118 | ||
24f1e32c FW |
119 | if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) |
120 | return -EBUSY; | |
121 | ||
122 | set_debugreg(info->address, i); | |
0a3aee0d | 123 | __this_cpu_write(cpu_debugreg[i], info->address); |
24f1e32c | 124 | |
28b4e0d8 | 125 | dr7 = &__get_cpu_var(cpu_dr7); |
24f1e32c FW |
126 | *dr7 |= encode_dr7(i, info->len, info->type); |
127 | ||
128 | set_debugreg(*dr7, 7); | |
129 | ||
130 | return 0; | |
0067f129 P |
131 | } |
132 | ||
133 | /* | |
24f1e32c FW |
134 | * Uninstall the breakpoint contained in the given counter. |
135 | * | |
136 | * First we search the debug address register it uses and then we disable | |
137 | * it. | |
138 | * | |
139 | * Atomic: we hold the counter->ctx->lock and we only handle variables | |
140 | * and registers local to this cpu. | |
0067f129 | 141 | */ |
24f1e32c | 142 | void arch_uninstall_hw_breakpoint(struct perf_event *bp) |
0067f129 | 143 | { |
24f1e32c FW |
144 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
145 | unsigned long *dr7; | |
146 | int i; | |
147 | ||
148 | for (i = 0; i < HBP_NUM; i++) { | |
149 | struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]); | |
150 | ||
151 | if (*slot == bp) { | |
152 | *slot = NULL; | |
153 | break; | |
154 | } | |
155 | } | |
156 | ||
157 | if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) | |
158 | return; | |
0067f129 | 159 | |
28b4e0d8 | 160 | dr7 = &__get_cpu_var(cpu_dr7); |
2c31b795 | 161 | *dr7 &= ~__encode_dr7(i, info->len, info->type); |
24f1e32c FW |
162 | |
163 | set_debugreg(*dr7, 7); | |
0067f129 P |
164 | } |
165 | ||
166 | static int get_hbp_len(u8 hbp_len) | |
167 | { | |
168 | unsigned int len_in_bytes = 0; | |
169 | ||
170 | switch (hbp_len) { | |
24f1e32c | 171 | case X86_BREAKPOINT_LEN_1: |
0067f129 P |
172 | len_in_bytes = 1; |
173 | break; | |
24f1e32c | 174 | case X86_BREAKPOINT_LEN_2: |
0067f129 P |
175 | len_in_bytes = 2; |
176 | break; | |
24f1e32c | 177 | case X86_BREAKPOINT_LEN_4: |
0067f129 P |
178 | len_in_bytes = 4; |
179 | break; | |
180 | #ifdef CONFIG_X86_64 | |
24f1e32c | 181 | case X86_BREAKPOINT_LEN_8: |
0067f129 P |
182 | len_in_bytes = 8; |
183 | break; | |
184 | #endif | |
185 | } | |
186 | return len_in_bytes; | |
187 | } | |
188 | ||
0067f129 P |
189 | /* |
190 | * Check for virtual address in kernel space. | |
191 | */ | |
b2812d03 | 192 | int arch_check_bp_in_kernelspace(struct perf_event *bp) |
0067f129 P |
193 | { |
194 | unsigned int len; | |
b2812d03 FW |
195 | unsigned long va; |
196 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
0067f129 | 197 | |
b2812d03 FW |
198 | va = info->address; |
199 | len = get_hbp_len(info->len); | |
0067f129 P |
200 | |
201 | return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); | |
202 | } | |
203 | ||
24f1e32c FW |
204 | int arch_bp_generic_fields(int x86_len, int x86_type, |
205 | int *gen_len, int *gen_type) | |
0067f129 | 206 | { |
89e45aac FW |
207 | /* Type */ |
208 | switch (x86_type) { | |
209 | case X86_BREAKPOINT_EXECUTE: | |
210 | if (x86_len != X86_BREAKPOINT_LEN_X) | |
211 | return -EINVAL; | |
212 | ||
213 | *gen_type = HW_BREAKPOINT_X; | |
f7809daf | 214 | *gen_len = sizeof(long); |
89e45aac FW |
215 | return 0; |
216 | case X86_BREAKPOINT_WRITE: | |
217 | *gen_type = HW_BREAKPOINT_W; | |
f7809daf | 218 | break; |
89e45aac FW |
219 | case X86_BREAKPOINT_RW: |
220 | *gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R; | |
221 | break; | |
222 | default: | |
223 | return -EINVAL; | |
224 | } | |
225 | ||
226 | /* Len */ | |
227 | switch (x86_len) { | |
24f1e32c FW |
228 | case X86_BREAKPOINT_LEN_1: |
229 | *gen_len = HW_BREAKPOINT_LEN_1; | |
230 | break; | |
231 | case X86_BREAKPOINT_LEN_2: | |
232 | *gen_len = HW_BREAKPOINT_LEN_2; | |
233 | break; | |
234 | case X86_BREAKPOINT_LEN_4: | |
235 | *gen_len = HW_BREAKPOINT_LEN_4; | |
236 | break; | |
237 | #ifdef CONFIG_X86_64 | |
238 | case X86_BREAKPOINT_LEN_8: | |
239 | *gen_len = HW_BREAKPOINT_LEN_8; | |
240 | break; | |
241 | #endif | |
242 | default: | |
243 | return -EINVAL; | |
244 | } | |
0067f129 | 245 | |
24f1e32c FW |
246 | return 0; |
247 | } | |
248 | ||
249 | ||
250 | static int arch_build_bp_info(struct perf_event *bp) | |
251 | { | |
252 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
253 | ||
254 | info->address = bp->attr.bp_addr; | |
255 | ||
f7809daf FW |
256 | /* Type */ |
257 | switch (bp->attr.bp_type) { | |
258 | case HW_BREAKPOINT_W: | |
259 | info->type = X86_BREAKPOINT_WRITE; | |
260 | break; | |
261 | case HW_BREAKPOINT_W | HW_BREAKPOINT_R: | |
262 | info->type = X86_BREAKPOINT_RW; | |
263 | break; | |
264 | case HW_BREAKPOINT_X: | |
265 | info->type = X86_BREAKPOINT_EXECUTE; | |
266 | /* | |
267 | * x86 inst breakpoints need to have a specific undefined len. | |
268 | * But we still need to check userspace is not trying to setup | |
269 | * an unsupported length, to get a range breakpoint for example. | |
270 | */ | |
271 | if (bp->attr.bp_len == sizeof(long)) { | |
272 | info->len = X86_BREAKPOINT_LEN_X; | |
273 | return 0; | |
274 | } | |
275 | default: | |
276 | return -EINVAL; | |
277 | } | |
278 | ||
24f1e32c FW |
279 | /* Len */ |
280 | switch (bp->attr.bp_len) { | |
0067f129 | 281 | case HW_BREAKPOINT_LEN_1: |
24f1e32c | 282 | info->len = X86_BREAKPOINT_LEN_1; |
0067f129 P |
283 | break; |
284 | case HW_BREAKPOINT_LEN_2: | |
24f1e32c | 285 | info->len = X86_BREAKPOINT_LEN_2; |
0067f129 P |
286 | break; |
287 | case HW_BREAKPOINT_LEN_4: | |
24f1e32c | 288 | info->len = X86_BREAKPOINT_LEN_4; |
0067f129 P |
289 | break; |
290 | #ifdef CONFIG_X86_64 | |
291 | case HW_BREAKPOINT_LEN_8: | |
24f1e32c FW |
292 | info->len = X86_BREAKPOINT_LEN_8; |
293 | break; | |
294 | #endif | |
295 | default: | |
296 | return -EINVAL; | |
297 | } | |
298 | ||
24f1e32c FW |
299 | return 0; |
300 | } | |
301 | /* | |
302 | * Validate the arch-specific HW Breakpoint register settings | |
303 | */ | |
b2812d03 | 304 | int arch_validate_hwbkpt_settings(struct perf_event *bp) |
24f1e32c FW |
305 | { |
306 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
307 | unsigned int align; | |
308 | int ret; | |
309 | ||
310 | ||
311 | ret = arch_build_bp_info(bp); | |
312 | if (ret) | |
313 | return ret; | |
314 | ||
315 | ret = -EINVAL; | |
316 | ||
24f1e32c FW |
317 | switch (info->len) { |
318 | case X86_BREAKPOINT_LEN_1: | |
319 | align = 0; | |
320 | break; | |
321 | case X86_BREAKPOINT_LEN_2: | |
322 | align = 1; | |
323 | break; | |
324 | case X86_BREAKPOINT_LEN_4: | |
325 | align = 3; | |
326 | break; | |
327 | #ifdef CONFIG_X86_64 | |
328 | case X86_BREAKPOINT_LEN_8: | |
0067f129 P |
329 | align = 7; |
330 | break; | |
331 | #endif | |
332 | default: | |
333 | return ret; | |
334 | } | |
335 | ||
0067f129 P |
336 | /* |
337 | * Check that the low-order bits of the address are appropriate | |
338 | * for the alignment implied by len. | |
339 | */ | |
24f1e32c | 340 | if (info->address & align) |
0067f129 P |
341 | return -EINVAL; |
342 | ||
0067f129 P |
343 | return 0; |
344 | } | |
345 | ||
9f6b3c2c FW |
346 | /* |
347 | * Dump the debug register contents to the user. | |
348 | * We can't dump our per cpu values because it | |
349 | * may contain cpu wide breakpoint, something that | |
350 | * doesn't belong to the current task. | |
351 | * | |
352 | * TODO: include non-ptrace user breakpoints (perf) | |
353 | */ | |
354 | void aout_dump_debugregs(struct user *dump) | |
355 | { | |
356 | int i; | |
357 | int dr7 = 0; | |
358 | struct perf_event *bp; | |
359 | struct arch_hw_breakpoint *info; | |
360 | struct thread_struct *thread = ¤t->thread; | |
361 | ||
362 | for (i = 0; i < HBP_NUM; i++) { | |
363 | bp = thread->ptrace_bps[i]; | |
364 | ||
365 | if (bp && !bp->attr.disabled) { | |
366 | dump->u_debugreg[i] = bp->attr.bp_addr; | |
367 | info = counter_arch_bp(bp); | |
368 | dr7 |= encode_dr7(i, info->len, info->type); | |
369 | } else { | |
370 | dump->u_debugreg[i] = 0; | |
371 | } | |
372 | } | |
373 | ||
374 | dump->u_debugreg[4] = 0; | |
375 | dump->u_debugreg[5] = 0; | |
376 | dump->u_debugreg[6] = current->thread.debugreg6; | |
377 | ||
378 | dump->u_debugreg[7] = dr7; | |
379 | } | |
68efa37d | 380 | EXPORT_SYMBOL_GPL(aout_dump_debugregs); |
9f6b3c2c | 381 | |
24f1e32c FW |
382 | /* |
383 | * Release the user breakpoints used by ptrace | |
384 | */ | |
385 | void flush_ptrace_hw_breakpoint(struct task_struct *tsk) | |
0067f129 | 386 | { |
24f1e32c FW |
387 | int i; |
388 | struct thread_struct *t = &tsk->thread; | |
389 | ||
390 | for (i = 0; i < HBP_NUM; i++) { | |
391 | unregister_hw_breakpoint(t->ptrace_bps[i]); | |
392 | t->ptrace_bps[i] = NULL; | |
393 | } | |
f7da04c9 ON |
394 | |
395 | t->debugreg6 = 0; | |
396 | t->ptrace_dr7 = 0; | |
0067f129 P |
397 | } |
398 | ||
24f1e32c | 399 | void hw_breakpoint_restore(void) |
0067f129 | 400 | { |
0a3aee0d TH |
401 | set_debugreg(__this_cpu_read(cpu_debugreg[0]), 0); |
402 | set_debugreg(__this_cpu_read(cpu_debugreg[1]), 1); | |
403 | set_debugreg(__this_cpu_read(cpu_debugreg[2]), 2); | |
404 | set_debugreg(__this_cpu_read(cpu_debugreg[3]), 3); | |
24f1e32c | 405 | set_debugreg(current->thread.debugreg6, 6); |
0a3aee0d | 406 | set_debugreg(__this_cpu_read(cpu_dr7), 7); |
0067f129 | 407 | } |
24f1e32c | 408 | EXPORT_SYMBOL_GPL(hw_breakpoint_restore); |
0067f129 P |
409 | |
410 | /* | |
411 | * Handle debug exception notifications. | |
412 | * | |
413 | * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below. | |
414 | * | |
415 | * NOTIFY_DONE returned if one of the following conditions is true. | |
416 | * i) When the causative address is from user-space and the exception | |
417 | * is a valid one, i.e. not triggered as a result of lazy debug register | |
418 | * switching | |
419 | * ii) When there are more bits than trap<n> set in DR6 register (such | |
420 | * as BD, BS or BT) indicating that more than one debug condition is | |
421 | * met and requires some more action in do_debug(). | |
422 | * | |
423 | * NOTIFY_STOP returned for all other cases | |
424 | * | |
425 | */ | |
9c54b616 | 426 | static int hw_breakpoint_handler(struct die_args *args) |
0067f129 P |
427 | { |
428 | int i, cpu, rc = NOTIFY_STOP; | |
24f1e32c | 429 | struct perf_event *bp; |
62edab90 P |
430 | unsigned long dr7, dr6; |
431 | unsigned long *dr6_p; | |
432 | ||
433 | /* The DR6 value is pointed by args->err */ | |
434 | dr6_p = (unsigned long *)ERR_PTR(args->err); | |
435 | dr6 = *dr6_p; | |
0067f129 | 436 | |
6c0aca28 FW |
437 | /* If it's a single step, TRAP bits are random */ |
438 | if (dr6 & DR_STEP) | |
439 | return NOTIFY_DONE; | |
440 | ||
0067f129 P |
441 | /* Do an early return if no trap bits are set in DR6 */ |
442 | if ((dr6 & DR_TRAP_BITS) == 0) | |
443 | return NOTIFY_DONE; | |
444 | ||
0067f129 P |
445 | get_debugreg(dr7, 7); |
446 | /* Disable breakpoints during exception handling */ | |
447 | set_debugreg(0UL, 7); | |
448 | /* | |
449 | * Assert that local interrupts are disabled | |
450 | * Reset the DRn bits in the virtualized register value. | |
451 | * The ptrace trigger routine will add in whatever is needed. | |
452 | */ | |
453 | current->thread.debugreg6 &= ~DR_TRAP_BITS; | |
454 | cpu = get_cpu(); | |
455 | ||
456 | /* Handle all the breakpoints that were triggered */ | |
457 | for (i = 0; i < HBP_NUM; ++i) { | |
458 | if (likely(!(dr6 & (DR_TRAP0 << i)))) | |
459 | continue; | |
24f1e32c | 460 | |
0067f129 | 461 | /* |
24f1e32c FW |
462 | * The counter may be concurrently released but that can only |
463 | * occur from a call_rcu() path. We can then safely fetch | |
464 | * the breakpoint, use its callback, touch its counter | |
465 | * while we are in an rcu_read_lock() path. | |
0067f129 | 466 | */ |
24f1e32c FW |
467 | rcu_read_lock(); |
468 | ||
469 | bp = per_cpu(bp_per_reg[i], cpu); | |
62edab90 P |
470 | /* |
471 | * Reset the 'i'th TRAP bit in dr6 to denote completion of | |
472 | * exception handling | |
473 | */ | |
474 | (*dr6_p) &= ~(DR_TRAP0 << i); | |
0067f129 P |
475 | /* |
476 | * bp can be NULL due to lazy debug register switching | |
24f1e32c | 477 | * or due to concurrent perf counter removing. |
0067f129 | 478 | */ |
24f1e32c FW |
479 | if (!bp) { |
480 | rcu_read_unlock(); | |
481 | break; | |
482 | } | |
483 | ||
b326e956 | 484 | perf_bp_event(bp, args->regs); |
0067f129 | 485 | |
0c4519e8 FW |
486 | /* |
487 | * Set up resume flag to avoid breakpoint recursion when | |
488 | * returning back to origin. | |
489 | */ | |
490 | if (bp->hw.info.type == X86_BREAKPOINT_EXECUTE) | |
491 | args->regs->flags |= X86_EFLAGS_RF; | |
492 | ||
24f1e32c | 493 | rcu_read_unlock(); |
0067f129 | 494 | } |
e0e53db6 P |
495 | /* |
496 | * Further processing in do_debug() is needed for a) user-space | |
497 | * breakpoints (to generate signals) and b) when the system has | |
498 | * taken exception due to multiple causes | |
499 | */ | |
500 | if ((current->thread.debugreg6 & DR_TRAP_BITS) || | |
501 | (dr6 & (~DR_TRAP_BITS))) | |
0067f129 P |
502 | rc = NOTIFY_DONE; |
503 | ||
504 | set_debugreg(dr7, 7); | |
eadb8a09 | 505 | put_cpu(); |
24f1e32c | 506 | |
0067f129 P |
507 | return rc; |
508 | } | |
509 | ||
510 | /* | |
511 | * Handle debug exception notifications. | |
512 | */ | |
9c54b616 | 513 | int hw_breakpoint_exceptions_notify( |
0067f129 P |
514 | struct notifier_block *unused, unsigned long val, void *data) |
515 | { | |
516 | if (val != DIE_DEBUG) | |
517 | return NOTIFY_DONE; | |
518 | ||
519 | return hw_breakpoint_handler(data); | |
520 | } | |
24f1e32c FW |
521 | |
522 | void hw_breakpoint_pmu_read(struct perf_event *bp) | |
523 | { | |
524 | /* TODO */ | |
525 | } |