[deliverable/linux.git] / arch / x86 / kernel / hw_breakpoint.c

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) 2007 Alan Stern
 * Copyright (C) 2009 IBM Corporation
 * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
 */

/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers.
 */

#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
#include <linux/irqflags.h>
#include <linux/notifier.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/percpu.h>
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/smp.h>

#include <asm/hw_breakpoint.h>
#include <asm/processor.h>
#include <asm/debugreg.h>

/* Per cpu debug control register value */
DEFINE_PER_CPU(unsigned long, dr7);
EXPORT_PER_CPU_SYMBOL(dr7);

/* Per cpu debug address registers values */
static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);

/*
 * Stores the breakpoints currently in use on each breakpoint address
 * register for each cpus
 */
static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);


/*
 * Encode the length, type, Exact, and Enable bits for a particular breakpoint
 * as stored in debug register 7.
 */
unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
{
	unsigned long bp_info;

	bp_info = (len | type) & 0xf;
	bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
	bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)) |
				DR_GLOBAL_SLOWDOWN;
	return bp_info;
}

/*
 * Decode the length and type bits for a particular breakpoint as
 * stored in debug register 7.  Return the "enabled" status.
 */
int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type)
{
	int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);

	*len = (bp_info & 0xc) | 0x40;
	*type = (bp_info & 0x3) | 0x80;

	return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
}

/*
 * Install a perf counter breakpoint.
 *
 * We seek a free debug address register and use it for this
 * breakpoint. Eventually we enable it in the debug control register.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
int arch_install_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);

		if (!*slot) {
			*slot = bp;
			break;
		}
	}

	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return -EBUSY;

	set_debugreg(info->address, i);
	__get_cpu_var(cpu_debugreg[i]) = info->address;

	dr7 = &__get_cpu_var(dr7);
	*dr7 |= encode_dr7(i, info->len, info->type);

	set_debugreg(*dr7, 7);

	return 0;
}

/*
 * Uninstall the breakpoint contained in the given counter.
 *
 * First we search the debug address register it uses and then we disable
 * it.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);

		if (*slot == bp) {
			*slot = NULL;
			break;
		}
	}

	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return;

	dr7 = &__get_cpu_var(dr7);
	*dr7 &= ~encode_dr7(i, info->len, info->type);

	set_debugreg(*dr7, 7);
}

static int get_hbp_len(u8 hbp_len)
{
	unsigned int len_in_bytes = 0;

	switch (hbp_len) {
	case X86_BREAKPOINT_LEN_1:
		len_in_bytes = 1;
		break;
	case X86_BREAKPOINT_LEN_2:
		len_in_bytes = 2;
		break;
	case X86_BREAKPOINT_LEN_4:
		len_in_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		len_in_bytes = 8;
		break;
#endif
	}
	return len_in_bytes;
}

/*
 * Check for virtual address in user space.
 */
int arch_check_va_in_userspace(unsigned long va, u8 hbp_len)
{
	unsigned int len;

	len = get_hbp_len(hbp_len);

	return (va <= TASK_SIZE - len);
}

/*
 * Check for virtual address in kernel space.
 */
static int arch_check_va_in_kernelspace(unsigned long va, u8 hbp_len)
{
	unsigned int len;

	len = get_hbp_len(hbp_len);

	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
}

/*
 * Store a breakpoint's encoded address, length, and type.
 */
static int arch_store_info(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	/*
	 * For kernel-addresses, either the address or symbol name can be
	 * specified.
	 */
	if (info->name)
		info->address = (unsigned long)
				kallsyms_lookup_name(info->name);
	if (info->address)
		return 0;

	return -EINVAL;
}

int arch_bp_generic_fields(int x86_len, int x86_type,
			   int *gen_len, int *gen_type)
{
	/* Len */
	switch (x86_len) {
	case X86_BREAKPOINT_LEN_1:
		*gen_len = HW_BREAKPOINT_LEN_1;
		break;
	case X86_BREAKPOINT_LEN_2:
		*gen_len = HW_BREAKPOINT_LEN_2;
		break;
	case X86_BREAKPOINT_LEN_4:
		*gen_len = HW_BREAKPOINT_LEN_4;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		*gen_len = HW_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		return -EINVAL;
	}

	/* Type */
	switch (x86_type) {
	case X86_BREAKPOINT_EXECUTE:
		*gen_type = HW_BREAKPOINT_X;
		break;
	case X86_BREAKPOINT_WRITE:
		*gen_type = HW_BREAKPOINT_W;
		break;
	case X86_BREAKPOINT_RW:
		*gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R;
		break;
	default:
		return -EINVAL;
	}

	return 0;
}


static int arch_build_bp_info(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);

	info->address = bp->attr.bp_addr;

	/* Len */
	switch (bp->attr.bp_len) {
	case HW_BREAKPOINT_LEN_1:
		info->len = X86_BREAKPOINT_LEN_1;
		break;
	case HW_BREAKPOINT_LEN_2:
		info->len = X86_BREAKPOINT_LEN_2;
		break;
	case HW_BREAKPOINT_LEN_4:
		info->len = X86_BREAKPOINT_LEN_4;
		break;
#ifdef CONFIG_X86_64
	case HW_BREAKPOINT_LEN_8:
		info->len = X86_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		return -EINVAL;
	}

	/* Type */
	switch (bp->attr.bp_type) {
	case HW_BREAKPOINT_W:
		info->type = X86_BREAKPOINT_WRITE;
		break;
	case HW_BREAKPOINT_W | HW_BREAKPOINT_R:
		info->type = X86_BREAKPOINT_RW;
		break;
	case HW_BREAKPOINT_X:
		info->type = X86_BREAKPOINT_EXECUTE;
		break;
	default:
		return -EINVAL;
	}

	return 0;
}
/*
 * Validate the arch-specific HW Breakpoint register settings
 */
int arch_validate_hwbkpt_settings(struct perf_event *bp,
				  struct task_struct *tsk)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned int align;
	int ret;


	ret = arch_build_bp_info(bp);
	if (ret)
		return ret;

	ret = -EINVAL;

	if (info->type == X86_BREAKPOINT_EXECUTE)
		/*
		 * Ptrace-refactoring code
		 * For now, we'll allow instruction breakpoint only for user-space
		 * addresses
		 */
		if ((!arch_check_va_in_userspace(info->address, info->len)) &&
			info->len != X86_BREAKPOINT_EXECUTE)
			return ret;

	switch (info->len) {
	case X86_BREAKPOINT_LEN_1:
		align = 0;
		break;
	case X86_BREAKPOINT_LEN_2:
		align = 1;
		break;
	case X86_BREAKPOINT_LEN_4:
		align = 3;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		align = 7;
		break;
#endif
	default:
		return ret;
	}

	if (bp->callback)
		ret = arch_store_info(bp);

	if (ret < 0)
		return ret;
	/*
	 * Check that the low-order bits of the address are appropriate
	 * for the alignment implied by len.
	 */
	if (info->address & align)
		return -EINVAL;

	/* Check that the virtual address is in the proper range */
	if (tsk) {
		if (!arch_check_va_in_userspace(info->address, info->len))
			return -EFAULT;
	} else {
		if (!arch_check_va_in_kernelspace(info->address, info->len))
			return -EFAULT;
	}

	return 0;
}

/*
 * Dump the debug register contents to the user.
 * We can't dump our per cpu values because it
 * may contain cpu wide breakpoint, something that
 * doesn't belong to the current task.
 *
 * TODO: include non-ptrace user breakpoints (perf)
 */
void aout_dump_debugregs(struct user *dump)
{
	int i;
	int dr7 = 0;
	struct perf_event *bp;
	struct arch_hw_breakpoint *info;
	struct thread_struct *thread = &current->thread;

	for (i = 0; i < HBP_NUM; i++) {
		bp = thread->ptrace_bps[i];

		if (bp && !bp->attr.disabled) {
			dump->u_debugreg[i] = bp->attr.bp_addr;
			info = counter_arch_bp(bp);
			dr7 |= encode_dr7(i, info->len, info->type);
		} else {
			dump->u_debugreg[i] = 0;
		}
	}

	dump->u_debugreg[4] = 0;
	dump->u_debugreg[5] = 0;
	dump->u_debugreg[6] = current->thread.debugreg6;

	dump->u_debugreg[7] = dr7;
}
EXPORT_SYMBOL_GPL(aout_dump_debugregs);

/*
 * Release the user breakpoints used by ptrace
 */
void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
{
	int i;
	struct thread_struct *t = &tsk->thread;

	for (i = 0; i < HBP_NUM; i++) {
		unregister_hw_breakpoint(t->ptrace_bps[i]);
		t->ptrace_bps[i] = NULL;
	}
}

void hw_breakpoint_restore(void)
{
	set_debugreg(__get_cpu_var(cpu_debugreg[0]), 0);
	set_debugreg(__get_cpu_var(cpu_debugreg[1]), 1);
	set_debugreg(__get_cpu_var(cpu_debugreg[2]), 2);
	set_debugreg(__get_cpu_var(cpu_debugreg[3]), 3);
	set_debugreg(current->thread.debugreg6, 6);
	set_debugreg(__get_cpu_var(dr7), 7);
}
EXPORT_SYMBOL_GPL(hw_breakpoint_restore);

/*
 * Handle debug exception notifications.
 *
 * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
 *
 * NOTIFY_DONE returned if one of the following conditions is true.
 * i) When the causative address is from user-space and the exception
 * is a valid one, i.e. not triggered as a result of lazy debug register
 * switching
 * ii) When there are more bits than trap<n> set in DR6 register (such
 * as BD, BS or BT) indicating that more than one debug condition is
 * met and requires some more action in do_debug().
 *
 * NOTIFY_STOP returned for all other cases
 *
 */
static int __kprobes hw_breakpoint_handler(struct die_args *args)
{
	int i, cpu, rc = NOTIFY_STOP;
	struct perf_event *bp;
	unsigned long dr7, dr6;
	unsigned long *dr6_p;

	/* The DR6 value is pointed by args->err */
	dr6_p = (unsigned long *)ERR_PTR(args->err);
	dr6 = *dr6_p;

	/* Do an early return if no trap bits are set in DR6 */
	if ((dr6 & DR_TRAP_BITS) == 0)
		return NOTIFY_DONE;

	get_debugreg(dr7, 7);
	/* Disable breakpoints during exception handling */
	set_debugreg(0UL, 7);
	/*
	 * Assert that local interrupts are disabled
	 * Reset the DRn bits in the virtualized register value.
	 * The ptrace trigger routine will add in whatever is needed.
	 */
	current->thread.debugreg6 &= ~DR_TRAP_BITS;
	cpu = get_cpu();

	/* Handle all the breakpoints that were triggered */
	for (i = 0; i < HBP_NUM; ++i) {
		if (likely(!(dr6 & (DR_TRAP0 << i))))
			continue;

		/*
		 * The counter may be concurrently released but that can only
		 * occur from a call_rcu() path. We can then safely fetch
		 * the breakpoint, use its callback, touch its counter
		 * while we are in an rcu_read_lock() path.
		 */
		rcu_read_lock();

		bp = per_cpu(bp_per_reg[i], cpu);
		if (bp)
			rc = NOTIFY_DONE;
		/*
		 * Reset the 'i'th TRAP bit in dr6 to denote completion of
		 * exception handling
		 */
		(*dr6_p) &= ~(DR_TRAP0 << i);
		/*
		 * bp can be NULL due to lazy debug register switching
		 * or due to concurrent perf counter removing.
		 */
		if (!bp) {
			rcu_read_unlock();
			break;
		}

		(bp->callback)(bp, args->regs);

		rcu_read_unlock();
	}
	if (dr6 & (~DR_TRAP_BITS))
		rc = NOTIFY_DONE;

	set_debugreg(dr7, 7);
	put_cpu();

	return rc;
}

/*
 * Handle debug exception notifications.
 */
int __kprobes hw_breakpoint_exceptions_notify(
		struct notifier_block *unused, unsigned long val, void *data)
{
	if (val != DIE_DEBUG)
		return NOTIFY_DONE;

	return hw_breakpoint_handler(data);
}

void hw_breakpoint_pmu_read(struct perf_event *bp)
{
	/* TODO */
}

void hw_breakpoint_pmu_unthrottle(struct perf_event *bp)
{
	/* TODO */
}
Commit	Line	Data
0067f129 P	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License as published by
	4	* the Free Software Foundation; either version 2 of the License, or
	5	* (at your option) any later version.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	* You should have received a copy of the GNU General Public License
	13	* along with this program; if not, write to the Free Software
	14	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	*
	16	* Copyright (C) 2007 Alan Stern
	17	* Copyright (C) 2009 IBM Corporation
24f1e32c	18	* Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
0067f129 P	19	*/
	20
	21	/*
	22	* HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
	23	* using the CPU's debug registers.
	24	*/
	25
24f1e32c FW	26	#include <linux/perf_event.h>
24f1e32c FW	27	#include <linux/hw_breakpoint.h>
0067f129 P	28	#include <linux/irqflags.h>
	29	#include <linux/notifier.h>
	30	#include <linux/kallsyms.h>
	31	#include <linux/kprobes.h>
	32	#include <linux/percpu.h>
	33	#include <linux/kdebug.h>
	34	#include <linux/kernel.h>
	35	#include <linux/module.h>
	36	#include <linux/sched.h>
	37	#include <linux/init.h>
	38	#include <linux/smp.h>
	39
	40	#include <asm/hw_breakpoint.h>
	41	#include <asm/processor.h>
	42	#include <asm/debugreg.h>
	43
24f1e32c FW	44	/* Per cpu debug control register value */
24f1e32c FW	45	DEFINE_PER_CPU(unsigned long, dr7);
68efa37d	46	EXPORT_PER_CPU_SYMBOL(dr7);
24f1e32c FW	47
	48	/* Per cpu debug address registers values */
	49	static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);
0067f129 P	50
0067f129 P	51	/*
24f1e32c FW	52	* Stores the breakpoints currently in use on each breakpoint address
24f1e32c FW	53	* register for each cpus
0067f129	54	*/
24f1e32c	55	static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);
0067f129 P	56
	57
	58	/*
	59	* Encode the length, type, Exact, and Enable bits for a particular breakpoint
	60	* as stored in debug register 7.
	61	*/
24f1e32c	62	unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
0067f129 P	63	{
	64	unsigned long bp_info;
	65
	66	bp_info = (len \| type) & 0xf;
	67	bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
	68	bp_info \|= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE)) \|
	69	DR_GLOBAL_SLOWDOWN;
	70	return bp_info;
	71	}
	72
24f1e32c FW	73	/*
	74	* Decode the length and type bits for a particular breakpoint as
	75	* stored in debug register 7. Return the "enabled" status.
	76	*/
	77	int decode_dr7(unsigned long dr7, int bpnum, unsigned len, unsigned type)
0067f129	78	{
24f1e32c	79	int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);
0067f129	80
24f1e32c FW	81	*len = (bp_info & 0xc) \| 0x40;
24f1e32c FW	82	*type = (bp_info & 0x3) \| 0x80;
0067f129	83
24f1e32c	84	return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
0067f129 P	85	}
	86
	87	/*
24f1e32c FW	88	* Install a perf counter breakpoint.
	89	*
	90	* We seek a free debug address register and use it for this
	91	* breakpoint. Eventually we enable it in the debug control register.
	92	*
	93	* Atomic: we hold the counter->ctx->lock and we only handle variables
	94	* and registers local to this cpu.
0067f129	95	*/
24f1e32c	96	int arch_install_hw_breakpoint(struct perf_event *bp)
0067f129	97	{
24f1e32c FW	98	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	99	unsigned long *dr7;
	100	int i;
	101
	102	for (i = 0; i < HBP_NUM; i++) {
	103	struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
	104
	105	if (!*slot) {
	106	*slot = bp;
	107	break;
	108	}
0067f129 P	109	}
0067f129 P	110
24f1e32c FW	111	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
	112	return -EBUSY;
	113
	114	set_debugreg(info->address, i);
	115	__get_cpu_var(cpu_debugreg[i]) = info->address;
	116
	117	dr7 = &__get_cpu_var(dr7);
	118	*dr7 \|= encode_dr7(i, info->len, info->type);
	119
	120	set_debugreg(*dr7, 7);
	121
	122	return 0;
0067f129 P	123	}
	124
	125	/*
24f1e32c FW	126	* Uninstall the breakpoint contained in the given counter.
	127	*
	128	* First we search the debug address register it uses and then we disable
	129	* it.
	130	*
	131	* Atomic: we hold the counter->ctx->lock and we only handle variables
	132	* and registers local to this cpu.
0067f129	133	*/
24f1e32c	134	void arch_uninstall_hw_breakpoint(struct perf_event *bp)
0067f129	135	{
24f1e32c FW	136	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	137	unsigned long *dr7;
	138	int i;
	139
	140	for (i = 0; i < HBP_NUM; i++) {
	141	struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
	142
	143	if (*slot == bp) {
	144	*slot = NULL;
	145	break;
	146	}
	147	}
	148
	149	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
	150	return;
0067f129	151
24f1e32c FW	152	dr7 = &__get_cpu_var(dr7);
	153	*dr7 &= ~encode_dr7(i, info->len, info->type);
	154
	155	set_debugreg(*dr7, 7);
0067f129 P	156	}
	157
	158	static int get_hbp_len(u8 hbp_len)
	159	{
	160	unsigned int len_in_bytes = 0;
	161
	162	switch (hbp_len) {
24f1e32c	163	case X86_BREAKPOINT_LEN_1:
0067f129 P	164	len_in_bytes = 1;
0067f129 P	165	break;
24f1e32c	166	case X86_BREAKPOINT_LEN_2:
0067f129 P	167	len_in_bytes = 2;
0067f129 P	168	break;
24f1e32c	169	case X86_BREAKPOINT_LEN_4:
0067f129 P	170	len_in_bytes = 4;
	171	break;
	172	#ifdef CONFIG_X86_64
24f1e32c	173	case X86_BREAKPOINT_LEN_8:
0067f129 P	174	len_in_bytes = 8;
	175	break;
	176	#endif
	177	}
	178	return len_in_bytes;
	179	}
	180
	181	/*
	182	* Check for virtual address in user space.
	183	*/
	184	int arch_check_va_in_userspace(unsigned long va, u8 hbp_len)
	185	{
	186	unsigned int len;
	187
	188	len = get_hbp_len(hbp_len);
	189
	190	return (va <= TASK_SIZE - len);
	191	}
	192
	193	/*
	194	* Check for virtual address in kernel space.
	195	*/
4555835b	196	static int arch_check_va_in_kernelspace(unsigned long va, u8 hbp_len)
0067f129 P	197	{
	198	unsigned int len;
	199
	200	len = get_hbp_len(hbp_len);
	201
	202	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
	203	}
	204
	205	/*
	206	* Store a breakpoint's encoded address, length, and type.
	207	*/
24f1e32c	208	static int arch_store_info(struct perf_event *bp)
0067f129	209	{
24f1e32c	210	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
0067f129 P	211	/*
	212	* For kernel-addresses, either the address or symbol name can be
	213	* specified.
	214	*/
24f1e32c FW	215	if (info->name)
	216	info->address = (unsigned long)
	217	kallsyms_lookup_name(info->name);
	218	if (info->address)
0067f129	219	return 0;
24f1e32c	220
0067f129 P	221	return -EINVAL;
	222	}
	223
24f1e32c FW	224	int arch_bp_generic_fields(int x86_len, int x86_type,
24f1e32c FW	225	int gen_len, int gen_type)
0067f129	226	{
24f1e32c FW	227	/* Len */
	228	switch (x86_len) {
	229	case X86_BREAKPOINT_LEN_1:
	230	*gen_len = HW_BREAKPOINT_LEN_1;
	231	break;
	232	case X86_BREAKPOINT_LEN_2:
	233	*gen_len = HW_BREAKPOINT_LEN_2;
	234	break;
	235	case X86_BREAKPOINT_LEN_4:
	236	*gen_len = HW_BREAKPOINT_LEN_4;
	237	break;
	238	#ifdef CONFIG_X86_64
	239	case X86_BREAKPOINT_LEN_8:
	240	*gen_len = HW_BREAKPOINT_LEN_8;
	241	break;
	242	#endif
	243	default:
	244	return -EINVAL;
	245	}
0067f129	246
24f1e32c FW	247	/* Type */
	248	switch (x86_type) {
	249	case X86_BREAKPOINT_EXECUTE:
	250	*gen_type = HW_BREAKPOINT_X;
0067f129	251	break;
24f1e32c FW	252	case X86_BREAKPOINT_WRITE:
24f1e32c FW	253	*gen_type = HW_BREAKPOINT_W;
0067f129	254	break;
24f1e32c FW	255	case X86_BREAKPOINT_RW:
24f1e32c FW	256	*gen_type = HW_BREAKPOINT_W \| HW_BREAKPOINT_R;
0067f129 P	257	break;
0067f129 P	258	default:
24f1e32c	259	return -EINVAL;
0067f129 P	260	}
0067f129 P	261
24f1e32c FW	262	return 0;
	263	}
	264
	265
	266	static int arch_build_bp_info(struct perf_event *bp)
	267	{
	268	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	269
	270	info->address = bp->attr.bp_addr;
	271
	272	/* Len */
	273	switch (bp->attr.bp_len) {
0067f129	274	case HW_BREAKPOINT_LEN_1:
24f1e32c	275	info->len = X86_BREAKPOINT_LEN_1;
0067f129 P	276	break;
0067f129 P	277	case HW_BREAKPOINT_LEN_2:
24f1e32c	278	info->len = X86_BREAKPOINT_LEN_2;
0067f129 P	279	break;
0067f129 P	280	case HW_BREAKPOINT_LEN_4:
24f1e32c	281	info->len = X86_BREAKPOINT_LEN_4;
0067f129 P	282	break;
	283	#ifdef CONFIG_X86_64
	284	case HW_BREAKPOINT_LEN_8:
24f1e32c FW	285	info->len = X86_BREAKPOINT_LEN_8;
	286	break;
	287	#endif
	288	default:
	289	return -EINVAL;
	290	}
	291
	292	/* Type */
	293	switch (bp->attr.bp_type) {
	294	case HW_BREAKPOINT_W:
	295	info->type = X86_BREAKPOINT_WRITE;
	296	break;
	297	case HW_BREAKPOINT_W \| HW_BREAKPOINT_R:
	298	info->type = X86_BREAKPOINT_RW;
	299	break;
	300	case HW_BREAKPOINT_X:
	301	info->type = X86_BREAKPOINT_EXECUTE;
	302	break;
	303	default:
	304	return -EINVAL;
	305	}
	306
	307	return 0;
	308	}
	309	/*
	310	* Validate the arch-specific HW Breakpoint register settings
	311	*/
	312	int arch_validate_hwbkpt_settings(struct perf_event *bp,
	313	struct task_struct *tsk)
	314	{
	315	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	316	unsigned int align;
	317	int ret;
	318
	319
	320	ret = arch_build_bp_info(bp);
	321	if (ret)
	322	return ret;
	323
	324	ret = -EINVAL;
	325
	326	if (info->type == X86_BREAKPOINT_EXECUTE)
	327	/*
	328	* Ptrace-refactoring code
	329	* For now, we'll allow instruction breakpoint only for user-space
	330	* addresses
	331	*/
	332	if ((!arch_check_va_in_userspace(info->address, info->len)) &&
	333	info->len != X86_BREAKPOINT_EXECUTE)
	334	return ret;
	335
	336	switch (info->len) {
	337	case X86_BREAKPOINT_LEN_1:
	338	align = 0;
	339	break;
	340	case X86_BREAKPOINT_LEN_2:
	341	align = 1;
	342	break;
	343	case X86_BREAKPOINT_LEN_4:
	344	align = 3;
	345	break;
	346	#ifdef CONFIG_X86_64
	347	case X86_BREAKPOINT_LEN_8:
0067f129 P	348	align = 7;
	349	break;
	350	#endif
	351	default:
	352	return ret;
	353	}
	354
24f1e32c FW	355	if (bp->callback)
24f1e32c FW	356	ret = arch_store_info(bp);
0067f129 P	357
	358	if (ret < 0)
	359	return ret;
	360	/*
	361	* Check that the low-order bits of the address are appropriate
	362	* for the alignment implied by len.
	363	*/
24f1e32c	364	if (info->address & align)
0067f129 P	365	return -EINVAL;
	366
	367	/* Check that the virtual address is in the proper range */
	368	if (tsk) {
24f1e32c	369	if (!arch_check_va_in_userspace(info->address, info->len))
0067f129 P	370	return -EFAULT;
0067f129 P	371	} else {
24f1e32c	372	if (!arch_check_va_in_kernelspace(info->address, info->len))
0067f129 P	373	return -EFAULT;
0067f129 P	374	}
24f1e32c	375
0067f129 P	376	return 0;
	377	}
	378
9f6b3c2c FW	379	/*
	380	* Dump the debug register contents to the user.
	381	* We can't dump our per cpu values because it
	382	* may contain cpu wide breakpoint, something that
	383	* doesn't belong to the current task.
	384	*
	385	* TODO: include non-ptrace user breakpoints (perf)
	386	*/
	387	void aout_dump_debugregs(struct user *dump)
	388	{
	389	int i;
	390	int dr7 = 0;
	391	struct perf_event *bp;
	392	struct arch_hw_breakpoint *info;
	393	struct thread_struct *thread = &current->thread;
	394
	395	for (i = 0; i < HBP_NUM; i++) {
	396	bp = thread->ptrace_bps[i];
	397
	398	if (bp && !bp->attr.disabled) {
	399	dump->u_debugreg[i] = bp->attr.bp_addr;
	400	info = counter_arch_bp(bp);
	401	dr7 \|= encode_dr7(i, info->len, info->type);
	402	} else {
	403	dump->u_debugreg[i] = 0;
	404	}
	405	}
	406
	407	dump->u_debugreg[4] = 0;
	408	dump->u_debugreg[5] = 0;
	409	dump->u_debugreg[6] = current->thread.debugreg6;
	410
	411	dump->u_debugreg[7] = dr7;
	412	}
68efa37d	413	EXPORT_SYMBOL_GPL(aout_dump_debugregs);
9f6b3c2c	414
24f1e32c FW	415	/*
	416	* Release the user breakpoints used by ptrace
	417	*/
	418	void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
0067f129	419	{
24f1e32c FW	420	int i;
	421	struct thread_struct *t = &tsk->thread;
	422
	423	for (i = 0; i < HBP_NUM; i++) {
	424	unregister_hw_breakpoint(t->ptrace_bps[i]);
	425	t->ptrace_bps[i] = NULL;
	426	}
0067f129 P	427	}
0067f129 P	428
24f1e32c	429	void hw_breakpoint_restore(void)
0067f129	430	{
24f1e32c FW	431	set_debugreg(__get_cpu_var(cpu_debugreg[0]), 0);
	432	set_debugreg(__get_cpu_var(cpu_debugreg[1]), 1);
	433	set_debugreg(__get_cpu_var(cpu_debugreg[2]), 2);
	434	set_debugreg(__get_cpu_var(cpu_debugreg[3]), 3);
	435	set_debugreg(current->thread.debugreg6, 6);
	436	set_debugreg(__get_cpu_var(dr7), 7);
0067f129	437	}
24f1e32c	438	EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
0067f129 P	439
	440	/*
	441	* Handle debug exception notifications.
	442	*
	443	* Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
	444	*
	445	* NOTIFY_DONE returned if one of the following conditions is true.
	446	* i) When the causative address is from user-space and the exception
	447	* is a valid one, i.e. not triggered as a result of lazy debug register
	448	* switching
	449	* ii) When there are more bits than trap<n> set in DR6 register (such
	450	* as BD, BS or BT) indicating that more than one debug condition is
	451	* met and requires some more action in do_debug().
	452	*
	453	* NOTIFY_STOP returned for all other cases
	454	*
	455	*/
4555835b	456	static int __kprobes hw_breakpoint_handler(struct die_args *args)
0067f129 P	457	{
0067f129 P	458	int i, cpu, rc = NOTIFY_STOP;
24f1e32c	459	struct perf_event *bp;
62edab90 P	460	unsigned long dr7, dr6;
	461	unsigned long *dr6_p;
	462
	463	/* The DR6 value is pointed by args->err */
	464	dr6_p = (unsigned long *)ERR_PTR(args->err);
	465	dr6 = *dr6_p;
0067f129 P	466
	467	/* Do an early return if no trap bits are set in DR6 */
	468	if ((dr6 & DR_TRAP_BITS) == 0)
	469	return NOTIFY_DONE;
	470
0067f129 P	471	get_debugreg(dr7, 7);
	472	/* Disable breakpoints during exception handling */
	473	set_debugreg(0UL, 7);
	474	/*
	475	* Assert that local interrupts are disabled
	476	* Reset the DRn bits in the virtualized register value.
	477	* The ptrace trigger routine will add in whatever is needed.
	478	*/
	479	current->thread.debugreg6 &= ~DR_TRAP_BITS;
	480	cpu = get_cpu();
	481
	482	/* Handle all the breakpoints that were triggered */
	483	for (i = 0; i < HBP_NUM; ++i) {
	484	if (likely(!(dr6 & (DR_TRAP0 << i))))
	485	continue;
24f1e32c	486
0067f129	487	/*
24f1e32c FW	488	* The counter may be concurrently released but that can only
	489	* occur from a call_rcu() path. We can then safely fetch
	490	* the breakpoint, use its callback, touch its counter
	491	* while we are in an rcu_read_lock() path.
0067f129	492	*/
24f1e32c FW	493	rcu_read_lock();
	494
	495	bp = per_cpu(bp_per_reg[i], cpu);
	496	if (bp)
	497	rc = NOTIFY_DONE;
62edab90 P	498	/*
	499	* Reset the 'i'th TRAP bit in dr6 to denote completion of
	500	* exception handling
	501	*/
	502	(*dr6_p) &= ~(DR_TRAP0 << i);
0067f129 P	503	/*
0067f129 P	504	* bp can be NULL due to lazy debug register switching
24f1e32c	505	* or due to concurrent perf counter removing.
0067f129	506	*/
24f1e32c FW	507	if (!bp) {
	508	rcu_read_unlock();
	509	break;
	510	}
	511
	512	(bp->callback)(bp, args->regs);
0067f129	513
24f1e32c	514	rcu_read_unlock();
0067f129 P	515	}
	516	if (dr6 & (~DR_TRAP_BITS))
	517	rc = NOTIFY_DONE;
	518
	519	set_debugreg(dr7, 7);
eadb8a09	520	put_cpu();
24f1e32c	521
0067f129 P	522	return rc;
	523	}
	524
	525	/*
	526	* Handle debug exception notifications.
	527	*/
	528	int __kprobes hw_breakpoint_exceptions_notify(
	529	struct notifier_block unused, unsigned long val, void data)
	530	{
	531	if (val != DIE_DEBUG)
	532	return NOTIFY_DONE;
	533
	534	return hw_breakpoint_handler(data);
	535	}
24f1e32c FW	536
	537	void hw_breakpoint_pmu_read(struct perf_event *bp)
	538	{
	539	/* TODO */
	540	}
	541
	542	void hw_breakpoint_pmu_unthrottle(struct perf_event *bp)
	543	{
	544	/* TODO */
	545	}