[deliverable/linux.git] / arch / x86 / kernel / hw_breakpoint.c

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) 2007 Alan Stern
 * Copyright (C) 2009 IBM Corporation
 * Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
 *
 * Authors: Alan Stern <stern@rowland.harvard.edu>
 *          K.Prasad <prasad@linux.vnet.ibm.com>
 *          Frederic Weisbecker <fweisbec@gmail.com>
 */

/*
 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
 * using the CPU's debug registers.
 */

#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
#include <linux/irqflags.h>
#include <linux/notifier.h>
#include <linux/kallsyms.h>
#include <linux/kprobes.h>
#include <linux/percpu.h>
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/smp.h>

#include <asm/hw_breakpoint.h>
#include <asm/processor.h>
#include <asm/debugreg.h>

/* Per cpu debug control register value */
DEFINE_PER_CPU(unsigned long, cpu_dr7);
EXPORT_PER_CPU_SYMBOL(cpu_dr7);

/* Per cpu debug address registers values */
static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);

/*
 * Stores the breakpoints currently in use on each breakpoint address
 * register for each cpus
 */
static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);


static inline unsigned long
__encode_dr7(int drnum, unsigned int len, unsigned int type)
{
	unsigned long bp_info;

	bp_info = (len | type) & 0xf;
	bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
	bp_info |= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE));

	return bp_info;
}

/*
 * Encode the length, type, Exact, and Enable bits for a particular breakpoint
 * as stored in debug register 7.
 */
unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
{
	return __encode_dr7(drnum, len, type) | DR_GLOBAL_SLOWDOWN;
}

/*
 * Decode the length and type bits for a particular breakpoint as
 * stored in debug register 7.  Return the "enabled" status.
 */
int decode_dr7(unsigned long dr7, int bpnum, unsigned *len, unsigned *type)
{
	int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);

	*len = (bp_info & 0xc) | 0x40;
	*type = (bp_info & 0x3) | 0x80;

	return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
}

/*
 * Install a perf counter breakpoint.
 *
 * We seek a free debug address register and use it for this
 * breakpoint. Eventually we enable it in the debug control register.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
int arch_install_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);

		if (!*slot) {
			*slot = bp;
			break;
		}
	}

	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return -EBUSY;

	set_debugreg(info->address, i);
	__get_cpu_var(cpu_debugreg[i]) = info->address;

	dr7 = &__get_cpu_var(cpu_dr7);
	*dr7 |= encode_dr7(i, info->len, info->type);

	set_debugreg(*dr7, 7);

	return 0;
}

/*
 * Uninstall the breakpoint contained in the given counter.
 *
 * First we search the debug address register it uses and then we disable
 * it.
 *
 * Atomic: we hold the counter->ctx->lock and we only handle variables
 * and registers local to this cpu.
 */
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned long *dr7;
	int i;

	for (i = 0; i < HBP_NUM; i++) {
		struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);

		if (*slot == bp) {
			*slot = NULL;
			break;
		}
	}

	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
		return;

	dr7 = &__get_cpu_var(cpu_dr7);
	*dr7 &= ~__encode_dr7(i, info->len, info->type);

	set_debugreg(*dr7, 7);
}

static int get_hbp_len(u8 hbp_len)
{
	unsigned int len_in_bytes = 0;

	switch (hbp_len) {
	case X86_BREAKPOINT_LEN_1:
		len_in_bytes = 1;
		break;
	case X86_BREAKPOINT_LEN_2:
		len_in_bytes = 2;
		break;
	case X86_BREAKPOINT_LEN_4:
		len_in_bytes = 4;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		len_in_bytes = 8;
		break;
#endif
	}
	return len_in_bytes;
}

/*
 * Check for virtual address in kernel space.
 */
int arch_check_bp_in_kernelspace(struct perf_event *bp)
{
	unsigned int len;
	unsigned long va;
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);

	va = info->address;
	len = get_hbp_len(info->len);

	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
}

int arch_bp_generic_fields(int x86_len, int x86_type,
			   int *gen_len, int *gen_type)
{
	/* Len */
	switch (x86_len) {
	case X86_BREAKPOINT_LEN_1:
		*gen_len = HW_BREAKPOINT_LEN_1;
		break;
	case X86_BREAKPOINT_LEN_2:
		*gen_len = HW_BREAKPOINT_LEN_2;
		break;
	case X86_BREAKPOINT_LEN_4:
		*gen_len = HW_BREAKPOINT_LEN_4;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		*gen_len = HW_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		return -EINVAL;
	}

	/* Type */
	switch (x86_type) {
	case X86_BREAKPOINT_EXECUTE:
		*gen_type = HW_BREAKPOINT_X;
		break;
	case X86_BREAKPOINT_WRITE:
		*gen_type = HW_BREAKPOINT_W;
		break;
	case X86_BREAKPOINT_RW:
		*gen_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R;
		break;
	default:
		return -EINVAL;
	}

	return 0;
}


static int arch_build_bp_info(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);

	info->address = bp->attr.bp_addr;

	/* Len */
	switch (bp->attr.bp_len) {
	case HW_BREAKPOINT_LEN_1:
		info->len = X86_BREAKPOINT_LEN_1;
		break;
	case HW_BREAKPOINT_LEN_2:
		info->len = X86_BREAKPOINT_LEN_2;
		break;
	case HW_BREAKPOINT_LEN_4:
		info->len = X86_BREAKPOINT_LEN_4;
		break;
#ifdef CONFIG_X86_64
	case HW_BREAKPOINT_LEN_8:
		info->len = X86_BREAKPOINT_LEN_8;
		break;
#endif
	default:
		return -EINVAL;
	}

	/* Type */
	switch (bp->attr.bp_type) {
	case HW_BREAKPOINT_W:
		info->type = X86_BREAKPOINT_WRITE;
		break;
	case HW_BREAKPOINT_W | HW_BREAKPOINT_R:
		info->type = X86_BREAKPOINT_RW;
		break;
	case HW_BREAKPOINT_X:
		info->type = X86_BREAKPOINT_EXECUTE;
		break;
	default:
		return -EINVAL;
	}

	return 0;
}
/*
 * Validate the arch-specific HW Breakpoint register settings
 */
int arch_validate_hwbkpt_settings(struct perf_event *bp)
{
	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	unsigned int align;
	int ret;


	ret = arch_build_bp_info(bp);
	if (ret)
		return ret;

	ret = -EINVAL;

	switch (info->len) {
	case X86_BREAKPOINT_LEN_1:
		align = 0;
		break;
	case X86_BREAKPOINT_LEN_2:
		align = 1;
		break;
	case X86_BREAKPOINT_LEN_4:
		align = 3;
		break;
#ifdef CONFIG_X86_64
	case X86_BREAKPOINT_LEN_8:
		align = 7;
		break;
#endif
	default:
		return ret;
	}

	/*
	 * Check that the low-order bits of the address are appropriate
	 * for the alignment implied by len.
	 */
	if (info->address & align)
		return -EINVAL;

	return 0;
}

/*
 * Dump the debug register contents to the user.
 * We can't dump our per cpu values because it
 * may contain cpu wide breakpoint, something that
 * doesn't belong to the current task.
 *
 * TODO: include non-ptrace user breakpoints (perf)
 */
void aout_dump_debugregs(struct user *dump)
{
	int i;
	int dr7 = 0;
	struct perf_event *bp;
	struct arch_hw_breakpoint *info;
	struct thread_struct *thread = &current->thread;

	for (i = 0; i < HBP_NUM; i++) {
		bp = thread->ptrace_bps[i];

		if (bp && !bp->attr.disabled) {
			dump->u_debugreg[i] = bp->attr.bp_addr;
			info = counter_arch_bp(bp);
			dr7 |= encode_dr7(i, info->len, info->type);
		} else {
			dump->u_debugreg[i] = 0;
		}
	}

	dump->u_debugreg[4] = 0;
	dump->u_debugreg[5] = 0;
	dump->u_debugreg[6] = current->thread.debugreg6;

	dump->u_debugreg[7] = dr7;
}
EXPORT_SYMBOL_GPL(aout_dump_debugregs);

/*
 * Release the user breakpoints used by ptrace
 */
void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
{
	int i;
	struct thread_struct *t = &tsk->thread;

	for (i = 0; i < HBP_NUM; i++) {
		unregister_hw_breakpoint(t->ptrace_bps[i]);
		t->ptrace_bps[i] = NULL;
	}
}

void hw_breakpoint_restore(void)
{
	set_debugreg(__get_cpu_var(cpu_debugreg[0]), 0);
	set_debugreg(__get_cpu_var(cpu_debugreg[1]), 1);
	set_debugreg(__get_cpu_var(cpu_debugreg[2]), 2);
	set_debugreg(__get_cpu_var(cpu_debugreg[3]), 3);
	set_debugreg(current->thread.debugreg6, 6);
	set_debugreg(__get_cpu_var(cpu_dr7), 7);
}
EXPORT_SYMBOL_GPL(hw_breakpoint_restore);

/*
 * Handle debug exception notifications.
 *
 * Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
 *
 * NOTIFY_DONE returned if one of the following conditions is true.
 * i) When the causative address is from user-space and the exception
 * is a valid one, i.e. not triggered as a result of lazy debug register
 * switching
 * ii) When there are more bits than trap<n> set in DR6 register (such
 * as BD, BS or BT) indicating that more than one debug condition is
 * met and requires some more action in do_debug().
 *
 * NOTIFY_STOP returned for all other cases
 *
 */
static int __kprobes hw_breakpoint_handler(struct die_args *args)
{
	int i, cpu, rc = NOTIFY_STOP;
	struct perf_event *bp;
	unsigned long dr7, dr6;
	unsigned long *dr6_p;

	/* The DR6 value is pointed by args->err */
	dr6_p = (unsigned long *)ERR_PTR(args->err);
	dr6 = *dr6_p;

	/* Do an early return if no trap bits are set in DR6 */
	if ((dr6 & DR_TRAP_BITS) == 0)
		return NOTIFY_DONE;

	get_debugreg(dr7, 7);
	/* Disable breakpoints during exception handling */
	set_debugreg(0UL, 7);
	/*
	 * Assert that local interrupts are disabled
	 * Reset the DRn bits in the virtualized register value.
	 * The ptrace trigger routine will add in whatever is needed.
	 */
	current->thread.debugreg6 &= ~DR_TRAP_BITS;
	cpu = get_cpu();

	/* Handle all the breakpoints that were triggered */
	for (i = 0; i < HBP_NUM; ++i) {
		if (likely(!(dr6 & (DR_TRAP0 << i))))
			continue;

		/*
		 * The counter may be concurrently released but that can only
		 * occur from a call_rcu() path. We can then safely fetch
		 * the breakpoint, use its callback, touch its counter
		 * while we are in an rcu_read_lock() path.
		 */
		rcu_read_lock();

		bp = per_cpu(bp_per_reg[i], cpu);
		/*
		 * Reset the 'i'th TRAP bit in dr6 to denote completion of
		 * exception handling
		 */
		(*dr6_p) &= ~(DR_TRAP0 << i);
		/*
		 * bp can be NULL due to lazy debug register switching
		 * or due to concurrent perf counter removing.
		 */
		if (!bp) {
			rcu_read_unlock();
			break;
		}

		perf_bp_event(bp, args->regs);

		rcu_read_unlock();
	}
	/*
	 * Further processing in do_debug() is needed for a) user-space
	 * breakpoints (to generate signals) and b) when the system has
	 * taken exception due to multiple causes
	 */
	if ((current->thread.debugreg6 & DR_TRAP_BITS) ||
	    (dr6 & (~DR_TRAP_BITS)))
		rc = NOTIFY_DONE;

	set_debugreg(dr7, 7);
	put_cpu();

	return rc;
}

/*
 * Handle debug exception notifications.
 */
int __kprobes hw_breakpoint_exceptions_notify(
		struct notifier_block *unused, unsigned long val, void *data)
{
	if (val != DIE_DEBUG)
		return NOTIFY_DONE;

	return hw_breakpoint_handler(data);
}

void hw_breakpoint_pmu_read(struct perf_event *bp)
{
	/* TODO */
}
Commit	Line	Data
0067f129 P	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License as published by
	4	* the Free Software Foundation; either version 2 of the License, or
	5	* (at your option) any later version.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	* You should have received a copy of the GNU General Public License
	13	* along with this program; if not, write to the Free Software
	14	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	*
	16	* Copyright (C) 2007 Alan Stern
	17	* Copyright (C) 2009 IBM Corporation
24f1e32c	18	* Copyright (C) 2009 Frederic Weisbecker <fweisbec@gmail.com>
ba6909b7 P	19	*
	20	* Authors: Alan Stern <stern@rowland.harvard.edu>
	21	* K.Prasad <prasad@linux.vnet.ibm.com>
	22	* Frederic Weisbecker <fweisbec@gmail.com>
0067f129 P	23	*/
	24
	25	/*
	26	* HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
	27	* using the CPU's debug registers.
	28	*/
	29
24f1e32c FW	30	#include <linux/perf_event.h>
24f1e32c FW	31	#include <linux/hw_breakpoint.h>
0067f129 P	32	#include <linux/irqflags.h>
	33	#include <linux/notifier.h>
	34	#include <linux/kallsyms.h>
	35	#include <linux/kprobes.h>
	36	#include <linux/percpu.h>
	37	#include <linux/kdebug.h>
	38	#include <linux/kernel.h>
	39	#include <linux/module.h>
	40	#include <linux/sched.h>
	41	#include <linux/init.h>
	42	#include <linux/smp.h>
	43
	44	#include <asm/hw_breakpoint.h>
	45	#include <asm/processor.h>
	46	#include <asm/debugreg.h>
	47
24f1e32c	48	/* Per cpu debug control register value */
28b4e0d8 TH	49	DEFINE_PER_CPU(unsigned long, cpu_dr7);
28b4e0d8 TH	50	EXPORT_PER_CPU_SYMBOL(cpu_dr7);
24f1e32c FW	51
	52	/* Per cpu debug address registers values */
	53	static DEFINE_PER_CPU(unsigned long, cpu_debugreg[HBP_NUM]);
0067f129 P	54
0067f129 P	55	/*
24f1e32c FW	56	* Stores the breakpoints currently in use on each breakpoint address
24f1e32c FW	57	* register for each cpus
0067f129	58	*/
24f1e32c	59	static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM]);
0067f129 P	60
0067f129 P	61
2c31b795 FW	62	static inline unsigned long
2c31b795 FW	63	__encode_dr7(int drnum, unsigned int len, unsigned int type)
0067f129 P	64	{
	65	unsigned long bp_info;
	66
	67	bp_info = (len \| type) & 0xf;
	68	bp_info <<= (DR_CONTROL_SHIFT + drnum * DR_CONTROL_SIZE);
2c31b795 FW	69	bp_info \|= (DR_GLOBAL_ENABLE << (drnum * DR_ENABLE_SIZE));
2c31b795 FW	70
0067f129 P	71	return bp_info;
	72	}
	73
2c31b795 FW	74	/*
	75	* Encode the length, type, Exact, and Enable bits for a particular breakpoint
	76	* as stored in debug register 7.
	77	*/
	78	unsigned long encode_dr7(int drnum, unsigned int len, unsigned int type)
	79	{
	80	return __encode_dr7(drnum, len, type) \| DR_GLOBAL_SLOWDOWN;
	81	}
	82
24f1e32c FW	83	/*
	84	* Decode the length and type bits for a particular breakpoint as
	85	* stored in debug register 7. Return the "enabled" status.
	86	*/
	87	int decode_dr7(unsigned long dr7, int bpnum, unsigned len, unsigned type)
0067f129	88	{
24f1e32c	89	int bp_info = dr7 >> (DR_CONTROL_SHIFT + bpnum * DR_CONTROL_SIZE);
0067f129	90
24f1e32c FW	91	*len = (bp_info & 0xc) \| 0x40;
24f1e32c FW	92	*type = (bp_info & 0x3) \| 0x80;
0067f129	93
24f1e32c	94	return (dr7 >> (bpnum * DR_ENABLE_SIZE)) & 0x3;
0067f129 P	95	}
	96
	97	/*
24f1e32c FW	98	* Install a perf counter breakpoint.
	99	*
	100	* We seek a free debug address register and use it for this
	101	* breakpoint. Eventually we enable it in the debug control register.
	102	*
	103	* Atomic: we hold the counter->ctx->lock and we only handle variables
	104	* and registers local to this cpu.
0067f129	105	*/
24f1e32c	106	int arch_install_hw_breakpoint(struct perf_event *bp)
0067f129	107	{
24f1e32c FW	108	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	109	unsigned long *dr7;
	110	int i;
	111
	112	for (i = 0; i < HBP_NUM; i++) {
	113	struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
	114
	115	if (!*slot) {
	116	*slot = bp;
	117	break;
	118	}
0067f129 P	119	}
0067f129 P	120
24f1e32c FW	121	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
	122	return -EBUSY;
	123
	124	set_debugreg(info->address, i);
	125	__get_cpu_var(cpu_debugreg[i]) = info->address;
	126
28b4e0d8	127	dr7 = &__get_cpu_var(cpu_dr7);
24f1e32c FW	128	*dr7 \|= encode_dr7(i, info->len, info->type);
	129
	130	set_debugreg(*dr7, 7);
	131
	132	return 0;
0067f129 P	133	}
	134
	135	/*
24f1e32c FW	136	* Uninstall the breakpoint contained in the given counter.
	137	*
	138	* First we search the debug address register it uses and then we disable
	139	* it.
	140	*
	141	* Atomic: we hold the counter->ctx->lock and we only handle variables
	142	* and registers local to this cpu.
0067f129	143	*/
24f1e32c	144	void arch_uninstall_hw_breakpoint(struct perf_event *bp)
0067f129	145	{
24f1e32c FW	146	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	147	unsigned long *dr7;
	148	int i;
	149
	150	for (i = 0; i < HBP_NUM; i++) {
	151	struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
	152
	153	if (*slot == bp) {
	154	*slot = NULL;
	155	break;
	156	}
	157	}
	158
	159	if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
	160	return;
0067f129	161
28b4e0d8	162	dr7 = &__get_cpu_var(cpu_dr7);
2c31b795	163	*dr7 &= ~__encode_dr7(i, info->len, info->type);
24f1e32c FW	164
24f1e32c FW	165	set_debugreg(*dr7, 7);
0067f129 P	166	}
	167
	168	static int get_hbp_len(u8 hbp_len)
	169	{
	170	unsigned int len_in_bytes = 0;
	171
	172	switch (hbp_len) {
24f1e32c	173	case X86_BREAKPOINT_LEN_1:
0067f129 P	174	len_in_bytes = 1;
0067f129 P	175	break;
24f1e32c	176	case X86_BREAKPOINT_LEN_2:
0067f129 P	177	len_in_bytes = 2;
0067f129 P	178	break;
24f1e32c	179	case X86_BREAKPOINT_LEN_4:
0067f129 P	180	len_in_bytes = 4;
	181	break;
	182	#ifdef CONFIG_X86_64
24f1e32c	183	case X86_BREAKPOINT_LEN_8:
0067f129 P	184	len_in_bytes = 8;
	185	break;
	186	#endif
	187	}
	188	return len_in_bytes;
	189	}
	190
0067f129 P	191	/*
	192	* Check for virtual address in kernel space.
	193	*/
b2812d03	194	int arch_check_bp_in_kernelspace(struct perf_event *bp)
0067f129 P	195	{
0067f129 P	196	unsigned int len;
b2812d03 FW	197	unsigned long va;
b2812d03 FW	198	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
0067f129	199
b2812d03 FW	200	va = info->address;
b2812d03 FW	201	len = get_hbp_len(info->len);
0067f129 P	202
	203	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
	204	}
	205
24f1e32c FW	206	int arch_bp_generic_fields(int x86_len, int x86_type,
24f1e32c FW	207	int gen_len, int gen_type)
0067f129	208	{
24f1e32c FW	209	/* Len */
	210	switch (x86_len) {
	211	case X86_BREAKPOINT_LEN_1:
	212	*gen_len = HW_BREAKPOINT_LEN_1;
	213	break;
	214	case X86_BREAKPOINT_LEN_2:
	215	*gen_len = HW_BREAKPOINT_LEN_2;
	216	break;
	217	case X86_BREAKPOINT_LEN_4:
	218	*gen_len = HW_BREAKPOINT_LEN_4;
	219	break;
	220	#ifdef CONFIG_X86_64
	221	case X86_BREAKPOINT_LEN_8:
	222	*gen_len = HW_BREAKPOINT_LEN_8;
	223	break;
	224	#endif
	225	default:
	226	return -EINVAL;
	227	}
0067f129	228
24f1e32c FW	229	/* Type */
	230	switch (x86_type) {
	231	case X86_BREAKPOINT_EXECUTE:
	232	*gen_type = HW_BREAKPOINT_X;
0067f129	233	break;
24f1e32c FW	234	case X86_BREAKPOINT_WRITE:
24f1e32c FW	235	*gen_type = HW_BREAKPOINT_W;
0067f129	236	break;
24f1e32c FW	237	case X86_BREAKPOINT_RW:
24f1e32c FW	238	*gen_type = HW_BREAKPOINT_W \| HW_BREAKPOINT_R;
0067f129 P	239	break;
0067f129 P	240	default:
24f1e32c	241	return -EINVAL;
0067f129 P	242	}
0067f129 P	243
24f1e32c FW	244	return 0;
	245	}
	246
	247
	248	static int arch_build_bp_info(struct perf_event *bp)
	249	{
	250	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	251
	252	info->address = bp->attr.bp_addr;
	253
	254	/* Len */
	255	switch (bp->attr.bp_len) {
0067f129	256	case HW_BREAKPOINT_LEN_1:
24f1e32c	257	info->len = X86_BREAKPOINT_LEN_1;
0067f129 P	258	break;
0067f129 P	259	case HW_BREAKPOINT_LEN_2:
24f1e32c	260	info->len = X86_BREAKPOINT_LEN_2;
0067f129 P	261	break;
0067f129 P	262	case HW_BREAKPOINT_LEN_4:
24f1e32c	263	info->len = X86_BREAKPOINT_LEN_4;
0067f129 P	264	break;
	265	#ifdef CONFIG_X86_64
	266	case HW_BREAKPOINT_LEN_8:
24f1e32c FW	267	info->len = X86_BREAKPOINT_LEN_8;
	268	break;
	269	#endif
	270	default:
	271	return -EINVAL;
	272	}
	273
	274	/* Type */
	275	switch (bp->attr.bp_type) {
	276	case HW_BREAKPOINT_W:
	277	info->type = X86_BREAKPOINT_WRITE;
	278	break;
	279	case HW_BREAKPOINT_W \| HW_BREAKPOINT_R:
	280	info->type = X86_BREAKPOINT_RW;
	281	break;
	282	case HW_BREAKPOINT_X:
	283	info->type = X86_BREAKPOINT_EXECUTE;
	284	break;
	285	default:
	286	return -EINVAL;
	287	}
	288
	289	return 0;
	290	}
	291	/*
	292	* Validate the arch-specific HW Breakpoint register settings
	293	*/
b2812d03	294	int arch_validate_hwbkpt_settings(struct perf_event *bp)
24f1e32c FW	295	{
	296	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
	297	unsigned int align;
	298	int ret;
	299
	300
	301	ret = arch_build_bp_info(bp);
	302	if (ret)
	303	return ret;
	304
	305	ret = -EINVAL;
	306
24f1e32c FW	307	switch (info->len) {
	308	case X86_BREAKPOINT_LEN_1:
	309	align = 0;
	310	break;
	311	case X86_BREAKPOINT_LEN_2:
	312	align = 1;
	313	break;
	314	case X86_BREAKPOINT_LEN_4:
	315	align = 3;
	316	break;
	317	#ifdef CONFIG_X86_64
	318	case X86_BREAKPOINT_LEN_8:
0067f129 P	319	align = 7;
	320	break;
	321	#endif
	322	default:
	323	return ret;
	324	}
	325
0067f129 P	326	/*
	327	* Check that the low-order bits of the address are appropriate
	328	* for the alignment implied by len.
	329	*/
24f1e32c	330	if (info->address & align)
0067f129 P	331	return -EINVAL;
0067f129 P	332
0067f129 P	333	return 0;
	334	}
	335
9f6b3c2c FW	336	/*
	337	* Dump the debug register contents to the user.
	338	* We can't dump our per cpu values because it
	339	* may contain cpu wide breakpoint, something that
	340	* doesn't belong to the current task.
	341	*
	342	* TODO: include non-ptrace user breakpoints (perf)
	343	*/
	344	void aout_dump_debugregs(struct user *dump)
	345	{
	346	int i;
	347	int dr7 = 0;
	348	struct perf_event *bp;
	349	struct arch_hw_breakpoint *info;
	350	struct thread_struct *thread = &current->thread;
	351
	352	for (i = 0; i < HBP_NUM; i++) {
	353	bp = thread->ptrace_bps[i];
	354
	355	if (bp && !bp->attr.disabled) {
	356	dump->u_debugreg[i] = bp->attr.bp_addr;
	357	info = counter_arch_bp(bp);
	358	dr7 \|= encode_dr7(i, info->len, info->type);
	359	} else {
	360	dump->u_debugreg[i] = 0;
	361	}
	362	}
	363
	364	dump->u_debugreg[4] = 0;
	365	dump->u_debugreg[5] = 0;
	366	dump->u_debugreg[6] = current->thread.debugreg6;
	367
	368	dump->u_debugreg[7] = dr7;
	369	}
68efa37d	370	EXPORT_SYMBOL_GPL(aout_dump_debugregs);
9f6b3c2c	371
24f1e32c FW	372	/*
	373	* Release the user breakpoints used by ptrace
	374	*/
	375	void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
0067f129	376	{
24f1e32c FW	377	int i;
	378	struct thread_struct *t = &tsk->thread;
	379
	380	for (i = 0; i < HBP_NUM; i++) {
	381	unregister_hw_breakpoint(t->ptrace_bps[i]);
	382	t->ptrace_bps[i] = NULL;
	383	}
0067f129 P	384	}
0067f129 P	385
24f1e32c	386	void hw_breakpoint_restore(void)
0067f129	387	{
24f1e32c FW	388	set_debugreg(__get_cpu_var(cpu_debugreg[0]), 0);
	389	set_debugreg(__get_cpu_var(cpu_debugreg[1]), 1);
	390	set_debugreg(__get_cpu_var(cpu_debugreg[2]), 2);
	391	set_debugreg(__get_cpu_var(cpu_debugreg[3]), 3);
	392	set_debugreg(current->thread.debugreg6, 6);
28b4e0d8	393	set_debugreg(__get_cpu_var(cpu_dr7), 7);
0067f129	394	}
24f1e32c	395	EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
0067f129 P	396
	397	/*
	398	* Handle debug exception notifications.
	399	*
	400	* Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
	401	*
	402	* NOTIFY_DONE returned if one of the following conditions is true.
	403	* i) When the causative address is from user-space and the exception
	404	* is a valid one, i.e. not triggered as a result of lazy debug register
	405	* switching
	406	* ii) When there are more bits than trap<n> set in DR6 register (such
	407	* as BD, BS or BT) indicating that more than one debug condition is
	408	* met and requires some more action in do_debug().
	409	*
	410	* NOTIFY_STOP returned for all other cases
	411	*
	412	*/
4555835b	413	static int __kprobes hw_breakpoint_handler(struct die_args *args)
0067f129 P	414	{
0067f129 P	415	int i, cpu, rc = NOTIFY_STOP;
24f1e32c	416	struct perf_event *bp;
62edab90 P	417	unsigned long dr7, dr6;
	418	unsigned long *dr6_p;
	419
	420	/* The DR6 value is pointed by args->err */
	421	dr6_p = (unsigned long *)ERR_PTR(args->err);
	422	dr6 = *dr6_p;
0067f129 P	423
	424	/* Do an early return if no trap bits are set in DR6 */
	425	if ((dr6 & DR_TRAP_BITS) == 0)
	426	return NOTIFY_DONE;
	427
0067f129 P	428	get_debugreg(dr7, 7);
	429	/* Disable breakpoints during exception handling */
	430	set_debugreg(0UL, 7);
	431	/*
	432	* Assert that local interrupts are disabled
	433	* Reset the DRn bits in the virtualized register value.
	434	* The ptrace trigger routine will add in whatever is needed.
	435	*/
	436	current->thread.debugreg6 &= ~DR_TRAP_BITS;
	437	cpu = get_cpu();
	438
	439	/* Handle all the breakpoints that were triggered */
	440	for (i = 0; i < HBP_NUM; ++i) {
	441	if (likely(!(dr6 & (DR_TRAP0 << i))))
	442	continue;
24f1e32c	443
0067f129	444	/*
24f1e32c FW	445	* The counter may be concurrently released but that can only
	446	* occur from a call_rcu() path. We can then safely fetch
	447	* the breakpoint, use its callback, touch its counter
	448	* while we are in an rcu_read_lock() path.
0067f129	449	*/
24f1e32c FW	450	rcu_read_lock();
	451
	452	bp = per_cpu(bp_per_reg[i], cpu);
62edab90 P	453	/*
	454	* Reset the 'i'th TRAP bit in dr6 to denote completion of
	455	* exception handling
	456	*/
	457	(*dr6_p) &= ~(DR_TRAP0 << i);
0067f129 P	458	/*
0067f129 P	459	* bp can be NULL due to lazy debug register switching
24f1e32c	460	* or due to concurrent perf counter removing.
0067f129	461	*/
24f1e32c FW	462	if (!bp) {
	463	rcu_read_unlock();
	464	break;
	465	}
	466
b326e956	467	perf_bp_event(bp, args->regs);
0067f129	468
24f1e32c	469	rcu_read_unlock();
0067f129	470	}
e0e53db6 P	471	/*
	472	* Further processing in do_debug() is needed for a) user-space
	473	* breakpoints (to generate signals) and b) when the system has
	474	* taken exception due to multiple causes
	475	*/
	476	if ((current->thread.debugreg6 & DR_TRAP_BITS) \|\|
	477	(dr6 & (~DR_TRAP_BITS)))
0067f129 P	478	rc = NOTIFY_DONE;
	479
	480	set_debugreg(dr7, 7);
eadb8a09	481	put_cpu();
24f1e32c	482
0067f129 P	483	return rc;
	484	}
	485
	486	/*
	487	* Handle debug exception notifications.
	488	*/
	489	int __kprobes hw_breakpoint_exceptions_notify(
	490	struct notifier_block unused, unsigned long val, void data)
	491	{
	492	if (val != DIE_DEBUG)
	493	return NOTIFY_DONE;
	494
	495	return hw_breakpoint_handler(data);
	496	}
24f1e32c FW	497
	498	void hw_breakpoint_pmu_read(struct perf_event *bp)
	499	{
	500	/* TODO */
	501	}