[deliverable/linux.git] / arch / x86 / kernel / mmiotrace / mmio-mod.c

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2005
 *               Jeff Muizelaar, 2006, 2007
 *               Pekka Paalanen, 2008 <pq@iki.fi>
 *
 * Derived from the read-mod example from relay-examples by Tom Zanussi.
 */
#define DEBUG 1

#include <linux/module.h>
#include <linux/relay.h>
#include <linux/debugfs.h>
#include <linux/proc_fs.h>
#include <asm/io.h>
#include <linux/version.h>
#include <linux/kallsyms.h>
#include <asm/pgtable.h>
#include <linux/mmiotrace.h>
#include <asm/e820.h> /* for ISA_START_ADDRESS */
#include <asm/atomic.h>
#include <linux/percpu.h>

#include "pf_in.h"

#define NAME "mmiotrace: "

/* This app's relay channel files will appear in /debug/mmio-trace */
static const char APP_DIR[] = "mmio-trace";
/* the marker injection file in /debug/APP_DIR */
static const char MARKER_FILE[] = "mmio-marker";

struct trap_reason {
	unsigned long addr;
	unsigned long ip;
	enum reason_type type;
	int active_traces;
};

struct remap_trace {
	struct list_head list;
	struct kmmio_probe probe;
	unsigned long phys;
	unsigned long id;
};

static const size_t subbuf_size = 256*1024;

/* Accessed per-cpu. */
static DEFINE_PER_CPU(struct trap_reason, pf_reason);
static DEFINE_PER_CPU(struct mm_io_header_rw, cpu_trace);

/* Access to this is not per-cpu. */
static DEFINE_PER_CPU(atomic_t, dropped);

static struct dentry *dir;
static struct dentry *enabled_file;
static struct dentry *marker_file;

static DEFINE_MUTEX(mmiotrace_mutex);
static DEFINE_SPINLOCK(trace_lock);
static atomic_t mmiotrace_enabled;
static LIST_HEAD(trace_list);		/* struct remap_trace */
static struct rchan *chan;

/*
 * Locking in this file:
 * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
 * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
 *   and trace_lock.
 * - Routines depending on is_enabled() must take trace_lock.
 * - trace_list users must hold trace_lock.
 * - is_enabled() guarantees that chan is valid.
 * - pre/post callbacks assume the effect of is_enabled() being true.
 */

/* module parameters */
static unsigned int	n_subbufs = 32*4;
static unsigned long	filter_offset;
static int		nommiotrace;
static int		ISA_trace;
static int		trace_pc;
static int		enable_now;

module_param(n_subbufs, uint, 0);
module_param(filter_offset, ulong, 0);
module_param(nommiotrace, bool, 0);
module_param(ISA_trace, bool, 0);
module_param(trace_pc, bool, 0);
module_param(enable_now, bool, 0);

MODULE_PARM_DESC(n_subbufs, "Number of 256kB buffers, default 128.");
MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing.");
MODULE_PARM_DESC(ISA_trace, "Do not exclude the low ISA range.");
MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
MODULE_PARM_DESC(enable_now, "Start mmiotracing immediately on module load.");

static bool is_enabled(void)
{
	return atomic_read(&mmiotrace_enabled);
}

static void record_timestamp(struct mm_io_header *header)
{
	struct timespec now;

	getnstimeofday(&now);
	header->sec = now.tv_sec;
	header->nsec = now.tv_nsec;
}

/*
 * Write callback for the debugfs entry:
 * Read a marker and write it to the mmio trace log
 */
static ssize_t write_marker(struct file *file, const char __user *buffer,
						size_t count, loff_t *ppos)
{
	char *event = NULL;
	struct mm_io_header *headp;
	ssize_t len = (count > 65535) ? 65535 : count;

	event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
	if (!event)
		return -ENOMEM;

	headp = (struct mm_io_header *)event;
	headp->type = MMIO_MAGIC | (MMIO_MARKER << MMIO_OPCODE_SHIFT);
	headp->data_len = len;
	record_timestamp(headp);

	if (copy_from_user(event + sizeof(*headp), buffer, len)) {
		kfree(event);
		return -EFAULT;
	}

	spin_lock_irq(&trace_lock);
	if (is_enabled())
		relay_write(chan, event, sizeof(*headp) + len);
	else
		len = -EINVAL;
	spin_unlock_irq(&trace_lock);
	kfree(event);
	return len;
}

static void print_pte(unsigned long address)
{
	int level;
	pte_t *pte = lookup_address(address, &level);

	if (!pte) {
		pr_err(NAME "Error in %s: no pte for page 0x%08lx\n",
							__func__, address);
		return;
	}

	if (level == PG_LEVEL_2M) {
		pr_emerg(NAME "4MB pages are not currently supported: "
							"0x%08lx\n", address);
		BUG();
	}
	pr_info(NAME "pte for 0x%lx: 0x%lx 0x%lx\n", address, pte_val(*pte),
						pte_val(*pte) & _PAGE_PRESENT);
}

/*
 * For some reason the pre/post pairs have been called in an
 * unmatched order. Report and die.
 */
static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
{
	const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
	pr_emerg(NAME "unexpected fault for address: 0x%08lx, "
					"last fault for address: 0x%08lx\n",
					addr, my_reason->addr);
	print_pte(addr);
	print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip);
	print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip);
#ifdef __i386__
	pr_emerg("eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
			regs->ax, regs->bx, regs->cx, regs->dx);
	pr_emerg("esi: %08lx   edi: %08lx   ebp: %08lx   esp: %08lx\n",
			regs->si, regs->di, regs->bp, regs->sp);
#else
	pr_emerg("rax: %016lx   rcx: %016lx   rdx: %016lx\n",
					regs->ax, regs->cx, regs->dx);
	pr_emerg("rsi: %016lx   rdi: %016lx   rbp: %016lx   rsp: %016lx\n",
				regs->si, regs->di, regs->bp, regs->sp);
#endif
	put_cpu_var(pf_reason);
	BUG();
}

static void pre(struct kmmio_probe *p, struct pt_regs *regs,
						unsigned long addr)
{
	struct trap_reason *my_reason = &get_cpu_var(pf_reason);
	struct mm_io_header_rw *my_trace = &get_cpu_var(cpu_trace);
	const unsigned long instptr = instruction_pointer(regs);
	const enum reason_type type = get_ins_type(instptr);

	/* it doesn't make sense to have more than one active trace per cpu */
	if (my_reason->active_traces)
		die_kmmio_nesting_error(regs, addr);
	else
		my_reason->active_traces++;

	my_reason->type = type;
	my_reason->addr = addr;
	my_reason->ip = instptr;

	my_trace->header.type = MMIO_MAGIC;
	my_trace->header.pid = 0;
	my_trace->header.data_len = sizeof(struct mm_io_rw);
	my_trace->rw.address = addr;
	/*
	 * struct remap_trace *trace = p->user_data;
	 * phys = addr - trace->probe.addr + trace->phys;
	 */

	/*
	 * Only record the program counter when requested.
	 * It may taint clean-room reverse engineering.
	 */
	if (trace_pc)
		my_trace->rw.pc = instptr;
	else
		my_trace->rw.pc = 0;

	record_timestamp(&my_trace->header);

	switch (type) {
	case REG_READ:
		my_trace->header.type |=
			(MMIO_READ << MMIO_OPCODE_SHIFT) |
			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
		break;
	case REG_WRITE:
		my_trace->header.type |=
			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
		my_trace->rw.value = get_ins_reg_val(instptr, regs);
		break;
	case IMM_WRITE:
		my_trace->header.type |=
			(MMIO_WRITE << MMIO_OPCODE_SHIFT) |
			(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
		my_trace->rw.value = get_ins_imm_val(instptr);
		break;
	default:
		{
			unsigned char *ip = (unsigned char *)instptr;
			my_trace->header.type |=
					(MMIO_UNKNOWN_OP << MMIO_OPCODE_SHIFT);
			my_trace->rw.value = (*ip) << 16 | *(ip + 1) << 8 |
								*(ip + 2);
		}
	}
	put_cpu_var(cpu_trace);
	put_cpu_var(pf_reason);
}

static void post(struct kmmio_probe *p, unsigned long condition,
							struct pt_regs *regs)
{
	struct trap_reason *my_reason = &get_cpu_var(pf_reason);
	struct mm_io_header_rw *my_trace = &get_cpu_var(cpu_trace);

	/* this should always return the active_trace count to 0 */
	my_reason->active_traces--;
	if (my_reason->active_traces) {
		pr_emerg(NAME "unexpected post handler");
		BUG();
	}

	switch (my_reason->type) {
	case REG_READ:
		my_trace->rw.value = get_ins_reg_val(my_reason->ip, regs);
		break;
	default:
		break;
	}
	relay_write(chan, my_trace, sizeof(*my_trace));
	put_cpu_var(cpu_trace);
	put_cpu_var(pf_reason);
}

/*
 * subbuf_start() relay callback.
 *
 * Defined so that we know when events are dropped due to the buffer-full
 * condition.
 */
static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf,
					void *prev_subbuf, size_t prev_padding)
{
	unsigned int cpu = buf->cpu;
	atomic_t *drop = &per_cpu(dropped, cpu);
	int count;
	if (relay_buf_full(buf)) {
		if (atomic_inc_return(drop) == 1)
			pr_err(NAME "cpu %d buffer full!\n", cpu);
		return 0;
	}
	count = atomic_read(drop);
	if (count) {
		pr_err(NAME "cpu %d buffer no longer full, missed %d events.\n",
								cpu, count);
		atomic_sub(count, drop);
	}

	return 1;
}

static struct file_operations mmio_fops = {
	.owner = THIS_MODULE,
};

/* file_create() callback.  Creates relay file in debugfs. */
static struct dentry *create_buf_file_handler(const char *filename,
						struct dentry *parent,
						int mode,
						struct rchan_buf *buf,
						int *is_global)
{
	struct dentry *buf_file;

	mmio_fops.read = relay_file_operations.read;
	mmio_fops.open = relay_file_operations.open;
	mmio_fops.poll = relay_file_operations.poll;
	mmio_fops.mmap = relay_file_operations.mmap;
	mmio_fops.release = relay_file_operations.release;
	mmio_fops.splice_read = relay_file_operations.splice_read;

	buf_file = debugfs_create_file(filename, mode, parent, buf,
								&mmio_fops);

	return buf_file;
}

/* file_remove() default callback.  Removes relay file in debugfs. */
static int remove_buf_file_handler(struct dentry *dentry)
{
	debugfs_remove(dentry);
	return 0;
}

static struct rchan_callbacks relay_callbacks = {
	.subbuf_start = subbuf_start_handler,
	.create_buf_file = create_buf_file_handler,
	.remove_buf_file = remove_buf_file_handler,
};

static void ioremap_trace_core(unsigned long offset, unsigned long size,
							void __iomem *addr)
{
	static atomic_t next_id;
	struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
	struct mm_io_header_map event = {
		.header = {
			.type = MMIO_MAGIC |
					(MMIO_PROBE << MMIO_OPCODE_SHIFT),
			.sec = 0,
			.nsec = 0,
			.pid = 0,
			.data_len = sizeof(struct mm_io_map)
		},
		.map = {
			.phys = offset,
			.addr = (unsigned long)addr,
			.len  = size,
			.pc   = 0
		}
	};
	record_timestamp(&event.header);

	if (!trace) {
		pr_err(NAME "kmalloc failed in ioremap\n");
		return;
	}

	*trace = (struct remap_trace) {
		.probe = {
			.addr = (unsigned long)addr,
			.len = size,
			.pre_handler = pre,
			.post_handler = post,
			.user_data = trace
		},
		.phys = offset,
		.id = atomic_inc_return(&next_id)
	};

	spin_lock_irq(&trace_lock);
	if (!is_enabled())
		goto not_enabled;

	relay_write(chan, &event, sizeof(event));
	list_add_tail(&trace->list, &trace_list);
	if (!nommiotrace)
		register_kmmio_probe(&trace->probe);

not_enabled:
	spin_unlock_irq(&trace_lock);
}

void
mmiotrace_ioremap(unsigned long offset, unsigned long size, void __iomem *addr)
{
	if (!is_enabled()) /* recheck and proper locking in *_core() */
		return;

	pr_debug(NAME "ioremap_*(0x%lx, 0x%lx) = %p\n", offset, size, addr);
	if ((filter_offset) && (offset != filter_offset))
		return;
	ioremap_trace_core(offset, size, addr);
}

static void iounmap_trace_core(volatile void __iomem *addr)
{
	struct mm_io_header_map event = {
		.header = {
			.type = MMIO_MAGIC |
				(MMIO_UNPROBE << MMIO_OPCODE_SHIFT),
			.sec = 0,
			.nsec = 0,
			.pid = 0,
			.data_len = sizeof(struct mm_io_map)
		},
		.map = {
			.phys = 0,
			.addr = (unsigned long)addr,
			.len  = 0,
			.pc   = 0
		}
	};
	struct remap_trace *trace;
	struct remap_trace *tmp;
	struct remap_trace *found_trace = NULL;

	pr_debug(NAME "Unmapping %p.\n", addr);
	record_timestamp(&event.header);

	spin_lock_irq(&trace_lock);
	if (!is_enabled())
		goto not_enabled;

	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
		if ((unsigned long)addr == trace->probe.addr) {
			if (!nommiotrace)
				unregister_kmmio_probe(&trace->probe);
			list_del(&trace->list);
			found_trace = trace;
			break;
		}
	}
	relay_write(chan, &event, sizeof(event));

not_enabled:
	spin_unlock_irq(&trace_lock);
	if (found_trace) {
		synchronize_rcu(); /* unregister_kmmio_probe() requirement */
		kfree(found_trace);
	}
}

void mmiotrace_iounmap(volatile void __iomem *addr)
{
	might_sleep();
	if (is_enabled()) /* recheck and proper locking in *_core() */
		iounmap_trace_core(addr);
}

static void clear_trace_list(void)
{
	struct remap_trace *trace;
	struct remap_trace *tmp;

	/*
	 * No locking required, because the caller ensures we are in a
	 * critical section via mutex, and is_enabled() is false,
	 * i.e. nothing can traverse or modify this list.
	 * Caller also ensures is_enabled() cannot change.
	 */
	list_for_each_entry(trace, &trace_list, list) {
		pr_notice(NAME "purging non-iounmapped "
					"trace @0x%08lx, size 0x%lx.\n",
					trace->probe.addr, trace->probe.len);
		if (!nommiotrace)
			unregister_kmmio_probe(&trace->probe);
	}
	synchronize_rcu(); /* unregister_kmmio_probe() requirement */

	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
		list_del(&trace->list);
		kfree(trace);
	}
}

static ssize_t read_enabled_file_bool(struct file *file,
		char __user *user_buf, size_t count, loff_t *ppos)
{
	char buf[3];

	if (is_enabled())
		buf[0] = '1';
	else
		buf[0] = '0';
	buf[1] = '\n';
	buf[2] = '\0';
	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
}

static void enable_mmiotrace(void);
static void disable_mmiotrace(void);

static ssize_t write_enabled_file_bool(struct file *file,
		const char __user *user_buf, size_t count, loff_t *ppos)
{
	char buf[32];
	int buf_size = min(count, (sizeof(buf)-1));

	if (copy_from_user(buf, user_buf, buf_size))
		return -EFAULT;

	switch (buf[0]) {
	case 'y':
	case 'Y':
	case '1':
		enable_mmiotrace();
		break;
	case 'n':
	case 'N':
	case '0':
		disable_mmiotrace();
		break;
	}

	return count;
}

/* this ripped from kernel/kprobes.c */
static struct file_operations fops_enabled = {
	.owner =	THIS_MODULE,
	.read =		read_enabled_file_bool,
	.write =	write_enabled_file_bool
};

static struct file_operations fops_marker = {
	.owner =	THIS_MODULE,
	.write =	write_marker
};

static void enable_mmiotrace(void)
{
	mutex_lock(&mmiotrace_mutex);
	if (is_enabled())
		goto out;

	chan = relay_open("cpu", dir, subbuf_size, n_subbufs,
						&relay_callbacks, NULL);
	if (!chan) {
		pr_err(NAME "relay app channel creation failed.\n");
		goto out;
	}

	reference_kmmio();

	marker_file = debugfs_create_file("marker", 0660, dir, NULL,
								&fops_marker);
	if (!marker_file)
		pr_err(NAME "marker file creation failed.\n");

	if (nommiotrace)
		pr_info(NAME "MMIO tracing disabled.\n");
	if (ISA_trace)
		pr_warning(NAME "Warning! low ISA range will be traced.\n");
	spin_lock_irq(&trace_lock);
	atomic_inc(&mmiotrace_enabled);
	spin_unlock_irq(&trace_lock);
	pr_info(NAME "enabled.\n");
out:
	mutex_unlock(&mmiotrace_mutex);
}

static void disable_mmiotrace(void)
{
	mutex_lock(&mmiotrace_mutex);
	if (!is_enabled())
		goto out;

	spin_lock_irq(&trace_lock);
	atomic_dec(&mmiotrace_enabled);
	BUG_ON(is_enabled());
	spin_unlock_irq(&trace_lock);

	clear_trace_list(); /* guarantees: no more kmmio callbacks */
	unreference_kmmio();
	if (marker_file) {
		debugfs_remove(marker_file);
		marker_file = NULL;
	}
	if (chan) {
		relay_close(chan);
		chan = NULL;
	}

	pr_info(NAME "disabled.\n");
out:
	mutex_unlock(&mmiotrace_mutex);
}

static int __init init(void)
{
	pr_debug(NAME "load...\n");
	if (n_subbufs < 2)
		return -EINVAL;

	dir = debugfs_create_dir(APP_DIR, NULL);
	if (!dir) {
		pr_err(NAME "Couldn't create relay app directory.\n");
		return -ENOMEM;
	}

	enabled_file = debugfs_create_file("enabled", 0600, dir, NULL,
								&fops_enabled);
	if (!enabled_file) {
		pr_err(NAME "Couldn't create enabled file.\n");
		debugfs_remove(dir);
		return -ENOMEM;
	}

	if (enable_now)
		enable_mmiotrace();

	return 0;
}

static void __exit cleanup(void)
{
	pr_debug(NAME "unload...\n");
	if (enabled_file)
		debugfs_remove(enabled_file);
	disable_mmiotrace();
	if (dir)
		debugfs_remove(dir);
}

module_init(init);
module_exit(cleanup);
MODULE_LICENSE("GPL");
Commit	Line	Data
8b7d89d0 PP	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License as published by
	4	* the Free Software Foundation; either version 2 of the License, or
	5	* (at your option) any later version.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	* You should have received a copy of the GNU General Public License
	13	* along with this program; if not, write to the Free Software
	14	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	*
	16	* Copyright (C) IBM Corporation, 2005
	17	* Jeff Muizelaar, 2006, 2007
	18	* Pekka Paalanen, 2008 <pq@iki.fi>
	19	*
	20	* Derived from the read-mod example from relay-examples by Tom Zanussi.
	21	*/
d61fc448 PP	22	#define DEBUG 1
d61fc448 PP	23
8b7d89d0 PP	24	#include <linux/module.h>
	25	#include <linux/relay.h>
	26	#include <linux/debugfs.h>
	27	#include <linux/proc_fs.h>
	28	#include <asm/io.h>
	29	#include <linux/version.h>
	30	#include <linux/kallsyms.h>
	31	#include <asm/pgtable.h>
	32	#include <linux/mmiotrace.h>
	33	#include <asm/e820.h> /* for ISA_START_ADDRESS */
fe1ffafa	34	#include <asm/atomic.h>
f5136380	35	#include <linux/percpu.h>
8b7d89d0	36
8b7d89d0 PP	37	#include "pf_in.h"
8b7d89d0 PP	38
d61fc448	39	#define NAME "mmiotrace: "
8b7d89d0	40
d61fc448 PP	41	/* This app's relay channel files will appear in /debug/mmio-trace */
	42	static const char APP_DIR[] = "mmio-trace";
	43	/* the marker injection file in /debug/APP_DIR */
	44	static const char MARKER_FILE[] = "mmio-marker";
8b7d89d0 PP	45
	46	struct trap_reason {
	47	unsigned long addr;
	48	unsigned long ip;
	49	enum reason_type type;
	50	int active_traces;
	51	};
	52
d61fc448 PP	53	struct remap_trace {
	54	struct list_head list;
	55	struct kmmio_probe probe;
	56	unsigned long phys;
	57	unsigned long id;
	58	};
	59
	60	static const size_t subbuf_size = 256*1024;
	61
fe1ffafa	62	/* Accessed per-cpu. */
f5136380 PP	63	static DEFINE_PER_CPU(struct trap_reason, pf_reason);
f5136380 PP	64	static DEFINE_PER_CPU(struct mm_io_header_rw, cpu_trace);
8b7d89d0	65
fe1ffafa	66	/* Access to this is not per-cpu. */
f5136380	67	static DEFINE_PER_CPU(atomic_t, dropped);
fe1ffafa	68
d61fc448 PP	69	static struct dentry *dir;
	70	static struct dentry *enabled_file;
	71	static struct dentry *marker_file;
8b7d89d0	72
d61fc448 PP	73	static DEFINE_MUTEX(mmiotrace_mutex);
	74	static DEFINE_SPINLOCK(trace_lock);
	75	static atomic_t mmiotrace_enabled;
	76	static LIST_HEAD(trace_list); /* struct remap_trace */
8b7d89d0	77	static struct rchan *chan;
d61fc448 PP	78
	79	/*
	80	* Locking in this file:
	81	* - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
	82	* - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
	83	* and trace_lock.
	84	* - Routines depending on is_enabled() must take trace_lock.
	85	* - trace_list users must hold trace_lock.
	86	* - is_enabled() guarantees that chan is valid.
	87	* - pre/post callbacks assume the effect of is_enabled() being true.
	88	*/
8b7d89d0 PP	89
8b7d89d0 PP	90	/* module parameters */
d61fc448 PP	91	static unsigned int n_subbufs = 32*4;
	92	static unsigned long filter_offset;
	93	static int nommiotrace;
	94	static int ISA_trace;
	95	static int trace_pc;
	96	static int enable_now;
8b7d89d0 PP	97
	98	module_param(n_subbufs, uint, 0);
	99	module_param(filter_offset, ulong, 0);
	100	module_param(nommiotrace, bool, 0);
	101	module_param(ISA_trace, bool, 0);
	102	module_param(trace_pc, bool, 0);
d61fc448	103	module_param(enable_now, bool, 0);
8b7d89d0 PP	104
	105	MODULE_PARM_DESC(n_subbufs, "Number of 256kB buffers, default 128.");
	106	MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
	107	MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing.");
	108	MODULE_PARM_DESC(ISA_trace, "Do not exclude the low ISA range.");
	109	MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
d61fc448 PP	110	MODULE_PARM_DESC(enable_now, "Start mmiotracing immediately on module load.");
	111
	112	static bool is_enabled(void)
	113	{
	114	return atomic_read(&mmiotrace_enabled);
	115	}
8b7d89d0 PP	116
	117	static void record_timestamp(struct mm_io_header *header)
	118	{
	119	struct timespec now;
	120
	121	getnstimeofday(&now);
	122	header->sec = now.tv_sec;
	123	header->nsec = now.tv_nsec;
	124	}
	125
	126	/*
d61fc448	127	* Write callback for the debugfs entry:
8b7d89d0 PP	128	* Read a marker and write it to the mmio trace log
8b7d89d0 PP	129	*/
d61fc448 PP	130	static ssize_t write_marker(struct file file, const char __user buffer,
d61fc448 PP	131	size_t count, loff_t *ppos)
8b7d89d0 PP	132	{
	133	char *event = NULL;
	134	struct mm_io_header *headp;
d61fc448	135	ssize_t len = (count > 65535) ? 65535 : count;
8b7d89d0 PP	136
	137	event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
	138	if (!event)
	139	return -ENOMEM;
	140
	141	headp = (struct mm_io_header *)event;
	142	headp->type = MMIO_MAGIC \| (MMIO_MARKER << MMIO_OPCODE_SHIFT);
	143	headp->data_len = len;
	144	record_timestamp(headp);
	145
	146	if (copy_from_user(event + sizeof(*headp), buffer, len)) {
	147	kfree(event);
	148	return -EFAULT;
	149	}
	150
d61fc448 PP	151	spin_lock_irq(&trace_lock);
	152	if (is_enabled())
	153	relay_write(chan, event, sizeof(*headp) + len);
	154	else
	155	len = -EINVAL;
	156	spin_unlock_irq(&trace_lock);
8b7d89d0 PP	157	kfree(event);
	158	return len;
	159	}
	160
	161	static void print_pte(unsigned long address)
	162	{
75bb8835 PP	163	int level;
	164	pte_t *pte = lookup_address(address, &level);
	165
	166	if (!pte) {
d61fc448	167	pr_err(NAME "Error in %s: no pte for page 0x%08lx\n",
0fd0e3da	168	__func__, address);
75bb8835 PP	169	return;
	170	}
	171
	172	if (level == PG_LEVEL_2M) {
d61fc448 PP	173	pr_emerg(NAME "4MB pages are not currently supported: "
d61fc448 PP	174	"0x%08lx\n", address);
8b7d89d0 PP	175	BUG();
8b7d89d0 PP	176	}
d61fc448 PP	177	pr_info(NAME "pte for 0x%lx: 0x%lx 0x%lx\n", address, pte_val(*pte),
d61fc448 PP	178	pte_val(*pte) & _PAGE_PRESENT);
8b7d89d0 PP	179	}
	180
	181	/*
	182	* For some reason the pre/post pairs have been called in an
	183	* unmatched order. Report and die.
	184	*/
	185	static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
	186	{
f5136380	187	const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
d61fc448 PP	188	pr_emerg(NAME "unexpected fault for address: 0x%08lx, "
d61fc448 PP	189	"last fault for address: 0x%08lx\n",
f5136380	190	addr, my_reason->addr);
8b7d89d0	191	print_pte(addr);
d61fc448 PP	192	print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip);
d61fc448 PP	193	print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip);
8b7d89d0	194	#ifdef __i386__
0fd0e3da	195	pr_emerg("eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n",
8b7d89d0	196	regs->ax, regs->bx, regs->cx, regs->dx);
0fd0e3da	197	pr_emerg("esi: %08lx edi: %08lx ebp: %08lx esp: %08lx\n",
8b7d89d0 PP	198	regs->si, regs->di, regs->bp, regs->sp);
8b7d89d0 PP	199	#else
0fd0e3da	200	pr_emerg("rax: %016lx rcx: %016lx rdx: %016lx\n",
8b7d89d0	201	regs->ax, regs->cx, regs->dx);
0fd0e3da	202	pr_emerg("rsi: %016lx rdi: %016lx rbp: %016lx rsp: %016lx\n",
8b7d89d0 PP	203	regs->si, regs->di, regs->bp, regs->sp);
8b7d89d0 PP	204	#endif
f5136380	205	put_cpu_var(pf_reason);
8b7d89d0 PP	206	BUG();
	207	}
	208
	209	static void pre(struct kmmio_probe p, struct pt_regs regs,
	210	unsigned long addr)
	211	{
f5136380 PP	212	struct trap_reason *my_reason = &get_cpu_var(pf_reason);
f5136380 PP	213	struct mm_io_header_rw *my_trace = &get_cpu_var(cpu_trace);
8b7d89d0 PP	214	const unsigned long instptr = instruction_pointer(regs);
	215	const enum reason_type type = get_ins_type(instptr);
	216
	217	/* it doesn't make sense to have more than one active trace per cpu */
f5136380	218	if (my_reason->active_traces)
8b7d89d0 PP	219	die_kmmio_nesting_error(regs, addr);
8b7d89d0 PP	220	else
f5136380	221	my_reason->active_traces++;
8b7d89d0	222
f5136380 PP	223	my_reason->type = type;
	224	my_reason->addr = addr;
	225	my_reason->ip = instptr;
8b7d89d0	226
f5136380 PP	227	my_trace->header.type = MMIO_MAGIC;
	228	my_trace->header.pid = 0;
	229	my_trace->header.data_len = sizeof(struct mm_io_rw);
	230	my_trace->rw.address = addr;
d61fc448 PP	231	/*
	232	* struct remap_trace *trace = p->user_data;
	233	* phys = addr - trace->probe.addr + trace->phys;
	234	*/
8b7d89d0 PP	235
	236	/*
	237	* Only record the program counter when requested.
	238	* It may taint clean-room reverse engineering.
	239	*/
	240	if (trace_pc)
f5136380	241	my_trace->rw.pc = instptr;
8b7d89d0	242	else
f5136380	243	my_trace->rw.pc = 0;
8b7d89d0	244
f5136380	245	record_timestamp(&my_trace->header);
8b7d89d0 PP	246
	247	switch (type) {
	248	case REG_READ:
f5136380	249	my_trace->header.type \|=
8b7d89d0 PP	250	(MMIO_READ << MMIO_OPCODE_SHIFT) \|
	251	(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
	252	break;
	253	case REG_WRITE:
f5136380	254	my_trace->header.type \|=
8b7d89d0 PP	255	(MMIO_WRITE << MMIO_OPCODE_SHIFT) \|
8b7d89d0 PP	256	(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
f5136380	257	my_trace->rw.value = get_ins_reg_val(instptr, regs);
8b7d89d0 PP	258	break;
8b7d89d0 PP	259	case IMM_WRITE:
f5136380	260	my_trace->header.type \|=
8b7d89d0 PP	261	(MMIO_WRITE << MMIO_OPCODE_SHIFT) \|
8b7d89d0 PP	262	(get_ins_mem_width(instptr) << MMIO_WIDTH_SHIFT);
f5136380	263	my_trace->rw.value = get_ins_imm_val(instptr);
8b7d89d0 PP	264	break;
	265	default:
	266	{
	267	unsigned char ip = (unsigned char )instptr;
f5136380	268	my_trace->header.type \|=
8b7d89d0	269	(MMIO_UNKNOWN_OP << MMIO_OPCODE_SHIFT);
f5136380 PP	270	my_trace->rw.value = (ip) << 16 \| (ip + 1) << 8 \|
f5136380 PP	271	*(ip + 2);
8b7d89d0 PP	272	}
8b7d89d0 PP	273	}
f5136380 PP	274	put_cpu_var(cpu_trace);
f5136380 PP	275	put_cpu_var(pf_reason);
8b7d89d0 PP	276	}
	277
	278	static void post(struct kmmio_probe *p, unsigned long condition,
	279	struct pt_regs *regs)
	280	{
f5136380 PP	281	struct trap_reason *my_reason = &get_cpu_var(pf_reason);
f5136380 PP	282	struct mm_io_header_rw *my_trace = &get_cpu_var(cpu_trace);
8b7d89d0 PP	283
8b7d89d0 PP	284	/* this should always return the active_trace count to 0 */
f5136380 PP	285	my_reason->active_traces--;
f5136380 PP	286	if (my_reason->active_traces) {
d61fc448	287	pr_emerg(NAME "unexpected post handler");
8b7d89d0 PP	288	BUG();
	289	}
	290
f5136380	291	switch (my_reason->type) {
8b7d89d0	292	case REG_READ:
f5136380	293	my_trace->rw.value = get_ins_reg_val(my_reason->ip, regs);
8b7d89d0 PP	294	break;
	295	default:
	296	break;
	297	}
f5136380 PP	298	relay_write(chan, my_trace, sizeof(*my_trace));
	299	put_cpu_var(cpu_trace);
	300	put_cpu_var(pf_reason);
8b7d89d0 PP	301	}
	302
	303	/*
	304	* subbuf_start() relay callback.
	305	*
	306	* Defined so that we know when events are dropped due to the buffer-full
	307	* condition.
	308	*/
	309	static int subbuf_start_handler(struct rchan_buf buf, void subbuf,
	310	void *prev_subbuf, size_t prev_padding)
	311	{
fe1ffafa	312	unsigned int cpu = buf->cpu;
f5136380	313	atomic_t *drop = &per_cpu(dropped, cpu);
fe1ffafa	314	int count;
8b7d89d0	315	if (relay_buf_full(buf)) {
0fd0e3da	316	if (atomic_inc_return(drop) == 1)
d61fc448	317	pr_err(NAME "cpu %d buffer full!\n", cpu);
8b7d89d0	318	return 0;
0fd0e3da PP	319	}
	320	count = atomic_read(drop);
	321	if (count) {
d61fc448 PP	322	pr_err(NAME "cpu %d buffer no longer full, missed %d events.\n",
d61fc448 PP	323	cpu, count);
fe1ffafa	324	atomic_sub(count, drop);
8b7d89d0 PP	325	}
	326
	327	return 1;
	328	}
	329
d61fc448 PP	330	static struct file_operations mmio_fops = {
	331	.owner = THIS_MODULE,
	332	};
	333
8b7d89d0 PP	334	/* file_create() callback. Creates relay file in debugfs. */
	335	static struct dentry create_buf_file_handler(const char filename,
	336	struct dentry *parent,
	337	int mode,
	338	struct rchan_buf *buf,
	339	int *is_global)
	340	{
	341	struct dentry *buf_file;
	342
	343	mmio_fops.read = relay_file_operations.read;
	344	mmio_fops.open = relay_file_operations.open;
	345	mmio_fops.poll = relay_file_operations.poll;
	346	mmio_fops.mmap = relay_file_operations.mmap;
	347	mmio_fops.release = relay_file_operations.release;
	348	mmio_fops.splice_read = relay_file_operations.splice_read;
	349
	350	buf_file = debugfs_create_file(filename, mode, parent, buf,
	351	&mmio_fops);
	352
	353	return buf_file;
	354	}
	355
	356	/* file_remove() default callback. Removes relay file in debugfs. */
	357	static int remove_buf_file_handler(struct dentry *dentry)
	358	{
	359	debugfs_remove(dentry);
	360	return 0;
	361	}
	362
	363	static struct rchan_callbacks relay_callbacks = {
	364	.subbuf_start = subbuf_start_handler,
	365	.create_buf_file = create_buf_file_handler,
	366	.remove_buf_file = remove_buf_file_handler,
	367	};
	368
d61fc448	369	static void ioremap_trace_core(unsigned long offset, unsigned long size,
8b7d89d0 PP	370	void __iomem *addr)
8b7d89d0 PP	371	{
d61fc448	372	static atomic_t next_id;
8b7d89d0 PP	373	struct remap_trace trace = kmalloc(sizeof(trace), GFP_KERNEL);
	374	struct mm_io_header_map event = {
	375	.header = {
	376	.type = MMIO_MAGIC \|
	377	(MMIO_PROBE << MMIO_OPCODE_SHIFT),
	378	.sec = 0,
	379	.nsec = 0,
	380	.pid = 0,
	381	.data_len = sizeof(struct mm_io_map)
	382	},
	383	.map = {
	384	.phys = offset,
	385	.addr = (unsigned long)addr,
	386	.len = size,
	387	.pc = 0
	388	}
	389	};
	390	record_timestamp(&event.header);
	391
d61fc448 PP	392	if (!trace) {
	393	pr_err(NAME "kmalloc failed in ioremap\n");
	394	return;
	395	}
	396
8b7d89d0 PP	397	*trace = (struct remap_trace) {
	398	.probe = {
	399	.addr = (unsigned long)addr,
	400	.len = size,
	401	.pre_handler = pre,
	402	.post_handler = post,
d61fc448 PP	403	.user_data = trace
	404	},
	405	.phys = offset,
	406	.id = atomic_inc_return(&next_id)
8b7d89d0 PP	407	};
8b7d89d0 PP	408
d61fc448 PP	409	spin_lock_irq(&trace_lock);
	410	if (!is_enabled())
	411	goto not_enabled;
	412
8b7d89d0	413	relay_write(chan, &event, sizeof(event));
8b7d89d0	414	list_add_tail(&trace->list, &trace_list);
8b7d89d0 PP	415	if (!nommiotrace)
8b7d89d0 PP	416	register_kmmio_probe(&trace->probe);
d61fc448 PP	417
	418	not_enabled:
	419	spin_unlock_irq(&trace_lock);
8b7d89d0 PP	420	}
8b7d89d0 PP	421
d61fc448 PP	422	void
d61fc448 PP	423	mmiotrace_ioremap(unsigned long offset, unsigned long size, void __iomem *addr)
8b7d89d0	424	{
d61fc448	425	if (!is_enabled()) /* recheck and proper locking in _core() /
8b7d89d0 PP	426	return;
8b7d89d0 PP	427
d61fc448 PP	428	pr_debug(NAME "ioremap_*(0x%lx, 0x%lx) = %p\n", offset, size, addr);
d61fc448 PP	429	if ((filter_offset) && (offset != filter_offset))
8b7d89d0	430	return;
d61fc448	431	ioremap_trace_core(offset, size, addr);
8b7d89d0	432	}
8b7d89d0	433
d61fc448	434	static void iounmap_trace_core(volatile void __iomem *addr)
8b7d89d0 PP	435	{
	436	struct mm_io_header_map event = {
	437	.header = {
	438	.type = MMIO_MAGIC \|
	439	(MMIO_UNPROBE << MMIO_OPCODE_SHIFT),
	440	.sec = 0,
	441	.nsec = 0,
	442	.pid = 0,
	443	.data_len = sizeof(struct mm_io_map)
	444	},
	445	.map = {
	446	.phys = 0,
	447	.addr = (unsigned long)addr,
	448	.len = 0,
	449	.pc = 0
	450	}
	451	};
	452	struct remap_trace *trace;
	453	struct remap_trace *tmp;
d61fc448 PP	454	struct remap_trace *found_trace = NULL;
	455
	456	pr_debug(NAME "Unmapping %p.\n", addr);
8b7d89d0 PP	457	record_timestamp(&event.header);
8b7d89d0 PP	458
d61fc448 PP	459	spin_lock_irq(&trace_lock);
	460	if (!is_enabled())
	461	goto not_enabled;
	462
8b7d89d0 PP	463	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
	464	if ((unsigned long)addr == trace->probe.addr) {
	465	if (!nommiotrace)
	466	unregister_kmmio_probe(&trace->probe);
	467	list_del(&trace->list);
d61fc448	468	found_trace = trace;
8b7d89d0 PP	469	break;
	470	}
	471	}
8b7d89d0	472	relay_write(chan, &event, sizeof(event));
d61fc448 PP	473
	474	not_enabled:
	475	spin_unlock_irq(&trace_lock);
	476	if (found_trace) {
	477	synchronize_rcu(); /* unregister_kmmio_probe() requirement */
	478	kfree(found_trace);
	479	}
	480	}
	481
	482	void mmiotrace_iounmap(volatile void __iomem *addr)
	483	{
	484	might_sleep();
	485	if (is_enabled()) /* recheck and proper locking in _core() /
	486	iounmap_trace_core(addr);
8b7d89d0	487	}
8b7d89d0 PP	488
	489	static void clear_trace_list(void)
	490	{
	491	struct remap_trace *trace;
	492	struct remap_trace *tmp;
	493
d61fc448 PP	494	/*
	495	* No locking required, because the caller ensures we are in a
	496	* critical section via mutex, and is_enabled() is false,
	497	* i.e. nothing can traverse or modify this list.
	498	* Caller also ensures is_enabled() cannot change.
	499	*/
	500	list_for_each_entry(trace, &trace_list, list) {
	501	pr_notice(NAME "purging non-iounmapped "
8b7d89d0 PP	502	"trace @0x%08lx, size 0x%lx.\n",
	503	trace->probe.addr, trace->probe.len);
	504	if (!nommiotrace)
	505	unregister_kmmio_probe(&trace->probe);
d61fc448 PP	506	}
	507	synchronize_rcu(); /* unregister_kmmio_probe() requirement */
	508
	509	list_for_each_entry_safe(trace, tmp, &trace_list, list) {
8b7d89d0 PP	510	list_del(&trace->list);
8b7d89d0 PP	511	kfree(trace);
d61fc448 PP	512	}
	513	}
	514
	515	static ssize_t read_enabled_file_bool(struct file *file,
	516	char __user user_buf, size_t count, loff_t ppos)
	517	{
	518	char buf[3];
	519
	520	if (is_enabled())
	521	buf[0] = '1';
	522	else
	523	buf[0] = '0';
	524	buf[1] = '\n';
	525	buf[2] = '\0';
	526	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
	527	}
	528
	529	static void enable_mmiotrace(void);
	530	static void disable_mmiotrace(void);
	531
	532	static ssize_t write_enabled_file_bool(struct file *file,
	533	const char __user user_buf, size_t count, loff_t ppos)
	534	{
	535	char buf[32];
	536	int buf_size = min(count, (sizeof(buf)-1));
	537
	538	if (copy_from_user(buf, user_buf, buf_size))
	539	return -EFAULT;
	540
	541	switch (buf[0]) {
	542	case 'y':
	543	case 'Y':
	544	case '1':
	545	enable_mmiotrace();
	546	break;
	547	case 'n':
	548	case 'N':
	549	case '0':
	550	disable_mmiotrace();
8b7d89d0 PP	551	break;
8b7d89d0 PP	552	}
d61fc448 PP	553
	554	return count;
	555	}
	556
	557	/* this ripped from kernel/kprobes.c */
	558	static struct file_operations fops_enabled = {
	559	.owner = THIS_MODULE,
	560	.read = read_enabled_file_bool,
	561	.write = write_enabled_file_bool
	562	};
	563
	564	static struct file_operations fops_marker = {
	565	.owner = THIS_MODULE,
	566	.write = write_marker
	567	};
	568
	569	static void enable_mmiotrace(void)
	570	{
	571	mutex_lock(&mmiotrace_mutex);
	572	if (is_enabled())
	573	goto out;
	574
	575	chan = relay_open("cpu", dir, subbuf_size, n_subbufs,
	576	&relay_callbacks, NULL);
	577	if (!chan) {
	578	pr_err(NAME "relay app channel creation failed.\n");
	579	goto out;
	580	}
	581
	582	reference_kmmio();
	583
	584	marker_file = debugfs_create_file("marker", 0660, dir, NULL,
	585	&fops_marker);
	586	if (!marker_file)
	587	pr_err(NAME "marker file creation failed.\n");
	588
	589	if (nommiotrace)
	590	pr_info(NAME "MMIO tracing disabled.\n");
	591	if (ISA_trace)
	592	pr_warning(NAME "Warning! low ISA range will be traced.\n");
	593	spin_lock_irq(&trace_lock);
	594	atomic_inc(&mmiotrace_enabled);
	595	spin_unlock_irq(&trace_lock);
	596	pr_info(NAME "enabled.\n");
	597	out:
	598	mutex_unlock(&mmiotrace_mutex);
	599	}
	600
	601	static void disable_mmiotrace(void)
	602	{
	603	mutex_lock(&mmiotrace_mutex);
	604	if (!is_enabled())
	605	goto out;
	606
	607	spin_lock_irq(&trace_lock);
	608	atomic_dec(&mmiotrace_enabled);
	609	BUG_ON(is_enabled());
	610	spin_unlock_irq(&trace_lock);
	611
	612	clear_trace_list(); /* guarantees: no more kmmio callbacks */
	613	unreference_kmmio();
	614	if (marker_file) {
	615	debugfs_remove(marker_file);
	616	marker_file = NULL;
617	}
618	if (chan) {
619	relay_close(chan);
620	chan = NULL;
621	}
622
623	pr_info(NAME "disabled.\n");
624	out:
625	mutex_unlock(&mmiotrace_mutex);
8b7d89d0 PP	626	}
	627
	628	static int __init init(void)
	629	{
d61fc448	630	pr_debug(NAME "load...\n");
8b7d89d0 PP	631	if (n_subbufs < 2)
	632	return -EINVAL;
	633
	634	dir = debugfs_create_dir(APP_DIR, NULL);
	635	if (!dir) {
d61fc448	636	pr_err(NAME "Couldn't create relay app directory.\n");
8b7d89d0 PP	637	return -ENOMEM;
	638	}
	639
d61fc448 PP	640	enabled_file = debugfs_create_file("enabled", 0600, dir, NULL,
	641	&fops_enabled);
	642	if (!enabled_file) {
	643	pr_err(NAME "Couldn't create enabled file.\n");
8b7d89d0	644	debugfs_remove(dir);
8b7d89d0 PP	645	return -ENOMEM;
	646	}
	647
d61fc448 PP	648	if (enable_now)
d61fc448 PP	649	enable_mmiotrace();
8b7d89d0	650
8b7d89d0 PP	651	return 0;
	652	}
	653
	654	static void __exit cleanup(void)
	655	{
d61fc448 PP	656	pr_debug(NAME "unload...\n");
	657	if (enabled_file)
	658	debugfs_remove(enabled_file);
	659	disable_mmiotrace();
8b7d89d0 PP	660	if (dir)
	661	debugfs_remove(dir);
	662	}
	663
	664	module_init(init);
	665	module_exit(cleanup);
	666	MODULE_LICENSE("GPL");