[deliverable/linux.git] / kernel / trace / trace_irqsoff.c

/*
 * trace irqs off criticall timings
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
 *
 * From code in the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
 *  Copyright (C) 2004 William Lee Irwin III
 */
#include <linux/kallsyms.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/ftrace.h>
#include <linux/fs.h>

#include "trace.h"

static struct trace_array		*irqsoff_trace __read_mostly;
static int				tracer_enabled __read_mostly;

static DEFINE_PER_CPU(int, tracing_cpu);

static DEFINE_SPINLOCK(max_trace_lock);

enum {
	TRACER_IRQS_OFF		= (1 << 1),
	TRACER_PREEMPT_OFF	= (1 << 2),
};

static int trace_type __read_mostly;

#ifdef CONFIG_PREEMPT_TRACER
static inline int
preempt_trace(void)
{
	return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count());
}
#else
# define preempt_trace() (0)
#endif

#ifdef CONFIG_IRQSOFF_TRACER
static inline int
irq_trace(void)
{
	return ((trace_type & TRACER_IRQS_OFF) &&
		irqs_disabled());
}
#else
# define irq_trace() (0)
#endif

/*
 * Sequence count - we record it when starting a measurement and
 * skip the latency if the sequence has changed - some other section
 * did a maximum and could disturb our measurement with serial console
 * printouts, etc. Truly coinciding maximum latencies should be rare
 * and what happens together happens separately as well, so this doesnt
 * decrease the validity of the maximum found:
 */
static __cacheline_aligned_in_smp	unsigned long max_sequence;

#ifdef CONFIG_FTRACE
/*
 * irqsoff uses its own tracer function to keep the overhead down:
 */
static void
irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
{
	struct trace_array *tr = irqsoff_trace;
	struct trace_array_cpu *data;
	unsigned long flags;
	long disabled;
	int cpu;

	/*
	 * Does not matter if we preempt. We test the flags
	 * afterward, to see if irqs are disabled or not.
	 * If we preempt and get a false positive, the flags
	 * test will fail.
	 */
	cpu = raw_smp_processor_id();
	if (likely(!per_cpu(tracing_cpu, cpu)))
		return;

	local_save_flags(flags);
	/* slight chance to get a false positive on tracing_cpu */
	if (!irqs_disabled_flags(flags))
		return;

	data = tr->data[cpu];
	disabled = atomic_inc_return(&data->disabled);

	if (likely(disabled == 1))
		trace_function(tr, data, ip, parent_ip, flags);

	atomic_dec(&data->disabled);
}

static struct ftrace_ops trace_ops __read_mostly =
{
	.func = irqsoff_tracer_call,
};
#endif /* CONFIG_FTRACE */

/*
 * Should this new latency be reported/recorded?
 */
static int report_latency(cycle_t delta)
{
	if (tracing_thresh) {
		if (delta < tracing_thresh)
			return 0;
	} else {
		if (delta <= tracing_max_latency)
			return 0;
	}
	return 1;
}

static void
check_critical_timing(struct trace_array *tr,
		      struct trace_array_cpu *data,
		      unsigned long parent_ip,
		      int cpu)
{
	unsigned long latency, t0, t1;
	cycle_t T0, T1, delta;
	unsigned long flags;

	/*
	 * usecs conversion is slow so we try to delay the conversion
	 * as long as possible:
	 */
	T0 = data->preempt_timestamp;
	T1 = ftrace_now(cpu);
	delta = T1-T0;

	local_save_flags(flags);

	if (!report_latency(delta))
		goto out;

	spin_lock_irqsave(&max_trace_lock, flags);

	/* check if we are still the max latency */
	if (!report_latency(delta))
		goto out_unlock;

	trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);

	latency = nsecs_to_usecs(delta);

	if (data->critical_sequence != max_sequence)
		goto out_unlock;

	tracing_max_latency = delta;
	t0 = nsecs_to_usecs(T0);
	t1 = nsecs_to_usecs(T1);

	data->critical_end = parent_ip;

	update_max_tr_single(tr, current, cpu);

	if (tracing_thresh) {
		printk(KERN_INFO "(%16s-%-5d|#%d):"
			" %lu us critical section violates %lu us threshold.\n",
				current->comm, current->pid,
				raw_smp_processor_id(),
				latency, nsecs_to_usecs(tracing_thresh));
	} else {
		printk(KERN_INFO "(%16s-%-5d|#%d):"
		       " new %lu us maximum-latency critical section.\n",
				current->comm, current->pid,
				raw_smp_processor_id(),
				latency);
	}

	max_sequence++;

out_unlock:
	spin_unlock_irqrestore(&max_trace_lock, flags);

out:
	data->critical_sequence = max_sequence;
	data->preempt_timestamp = ftrace_now(cpu);
	tracing_reset(data);
	trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);
}

static inline void
start_critical_timing(unsigned long ip, unsigned long parent_ip)
{
	int cpu;
	struct trace_array *tr = irqsoff_trace;
	struct trace_array_cpu *data;
	unsigned long flags;

	if (likely(!tracer_enabled))
		return;

	if (__get_cpu_var(tracing_cpu))
		return;

	cpu = raw_smp_processor_id();
	data = tr->data[cpu];

	if (unlikely(!data) || unlikely(!head_page(data)) ||
	    atomic_read(&data->disabled))
		return;

	atomic_inc(&data->disabled);

	data->critical_sequence = max_sequence;
	data->preempt_timestamp = ftrace_now(cpu);
	data->critical_start = parent_ip ? : ip;
	tracing_reset(data);

	local_save_flags(flags);

	trace_function(tr, data, ip, parent_ip, flags);

	__get_cpu_var(tracing_cpu) = 1;

	atomic_dec(&data->disabled);
}

static inline void
stop_critical_timing(unsigned long ip, unsigned long parent_ip)
{
	int cpu;
	struct trace_array *tr = irqsoff_trace;
	struct trace_array_cpu *data;
	unsigned long flags;

	/* Always clear the tracing cpu on stopping the trace */
	if (unlikely(__get_cpu_var(tracing_cpu)))
		__get_cpu_var(tracing_cpu) = 0;
	else
		return;

	if (!tracer_enabled)
		return;

	cpu = raw_smp_processor_id();
	data = tr->data[cpu];

	if (unlikely(!data) || unlikely(!head_page(data)) ||
	    !data->critical_start || atomic_read(&data->disabled))
		return;

	atomic_inc(&data->disabled);
	local_save_flags(flags);
	trace_function(tr, data, ip, parent_ip, flags);
	check_critical_timing(tr, data, parent_ip ? : ip, cpu);
	data->critical_start = 0;
	atomic_dec(&data->disabled);
}

/* start and stop critical timings used to for stoppage (in idle) */
void start_critical_timings(void)
{
	if (preempt_trace() || irq_trace())
		start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}

void stop_critical_timings(void)
{
	if (preempt_trace() || irq_trace())
		stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}

#ifdef CONFIG_IRQSOFF_TRACER
#ifdef CONFIG_PROVE_LOCKING
void time_hardirqs_on(unsigned long a0, unsigned long a1)
{
	if (!preempt_trace() && irq_trace())
		stop_critical_timing(a0, a1);
}

void time_hardirqs_off(unsigned long a0, unsigned long a1)
{
	if (!preempt_trace() && irq_trace())
		start_critical_timing(a0, a1);
}

#else /* !CONFIG_PROVE_LOCKING */

/*
 * Stubs:
 */

void early_boot_irqs_off(void)
{
}

void early_boot_irqs_on(void)
{
}

void trace_softirqs_on(unsigned long ip)
{
}

void trace_softirqs_off(unsigned long ip)
{
}

inline void print_irqtrace_events(struct task_struct *curr)
{
}

/*
 * We are only interested in hardirq on/off events:
 */
void trace_hardirqs_on(void)
{
	if (!preempt_trace() && irq_trace())
		stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}
EXPORT_SYMBOL(trace_hardirqs_on);

void trace_hardirqs_off(void)
{
	if (!preempt_trace() && irq_trace())
		start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
}
EXPORT_SYMBOL(trace_hardirqs_off);

void trace_hardirqs_on_caller(unsigned long caller_addr)
{
	if (!preempt_trace() && irq_trace())
		stop_critical_timing(CALLER_ADDR0, caller_addr);
}
EXPORT_SYMBOL(trace_hardirqs_on_caller);

void trace_hardirqs_off_caller(unsigned long caller_addr)
{
	if (!preempt_trace() && irq_trace())
		start_critical_timing(CALLER_ADDR0, caller_addr);
}
EXPORT_SYMBOL(trace_hardirqs_off_caller);

#endif /* CONFIG_PROVE_LOCKING */
#endif /*  CONFIG_IRQSOFF_TRACER */

#ifdef CONFIG_PREEMPT_TRACER
void trace_preempt_on(unsigned long a0, unsigned long a1)
{
	stop_critical_timing(a0, a1);
}

void trace_preempt_off(unsigned long a0, unsigned long a1)
{
	start_critical_timing(a0, a1);
}
#endif /* CONFIG_PREEMPT_TRACER */

static void start_irqsoff_tracer(struct trace_array *tr)
{
	register_ftrace_function(&trace_ops);
	tracer_enabled = 1;
}

static void stop_irqsoff_tracer(struct trace_array *tr)
{
	tracer_enabled = 0;
	unregister_ftrace_function(&trace_ops);
}

static void __irqsoff_tracer_init(struct trace_array *tr)
{
	irqsoff_trace = tr;
	/* make sure that the tracer is visibel */
	smp_wmb();

	if (tr->ctrl)
		start_irqsoff_tracer(tr);
}

static void irqsoff_tracer_reset(struct trace_array *tr)
{
	if (tr->ctrl)
		stop_irqsoff_tracer(tr);
}

static void irqsoff_tracer_ctrl_update(struct trace_array *tr)
{
	if (tr->ctrl)
		start_irqsoff_tracer(tr);
	else
		stop_irqsoff_tracer(tr);
}

static void irqsoff_tracer_open(struct trace_iterator *iter)
{
	/* stop the trace while dumping */
	if (iter->tr->ctrl)
		stop_irqsoff_tracer(iter->tr);
}

static void irqsoff_tracer_close(struct trace_iterator *iter)
{
	if (iter->tr->ctrl)
		start_irqsoff_tracer(iter->tr);
}

#ifdef CONFIG_IRQSOFF_TRACER
static void irqsoff_tracer_init(struct trace_array *tr)
{
	trace_type = TRACER_IRQS_OFF;

	__irqsoff_tracer_init(tr);
}
static struct tracer irqsoff_tracer __read_mostly =
{
	.name		= "irqsoff",
	.init		= irqsoff_tracer_init,
	.reset		= irqsoff_tracer_reset,
	.open		= irqsoff_tracer_open,
	.close		= irqsoff_tracer_close,
	.ctrl_update	= irqsoff_tracer_ctrl_update,
	.print_max	= 1,
#ifdef CONFIG_FTRACE_SELFTEST
	.selftest    = trace_selftest_startup_irqsoff,
#endif
};
# define register_irqsoff(trace) register_tracer(&trace)
#else
# define register_irqsoff(trace) do { } while (0)
#endif

#ifdef CONFIG_PREEMPT_TRACER
static void preemptoff_tracer_init(struct trace_array *tr)
{
	trace_type = TRACER_PREEMPT_OFF;

	__irqsoff_tracer_init(tr);
}

static struct tracer preemptoff_tracer __read_mostly =
{
	.name		= "preemptoff",
	.init		= preemptoff_tracer_init,
	.reset		= irqsoff_tracer_reset,
	.open		= irqsoff_tracer_open,
	.close		= irqsoff_tracer_close,
	.ctrl_update	= irqsoff_tracer_ctrl_update,
	.print_max	= 1,
#ifdef CONFIG_FTRACE_SELFTEST
	.selftest    = trace_selftest_startup_preemptoff,
#endif
};
# define register_preemptoff(trace) register_tracer(&trace)
#else
# define register_preemptoff(trace) do { } while (0)
#endif

#if defined(CONFIG_IRQSOFF_TRACER) && \
	defined(CONFIG_PREEMPT_TRACER)

static void preemptirqsoff_tracer_init(struct trace_array *tr)
{
	trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;

	__irqsoff_tracer_init(tr);
}

static struct tracer preemptirqsoff_tracer __read_mostly =
{
	.name		= "preemptirqsoff",
	.init		= preemptirqsoff_tracer_init,
	.reset		= irqsoff_tracer_reset,
	.open		= irqsoff_tracer_open,
	.close		= irqsoff_tracer_close,
	.ctrl_update	= irqsoff_tracer_ctrl_update,
	.print_max	= 1,
#ifdef CONFIG_FTRACE_SELFTEST
	.selftest    = trace_selftest_startup_preemptirqsoff,
#endif
};

# define register_preemptirqsoff(trace) register_tracer(&trace)
#else
# define register_preemptirqsoff(trace) do { } while (0)
#endif

__init static int init_irqsoff_tracer(void)
{
	register_irqsoff(irqsoff_tracer);
	register_preemptoff(preemptoff_tracer);
	register_preemptirqsoff(preemptirqsoff_tracer);

	return 0;
}
device_initcall(init_irqsoff_tracer);
Commit	Line	Data
	1	/*
	2	* trace irqs off criticall timings
	3	*
	4	* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
	5	* Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
	6	*
	7	* From code in the latency_tracer, that is:
	8	*
	9	* Copyright (C) 2004-2006 Ingo Molnar
	10	* Copyright (C) 2004 William Lee Irwin III
	11	*/
	12	#include <linux/kallsyms.h>
	13	#include <linux/debugfs.h>
	14	#include <linux/uaccess.h>
	15	#include <linux/module.h>
	16	#include <linux/ftrace.h>
	17	#include <linux/fs.h>
	18
	19	#include "trace.h"
	20
	21	static struct trace_array *irqsoff_trace __read_mostly;
	22	static int tracer_enabled __read_mostly;
	23
	24	static DEFINE_PER_CPU(int, tracing_cpu);
	25
	26	static DEFINE_SPINLOCK(max_trace_lock);
	27
	28	enum {
	29	TRACER_IRQS_OFF = (1 << 1),
	30	TRACER_PREEMPT_OFF = (1 << 2),
	31	};
	32
	33	static int trace_type __read_mostly;
	34
	35	#ifdef CONFIG_PREEMPT_TRACER
	36	static inline int
	37	preempt_trace(void)
	38	{
	39	return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count());
	40	}
	41	#else
	42	# define preempt_trace() (0)
	43	#endif
	44
	45	#ifdef CONFIG_IRQSOFF_TRACER
	46	static inline int
	47	irq_trace(void)
	48	{
	49	return ((trace_type & TRACER_IRQS_OFF) &&
	50	irqs_disabled());
	51	}
	52	#else
	53	# define irq_trace() (0)
	54	#endif
	55
	56	/*
	57	* Sequence count - we record it when starting a measurement and
	58	* skip the latency if the sequence has changed - some other section
	59	* did a maximum and could disturb our measurement with serial console
	60	* printouts, etc. Truly coinciding maximum latencies should be rare
	61	* and what happens together happens separately as well, so this doesnt
	62	* decrease the validity of the maximum found:
	63	*/
	64	static __cacheline_aligned_in_smp unsigned long max_sequence;
	65
	66	#ifdef CONFIG_FTRACE
	67	/*
	68	* irqsoff uses its own tracer function to keep the overhead down:
	69	*/
	70	static void
	71	irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
	72	{
	73	struct trace_array *tr = irqsoff_trace;
	74	struct trace_array_cpu *data;
	75	unsigned long flags;
	76	long disabled;
	77	int cpu;
	78
	79	/*
	80	* Does not matter if we preempt. We test the flags
	81	* afterward, to see if irqs are disabled or not.
	82	* If we preempt and get a false positive, the flags
	83	* test will fail.
	84	*/
	85	cpu = raw_smp_processor_id();
	86	if (likely(!per_cpu(tracing_cpu, cpu)))
	87	return;
	88
	89	local_save_flags(flags);
	90	/* slight chance to get a false positive on tracing_cpu */
	91	if (!irqs_disabled_flags(flags))
	92	return;
	93
	94	data = tr->data[cpu];
	95	disabled = atomic_inc_return(&data->disabled);
	96
	97	if (likely(disabled == 1))
	98	trace_function(tr, data, ip, parent_ip, flags);
	99
	100	atomic_dec(&data->disabled);
	101	}
	102
	103	static struct ftrace_ops trace_ops __read_mostly =
	104	{
	105	.func = irqsoff_tracer_call,
	106	};
	107	#endif /* CONFIG_FTRACE */
	108
	109	/*
	110	* Should this new latency be reported/recorded?
	111	*/
	112	static int report_latency(cycle_t delta)
	113	{
	114	if (tracing_thresh) {
	115	if (delta < tracing_thresh)
	116	return 0;
	117	} else {
	118	if (delta <= tracing_max_latency)
	119	return 0;
	120	}
	121	return 1;
	122	}
	123
	124	static void
	125	check_critical_timing(struct trace_array *tr,
	126	struct trace_array_cpu *data,
	127	unsigned long parent_ip,
	128	int cpu)
	129	{
	130	unsigned long latency, t0, t1;
	131	cycle_t T0, T1, delta;
	132	unsigned long flags;
	133
	134	/*
	135	* usecs conversion is slow so we try to delay the conversion
	136	* as long as possible:
	137	*/
	138	T0 = data->preempt_timestamp;
	139	T1 = ftrace_now(cpu);
	140	delta = T1-T0;
	141
	142	local_save_flags(flags);
	143
	144	if (!report_latency(delta))
	145	goto out;
	146
	147	spin_lock_irqsave(&max_trace_lock, flags);
	148
	149	/* check if we are still the max latency */
	150	if (!report_latency(delta))
	151	goto out_unlock;
	152
	153	trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);
	154
	155	latency = nsecs_to_usecs(delta);
	156
	157	if (data->critical_sequence != max_sequence)
	158	goto out_unlock;
	159
	160	tracing_max_latency = delta;
	161	t0 = nsecs_to_usecs(T0);
	162	t1 = nsecs_to_usecs(T1);
	163
	164	data->critical_end = parent_ip;
	165
	166	update_max_tr_single(tr, current, cpu);
	167
	168	if (tracing_thresh) {
	169	printk(KERN_INFO "(%16s-%-5d\|#%d):"
	170	" %lu us critical section violates %lu us threshold.\n",
	171	current->comm, current->pid,
	172	raw_smp_processor_id(),
	173	latency, nsecs_to_usecs(tracing_thresh));
	174	} else {
	175	printk(KERN_INFO "(%16s-%-5d\|#%d):"
	176	" new %lu us maximum-latency critical section.\n",
	177	current->comm, current->pid,
	178	raw_smp_processor_id(),
	179	latency);
	180	}
	181
	182	max_sequence++;
	183
	184	out_unlock:
	185	spin_unlock_irqrestore(&max_trace_lock, flags);
	186
	187	out:
	188	data->critical_sequence = max_sequence;
	189	data->preempt_timestamp = ftrace_now(cpu);
	190	tracing_reset(data);
	191	trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);
	192	}
	193
	194	static inline void
	195	start_critical_timing(unsigned long ip, unsigned long parent_ip)
	196	{
	197	int cpu;
	198	struct trace_array *tr = irqsoff_trace;
	199	struct trace_array_cpu *data;
	200	unsigned long flags;
	201
	202	if (likely(!tracer_enabled))
	203	return;
	204
	205	if (__get_cpu_var(tracing_cpu))
	206	return;
	207
	208	cpu = raw_smp_processor_id();
	209	data = tr->data[cpu];
	210
	211	if (unlikely(!data) \|\| unlikely(!head_page(data)) \|\|
	212	atomic_read(&data->disabled))
	213	return;
	214
	215	atomic_inc(&data->disabled);
	216
	217	data->critical_sequence = max_sequence;
	218	data->preempt_timestamp = ftrace_now(cpu);
	219	data->critical_start = parent_ip ? : ip;
	220	tracing_reset(data);
	221
	222	local_save_flags(flags);
	223
	224	trace_function(tr, data, ip, parent_ip, flags);
	225
	226	__get_cpu_var(tracing_cpu) = 1;
	227
	228	atomic_dec(&data->disabled);
	229	}
	230
	231	static inline void
	232	stop_critical_timing(unsigned long ip, unsigned long parent_ip)
	233	{
	234	int cpu;
	235	struct trace_array *tr = irqsoff_trace;
	236	struct trace_array_cpu *data;
	237	unsigned long flags;
	238
	239	/* Always clear the tracing cpu on stopping the trace */
	240	if (unlikely(__get_cpu_var(tracing_cpu)))
	241	__get_cpu_var(tracing_cpu) = 0;
	242	else
	243	return;
	244
	245	if (!tracer_enabled)
	246	return;
	247
	248	cpu = raw_smp_processor_id();
	249	data = tr->data[cpu];
	250
	251	if (unlikely(!data) \|\| unlikely(!head_page(data)) \|\|
	252	!data->critical_start \|\| atomic_read(&data->disabled))
	253	return;
	254
	255	atomic_inc(&data->disabled);
	256	local_save_flags(flags);
	257	trace_function(tr, data, ip, parent_ip, flags);
	258	check_critical_timing(tr, data, parent_ip ? : ip, cpu);
	259	data->critical_start = 0;
	260	atomic_dec(&data->disabled);
	261	}
	262
	263	/* start and stop critical timings used to for stoppage (in idle) */
	264	void start_critical_timings(void)
	265	{
	266	if (preempt_trace() \|\| irq_trace())
	267	start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
	268	}
	269
	270	void stop_critical_timings(void)
	271	{
	272	if (preempt_trace() \|\| irq_trace())
	273	stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
	274	}
	275
	276	#ifdef CONFIG_IRQSOFF_TRACER
	277	#ifdef CONFIG_PROVE_LOCKING
	278	void time_hardirqs_on(unsigned long a0, unsigned long a1)
	279	{
	280	if (!preempt_trace() && irq_trace())
	281	stop_critical_timing(a0, a1);
	282	}
	283
	284	void time_hardirqs_off(unsigned long a0, unsigned long a1)
	285	{
	286	if (!preempt_trace() && irq_trace())
	287	start_critical_timing(a0, a1);
	288	}
	289
	290	#else /* !CONFIG_PROVE_LOCKING */
	291
	292	/*
	293	* Stubs:
	294	*/
	295
	296	void early_boot_irqs_off(void)
	297	{
	298	}
	299
	300	void early_boot_irqs_on(void)
	301	{
	302	}
	303
	304	void trace_softirqs_on(unsigned long ip)
	305	{
	306	}
	307
	308	void trace_softirqs_off(unsigned long ip)
	309	{
	310	}
	311
	312	inline void print_irqtrace_events(struct task_struct *curr)
	313	{
	314	}
	315
	316	/*
	317	* We are only interested in hardirq on/off events:
	318	*/
	319	void trace_hardirqs_on(void)
	320	{
	321	if (!preempt_trace() && irq_trace())
	322	stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
	323	}
	324	EXPORT_SYMBOL(trace_hardirqs_on);
	325
	326	void trace_hardirqs_off(void)
	327	{
	328	if (!preempt_trace() && irq_trace())
	329	start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
	330	}
	331	EXPORT_SYMBOL(trace_hardirqs_off);
	332
	333	void trace_hardirqs_on_caller(unsigned long caller_addr)
	334	{
	335	if (!preempt_trace() && irq_trace())
	336	stop_critical_timing(CALLER_ADDR0, caller_addr);
	337	}
	338	EXPORT_SYMBOL(trace_hardirqs_on_caller);
	339
	340	void trace_hardirqs_off_caller(unsigned long caller_addr)
	341	{
	342	if (!preempt_trace() && irq_trace())
	343	start_critical_timing(CALLER_ADDR0, caller_addr);
	344	}
	345	EXPORT_SYMBOL(trace_hardirqs_off_caller);
	346
	347	#endif /* CONFIG_PROVE_LOCKING */
	348	#endif /* CONFIG_IRQSOFF_TRACER */
	349
	350	#ifdef CONFIG_PREEMPT_TRACER
	351	void trace_preempt_on(unsigned long a0, unsigned long a1)
	352	{
	353	stop_critical_timing(a0, a1);
	354	}
	355
	356	void trace_preempt_off(unsigned long a0, unsigned long a1)
	357	{
	358	start_critical_timing(a0, a1);
	359	}
	360	#endif /* CONFIG_PREEMPT_TRACER */
	361
	362	static void start_irqsoff_tracer(struct trace_array *tr)
	363	{
	364	register_ftrace_function(&trace_ops);
	365	tracer_enabled = 1;
	366	}
	367
	368	static void stop_irqsoff_tracer(struct trace_array *tr)
	369	{
	370	tracer_enabled = 0;
	371	unregister_ftrace_function(&trace_ops);
	372	}
	373
	374	static void __irqsoff_tracer_init(struct trace_array *tr)
	375	{
	376	irqsoff_trace = tr;
	377	/* make sure that the tracer is visibel */
	378	smp_wmb();
	379
	380	if (tr->ctrl)
	381	start_irqsoff_tracer(tr);
	382	}
	383
	384	static void irqsoff_tracer_reset(struct trace_array *tr)
	385	{
	386	if (tr->ctrl)
	387	stop_irqsoff_tracer(tr);
	388	}
	389
	390	static void irqsoff_tracer_ctrl_update(struct trace_array *tr)
	391	{
	392	if (tr->ctrl)
	393	start_irqsoff_tracer(tr);
	394	else
	395	stop_irqsoff_tracer(tr);
	396	}
	397
	398	static void irqsoff_tracer_open(struct trace_iterator *iter)
	399	{
	400	/* stop the trace while dumping */
	401	if (iter->tr->ctrl)
	402	stop_irqsoff_tracer(iter->tr);
	403	}
	404
	405	static void irqsoff_tracer_close(struct trace_iterator *iter)
	406	{
	407	if (iter->tr->ctrl)
	408	start_irqsoff_tracer(iter->tr);
	409	}
	410
	411	#ifdef CONFIG_IRQSOFF_TRACER
	412	static void irqsoff_tracer_init(struct trace_array *tr)
	413	{
	414	trace_type = TRACER_IRQS_OFF;
	415
	416	__irqsoff_tracer_init(tr);
	417	}
	418	static struct tracer irqsoff_tracer __read_mostly =
	419	{
	420	.name = "irqsoff",
	421	.init = irqsoff_tracer_init,
	422	.reset = irqsoff_tracer_reset,
	423	.open = irqsoff_tracer_open,
	424	.close = irqsoff_tracer_close,
	425	.ctrl_update = irqsoff_tracer_ctrl_update,
	426	.print_max = 1,
	427	#ifdef CONFIG_FTRACE_SELFTEST
	428	.selftest = trace_selftest_startup_irqsoff,
	429	#endif
	430	};
	431	# define register_irqsoff(trace) register_tracer(&trace)
	432	#else
	433	# define register_irqsoff(trace) do { } while (0)
	434	#endif
	435
	436	#ifdef CONFIG_PREEMPT_TRACER
	437	static void preemptoff_tracer_init(struct trace_array *tr)
	438	{
	439	trace_type = TRACER_PREEMPT_OFF;
	440
	441	__irqsoff_tracer_init(tr);
	442	}
	443
	444	static struct tracer preemptoff_tracer __read_mostly =
	445	{
	446	.name = "preemptoff",
	447	.init = preemptoff_tracer_init,
	448	.reset = irqsoff_tracer_reset,
	449	.open = irqsoff_tracer_open,
	450	.close = irqsoff_tracer_close,
	451	.ctrl_update = irqsoff_tracer_ctrl_update,
	452	.print_max = 1,
	453	#ifdef CONFIG_FTRACE_SELFTEST
	454	.selftest = trace_selftest_startup_preemptoff,
	455	#endif
	456	};
	457	# define register_preemptoff(trace) register_tracer(&trace)
	458	#else
	459	# define register_preemptoff(trace) do { } while (0)
	460	#endif
	461
	462	#if defined(CONFIG_IRQSOFF_TRACER) && \
	463	defined(CONFIG_PREEMPT_TRACER)
	464
	465	static void preemptirqsoff_tracer_init(struct trace_array *tr)
	466	{
	467	trace_type = TRACER_IRQS_OFF \| TRACER_PREEMPT_OFF;
	468
	469	__irqsoff_tracer_init(tr);
	470	}
	471
	472	static struct tracer preemptirqsoff_tracer __read_mostly =
	473	{
	474	.name = "preemptirqsoff",
	475	.init = preemptirqsoff_tracer_init,
	476	.reset = irqsoff_tracer_reset,
	477	.open = irqsoff_tracer_open,
	478	.close = irqsoff_tracer_close,
	479	.ctrl_update = irqsoff_tracer_ctrl_update,
	480	.print_max = 1,
	481	#ifdef CONFIG_FTRACE_SELFTEST
	482	.selftest = trace_selftest_startup_preemptirqsoff,
	483	#endif
	484	};
	485
	486	# define register_preemptirqsoff(trace) register_tracer(&trace)
	487	#else
	488	# define register_preemptirqsoff(trace) do { } while (0)
	489	#endif
	490
	491	__init static int init_irqsoff_tracer(void)
	492	{
	493	register_irqsoff(irqsoff_tracer);
	494	register_preemptoff(preemptoff_tracer);
	495	register_preemptirqsoff(preemptirqsoff_tracer);
	496
	497	return 0;
	498	}
	499	device_initcall(init_irqsoff_tracer);