[deliverable/linux.git] / include / linux / lockdep.h

/*
 * Runtime locking correctness validator
 *
 *  Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
 *
 * see Documentation/lockdep-design.txt for more details.
 */
#ifndef __LINUX_LOCKDEP_H
#define __LINUX_LOCKDEP_H

struct task_struct;
struct lockdep_map;

#ifdef CONFIG_LOCKDEP

#include <linux/linkage.h>
#include <linux/list.h>
#include <linux/debug_locks.h>
#include <linux/stacktrace.h>

/*
 * Lock-class usage-state bits:
 */
enum lock_usage_bit
{
	LOCK_USED = 0,
	LOCK_USED_IN_HARDIRQ,
	LOCK_USED_IN_SOFTIRQ,
	LOCK_ENABLED_SOFTIRQS,
	LOCK_ENABLED_HARDIRQS,
	LOCK_USED_IN_HARDIRQ_READ,
	LOCK_USED_IN_SOFTIRQ_READ,
	LOCK_ENABLED_SOFTIRQS_READ,
	LOCK_ENABLED_HARDIRQS_READ,
	LOCK_USAGE_STATES
};

/*
 * Usage-state bitmasks:
 */
#define LOCKF_USED			(1 << LOCK_USED)
#define LOCKF_USED_IN_HARDIRQ		(1 << LOCK_USED_IN_HARDIRQ)
#define LOCKF_USED_IN_SOFTIRQ		(1 << LOCK_USED_IN_SOFTIRQ)
#define LOCKF_ENABLED_HARDIRQS		(1 << LOCK_ENABLED_HARDIRQS)
#define LOCKF_ENABLED_SOFTIRQS		(1 << LOCK_ENABLED_SOFTIRQS)

#define LOCKF_ENABLED_IRQS (LOCKF_ENABLED_HARDIRQS | LOCKF_ENABLED_SOFTIRQS)
#define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ | LOCKF_USED_IN_SOFTIRQ)

#define LOCKF_USED_IN_HARDIRQ_READ	(1 << LOCK_USED_IN_HARDIRQ_READ)
#define LOCKF_USED_IN_SOFTIRQ_READ	(1 << LOCK_USED_IN_SOFTIRQ_READ)
#define LOCKF_ENABLED_HARDIRQS_READ	(1 << LOCK_ENABLED_HARDIRQS_READ)
#define LOCKF_ENABLED_SOFTIRQS_READ	(1 << LOCK_ENABLED_SOFTIRQS_READ)

#define LOCKF_ENABLED_IRQS_READ \
		(LOCKF_ENABLED_HARDIRQS_READ | LOCKF_ENABLED_SOFTIRQS_READ)
#define LOCKF_USED_IN_IRQ_READ \
		(LOCKF_USED_IN_HARDIRQ_READ | LOCKF_USED_IN_SOFTIRQ_READ)

#define MAX_LOCKDEP_SUBCLASSES		8UL

/*
 * Lock-classes are keyed via unique addresses, by embedding the
 * lockclass-key into the kernel (or module) .data section. (For
 * static locks we use the lock address itself as the key.)
 */
struct lockdep_subclass_key {
	char __one_byte;
} __attribute__ ((__packed__));

struct lock_class_key {
	struct lockdep_subclass_key	subkeys[MAX_LOCKDEP_SUBCLASSES];
};

#define LOCKSTAT_POINTS		4

/*
 * The lock-class itself:
 */
struct lock_class {
	/*
	 * class-hash:
	 */
	struct list_head		hash_entry;

	/*
	 * global list of all lock-classes:
	 */
	struct list_head		lock_entry;

	struct lockdep_subclass_key	*key;
	unsigned int			subclass;
	unsigned int			dep_gen_id;

	/*
	 * IRQ/softirq usage tracking bits:
	 */
	unsigned long			usage_mask;
	struct stack_trace		usage_traces[LOCK_USAGE_STATES];

	/*
	 * These fields represent a directed graph of lock dependencies,
	 * to every node we attach a list of "forward" and a list of
	 * "backward" graph nodes.
	 */
	struct list_head		locks_after, locks_before;

	/*
	 * Generation counter, when doing certain classes of graph walking,
	 * to ensure that we check one node only once:
	 */
	unsigned int			version;

	/*
	 * Statistics counter:
	 */
	unsigned long			ops;

	const char			*name;
	int				name_version;

#ifdef CONFIG_LOCK_STAT
	unsigned long			contention_point[LOCKSTAT_POINTS];
	unsigned long			contending_point[LOCKSTAT_POINTS];
#endif
};

#ifdef CONFIG_LOCK_STAT
struct lock_time {
	s64				min;
	s64				max;
	s64				total;
	unsigned long			nr;
};

enum bounce_type {
	bounce_acquired_write,
	bounce_acquired_read,
	bounce_contended_write,
	bounce_contended_read,
	nr_bounce_types,

	bounce_acquired = bounce_acquired_write,
	bounce_contended = bounce_contended_write,
};

struct lock_class_stats {
	unsigned long			contention_point[4];
	unsigned long			contending_point[4];
	struct lock_time		read_waittime;
	struct lock_time		write_waittime;
	struct lock_time		read_holdtime;
	struct lock_time		write_holdtime;
	unsigned long			bounces[nr_bounce_types];
};

struct lock_class_stats lock_stats(struct lock_class *class);
void clear_lock_stats(struct lock_class *class);
#endif

/*
 * Map the lock object (the lock instance) to the lock-class object.
 * This is embedded into specific lock instances:
 */
struct lockdep_map {
	struct lock_class_key		*key;
	struct lock_class		*class_cache;
	const char			*name;
#ifdef CONFIG_LOCK_STAT
	int				cpu;
	unsigned long			ip;
#endif
};

/*
 * Every lock has a list of other locks that were taken after it.
 * We only grow the list, never remove from it:
 */
struct lock_list {
	struct list_head		entry;
	struct lock_class		*class;
	struct stack_trace		trace;
	int				distance;
};

/*
 * We record lock dependency chains, so that we can cache them:
 */
struct lock_chain {
	u8				irq_context;
	u8				depth;
	u16				base;
	struct list_head		entry;
	u64				chain_key;
};

#define MAX_LOCKDEP_KEYS_BITS		13
/*
 * Subtract one because we offset hlock->class_idx by 1 in order
 * to make 0 mean no class. This avoids overflowing the class_idx
 * bitfield and hitting the BUG in hlock_class().
 */
#define MAX_LOCKDEP_KEYS		((1UL << MAX_LOCKDEP_KEYS_BITS) - 1)

struct held_lock {
	/*
	 * One-way hash of the dependency chain up to this point. We
	 * hash the hashes step by step as the dependency chain grows.
	 *
	 * We use it for dependency-caching and we skip detection
	 * passes and dependency-updates if there is a cache-hit, so
	 * it is absolutely critical for 100% coverage of the validator
	 * to have a unique key value for every unique dependency path
	 * that can occur in the system, to make a unique hash value
	 * as likely as possible - hence the 64-bit width.
	 *
	 * The task struct holds the current hash value (initialized
	 * with zero), here we store the previous hash value:
	 */
	u64				prev_chain_key;
	unsigned long			acquire_ip;
	struct lockdep_map		*instance;
	struct lockdep_map		*nest_lock;
#ifdef CONFIG_LOCK_STAT
	u64 				waittime_stamp;
	u64				holdtime_stamp;
#endif
	unsigned int			class_idx:MAX_LOCKDEP_KEYS_BITS;
	/*
	 * The lock-stack is unified in that the lock chains of interrupt
	 * contexts nest ontop of process context chains, but we 'separate'
	 * the hashes by starting with 0 if we cross into an interrupt
	 * context, and we also keep do not add cross-context lock
	 * dependencies - the lock usage graph walking covers that area
	 * anyway, and we'd just unnecessarily increase the number of
	 * dependencies otherwise. [Note: hardirq and softirq contexts
	 * are separated from each other too.]
	 *
	 * The following field is used to detect when we cross into an
	 * interrupt context:
	 */
	unsigned int irq_context:2; /* bit 0 - soft, bit 1 - hard */
	unsigned int trylock:1;
	unsigned int read:2;        /* see lock_acquire() comment */
	unsigned int check:2;       /* see lock_acquire() comment */
	unsigned int hardirqs_off:1;
};

/*
 * Initialization, self-test and debugging-output methods:
 */
extern void lockdep_init(void);
extern void lockdep_info(void);
extern void lockdep_reset(void);
extern void lockdep_reset_lock(struct lockdep_map *lock);
extern void lockdep_free_key_range(void *start, unsigned long size);
extern void lockdep_sys_exit(void);

extern void lockdep_off(void);
extern void lockdep_on(void);

/*
 * These methods are used by specific locking variants (spinlocks,
 * rwlocks, mutexes and rwsems) to pass init/acquire/release events
 * to lockdep:
 */

extern void lockdep_init_map(struct lockdep_map *lock, const char *name,
			     struct lock_class_key *key, int subclass);

/*
 * To initialize a lockdep_map statically use this macro.
 * Note that _name must not be NULL.
 */
#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
	{ .name = (_name), .key = (void *)(_key), }

/*
 * Reinitialize a lock key - for cases where there is special locking or
 * special initialization of locks so that the validator gets the scope
 * of dependencies wrong: they are either too broad (they need a class-split)
 * or they are too narrow (they suffer from a false class-split):
 */
#define lockdep_set_class(lock, key) \
		lockdep_init_map(&(lock)->dep_map, #key, key, 0)
#define lockdep_set_class_and_name(lock, key, name) \
		lockdep_init_map(&(lock)->dep_map, name, key, 0)
#define lockdep_set_class_and_subclass(lock, key, sub) \
		lockdep_init_map(&(lock)->dep_map, #key, key, sub)
#define lockdep_set_subclass(lock, sub)	\
		lockdep_init_map(&(lock)->dep_map, #lock, \
				 (lock)->dep_map.key, sub)

/*
 * Acquire a lock.
 *
 * Values for "read":
 *
 *   0: exclusive (write) acquire
 *   1: read-acquire (no recursion allowed)
 *   2: read-acquire with same-instance recursion allowed
 *
 * Values for check:
 *
 *   0: disabled
 *   1: simple checks (freeing, held-at-exit-time, etc.)
 *   2: full validation
 */
extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
			 int trylock, int read, int check,
			 struct lockdep_map *nest_lock, unsigned long ip);

extern void lock_release(struct lockdep_map *lock, int nested,
			 unsigned long ip);

extern void lock_set_class(struct lockdep_map *lock, const char *name,
			   struct lock_class_key *key, unsigned int subclass,
			   unsigned long ip);

static inline void lock_set_subclass(struct lockdep_map *lock,
		unsigned int subclass, unsigned long ip)
{
	lock_set_class(lock, lock->name, lock->key, subclass, ip);
}

# define INIT_LOCKDEP				.lockdep_recursion = 0,

#define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)

#else /* !LOCKDEP */

static inline void lockdep_off(void)
{
}

static inline void lockdep_on(void)
{
}

# define lock_acquire(l, s, t, r, c, n, i)	do { } while (0)
# define lock_release(l, n, i)			do { } while (0)
# define lock_set_class(l, n, k, s, i)		do { } while (0)
# define lock_set_subclass(l, s, i)		do { } while (0)
# define lockdep_init()				do { } while (0)
# define lockdep_info()				do { } while (0)
# define lockdep_init_map(lock, name, key, sub) \
		do { (void)(name); (void)(key); } while (0)
# define lockdep_set_class(lock, key)		do { (void)(key); } while (0)
# define lockdep_set_class_and_name(lock, key, name) \
		do { (void)(key); (void)(name); } while (0)
#define lockdep_set_class_and_subclass(lock, key, sub) \
		do { (void)(key); } while (0)
#define lockdep_set_subclass(lock, sub)		do { } while (0)

# define INIT_LOCKDEP
# define lockdep_reset()		do { debug_locks = 1; } while (0)
# define lockdep_free_key_range(start, size)	do { } while (0)
# define lockdep_sys_exit() 			do { } while (0)
/*
 * The class key takes no space if lockdep is disabled:
 */
struct lock_class_key { };

#define lockdep_depth(tsk)	(0)

#endif /* !LOCKDEP */

#ifdef CONFIG_LOCK_STAT

extern void lock_contended(struct lockdep_map *lock, unsigned long ip);
extern void lock_acquired(struct lockdep_map *lock, unsigned long ip);

#define LOCK_CONTENDED(_lock, try, lock)			\
do {								\
	if (!try(_lock)) {					\
		lock_contended(&(_lock)->dep_map, _RET_IP_);	\
		lock(_lock);					\
	}							\
	lock_acquired(&(_lock)->dep_map, _RET_IP_);			\
} while (0)

#else /* CONFIG_LOCK_STAT */

#define lock_contended(lockdep_map, ip) do {} while (0)
#define lock_acquired(lockdep_map, ip) do {} while (0)

#define LOCK_CONTENDED(_lock, try, lock) \
	lock(_lock)

#endif /* CONFIG_LOCK_STAT */

#ifdef CONFIG_GENERIC_HARDIRQS
extern void early_init_irq_lock_class(void);
#else
static inline void early_init_irq_lock_class(void)
{
}
#endif

#ifdef CONFIG_TRACE_IRQFLAGS
extern void early_boot_irqs_off(void);
extern void early_boot_irqs_on(void);
extern void print_irqtrace_events(struct task_struct *curr);
#else
static inline void early_boot_irqs_off(void)
{
}
static inline void early_boot_irqs_on(void)
{
}
static inline void print_irqtrace_events(struct task_struct *curr)
{
}
#endif

/*
 * For trivial one-depth nesting of a lock-class, the following
 * global define can be used. (Subsystems with multiple levels
 * of nesting should define their own lock-nesting subclasses.)
 */
#define SINGLE_DEPTH_NESTING			1

/*
 * Map the dependency ops to NOP or to real lockdep ops, depending
 * on the per lock-class debug mode:
 */

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define spin_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
#  define spin_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 2, n, i)
# else
#  define spin_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
#  define spin_acquire_nest(l, s, t, n, i)	lock_acquire(l, s, t, 0, 1, NULL, i)
# endif
# define spin_release(l, n, i)			lock_release(l, n, i)
#else
# define spin_acquire(l, s, t, i)		do { } while (0)
# define spin_release(l, n, i)			do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define rwlock_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
#  define rwlock_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 2, 2, NULL, i)
# else
#  define rwlock_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
#  define rwlock_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 2, 1, NULL, i)
# endif
# define rwlock_release(l, n, i)		lock_release(l, n, i)
#else
# define rwlock_acquire(l, s, t, i)		do { } while (0)
# define rwlock_acquire_read(l, s, t, i)	do { } while (0)
# define rwlock_release(l, n, i)		do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define mutex_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
# else
#  define mutex_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
# endif
# define mutex_release(l, n, i)			lock_release(l, n, i)
#else
# define mutex_acquire(l, s, t, i)		do { } while (0)
# define mutex_release(l, n, i)			do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define rwsem_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 2, NULL, i)
#  define rwsem_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 1, 2, NULL, i)
# else
#  define rwsem_acquire(l, s, t, i)		lock_acquire(l, s, t, 0, 1, NULL, i)
#  define rwsem_acquire_read(l, s, t, i)	lock_acquire(l, s, t, 1, 1, NULL, i)
# endif
# define rwsem_release(l, n, i)			lock_release(l, n, i)
#else
# define rwsem_acquire(l, s, t, i)		do { } while (0)
# define rwsem_acquire_read(l, s, t, i)		do { } while (0)
# define rwsem_release(l, n, i)			do { } while (0)
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
#  define lock_map_acquire(l)		lock_acquire(l, 0, 0, 0, 2, NULL, _THIS_IP_)
# else
#  define lock_map_acquire(l)		lock_acquire(l, 0, 0, 0, 1, NULL, _THIS_IP_)
# endif
# define lock_map_release(l)			lock_release(l, 1, _THIS_IP_)
#else
# define lock_map_acquire(l)			do { } while (0)
# define lock_map_release(l)			do { } while (0)
#endif

#ifdef CONFIG_PROVE_LOCKING
# define might_lock(lock) 						\
do {									\
	typecheck(struct lockdep_map *, &(lock)->dep_map);		\
	lock_acquire(&(lock)->dep_map, 0, 0, 0, 2, NULL, _THIS_IP_);	\
	lock_release(&(lock)->dep_map, 0, _THIS_IP_);			\
} while (0)
# define might_lock_read(lock) 						\
do {									\
	typecheck(struct lockdep_map *, &(lock)->dep_map);		\
	lock_acquire(&(lock)->dep_map, 0, 0, 1, 2, NULL, _THIS_IP_);	\
	lock_release(&(lock)->dep_map, 0, _THIS_IP_);			\
} while (0)
#else
# define might_lock(lock) do { } while (0)
# define might_lock_read(lock) do { } while (0)
#endif

#endif /* __LINUX_LOCKDEP_H */
Commit	Line	Data
fbb9ce95 IM	1	/*
	2	* Runtime locking correctness validator
	3	*
4b32d0a4 PZ	4	* Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
4b32d0a4 PZ	5	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
fbb9ce95 IM	6	*
	7	* see Documentation/lockdep-design.txt for more details.
	8	*/
	9	#ifndef __LINUX_LOCKDEP_H
	10	#define __LINUX_LOCKDEP_H
	11
a1e96b03	12	struct task_struct;
f20786ff	13	struct lockdep_map;
a1e96b03	14
db0b0ead MT	15	#ifdef CONFIG_LOCKDEP
db0b0ead MT	16
fbb9ce95 IM	17	#include <linux/linkage.h>
	18	#include <linux/list.h>
	19	#include <linux/debug_locks.h>
	20	#include <linux/stacktrace.h>
	21
fbb9ce95 IM	22	/*
	23	* Lock-class usage-state bits:
	24	*/
	25	enum lock_usage_bit
	26	{
	27	LOCK_USED = 0,
	28	LOCK_USED_IN_HARDIRQ,
	29	LOCK_USED_IN_SOFTIRQ,
	30	LOCK_ENABLED_SOFTIRQS,
	31	LOCK_ENABLED_HARDIRQS,
	32	LOCK_USED_IN_HARDIRQ_READ,
	33	LOCK_USED_IN_SOFTIRQ_READ,
	34	LOCK_ENABLED_SOFTIRQS_READ,
	35	LOCK_ENABLED_HARDIRQS_READ,
	36	LOCK_USAGE_STATES
	37	};
	38
	39	/*
	40	* Usage-state bitmasks:
	41	*/
	42	#define LOCKF_USED (1 << LOCK_USED)
	43	#define LOCKF_USED_IN_HARDIRQ (1 << LOCK_USED_IN_HARDIRQ)
	44	#define LOCKF_USED_IN_SOFTIRQ (1 << LOCK_USED_IN_SOFTIRQ)
	45	#define LOCKF_ENABLED_HARDIRQS (1 << LOCK_ENABLED_HARDIRQS)
	46	#define LOCKF_ENABLED_SOFTIRQS (1 << LOCK_ENABLED_SOFTIRQS)
	47
	48	#define LOCKF_ENABLED_IRQS (LOCKF_ENABLED_HARDIRQS \| LOCKF_ENABLED_SOFTIRQS)
	49	#define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ \| LOCKF_USED_IN_SOFTIRQ)
	50
	51	#define LOCKF_USED_IN_HARDIRQ_READ (1 << LOCK_USED_IN_HARDIRQ_READ)
	52	#define LOCKF_USED_IN_SOFTIRQ_READ (1 << LOCK_USED_IN_SOFTIRQ_READ)
	53	#define LOCKF_ENABLED_HARDIRQS_READ (1 << LOCK_ENABLED_HARDIRQS_READ)
	54	#define LOCKF_ENABLED_SOFTIRQS_READ (1 << LOCK_ENABLED_SOFTIRQS_READ)
	55
	56	#define LOCKF_ENABLED_IRQS_READ \
	57	(LOCKF_ENABLED_HARDIRQS_READ \| LOCKF_ENABLED_SOFTIRQS_READ)
	58	#define LOCKF_USED_IN_IRQ_READ \
	59	(LOCKF_USED_IN_HARDIRQ_READ \| LOCKF_USED_IN_SOFTIRQ_READ)
	60
	61	#define MAX_LOCKDEP_SUBCLASSES 8UL
	62
	63	/*
	64	* Lock-classes are keyed via unique addresses, by embedding the
	65	* lockclass-key into the kernel (or module) .data section. (For
	66	* static locks we use the lock address itself as the key.)
	67	*/
	68	struct lockdep_subclass_key {
	69	char __one_byte;
	70	} __attribute__ ((__packed__));
	71
	72	struct lock_class_key {
	73	struct lockdep_subclass_key subkeys[MAX_LOCKDEP_SUBCLASSES];
	74	};
	75
c7e78cff PZ	76	#define LOCKSTAT_POINTS 4
c7e78cff PZ	77
fbb9ce95 IM	78	/*
	79	* The lock-class itself:
	80	*/
	81	struct lock_class {
	82	/*
	83	* class-hash:
	84	*/
	85	struct list_head hash_entry;
	86
	87	/*
	88	* global list of all lock-classes:
	89	*/
	90	struct list_head lock_entry;
	91
	92	struct lockdep_subclass_key *key;
	93	unsigned int subclass;
419ca3f1	94	unsigned int dep_gen_id;
fbb9ce95 IM	95
	96	/*
	97	* IRQ/softirq usage tracking bits:
	98	*/
	99	unsigned long usage_mask;
	100	struct stack_trace usage_traces[LOCK_USAGE_STATES];
	101
	102	/*
	103	* These fields represent a directed graph of lock dependencies,
	104	* to every node we attach a list of "forward" and a list of
	105	* "backward" graph nodes.
	106	*/
	107	struct list_head locks_after, locks_before;
	108
	109	/*
	110	* Generation counter, when doing certain classes of graph walking,
	111	* to ensure that we check one node only once:
	112	*/
	113	unsigned int version;
	114
	115	/*
	116	* Statistics counter:
	117	*/
	118	unsigned long ops;
	119
	120	const char *name;
	121	int name_version;
f20786ff PZ	122
f20786ff PZ	123	#ifdef CONFIG_LOCK_STAT
c7e78cff PZ	124	unsigned long contention_point[LOCKSTAT_POINTS];
c7e78cff PZ	125	unsigned long contending_point[LOCKSTAT_POINTS];
f20786ff PZ	126	#endif
	127	};
	128
	129	#ifdef CONFIG_LOCK_STAT
	130	struct lock_time {
	131	s64 min;
	132	s64 max;
	133	s64 total;
	134	unsigned long nr;
fbb9ce95 IM	135	};
fbb9ce95 IM	136
96645678 PZ	137	enum bounce_type {
	138	bounce_acquired_write,
	139	bounce_acquired_read,
	140	bounce_contended_write,
	141	bounce_contended_read,
	142	nr_bounce_types,
	143
	144	bounce_acquired = bounce_acquired_write,
	145	bounce_contended = bounce_contended_write,
	146	};
	147
f20786ff PZ	148	struct lock_class_stats {
f20786ff PZ	149	unsigned long contention_point[4];
c7e78cff	150	unsigned long contending_point[4];
f20786ff PZ	151	struct lock_time read_waittime;
	152	struct lock_time write_waittime;
	153	struct lock_time read_holdtime;
	154	struct lock_time write_holdtime;
96645678	155	unsigned long bounces[nr_bounce_types];
f20786ff PZ	156	};
	157
	158	struct lock_class_stats lock_stats(struct lock_class *class);
	159	void clear_lock_stats(struct lock_class *class);
	160	#endif
	161
fbb9ce95 IM	162	/*
	163	* Map the lock object (the lock instance) to the lock-class object.
	164	* This is embedded into specific lock instances:
	165	*/
	166	struct lockdep_map {
	167	struct lock_class_key *key;
d6d897ce	168	struct lock_class *class_cache;
fbb9ce95	169	const char *name;
96645678 PZ	170	#ifdef CONFIG_LOCK_STAT
96645678 PZ	171	int cpu;
c7e78cff	172	unsigned long ip;
96645678	173	#endif
fbb9ce95 IM	174	};
	175
	176	/*
	177	* Every lock has a list of other locks that were taken after it.
	178	* We only grow the list, never remove from it:
	179	*/
	180	struct lock_list {
	181	struct list_head entry;
	182	struct lock_class *class;
	183	struct stack_trace trace;
068135e6	184	int distance;
fbb9ce95 IM	185	};
	186
	187	/*
	188	* We record lock dependency chains, so that we can cache them:
	189	*/
	190	struct lock_chain {
443cd507 HY	191	u8 irq_context;
	192	u8 depth;
	193	u16 base;
fbb9ce95 IM	194	struct list_head entry;
	195	u64 chain_key;
	196	};
	197
e5f363e3	198	#define MAX_LOCKDEP_KEYS_BITS 13
b42e737e PZ	199	/*
	200	* Subtract one because we offset hlock->class_idx by 1 in order
	201	* to make 0 mean no class. This avoids overflowing the class_idx
	202	* bitfield and hitting the BUG in hlock_class().
	203	*/
	204	#define MAX_LOCKDEP_KEYS ((1UL << MAX_LOCKDEP_KEYS_BITS) - 1)
f82b217e	205
fbb9ce95 IM	206	struct held_lock {
	207	/*
	208	* One-way hash of the dependency chain up to this point. We
	209	* hash the hashes step by step as the dependency chain grows.
	210	*
	211	* We use it for dependency-caching and we skip detection
	212	* passes and dependency-updates if there is a cache-hit, so
	213	* it is absolutely critical for 100% coverage of the validator
	214	* to have a unique key value for every unique dependency path
	215	* that can occur in the system, to make a unique hash value
	216	* as likely as possible - hence the 64-bit width.
	217	*
	218	* The task struct holds the current hash value (initialized
	219	* with zero), here we store the previous hash value:
	220	*/
	221	u64 prev_chain_key;
fbb9ce95 IM	222	unsigned long acquire_ip;
fbb9ce95 IM	223	struct lockdep_map *instance;
7531e2f3	224	struct lockdep_map *nest_lock;
f20786ff PZ	225	#ifdef CONFIG_LOCK_STAT
	226	u64 waittime_stamp;
	227	u64 holdtime_stamp;
	228	#endif
f82b217e	229	unsigned int class_idx:MAX_LOCKDEP_KEYS_BITS;
fbb9ce95 IM	230	/*
	231	* The lock-stack is unified in that the lock chains of interrupt
	232	* contexts nest ontop of process context chains, but we 'separate'
	233	* the hashes by starting with 0 if we cross into an interrupt
	234	* context, and we also keep do not add cross-context lock
	235	* dependencies - the lock usage graph walking covers that area
	236	* anyway, and we'd just unnecessarily increase the number of
	237	* dependencies otherwise. [Note: hardirq and softirq contexts
	238	* are separated from each other too.]
	239	*
	240	* The following field is used to detect when we cross into an
	241	* interrupt context:
	242	*/
f82b217e DJ	243	unsigned int irq_context:2; /* bit 0 - soft, bit 1 - hard */
	244	unsigned int trylock:1;
	245	unsigned int read:2; /* see lock_acquire() comment */
	246	unsigned int check:2; /* see lock_acquire() comment */
	247	unsigned int hardirqs_off:1;
fbb9ce95 IM	248	};
	249
	250	/*
	251	* Initialization, self-test and debugging-output methods:
	252	*/
	253	extern void lockdep_init(void);
	254	extern void lockdep_info(void);
	255	extern void lockdep_reset(void);
	256	extern void lockdep_reset_lock(struct lockdep_map *lock);
	257	extern void lockdep_free_key_range(void *start, unsigned long size);
b351d164	258	extern void lockdep_sys_exit(void);
fbb9ce95 IM	259
	260	extern void lockdep_off(void);
	261	extern void lockdep_on(void);
fbb9ce95 IM	262
	263	/*
	264	* These methods are used by specific locking variants (spinlocks,
	265	* rwlocks, mutexes and rwsems) to pass init/acquire/release events
	266	* to lockdep:
	267	*/
	268
	269	extern void lockdep_init_map(struct lockdep_map lock, const char name,
4dfbb9d8	270	struct lock_class_key *key, int subclass);
fbb9ce95	271
851a67b8 PZ	272	/*
	273	* To initialize a lockdep_map statically use this macro.
	274	* Note that _name must not be NULL.
	275	*/
	276	#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
	277	{ .name = (_name), .key = (void *)(_key), }
	278
fbb9ce95 IM	279	/*
	280	* Reinitialize a lock key - for cases where there is special locking or
	281	* special initialization of locks so that the validator gets the scope
	282	* of dependencies wrong: they are either too broad (they need a class-split)
	283	* or they are too narrow (they suffer from a false class-split):
	284	*/
	285	#define lockdep_set_class(lock, key) \
4dfbb9d8	286	lockdep_init_map(&(lock)->dep_map, #key, key, 0)
fbb9ce95	287	#define lockdep_set_class_and_name(lock, key, name) \
4dfbb9d8 PZ	288	lockdep_init_map(&(lock)->dep_map, name, key, 0)
	289	#define lockdep_set_class_and_subclass(lock, key, sub) \
	290	lockdep_init_map(&(lock)->dep_map, #key, key, sub)
	291	#define lockdep_set_subclass(lock, sub) \
	292	lockdep_init_map(&(lock)->dep_map, #lock, \
	293	(lock)->dep_map.key, sub)
fbb9ce95 IM	294
	295	/*
	296	* Acquire a lock.
	297	*
	298	* Values for "read":
	299	*
	300	* 0: exclusive (write) acquire
	301	* 1: read-acquire (no recursion allowed)
	302	* 2: read-acquire with same-instance recursion allowed
	303	*
	304	* Values for check:
	305	*
	306	* 0: disabled
	307	* 1: simple checks (freeing, held-at-exit-time, etc.)
	308	* 2: full validation
	309	*/
	310	extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
7531e2f3 PZ	311	int trylock, int read, int check,
7531e2f3 PZ	312	struct lockdep_map *nest_lock, unsigned long ip);
fbb9ce95 IM	313
	314	extern void lock_release(struct lockdep_map *lock, int nested,
	315	unsigned long ip);
	316
00ef9f73 PZ	317	extern void lock_set_class(struct lockdep_map lock, const char name,
	318	struct lock_class_key *key, unsigned int subclass,
	319	unsigned long ip);
	320
	321	static inline void lock_set_subclass(struct lockdep_map *lock,
	322	unsigned int subclass, unsigned long ip)
	323	{
	324	lock_set_class(lock, lock->name, lock->key, subclass, ip);
	325	}
64aa348e	326
fbb9ce95 IM	327	# define INIT_LOCKDEP .lockdep_recursion = 0,
fbb9ce95 IM	328
e3a55fd1	329	#define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0)
d5abe669	330
fbb9ce95 IM	331	#else /* !LOCKDEP */
	332
	333	static inline void lockdep_off(void)
	334	{
	335	}
	336
	337	static inline void lockdep_on(void)
	338	{
	339	}
	340
7531e2f3	341	# define lock_acquire(l, s, t, r, c, n, i) do { } while (0)
fbb9ce95	342	# define lock_release(l, n, i) do { } while (0)
00ef9f73	343	# define lock_set_class(l, n, k, s, i) do { } while (0)
64aa348e	344	# define lock_set_subclass(l, s, i) do { } while (0)
fbb9ce95 IM	345	# define lockdep_init() do { } while (0)
fbb9ce95 IM	346	# define lockdep_info() do { } while (0)
e25cf3db IM	347	# define lockdep_init_map(lock, name, key, sub) \
e25cf3db IM	348	do { (void)(name); (void)(key); } while (0)
fbb9ce95 IM	349	# define lockdep_set_class(lock, key) do { (void)(key); } while (0)
fbb9ce95 IM	350	# define lockdep_set_class_and_name(lock, key, name) \
e25cf3db	351	do { (void)(key); (void)(name); } while (0)
4dfbb9d8 PZ	352	#define lockdep_set_class_and_subclass(lock, key, sub) \
4dfbb9d8 PZ	353	do { (void)(key); } while (0)
07646e21 AM	354	#define lockdep_set_subclass(lock, sub) do { } while (0)
07646e21 AM	355
fbb9ce95 IM	356	# define INIT_LOCKDEP
	357	# define lockdep_reset() do { debug_locks = 1; } while (0)
	358	# define lockdep_free_key_range(start, size) do { } while (0)
b351d164	359	# define lockdep_sys_exit() do { } while (0)
fbb9ce95 IM	360	/*
	361	* The class key takes no space if lockdep is disabled:
	362	*/
	363	struct lock_class_key { };
d5abe669 PZ	364
	365	#define lockdep_depth(tsk) (0)
	366
fbb9ce95 IM	367	#endif /* !LOCKDEP */
fbb9ce95 IM	368
f20786ff PZ	369	#ifdef CONFIG_LOCK_STAT
	370
	371	extern void lock_contended(struct lockdep_map *lock, unsigned long ip);
c7e78cff	372	extern void lock_acquired(struct lockdep_map *lock, unsigned long ip);
f20786ff PZ	373
	374	#define LOCK_CONTENDED(_lock, try, lock) \
	375	do { \
	376	if (!try(_lock)) { \
	377	lock_contended(&(_lock)->dep_map, _RET_IP_); \
	378	lock(_lock); \
f20786ff	379	} \
c7e78cff	380	lock_acquired(&(_lock)->dep_map, _RET_IP_); \
f20786ff PZ	381	} while (0)
	382
	383	#else /* CONFIG_LOCK_STAT */
	384
	385	#define lock_contended(lockdep_map, ip) do {} while (0)
c7e78cff	386	#define lock_acquired(lockdep_map, ip) do {} while (0)
f20786ff PZ	387
	388	#define LOCK_CONTENDED(_lock, try, lock) \
	389	lock(_lock)
	390
	391	#endif /* CONFIG_LOCK_STAT */
	392
74c8a613	393	#ifdef CONFIG_GENERIC_HARDIRQS
243c7621 IM	394	extern void early_init_irq_lock_class(void);
243c7621 IM	395	#else
3117df04 IM	396	static inline void early_init_irq_lock_class(void)
	397	{
	398	}
243c7621 IM	399	#endif
243c7621 IM	400
fbb9ce95 IM	401	#ifdef CONFIG_TRACE_IRQFLAGS
	402	extern void early_boot_irqs_off(void);
	403	extern void early_boot_irqs_on(void);
3117df04	404	extern void print_irqtrace_events(struct task_struct *curr);
fbb9ce95	405	#else
3117df04 IM	406	static inline void early_boot_irqs_off(void)
	407	{
	408	}
	409	static inline void early_boot_irqs_on(void)
	410	{
	411	}
	412	static inline void print_irqtrace_events(struct task_struct *curr)
	413	{
	414	}
fbb9ce95 IM	415	#endif
	416
	417	/*
	418	* For trivial one-depth nesting of a lock-class, the following
	419	* global define can be used. (Subsystems with multiple levels
	420	* of nesting should define their own lock-nesting subclasses.)
	421	*/
	422	#define SINGLE_DEPTH_NESTING 1
	423
	424	/*
	425	* Map the dependency ops to NOP or to real lockdep ops, depending
	426	* on the per lock-class debug mode:
	427	*/
	428
	429	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	430	# ifdef CONFIG_PROVE_LOCKING
7531e2f3	431	# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
b7d39aff	432	# define spin_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 2, n, i)
fbb9ce95	433	# else
7531e2f3	434	# define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
b7d39aff	435	# define spin_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 1, NULL, i)
fbb9ce95 IM	436	# endif
	437	# define spin_release(l, n, i) lock_release(l, n, i)
	438	#else
	439	# define spin_acquire(l, s, t, i) do { } while (0)
	440	# define spin_release(l, n, i) do { } while (0)
	441	#endif
	442
	443	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	444	# ifdef CONFIG_PROVE_LOCKING
7531e2f3 PZ	445	# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
7531e2f3 PZ	446	# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 2, NULL, i)
fbb9ce95	447	# else
7531e2f3 PZ	448	# define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
7531e2f3 PZ	449	# define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 1, NULL, i)
fbb9ce95 IM	450	# endif
	451	# define rwlock_release(l, n, i) lock_release(l, n, i)
	452	#else
	453	# define rwlock_acquire(l, s, t, i) do { } while (0)
	454	# define rwlock_acquire_read(l, s, t, i) do { } while (0)
	455	# define rwlock_release(l, n, i) do { } while (0)
	456	#endif
	457
	458	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	459	# ifdef CONFIG_PROVE_LOCKING
7531e2f3	460	# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
fbb9ce95	461	# else
7531e2f3	462	# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
fbb9ce95 IM	463	# endif
	464	# define mutex_release(l, n, i) lock_release(l, n, i)
	465	#else
	466	# define mutex_acquire(l, s, t, i) do { } while (0)
	467	# define mutex_release(l, n, i) do { } while (0)
	468	#endif
	469
	470	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	471	# ifdef CONFIG_PROVE_LOCKING
7531e2f3 PZ	472	# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
7531e2f3 PZ	473	# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 2, NULL, i)
fbb9ce95	474	# else
7531e2f3 PZ	475	# define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
7531e2f3 PZ	476	# define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 1, NULL, i)
fbb9ce95 IM	477	# endif
	478	# define rwsem_release(l, n, i) lock_release(l, n, i)
	479	#else
	480	# define rwsem_acquire(l, s, t, i) do { } while (0)
	481	# define rwsem_acquire_read(l, s, t, i) do { } while (0)
	482	# define rwsem_release(l, n, i) do { } while (0)
	483	#endif
	484
4f3e7524 PZ	485	#ifdef CONFIG_DEBUG_LOCK_ALLOC
4f3e7524 PZ	486	# ifdef CONFIG_PROVE_LOCKING
3295f0ef	487	# define lock_map_acquire(l) lock_acquire(l, 0, 0, 0, 2, NULL, _THIS_IP_)
4f3e7524	488	# else
3295f0ef	489	# define lock_map_acquire(l) lock_acquire(l, 0, 0, 0, 1, NULL, _THIS_IP_)
4f3e7524	490	# endif
3295f0ef	491	# define lock_map_release(l) lock_release(l, 1, _THIS_IP_)
4f3e7524	492	#else
3295f0ef IM	493	# define lock_map_acquire(l) do { } while (0)
3295f0ef IM	494	# define lock_map_release(l) do { } while (0)
4f3e7524 PZ	495	#endif
4f3e7524 PZ	496
76b189e9 PZ	497	#ifdef CONFIG_PROVE_LOCKING
	498	# define might_lock(lock) \
	499	do { \
	500	typecheck(struct lockdep_map *, &(lock)->dep_map); \
	501	lock_acquire(&(lock)->dep_map, 0, 0, 0, 2, NULL, _THIS_IP_); \
	502	lock_release(&(lock)->dep_map, 0, _THIS_IP_); \
	503	} while (0)
	504	# define might_lock_read(lock) \
	505	do { \
	506	typecheck(struct lockdep_map *, &(lock)->dep_map); \
	507	lock_acquire(&(lock)->dep_map, 0, 0, 1, 2, NULL, _THIS_IP_); \
	508	lock_release(&(lock)->dep_map, 0, _THIS_IP_); \
	509	} while (0)
	510	#else
	511	# define might_lock(lock) do { } while (0)
	512	# define might_lock_read(lock) do { } while (0)
	513	#endif
	514
fbb9ce95	515	#endif /* __LINUX_LOCKDEP_H */