[deliverable/linux.git] / include / linux / percpu-defs.h

/*
 * linux/percpu-defs.h - basic definitions for percpu areas
 *
 * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
 *
 * This file is separate from linux/percpu.h to avoid cyclic inclusion
 * dependency from arch header files.  Only to be included from
 * asm/percpu.h.
 *
 * This file includes macros necessary to declare percpu sections and
 * variables, and definitions of percpu accessors and operations.  It
 * should provide enough percpu features to arch header files even when
 * they can only include asm/percpu.h to avoid cyclic inclusion dependency.
 */

#ifndef _LINUX_PERCPU_DEFS_H
#define _LINUX_PERCPU_DEFS_H

#ifdef CONFIG_SMP

#ifdef MODULE
#define PER_CPU_SHARED_ALIGNED_SECTION ""
#define PER_CPU_ALIGNED_SECTION ""
#else
#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
#endif
#define PER_CPU_FIRST_SECTION "..first"

#else

#define PER_CPU_SHARED_ALIGNED_SECTION ""
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
#define PER_CPU_FIRST_SECTION ""

#endif

/*
 * Base implementations of per-CPU variable declarations and definitions, where
 * the section in which the variable is to be placed is provided by the
 * 'sec' argument.  This may be used to affect the parameters governing the
 * variable's storage.
 *
 * NOTE!  The sections for the DECLARE and for the DEFINE must match, lest
 * linkage errors occur due the compiler generating the wrong code to access
 * that section.
 */
#define __PCPU_ATTRS(sec)						\
	__percpu __attribute__((section(PER_CPU_BASE_SECTION sec)))	\
	PER_CPU_ATTRIBUTES

#define __PCPU_DUMMY_ATTRS						\
	__attribute__((section(".discard"), unused))

/*
 * s390 and alpha modules require percpu variables to be defined as
 * weak to force the compiler to generate GOT based external
 * references for them.  This is necessary because percpu sections
 * will be located outside of the usually addressable area.
 *
 * This definition puts the following two extra restrictions when
 * defining percpu variables.
 *
 * 1. The symbol must be globally unique, even the static ones.
 * 2. Static percpu variables cannot be defined inside a function.
 *
 * Archs which need weak percpu definitions should define
 * ARCH_NEEDS_WEAK_PER_CPU in asm/percpu.h when necessary.
 *
 * To ensure that the generic code observes the above two
 * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
 * definition is used for all cases.
 */
#if defined(ARCH_NEEDS_WEAK_PER_CPU) || defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
/*
 * __pcpu_scope_* dummy variable is used to enforce scope.  It
 * receives the static modifier when it's used in front of
 * DEFINE_PER_CPU() and will trigger build failure if
 * DECLARE_PER_CPU() is used for the same variable.
 *
 * __pcpu_unique_* dummy variable is used to enforce symbol uniqueness
 * such that hidden weak symbol collision, which will cause unrelated
 * variables to share the same address, can be detected during build.
 */
#define DECLARE_PER_CPU_SECTION(type, name, sec)			\
	extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name;		\
	extern __PCPU_ATTRS(sec) __typeof__(type) name

#define DEFINE_PER_CPU_SECTION(type, name, sec)				\
	__PCPU_DUMMY_ATTRS char __pcpu_scope_##name;			\
	extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;		\
	__PCPU_DUMMY_ATTRS char __pcpu_unique_##name;			\
	extern __PCPU_ATTRS(sec) __typeof__(type) name;			\
	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak			\
	__typeof__(type) name
#else
/*
 * Normal declaration and definition macros.
 */
#define DECLARE_PER_CPU_SECTION(type, name, sec)			\
	extern __PCPU_ATTRS(sec) __typeof__(type) name

#define DEFINE_PER_CPU_SECTION(type, name, sec)				\
	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES			\
	__typeof__(type) name
#endif

/*
 * Variant on the per-CPU variable declaration/definition theme used for
 * ordinary per-CPU variables.
 */
#define DECLARE_PER_CPU(type, name)					\
	DECLARE_PER_CPU_SECTION(type, name, "")

#define DEFINE_PER_CPU(type, name)					\
	DEFINE_PER_CPU_SECTION(type, name, "")

/*
 * Declaration/definition used for per-CPU variables that must come first in
 * the set of variables.
 */
#define DECLARE_PER_CPU_FIRST(type, name)				\
	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)

#define DEFINE_PER_CPU_FIRST(type, name)				\
	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)

/*
 * Declaration/definition used for per-CPU variables that must be cacheline
 * aligned under SMP conditions so that, whilst a particular instance of the
 * data corresponds to a particular CPU, inefficiencies due to direct access by
 * other CPUs are reduced by preventing the data from unnecessarily spanning
 * cachelines.
 *
 * An example of this would be statistical data, where each CPU's set of data
 * is updated by that CPU alone, but the data from across all CPUs is collated
 * by a CPU processing a read from a proc file.
 */
#define DECLARE_PER_CPU_SHARED_ALIGNED(type, name)			\
	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
	____cacheline_aligned_in_smp

#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)			\
	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
	____cacheline_aligned_in_smp

#define DECLARE_PER_CPU_ALIGNED(type, name)				\
	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION)	\
	____cacheline_aligned

#define DEFINE_PER_CPU_ALIGNED(type, name)				\
	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION)	\
	____cacheline_aligned

/*
 * Declaration/definition used for per-CPU variables that must be page aligned.
 */
#define DECLARE_PER_CPU_PAGE_ALIGNED(type, name)			\
	DECLARE_PER_CPU_SECTION(type, name, "..page_aligned")		\
	__aligned(PAGE_SIZE)

#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)				\
	DEFINE_PER_CPU_SECTION(type, name, "..page_aligned")		\
	__aligned(PAGE_SIZE)

/*
 * Declaration/definition used for per-CPU variables that must be read mostly.
 */
#define DECLARE_PER_CPU_READ_MOSTLY(type, name)			\
	DECLARE_PER_CPU_SECTION(type, name, "..read_mostly")

#define DEFINE_PER_CPU_READ_MOSTLY(type, name)				\
	DEFINE_PER_CPU_SECTION(type, name, "..read_mostly")

/*
 * Intermodule exports for per-CPU variables.  sparse forgets about
 * address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to
 * noop if __CHECKER__.
 */
#ifndef __CHECKER__
#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)
#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)
#else
#define EXPORT_PER_CPU_SYMBOL(var)
#define EXPORT_PER_CPU_SYMBOL_GPL(var)
#endif

/*
 * Accessors and operations.
 */
#ifndef __ASSEMBLY__

/*
 * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
 * @ptr and is invoked once before a percpu area is accessed by all
 * accessors and operations.  This is performed in the generic part of
 * percpu and arch overrides don't need to worry about it; however, if an
 * arch wants to implement an arch-specific percpu accessor or operation,
 * it may use __verify_pcpu_ptr() to verify the parameters.
 *
 * + 0 is required in order to convert the pointer type from a
 * potential array type to a pointer to a single item of the array.
 */
#define __verify_pcpu_ptr(ptr)						\
do {									\
	const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL;	\
	(void)__vpp_verify;						\
} while (0)

#ifdef CONFIG_SMP

/*
 * Add an offset to a pointer but keep the pointer as-is.  Use RELOC_HIDE()
 * to prevent the compiler from making incorrect assumptions about the
 * pointer value.  The weird cast keeps both GCC and sparse happy.
 */
#define SHIFT_PERCPU_PTR(__p, __offset)					\
	RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))

#define per_cpu_ptr(ptr, cpu)						\
({									\
	__verify_pcpu_ptr(ptr);						\
	SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)));			\
})

#define raw_cpu_ptr(ptr)						\
({									\
	__verify_pcpu_ptr(ptr);						\
	arch_raw_cpu_ptr(ptr);						\
})

#ifdef CONFIG_DEBUG_PREEMPT
#define this_cpu_ptr(ptr)						\
({									\
	__verify_pcpu_ptr(ptr);						\
	SHIFT_PERCPU_PTR(ptr, my_cpu_offset);				\
})
#else
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
#endif

#else	/* CONFIG_SMP */

#define VERIFY_PERCPU_PTR(__p)						\
({									\
	__verify_pcpu_ptr(__p);						\
	(typeof(*(__p)) __kernel __force *)(__p);			\
})

#define per_cpu_ptr(ptr, cpu)	({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
#define raw_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)
#define this_cpu_ptr(ptr)	raw_cpu_ptr(ptr)

#endif	/* CONFIG_SMP */

#define per_cpu(var, cpu)	(*per_cpu_ptr(&(var), cpu))
#define __raw_get_cpu_var(var)	(*raw_cpu_ptr(&(var)))
#define __get_cpu_var(var)	(*this_cpu_ptr(&(var)))

/*
 * Must be an lvalue. Since @var must be a simple identifier,
 * we force a syntax error here if it isn't.
 */
#define get_cpu_var(var)						\
(*({									\
	preempt_disable();						\
	this_cpu_ptr(&var);						\
}))

/*
 * The weird & is necessary because sparse considers (void)(var) to be
 * a direct dereference of percpu variable (var).
 */
#define put_cpu_var(var)						\
do {									\
	(void)&(var);							\
	preempt_enable();						\
} while (0)

#define get_cpu_ptr(var)						\
({									\
	preempt_disable();						\
	this_cpu_ptr(var);						\
})

#define put_cpu_ptr(var)						\
do {									\
	(void)(var);							\
	preempt_enable();						\
} while (0)

/*
 * Branching function to split up a function into a set of functions that
 * are called for different scalar sizes of the objects handled.
 */

extern void __bad_size_call_parameter(void);

#ifdef CONFIG_DEBUG_PREEMPT
extern void __this_cpu_preempt_check(const char *op);
#else
static inline void __this_cpu_preempt_check(const char *op) { }
#endif

#define __pcpu_size_call_return(stem, variable)				\
({									\
	typeof(variable) pscr_ret__;					\
	__verify_pcpu_ptr(&(variable));					\
	switch(sizeof(variable)) {					\
	case 1: pscr_ret__ = stem##1(variable); break;			\
	case 2: pscr_ret__ = stem##2(variable); break;			\
	case 4: pscr_ret__ = stem##4(variable); break;			\
	case 8: pscr_ret__ = stem##8(variable); break;			\
	default:							\
		__bad_size_call_parameter(); break;			\
	}								\
	pscr_ret__;							\
})

#define __pcpu_size_call_return2(stem, variable, ...)			\
({									\
	typeof(variable) pscr2_ret__;					\
	__verify_pcpu_ptr(&(variable));					\
	switch(sizeof(variable)) {					\
	case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break;	\
	case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break;	\
	case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break;	\
	case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break;	\
	default:							\
		__bad_size_call_parameter(); break;			\
	}								\
	pscr2_ret__;							\
})

/*
 * Special handling for cmpxchg_double.  cmpxchg_double is passed two
 * percpu variables.  The first has to be aligned to a double word
 * boundary and the second has to follow directly thereafter.
 * We enforce this on all architectures even if they don't support
 * a double cmpxchg instruction, since it's a cheap requirement, and it
 * avoids breaking the requirement for architectures with the instruction.
 */
#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...)		\
({									\
	bool pdcrb_ret__;						\
	__verify_pcpu_ptr(&(pcp1));					\
	BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2));			\
	VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1)));	\
	VM_BUG_ON((unsigned long)(&(pcp2)) !=				\
		  (unsigned long)(&(pcp1)) + sizeof(pcp1));		\
	switch(sizeof(pcp1)) {						\
	case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break;	\
	case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break;	\
	case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break;	\
	case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break;	\
	default:							\
		__bad_size_call_parameter(); break;			\
	}								\
	pdcrb_ret__;							\
})

#define __pcpu_size_call(stem, variable, ...)				\
do {									\
	__verify_pcpu_ptr(&(variable));					\
	switch(sizeof(variable)) {					\
		case 1: stem##1(variable, __VA_ARGS__);break;		\
		case 2: stem##2(variable, __VA_ARGS__);break;		\
		case 4: stem##4(variable, __VA_ARGS__);break;		\
		case 8: stem##8(variable, __VA_ARGS__);break;		\
		default: 						\
			__bad_size_call_parameter();break;		\
	}								\
} while (0)

/*
 * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
 *
 * Optimized manipulation for memory allocated through the per cpu
 * allocator or for addresses of per cpu variables.
 *
 * These operation guarantee exclusivity of access for other operations
 * on the *same* processor. The assumption is that per cpu data is only
 * accessed by a single processor instance (the current one).
 *
 * The arch code can provide optimized implementation by defining macros
 * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
 * cpu atomic operations for 2 byte sized RMW actions. If arch code does
 * not provide operations for a scalar size then the fallback in the
 * generic code will be used.
 *
 * cmpxchg_double replaces two adjacent scalars at once.  The first two
 * parameters are per cpu variables which have to be of the same size.  A
 * truth value is returned to indicate success or failure (since a double
 * register result is difficult to handle).  There is very limited hardware
 * support for these operations, so only certain sizes may work.
 */

/*
 * Operations for contexts where we do not want to do any checks for
 * preemptions.  Unless strictly necessary, always use [__]this_cpu_*()
 * instead.
 *
 * If there is no other protection through preempt disable and/or disabling
 * interupts then one of these RMW operations can show unexpected behavior
 * because the execution thread was rescheduled on another processor or an
 * interrupt occurred and the same percpu variable was modified from the
 * interrupt context.
 */
#define raw_cpu_read(pcp)		__pcpu_size_call_return(raw_cpu_read_, pcp)
#define raw_cpu_write(pcp, val)		__pcpu_size_call(raw_cpu_write_, pcp, val)
#define raw_cpu_add(pcp, val)		__pcpu_size_call(raw_cpu_add_, pcp, val)
#define raw_cpu_and(pcp, val)		__pcpu_size_call(raw_cpu_and_, pcp, val)
#define raw_cpu_or(pcp, val)		__pcpu_size_call(raw_cpu_or_, pcp, val)
#define raw_cpu_add_return(pcp, val)	__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
#define raw_cpu_xchg(pcp, nval)		__pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
#define raw_cpu_cmpxchg(pcp, oval, nval) \
	__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)

#define raw_cpu_sub(pcp, val)		raw_cpu_add(pcp, -(val))
#define raw_cpu_inc(pcp)		raw_cpu_add(pcp, 1)
#define raw_cpu_dec(pcp)		raw_cpu_sub(pcp, 1)
#define raw_cpu_sub_return(pcp, val)	raw_cpu_add_return(pcp, -(typeof(pcp))(val))
#define raw_cpu_inc_return(pcp)		raw_cpu_add_return(pcp, 1)
#define raw_cpu_dec_return(pcp)		raw_cpu_add_return(pcp, -1)

/*
 * Operations for contexts that are safe from preemption/interrupts.  These
 * operations verify that preemption is disabled.
 */
#define __this_cpu_read(pcp)						\
({									\
	__this_cpu_preempt_check("read");				\
	raw_cpu_read(pcp);						\
})

#define __this_cpu_write(pcp, val)					\
({									\
	__this_cpu_preempt_check("write");				\
	raw_cpu_write(pcp, val);					\
})

#define __this_cpu_add(pcp, val)					\
({									\
	__this_cpu_preempt_check("add");				\
	raw_cpu_add(pcp, val);						\
})

#define __this_cpu_and(pcp, val)					\
({									\
	__this_cpu_preempt_check("and");				\
	raw_cpu_and(pcp, val);						\
})

#define __this_cpu_or(pcp, val)						\
({									\
	__this_cpu_preempt_check("or");					\
	raw_cpu_or(pcp, val);						\
})

#define __this_cpu_add_return(pcp, val)					\
({									\
	__this_cpu_preempt_check("add_return");				\
	raw_cpu_add_return(pcp, val);					\
})

#define __this_cpu_xchg(pcp, nval)					\
({									\
	__this_cpu_preempt_check("xchg");				\
	raw_cpu_xchg(pcp, nval);					\
})

#define __this_cpu_cmpxchg(pcp, oval, nval)				\
({									\
	__this_cpu_preempt_check("cmpxchg");				\
	raw_cpu_cmpxchg(pcp, oval, nval);				\
})

#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({	__this_cpu_preempt_check("cmpxchg_double");			\
	raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2);	\
})

#define __this_cpu_sub(pcp, val)	__this_cpu_add(pcp, -(typeof(pcp))(val))
#define __this_cpu_inc(pcp)		__this_cpu_add(pcp, 1)
#define __this_cpu_dec(pcp)		__this_cpu_sub(pcp, 1)
#define __this_cpu_sub_return(pcp, val)	__this_cpu_add_return(pcp, -(typeof(pcp))(val))
#define __this_cpu_inc_return(pcp)	__this_cpu_add_return(pcp, 1)
#define __this_cpu_dec_return(pcp)	__this_cpu_add_return(pcp, -1)

/*
 * Operations with implied preemption protection.  These operations can be
 * used without worrying about preemption.  Note that interrupts may still
 * occur while an operation is in progress and if the interrupt modifies
 * the variable too then RMW actions may not be reliable.
 */
#define this_cpu_read(pcp)		__pcpu_size_call_return(this_cpu_read_, pcp)
#define this_cpu_write(pcp, val)	__pcpu_size_call(this_cpu_write_, pcp, val)
#define this_cpu_add(pcp, val)		__pcpu_size_call(this_cpu_add_, pcp, val)
#define this_cpu_and(pcp, val)		__pcpu_size_call(this_cpu_and_, pcp, val)
#define this_cpu_or(pcp, val)		__pcpu_size_call(this_cpu_or_, pcp, val)
#define this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
#define this_cpu_xchg(pcp, nval)	__pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
#define this_cpu_cmpxchg(pcp, oval, nval) \
	__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)

#define this_cpu_sub(pcp, val)		this_cpu_add(pcp, -(typeof(pcp))(val))
#define this_cpu_inc(pcp)		this_cpu_add(pcp, 1)
#define this_cpu_dec(pcp)		this_cpu_sub(pcp, 1)
#define this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(typeof(pcp))(val))
#define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)
#define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)

#endif /* __ASSEMBLY__ */
#endif /* _LINUX_PERCPU_DEFS_H */
Commit	Line	Data
62fde541 TH	1	/*
	2	* linux/percpu-defs.h - basic definitions for percpu areas
	3	*
	4	* DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
	5	*
	6	* This file is separate from linux/percpu.h to avoid cyclic inclusion
	7	* dependency from arch header files. Only to be included from
	8	* asm/percpu.h.
	9	*
	10	* This file includes macros necessary to declare percpu sections and
	11	* variables, and definitions of percpu accessors and operations. It
	12	* should provide enough percpu features to arch header files even when
	13	* they can only include asm/percpu.h to avoid cyclic inclusion dependency.
	14	*/
	15
5028eaa9 DH	16	#ifndef _LINUX_PERCPU_DEFS_H
	17	#define _LINUX_PERCPU_DEFS_H
	18
62fde541 TH	19	#ifdef CONFIG_SMP
	20
	21	#ifdef MODULE
	22	#define PER_CPU_SHARED_ALIGNED_SECTION ""
	23	#define PER_CPU_ALIGNED_SECTION ""
	24	#else
	25	#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
	26	#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
	27	#endif
	28	#define PER_CPU_FIRST_SECTION "..first"
	29
	30	#else
	31
	32	#define PER_CPU_SHARED_ALIGNED_SECTION ""
	33	#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
	34	#define PER_CPU_FIRST_SECTION ""
	35
	36	#endif
	37
5028eaa9 DH	38	/*
	39	* Base implementations of per-CPU variable declarations and definitions, where
	40	* the section in which the variable is to be placed is provided by the
7c756e6e	41	* 'sec' argument. This may be used to affect the parameters governing the
5028eaa9 DH	42	* variable's storage.
	43	*
	44	* NOTE! The sections for the DECLARE and for the DEFINE must match, lest
	45	* linkage errors occur due the compiler generating the wrong code to access
	46	* that section.
	47	*/
7c756e6e	48	#define __PCPU_ATTRS(sec) \
e0fdb0e0	49	__percpu __attribute__((section(PER_CPU_BASE_SECTION sec))) \
7c756e6e TH	50	PER_CPU_ATTRIBUTES
	51
	52	#define __PCPU_DUMMY_ATTRS \
	53	__attribute__((section(".discard"), unused))
	54
	55	/*
	56	* s390 and alpha modules require percpu variables to be defined as
	57	* weak to force the compiler to generate GOT based external
	58	* references for them. This is necessary because percpu sections
	59	* will be located outside of the usually addressable area.
	60	*
	61	* This definition puts the following two extra restrictions when
	62	* defining percpu variables.
	63	*
	64	* 1. The symbol must be globally unique, even the static ones.
	65	* 2. Static percpu variables cannot be defined inside a function.
	66	*
	67	* Archs which need weak percpu definitions should define
	68	* ARCH_NEEDS_WEAK_PER_CPU in asm/percpu.h when necessary.
	69	*
	70	* To ensure that the generic code observes the above two
	71	* restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
	72	* definition is used for all cases.
	73	*/
	74	#if defined(ARCH_NEEDS_WEAK_PER_CPU) \|\| defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
	75	/*
	76	* __pcpu_scope_* dummy variable is used to enforce scope. It
	77	* receives the static modifier when it's used in front of
	78	* DEFINE_PER_CPU() and will trigger build failure if
	79	* DECLARE_PER_CPU() is used for the same variable.
	80	*
	81	* __pcpu_unique_* dummy variable is used to enforce symbol uniqueness
	82	* such that hidden weak symbol collision, which will cause unrelated
	83	* variables to share the same address, can be detected during build.
	84	*/
	85	#define DECLARE_PER_CPU_SECTION(type, name, sec) \
	86	extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \
dd17c8f7	87	extern __PCPU_ATTRS(sec) __typeof__(type) name
7c756e6e TH	88
	89	#define DEFINE_PER_CPU_SECTION(type, name, sec) \
	90	__PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \
0f5e4816	91	extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
7c756e6e	92	__PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
b1a0fbfd	93	extern __PCPU_ATTRS(sec) __typeof__(type) name; \
c43768cb	94	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \
dd17c8f7	95	__typeof__(type) name
7c756e6e TH	96	#else
	97	/*
	98	* Normal declaration and definition macros.
	99	*/
	100	#define DECLARE_PER_CPU_SECTION(type, name, sec) \
dd17c8f7	101	extern __PCPU_ATTRS(sec) __typeof__(type) name
7c756e6e TH	102
7c756e6e TH	103	#define DEFINE_PER_CPU_SECTION(type, name, sec) \
c43768cb	104	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES \
dd17c8f7	105	__typeof__(type) name
7c756e6e	106	#endif
5028eaa9 DH	107
	108	/*
	109	* Variant on the per-CPU variable declaration/definition theme used for
	110	* ordinary per-CPU variables.
	111	*/
	112	#define DECLARE_PER_CPU(type, name) \
	113	DECLARE_PER_CPU_SECTION(type, name, "")
	114
	115	#define DEFINE_PER_CPU(type, name) \
	116	DEFINE_PER_CPU_SECTION(type, name, "")
	117
	118	/*
	119	* Declaration/definition used for per-CPU variables that must come first in
	120	* the set of variables.
	121	*/
	122	#define DECLARE_PER_CPU_FIRST(type, name) \
	123	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
	124
	125	#define DEFINE_PER_CPU_FIRST(type, name) \
	126	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
	127
	128	/*
	129	* Declaration/definition used for per-CPU variables that must be cacheline
	130	* aligned under SMP conditions so that, whilst a particular instance of the
	131	* data corresponds to a particular CPU, inefficiencies due to direct access by
	132	* other CPUs are reduced by preventing the data from unnecessarily spanning
	133	* cachelines.
	134	*
	135	* An example of this would be statistical data, where each CPU's set of data
	136	* is updated by that CPU alone, but the data from across all CPUs is collated
	137	* by a CPU processing a read from a proc file.
	138	*/
	139	#define DECLARE_PER_CPU_SHARED_ALIGNED(type, name) \
	140	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
	141	____cacheline_aligned_in_smp
	142
	143	#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \
	144	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
	145	____cacheline_aligned_in_smp
	146
53f82452 JF	147	#define DECLARE_PER_CPU_ALIGNED(type, name) \
	148	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION) \
	149	____cacheline_aligned
	150
	151	#define DEFINE_PER_CPU_ALIGNED(type, name) \
	152	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION) \
	153	____cacheline_aligned
	154
5028eaa9 DH	155	/*
	156	* Declaration/definition used for per-CPU variables that must be page aligned.
	157	*/
3e352aa8	158	#define DECLARE_PER_CPU_PAGE_ALIGNED(type, name) \
3d9a854c	159	DECLARE_PER_CPU_SECTION(type, name, "..page_aligned") \
3e352aa8	160	__aligned(PAGE_SIZE)
5028eaa9 DH	161
5028eaa9 DH	162	#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \
3d9a854c	163	DEFINE_PER_CPU_SECTION(type, name, "..page_aligned") \
3e352aa8	164	__aligned(PAGE_SIZE)
5028eaa9	165
c957ef2c SL	166	/*
	167	* Declaration/definition used for per-CPU variables that must be read mostly.
	168	*/
	169	#define DECLARE_PER_CPU_READ_MOSTLY(type, name) \
330d2822	170	DECLARE_PER_CPU_SECTION(type, name, "..read_mostly")
c957ef2c SL	171
c957ef2c SL	172	#define DEFINE_PER_CPU_READ_MOSTLY(type, name) \
330d2822	173	DEFINE_PER_CPU_SECTION(type, name, "..read_mostly")
c957ef2c	174
5028eaa9	175	/*
545695fb TH	176	* Intermodule exports for per-CPU variables. sparse forgets about
	177	* address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to
	178	* noop if __CHECKER__.
5028eaa9	179	*/
545695fb	180	#ifndef __CHECKER__
dd17c8f7 RR	181	#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)
dd17c8f7 RR	182	#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)
545695fb TH	183	#else
	184	#define EXPORT_PER_CPU_SYMBOL(var)
	185	#define EXPORT_PER_CPU_SYMBOL_GPL(var)
	186	#endif
5028eaa9	187
62fde541 TH	188	/*
	189	* Accessors and operations.
	190	*/
	191	#ifndef __ASSEMBLY__
	192
9c28278a	193	/*
6fbc07bb TH	194	* __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
	195	* @ptr and is invoked once before a percpu area is accessed by all
	196	* accessors and operations. This is performed in the generic part of
	197	* percpu and arch overrides don't need to worry about it; however, if an
	198	* arch wants to implement an arch-specific percpu accessor or operation,
	199	* it may use __verify_pcpu_ptr() to verify the parameters.
9c28278a TH	200	*
	201	* + 0 is required in order to convert the pointer type from a
	202	* potential array type to a pointer to a single item of the array.
	203	*/
eba11788 TH	204	#define __verify_pcpu_ptr(ptr) \
eba11788 TH	205	do { \
9c28278a TH	206	const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
	207	(void)__vpp_verify; \
	208	} while (0)
	209
62fde541 TH	210	#ifdef CONFIG_SMP
	211
	212	/*
	213	* Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE()
	214	* to prevent the compiler from making incorrect assumptions about the
	215	* pointer value. The weird cast keeps both GCC and sparse happy.
	216	*/
eba11788	217	#define SHIFT_PERCPU_PTR(__p, __offset) \
6fbc07bb TH	218	RELOC_HIDE((typeof((__p)) __kernel __force )(__p), (__offset))
	219
	220	#define per_cpu_ptr(ptr, cpu) \
eba11788	221	({ \
6fbc07bb TH	222	__verify_pcpu_ptr(ptr); \
6fbc07bb TH	223	SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
62fde541 TH	224	})
62fde541 TH	225
6fbc07bb TH	226	#define raw_cpu_ptr(ptr) \
	227	({ \
	228	__verify_pcpu_ptr(ptr); \
	229	arch_raw_cpu_ptr(ptr); \
	230	})
62fde541 TH	231
62fde541 TH	232	#ifdef CONFIG_DEBUG_PREEMPT
6fbc07bb TH	233	#define this_cpu_ptr(ptr) \
	234	({ \
	235	__verify_pcpu_ptr(ptr); \
	236	SHIFT_PERCPU_PTR(ptr, my_cpu_offset); \
	237	})
62fde541 TH	238	#else
	239	#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
	240	#endif
	241
62fde541 TH	242	#else /* CONFIG_SMP */
62fde541 TH	243
eba11788 TH	244	#define VERIFY_PERCPU_PTR(__p) \
	245	({ \
	246	__verify_pcpu_ptr(__p); \
	247	(typeof((__p)) __kernel __force )(__p); \
62fde541 TH	248	})
62fde541 TH	249
eba11788	250	#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
3b8ed91d TH	251	#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
3b8ed91d TH	252	#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
62fde541 TH	253
	254	#endif /* CONFIG_SMP */
	255
3b8ed91d TH	256	#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
	257	#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
	258	#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
	259
9defda18 TH	260	/*
	261	* Must be an lvalue. Since @var must be a simple identifier,
	262	* we force a syntax error here if it isn't.
	263	*/
eba11788 TH	264	#define get_cpu_var(var) \
	265	(*({ \
	266	preempt_disable(); \
	267	this_cpu_ptr(&var); \
	268	}))
9defda18 TH	269
	270	/*
	271	* The weird & is necessary because sparse considers (void)(var) to be
	272	* a direct dereference of percpu variable (var).
	273	*/
eba11788 TH	274	#define put_cpu_var(var) \
	275	do { \
	276	(void)&(var); \
	277	preempt_enable(); \
9defda18 TH	278	} while (0)
9defda18 TH	279
eba11788 TH	280	#define get_cpu_ptr(var) \
	281	({ \
	282	preempt_disable(); \
	283	this_cpu_ptr(var); \
	284	})
9defda18	285
eba11788 TH	286	#define put_cpu_ptr(var) \
	287	do { \
	288	(void)(var); \
	289	preempt_enable(); \
9defda18 TH	290	} while (0)
9defda18 TH	291
a32f8d8e TH	292	/*
	293	* Branching function to split up a function into a set of functions that
	294	* are called for different scalar sizes of the objects handled.
	295	*/
	296
	297	extern void __bad_size_call_parameter(void);
	298
	299	#ifdef CONFIG_DEBUG_PREEMPT
	300	extern void __this_cpu_preempt_check(const char *op);
	301	#else
	302	static inline void __this_cpu_preempt_check(const char *op) { }
	303	#endif
	304
	305	#define __pcpu_size_call_return(stem, variable) \
eba11788 TH	306	({ \
eba11788 TH	307	typeof(variable) pscr_ret__; \
a32f8d8e TH	308	__verify_pcpu_ptr(&(variable)); \
a32f8d8e TH	309	switch(sizeof(variable)) { \
eba11788 TH	310	case 1: pscr_ret__ = stem##1(variable); break; \
	311	case 2: pscr_ret__ = stem##2(variable); break; \
	312	case 4: pscr_ret__ = stem##4(variable); break; \
	313	case 8: pscr_ret__ = stem##8(variable); break; \
a32f8d8e	314	default: \
eba11788	315	__bad_size_call_parameter(); break; \
a32f8d8e TH	316	} \
	317	pscr_ret__; \
	318	})
	319
	320	#define __pcpu_size_call_return2(stem, variable, ...) \
	321	({ \
	322	typeof(variable) pscr2_ret__; \
	323	__verify_pcpu_ptr(&(variable)); \
	324	switch(sizeof(variable)) { \
	325	case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
	326	case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
	327	case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
	328	case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
	329	default: \
	330	__bad_size_call_parameter(); break; \
	331	} \
	332	pscr2_ret__; \
	333	})
	334
	335	/*
	336	* Special handling for cmpxchg_double. cmpxchg_double is passed two
	337	* percpu variables. The first has to be aligned to a double word
	338	* boundary and the second has to follow directly thereafter.
	339	* We enforce this on all architectures even if they don't support
	340	* a double cmpxchg instruction, since it's a cheap requirement, and it
	341	* avoids breaking the requirement for architectures with the instruction.
	342	*/
	343	#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
	344	({ \
	345	bool pdcrb_ret__; \
eba11788	346	__verify_pcpu_ptr(&(pcp1)); \
a32f8d8e	347	BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
eba11788 TH	348	VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
	349	VM_BUG_ON((unsigned long)(&(pcp2)) != \
	350	(unsigned long)(&(pcp1)) + sizeof(pcp1)); \
a32f8d8e TH	351	switch(sizeof(pcp1)) { \
	352	case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
	353	case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
	354	case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
	355	case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
	356	default: \
	357	__bad_size_call_parameter(); break; \
	358	} \
	359	pdcrb_ret__; \
	360	})
	361
	362	#define __pcpu_size_call(stem, variable, ...) \
	363	do { \
	364	__verify_pcpu_ptr(&(variable)); \
	365	switch(sizeof(variable)) { \
	366	case 1: stem##1(variable, __VA_ARGS__);break; \
	367	case 2: stem##2(variable, __VA_ARGS__);break; \
	368	case 4: stem##4(variable, __VA_ARGS__);break; \
	369	case 8: stem##8(variable, __VA_ARGS__);break; \
	370	default: \
	371	__bad_size_call_parameter();break; \
	372	} \
	373	} while (0)
	374
	375	/*
	376	* this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
	377	*
	378	* Optimized manipulation for memory allocated through the per cpu
	379	* allocator or for addresses of per cpu variables.
	380	*
	381	* These operation guarantee exclusivity of access for other operations
	382	* on the same processor. The assumption is that per cpu data is only
	383	* accessed by a single processor instance (the current one).
	384	*
	385	* The arch code can provide optimized implementation by defining macros
	386	* for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
	387	* cpu atomic operations for 2 byte sized RMW actions. If arch code does
	388	* not provide operations for a scalar size then the fallback in the
	389	* generic code will be used.
eba11788 TH	390	*
	391	* cmpxchg_double replaces two adjacent scalars at once. The first two
	392	* parameters are per cpu variables which have to be of the same size. A
	393	* truth value is returned to indicate success or failure (since a double
	394	* register result is difficult to handle). There is very limited hardware
	395	* support for these operations, so only certain sizes may work.
a32f8d8e TH	396	*/
	397
	398	/*
eba11788 TH	399	* Operations for contexts where we do not want to do any checks for
	400	* preemptions. Unless strictly necessary, always use [__]this_cpu_*()
	401	* instead.
a32f8d8e	402	*
eba11788 TH	403	* If there is no other protection through preempt disable and/or disabling
	404	* interupts then one of these RMW operations can show unexpected behavior
	405	* because the execution thread was rescheduled on another processor or an
	406	* interrupt occurred and the same percpu variable was modified from the
	407	* interrupt context.
a32f8d8e	408	*/
eba11788 TH	409	#define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, pcp)
	410	#define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, pcp, val)
	411	#define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, pcp, val)
	412	#define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, pcp, val)
	413	#define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, pcp, val)
	414	#define raw_cpu_add_return(pcp, val) __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
	415	#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
	416	#define raw_cpu_cmpxchg(pcp, oval, nval) \
a32f8d8e	417	__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
eba11788 TH	418	#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	419	__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
	420
	421	#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
	422	#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
	423	#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
	424	#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
	425	#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
	426	#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
a32f8d8e TH	427
a32f8d8e TH	428	/*
eba11788 TH	429	* Operations for contexts that are safe from preemption/interrupts. These
eba11788 TH	430	* operations verify that preemption is disabled.
a32f8d8e	431	*/
eba11788 TH	432	#define __this_cpu_read(pcp) \
	433	({ \
	434	__this_cpu_preempt_check("read"); \
	435	raw_cpu_read(pcp); \
	436	})
a32f8d8e	437
eba11788 TH	438	#define __this_cpu_write(pcp, val) \
	439	({ \
	440	__this_cpu_preempt_check("write"); \
	441	raw_cpu_write(pcp, val); \
	442	})
a32f8d8e	443
eba11788 TH	444	#define __this_cpu_add(pcp, val) \
	445	({ \
	446	__this_cpu_preempt_check("add"); \
cadb1c4d	447	raw_cpu_add(pcp, val); \
eba11788	448	})
a32f8d8e	449
eba11788 TH	450	#define __this_cpu_and(pcp, val) \
	451	({ \
	452	__this_cpu_preempt_check("and"); \
cadb1c4d	453	raw_cpu_and(pcp, val); \
eba11788	454	})
a32f8d8e	455
eba11788 TH	456	#define __this_cpu_or(pcp, val) \
	457	({ \
	458	__this_cpu_preempt_check("or"); \
cadb1c4d	459	raw_cpu_or(pcp, val); \
eba11788	460	})
a32f8d8e	461
eba11788 TH	462	#define __this_cpu_add_return(pcp, val) \
	463	({ \
	464	__this_cpu_preempt_check("add_return"); \
	465	raw_cpu_add_return(pcp, val); \
	466	})
a32f8d8e	467
eba11788 TH	468	#define __this_cpu_xchg(pcp, nval) \
	469	({ \
	470	__this_cpu_preempt_check("xchg"); \
	471	raw_cpu_xchg(pcp, nval); \
	472	})
a32f8d8e	473
eba11788 TH	474	#define __this_cpu_cmpxchg(pcp, oval, nval) \
	475	({ \
	476	__this_cpu_preempt_check("cmpxchg"); \
	477	raw_cpu_cmpxchg(pcp, oval, nval); \
	478	})
a32f8d8e	479
eba11788 TH	480	#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	481	({ __this_cpu_preempt_check("cmpxchg_double"); \
	482	raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
	483	})
a32f8d8e	484
eba11788 TH	485	#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
	486	#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
	487	#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
	488	#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
	489	#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
	490	#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
a32f8d8e TH	491
a32f8d8e TH	492	/*
eba11788 TH	493	* Operations with implied preemption protection. These operations can be
	494	* used without worrying about preemption. Note that interrupts may still
	495	* occur while an operation is in progress and if the interrupt modifies
	496	* the variable too then RMW actions may not be reliable.
a32f8d8e	497	*/
eba11788 TH	498	#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
	499	#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
	500	#define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
	501	#define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, pcp, val)
	502	#define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, pcp, val)
	503	#define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
	504	#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
	505	#define this_cpu_cmpxchg(pcp, oval, nval) \
	506	__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
	507	#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	508	__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
	509
	510	#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
	511	#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
	512	#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
a32f8d8e TH	513	#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
	514	#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
	515	#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
a32f8d8e	516
62fde541	517	#endif /* __ASSEMBLY__ */
5028eaa9	518	#endif /* _LINUX_PERCPU_DEFS_H */