[deliverable/linux.git] / arch / arm64 / include / asm / percpu.h

/*
 * Copyright (C) 2013 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_PERCPU_H
#define __ASM_PERCPU_H

#ifdef CONFIG_SMP

static inline void set_my_cpu_offset(unsigned long off)
{
	asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
}

static inline unsigned long __my_cpu_offset(void)
{
	unsigned long off;

	/*
	 * We want to allow caching the value, so avoid using volatile and
	 * instead use a fake stack read to hazard against barrier().
	 */
	asm("mrs %0, tpidr_el1" : "=r" (off) :
		"Q" (*(const unsigned long *)current_stack_pointer));

	return off;
}
#define __my_cpu_offset __my_cpu_offset()

#else	/* !CONFIG_SMP */

#define set_my_cpu_offset(x)	do { } while (0)

#endif /* CONFIG_SMP */

#define PERCPU_OP(op, asm_op)						\
static inline unsigned long __percpu_##op(void *ptr,			\
			unsigned long val, int size)			\
{									\
	unsigned long loop, ret;					\
									\
	switch (size) {							\
	case 1:								\
		do {							\
			asm ("//__per_cpu_" #op "_1\n"			\
			"ldxrb	  %w[ret], %[ptr]\n"			\
			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
			"stxrb	  %w[loop], %w[ret], %[ptr]\n"		\
			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
			  [ptr] "+Q"(*(u8 *)ptr)			\
			: [val] "Ir" (val));				\
		} while (loop);						\
		break;							\
	case 2:								\
		do {							\
			asm ("//__per_cpu_" #op "_2\n"			\
			"ldxrh	  %w[ret], %[ptr]\n"			\
			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
			"stxrh	  %w[loop], %w[ret], %[ptr]\n"		\
			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
			  [ptr]  "+Q"(*(u16 *)ptr)			\
			: [val] "Ir" (val));				\
		} while (loop);						\
		break;							\
	case 4:								\
		do {							\
			asm ("//__per_cpu_" #op "_4\n"			\
			"ldxr	  %w[ret], %[ptr]\n"			\
			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
			"stxr	  %w[loop], %w[ret], %[ptr]\n"		\
			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
			  [ptr] "+Q"(*(u32 *)ptr)			\
			: [val] "Ir" (val));				\
		} while (loop);						\
		break;							\
	case 8:								\
		do {							\
			asm ("//__per_cpu_" #op "_8\n"			\
			"ldxr	  %[ret], %[ptr]\n"			\
			#asm_op " %[ret], %[ret], %[val]\n"		\
			"stxr	  %w[loop], %[ret], %[ptr]\n"		\
			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
			  [ptr] "+Q"(*(u64 *)ptr)			\
			: [val] "Ir" (val));				\
		} while (loop);						\
		break;							\
	default:							\
		BUILD_BUG();						\
	}								\
									\
	return ret;							\
}

PERCPU_OP(add, add)
PERCPU_OP(and, and)
PERCPU_OP(or, orr)
#undef PERCPU_OP

static inline unsigned long __percpu_read(void *ptr, int size)
{
	unsigned long ret;

	switch (size) {
	case 1:
		ret = ACCESS_ONCE(*(u8 *)ptr);
		break;
	case 2:
		ret = ACCESS_ONCE(*(u16 *)ptr);
		break;
	case 4:
		ret = ACCESS_ONCE(*(u32 *)ptr);
		break;
	case 8:
		ret = ACCESS_ONCE(*(u64 *)ptr);
		break;
	default:
		BUILD_BUG();
	}

	return ret;
}

static inline void __percpu_write(void *ptr, unsigned long val, int size)
{
	switch (size) {
	case 1:
		ACCESS_ONCE(*(u8 *)ptr) = (u8)val;
		break;
	case 2:
		ACCESS_ONCE(*(u16 *)ptr) = (u16)val;
		break;
	case 4:
		ACCESS_ONCE(*(u32 *)ptr) = (u32)val;
		break;
	case 8:
		ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
		break;
	default:
		BUILD_BUG();
	}
}

static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
						int size)
{
	unsigned long ret, loop;

	switch (size) {
	case 1:
		do {
			asm ("//__percpu_xchg_1\n"
			"ldxrb %w[ret], %[ptr]\n"
			"stxrb %w[loop], %w[val], %[ptr]\n"
			: [loop] "=&r"(loop), [ret] "=&r"(ret),
			  [ptr] "+Q"(*(u8 *)ptr)
			: [val] "r" (val));
		} while (loop);
		break;
	case 2:
		do {
			asm ("//__percpu_xchg_2\n"
			"ldxrh %w[ret], %[ptr]\n"
			"stxrh %w[loop], %w[val], %[ptr]\n"
			: [loop] "=&r"(loop), [ret] "=&r"(ret),
			  [ptr] "+Q"(*(u16 *)ptr)
			: [val] "r" (val));
		} while (loop);
		break;
	case 4:
		do {
			asm ("//__percpu_xchg_4\n"
			"ldxr %w[ret], %[ptr]\n"
			"stxr %w[loop], %w[val], %[ptr]\n"
			: [loop] "=&r"(loop), [ret] "=&r"(ret),
			  [ptr] "+Q"(*(u32 *)ptr)
			: [val] "r" (val));
		} while (loop);
		break;
	case 8:
		do {
			asm ("//__percpu_xchg_8\n"
			"ldxr %[ret], %[ptr]\n"
			"stxr %w[loop], %[val], %[ptr]\n"
			: [loop] "=&r"(loop), [ret] "=&r"(ret),
			  [ptr] "+Q"(*(u64 *)ptr)
			: [val] "r" (val));
		} while (loop);
		break;
	default:
		BUILD_BUG();
	}

	return ret;
}

#define _percpu_add(pcp, val) \
	__percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))

#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))

#define _percpu_and(pcp, val) \
	__percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))

#define _percpu_or(pcp, val) \
	__percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))

#define _percpu_read(pcp) (typeof(pcp))	\
	(__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))

#define _percpu_write(pcp, val) \
	__percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))

#define _percpu_xchg(pcp, val) (typeof(pcp)) \
	(__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))

#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)

#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)

#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)

#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)

#define this_cpu_read_1(pcp) _percpu_read(pcp)
#define this_cpu_read_2(pcp) _percpu_read(pcp)
#define this_cpu_read_4(pcp) _percpu_read(pcp)
#define this_cpu_read_8(pcp) _percpu_read(pcp)

#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)

#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)

#include <asm-generic/percpu.h>

#endif /* __ASM_PERCPU_H */
Commit	Line	Data
71586276 WD	1	/*
	2	* Copyright (C) 2013 ARM Ltd.
	3	*
	4	* This program is free software; you can redistribute it and/or modify
	5	* it under the terms of the GNU General Public License version 2 as
	6	* published by the Free Software Foundation.
	7	*
	8	* This program is distributed in the hope that it will be useful,
	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	* GNU General Public License for more details.
	12	*
	13	* You should have received a copy of the GNU General Public License
	14	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	15	*/
	16	#ifndef __ASM_PERCPU_H
	17	#define __ASM_PERCPU_H
	18
b57fc9e8 CM	19	#ifdef CONFIG_SMP
b57fc9e8 CM	20
71586276 WD	21	static inline void set_my_cpu_offset(unsigned long off)
	22	{
	23	asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
	24	}
	25
	26	static inline unsigned long __my_cpu_offset(void)
	27	{
	28	unsigned long off;
71586276 WD	29
	30	/*
	31	* We want to allow caching the value, so avoid using volatile and
	32	* instead use a fake stack read to hazard against barrier().
	33	*/
34ccf8f4 MC	34	asm("mrs %0, tpidr_el1" : "=r" (off) :
34ccf8f4 MC	35	"Q" ((const unsigned long )current_stack_pointer));
71586276 WD	36
	37	return off;
	38	}
	39	#define __my_cpu_offset __my_cpu_offset()
	40
b57fc9e8 CM	41	#else /* !CONFIG_SMP */
	42
	43	#define set_my_cpu_offset(x) do { } while (0)
	44
	45	#endif /* CONFIG_SMP */
	46
f97fc810 SC	47	#define PERCPU_OP(op, asm_op) \
	48	static inline unsigned long __percpu_##op(void *ptr, \
	49	unsigned long val, int size) \
	50	{ \
	51	unsigned long loop, ret; \
	52	\
	53	switch (size) { \
	54	case 1: \
	55	do { \
	56	asm ("//__per_cpu_" #op "_1\n" \
	57	"ldxrb %w[ret], %[ptr]\n" \
	58	#asm_op " %w[ret], %w[ret], %w[val]\n" \
	59	"stxrb %w[loop], %w[ret], %[ptr]\n" \
	60	: [loop] "=&r" (loop), [ret] "=&r" (ret), \
	61	[ptr] "+Q"((u8 )ptr) \
	62	: [val] "Ir" (val)); \
	63	} while (loop); \
	64	break; \
	65	case 2: \
	66	do { \
	67	asm ("//__per_cpu_" #op "_2\n" \
	68	"ldxrh %w[ret], %[ptr]\n" \
	69	#asm_op " %w[ret], %w[ret], %w[val]\n" \
	70	"stxrh %w[loop], %w[ret], %[ptr]\n" \
	71	: [loop] "=&r" (loop), [ret] "=&r" (ret), \
	72	[ptr] "+Q"((u16 )ptr) \
	73	: [val] "Ir" (val)); \
	74	} while (loop); \
	75	break; \
	76	case 4: \
	77	do { \
	78	asm ("//__per_cpu_" #op "_4\n" \
	79	"ldxr %w[ret], %[ptr]\n" \
	80	#asm_op " %w[ret], %w[ret], %w[val]\n" \
	81	"stxr %w[loop], %w[ret], %[ptr]\n" \
	82	: [loop] "=&r" (loop), [ret] "=&r" (ret), \
	83	[ptr] "+Q"((u32 )ptr) \
	84	: [val] "Ir" (val)); \
	85	} while (loop); \
	86	break; \
	87	case 8: \
	88	do { \
	89	asm ("//__per_cpu_" #op "_8\n" \
	90	"ldxr %[ret], %[ptr]\n" \
	91	#asm_op " %[ret], %[ret], %[val]\n" \
	92	"stxr %w[loop], %[ret], %[ptr]\n" \
	93	: [loop] "=&r" (loop), [ret] "=&r" (ret), \
	94	[ptr] "+Q"((u64 )ptr) \
	95	: [val] "Ir" (val)); \
	96	} while (loop); \
	97	break; \
	98	default: \
	99	BUILD_BUG(); \
	100	} \
	101	\
	102	return ret; \
	103	}
	104
	105	PERCPU_OP(add, add)
	106	PERCPU_OP(and, and)
	107	PERCPU_OP(or, orr)
	108	#undef PERCPU_OP
	109
	110	static inline unsigned long __percpu_read(void *ptr, int size)
111	{
112	unsigned long ret;
113
114	switch (size) {
115	case 1:
116	ret = ACCESS_ONCE((u8 )ptr);
117	break;
118	case 2:
119	ret = ACCESS_ONCE((u16 )ptr);
120	break;
121	case 4:
122	ret = ACCESS_ONCE((u32 )ptr);
123	break;
124	case 8:
125	ret = ACCESS_ONCE((u64 )ptr);
126	break;
127	default:
128	BUILD_BUG();
129	}
130
131	return ret;
132	}
133
134	static inline void __percpu_write(void *ptr, unsigned long val, int size)
135	{
136	switch (size) {
137	case 1:
138	ACCESS_ONCE((u8 )ptr) = (u8)val;
139	break;
140	case 2:
141	ACCESS_ONCE((u16 )ptr) = (u16)val;
142	break;
143	case 4:
144	ACCESS_ONCE((u32 )ptr) = (u32)val;
145	break;
146	case 8:
147	ACCESS_ONCE((u64 )ptr) = (u64)val;
148	break;
149	default:
150	BUILD_BUG();
151	}
152	}
153
154	static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
155	int size)
156	{
157	unsigned long ret, loop;
158
159	switch (size) {
160	case 1:
161	do {
162	asm ("//__percpu_xchg_1\n"
163	"ldxrb %w[ret], %[ptr]\n"
164	"stxrb %w[loop], %w[val], %[ptr]\n"
165	: [loop] "=&r"(loop), [ret] "=&r"(ret),
166	[ptr] "+Q"((u8 )ptr)
167	: [val] "r" (val));
168	} while (loop);
169	break;
170	case 2:
171	do {
172	asm ("//__percpu_xchg_2\n"
173	"ldxrh %w[ret], %[ptr]\n"
174	"stxrh %w[loop], %w[val], %[ptr]\n"
175	: [loop] "=&r"(loop), [ret] "=&r"(ret),
176	[ptr] "+Q"((u16 )ptr)
177	: [val] "r" (val));
178	} while (loop);
179	break;
180	case 4:
181	do {
182	asm ("//__percpu_xchg_4\n"
183	"ldxr %w[ret], %[ptr]\n"
184	"stxr %w[loop], %w[val], %[ptr]\n"
185	: [loop] "=&r"(loop), [ret] "=&r"(ret),
186	[ptr] "+Q"((u32 )ptr)
187	: [val] "r" (val));
188	} while (loop);
189	break;
190	case 8:
191	do {
192	asm ("//__percpu_xchg_8\n"
193	"ldxr %[ret], %[ptr]\n"
194	"stxr %w[loop], %[val], %[ptr]\n"
195	: [loop] "=&r"(loop), [ret] "=&r"(ret),
196	[ptr] "+Q"((u64 )ptr)
197	: [val] "r" (val));
198	} while (loop);
199	break;
200	default:
201	BUILD_BUG();
202	}
203
204	return ret;
205	}
206
207	#define _percpu_add(pcp, val) \
208	__percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
209
210	#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
211
212	#define _percpu_and(pcp, val) \
213	__percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
214
215	#define _percpu_or(pcp, val) \
216	__percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
217
218	#define _percpu_read(pcp) (typeof(pcp)) \
219	(__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
220
221	#define _percpu_write(pcp, val) \
222	__percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
223
224	#define _percpu_xchg(pcp, val) (typeof(pcp)) \
225	(__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
226
227	#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
228	#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
229	#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
230	#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
231
232	#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
233	#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
234	#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
235	#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
236
237	#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
238	#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
239	#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
240	#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
241
242	#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
243	#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
244	#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
245	#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
246
247	#define this_cpu_read_1(pcp) _percpu_read(pcp)
248	#define this_cpu_read_2(pcp) _percpu_read(pcp)
249	#define this_cpu_read_4(pcp) _percpu_read(pcp)
250	#define this_cpu_read_8(pcp) _percpu_read(pcp)
251
252	#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
253	#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
254	#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
255	#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
256
257	#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
258	#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
259	#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
260	#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
261
71586276 WD	262	#include <asm-generic/percpu.h>
	263
	264	#endif /* __ASM_PERCPU_H */