Commit | Line | Data |
---|---|---|
a2c9dbe8 HC |
1 | /* |
2 | * Copyright IBM Corp. 1999, 2011 | |
3 | * | |
4 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, | |
5 | */ | |
6 | ||
7 | #ifndef __ASM_CMPXCHG_H | |
8 | #define __ASM_CMPXCHG_H | |
9 | ||
10 | #include <linux/types.h> | |
11 | ||
12 | extern void __xchg_called_with_bad_pointer(void); | |
13 | ||
14 | static inline unsigned long __xchg(unsigned long x, void *ptr, int size) | |
15 | { | |
16 | unsigned long addr, old; | |
17 | int shift; | |
18 | ||
19 | switch (size) { | |
20 | case 1: | |
21 | addr = (unsigned long) ptr; | |
22 | shift = (3 ^ (addr & 3)) << 3; | |
23 | addr ^= addr & 3; | |
24 | asm volatile( | |
25 | " l %0,%4\n" | |
26 | "0: lr 0,%0\n" | |
27 | " nr 0,%3\n" | |
28 | " or 0,%2\n" | |
29 | " cs %0,0,%4\n" | |
30 | " jl 0b\n" | |
31 | : "=&d" (old), "=Q" (*(int *) addr) | |
32 | : "d" (x << shift), "d" (~(255 << shift)), | |
33 | "Q" (*(int *) addr) : "memory", "cc", "0"); | |
34 | return old >> shift; | |
35 | case 2: | |
36 | addr = (unsigned long) ptr; | |
37 | shift = (2 ^ (addr & 2)) << 3; | |
38 | addr ^= addr & 2; | |
39 | asm volatile( | |
40 | " l %0,%4\n" | |
41 | "0: lr 0,%0\n" | |
42 | " nr 0,%3\n" | |
43 | " or 0,%2\n" | |
44 | " cs %0,0,%4\n" | |
45 | " jl 0b\n" | |
46 | : "=&d" (old), "=Q" (*(int *) addr) | |
47 | : "d" (x << shift), "d" (~(65535 << shift)), | |
48 | "Q" (*(int *) addr) : "memory", "cc", "0"); | |
49 | return old >> shift; | |
50 | case 4: | |
51 | asm volatile( | |
52 | " l %0,%3\n" | |
53 | "0: cs %0,%2,%3\n" | |
54 | " jl 0b\n" | |
55 | : "=&d" (old), "=Q" (*(int *) ptr) | |
56 | : "d" (x), "Q" (*(int *) ptr) | |
57 | : "memory", "cc"); | |
58 | return old; | |
59 | #ifdef CONFIG_64BIT | |
60 | case 8: | |
61 | asm volatile( | |
62 | " lg %0,%3\n" | |
63 | "0: csg %0,%2,%3\n" | |
64 | " jl 0b\n" | |
65 | : "=&d" (old), "=m" (*(long *) ptr) | |
66 | : "d" (x), "Q" (*(long *) ptr) | |
67 | : "memory", "cc"); | |
68 | return old; | |
69 | #endif /* CONFIG_64BIT */ | |
70 | } | |
71 | __xchg_called_with_bad_pointer(); | |
72 | return x; | |
73 | } | |
74 | ||
75 | #define xchg(ptr, x) \ | |
76 | ({ \ | |
77 | __typeof__(*(ptr)) __ret; \ | |
78 | __ret = (__typeof__(*(ptr))) \ | |
79 | __xchg((unsigned long)(x), (void *)(ptr), sizeof(*(ptr)));\ | |
80 | __ret; \ | |
81 | }) | |
82 | ||
83 | /* | |
84 | * Atomic compare and exchange. Compare OLD with MEM, if identical, | |
85 | * store NEW in MEM. Return the initial value in MEM. Success is | |
86 | * indicated by comparing RETURN with OLD. | |
87 | */ | |
88 | ||
89 | #define __HAVE_ARCH_CMPXCHG | |
90 | ||
91 | extern void __cmpxchg_called_with_bad_pointer(void); | |
92 | ||
93 | static inline unsigned long __cmpxchg(void *ptr, unsigned long old, | |
94 | unsigned long new, int size) | |
95 | { | |
96 | unsigned long addr, prev, tmp; | |
97 | int shift; | |
98 | ||
99 | switch (size) { | |
100 | case 1: | |
101 | addr = (unsigned long) ptr; | |
102 | shift = (3 ^ (addr & 3)) << 3; | |
103 | addr ^= addr & 3; | |
104 | asm volatile( | |
105 | " l %0,%2\n" | |
106 | "0: nr %0,%5\n" | |
107 | " lr %1,%0\n" | |
108 | " or %0,%3\n" | |
109 | " or %1,%4\n" | |
110 | " cs %0,%1,%2\n" | |
111 | " jnl 1f\n" | |
112 | " xr %1,%0\n" | |
113 | " nr %1,%5\n" | |
114 | " jnz 0b\n" | |
115 | "1:" | |
116 | : "=&d" (prev), "=&d" (tmp), "=Q" (*(int *) ptr) | |
117 | : "d" (old << shift), "d" (new << shift), | |
118 | "d" (~(255 << shift)), "Q" (*(int *) ptr) | |
119 | : "memory", "cc"); | |
120 | return prev >> shift; | |
121 | case 2: | |
122 | addr = (unsigned long) ptr; | |
123 | shift = (2 ^ (addr & 2)) << 3; | |
124 | addr ^= addr & 2; | |
125 | asm volatile( | |
126 | " l %0,%2\n" | |
127 | "0: nr %0,%5\n" | |
128 | " lr %1,%0\n" | |
129 | " or %0,%3\n" | |
130 | " or %1,%4\n" | |
131 | " cs %0,%1,%2\n" | |
132 | " jnl 1f\n" | |
133 | " xr %1,%0\n" | |
134 | " nr %1,%5\n" | |
135 | " jnz 0b\n" | |
136 | "1:" | |
137 | : "=&d" (prev), "=&d" (tmp), "=Q" (*(int *) ptr) | |
138 | : "d" (old << shift), "d" (new << shift), | |
139 | "d" (~(65535 << shift)), "Q" (*(int *) ptr) | |
140 | : "memory", "cc"); | |
141 | return prev >> shift; | |
142 | case 4: | |
143 | asm volatile( | |
144 | " cs %0,%3,%1\n" | |
145 | : "=&d" (prev), "=Q" (*(int *) ptr) | |
146 | : "0" (old), "d" (new), "Q" (*(int *) ptr) | |
147 | : "memory", "cc"); | |
148 | return prev; | |
149 | #ifdef CONFIG_64BIT | |
150 | case 8: | |
151 | asm volatile( | |
152 | " csg %0,%3,%1\n" | |
153 | : "=&d" (prev), "=Q" (*(long *) ptr) | |
154 | : "0" (old), "d" (new), "Q" (*(long *) ptr) | |
155 | : "memory", "cc"); | |
156 | return prev; | |
157 | #endif /* CONFIG_64BIT */ | |
158 | } | |
159 | __cmpxchg_called_with_bad_pointer(); | |
160 | return old; | |
161 | } | |
162 | ||
163 | #define cmpxchg(ptr, o, n) \ | |
164 | ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ | |
165 | (unsigned long)(n), sizeof(*(ptr)))) | |
166 | ||
54eaae30 HC |
167 | #ifdef CONFIG_64BIT |
168 | #define cmpxchg64(ptr, o, n) \ | |
169 | ({ \ | |
54eaae30 HC |
170 | cmpxchg((ptr), (o), (n)); \ |
171 | }) | |
172 | #else /* CONFIG_64BIT */ | |
173 | static inline unsigned long long __cmpxchg64(void *ptr, | |
174 | unsigned long long old, | |
175 | unsigned long long new) | |
176 | { | |
177 | register_pair rp_old = {.pair = old}; | |
178 | register_pair rp_new = {.pair = new}; | |
179 | ||
180 | asm volatile( | |
181 | " cds %0,%2,%1" | |
182 | : "+&d" (rp_old), "=Q" (ptr) | |
183 | : "d" (rp_new), "Q" (ptr) | |
184 | : "cc"); | |
185 | return rp_old.pair; | |
186 | } | |
187 | #define cmpxchg64(ptr, o, n) \ | |
188 | ((__typeof__(*(ptr)))__cmpxchg64((ptr), \ | |
189 | (unsigned long long)(o), \ | |
190 | (unsigned long long)(n))) | |
191 | #endif /* CONFIG_64BIT */ | |
192 | ||
a2c9dbe8 HC |
193 | #include <asm-generic/cmpxchg-local.h> |
194 | ||
195 | static inline unsigned long __cmpxchg_local(void *ptr, | |
196 | unsigned long old, | |
197 | unsigned long new, int size) | |
198 | { | |
199 | switch (size) { | |
200 | case 1: | |
201 | case 2: | |
202 | case 4: | |
203 | #ifdef CONFIG_64BIT | |
204 | case 8: | |
205 | #endif | |
206 | return __cmpxchg(ptr, old, new, size); | |
207 | default: | |
208 | return __cmpxchg_local_generic(ptr, old, new, size); | |
209 | } | |
210 | ||
211 | return old; | |
212 | } | |
213 | ||
214 | /* | |
215 | * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make | |
216 | * them available. | |
217 | */ | |
218 | #define cmpxchg_local(ptr, o, n) \ | |
219 | ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ | |
220 | (unsigned long)(n), sizeof(*(ptr)))) | |
54eaae30 HC |
221 | |
222 | #define cmpxchg64_local(ptr, o, n) cmpxchg64((ptr), (o), (n)) | |
a2c9dbe8 HC |
223 | |
224 | #endif /* __ASM_CMPXCHG_H */ |