Commit | Line | Data |
---|---|---|
9f97da78 DH |
1 | #ifndef __ASM_ARM_CMPXCHG_H |
2 | #define __ASM_ARM_CMPXCHG_H | |
3 | ||
4 | #include <linux/irqflags.h> | |
c32ffce0 | 5 | #include <linux/prefetch.h> |
9f97da78 DH |
6 | #include <asm/barrier.h> |
7 | ||
8 | #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) | |
9 | /* | |
10 | * On the StrongARM, "swp" is terminally broken since it bypasses the | |
11 | * cache totally. This means that the cache becomes inconsistent, and, | |
12 | * since we use normal loads/stores as well, this is really bad. | |
13 | * Typically, this causes oopsen in filp_close, but could have other, | |
14 | * more disastrous effects. There are two work-arounds: | |
15 | * 1. Disable interrupts and emulate the atomic swap | |
16 | * 2. Clean the cache, perform atomic swap, flush the cache | |
17 | * | |
18 | * We choose (1) since its the "easiest" to achieve here and is not | |
19 | * dependent on the processor type. | |
20 | * | |
21 | * NOTE that this solution won't work on an SMP system, so explcitly | |
22 | * forbid it here. | |
23 | */ | |
24 | #define swp_is_buggy | |
25 | #endif | |
26 | ||
27 | static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) | |
28 | { | |
29 | extern void __bad_xchg(volatile void *, int); | |
30 | unsigned long ret; | |
31 | #ifdef swp_is_buggy | |
32 | unsigned long flags; | |
33 | #endif | |
34 | #if __LINUX_ARM_ARCH__ >= 6 | |
35 | unsigned int tmp; | |
36 | #endif | |
37 | ||
38 | smp_mb(); | |
c32ffce0 | 39 | prefetchw((const void *)ptr); |
9f97da78 DH |
40 | |
41 | switch (size) { | |
42 | #if __LINUX_ARM_ARCH__ >= 6 | |
43 | case 1: | |
44 | asm volatile("@ __xchg1\n" | |
45 | "1: ldrexb %0, [%3]\n" | |
46 | " strexb %1, %2, [%3]\n" | |
47 | " teq %1, #0\n" | |
48 | " bne 1b" | |
49 | : "=&r" (ret), "=&r" (tmp) | |
50 | : "r" (x), "r" (ptr) | |
51 | : "memory", "cc"); | |
52 | break; | |
53 | case 4: | |
54 | asm volatile("@ __xchg4\n" | |
55 | "1: ldrex %0, [%3]\n" | |
56 | " strex %1, %2, [%3]\n" | |
57 | " teq %1, #0\n" | |
58 | " bne 1b" | |
59 | : "=&r" (ret), "=&r" (tmp) | |
60 | : "r" (x), "r" (ptr) | |
61 | : "memory", "cc"); | |
62 | break; | |
63 | #elif defined(swp_is_buggy) | |
64 | #ifdef CONFIG_SMP | |
65 | #error SMP is not supported on this platform | |
66 | #endif | |
67 | case 1: | |
68 | raw_local_irq_save(flags); | |
69 | ret = *(volatile unsigned char *)ptr; | |
70 | *(volatile unsigned char *)ptr = x; | |
71 | raw_local_irq_restore(flags); | |
72 | break; | |
73 | ||
74 | case 4: | |
75 | raw_local_irq_save(flags); | |
76 | ret = *(volatile unsigned long *)ptr; | |
77 | *(volatile unsigned long *)ptr = x; | |
78 | raw_local_irq_restore(flags); | |
79 | break; | |
80 | #else | |
81 | case 1: | |
82 | asm volatile("@ __xchg1\n" | |
83 | " swpb %0, %1, [%2]" | |
84 | : "=&r" (ret) | |
85 | : "r" (x), "r" (ptr) | |
86 | : "memory", "cc"); | |
87 | break; | |
88 | case 4: | |
89 | asm volatile("@ __xchg4\n" | |
90 | " swp %0, %1, [%2]" | |
91 | : "=&r" (ret) | |
92 | : "r" (x), "r" (ptr) | |
93 | : "memory", "cc"); | |
94 | break; | |
95 | #endif | |
96 | default: | |
31cd08c3 | 97 | /* Cause a link-time error, the xchg() size is not supported */ |
9f97da78 DH |
98 | __bad_xchg(ptr, size), ret = 0; |
99 | break; | |
100 | } | |
101 | smp_mb(); | |
102 | ||
103 | return ret; | |
104 | } | |
105 | ||
e001bbae RK |
106 | #define xchg(ptr, x) ({ \ |
107 | (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ | |
108 | sizeof(*(ptr))); \ | |
109 | }) | |
9f97da78 DH |
110 | |
111 | #include <asm-generic/cmpxchg-local.h> | |
112 | ||
113 | #if __LINUX_ARM_ARCH__ < 6 | |
114 | /* min ARCH < ARMv6 */ | |
115 | ||
116 | #ifdef CONFIG_SMP | |
117 | #error "SMP is not supported on this platform" | |
118 | #endif | |
119 | ||
120 | /* | |
121 | * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make | |
122 | * them available. | |
123 | */ | |
e001bbae RK |
124 | #define cmpxchg_local(ptr, o, n) ({ \ |
125 | (__typeof(*ptr))__cmpxchg_local_generic((ptr), \ | |
126 | (unsigned long)(o), \ | |
127 | (unsigned long)(n), \ | |
128 | sizeof(*(ptr))); \ | |
129 | }) | |
130 | ||
9f97da78 DH |
131 | #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) |
132 | ||
9f97da78 | 133 | #include <asm-generic/cmpxchg.h> |
9f97da78 DH |
134 | |
135 | #else /* min ARCH >= ARMv6 */ | |
136 | ||
137 | extern void __bad_cmpxchg(volatile void *ptr, int size); | |
138 | ||
139 | /* | |
140 | * cmpxchg only support 32-bits operands on ARMv6. | |
141 | */ | |
142 | ||
143 | static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, | |
144 | unsigned long new, int size) | |
145 | { | |
146 | unsigned long oldval, res; | |
147 | ||
c32ffce0 WD |
148 | prefetchw((const void *)ptr); |
149 | ||
9f97da78 DH |
150 | switch (size) { |
151 | #ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ | |
152 | case 1: | |
153 | do { | |
154 | asm volatile("@ __cmpxchg1\n" | |
155 | " ldrexb %1, [%2]\n" | |
156 | " mov %0, #0\n" | |
157 | " teq %1, %3\n" | |
158 | " strexbeq %0, %4, [%2]\n" | |
159 | : "=&r" (res), "=&r" (oldval) | |
160 | : "r" (ptr), "Ir" (old), "r" (new) | |
161 | : "memory", "cc"); | |
162 | } while (res); | |
163 | break; | |
164 | case 2: | |
165 | do { | |
166 | asm volatile("@ __cmpxchg1\n" | |
167 | " ldrexh %1, [%2]\n" | |
168 | " mov %0, #0\n" | |
169 | " teq %1, %3\n" | |
170 | " strexheq %0, %4, [%2]\n" | |
171 | : "=&r" (res), "=&r" (oldval) | |
172 | : "r" (ptr), "Ir" (old), "r" (new) | |
173 | : "memory", "cc"); | |
174 | } while (res); | |
175 | break; | |
176 | #endif | |
177 | case 4: | |
178 | do { | |
179 | asm volatile("@ __cmpxchg4\n" | |
180 | " ldrex %1, [%2]\n" | |
181 | " mov %0, #0\n" | |
182 | " teq %1, %3\n" | |
183 | " strexeq %0, %4, [%2]\n" | |
184 | : "=&r" (res), "=&r" (oldval) | |
185 | : "r" (ptr), "Ir" (old), "r" (new) | |
186 | : "memory", "cc"); | |
187 | } while (res); | |
188 | break; | |
189 | default: | |
190 | __bad_cmpxchg(ptr, size); | |
191 | oldval = 0; | |
192 | } | |
193 | ||
194 | return oldval; | |
195 | } | |
196 | ||
197 | static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, | |
198 | unsigned long new, int size) | |
199 | { | |
200 | unsigned long ret; | |
201 | ||
202 | smp_mb(); | |
203 | ret = __cmpxchg(ptr, old, new, size); | |
204 | smp_mb(); | |
205 | ||
206 | return ret; | |
207 | } | |
208 | ||
e001bbae RK |
209 | #define cmpxchg(ptr,o,n) ({ \ |
210 | (__typeof__(*(ptr)))__cmpxchg_mb((ptr), \ | |
211 | (unsigned long)(o), \ | |
212 | (unsigned long)(n), \ | |
213 | sizeof(*(ptr))); \ | |
214 | }) | |
9f97da78 DH |
215 | |
216 | static inline unsigned long __cmpxchg_local(volatile void *ptr, | |
217 | unsigned long old, | |
218 | unsigned long new, int size) | |
219 | { | |
220 | unsigned long ret; | |
221 | ||
222 | switch (size) { | |
223 | #ifdef CONFIG_CPU_V6 /* min ARCH == ARMv6 */ | |
224 | case 1: | |
225 | case 2: | |
226 | ret = __cmpxchg_local_generic(ptr, old, new, size); | |
227 | break; | |
228 | #endif | |
229 | default: | |
230 | ret = __cmpxchg(ptr, old, new, size); | |
231 | } | |
232 | ||
233 | return ret; | |
234 | } | |
235 | ||
e001bbae RK |
236 | #define cmpxchg_local(ptr, o, n) ({ \ |
237 | (__typeof(*ptr))__cmpxchg_local((ptr), \ | |
238 | (unsigned long)(o), \ | |
239 | (unsigned long)(n), \ | |
240 | sizeof(*(ptr))); \ | |
241 | }) | |
242 | ||
2523c67b WD |
243 | static inline unsigned long long __cmpxchg64(unsigned long long *ptr, |
244 | unsigned long long old, | |
245 | unsigned long long new) | |
246 | { | |
247 | unsigned long long oldval; | |
248 | unsigned long res; | |
249 | ||
c32ffce0 WD |
250 | prefetchw(ptr); |
251 | ||
2523c67b WD |
252 | __asm__ __volatile__( |
253 | "1: ldrexd %1, %H1, [%3]\n" | |
254 | " teq %1, %4\n" | |
255 | " teqeq %H1, %H4\n" | |
256 | " bne 2f\n" | |
257 | " strexd %0, %5, %H5, [%3]\n" | |
258 | " teq %0, #0\n" | |
259 | " bne 1b\n" | |
260 | "2:" | |
261 | : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) | |
262 | : "r" (ptr), "r" (old), "r" (new) | |
263 | : "cc"); | |
264 | ||
265 | return oldval; | |
266 | } | |
267 | ||
e001bbae RK |
268 | #define cmpxchg64_relaxed(ptr, o, n) ({ \ |
269 | (__typeof__(*(ptr)))__cmpxchg64((ptr), \ | |
270 | (unsigned long long)(o), \ | |
271 | (unsigned long long)(n)); \ | |
272 | }) | |
273 | ||
274 | #define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) | |
275 | ||
2523c67b WD |
276 | static inline unsigned long long __cmpxchg64_mb(unsigned long long *ptr, |
277 | unsigned long long old, | |
278 | unsigned long long new) | |
279 | { | |
280 | unsigned long long ret; | |
281 | ||
282 | smp_mb(); | |
283 | ret = __cmpxchg64(ptr, old, new); | |
284 | smp_mb(); | |
285 | ||
286 | return ret; | |
287 | } | |
288 | ||
e001bbae RK |
289 | #define cmpxchg64(ptr, o, n) ({ \ |
290 | (__typeof__(*(ptr)))__cmpxchg64_mb((ptr), \ | |
291 | (unsigned long long)(o), \ | |
292 | (unsigned long long)(n)); \ | |
293 | }) | |
775ebcc1 | 294 | |
9f97da78 DH |
295 | #endif /* __LINUX_ARM_ARCH__ >= 6 */ |
296 | ||
297 | #endif /* __ASM_ARM_CMPXCHG_H */ |