Commit | Line | Data |
---|---|---|
1965aae3 PA |
1 | #ifndef _ASM_X86_CMPXCHG_32_H |
2 | #define _ASM_X86_CMPXCHG_32_H | |
a436ed9c JD |
3 | |
4 | #include <linux/bitops.h> /* for LOCK_PREFIX */ | |
5 | ||
2d9ce177 AK |
6 | /* |
7 | * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you | |
8 | * you need to test for the feature in boot_cpu_data. | |
9 | */ | |
10 | ||
f3834b9e PZ |
11 | extern void __xchg_wrong_size(void); |
12 | ||
13 | /* | |
4532b305 PA |
14 | * Note: no "lock" prefix even on SMP: xchg always implies lock anyway. |
15 | * Since this is generally used to protect other memory information, we | |
16 | * use "asm volatile" and "memory" clobbers to prevent gcc from moving | |
17 | * information around. | |
f3834b9e | 18 | */ |
f3834b9e PZ |
19 | #define __xchg(x, ptr, size) \ |
20 | ({ \ | |
21 | __typeof(*(ptr)) __x = (x); \ | |
22 | switch (size) { \ | |
23 | case 1: \ | |
4532b305 PA |
24 | { \ |
25 | volatile u8 *__ptr = (volatile u8 *)(ptr); \ | |
26 | asm volatile("xchgb %0,%1" \ | |
27 | : "=q" (__x), "+m" (*__ptr) \ | |
113fc5a6 | 28 | : "0" (__x) \ |
f3834b9e PZ |
29 | : "memory"); \ |
30 | break; \ | |
4532b305 | 31 | } \ |
f3834b9e | 32 | case 2: \ |
4532b305 PA |
33 | { \ |
34 | volatile u16 *__ptr = (volatile u16 *)(ptr); \ | |
35 | asm volatile("xchgw %0,%1" \ | |
36 | : "=r" (__x), "+m" (*__ptr) \ | |
113fc5a6 | 37 | : "0" (__x) \ |
f3834b9e PZ |
38 | : "memory"); \ |
39 | break; \ | |
4532b305 | 40 | } \ |
f3834b9e | 41 | case 4: \ |
4532b305 PA |
42 | { \ |
43 | volatile u32 *__ptr = (volatile u32 *)(ptr); \ | |
f3834b9e | 44 | asm volatile("xchgl %0,%1" \ |
4532b305 | 45 | : "=r" (__x), "+m" (*__ptr) \ |
113fc5a6 | 46 | : "0" (__x) \ |
f3834b9e PZ |
47 | : "memory"); \ |
48 | break; \ | |
4532b305 | 49 | } \ |
f3834b9e PZ |
50 | default: \ |
51 | __xchg_wrong_size(); \ | |
52 | } \ | |
53 | __x; \ | |
54 | }) | |
55 | ||
56 | #define xchg(ptr, v) \ | |
57 | __xchg((v), (ptr), sizeof(*ptr)) | |
58 | ||
a436ed9c | 59 | /* |
69309a05 PA |
60 | * CMPXCHG8B only writes to the target if we had the previous |
61 | * value in registers, otherwise it acts as a read and gives us the | |
62 | * "new previous" value. That is why there is a loop. Preloading | |
63 | * EDX:EAX is a performance optimization: in the common case it means | |
64 | * we need only one locked operation. | |
a436ed9c | 65 | * |
69309a05 PA |
66 | * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very |
67 | * least an FPU save and/or %cr0.ts manipulation. | |
68 | * | |
69 | * cmpxchg8b must be used with the lock prefix here to allow the | |
70 | * instruction to be executed atomically. We need to have the reader | |
71 | * side to see the coherent 64bit value. | |
a436ed9c | 72 | */ |
69309a05 | 73 | static inline void set_64bit(volatile u64 *ptr, u64 value) |
a436ed9c | 74 | { |
69309a05 PA |
75 | u32 low = value; |
76 | u32 high = value >> 32; | |
77 | u64 prev = *ptr; | |
78 | ||
8121019c | 79 | asm volatile("\n1:\t" |
69309a05 | 80 | LOCK_PREFIX "cmpxchg8b %0\n\t" |
8121019c | 81 | "jnz 1b" |
69309a05 PA |
82 | : "=m" (*ptr), "+A" (prev) |
83 | : "b" (low), "c" (high) | |
84 | : "memory"); | |
a436ed9c | 85 | } |
a436ed9c | 86 | |
f3834b9e | 87 | extern void __cmpxchg_wrong_size(void); |
a436ed9c JD |
88 | |
89 | /* | |
90 | * Atomic compare and exchange. Compare OLD with MEM, if identical, | |
91 | * store NEW in MEM. Return the initial value in MEM. Success is | |
92 | * indicated by comparing RETURN with OLD. | |
93 | */ | |
f3834b9e PZ |
94 | #define __raw_cmpxchg(ptr, old, new, size, lock) \ |
95 | ({ \ | |
96 | __typeof__(*(ptr)) __ret; \ | |
97 | __typeof__(*(ptr)) __old = (old); \ | |
98 | __typeof__(*(ptr)) __new = (new); \ | |
99 | switch (size) { \ | |
100 | case 1: \ | |
4532b305 PA |
101 | { \ |
102 | volatile u8 *__ptr = (volatile u8 *)(ptr); \ | |
103 | asm volatile(lock "cmpxchgb %2,%1" \ | |
104 | : "=a" (__ret), "+m" (*__ptr) \ | |
113fc5a6 | 105 | : "q" (__new), "0" (__old) \ |
f3834b9e PZ |
106 | : "memory"); \ |
107 | break; \ | |
4532b305 | 108 | } \ |
f3834b9e | 109 | case 2: \ |
4532b305 PA |
110 | { \ |
111 | volatile u16 *__ptr = (volatile u16 *)(ptr); \ | |
112 | asm volatile(lock "cmpxchgw %2,%1" \ | |
113 | : "=a" (__ret), "+m" (*__ptr) \ | |
113fc5a6 | 114 | : "r" (__new), "0" (__old) \ |
f3834b9e PZ |
115 | : "memory"); \ |
116 | break; \ | |
4532b305 | 117 | } \ |
f3834b9e | 118 | case 4: \ |
4532b305 PA |
119 | { \ |
120 | volatile u32 *__ptr = (volatile u32 *)(ptr); \ | |
113fc5a6 | 121 | asm volatile(lock "cmpxchgl %2,%1" \ |
4532b305 | 122 | : "=a" (__ret), "+m" (*__ptr) \ |
113fc5a6 | 123 | : "r" (__new), "0" (__old) \ |
f3834b9e PZ |
124 | : "memory"); \ |
125 | break; \ | |
4532b305 | 126 | } \ |
f3834b9e PZ |
127 | default: \ |
128 | __cmpxchg_wrong_size(); \ | |
129 | } \ | |
130 | __ret; \ | |
131 | }) | |
132 | ||
133 | #define __cmpxchg(ptr, old, new, size) \ | |
134 | __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX) | |
135 | ||
136 | #define __sync_cmpxchg(ptr, old, new, size) \ | |
137 | __raw_cmpxchg((ptr), (old), (new), (size), "lock; ") | |
138 | ||
139 | #define __cmpxchg_local(ptr, old, new, size) \ | |
140 | __raw_cmpxchg((ptr), (old), (new), (size), "") | |
a436ed9c JD |
141 | |
142 | #ifdef CONFIG_X86_CMPXCHG | |
143 | #define __HAVE_ARCH_CMPXCHG 1 | |
f3834b9e PZ |
144 | |
145 | #define cmpxchg(ptr, old, new) \ | |
146 | __cmpxchg((ptr), (old), (new), sizeof(*ptr)) | |
147 | ||
148 | #define sync_cmpxchg(ptr, old, new) \ | |
149 | __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr)) | |
150 | ||
151 | #define cmpxchg_local(ptr, old, new) \ | |
152 | __cmpxchg_local((ptr), (old), (new), sizeof(*ptr)) | |
2c0b8a75 MD |
153 | #endif |
154 | ||
155 | #ifdef CONFIG_X86_CMPXCHG64 | |
8121019c JP |
156 | #define cmpxchg64(ptr, o, n) \ |
157 | ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ | |
158 | (unsigned long long)(n))) | |
159 | #define cmpxchg64_local(ptr, o, n) \ | |
160 | ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ | |
161 | (unsigned long long)(n))) | |
a436ed9c JD |
162 | #endif |
163 | ||
4532b305 | 164 | static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) |
2c0b8a75 | 165 | { |
4532b305 | 166 | u64 prev; |
113fc5a6 PA |
167 | asm volatile(LOCK_PREFIX "cmpxchg8b %1" |
168 | : "=A" (prev), | |
4532b305 PA |
169 | "+m" (*ptr) |
170 | : "b" ((u32)new), | |
171 | "c" ((u32)(new >> 32)), | |
113fc5a6 | 172 | "0" (old) |
8121019c | 173 | : "memory"); |
2c0b8a75 MD |
174 | return prev; |
175 | } | |
176 | ||
4532b305 | 177 | static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) |
2c0b8a75 | 178 | { |
4532b305 | 179 | u64 prev; |
113fc5a6 PA |
180 | asm volatile("cmpxchg8b %1" |
181 | : "=A" (prev), | |
4532b305 PA |
182 | "+m" (*ptr) |
183 | : "b" ((u32)new), | |
184 | "c" ((u32)(new >> 32)), | |
113fc5a6 | 185 | "0" (old) |
8121019c | 186 | : "memory"); |
2c0b8a75 MD |
187 | return prev; |
188 | } | |
189 | ||
a436ed9c JD |
190 | #ifndef CONFIG_X86_CMPXCHG |
191 | /* | |
192 | * Building a kernel capable running on 80386. It may be necessary to | |
193 | * simulate the cmpxchg on the 80386 CPU. For that purpose we define | |
194 | * a function for each of the sizes we support. | |
195 | */ | |
196 | ||
197 | extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8); | |
198 | extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16); | |
199 | extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32); | |
200 | ||
201 | static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old, | |
8121019c | 202 | unsigned long new, int size) |
a436ed9c JD |
203 | { |
204 | switch (size) { | |
205 | case 1: | |
206 | return cmpxchg_386_u8(ptr, old, new); | |
207 | case 2: | |
208 | return cmpxchg_386_u16(ptr, old, new); | |
209 | case 4: | |
210 | return cmpxchg_386_u32(ptr, old, new); | |
211 | } | |
212 | return old; | |
213 | } | |
214 | ||
2c0b8a75 | 215 | #define cmpxchg(ptr, o, n) \ |
a436ed9c JD |
216 | ({ \ |
217 | __typeof__(*(ptr)) __ret; \ | |
218 | if (likely(boot_cpu_data.x86 > 3)) \ | |
3078b79d MD |
219 | __ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \ |
220 | (unsigned long)(o), (unsigned long)(n), \ | |
221 | sizeof(*(ptr))); \ | |
a436ed9c | 222 | else \ |
3078b79d MD |
223 | __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ |
224 | (unsigned long)(o), (unsigned long)(n), \ | |
225 | sizeof(*(ptr))); \ | |
a436ed9c JD |
226 | __ret; \ |
227 | }) | |
2c0b8a75 | 228 | #define cmpxchg_local(ptr, o, n) \ |
a436ed9c JD |
229 | ({ \ |
230 | __typeof__(*(ptr)) __ret; \ | |
231 | if (likely(boot_cpu_data.x86 > 3)) \ | |
3078b79d MD |
232 | __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \ |
233 | (unsigned long)(o), (unsigned long)(n), \ | |
234 | sizeof(*(ptr))); \ | |
a436ed9c | 235 | else \ |
3078b79d MD |
236 | __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ |
237 | (unsigned long)(o), (unsigned long)(n), \ | |
238 | sizeof(*(ptr))); \ | |
a436ed9c JD |
239 | __ret; \ |
240 | }) | |
241 | #endif | |
242 | ||
2c0b8a75 MD |
243 | #ifndef CONFIG_X86_CMPXCHG64 |
244 | /* | |
245 | * Building a kernel capable running on 80386 and 80486. It may be necessary | |
246 | * to simulate the cmpxchg8b on the 80386 and 80486 CPU. | |
247 | */ | |
a436ed9c | 248 | |
79e1dd05 AV |
249 | #define cmpxchg64(ptr, o, n) \ |
250 | ({ \ | |
251 | __typeof__(*(ptr)) __ret; \ | |
252 | __typeof__(*(ptr)) __old = (o); \ | |
253 | __typeof__(*(ptr)) __new = (n); \ | |
9c76b384 LB |
254 | alternative_io(LOCK_PREFIX_HERE \ |
255 | "call cmpxchg8b_emu", \ | |
79e1dd05 AV |
256 | "lock; cmpxchg8b (%%esi)" , \ |
257 | X86_FEATURE_CX8, \ | |
258 | "=A" (__ret), \ | |
259 | "S" ((ptr)), "0" (__old), \ | |
260 | "b" ((unsigned int)__new), \ | |
261 | "c" ((unsigned int)(__new>>32)) \ | |
262 | : "memory"); \ | |
263 | __ret; }) | |
264 | ||
265 | ||
a378d933 PA |
266 | #define cmpxchg64_local(ptr, o, n) \ |
267 | ({ \ | |
268 | __typeof__(*(ptr)) __ret; \ | |
269 | __typeof__(*(ptr)) __old = (o); \ | |
270 | __typeof__(*(ptr)) __new = (n); \ | |
271 | alternative_io("call cmpxchg8b_emu", \ | |
272 | "cmpxchg8b (%%esi)" , \ | |
273 | X86_FEATURE_CX8, \ | |
274 | "=A" (__ret), \ | |
275 | "S" ((ptr)), "0" (__old), \ | |
276 | "b" ((unsigned int)__new), \ | |
277 | "c" ((unsigned int)(__new>>32)) \ | |
278 | : "memory"); \ | |
279 | __ret; }) | |
2c0b8a75 MD |
280 | |
281 | #endif | |
a436ed9c | 282 | |
1965aae3 | 283 | #endif /* _ASM_X86_CMPXCHG_32_H */ |