Commit | Line | Data |
---|---|---|
1c54d770 JF |
1 | #ifndef _ASM_X86_BITOPS_H |
2 | #define _ASM_X86_BITOPS_H | |
3 | ||
4 | /* | |
5 | * Copyright 1992, Linus Torvalds. | |
6 | */ | |
7 | ||
8 | #ifndef _LINUX_BITOPS_H | |
9 | #error only <linux/bitops.h> can be included directly | |
10 | #endif | |
11 | ||
12 | #include <linux/compiler.h> | |
13 | #include <asm/alternative.h> | |
14 | ||
15 | /* | |
16 | * These have to be done with inline assembly: that way the bit-setting | |
17 | * is guaranteed to be atomic. All bit operations return 0 if the bit | |
18 | * was cleared before the operation and != 0 if it was not. | |
19 | * | |
20 | * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1). | |
21 | */ | |
22 | ||
23 | #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1) | |
24 | /* Technically wrong, but this avoids compilation errors on some gcc | |
25 | versions. */ | |
286275c9 JP |
26 | #define ADDR "=m" (*(volatile long *)addr) |
27 | #define BIT_ADDR "=m" (((volatile int *)addr)[nr >> 5]) | |
1c54d770 JF |
28 | #else |
29 | #define ADDR "+m" (*(volatile long *) addr) | |
286275c9 | 30 | #define BIT_ADDR "+m" (((volatile int *)addr)[nr >> 5]) |
1c54d770 | 31 | #endif |
286275c9 | 32 | #define BASE_ADDR "m" (*(volatile int *)addr) |
1c54d770 JF |
33 | |
34 | /** | |
35 | * set_bit - Atomically set a bit in memory | |
36 | * @nr: the bit to set | |
37 | * @addr: the address to start counting from | |
38 | * | |
39 | * This function is atomic and may not be reordered. See __set_bit() | |
40 | * if you do not require the atomic guarantees. | |
41 | * | |
42 | * Note: there are no guarantees that this function will not be reordered | |
43 | * on non x86 architectures, so if you are writing portable code, | |
44 | * make sure not to rely on its reordering guarantees. | |
45 | * | |
46 | * Note that @nr may be almost arbitrarily large; this function is not | |
47 | * restricted to acting on a single-word quantity. | |
48 | */ | |
26996dd2 | 49 | static inline void set_bit(int nr, volatile void *addr) |
1c54d770 | 50 | { |
286275c9 | 51 | asm volatile(LOCK_PREFIX "bts %1,%0" : ADDR : "Ir" (nr) : "memory"); |
1c54d770 JF |
52 | } |
53 | ||
54 | /** | |
55 | * __set_bit - Set a bit in memory | |
56 | * @nr: the bit to set | |
57 | * @addr: the address to start counting from | |
58 | * | |
59 | * Unlike set_bit(), this function is non-atomic and may be reordered. | |
60 | * If it's called on the same region of memory simultaneously, the effect | |
61 | * may be that only one operation succeeds. | |
62 | */ | |
26996dd2 | 63 | static inline void __set_bit(int nr, volatile void *addr) |
1c54d770 JF |
64 | { |
65 | asm volatile("bts %1,%0" | |
66 | : ADDR | |
67 | : "Ir" (nr) : "memory"); | |
68 | } | |
69 | ||
70 | ||
71 | /** | |
72 | * clear_bit - Clears a bit in memory | |
73 | * @nr: Bit to clear | |
74 | * @addr: Address to start counting from | |
75 | * | |
76 | * clear_bit() is atomic and may not be reordered. However, it does | |
77 | * not contain a memory barrier, so if it is used for locking purposes, | |
78 | * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() | |
79 | * in order to ensure changes are visible on other processors. | |
80 | */ | |
26996dd2 | 81 | static inline void clear_bit(int nr, volatile void *addr) |
1c54d770 | 82 | { |
286275c9 | 83 | asm volatile(LOCK_PREFIX "btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); |
1c54d770 JF |
84 | } |
85 | ||
86 | /* | |
87 | * clear_bit_unlock - Clears a bit in memory | |
88 | * @nr: Bit to clear | |
89 | * @addr: Address to start counting from | |
90 | * | |
91 | * clear_bit() is atomic and implies release semantics before the memory | |
92 | * operation. It can be used for an unlock. | |
93 | */ | |
26996dd2 | 94 | static inline void clear_bit_unlock(unsigned nr, volatile void *addr) |
1c54d770 JF |
95 | { |
96 | barrier(); | |
97 | clear_bit(nr, addr); | |
98 | } | |
99 | ||
26996dd2 | 100 | static inline void __clear_bit(int nr, volatile void *addr) |
1c54d770 | 101 | { |
709f744f | 102 | asm volatile("btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); |
1c54d770 JF |
103 | } |
104 | ||
105 | /* | |
106 | * __clear_bit_unlock - Clears a bit in memory | |
107 | * @nr: Bit to clear | |
108 | * @addr: Address to start counting from | |
109 | * | |
110 | * __clear_bit() is non-atomic and implies release semantics before the memory | |
111 | * operation. It can be used for an unlock if no other CPUs can concurrently | |
112 | * modify other bits in the word. | |
113 | * | |
114 | * No memory barrier is required here, because x86 cannot reorder stores past | |
115 | * older loads. Same principle as spin_unlock. | |
116 | */ | |
26996dd2 | 117 | static inline void __clear_bit_unlock(unsigned nr, volatile void *addr) |
1c54d770 JF |
118 | { |
119 | barrier(); | |
120 | __clear_bit(nr, addr); | |
121 | } | |
122 | ||
123 | #define smp_mb__before_clear_bit() barrier() | |
124 | #define smp_mb__after_clear_bit() barrier() | |
125 | ||
126 | /** | |
127 | * __change_bit - Toggle a bit in memory | |
128 | * @nr: the bit to change | |
129 | * @addr: the address to start counting from | |
130 | * | |
131 | * Unlike change_bit(), this function is non-atomic and may be reordered. | |
132 | * If it's called on the same region of memory simultaneously, the effect | |
133 | * may be that only one operation succeeds. | |
134 | */ | |
26996dd2 | 135 | static inline void __change_bit(int nr, volatile void *addr) |
1c54d770 | 136 | { |
709f744f | 137 | asm volatile("btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); |
1c54d770 JF |
138 | } |
139 | ||
140 | /** | |
141 | * change_bit - Toggle a bit in memory | |
142 | * @nr: Bit to change | |
143 | * @addr: Address to start counting from | |
144 | * | |
145 | * change_bit() is atomic and may not be reordered. | |
146 | * Note that @nr may be almost arbitrarily large; this function is not | |
147 | * restricted to acting on a single-word quantity. | |
148 | */ | |
26996dd2 | 149 | static inline void change_bit(int nr, volatile void *addr) |
1c54d770 | 150 | { |
286275c9 | 151 | asm volatile(LOCK_PREFIX "btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR); |
1c54d770 JF |
152 | } |
153 | ||
154 | /** | |
155 | * test_and_set_bit - Set a bit and return its old value | |
156 | * @nr: Bit to set | |
157 | * @addr: Address to count from | |
158 | * | |
159 | * This operation is atomic and cannot be reordered. | |
160 | * It also implies a memory barrier. | |
161 | */ | |
26996dd2 | 162 | static inline int test_and_set_bit(int nr, volatile void *addr) |
1c54d770 JF |
163 | { |
164 | int oldbit; | |
165 | ||
166 | asm volatile(LOCK_PREFIX "bts %2,%1\n\t" | |
286275c9 | 167 | "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); |
1c54d770 JF |
168 | |
169 | return oldbit; | |
170 | } | |
171 | ||
172 | /** | |
173 | * test_and_set_bit_lock - Set a bit and return its old value for lock | |
174 | * @nr: Bit to set | |
175 | * @addr: Address to count from | |
176 | * | |
177 | * This is the same as test_and_set_bit on x86. | |
178 | */ | |
26996dd2 | 179 | static inline int test_and_set_bit_lock(int nr, volatile void *addr) |
1c54d770 JF |
180 | { |
181 | return test_and_set_bit(nr, addr); | |
182 | } | |
183 | ||
184 | /** | |
185 | * __test_and_set_bit - Set a bit and return its old value | |
186 | * @nr: Bit to set | |
187 | * @addr: Address to count from | |
188 | * | |
189 | * This operation is non-atomic and can be reordered. | |
190 | * If two examples of this operation race, one can appear to succeed | |
191 | * but actually fail. You must protect multiple accesses with a lock. | |
192 | */ | |
26996dd2 | 193 | static inline int __test_and_set_bit(int nr, volatile void *addr) |
1c54d770 JF |
194 | { |
195 | int oldbit; | |
196 | ||
709f744f JB |
197 | asm volatile("bts %2,%3\n\t" |
198 | "sbb %0,%0" | |
286275c9 | 199 | : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR); |
1c54d770 JF |
200 | return oldbit; |
201 | } | |
202 | ||
203 | /** | |
204 | * test_and_clear_bit - Clear a bit and return its old value | |
205 | * @nr: Bit to clear | |
206 | * @addr: Address to count from | |
207 | * | |
208 | * This operation is atomic and cannot be reordered. | |
209 | * It also implies a memory barrier. | |
210 | */ | |
26996dd2 | 211 | static inline int test_and_clear_bit(int nr, volatile void *addr) |
1c54d770 JF |
212 | { |
213 | int oldbit; | |
214 | ||
215 | asm volatile(LOCK_PREFIX "btr %2,%1\n\t" | |
216 | "sbb %0,%0" | |
286275c9 | 217 | : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); |
1c54d770 JF |
218 | |
219 | return oldbit; | |
220 | } | |
221 | ||
222 | /** | |
223 | * __test_and_clear_bit - Clear a bit and return its old value | |
224 | * @nr: Bit to clear | |
225 | * @addr: Address to count from | |
226 | * | |
227 | * This operation is non-atomic and can be reordered. | |
228 | * If two examples of this operation race, one can appear to succeed | |
229 | * but actually fail. You must protect multiple accesses with a lock. | |
230 | */ | |
26996dd2 | 231 | static inline int __test_and_clear_bit(int nr, volatile void *addr) |
1c54d770 JF |
232 | { |
233 | int oldbit; | |
234 | ||
709f744f | 235 | asm volatile("btr %2,%3\n\t" |
1c54d770 | 236 | "sbb %0,%0" |
286275c9 | 237 | : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR); |
1c54d770 JF |
238 | return oldbit; |
239 | } | |
240 | ||
241 | /* WARNING: non atomic and it can be reordered! */ | |
26996dd2 | 242 | static inline int __test_and_change_bit(int nr, volatile void *addr) |
1c54d770 JF |
243 | { |
244 | int oldbit; | |
245 | ||
709f744f | 246 | asm volatile("btc %2,%3\n\t" |
1c54d770 | 247 | "sbb %0,%0" |
286275c9 | 248 | : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR); |
1c54d770 JF |
249 | |
250 | return oldbit; | |
251 | } | |
252 | ||
253 | /** | |
254 | * test_and_change_bit - Change a bit and return its old value | |
255 | * @nr: Bit to change | |
256 | * @addr: Address to count from | |
257 | * | |
258 | * This operation is atomic and cannot be reordered. | |
259 | * It also implies a memory barrier. | |
260 | */ | |
26996dd2 | 261 | static inline int test_and_change_bit(int nr, volatile void *addr) |
1c54d770 JF |
262 | { |
263 | int oldbit; | |
264 | ||
265 | asm volatile(LOCK_PREFIX "btc %2,%1\n\t" | |
266 | "sbb %0,%0" | |
286275c9 | 267 | : "=r" (oldbit), ADDR : "Ir" (nr) : "memory"); |
1c54d770 JF |
268 | |
269 | return oldbit; | |
270 | } | |
271 | ||
26996dd2 | 272 | static inline int constant_test_bit(int nr, const volatile void *addr) |
1c54d770 | 273 | { |
26996dd2 GOC |
274 | return ((1UL << (nr % BITS_PER_LONG)) & |
275 | (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0; | |
1c54d770 JF |
276 | } |
277 | ||
26996dd2 | 278 | static inline int variable_test_bit(int nr, volatile const void *addr) |
1c54d770 JF |
279 | { |
280 | int oldbit; | |
281 | ||
709f744f | 282 | asm volatile("bt %2,%3\n\t" |
1c54d770 JF |
283 | "sbb %0,%0" |
284 | : "=r" (oldbit) | |
709f744f JB |
285 | : "m" (((volatile const int *)addr)[nr >> 5]), |
286 | "Ir" (nr), BASE_ADDR); | |
1c54d770 JF |
287 | |
288 | return oldbit; | |
289 | } | |
290 | ||
291 | #if 0 /* Fool kernel-doc since it doesn't do macros yet */ | |
292 | /** | |
293 | * test_bit - Determine whether a bit is set | |
294 | * @nr: bit number to test | |
295 | * @addr: Address to start counting from | |
296 | */ | |
297 | static int test_bit(int nr, const volatile unsigned long *addr); | |
298 | #endif | |
299 | ||
300 | #define test_bit(nr,addr) \ | |
301 | (__builtin_constant_p(nr) ? \ | |
302 | constant_test_bit((nr),(addr)) : \ | |
303 | variable_test_bit((nr),(addr))) | |
304 | ||
709f744f JB |
305 | #undef BASE_ADDR |
306 | #undef BIT_ADDR | |
1c54d770 JF |
307 | #undef ADDR |
308 | ||
96a388de TG |
309 | #ifdef CONFIG_X86_32 |
310 | # include "bitops_32.h" | |
311 | #else | |
312 | # include "bitops_64.h" | |
313 | #endif | |
1c54d770 JF |
314 | |
315 | #endif /* _ASM_X86_BITOPS_H */ |