Commit | Line | Data |
---|---|---|
a0e60b20 DG |
1 | /* |
2 | * PowerPC atomic bit operations. | |
3 | * | |
4 | * Merged version by David Gibson <david@gibson.dropbear.id.au>. | |
5 | * Based on ppc64 versions by: Dave Engebretsen, Todd Inglett, Don | |
6 | * Reed, Pat McCarthy, Peter Bergner, Anton Blanchard. They | |
7 | * originally took it from the ppc32 code. | |
8 | * | |
9 | * Within a word, bits are numbered LSB first. Lot's of places make | |
10 | * this assumption by directly testing bits with (val & (1<<nr)). | |
11 | * This can cause confusion for large (> 1 word) bitmaps on a | |
12 | * big-endian system because, unlike little endian, the number of each | |
13 | * bit depends on the word size. | |
14 | * | |
15 | * The bitop functions are defined to work on unsigned longs, so for a | |
16 | * ppc64 system the bits end up numbered: | |
17 | * |63..............0|127............64|191...........128|255...........196| | |
18 | * and on ppc32: | |
19 | * |31.....0|63....31|95....64|127...96|159..128|191..160|223..192|255..224| | |
20 | * | |
21 | * There are a few little-endian macros used mostly for filesystem | |
22 | * bitmaps, these work on similar bit arrays layouts, but | |
23 | * byte-oriented: | |
24 | * |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56| | |
25 | * | |
26 | * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit | |
27 | * number field needs to be reversed compared to the big-endian bit | |
28 | * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b). | |
29 | * | |
30 | * This program is free software; you can redistribute it and/or | |
31 | * modify it under the terms of the GNU General Public License | |
32 | * as published by the Free Software Foundation; either version | |
33 | * 2 of the License, or (at your option) any later version. | |
34 | */ | |
35 | ||
36 | #ifndef _ASM_POWERPC_BITOPS_H | |
37 | #define _ASM_POWERPC_BITOPS_H | |
38 | ||
39 | #ifdef __KERNEL__ | |
40 | ||
41 | #include <linux/compiler.h> | |
42 | #include <asm/atomic.h> | |
3ddfbcf1 | 43 | #include <asm/asm-compat.h> |
a0e60b20 DG |
44 | #include <asm/synch.h> |
45 | ||
46 | /* | |
47 | * clear_bit doesn't imply a memory barrier | |
48 | */ | |
49 | #define smp_mb__before_clear_bit() smp_mb() | |
50 | #define smp_mb__after_clear_bit() smp_mb() | |
51 | ||
52 | #define BITOP_MASK(nr) (1UL << ((nr) % BITS_PER_LONG)) | |
53 | #define BITOP_WORD(nr) ((nr) / BITS_PER_LONG) | |
54 | #define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) | |
55 | ||
a0e60b20 DG |
56 | static __inline__ void set_bit(int nr, volatile unsigned long *addr) |
57 | { | |
58 | unsigned long old; | |
59 | unsigned long mask = BITOP_MASK(nr); | |
60 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
61 | ||
62 | __asm__ __volatile__( | |
3ddfbcf1 | 63 | "1:" PPC_LLARX "%0,0,%3 # set_bit\n" |
a0e60b20 DG |
64 | "or %0,%0,%2\n" |
65 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 66 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 DG |
67 | "bne- 1b" |
68 | : "=&r"(old), "=m"(*p) | |
69 | : "r"(mask), "r"(p), "m"(*p) | |
70 | : "cc" ); | |
71 | } | |
72 | ||
73 | static __inline__ void clear_bit(int nr, volatile unsigned long *addr) | |
74 | { | |
75 | unsigned long old; | |
76 | unsigned long mask = BITOP_MASK(nr); | |
77 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
78 | ||
79 | __asm__ __volatile__( | |
3ddfbcf1 | 80 | "1:" PPC_LLARX "%0,0,%3 # clear_bit\n" |
a0e60b20 DG |
81 | "andc %0,%0,%2\n" |
82 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 83 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 DG |
84 | "bne- 1b" |
85 | : "=&r"(old), "=m"(*p) | |
86 | : "r"(mask), "r"(p), "m"(*p) | |
87 | : "cc" ); | |
88 | } | |
89 | ||
90 | static __inline__ void change_bit(int nr, volatile unsigned long *addr) | |
91 | { | |
92 | unsigned long old; | |
93 | unsigned long mask = BITOP_MASK(nr); | |
94 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
95 | ||
96 | __asm__ __volatile__( | |
3ddfbcf1 | 97 | "1:" PPC_LLARX "%0,0,%3 # change_bit\n" |
a0e60b20 DG |
98 | "xor %0,%0,%2\n" |
99 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 100 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 DG |
101 | "bne- 1b" |
102 | : "=&r"(old), "=m"(*p) | |
103 | : "r"(mask), "r"(p), "m"(*p) | |
104 | : "cc" ); | |
105 | } | |
106 | ||
107 | static __inline__ int test_and_set_bit(unsigned long nr, | |
108 | volatile unsigned long *addr) | |
109 | { | |
110 | unsigned long old, t; | |
111 | unsigned long mask = BITOP_MASK(nr); | |
112 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
113 | ||
114 | __asm__ __volatile__( | |
115 | EIEIO_ON_SMP | |
3ddfbcf1 | 116 | "1:" PPC_LLARX "%0,0,%3 # test_and_set_bit\n" |
a0e60b20 DG |
117 | "or %1,%0,%2 \n" |
118 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 119 | PPC_STLCX "%1,0,%3 \n" |
a0e60b20 DG |
120 | "bne- 1b" |
121 | ISYNC_ON_SMP | |
122 | : "=&r" (old), "=&r" (t) | |
123 | : "r" (mask), "r" (p) | |
124 | : "cc", "memory"); | |
125 | ||
126 | return (old & mask) != 0; | |
127 | } | |
128 | ||
129 | static __inline__ int test_and_clear_bit(unsigned long nr, | |
130 | volatile unsigned long *addr) | |
131 | { | |
132 | unsigned long old, t; | |
133 | unsigned long mask = BITOP_MASK(nr); | |
134 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
135 | ||
136 | __asm__ __volatile__( | |
137 | EIEIO_ON_SMP | |
3ddfbcf1 | 138 | "1:" PPC_LLARX "%0,0,%3 # test_and_clear_bit\n" |
a0e60b20 DG |
139 | "andc %1,%0,%2 \n" |
140 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 141 | PPC_STLCX "%1,0,%3 \n" |
a0e60b20 DG |
142 | "bne- 1b" |
143 | ISYNC_ON_SMP | |
144 | : "=&r" (old), "=&r" (t) | |
145 | : "r" (mask), "r" (p) | |
146 | : "cc", "memory"); | |
147 | ||
148 | return (old & mask) != 0; | |
149 | } | |
150 | ||
151 | static __inline__ int test_and_change_bit(unsigned long nr, | |
152 | volatile unsigned long *addr) | |
153 | { | |
154 | unsigned long old, t; | |
155 | unsigned long mask = BITOP_MASK(nr); | |
156 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
157 | ||
158 | __asm__ __volatile__( | |
159 | EIEIO_ON_SMP | |
3ddfbcf1 | 160 | "1:" PPC_LLARX "%0,0,%3 # test_and_change_bit\n" |
a0e60b20 DG |
161 | "xor %1,%0,%2 \n" |
162 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 163 | PPC_STLCX "%1,0,%3 \n" |
a0e60b20 DG |
164 | "bne- 1b" |
165 | ISYNC_ON_SMP | |
166 | : "=&r" (old), "=&r" (t) | |
167 | : "r" (mask), "r" (p) | |
168 | : "cc", "memory"); | |
169 | ||
170 | return (old & mask) != 0; | |
171 | } | |
172 | ||
173 | static __inline__ void set_bits(unsigned long mask, unsigned long *addr) | |
174 | { | |
175 | unsigned long old; | |
176 | ||
177 | __asm__ __volatile__( | |
3ddfbcf1 | 178 | "1:" PPC_LLARX "%0,0,%3 # set_bits\n" |
a0e60b20 | 179 | "or %0,%0,%2\n" |
3ddfbcf1 | 180 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 DG |
181 | "bne- 1b" |
182 | : "=&r" (old), "=m" (*addr) | |
183 | : "r" (mask), "r" (addr), "m" (*addr) | |
184 | : "cc"); | |
185 | } | |
186 | ||
187 | /* Non-atomic versions */ | |
188 | static __inline__ int test_bit(unsigned long nr, | |
189 | __const__ volatile unsigned long *addr) | |
190 | { | |
191 | return 1UL & (addr[BITOP_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); | |
192 | } | |
193 | ||
194 | static __inline__ void __set_bit(unsigned long nr, | |
195 | volatile unsigned long *addr) | |
196 | { | |
197 | unsigned long mask = BITOP_MASK(nr); | |
198 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
199 | ||
200 | *p |= mask; | |
201 | } | |
202 | ||
203 | static __inline__ void __clear_bit(unsigned long nr, | |
204 | volatile unsigned long *addr) | |
205 | { | |
206 | unsigned long mask = BITOP_MASK(nr); | |
207 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
208 | ||
209 | *p &= ~mask; | |
210 | } | |
211 | ||
212 | static __inline__ void __change_bit(unsigned long nr, | |
213 | volatile unsigned long *addr) | |
214 | { | |
215 | unsigned long mask = BITOP_MASK(nr); | |
216 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
217 | ||
218 | *p ^= mask; | |
219 | } | |
220 | ||
221 | static __inline__ int __test_and_set_bit(unsigned long nr, | |
222 | volatile unsigned long *addr) | |
223 | { | |
224 | unsigned long mask = BITOP_MASK(nr); | |
225 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
226 | unsigned long old = *p; | |
227 | ||
228 | *p = old | mask; | |
229 | return (old & mask) != 0; | |
230 | } | |
231 | ||
232 | static __inline__ int __test_and_clear_bit(unsigned long nr, | |
233 | volatile unsigned long *addr) | |
234 | { | |
235 | unsigned long mask = BITOP_MASK(nr); | |
236 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
237 | unsigned long old = *p; | |
238 | ||
239 | *p = old & ~mask; | |
240 | return (old & mask) != 0; | |
241 | } | |
242 | ||
243 | static __inline__ int __test_and_change_bit(unsigned long nr, | |
244 | volatile unsigned long *addr) | |
245 | { | |
246 | unsigned long mask = BITOP_MASK(nr); | |
247 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
248 | unsigned long old = *p; | |
249 | ||
250 | *p = old ^ mask; | |
251 | return (old & mask) != 0; | |
252 | } | |
253 | ||
254 | /* | |
255 | * Return the zero-based bit position (LE, not IBM bit numbering) of | |
256 | * the most significant 1-bit in a double word. | |
257 | */ | |
258 | static __inline__ int __ilog2(unsigned long x) | |
259 | { | |
260 | int lz; | |
261 | ||
3ddfbcf1 | 262 | asm (PPC_CNTLZL "%0,%1" : "=r" (lz) : "r" (x)); |
a0e60b20 DG |
263 | return BITS_PER_LONG - 1 - lz; |
264 | } | |
265 | ||
266 | /* | |
267 | * Determines the bit position of the least significant 0 bit in the | |
268 | * specified double word. The returned bit position will be | |
269 | * zero-based, starting from the right side (63/31 - 0). | |
270 | */ | |
271 | static __inline__ unsigned long ffz(unsigned long x) | |
272 | { | |
273 | /* no zero exists anywhere in the 8 byte area. */ | |
274 | if ((x = ~x) == 0) | |
275 | return BITS_PER_LONG; | |
276 | ||
277 | /* | |
278 | * Calculate the bit position of the least signficant '1' bit in x | |
279 | * (since x has been changed this will actually be the least signficant | |
280 | * '0' bit in * the original x). Note: (x & -x) gives us a mask that | |
281 | * is the least significant * (RIGHT-most) 1-bit of the value in x. | |
282 | */ | |
283 | return __ilog2(x & -x); | |
284 | } | |
285 | ||
286 | static __inline__ int __ffs(unsigned long x) | |
287 | { | |
288 | return __ilog2(x & -x); | |
289 | } | |
290 | ||
291 | /* | |
292 | * ffs: find first bit set. This is defined the same way as | |
293 | * the libc and compiler builtin ffs routines, therefore | |
294 | * differs in spirit from the above ffz (man ffs). | |
295 | */ | |
296 | static __inline__ int ffs(int x) | |
297 | { | |
298 | unsigned long i = (unsigned long)x; | |
299 | return __ilog2(i & -i) + 1; | |
300 | } | |
301 | ||
302 | /* | |
303 | * fls: find last (most-significant) bit set. | |
304 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. | |
305 | */ | |
306 | static __inline__ int fls(unsigned int x) | |
307 | { | |
308 | int lz; | |
309 | ||
310 | asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x)); | |
311 | return 32 - lz; | |
312 | } | |
3821af2f | 313 | #define fls64(x) generic_fls64(x) |
a0e60b20 DG |
314 | |
315 | /* | |
316 | * hweightN: returns the hamming weight (i.e. the number | |
317 | * of bits set) of a N-bit word | |
318 | */ | |
319 | #define hweight64(x) generic_hweight64(x) | |
320 | #define hweight32(x) generic_hweight32(x) | |
321 | #define hweight16(x) generic_hweight16(x) | |
322 | #define hweight8(x) generic_hweight8(x) | |
323 | ||
324 | #define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0) | |
325 | unsigned long find_next_zero_bit(const unsigned long *addr, | |
326 | unsigned long size, unsigned long offset); | |
327 | /** | |
328 | * find_first_bit - find the first set bit in a memory region | |
329 | * @addr: The address to start the search at | |
330 | * @size: The maximum size to search | |
331 | * | |
332 | * Returns the bit-number of the first set bit, not the number of the byte | |
333 | * containing a bit. | |
334 | */ | |
335 | #define find_first_bit(addr, size) find_next_bit((addr), (size), 0) | |
336 | unsigned long find_next_bit(const unsigned long *addr, | |
337 | unsigned long size, unsigned long offset); | |
338 | ||
339 | /* Little-endian versions */ | |
340 | ||
341 | static __inline__ int test_le_bit(unsigned long nr, | |
342 | __const__ unsigned long *addr) | |
343 | { | |
344 | __const__ unsigned char *tmp = (__const__ unsigned char *) addr; | |
345 | return (tmp[nr >> 3] >> (nr & 7)) & 1; | |
346 | } | |
347 | ||
348 | #define __set_le_bit(nr, addr) \ | |
349 | __set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
350 | #define __clear_le_bit(nr, addr) \ | |
351 | __clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
352 | ||
353 | #define test_and_set_le_bit(nr, addr) \ | |
354 | test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
355 | #define test_and_clear_le_bit(nr, addr) \ | |
356 | test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
357 | ||
358 | #define __test_and_set_le_bit(nr, addr) \ | |
359 | __test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
360 | #define __test_and_clear_le_bit(nr, addr) \ | |
361 | __test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
362 | ||
363 | #define find_first_zero_le_bit(addr, size) find_next_zero_le_bit((addr), (size), 0) | |
364 | unsigned long find_next_zero_le_bit(const unsigned long *addr, | |
365 | unsigned long size, unsigned long offset); | |
366 | ||
367 | /* Bitmap functions for the ext2 filesystem */ | |
368 | ||
369 | #define ext2_set_bit(nr,addr) \ | |
370 | __test_and_set_le_bit((nr), (unsigned long*)addr) | |
371 | #define ext2_clear_bit(nr, addr) \ | |
372 | __test_and_clear_le_bit((nr), (unsigned long*)addr) | |
373 | ||
374 | #define ext2_set_bit_atomic(lock, nr, addr) \ | |
375 | test_and_set_le_bit((nr), (unsigned long*)addr) | |
376 | #define ext2_clear_bit_atomic(lock, nr, addr) \ | |
377 | test_and_clear_le_bit((nr), (unsigned long*)addr) | |
378 | ||
379 | #define ext2_test_bit(nr, addr) test_le_bit((nr),(unsigned long*)addr) | |
380 | ||
381 | #define ext2_find_first_zero_bit(addr, size) \ | |
382 | find_first_zero_le_bit((unsigned long*)addr, size) | |
383 | #define ext2_find_next_zero_bit(addr, size, off) \ | |
384 | find_next_zero_le_bit((unsigned long*)addr, size, off) | |
385 | ||
386 | /* Bitmap functions for the minix filesystem. */ | |
387 | ||
388 | #define minix_test_and_set_bit(nr,addr) \ | |
389 | __test_and_set_le_bit(nr, (unsigned long *)addr) | |
390 | #define minix_set_bit(nr,addr) \ | |
391 | __set_le_bit(nr, (unsigned long *)addr) | |
392 | #define minix_test_and_clear_bit(nr,addr) \ | |
393 | __test_and_clear_le_bit(nr, (unsigned long *)addr) | |
394 | #define minix_test_bit(nr,addr) \ | |
395 | test_le_bit(nr, (unsigned long *)addr) | |
396 | ||
397 | #define minix_find_first_zero_bit(addr,size) \ | |
398 | find_first_zero_le_bit((unsigned long *)addr, size) | |
399 | ||
400 | /* | |
401 | * Every architecture must define this function. It's the fastest | |
402 | * way of searching a 140-bit bitmap where the first 100 bits are | |
403 | * unlikely to be set. It's guaranteed that at least one of the 140 | |
404 | * bits is cleared. | |
405 | */ | |
406 | static inline int sched_find_first_bit(const unsigned long *b) | |
407 | { | |
408 | #ifdef CONFIG_PPC64 | |
409 | if (unlikely(b[0])) | |
410 | return __ffs(b[0]); | |
411 | if (unlikely(b[1])) | |
412 | return __ffs(b[1]) + 64; | |
413 | return __ffs(b[2]) + 128; | |
414 | #else | |
415 | if (unlikely(b[0])) | |
416 | return __ffs(b[0]); | |
417 | if (unlikely(b[1])) | |
418 | return __ffs(b[1]) + 32; | |
419 | if (unlikely(b[2])) | |
420 | return __ffs(b[2]) + 64; | |
421 | if (b[3]) | |
422 | return __ffs(b[3]) + 96; | |
423 | return __ffs(b[4]) + 128; | |
424 | #endif | |
425 | } | |
426 | ||
427 | #endif /* __KERNEL__ */ | |
428 | ||
429 | #endif /* _ASM_POWERPC_BITOPS_H */ |