include/asm-x86/bitops.h

   1 #ifndef _ASM_X86_BITOPS_H
   2 #define _ASM_X86_BITOPS_H
   3
   4 /*
   5  * Copyright 1992, Linus Torvalds.
   6  */
   7
   8 #ifndef _LINUX_BITOPS_H
   9 #error only <linux/bitops.h> can be included directly
  10 #endif
  11
  12 #include <linux/compiler.h>
  13 #include <asm/alternative.h>
  14
  15 /*
  16  * These have to be done with inline assembly: that way the bit-setting
  17  * is guaranteed to be atomic. All bit operations return 0 if the bit
  18  * was cleared before the operation and != 0 if it was not.
  19  *
  20  * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
  21  */
  22
  23 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
  24 /* Technically wrong, but this avoids compilation errors on some gcc
  25    versions. */
  26 #define ADDR "=m" (*(volatile long *)addr)
  27 #define BIT_ADDR "=m" (((volatile int *)addr)[nr >> 5])
  28 #else
  29 #define ADDR "+m" (*(volatile long *) addr)
  30 #define BIT_ADDR "+m" (((volatile int *)addr)[nr >> 5])
  31 #endif
  32 #define BASE_ADDR "m" (*(volatile int *)addr)
  33
  34 /**
  35  * set_bit - Atomically set a bit in memory
  36  * @nr: the bit to set
  37  * @addr: the address to start counting from
  38  *
  39  * This function is atomic and may not be reordered.  See __set_bit()
  40  * if you do not require the atomic guarantees.
  41  *
  42  * Note: there are no guarantees that this function will not be reordered
  43  * on non x86 architectures, so if you are writing portable code,
  44  * make sure not to rely on its reordering guarantees.
  45  *
  46  * Note that @nr may be almost arbitrarily large; this function is not
  47  * restricted to acting on a single-word quantity.
  48  */
  49 static inline void set_bit(int nr, volatile void *addr)
  50 {
  51         asm volatile(LOCK_PREFIX "bts %1,%0" : ADDR : "Ir" (nr) : "memory");
  52 }
  53
  54 /**
  55  * __set_bit - Set a bit in memory
  56  * @nr: the bit to set
  57  * @addr: the address to start counting from
  58  *
  59  * Unlike set_bit(), this function is non-atomic and may be reordered.
  60  * If it's called on the same region of memory simultaneously, the effect
  61  * may be that only one operation succeeds.
  62  */
  63 static inline void __set_bit(int nr, volatile void *addr)
  64 {
  65         asm volatile("bts %1,%0"
  66                      : ADDR
  67                      : "Ir" (nr) : "memory");
  68 }
  69
  70 /**
  71  * clear_bit - Clears a bit in memory
  72  * @nr: Bit to clear
  73  * @addr: Address to start counting from
  74  *
  75  * clear_bit() is atomic and may not be reordered.  However, it does
  76  * not contain a memory barrier, so if it is used for locking purposes,
  77  * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
  78  * in order to ensure changes are visible on other processors.
  79  */
  80 static inline void clear_bit(int nr, volatile void *addr)
  81 {
  82         asm volatile(LOCK_PREFIX "btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
  83 }
  84
  85 /*
  86  * clear_bit_unlock - Clears a bit in memory
  87  * @nr: Bit to clear
  88  * @addr: Address to start counting from
  89  *
  90  * clear_bit() is atomic and implies release semantics before the memory
  91  * operation. It can be used for an unlock.
  92  */
  93 static inline void clear_bit_unlock(unsigned nr, volatile void *addr)
  94 {
  95         barrier();
  96         clear_bit(nr, addr);
  97 }
  98
  99 static inline void __clear_bit(int nr, volatile void *addr)
 100 {
 101         asm volatile("btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 102 }
 103
 104 /*
 105  * __clear_bit_unlock - Clears a bit in memory
 106  * @nr: Bit to clear
 107  * @addr: Address to start counting from
 108  *
 109  * __clear_bit() is non-atomic and implies release semantics before the memory
 110  * operation. It can be used for an unlock if no other CPUs can concurrently
 111  * modify other bits in the word.
 112  *
 113  * No memory barrier is required here, because x86 cannot reorder stores past
 114  * older loads. Same principle as spin_unlock.
 115  */
 116 static inline void __clear_bit_unlock(unsigned nr, volatile void *addr)
 117 {
 118         barrier();
 119         __clear_bit(nr, addr);
 120 }
 121
 122 #define smp_mb__before_clear_bit()      barrier()
 123 #define smp_mb__after_clear_bit()       barrier()
 124
 125 /**
 126  * __change_bit - Toggle a bit in memory
 127  * @nr: the bit to change
 128  * @addr: the address to start counting from
 129  *
 130  * Unlike change_bit(), this function is non-atomic and may be reordered.
 131  * If it's called on the same region of memory simultaneously, the effect
 132  * may be that only one operation succeeds.
 133  */
 134 static inline void __change_bit(int nr, volatile void *addr)
 135 {
 136         asm volatile("btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 137 }
 138
 139 /**
 140  * change_bit - Toggle a bit in memory
 141  * @nr: Bit to change
 142  * @addr: Address to start counting from
 143  *
 144  * change_bit() is atomic and may not be reordered.
 145  * Note that @nr may be almost arbitrarily large; this function is not
 146  * restricted to acting on a single-word quantity.
 147  */
 148 static inline void change_bit(int nr, volatile void *addr)
 149 {
 150         asm volatile(LOCK_PREFIX "btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
 151 }
 152
 153 /**
 154  * test_and_set_bit - Set a bit and return its old value
 155  * @nr: Bit to set
 156  * @addr: Address to count from
 157  *
 158  * This operation is atomic and cannot be reordered.
 159  * It also implies a memory barrier.
 160  */
 161 static inline int test_and_set_bit(int nr, volatile void *addr)
 162 {
 163         int oldbit;
 164
 165         asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
 166                      "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
 167
 168         return oldbit;
 169 }
 170
 171 /**
 172  * test_and_set_bit_lock - Set a bit and return its old value for lock
 173  * @nr: Bit to set
 174  * @addr: Address to count from
 175  *
 176  * This is the same as test_and_set_bit on x86.
 177  */
 178 static inline int test_and_set_bit_lock(int nr, volatile void *addr)
 179 {
 180         return test_and_set_bit(nr, addr);
 181 }
 182
 183 /**
 184  * __test_and_set_bit - Set a bit and return its old value
 185  * @nr: Bit to set
 186  * @addr: Address to count from
 187  *
 188  * This operation is non-atomic and can be reordered.
 189  * If two examples of this operation race, one can appear to succeed
 190  * but actually fail.  You must protect multiple accesses with a lock.
 191  */
 192 static inline int __test_and_set_bit(int nr, volatile void *addr)
 193 {
 194         int oldbit;
 195
 196         asm volatile("bts %2,%3\n\t"
 197                      "sbb %0,%0"
 198                      : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
 199         return oldbit;
 200 }
 201
 202 /**
 203  * test_and_clear_bit - Clear a bit and return its old value
 204  * @nr: Bit to clear
 205  * @addr: Address to count from
 206  *
 207  * This operation is atomic and cannot be reordered.
 208  * It also implies a memory barrier.
 209  */
 210 static inline int test_and_clear_bit(int nr, volatile void *addr)
 211 {
 212         int oldbit;
 213
 214         asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
 215                      "sbb %0,%0"
 216                      : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
 217
 218         return oldbit;
 219 }
 220
 221 /**
 222  * __test_and_clear_bit - Clear a bit and return its old value
 223  * @nr: Bit to clear
 224  * @addr: Address to count from
 225  *
 226  * This operation is non-atomic and can be reordered.
 227  * If two examples of this operation race, one can appear to succeed
 228  * but actually fail.  You must protect multiple accesses with a lock.
 229  */
 230 static inline int __test_and_clear_bit(int nr, volatile void *addr)
 231 {
 232         int oldbit;
 233
 234         asm volatile("btr %2,%3\n\t"
 235                      "sbb %0,%0"
 236                      : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
 237         return oldbit;
 238 }
 239
 240 /* WARNING: non atomic and it can be reordered! */
 241 static inline int __test_and_change_bit(int nr, volatile void *addr)
 242 {
 243         int oldbit;
 244
 245         asm volatile("btc %2,%3\n\t"
 246                      "sbb %0,%0"
 247                      : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
 248
 249         return oldbit;
 250 }
 251
 252 /**
 253  * test_and_change_bit - Change a bit and return its old value
 254  * @nr: Bit to change
 255  * @addr: Address to count from
 256  *
 257  * This operation is atomic and cannot be reordered.
 258  * It also implies a memory barrier.
 259  */
 260 static inline int test_and_change_bit(int nr, volatile void *addr)
 261 {
 262         int oldbit;
 263
 264         asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
 265                      "sbb %0,%0"
 266                      : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
 267
 268         return oldbit;
 269 }
 270
 271 static inline int constant_test_bit(int nr, const volatile void *addr)
 272 {
 273         return ((1UL << (nr % BITS_PER_LONG)) &
 274                 (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
 275 }
 276
 277 static inline int variable_test_bit(int nr, volatile const void *addr)
 278 {
 279         int oldbit;
 280
 281         asm volatile("bt %2,%3\n\t"
 282                      "sbb %0,%0"
 283                      : "=r" (oldbit)
 284                      : "m" (((volatile const int *)addr)[nr >> 5]),
 285                        "Ir" (nr), BASE_ADDR);
 286
 287         return oldbit;
 288 }
 289
 290 #if 0 /* Fool kernel-doc since it doesn't do macros yet */
 291 /**
 292  * test_bit - Determine whether a bit is set
 293  * @nr: bit number to test
 294  * @addr: Address to start counting from
 295  */
 296 static int test_bit(int nr, const volatile unsigned long *addr);
 297 #endif
 298
 299 #define test_bit(nr,addr)                       \
 300         (__builtin_constant_p(nr) ?             \
 301          constant_test_bit((nr),(addr)) :       \
 302          variable_test_bit((nr),(addr)))
 303
 304 #undef BASE_ADDR
 305 #undef BIT_ADDR
 306 /**
 307  * __ffs - find first set bit in word
 308  * @word: The word to search
 309  *
 310  * Undefined if no bit exists, so code should check against 0 first.
 311  */
 312 static inline unsigned long __ffs(unsigned long word)
 313 {
 314         __asm__("bsf %1,%0"
 315                 :"=r" (word)
 316                 :"rm" (word));
 317         return word;
 318 }
 319
 320 /**
 321  * ffz - find first zero bit in word
 322  * @word: The word to search
 323  *
 324  * Undefined if no zero exists, so code should check against ~0UL first.
 325  */
 326 static inline unsigned long ffz(unsigned long word)
 327 {
 328         __asm__("bsf %1,%0"
 329                 :"=r" (word)
 330                 :"r" (~word));
 331         return word;
 332 }
 333
 334 /*
 335  * __fls: find last set bit in word
 336  * @word: The word to search
 337  *
 338  * Undefined if no zero exists, so code should check against ~0UL first.
 339  */
 340 static inline unsigned long __fls(unsigned long word)
 341 {
 342         __asm__("bsr %1,%0"
 343                 :"=r" (word)
 344                 :"rm" (word));
 345         return word;
 346 }
 347
 348 #ifdef __KERNEL__
 349 /**
 350  * ffs - find first set bit in word
 351  * @x: the word to search
 352  *
 353  * This is defined the same way as the libc and compiler builtin ffs
 354  * routines, therefore differs in spirit from the other bitops.
 355  *
 356  * ffs(value) returns 0 if value is 0 or the position of the first
 357  * set bit if value is nonzero. The first (least significant) bit
 358  * is at position 1.
 359  */
 360 static inline int ffs(int x)
 361 {
 362         int r;
 363 #ifdef CONFIG_X86_CMOV
 364         __asm__("bsfl %1,%0\n\t"
 365                 "cmovzl %2,%0"
 366                 : "=r" (r) : "rm" (x), "r" (-1));
 367 #else
 368         __asm__("bsfl %1,%0\n\t"
 369                 "jnz 1f\n\t"
 370                 "movl $-1,%0\n"
 371                 "1:" : "=r" (r) : "rm" (x));
 372 #endif
 373         return r + 1;
 374 }
 375
 376 /**
 377  * fls - find last set bit in word
 378  * @x: the word to search
 379  *
 380  * This is defined in a similar way as the libc and compiler builtin
 381  * ffs, but returns the position of the most significant set bit.
 382  *
 383  * fls(value) returns 0 if value is 0 or the position of the last
 384  * set bit if value is nonzero. The last (most significant) bit is
 385  * at position 32.
 386  */
 387 static inline int fls(int x)
 388 {
 389         int r;
 390 #ifdef CONFIG_X86_CMOV
 391         __asm__("bsrl %1,%0\n\t"
 392                 "cmovzl %2,%0"
 393                 : "=&r" (r) : "rm" (x), "rm" (-1));
 394 #else
 395         __asm__("bsrl %1,%0\n\t"
 396                 "jnz 1f\n\t"
 397                 "movl $-1,%0\n"
 398                 "1:" : "=r" (r) : "rm" (x));
 399 #endif
 400         return r + 1;
 401 }
 402 #endif /* __KERNEL__ */
 403
 404 #undef ADDR
 405
 406 static inline void set_bit_string(unsigned long *bitmap,
 407                 unsigned long i, int len)
 408 {
 409         unsigned long end = i + len;
 410         while (i < end) {
 411                 __set_bit(i, bitmap);
 412                 i++;
 413         }
 414 }
 415
 416 #ifdef __KERNEL__
 417
 418 #include <asm-generic/bitops/sched.h>
 419
 420 #define ARCH_HAS_FAST_MULTIPLIER 1
 421
 422 #include <asm-generic/bitops/hweight.h>
 423
 424 #endif /* __KERNEL__ */
 425
 426 #include <asm-generic/bitops/fls64.h>
 427
 428 #ifdef __KERNEL__
 429
 430 #include <asm-generic/bitops/ext2-non-atomic.h>
 431
 432 #define ext2_set_bit_atomic(lock, nr, addr)                     \
 433         test_and_set_bit((nr), (unsigned long *)(addr))
 434 #define ext2_clear_bit_atomic(lock, nr, addr)                   \
 435         test_and_clear_bit((nr), (unsigned long *)(addr))
 436
 437 #include <asm-generic/bitops/minix.h>
 438
 439 #endif /* __KERNEL__ */
 440 #endif  /* _ASM_X86_BITOPS_H */