include/asm-x86/bitops_64.h

   1 #ifndef _X86_64_BITOPS_H
   2 #define _X86_64_BITOPS_H
   3
   4 /*
   5  * Copyright 1992, Linus Torvalds.
   6  */
   7
   8 #ifndef _LINUX_BITOPS_H
   9 #error only <linux/bitops.h> can be included directly
  10 #endif
  11
  12 #include <asm/alternative.h>
  13
  14 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
  15 /* Technically wrong, but this avoids compilation errors on some gcc
  16    versions. */
  17 #define ADDR "=m" (*(volatile long *) addr)
  18 #else
  19 #define ADDR "+m" (*(volatile long *) addr)
  20 #endif
  21
  22 /**
  23  * set_bit - Atomically set a bit in memory
  24  * @nr: the bit to set
  25  * @addr: the address to start counting from
  26  *
  27  * This function is atomic and may not be reordered.  See __set_bit()
  28  * if you do not require the atomic guarantees.
  29  * Note that @nr may be almost arbitrarily large; this function is not
  30  * restricted to acting on a single-word quantity.
  31  */
  32 static __inline__ void set_bit(int nr, volatile void * addr)
  33 {
  34         __asm__ __volatile__( LOCK_PREFIX
  35                 "btsl %1,%0"
  36                 :ADDR
  37                 :"dIr" (nr) : "memory");
  38 }
  39
  40 /**
  41  * __set_bit - Set a bit in memory
  42  * @nr: the bit to set
  43  * @addr: the address to start counting from
  44  *
  45  * Unlike set_bit(), this function is non-atomic and may be reordered.
  46  * If it's called on the same region of memory simultaneously, the effect
  47  * may be that only one operation succeeds.
  48  */
  49 static __inline__ void __set_bit(int nr, volatile void * addr)
  50 {
  51         __asm__ volatile(
  52                 "btsl %1,%0"
  53                 :ADDR
  54                 :"dIr" (nr) : "memory");
  55 }
  56
  57 /**
  58  * clear_bit - Clears a bit in memory
  59  * @nr: Bit to clear
  60  * @addr: Address to start counting from
  61  *
  62  * clear_bit() is atomic and may not be reordered.  However, it does
  63  * not contain a memory barrier, so if it is used for locking purposes,
  64  * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
  65  * in order to ensure changes are visible on other processors.
  66  */
  67 static __inline__ void clear_bit(int nr, volatile void * addr)
  68 {
  69         __asm__ __volatile__( LOCK_PREFIX
  70                 "btrl %1,%0"
  71                 :ADDR
  72                 :"dIr" (nr));
  73 }
  74
  75 /*
  76  * clear_bit_unlock - Clears a bit in memory
  77  * @nr: Bit to clear
  78  * @addr: Address to start counting from
  79  *
  80  * clear_bit() is atomic and implies release semantics before the memory
  81  * operation. It can be used for an unlock.
  82  */
  83 static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
  84 {
  85         barrier();
  86         clear_bit(nr, addr);
  87 }
  88
  89 static __inline__ void __clear_bit(int nr, volatile void * addr)
  90 {
  91         __asm__ __volatile__(
  92                 "btrl %1,%0"
  93                 :ADDR
  94                 :"dIr" (nr));
  95 }
  96
  97 /*
  98  * __clear_bit_unlock - Clears a bit in memory
  99  * @nr: Bit to clear
 100  * @addr: Address to start counting from
 101  *
 102  * __clear_bit() is non-atomic and implies release semantics before the memory
 103  * operation. It can be used for an unlock if no other CPUs can concurrently
 104  * modify other bits in the word.
 105  *
 106  * No memory barrier is required here, because x86 cannot reorder stores past
 107  * older loads. Same principle as spin_unlock.
 108  */
 109 static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
 110 {
 111         barrier();
 112         __clear_bit(nr, addr);
 113 }
 114
 115 #define smp_mb__before_clear_bit()      barrier()
 116 #define smp_mb__after_clear_bit()       barrier()
 117
 118 /**
 119  * __change_bit - Toggle a bit in memory
 120  * @nr: the bit to change
 121  * @addr: the address to start counting from
 122  *
 123  * Unlike change_bit(), this function is non-atomic and may be reordered.
 124  * If it's called on the same region of memory simultaneously, the effect
 125  * may be that only one operation succeeds.
 126  */
 127 static __inline__ void __change_bit(int nr, volatile void * addr)
 128 {
 129         __asm__ __volatile__(
 130                 "btcl %1,%0"
 131                 :ADDR
 132                 :"dIr" (nr));
 133 }
 134
 135 /**
 136  * change_bit - Toggle a bit in memory
 137  * @nr: Bit to change
 138  * @addr: Address to start counting from
 139  *
 140  * change_bit() is atomic and may not be reordered.
 141  * Note that @nr may be almost arbitrarily large; this function is not
 142  * restricted to acting on a single-word quantity.
 143  */
 144 static __inline__ void change_bit(int nr, volatile void * addr)
 145 {
 146         __asm__ __volatile__( LOCK_PREFIX
 147                 "btcl %1,%0"
 148                 :ADDR
 149                 :"dIr" (nr));
 150 }
 151
 152 /**
 153  * test_and_set_bit - Set a bit and return its old value
 154  * @nr: Bit to set
 155  * @addr: Address to count from
 156  *
 157  * This operation is atomic and cannot be reordered.
 158  * It also implies a memory barrier.
 159  */
 160 static __inline__ int test_and_set_bit(int nr, volatile void * addr)
 161 {
 162         int oldbit;
 163
 164         __asm__ __volatile__( LOCK_PREFIX
 165                 "btsl %2,%1\n\tsbbl %0,%0"
 166                 :"=r" (oldbit),ADDR
 167                 :"dIr" (nr) : "memory");
 168         return oldbit;
 169 }
 170
 171 /**
 172  * test_and_set_bit_lock - Set a bit and return its old value for lock
 173  * @nr: Bit to set
 174  * @addr: Address to count from
 175  *
 176  * This is the same as test_and_set_bit on x86
 177  */
 178 #define test_and_set_bit_lock test_and_set_bit
 179
 180 /**
 181  * __test_and_set_bit - Set a bit and return its old value
 182  * @nr: Bit to set
 183  * @addr: Address to count from
 184  *
 185  * This operation is non-atomic and can be reordered.
 186  * If two examples of this operation race, one can appear to succeed
 187  * but actually fail.  You must protect multiple accesses with a lock.
 188  */
 189 static __inline__ int __test_and_set_bit(int nr, volatile void * addr)
 190 {
 191         int oldbit;
 192
 193         __asm__(
 194                 "btsl %2,%1\n\tsbbl %0,%0"
 195                 :"=r" (oldbit),ADDR
 196                 :"dIr" (nr));
 197         return oldbit;
 198 }
 199
 200 /**
 201  * test_and_clear_bit - Clear a bit and return its old value
 202  * @nr: Bit to clear
 203  * @addr: Address to count from
 204  *
 205  * This operation is atomic and cannot be reordered.
 206  * It also implies a memory barrier.
 207  */
 208 static __inline__ int test_and_clear_bit(int nr, volatile void * addr)
 209 {
 210         int oldbit;
 211
 212         __asm__ __volatile__( LOCK_PREFIX
 213                 "btrl %2,%1\n\tsbbl %0,%0"
 214                 :"=r" (oldbit),ADDR
 215                 :"dIr" (nr) : "memory");
 216         return oldbit;
 217 }
 218
 219 /**
 220  * __test_and_clear_bit - Clear a bit and return its old value
 221  * @nr: Bit to clear
 222  * @addr: Address to count from
 223  *
 224  * This operation is non-atomic and can be reordered.
 225  * If two examples of this operation race, one can appear to succeed
 226  * but actually fail.  You must protect multiple accesses with a lock.
 227  */
 228 static __inline__ int __test_and_clear_bit(int nr, volatile void * addr)
 229 {
 230         int oldbit;
 231
 232         __asm__(
 233                 "btrl %2,%1\n\tsbbl %0,%0"
 234                 :"=r" (oldbit),ADDR
 235                 :"dIr" (nr));
 236         return oldbit;
 237 }
 238
 239 /* WARNING: non atomic and it can be reordered! */
 240 static __inline__ int __test_and_change_bit(int nr, volatile void * addr)
 241 {
 242         int oldbit;
 243
 244         __asm__ __volatile__(
 245                 "btcl %2,%1\n\tsbbl %0,%0"
 246                 :"=r" (oldbit),ADDR
 247                 :"dIr" (nr) : "memory");
 248         return oldbit;
 249 }
 250
 251 /**
 252  * test_and_change_bit - Change a bit and return its old value
 253  * @nr: Bit to change
 254  * @addr: Address to count from
 255  *
 256  * This operation is atomic and cannot be reordered.
 257  * It also implies a memory barrier.
 258  */
 259 static __inline__ int test_and_change_bit(int nr, volatile void * addr)
 260 {
 261         int oldbit;
 262
 263         __asm__ __volatile__( LOCK_PREFIX
 264                 "btcl %2,%1\n\tsbbl %0,%0"
 265                 :"=r" (oldbit),ADDR
 266                 :"dIr" (nr) : "memory");
 267         return oldbit;
 268 }
 269
 270 #if 0 /* Fool kernel-doc since it doesn't do macros yet */
 271 /**
 272  * test_bit - Determine whether a bit is set
 273  * @nr: bit number to test
 274  * @addr: Address to start counting from
 275  */
 276 static int test_bit(int nr, const volatile void * addr);
 277 #endif
 278
 279 static __inline__ int constant_test_bit(int nr, const volatile void * addr)
 280 {
 281         return ((1UL << (nr & 31)) & (((const volatile unsigned int *) addr)[nr >> 5])) != 0;
 282 }
 283
 284 static __inline__ int variable_test_bit(int nr, volatile const void * addr)
 285 {
 286         int oldbit;
 287
 288         __asm__ __volatile__(
 289                 "btl %2,%1\n\tsbbl %0,%0"
 290                 :"=r" (oldbit)
 291                 :"m" (*(volatile long *)addr),"dIr" (nr));
 292         return oldbit;
 293 }
 294
 295 #define test_bit(nr,addr) \
 296 (__builtin_constant_p(nr) ? \
 297  constant_test_bit((nr),(addr)) : \
 298  variable_test_bit((nr),(addr)))
 299
 300 #undef ADDR
 301
 302 extern long find_first_zero_bit(const unsigned long * addr, unsigned long size);
 303 extern long find_next_zero_bit (const unsigned long * addr, long size, long offset);
 304 extern long find_first_bit(const unsigned long * addr, unsigned long size);
 305 extern long find_next_bit(const unsigned long * addr, long size, long offset);
 306
 307 /* return index of first bet set in val or max when no bit is set */
 308 static inline long __scanbit(unsigned long val, unsigned long max)
 309 {
 310         asm("bsfq %1,%0 ; cmovz %2,%0" : "=&r" (val) : "r" (val), "r" (max));
 311         return val;
 312 }
 313
 314 #define find_first_bit(addr,size) \
 315 ((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \
 316   (__scanbit(*(unsigned long *)addr,(size))) : \
 317   find_first_bit(addr,size)))
 318
 319 #define find_next_bit(addr,size,off) \
 320 ((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ?         \
 321   ((off) + (__scanbit((*(unsigned long *)addr) >> (off),(size)-(off)))) : \
 322         find_next_bit(addr,size,off)))
 323
 324 #define find_first_zero_bit(addr,size) \
 325 ((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ? \
 326   (__scanbit(~*(unsigned long *)addr,(size))) : \
 327         find_first_zero_bit(addr,size)))
 328
 329 #define find_next_zero_bit(addr,size,off) \
 330 ((__builtin_constant_p(size) && (size) <= BITS_PER_LONG ?         \
 331   ((off)+(__scanbit(~(((*(unsigned long *)addr)) >> (off)),(size)-(off)))) : \
 332         find_next_zero_bit(addr,size,off)))
 333
 334 /*
 335  * Find string of zero bits in a bitmap. -1 when not found.
 336  */
 337 extern unsigned long
 338 find_next_zero_string(unsigned long *bitmap, long start, long nbits, int len);
 339
 340 static inline void set_bit_string(unsigned long *bitmap, unsigned long i,
 341                                   int len)
 342 {
 343         unsigned long end = i + len;
 344         while (i < end) {
 345                 __set_bit(i, bitmap);
 346                 i++;
 347         }
 348 }
 349
 350 static inline void __clear_bit_string(unsigned long *bitmap, unsigned long i,
 351                                     int len)
 352 {
 353         unsigned long end = i + len;
 354         while (i < end) {
 355                 __clear_bit(i, bitmap);
 356                 i++;
 357         }
 358 }
 359
 360 /**
 361  * ffz - find first zero in word.
 362  * @word: The word to search
 363  *
 364  * Undefined if no zero exists, so code should check against ~0UL first.
 365  */
 366 static __inline__ unsigned long ffz(unsigned long word)
 367 {
 368         __asm__("bsfq %1,%0"
 369                 :"=r" (word)
 370                 :"r" (~word));
 371         return word;
 372 }
 373
 374 /**
 375  * __ffs - find first bit in word.
 376  * @word: The word to search
 377  *
 378  * Undefined if no bit exists, so code should check against 0 first.
 379  */
 380 static __inline__ unsigned long __ffs(unsigned long word)
 381 {
 382         __asm__("bsfq %1,%0"
 383                 :"=r" (word)
 384                 :"rm" (word));
 385         return word;
 386 }
 387
 388 /*
 389  * __fls: find last bit set.
 390  * @word: The word to search
 391  *
 392  * Undefined if no zero exists, so code should check against ~0UL first.
 393  */
 394 static __inline__ unsigned long __fls(unsigned long word)
 395 {
 396         __asm__("bsrq %1,%0"
 397                 :"=r" (word)
 398                 :"rm" (word));
 399         return word;
 400 }
 401
 402 #ifdef __KERNEL__
 403
 404 #include <asm-generic/bitops/sched.h>
 405
 406 /**
 407  * ffs - find first bit set
 408  * @x: the word to search
 409  *
 410  * This is defined the same way as
 411  * the libc and compiler builtin ffs routines, therefore
 412  * differs in spirit from the above ffz (man ffs).
 413  */
 414 static __inline__ int ffs(int x)
 415 {
 416         int r;
 417
 418         __asm__("bsfl %1,%0\n\t"
 419                 "cmovzl %2,%0"
 420                 : "=r" (r) : "rm" (x), "r" (-1));
 421         return r+1;
 422 }
 423
 424 /**
 425  * fls64 - find last bit set in 64 bit word
 426  * @x: the word to search
 427  *
 428  * This is defined the same way as fls.
 429  */
 430 static __inline__ int fls64(__u64 x)
 431 {
 432         if (x == 0)
 433                 return 0;
 434         return __fls(x) + 1;
 435 }
 436
 437 /**
 438  * fls - find last bit set
 439  * @x: the word to search
 440  *
 441  * This is defined the same way as ffs.
 442  */
 443 static __inline__ int fls(int x)
 444 {
 445         int r;
 446
 447         __asm__("bsrl %1,%0\n\t"
 448                 "cmovzl %2,%0"
 449                 : "=&r" (r) : "rm" (x), "rm" (-1));
 450         return r+1;
 451 }
 452
 453 #define ARCH_HAS_FAST_MULTIPLIER 1
 454
 455 #include <asm-generic/bitops/hweight.h>
 456
 457 #endif /* __KERNEL__ */
 458
 459 #ifdef __KERNEL__
 460
 461 #include <asm-generic/bitops/ext2-non-atomic.h>
 462
 463 #define ext2_set_bit_atomic(lock,nr,addr) \
 464                 test_and_set_bit((nr),(unsigned long*)addr)
 465 #define ext2_clear_bit_atomic(lock,nr,addr) \
 466                 test_and_clear_bit((nr),(unsigned long*)addr)
 467
 468 #include <asm-generic/bitops/minix.h>
 469
 470 #endif /* __KERNEL__ */
 471
 472 #endif /* _X86_64_BITOPS_H */