arch/x86/lib/bitops_64.c

   1 #include <linux/bitops.h>
   2
   3 #undef find_first_zero_bit
   4 #undef find_first_bit
   5
   6 static inline long
   7 __find_first_zero_bit(const unsigned long * addr, unsigned long size)
   8 {
   9         long d0, d1, d2;
  10         long res;
  11
  12         /*
  13          * We must test the size in words, not in bits, because
  14          * otherwise incoming sizes in the range -63..-1 will not run
  15          * any scasq instructions, and then the flags used by the je
  16          * instruction will have whatever random value was in place
  17          * before.  Nobody should call us like that, but
  18          * find_next_zero_bit() does when offset and size are at the
  19          * same word and it fails to find a zero itself.
  20          */
  21         size += 63;
  22         size >>= 6;
  23         if (!size)
  24                 return 0;
  25         asm volatile(
  26                 "  repe; scasq\n"
  27                 "  je 1f\n"
  28                 "  xorq -8(%%rdi),%%rax\n"
  29                 "  subq $8,%%rdi\n"
  30                 "  bsfq %%rax,%%rdx\n"
  31                 "1:  subq %[addr],%%rdi\n"
  32                 "  shlq $3,%%rdi\n"
  33                 "  addq %%rdi,%%rdx"
  34                 :"=d" (res), "=&c" (d0), "=&D" (d1), "=&a" (d2)
  35                 :"0" (0ULL), "1" (size), "2" (addr), "3" (-1ULL),
  36                  [addr] "S" (addr) : "memory");
  37         /*
  38          * Any register would do for [addr] above, but GCC tends to
  39          * prefer rbx over rsi, even though rsi is readily available
  40          * and doesn't have to be saved.
  41          */
  42         return res;
  43 }
  44
  45 /**
  46  * find_first_zero_bit - find the first zero bit in a memory region
  47  * @addr: The address to start the search at
  48  * @size: The maximum size to search
  49  *
  50  * Returns the bit-number of the first zero bit, not the number of the byte
  51  * containing a bit.
  52  */
  53 long find_first_zero_bit(const unsigned long * addr, unsigned long size)
  54 {
  55         return __find_first_zero_bit (addr, size);
  56 }
  57
  58 static inline long
  59 __find_first_bit(const unsigned long * addr, unsigned long size)
  60 {
  61         long d0, d1;
  62         long res;
  63
  64         /*
  65          * We must test the size in words, not in bits, because
  66          * otherwise incoming sizes in the range -63..-1 will not run
  67          * any scasq instructions, and then the flags used by the jz
  68          * instruction will have whatever random value was in place
  69          * before.  Nobody should call us like that, but
  70          * find_next_bit() does when offset and size are at the same
  71          * word and it fails to find a one itself.
  72          */
  73         size += 63;
  74         size >>= 6;
  75         if (!size)
  76                 return 0;
  77         asm volatile(
  78                 "   repe; scasq\n"
  79                 "   jz 1f\n"
  80                 "   subq $8,%%rdi\n"
  81                 "   bsfq (%%rdi),%%rax\n"
  82                 "1: subq %[addr],%%rdi\n"
  83                 "   shlq $3,%%rdi\n"
  84                 "   addq %%rdi,%%rax"
  85                 :"=a" (res), "=&c" (d0), "=&D" (d1)
  86                 :"0" (0ULL), "1" (size), "2" (addr),
  87                  [addr] "r" (addr) : "memory");
  88         return res;
  89 }
  90
  91 /**
  92  * find_first_bit - find the first set bit in a memory region
  93  * @addr: The address to start the search at
  94  * @size: The maximum size to search
  95  *
  96  * Returns the bit-number of the first set bit, not the number of the byte
  97  * containing a bit.
  98  */
  99 long find_first_bit(const unsigned long * addr, unsigned long size)
 100 {
 101         return __find_first_bit(addr,size);
 102 }
 103
 104 #include <linux/module.h>
 105
 106 EXPORT_SYMBOL(find_first_bit);
 107 EXPORT_SYMBOL(find_first_zero_bit);