Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #include <linux/bitops.h> |
2 | ||
3 | #undef find_first_zero_bit | |
1da177e4 | 4 | #undef find_first_bit |
1da177e4 | 5 | |
06024f21 AO |
6 | static inline long |
7 | __find_first_zero_bit(const unsigned long * addr, unsigned long size) | |
1da177e4 LT |
8 | { |
9 | long d0, d1, d2; | |
10 | long res; | |
11 | ||
06024f21 AO |
12 | /* |
13 | * We must test the size in words, not in bits, because | |
14 | * otherwise incoming sizes in the range -63..-1 will not run | |
15 | * any scasq instructions, and then the flags used by the je | |
16 | * instruction will have whatever random value was in place | |
17 | * before. Nobody should call us like that, but | |
18 | * find_next_zero_bit() does when offset and size are at the | |
19 | * same word and it fails to find a zero itself. | |
20 | */ | |
21 | size += 63; | |
22 | size >>= 6; | |
1da177e4 LT |
23 | if (!size) |
24 | return 0; | |
25 | asm volatile( | |
26 | " repe; scasq\n" | |
27 | " je 1f\n" | |
28 | " xorq -8(%%rdi),%%rax\n" | |
29 | " subq $8,%%rdi\n" | |
30 | " bsfq %%rax,%%rdx\n" | |
31 | "1: subq %[addr],%%rdi\n" | |
32 | " shlq $3,%%rdi\n" | |
33 | " addq %%rdi,%%rdx" | |
34 | :"=d" (res), "=&c" (d0), "=&D" (d1), "=&a" (d2) | |
06024f21 AO |
35 | :"0" (0ULL), "1" (size), "2" (addr), "3" (-1ULL), |
36 | [addr] "S" (addr) : "memory"); | |
37 | /* | |
38 | * Any register would do for [addr] above, but GCC tends to | |
39 | * prefer rbx over rsi, even though rsi is readily available | |
40 | * and doesn't have to be saved. | |
41 | */ | |
1da177e4 LT |
42 | return res; |
43 | } | |
44 | ||
06024f21 AO |
45 | /** |
46 | * find_first_zero_bit - find the first zero bit in a memory region | |
47 | * @addr: The address to start the search at | |
48 | * @size: The maximum size to search | |
49 | * | |
50 | * Returns the bit-number of the first zero bit, not the number of the byte | |
51 | * containing a bit. | |
52 | */ | |
53 | long find_first_zero_bit(const unsigned long * addr, unsigned long size) | |
54 | { | |
55 | return __find_first_zero_bit (addr, size); | |
56 | } | |
57 | ||
1da177e4 LT |
58 | static inline long |
59 | __find_first_bit(const unsigned long * addr, unsigned long size) | |
60 | { | |
61 | long d0, d1; | |
62 | long res; | |
63 | ||
06024f21 AO |
64 | /* |
65 | * We must test the size in words, not in bits, because | |
66 | * otherwise incoming sizes in the range -63..-1 will not run | |
67 | * any scasq instructions, and then the flags used by the jz | |
68 | * instruction will have whatever random value was in place | |
69 | * before. Nobody should call us like that, but | |
70 | * find_next_bit() does when offset and size are at the same | |
71 | * word and it fails to find a one itself. | |
72 | */ | |
73 | size += 63; | |
74 | size >>= 6; | |
75 | if (!size) | |
76 | return 0; | |
1da177e4 LT |
77 | asm volatile( |
78 | " repe; scasq\n" | |
79 | " jz 1f\n" | |
80 | " subq $8,%%rdi\n" | |
81 | " bsfq (%%rdi),%%rax\n" | |
82 | "1: subq %[addr],%%rdi\n" | |
83 | " shlq $3,%%rdi\n" | |
84 | " addq %%rdi,%%rax" | |
85 | :"=a" (res), "=&c" (d0), "=&D" (d1) | |
06024f21 | 86 | :"0" (0ULL), "1" (size), "2" (addr), |
1da177e4 LT |
87 | [addr] "r" (addr) : "memory"); |
88 | return res; | |
89 | } | |
90 | ||
91 | /** | |
92 | * find_first_bit - find the first set bit in a memory region | |
93 | * @addr: The address to start the search at | |
94 | * @size: The maximum size to search | |
95 | * | |
96 | * Returns the bit-number of the first set bit, not the number of the byte | |
97 | * containing a bit. | |
98 | */ | |
99 | long find_first_bit(const unsigned long * addr, unsigned long size) | |
100 | { | |
101 | return __find_first_bit(addr,size); | |
102 | } | |
103 | ||
1da177e4 LT |
104 | #include <linux/module.h> |
105 | ||
1da177e4 LT |
106 | EXPORT_SYMBOL(find_first_bit); |
107 | EXPORT_SYMBOL(find_first_zero_bit); |