Commit | Line | Data |
---|---|---|
1965aae3 PA |
1 | #ifndef _ASM_X86_CPUFEATURE_H |
2 | #define _ASM_X86_CPUFEATURE_H | |
7b11fb51 | 3 | |
cd4d09ec | 4 | #include <asm/processor.h> |
e2604b49 | 5 | |
fa1408e4 PA |
6 | #if defined(__KERNEL__) && !defined(__ASSEMBLY__) |
7 | ||
a3c8acd0 | 8 | #include <asm/asm.h> |
fa1408e4 PA |
9 | #include <linux/bitops.h> |
10 | ||
39c06df4 BP |
11 | enum cpuid_leafs |
12 | { | |
13 | CPUID_1_EDX = 0, | |
14 | CPUID_8000_0001_EDX, | |
15 | CPUID_8086_0001_EDX, | |
16 | CPUID_LNX_1, | |
17 | CPUID_1_ECX, | |
18 | CPUID_C000_0001_EDX, | |
19 | CPUID_8000_0001_ECX, | |
20 | CPUID_LNX_2, | |
21 | CPUID_LNX_3, | |
22 | CPUID_7_0_EBX, | |
23 | CPUID_D_1_EAX, | |
24 | CPUID_F_0_EDX, | |
25 | CPUID_F_1_EDX, | |
26 | CPUID_8000_0008_EBX, | |
27 | CPUID_6_EAX, | |
28 | CPUID_8000_000A_EDX, | |
29 | }; | |
30 | ||
9def39be | 31 | #ifdef CONFIG_X86_FEATURE_NAMES |
fa1408e4 PA |
32 | extern const char * const x86_cap_flags[NCAPINTS*32]; |
33 | extern const char * const x86_power_flags[32]; | |
9def39be JT |
34 | #define X86_CAP_FMT "%s" |
35 | #define x86_cap_flag(flag) x86_cap_flags[flag] | |
36 | #else | |
37 | #define X86_CAP_FMT "%d:%d" | |
38 | #define x86_cap_flag(flag) ((flag) >> 5), ((flag) & 31) | |
39 | #endif | |
fa1408e4 | 40 | |
80a208bd BP |
41 | /* |
42 | * In order to save room, we index into this array by doing | |
43 | * X86_BUG_<name> - NCAPINTS*32. | |
44 | */ | |
45 | extern const char * const x86_bug_flags[NBUGINTS*32]; | |
46 | ||
0f8d2b92 IM |
47 | #define test_cpu_cap(c, bit) \ |
48 | test_bit(bit, (unsigned long *)((c)->x86_capability)) | |
49 | ||
349c004e | 50 | #define REQUIRED_MASK_BIT_SET(bit) \ |
7b11fb51 PA |
51 | ( (((bit)>>5)==0 && (1UL<<((bit)&31) & REQUIRED_MASK0)) || \ |
52 | (((bit)>>5)==1 && (1UL<<((bit)&31) & REQUIRED_MASK1)) || \ | |
53 | (((bit)>>5)==2 && (1UL<<((bit)&31) & REQUIRED_MASK2)) || \ | |
54 | (((bit)>>5)==3 && (1UL<<((bit)&31) & REQUIRED_MASK3)) || \ | |
55 | (((bit)>>5)==4 && (1UL<<((bit)&31) & REQUIRED_MASK4)) || \ | |
56 | (((bit)>>5)==5 && (1UL<<((bit)&31) & REQUIRED_MASK5)) || \ | |
57 | (((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6)) || \ | |
bdc802dc PA |
58 | (((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7)) || \ |
59 | (((bit)>>5)==8 && (1UL<<((bit)&31) & REQUIRED_MASK8)) || \ | |
349c004e CL |
60 | (((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9)) ) |
61 | ||
381aa07a DH |
62 | #define DISABLED_MASK_BIT_SET(bit) \ |
63 | ( (((bit)>>5)==0 && (1UL<<((bit)&31) & DISABLED_MASK0)) || \ | |
64 | (((bit)>>5)==1 && (1UL<<((bit)&31) & DISABLED_MASK1)) || \ | |
65 | (((bit)>>5)==2 && (1UL<<((bit)&31) & DISABLED_MASK2)) || \ | |
66 | (((bit)>>5)==3 && (1UL<<((bit)&31) & DISABLED_MASK3)) || \ | |
67 | (((bit)>>5)==4 && (1UL<<((bit)&31) & DISABLED_MASK4)) || \ | |
68 | (((bit)>>5)==5 && (1UL<<((bit)&31) & DISABLED_MASK5)) || \ | |
69 | (((bit)>>5)==6 && (1UL<<((bit)&31) & DISABLED_MASK6)) || \ | |
70 | (((bit)>>5)==7 && (1UL<<((bit)&31) & DISABLED_MASK7)) || \ | |
71 | (((bit)>>5)==8 && (1UL<<((bit)&31) & DISABLED_MASK8)) || \ | |
72 | (((bit)>>5)==9 && (1UL<<((bit)&31) & DISABLED_MASK9)) ) | |
73 | ||
349c004e CL |
74 | #define cpu_has(c, bit) \ |
75 | (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \ | |
0f8d2b92 IM |
76 | test_cpu_cap(c, bit)) |
77 | ||
349c004e CL |
78 | #define this_cpu_has(bit) \ |
79 | (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \ | |
80 | x86_this_cpu_test_bit(bit, (unsigned long *)&cpu_info.x86_capability)) | |
81 | ||
381aa07a DH |
82 | /* |
83 | * This macro is for detection of features which need kernel | |
84 | * infrastructure to be used. It may *not* directly test the CPU | |
85 | * itself. Use the cpu_has() family if you want true runtime | |
86 | * testing of CPU features, like in hypervisor code where you are | |
87 | * supporting a possible guest feature where host support for it | |
88 | * is not relevant. | |
89 | */ | |
90 | #define cpu_feature_enabled(bit) \ | |
91 | (__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : \ | |
92 | cpu_has(&boot_cpu_data, bit)) | |
93 | ||
7b11fb51 PA |
94 | #define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit) |
95 | ||
53756d37 JF |
96 | #define set_cpu_cap(c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability)) |
97 | #define clear_cpu_cap(c, bit) clear_bit(bit, (unsigned long *)((c)->x86_capability)) | |
7d851c8d AK |
98 | #define setup_clear_cpu_cap(bit) do { \ |
99 | clear_cpu_cap(&boot_cpu_data, bit); \ | |
3e0c3737 | 100 | set_bit(bit, (unsigned long *)cpu_caps_cleared); \ |
7d851c8d | 101 | } while (0) |
404ee5b1 AK |
102 | #define setup_force_cpu_cap(bit) do { \ |
103 | set_cpu_cap(&boot_cpu_data, bit); \ | |
3e0c3737 | 104 | set_bit(bit, (unsigned long *)cpu_caps_set); \ |
404ee5b1 | 105 | } while (0) |
53756d37 | 106 | |
7b11fb51 | 107 | #define cpu_has_fpu boot_cpu_has(X86_FEATURE_FPU) |
7b11fb51 PA |
108 | #define cpu_has_pse boot_cpu_has(X86_FEATURE_PSE) |
109 | #define cpu_has_tsc boot_cpu_has(X86_FEATURE_TSC) | |
7b11fb51 PA |
110 | #define cpu_has_pge boot_cpu_has(X86_FEATURE_PGE) |
111 | #define cpu_has_apic boot_cpu_has(X86_FEATURE_APIC) | |
7b11fb51 PA |
112 | #define cpu_has_fxsr boot_cpu_has(X86_FEATURE_FXSR) |
113 | #define cpu_has_xmm boot_cpu_has(X86_FEATURE_XMM) | |
114 | #define cpu_has_xmm2 boot_cpu_has(X86_FEATURE_XMM2) | |
54b6a1bd | 115 | #define cpu_has_aes boot_cpu_has(X86_FEATURE_AES) |
66be8951 | 116 | #define cpu_has_avx boot_cpu_has(X86_FEATURE_AVX) |
60488010 | 117 | #define cpu_has_avx2 boot_cpu_has(X86_FEATURE_AVX2) |
840d2830 | 118 | #define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLUSH) |
019c3e7c | 119 | #define cpu_has_gbpages boot_cpu_has(X86_FEATURE_GBPAGES) |
86975101 | 120 | #define cpu_has_arch_perfmon boot_cpu_has(X86_FEATURE_ARCH_PERFMON) |
2e5d9c85 | 121 | #define cpu_has_pat boot_cpu_has(X86_FEATURE_PAT) |
32e1d0a0 | 122 | #define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC) |
f1240c00 | 123 | #define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE) |
6229ad27 | 124 | #define cpu_has_xsaves boot_cpu_has(X86_FEATURE_XSAVES) |
66be8951 | 125 | #define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE) |
49ab56ac | 126 | #define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) |
362f924b BP |
127 | /* |
128 | * Do not add any more of those clumsy macros - use static_cpu_has_safe() for | |
129 | * fast paths and boot_cpu_has() otherwise! | |
130 | */ | |
7b11fb51 | 131 | |
6e1315fe | 132 | #if __GNUC__ >= 4 && defined(CONFIG_X86_FAST_FEATURE_TESTS) |
5700f743 | 133 | extern void warn_pre_alternatives(void); |
4a90a99c | 134 | extern bool __static_cpu_has_safe(u16 bit); |
5700f743 | 135 | |
a3c8acd0 PA |
136 | /* |
137 | * Static testing of CPU features. Used the same as boot_cpu_has(). | |
138 | * These are only valid after alternatives have run, but will statically | |
139 | * patch the target code for additional performance. | |
a3c8acd0 | 140 | */ |
83a7a2ad | 141 | static __always_inline __pure bool __static_cpu_has(u16 bit) |
a3c8acd0 | 142 | { |
62122fd7 | 143 | #ifdef CC_HAVE_ASM_GOTO |
5700f743 BP |
144 | |
145 | #ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS | |
62122fd7 | 146 | |
5700f743 BP |
147 | /* |
148 | * Catch too early usage of this before alternatives | |
149 | * have run. | |
150 | */ | |
3f0116c3 | 151 | asm_volatile_goto("1: jmp %l[t_warn]\n" |
5700f743 BP |
152 | "2:\n" |
153 | ".section .altinstructions,\"a\"\n" | |
154 | " .long 1b - .\n" | |
155 | " .long 0\n" /* no replacement */ | |
156 | " .word %P0\n" /* 1: do replace */ | |
157 | " .byte 2b - 1b\n" /* source len */ | |
158 | " .byte 0\n" /* replacement len */ | |
4332195c | 159 | " .byte 0\n" /* pad len */ |
5700f743 BP |
160 | ".previous\n" |
161 | /* skipping size check since replacement size = 0 */ | |
162 | : : "i" (X86_FEATURE_ALWAYS) : : t_warn); | |
62122fd7 | 163 | |
5700f743 BP |
164 | #endif |
165 | ||
3f0116c3 | 166 | asm_volatile_goto("1: jmp %l[t_no]\n" |
a3c8acd0 PA |
167 | "2:\n" |
168 | ".section .altinstructions,\"a\"\n" | |
59e97e4d AL |
169 | " .long 1b - .\n" |
170 | " .long 0\n" /* no replacement */ | |
83a7a2ad | 171 | " .word %P0\n" /* feature bit */ |
a3c8acd0 PA |
172 | " .byte 2b - 1b\n" /* source len */ |
173 | " .byte 0\n" /* replacement len */ | |
4332195c | 174 | " .byte 0\n" /* pad len */ |
a3c8acd0 | 175 | ".previous\n" |
83a7a2ad | 176 | /* skipping size check since replacement size = 0 */ |
a3c8acd0 PA |
177 | : : "i" (bit) : : t_no); |
178 | return true; | |
179 | t_no: | |
180 | return false; | |
5700f743 BP |
181 | |
182 | #ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS | |
183 | t_warn: | |
184 | warn_pre_alternatives(); | |
185 | return false; | |
186 | #endif | |
62122fd7 BP |
187 | |
188 | #else /* CC_HAVE_ASM_GOTO */ | |
189 | ||
a3c8acd0 PA |
190 | u8 flag; |
191 | /* Open-coded due to __stringify() in ALTERNATIVE() */ | |
192 | asm volatile("1: movb $0,%0\n" | |
193 | "2:\n" | |
194 | ".section .altinstructions,\"a\"\n" | |
59e97e4d AL |
195 | " .long 1b - .\n" |
196 | " .long 3f - .\n" | |
83a7a2ad | 197 | " .word %P1\n" /* feature bit */ |
a3c8acd0 PA |
198 | " .byte 2b - 1b\n" /* source len */ |
199 | " .byte 4f - 3f\n" /* replacement len */ | |
4332195c | 200 | " .byte 0\n" /* pad len */ |
83a7a2ad PA |
201 | ".previous\n" |
202 | ".section .discard,\"aw\",@progbits\n" | |
203 | " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ | |
a3c8acd0 PA |
204 | ".previous\n" |
205 | ".section .altinstr_replacement,\"ax\"\n" | |
206 | "3: movb $1,%0\n" | |
207 | "4:\n" | |
208 | ".previous\n" | |
209 | : "=qm" (flag) : "i" (bit)); | |
210 | return flag; | |
62122fd7 BP |
211 | |
212 | #endif /* CC_HAVE_ASM_GOTO */ | |
a3c8acd0 PA |
213 | } |
214 | ||
215 | #define static_cpu_has(bit) \ | |
216 | ( \ | |
217 | __builtin_constant_p(boot_cpu_has(bit)) ? \ | |
218 | boot_cpu_has(bit) : \ | |
83a7a2ad | 219 | __builtin_constant_p(bit) ? \ |
a3c8acd0 PA |
220 | __static_cpu_has(bit) : \ |
221 | boot_cpu_has(bit) \ | |
222 | ) | |
4a90a99c BP |
223 | |
224 | static __always_inline __pure bool _static_cpu_has_safe(u16 bit) | |
225 | { | |
62122fd7 | 226 | #ifdef CC_HAVE_ASM_GOTO |
48c7a250 | 227 | asm_volatile_goto("1: jmp %l[t_dynamic]\n" |
4a90a99c | 228 | "2:\n" |
4332195c BP |
229 | ".skip -(((5f-4f) - (2b-1b)) > 0) * " |
230 | "((5f-4f) - (2b-1b)),0x90\n" | |
231 | "3:\n" | |
4a90a99c BP |
232 | ".section .altinstructions,\"a\"\n" |
233 | " .long 1b - .\n" /* src offset */ | |
4332195c | 234 | " .long 4f - .\n" /* repl offset */ |
4a90a99c | 235 | " .word %P1\n" /* always replace */ |
4332195c BP |
236 | " .byte 3b - 1b\n" /* src len */ |
237 | " .byte 5f - 4f\n" /* repl len */ | |
238 | " .byte 3b - 2b\n" /* pad len */ | |
4a90a99c BP |
239 | ".previous\n" |
240 | ".section .altinstr_replacement,\"ax\"\n" | |
48c7a250 | 241 | "4: jmp %l[t_no]\n" |
4332195c | 242 | "5:\n" |
4a90a99c BP |
243 | ".previous\n" |
244 | ".section .altinstructions,\"a\"\n" | |
245 | " .long 1b - .\n" /* src offset */ | |
246 | " .long 0\n" /* no replacement */ | |
247 | " .word %P0\n" /* feature bit */ | |
4332195c | 248 | " .byte 3b - 1b\n" /* src len */ |
4a90a99c | 249 | " .byte 0\n" /* repl len */ |
4332195c | 250 | " .byte 0\n" /* pad len */ |
4a90a99c BP |
251 | ".previous\n" |
252 | : : "i" (bit), "i" (X86_FEATURE_ALWAYS) | |
253 | : : t_dynamic, t_no); | |
254 | return true; | |
255 | t_no: | |
256 | return false; | |
257 | t_dynamic: | |
258 | return __static_cpu_has_safe(bit); | |
62122fd7 | 259 | #else |
4a90a99c BP |
260 | u8 flag; |
261 | /* Open-coded due to __stringify() in ALTERNATIVE() */ | |
262 | asm volatile("1: movb $2,%0\n" | |
263 | "2:\n" | |
264 | ".section .altinstructions,\"a\"\n" | |
265 | " .long 1b - .\n" /* src offset */ | |
266 | " .long 3f - .\n" /* repl offset */ | |
267 | " .word %P2\n" /* always replace */ | |
268 | " .byte 2b - 1b\n" /* source len */ | |
269 | " .byte 4f - 3f\n" /* replacement len */ | |
4332195c | 270 | " .byte 0\n" /* pad len */ |
4a90a99c BP |
271 | ".previous\n" |
272 | ".section .discard,\"aw\",@progbits\n" | |
273 | " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ | |
274 | ".previous\n" | |
275 | ".section .altinstr_replacement,\"ax\"\n" | |
276 | "3: movb $0,%0\n" | |
277 | "4:\n" | |
278 | ".previous\n" | |
279 | ".section .altinstructions,\"a\"\n" | |
280 | " .long 1b - .\n" /* src offset */ | |
281 | " .long 5f - .\n" /* repl offset */ | |
282 | " .word %P1\n" /* feature bit */ | |
283 | " .byte 4b - 3b\n" /* src len */ | |
284 | " .byte 6f - 5f\n" /* repl len */ | |
4332195c | 285 | " .byte 0\n" /* pad len */ |
4a90a99c BP |
286 | ".previous\n" |
287 | ".section .discard,\"aw\",@progbits\n" | |
288 | " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */ | |
289 | ".previous\n" | |
290 | ".section .altinstr_replacement,\"ax\"\n" | |
291 | "5: movb $1,%0\n" | |
292 | "6:\n" | |
293 | ".previous\n" | |
294 | : "=qm" (flag) | |
295 | : "i" (bit), "i" (X86_FEATURE_ALWAYS)); | |
296 | return (flag == 2 ? __static_cpu_has_safe(bit) : flag); | |
62122fd7 | 297 | #endif /* CC_HAVE_ASM_GOTO */ |
4a90a99c BP |
298 | } |
299 | ||
300 | #define static_cpu_has_safe(bit) \ | |
301 | ( \ | |
302 | __builtin_constant_p(boot_cpu_has(bit)) ? \ | |
303 | boot_cpu_has(bit) : \ | |
304 | _static_cpu_has_safe(bit) \ | |
305 | ) | |
1ba4f22c PA |
306 | #else |
307 | /* | |
308 | * gcc 3.x is too stupid to do the static test; fall back to dynamic. | |
309 | */ | |
4a90a99c BP |
310 | #define static_cpu_has(bit) boot_cpu_has(bit) |
311 | #define static_cpu_has_safe(bit) boot_cpu_has(bit) | |
1ba4f22c | 312 | #endif |
a3c8acd0 | 313 | |
9b13a93d BP |
314 | #define cpu_has_bug(c, bit) cpu_has(c, (bit)) |
315 | #define set_cpu_bug(c, bit) set_cpu_cap(c, (bit)) | |
316 | #define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit)) | |
65fc985b | 317 | |
9b13a93d BP |
318 | #define static_cpu_has_bug(bit) static_cpu_has((bit)) |
319 | #define static_cpu_has_bug_safe(bit) static_cpu_has_safe((bit)) | |
320 | #define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit)) | |
65fc985b | 321 | |
9b13a93d BP |
322 | #define MAX_CPU_FEATURES (NCAPINTS * 32) |
323 | #define cpu_have_feature boot_cpu_has | |
2b9c1f03 | 324 | |
9b13a93d BP |
325 | #define CPU_FEATURE_TYPEFMT "x86,ven%04Xfam%04Xmod%04X" |
326 | #define CPU_FEATURE_TYPEVAL boot_cpu_data.x86_vendor, boot_cpu_data.x86, \ | |
327 | boot_cpu_data.x86_model | |
2b9c1f03 | 328 | |
fa1408e4 | 329 | #endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */ |
1965aae3 | 330 | #endif /* _ASM_X86_CPUFEATURE_H */ |