Commit | Line | Data |
---|---|---|
1965aae3 PA |
1 | #ifndef _ASM_X86_MSR_H |
2 | #define _ASM_X86_MSR_H | |
be7baf80 | 3 | |
b72e7464 | 4 | #include "msr-index.h" |
be7baf80 | 5 | |
8f12dea6 | 6 | #ifndef __ASSEMBLY__ |
c210d249 GOC |
7 | |
8 | #include <asm/asm.h> | |
9 | #include <asm/errno.h> | |
6bc1096d | 10 | #include <asm/cpumask.h> |
b72e7464 | 11 | #include <uapi/asm/msr.h> |
6bc1096d BP |
12 | |
13 | struct msr { | |
14 | union { | |
15 | struct { | |
16 | u32 l; | |
17 | u32 h; | |
18 | }; | |
19 | u64 q; | |
20 | }; | |
21 | }; | |
c210d249 | 22 | |
6ede31e0 BP |
23 | struct msr_info { |
24 | u32 msr_no; | |
25 | struct msr reg; | |
26 | struct msr *msrs; | |
27 | int err; | |
28 | }; | |
29 | ||
30 | struct msr_regs_info { | |
31 | u32 *regs; | |
32 | int err; | |
33 | }; | |
34 | ||
1e160cc3 | 35 | static inline unsigned long long native_read_tscp(unsigned int *aux) |
8f12dea6 GOC |
36 | { |
37 | unsigned long low, high; | |
abb0ade0 JP |
38 | asm volatile(".byte 0x0f,0x01,0xf9" |
39 | : "=a" (low), "=d" (high), "=c" (*aux)); | |
41aefdcc | 40 | return low | ((u64)high << 32); |
8f12dea6 GOC |
41 | } |
42 | ||
c210d249 | 43 | /* |
d4f1b103 JS |
44 | * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A" |
45 | * constraint has different meanings. For i386, "A" means exactly | |
46 | * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead, | |
47 | * it means rax *or* rdx. | |
c210d249 GOC |
48 | */ |
49 | #ifdef CONFIG_X86_64 | |
50 | #define DECLARE_ARGS(val, low, high) unsigned low, high | |
abb0ade0 | 51 | #define EAX_EDX_VAL(val, low, high) ((low) | ((u64)(high) << 32)) |
c210d249 GOC |
52 | #define EAX_EDX_ARGS(val, low, high) "a" (low), "d" (high) |
53 | #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) | |
54 | #else | |
55 | #define DECLARE_ARGS(val, low, high) unsigned long long val | |
56 | #define EAX_EDX_VAL(val, low, high) (val) | |
57 | #define EAX_EDX_ARGS(val, low, high) "A" (val) | |
58 | #define EAX_EDX_RET(val, low, high) "=A" (val) | |
8f12dea6 GOC |
59 | #endif |
60 | ||
be7baf80 TG |
61 | static inline unsigned long long native_read_msr(unsigned int msr) |
62 | { | |
c210d249 | 63 | DECLARE_ARGS(val, low, high); |
be7baf80 | 64 | |
c210d249 GOC |
65 | asm volatile("rdmsr" : EAX_EDX_RET(val, low, high) : "c" (msr)); |
66 | return EAX_EDX_VAL(val, low, high); | |
be7baf80 TG |
67 | } |
68 | ||
69 | static inline unsigned long long native_read_msr_safe(unsigned int msr, | |
70 | int *err) | |
71 | { | |
c210d249 | 72 | DECLARE_ARGS(val, low, high); |
be7baf80 | 73 | |
08970fc4 | 74 | asm volatile("2: rdmsr ; xor %[err],%[err]\n" |
be7baf80 TG |
75 | "1:\n\t" |
76 | ".section .fixup,\"ax\"\n\t" | |
08970fc4 | 77 | "3: mov %[fault],%[err] ; jmp 1b\n\t" |
be7baf80 | 78 | ".previous\n\t" |
abb0ade0 | 79 | _ASM_EXTABLE(2b, 3b) |
08970fc4 | 80 | : [err] "=r" (*err), EAX_EDX_RET(val, low, high) |
0cc0213e | 81 | : "c" (msr), [fault] "i" (-EIO)); |
c210d249 | 82 | return EAX_EDX_VAL(val, low, high); |
be7baf80 TG |
83 | } |
84 | ||
c9dcda5c GOC |
85 | static inline void native_write_msr(unsigned int msr, |
86 | unsigned low, unsigned high) | |
be7baf80 | 87 | { |
af2b1c60 | 88 | asm volatile("wrmsr" : : "c" (msr), "a"(low), "d" (high) : "memory"); |
be7baf80 TG |
89 | } |
90 | ||
0ca59dd9 FW |
91 | /* Can be uninlined because referenced by paravirt */ |
92 | notrace static inline int native_write_msr_safe(unsigned int msr, | |
c9dcda5c | 93 | unsigned low, unsigned high) |
be7baf80 TG |
94 | { |
95 | int err; | |
08970fc4 | 96 | asm volatile("2: wrmsr ; xor %[err],%[err]\n" |
be7baf80 TG |
97 | "1:\n\t" |
98 | ".section .fixup,\"ax\"\n\t" | |
08970fc4 | 99 | "3: mov %[fault],%[err] ; jmp 1b\n\t" |
be7baf80 | 100 | ".previous\n\t" |
abb0ade0 | 101 | _ASM_EXTABLE(2b, 3b) |
08970fc4 | 102 | : [err] "=a" (err) |
c9dcda5c | 103 | : "c" (msr), "0" (low), "d" (high), |
0cc0213e | 104 | [fault] "i" (-EIO) |
af2b1c60 | 105 | : "memory"); |
be7baf80 TG |
106 | return err; |
107 | } | |
108 | ||
cdc7957d | 109 | extern unsigned long long native_read_tsc(void); |
be7baf80 | 110 | |
1f975f78 AP |
111 | extern int rdmsr_safe_regs(u32 regs[8]); |
112 | extern int wrmsr_safe_regs(u32 regs[8]); | |
132ec92f | 113 | |
92767af0 IM |
114 | static __always_inline unsigned long long __native_read_tsc(void) |
115 | { | |
116 | DECLARE_ARGS(val, low, high); | |
117 | ||
92767af0 | 118 | asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); |
92767af0 IM |
119 | |
120 | return EAX_EDX_VAL(val, low, high); | |
121 | } | |
122 | ||
b8d1fae7 | 123 | static inline unsigned long long native_read_pmc(int counter) |
be7baf80 | 124 | { |
c210d249 GOC |
125 | DECLARE_ARGS(val, low, high); |
126 | ||
127 | asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); | |
128 | return EAX_EDX_VAL(val, low, high); | |
be7baf80 TG |
129 | } |
130 | ||
131 | #ifdef CONFIG_PARAVIRT | |
132 | #include <asm/paravirt.h> | |
96a388de | 133 | #else |
be7baf80 TG |
134 | #include <linux/errno.h> |
135 | /* | |
136 | * Access to machine-specific registers (available on 586 and better only) | |
137 | * Note: the rd* operations modify the parameters directly (without using | |
138 | * pointer indirection), this allows gcc to optimize better | |
139 | */ | |
140 | ||
1423bed2 | 141 | #define rdmsr(msr, low, high) \ |
abb0ade0 JP |
142 | do { \ |
143 | u64 __val = native_read_msr((msr)); \ | |
1423bed2 BP |
144 | (void)((low) = (u32)__val); \ |
145 | (void)((high) = (u32)(__val >> 32)); \ | |
abb0ade0 | 146 | } while (0) |
be7baf80 | 147 | |
c9dcda5c | 148 | static inline void wrmsr(unsigned msr, unsigned low, unsigned high) |
be7baf80 | 149 | { |
c9dcda5c | 150 | native_write_msr(msr, low, high); |
be7baf80 TG |
151 | } |
152 | ||
abb0ade0 JP |
153 | #define rdmsrl(msr, val) \ |
154 | ((val) = native_read_msr((msr))) | |
be7baf80 | 155 | |
c210d249 | 156 | #define wrmsrl(msr, val) \ |
abb0ade0 | 157 | native_write_msr((msr), (u32)((u64)(val)), (u32)((u64)(val) >> 32)) |
be7baf80 TG |
158 | |
159 | /* wrmsr with exception handling */ | |
c9dcda5c | 160 | static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) |
be7baf80 | 161 | { |
c9dcda5c | 162 | return native_write_msr_safe(msr, low, high); |
be7baf80 TG |
163 | } |
164 | ||
060feb65 | 165 | /* rdmsr with exception handling */ |
1423bed2 | 166 | #define rdmsr_safe(msr, low, high) \ |
abb0ade0 JP |
167 | ({ \ |
168 | int __err; \ | |
169 | u64 __val = native_read_msr_safe((msr), &__err); \ | |
1423bed2 BP |
170 | (*low) = (u32)__val; \ |
171 | (*high) = (u32)(__val >> 32); \ | |
abb0ade0 JP |
172 | __err; \ |
173 | }) | |
be7baf80 | 174 | |
1de87bd4 AK |
175 | static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) |
176 | { | |
177 | int err; | |
178 | ||
179 | *p = native_read_msr_safe(msr, &err); | |
180 | return err; | |
181 | } | |
177fed1e | 182 | |
be7baf80 | 183 | #define rdtscl(low) \ |
205516c1 | 184 | ((low) = (u32)__native_read_tsc()) |
be7baf80 TG |
185 | |
186 | #define rdtscll(val) \ | |
205516c1 | 187 | ((val) = __native_read_tsc()) |
be7baf80 | 188 | |
abb0ade0 JP |
189 | #define rdpmc(counter, low, high) \ |
190 | do { \ | |
191 | u64 _l = native_read_pmc((counter)); \ | |
192 | (low) = (u32)_l; \ | |
193 | (high) = (u32)(_l >> 32); \ | |
194 | } while (0) | |
be7baf80 | 195 | |
1ff4d58a AK |
196 | #define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) |
197 | ||
abb0ade0 JP |
198 | #define rdtscp(low, high, aux) \ |
199 | do { \ | |
200 | unsigned long long _val = native_read_tscp(&(aux)); \ | |
201 | (low) = (u32)_val; \ | |
202 | (high) = (u32)(_val >> 32); \ | |
203 | } while (0) | |
be7baf80 | 204 | |
c210d249 | 205 | #define rdtscpll(val, aux) (val) = native_read_tscp(&(aux)) |
be7baf80 | 206 | |
c210d249 | 207 | #endif /* !CONFIG_PARAVIRT */ |
be7baf80 | 208 | |
cf991de2 AL |
209 | /* |
210 | * 64-bit version of wrmsr_safe(): | |
211 | */ | |
212 | static inline int wrmsrl_safe(u32 msr, u64 val) | |
213 | { | |
214 | return wrmsr_safe(msr, (u32)val, (u32)(val >> 32)); | |
215 | } | |
be7baf80 | 216 | |
1423bed2 | 217 | #define write_tsc(low, high) wrmsr(MSR_IA32_TSC, (low), (high)) |
be7baf80 | 218 | |
5df97400 | 219 | #define write_rdtscp_aux(val) wrmsr(MSR_TSC_AUX, (val), 0) |
be7baf80 | 220 | |
50542251 BP |
221 | struct msr *msrs_alloc(void); |
222 | void msrs_free(struct msr *msrs); | |
22085a66 BP |
223 | int msr_set_bit(u32 msr, u8 bit); |
224 | int msr_clear_bit(u32 msr, u8 bit); | |
50542251 | 225 | |
be7baf80 | 226 | #ifdef CONFIG_SMP |
c6f31932 PA |
227 | int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); |
228 | int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); | |
1a6b991a JP |
229 | int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); |
230 | int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q); | |
b8a47541 BP |
231 | void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); |
232 | void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); | |
be7baf80 TG |
233 | int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); |
234 | int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); | |
1a6b991a JP |
235 | int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); |
236 | int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); | |
8b956bf1 PA |
237 | int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); |
238 | int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); | |
be7baf80 | 239 | #else /* CONFIG_SMP */ |
c6f31932 | 240 | static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) |
be7baf80 TG |
241 | { |
242 | rdmsr(msr_no, *l, *h); | |
c6f31932 | 243 | return 0; |
be7baf80 | 244 | } |
c6f31932 | 245 | static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) |
be7baf80 TG |
246 | { |
247 | wrmsr(msr_no, l, h); | |
c6f31932 | 248 | return 0; |
be7baf80 | 249 | } |
1a6b991a JP |
250 | static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) |
251 | { | |
252 | rdmsrl(msr_no, *q); | |
253 | return 0; | |
254 | } | |
255 | static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) | |
256 | { | |
257 | wrmsrl(msr_no, q); | |
258 | return 0; | |
259 | } | |
0d0fbbdd | 260 | static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no, |
b034c19f BP |
261 | struct msr *msrs) |
262 | { | |
263 | rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h)); | |
264 | } | |
0d0fbbdd | 265 | static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no, |
b034c19f BP |
266 | struct msr *msrs) |
267 | { | |
268 | wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h); | |
269 | } | |
abb0ade0 JP |
270 | static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, |
271 | u32 *l, u32 *h) | |
be7baf80 TG |
272 | { |
273 | return rdmsr_safe(msr_no, l, h); | |
274 | } | |
275 | static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) | |
276 | { | |
277 | return wrmsr_safe(msr_no, l, h); | |
278 | } | |
1a6b991a JP |
279 | static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) |
280 | { | |
281 | return rdmsrl_safe(msr_no, q); | |
282 | } | |
283 | static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) | |
284 | { | |
285 | return wrmsrl_safe(msr_no, q); | |
286 | } | |
8b956bf1 PA |
287 | static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) |
288 | { | |
289 | return rdmsr_safe_regs(regs); | |
290 | } | |
291 | static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) | |
292 | { | |
293 | return wrmsr_safe_regs(regs); | |
294 | } | |
be7baf80 | 295 | #endif /* CONFIG_SMP */ |
ff55df53 | 296 | #endif /* __ASSEMBLY__ */ |
1965aae3 | 297 | #endif /* _ASM_X86_MSR_H */ |