arch/x86/include/asm/fpu/internal.h

   1 /*
   2  * Copyright (C) 1994 Linus Torvalds
   3  *
   4  * Pentium III FXSR, SSE support
   5  * General FPU state handling cleanups
   6  *      Gareth Hughes <gareth@valinux.com>, May 2000
   7  * x86-64 work by Andi Kleen 2002
   8  */
   9
  10 #ifndef _ASM_X86_FPU_INTERNAL_H
  11 #define _ASM_X86_FPU_INTERNAL_H
  12
  13 #include <linux/regset.h>
  14 #include <linux/compat.h>
  15 #include <linux/sched.h>
  16 #include <linux/slab.h>
  17
  18 #include <asm/user.h>
  19 #include <asm/fpu/api.h>
  20 #include <asm/fpu/xstate.h>
  21
  22 #define MXCSR_DEFAULT           0x1f80
  23
  24 extern unsigned int mxcsr_feature_mask;
  25
  26 extern void fpu__init_cpu(void);
  27 extern void fpu__init_system_xstate(void);
  28 extern void fpu__init_cpu_xstate(void);
  29 extern void fpu__init_system(struct cpuinfo_x86 *c);
  30
  31 extern void fpu__activate_curr(struct fpu *fpu);
  32 extern void fpstate_init(struct fpu *fpu);
  33
  34 extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
  35
  36 /*
  37  * High level FPU state handling functions:
  38  */
  39 extern void fpu__save(struct fpu *fpu);
  40 extern void fpu__restore(void);
  41 extern int  fpu__restore_sig(void __user *buf, int ia32_frame);
  42 extern void fpu__drop(struct fpu *fpu);
  43 extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
  44 extern void fpu__clear(struct fpu *fpu);
  45
  46 extern void fpu__init_check_bugs(void);
  47 extern void fpu__resume_cpu(void);
  48
  49 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
  50
  51 extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;
  52 extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
  53                                 xstateregs_get;
  54 extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
  55                                  xstateregs_set;
  56
  57 /*
  58  * xstateregs_active == regset_fpregs_active. Please refer to the comment
  59  * at the definition of regset_fpregs_active.
  60  */
  61 #define xstateregs_active       regset_fpregs_active
  62
  63 #ifdef CONFIG_MATH_EMULATION
  64 extern void finit_soft_fpu(struct i387_soft_struct *soft);
  65 #else
  66 static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
  67 #endif
  68
  69 /*
  70  * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx,
  71  * on this CPU.
  72  *
  73  * This will disable any lazy FPU state restore of the current FPU state,
  74  * but if the current thread owns the FPU, it will still be saved by.
  75  */
  76 static inline void __cpu_disable_lazy_restore(unsigned int cpu)
  77 {
  78         per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL;
  79 }
  80
  81 static inline int fpu_want_lazy_restore(struct fpu *fpu, unsigned int cpu)
  82 {
  83         return fpu == this_cpu_read_stable(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
  84 }
  85
  86 #define X87_FSW_ES (1 << 7)     /* Exception Summary */
  87
  88 static __always_inline __pure bool use_eager_fpu(void)
  89 {
  90         return static_cpu_has_safe(X86_FEATURE_EAGER_FPU);
  91 }
  92
  93 static __always_inline __pure bool use_xsaveopt(void)
  94 {
  95         return static_cpu_has_safe(X86_FEATURE_XSAVEOPT);
  96 }
  97
  98 static __always_inline __pure bool use_xsave(void)
  99 {
 100         return static_cpu_has_safe(X86_FEATURE_XSAVE);
 101 }
 102
 103 static __always_inline __pure bool use_fxsr(void)
 104 {
 105         return static_cpu_has_safe(X86_FEATURE_FXSR);
 106 }
 107
 108 static inline void fx_finit(struct i387_fxsave_struct *fx)
 109 {
 110         fx->cwd = 0x37f;
 111         fx->mxcsr = MXCSR_DEFAULT;
 112 }
 113
 114 extern void fpstate_sanitize_xstate(struct fpu *fpu);
 115
 116 #define user_insn(insn, output, input...)                               \
 117 ({                                                                      \
 118         int err;                                                        \
 119         asm volatile(ASM_STAC "\n"                                      \
 120                      "1:" #insn "\n\t"                                  \
 121                      "2: " ASM_CLAC "\n"                                \
 122                      ".section .fixup,\"ax\"\n"                         \
 123                      "3:  movl $-1,%[err]\n"                            \
 124                      "    jmp  2b\n"                                    \
 125                      ".previous\n"                                      \
 126                      _ASM_EXTABLE(1b, 3b)                               \
 127                      : [err] "=r" (err), output                         \
 128                      : "0"(0), input);                                  \
 129         err;                                                            \
 130 })
 131
 132 #define check_insn(insn, output, input...)                              \
 133 ({                                                                      \
 134         int err;                                                        \
 135         asm volatile("1:" #insn "\n\t"                                  \
 136                      "2:\n"                                             \
 137                      ".section .fixup,\"ax\"\n"                         \
 138                      "3:  movl $-1,%[err]\n"                            \
 139                      "    jmp  2b\n"                                    \
 140                      ".previous\n"                                      \
 141                      _ASM_EXTABLE(1b, 3b)                               \
 142                      : [err] "=r" (err), output                         \
 143                      : "0"(0), input);                                  \
 144         err;                                                            \
 145 })
 146
 147 static inline int fsave_user(struct i387_fsave_struct __user *fx)
 148 {
 149         return user_insn(fnsave %[fx]; fwait,  [fx] "=m" (*fx), "m" (*fx));
 150 }
 151
 152 static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
 153 {
 154         if (config_enabled(CONFIG_X86_32))
 155                 return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
 156         else if (config_enabled(CONFIG_AS_FXSAVEQ))
 157                 return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));
 158
 159         /* See comment in fpu_fxsave() below. */
 160         return user_insn(rex64/fxsave (%[fx]), "=m" (*fx), [fx] "R" (fx));
 161 }
 162
 163 static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
 164 {
 165         if (config_enabled(CONFIG_X86_32))
 166                 return check_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 167         else if (config_enabled(CONFIG_AS_FXSAVEQ))
 168                 return check_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
 169
 170         /* See comment in fpu_fxsave() below. */
 171         return check_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx),
 172                           "m" (*fx));
 173 }
 174
 175 static inline int fxrstor_user(struct i387_fxsave_struct __user *fx)
 176 {
 177         if (config_enabled(CONFIG_X86_32))
 178                 return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 179         else if (config_enabled(CONFIG_AS_FXSAVEQ))
 180                 return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
 181
 182         /* See comment in fpu_fxsave() below. */
 183         return user_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx),
 184                           "m" (*fx));
 185 }
 186
 187 static inline int frstor_checking(struct i387_fsave_struct *fx)
 188 {
 189         return check_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 190 }
 191
 192 static inline int frstor_user(struct i387_fsave_struct __user *fx)
 193 {
 194         return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
 195 }
 196
 197 static inline void fpu_fxsave(struct fpu *fpu)
 198 {
 199         if (config_enabled(CONFIG_X86_32))
 200                 asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state.fxsave));
 201         else if (config_enabled(CONFIG_AS_FXSAVEQ))
 202                 asm volatile("fxsaveq %[fx]" : [fx] "=m" (fpu->state.fxsave));
 203         else {
 204                 /* Using "rex64; fxsave %0" is broken because, if the memory
 205                  * operand uses any extended registers for addressing, a second
 206                  * REX prefix will be generated (to the assembler, rex64
 207                  * followed by semicolon is a separate instruction), and hence
 208                  * the 64-bitness is lost.
 209                  *
 210                  * Using "fxsaveq %0" would be the ideal choice, but is only
 211                  * supported starting with gas 2.16.
 212                  *
 213                  * Using, as a workaround, the properly prefixed form below
 214                  * isn't accepted by any binutils version so far released,
 215                  * complaining that the same type of prefix is used twice if
 216                  * an extended register is needed for addressing (fix submitted
 217                  * to mainline 2005-11-21).
 218                  *
 219                  *  asm volatile("rex64/fxsave %0" : "=m" (fpu->state.fxsave));
 220                  *
 221                  * This, however, we can work around by forcing the compiler to
 222                  * select an addressing mode that doesn't require extended
 223                  * registers.
 224                  */
 225                 asm volatile( "rex64/fxsave (%[fx])"
 226                              : "=m" (fpu->state.fxsave)
 227                              : [fx] "R" (&fpu->state.fxsave));
 228         }
 229 }
 230
 231 /*
 232  * These must be called with preempt disabled. Returns
 233  * 'true' if the FPU state is still intact and we can
 234  * keep registers active.
 235  *
 236  * The legacy FNSAVE instruction cleared all FPU state
 237  * unconditionally, so registers are essentially destroyed.
 238  * Modern FPU state can be kept in registers, if there are
 239  * no pending FP exceptions.
 240  */
 241 static inline int copy_fpregs_to_fpstate(struct fpu *fpu)
 242 {
 243         if (likely(use_xsave())) {
 244                 xsave_state(&fpu->state.xsave);
 245                 return 1;
 246         }
 247
 248         if (likely(use_fxsr())) {
 249                 fpu_fxsave(fpu);
 250                 return 1;
 251         }
 252
 253         /*
 254          * Legacy FPU register saving, FNSAVE always clears FPU registers,
 255          * so we have to mark them inactive:
 256          */
 257         asm volatile("fnsave %[fx]; fwait" : [fx] "=m" (fpu->state.fsave));
 258
 259         return 0;
 260 }
 261
 262 static inline int __copy_fpstate_to_fpregs(struct fpu *fpu)
 263 {
 264         if (use_xsave())
 265                 return fpu_xrstor_checking(&fpu->state.xsave);
 266         else if (use_fxsr())
 267                 return fxrstor_checking(&fpu->state.fxsave);
 268         else
 269                 return frstor_checking(&fpu->state.fsave);
 270 }
 271
 272 static inline int copy_fpstate_to_fpregs(struct fpu *fpu)
 273 {
 274         /*
 275          * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is
 276          * pending. Clear the x87 state here by setting it to fixed values.
 277          * "m" is a random variable that should be in L1.
 278          */
 279         if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK))) {
 280                 asm volatile(
 281                         "fnclex\n\t"
 282                         "emms\n\t"
 283                         "fildl %P[addr]"        /* set F?P to defined value */
 284                         : : [addr] "m" (fpu->fpregs_active));
 285         }
 286
 287         return __copy_fpstate_to_fpregs(fpu);
 288 }
 289
 290 /*
 291  * Wrap lazy FPU TS handling in a 'hw fpregs activation/deactivation'
 292  * idiom, which is then paired with the sw-flag (fpregs_active) later on:
 293  */
 294
 295 static inline void __fpregs_activate_hw(void)
 296 {
 297         if (!use_eager_fpu())
 298                 clts();
 299 }
 300
 301 static inline void __fpregs_deactivate_hw(void)
 302 {
 303         if (!use_eager_fpu())
 304                 stts();
 305 }
 306
 307 /* Must be paired with an 'stts' (fpregs_deactivate_hw()) after! */
 308 static inline void __fpregs_deactivate(struct fpu *fpu)
 309 {
 310         fpu->fpregs_active = 0;
 311         this_cpu_write(fpu_fpregs_owner_ctx, NULL);
 312 }
 313
 314 /* Must be paired with a 'clts' (fpregs_activate_hw()) before! */
 315 static inline void __fpregs_activate(struct fpu *fpu)
 316 {
 317         fpu->fpregs_active = 1;
 318         this_cpu_write(fpu_fpregs_owner_ctx, fpu);
 319 }
 320
 321 /*
 322  * The question "does this thread have fpu access?"
 323  * is slightly racy, since preemption could come in
 324  * and revoke it immediately after the test.
 325  *
 326  * However, even in that very unlikely scenario,
 327  * we can just assume we have FPU access - typically
 328  * to save the FP state - we'll just take a #NM
 329  * fault and get the FPU access back.
 330  */
 331 static inline int fpregs_active(void)
 332 {
 333         return current->thread.fpu.fpregs_active;
 334 }
 335
 336 /*
 337  * Encapsulate the CR0.TS handling together with the
 338  * software flag.
 339  *
 340  * These generally need preemption protection to work,
 341  * do try to avoid using these on their own.
 342  */
 343 static inline void fpregs_activate(struct fpu *fpu)
 344 {
 345         __fpregs_activate_hw();
 346         __fpregs_activate(fpu);
 347 }
 348
 349 static inline void fpregs_deactivate(struct fpu *fpu)
 350 {
 351         __fpregs_deactivate(fpu);
 352         __fpregs_deactivate_hw();
 353 }
 354
 355 static inline void restore_init_xstate(void)
 356 {
 357         if (use_xsave())
 358                 xrstor_state(&init_xstate_ctx, -1);
 359         else
 360                 fxrstor_checking(&init_xstate_ctx.i387);
 361 }
 362
 363 /*
 364  * Definitions for the eXtended Control Register instructions
 365  */
 366
 367 #define XCR_XFEATURE_ENABLED_MASK       0x00000000
 368
 369 static inline u64 xgetbv(u32 index)
 370 {
 371         u32 eax, edx;
 372
 373         asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
 374                      : "=a" (eax), "=d" (edx)
 375                      : "c" (index));
 376         return eax + ((u64)edx << 32);
 377 }
 378
 379 static inline void xsetbv(u32 index, u64 value)
 380 {
 381         u32 eax = value;
 382         u32 edx = value >> 32;
 383
 384         asm volatile(".byte 0x0f,0x01,0xd1" /* xsetbv */
 385                      : : "a" (eax), "d" (edx), "c" (index));
 386 }
 387
 388 /*
 389  * FPU state switching for scheduling.
 390  *
 391  * This is a two-stage process:
 392  *
 393  *  - switch_fpu_prepare() saves the old state and
 394  *    sets the new state of the CR0.TS bit. This is
 395  *    done within the context of the old process.
 396  *
 397  *  - switch_fpu_finish() restores the new state as
 398  *    necessary.
 399  */
 400 typedef struct { int preload; } fpu_switch_t;
 401
 402 static inline fpu_switch_t
 403 switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
 404 {
 405         fpu_switch_t fpu;
 406
 407         /*
 408          * If the task has used the math, pre-load the FPU on xsave processors
 409          * or if the past 5 consecutive context-switches used math.
 410          */
 411         fpu.preload = new_fpu->fpstate_active &&
 412                       (use_eager_fpu() || new_fpu->counter > 5);
 413
 414         if (old_fpu->fpregs_active) {
 415                 if (!copy_fpregs_to_fpstate(old_fpu))
 416                         old_fpu->last_cpu = -1;
 417                 else
 418                         old_fpu->last_cpu = cpu;
 419
 420                 /* But leave fpu_fpregs_owner_ctx! */
 421                 old_fpu->fpregs_active = 0;
 422
 423                 /* Don't change CR0.TS if we just switch! */
 424                 if (fpu.preload) {
 425                         new_fpu->counter++;
 426                         __fpregs_activate(new_fpu);
 427                         prefetch(&new_fpu->state);
 428                 } else {
 429                         __fpregs_deactivate_hw();
 430                 }
 431         } else {
 432                 old_fpu->counter = 0;
 433                 old_fpu->last_cpu = -1;
 434                 if (fpu.preload) {
 435                         new_fpu->counter++;
 436                         if (fpu_want_lazy_restore(new_fpu, cpu))
 437                                 fpu.preload = 0;
 438                         else
 439                                 prefetch(&new_fpu->state);
 440                         fpregs_activate(new_fpu);
 441                 }
 442         }
 443         return fpu;
 444 }
 445
 446 /*
 447  * By the time this gets called, we've already cleared CR0.TS and
 448  * given the process the FPU if we are going to preload the FPU
 449  * state - all we need to do is to conditionally restore the register
 450  * state itself.
 451  */
 452 static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switch)
 453 {
 454         if (fpu_switch.preload) {
 455                 if (unlikely(copy_fpstate_to_fpregs(new_fpu)))
 456                         fpu__clear(new_fpu);
 457         }
 458 }
 459
 460 /*
 461  * Signal frame handlers...
 462  */
 463 extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fx, int size);
 464
 465 /*
 466  * Needs to be preemption-safe.
 467  *
 468  * NOTE! user_fpu_begin() must be used only immediately before restoring
 469  * the save state. It does not do any saving/restoring on its own. In
 470  * lazy FPU mode, it is just an optimization to avoid a #NM exception,
 471  * the task can lose the FPU right after preempt_enable().
 472  */
 473 static inline void user_fpu_begin(void)
 474 {
 475         struct fpu *fpu = &current->thread.fpu;
 476
 477         preempt_disable();
 478         if (!fpregs_active())
 479                 fpregs_activate(fpu);
 480         preempt_enable();
 481 }
 482
 483 /*
 484  * i387 state interaction
 485  */
 486 static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
 487 {
 488         if (cpu_has_fxsr) {
 489                 return tsk->thread.fpu.state.fxsave.cwd;
 490         } else {
 491                 return (unsigned short)tsk->thread.fpu.state.fsave.cwd;
 492         }
 493 }
 494
 495 static inline unsigned short get_fpu_swd(struct task_struct *tsk)
 496 {
 497         if (cpu_has_fxsr) {
 498                 return tsk->thread.fpu.state.fxsave.swd;
 499         } else {
 500                 return (unsigned short)tsk->thread.fpu.state.fsave.swd;
 501         }
 502 }
 503
 504 static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
 505 {
 506         if (cpu_has_xmm) {
 507                 return tsk->thread.fpu.state.fxsave.mxcsr;
 508         } else {
 509                 return MXCSR_DEFAULT;
 510         }
 511 }
 512
 513 #endif /* _ASM_X86_FPU_INTERNAL_H */