arch/x86/kernel/i387.c

   1 /*
   2  *  Copyright (C) 1994 Linus Torvalds
   3  *
   4  *  Pentium III FXSR, SSE support
   5  *  General FPU state handling cleanups
   6  *      Gareth Hughes <gareth@valinux.com>, May 2000
   7  */
   8 #include <linux/module.h>
   9 #include <linux/regset.h>
  10 #include <linux/sched.h>
  11 #include <linux/slab.h>
  12
  13 #include <asm/sigcontext.h>
  14 #include <asm/processor.h>
  15 #include <asm/math_emu.h>
  16 #include <asm/tlbflush.h>
  17 #include <asm/uaccess.h>
  18 #include <asm/ptrace.h>
  19 #include <asm/i387.h>
  20 #include <asm/fpu-internal.h>
  21 #include <asm/user.h>
  22
  23 /*
  24  * Were we in an interrupt that interrupted kernel mode?
  25  *
  26  * On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
  27  * pair does nothing at all: the thread must not have fpu (so
  28  * that we don't try to save the FPU state), and TS must
  29  * be set (so that the clts/stts pair does nothing that is
  30  * visible in the interrupted kernel thread).
  31  *
  32  * Except for the eagerfpu case when we return 1 unless we've already
  33  * been eager and saved the state in kernel_fpu_begin().
  34  */
  35 static inline bool interrupted_kernel_fpu_idle(void)
  36 {
  37         if (use_eager_fpu())
  38                 return __thread_has_fpu(current);
  39
  40         return !__thread_has_fpu(current) &&
  41                 (read_cr0() & X86_CR0_TS);
  42 }
  43
  44 /*
  45  * Were we in user mode (or vm86 mode) when we were
  46  * interrupted?
  47  *
  48  * Doing kernel_fpu_begin/end() is ok if we are running
  49  * in an interrupt context from user mode - we'll just
  50  * save the FPU state as required.
  51  */
  52 static inline bool interrupted_user_mode(void)
  53 {
  54         struct pt_regs *regs = get_irq_regs();
  55         return regs && user_mode_vm(regs);
  56 }
  57
  58 /*
  59  * Can we use the FPU in kernel mode with the
  60  * whole "kernel_fpu_begin/end()" sequence?
  61  *
  62  * It's always ok in process context (ie "not interrupt")
  63  * but it is sometimes ok even from an irq.
  64  */
  65 bool irq_fpu_usable(void)
  66 {
  67         return !in_interrupt() ||
  68                 interrupted_user_mode() ||
  69                 interrupted_kernel_fpu_idle();
  70 }
  71 EXPORT_SYMBOL(irq_fpu_usable);
  72
  73 void __kernel_fpu_begin(void)
  74 {
  75         struct task_struct *me = current;
  76
  77         if (__thread_has_fpu(me)) {
  78                 __thread_clear_has_fpu(me);
  79                 __save_init_fpu(me);
  80                 /* We do 'stts()' in __kernel_fpu_end() */
  81         } else if (!use_eager_fpu()) {
  82                 this_cpu_write(fpu_owner_task, NULL);
  83                 clts();
  84         }
  85 }
  86 EXPORT_SYMBOL(__kernel_fpu_begin);
  87
  88 void __kernel_fpu_end(void)
  89 {
  90         if (use_eager_fpu()) {
  91                 /*
  92                  * For eager fpu, most the time, tsk_used_math() is true.
  93                  * Restore the user math as we are done with the kernel usage.
  94                  * At few instances during thread exit, signal handling etc,
  95                  * tsk_used_math() is false. Those few places will take proper
  96                  * actions, so we don't need to restore the math here.
  97                  */
  98                 if (likely(tsk_used_math(current)))
  99                         math_state_restore();
 100         } else {
 101                 stts();
 102         }
 103 }
 104 EXPORT_SYMBOL(__kernel_fpu_end);
 105
 106 void unlazy_fpu(struct task_struct *tsk)
 107 {
 108         preempt_disable();
 109         if (__thread_has_fpu(tsk)) {
 110                 __save_init_fpu(tsk);
 111                 __thread_fpu_end(tsk);
 112         } else
 113                 tsk->thread.fpu_counter = 0;
 114         preempt_enable();
 115 }
 116 EXPORT_SYMBOL(unlazy_fpu);
 117
 118 unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
 119 unsigned int xstate_size;
 120 EXPORT_SYMBOL_GPL(xstate_size);
 121 static struct i387_fxsave_struct fx_scratch;
 122
 123 static void mxcsr_feature_mask_init(void)
 124 {
 125         unsigned long mask = 0;
 126
 127         if (cpu_has_fxsr) {
 128                 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
 129                 asm volatile("fxsave %0" : "+m" (fx_scratch));
 130                 mask = fx_scratch.mxcsr_mask;
 131                 if (mask == 0)
 132                         mask = 0x0000ffbf;
 133         }
 134         mxcsr_feature_mask &= mask;
 135 }
 136
 137 static void init_thread_xstate(void)
 138 {
 139         /*
 140          * Note that xstate_size might be overwriten later during
 141          * xsave_init().
 142          */
 143
 144         if (!cpu_has_fpu) {
 145                 /*
 146                  * Disable xsave as we do not support it if i387
 147                  * emulation is enabled.
 148                  */
 149                 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
 150                 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
 151                 xstate_size = sizeof(struct i387_soft_struct);
 152                 return;
 153         }
 154
 155         if (cpu_has_fxsr)
 156                 xstate_size = sizeof(struct i387_fxsave_struct);
 157         else
 158                 xstate_size = sizeof(struct i387_fsave_struct);
 159 }
 160
 161 /*
 162  * Called at bootup to set up the initial FPU state that is later cloned
 163  * into all processes.
 164  */
 165
 166 void fpu_init(void)
 167 {
 168         unsigned long cr0;
 169         unsigned long cr4_mask = 0;
 170
 171 #ifndef CONFIG_MATH_EMULATION
 172         if (!cpu_has_fpu) {
 173                 pr_emerg("No FPU found and no math emulation present\n");
 174                 pr_emerg("Giving up\n");
 175                 for (;;)
 176                         asm volatile("hlt");
 177         }
 178 #endif
 179         if (cpu_has_fxsr)
 180                 cr4_mask |= X86_CR4_OSFXSR;
 181         if (cpu_has_xmm)
 182                 cr4_mask |= X86_CR4_OSXMMEXCPT;
 183         if (cr4_mask)
 184                 cr4_set_bits(cr4_mask);
 185
 186         cr0 = read_cr0();
 187         cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
 188         if (!cpu_has_fpu)
 189                 cr0 |= X86_CR0_EM;
 190         write_cr0(cr0);
 191
 192         /*
 193          * init_thread_xstate is only called once to avoid overriding
 194          * xstate_size during boot time or during CPU hotplug.
 195          */
 196         if (xstate_size == 0)
 197                 init_thread_xstate();
 198
 199         mxcsr_feature_mask_init();
 200         xsave_init();
 201         eager_fpu_init();
 202 }
 203
 204 void fpu_finit(struct fpu *fpu)
 205 {
 206         if (!cpu_has_fpu) {
 207                 finit_soft_fpu(&fpu->state->soft);
 208                 return;
 209         }
 210
 211         if (cpu_has_fxsr) {
 212                 fx_finit(&fpu->state->fxsave);
 213         } else {
 214                 struct i387_fsave_struct *fp = &fpu->state->fsave;
 215                 memset(fp, 0, xstate_size);
 216                 fp->cwd = 0xffff037fu;
 217                 fp->swd = 0xffff0000u;
 218                 fp->twd = 0xffffffffu;
 219                 fp->fos = 0xffff0000u;
 220         }
 221 }
 222 EXPORT_SYMBOL_GPL(fpu_finit);
 223
 224 /*
 225  * The _current_ task is using the FPU for the first time
 226  * so initialize it and set the mxcsr to its default
 227  * value at reset if we support XMM instructions and then
 228  * remember the current task has used the FPU.
 229  */
 230 int init_fpu(struct task_struct *tsk)
 231 {
 232         int ret;
 233
 234         if (tsk_used_math(tsk)) {
 235                 if (cpu_has_fpu && tsk == current)
 236                         unlazy_fpu(tsk);
 237                 tsk->thread.fpu.last_cpu = ~0;
 238                 return 0;
 239         }
 240
 241         /*
 242          * Memory allocation at the first usage of the FPU and other state.
 243          */
 244         ret = fpu_alloc(&tsk->thread.fpu);
 245         if (ret)
 246                 return ret;
 247
 248         fpu_finit(&tsk->thread.fpu);
 249
 250         set_stopped_child_used_math(tsk);
 251         return 0;
 252 }
 253 EXPORT_SYMBOL_GPL(init_fpu);
 254
 255 /*
 256  * The xstateregs_active() routine is the same as the fpregs_active() routine,
 257  * as the "regset->n" for the xstate regset will be updated based on the feature
 258  * capabilites supported by the xsave.
 259  */
 260 int fpregs_active(struct task_struct *target, const struct user_regset *regset)
 261 {
 262         return tsk_used_math(target) ? regset->n : 0;
 263 }
 264
 265 int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
 266 {
 267         return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
 268 }
 269
 270 int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
 271                 unsigned int pos, unsigned int count,
 272                 void *kbuf, void __user *ubuf)
 273 {
 274         int ret;
 275
 276         if (!cpu_has_fxsr)
 277                 return -ENODEV;
 278
 279         ret = init_fpu(target);
 280         if (ret)
 281                 return ret;
 282
 283         sanitize_i387_state(target);
 284
 285         return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 286                                    &target->thread.fpu.state->fxsave, 0, -1);
 287 }
 288
 289 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
 290                 unsigned int pos, unsigned int count,
 291                 const void *kbuf, const void __user *ubuf)
 292 {
 293         int ret;
 294
 295         if (!cpu_has_fxsr)
 296                 return -ENODEV;
 297
 298         ret = init_fpu(target);
 299         if (ret)
 300                 return ret;
 301
 302         sanitize_i387_state(target);
 303
 304         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 305                                  &target->thread.fpu.state->fxsave, 0, -1);
 306
 307         /*
 308          * mxcsr reserved bits must be masked to zero for security reasons.
 309          */
 310         target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 311
 312         /*
 313          * update the header bits in the xsave header, indicating the
 314          * presence of FP and SSE state.
 315          */
 316         if (cpu_has_xsave)
 317                 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
 318
 319         return ret;
 320 }
 321
 322 int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
 323                 unsigned int pos, unsigned int count,
 324                 void *kbuf, void __user *ubuf)
 325 {
 326         int ret;
 327
 328         if (!cpu_has_xsave)
 329                 return -ENODEV;
 330
 331         ret = init_fpu(target);
 332         if (ret)
 333                 return ret;
 334
 335         /*
 336          * Copy the 48bytes defined by the software first into the xstate
 337          * memory layout in the thread struct, so that we can copy the entire
 338          * xstateregs to the user using one user_regset_copyout().
 339          */
 340         memcpy(&target->thread.fpu.state->fxsave.sw_reserved,
 341                xstate_fx_sw_bytes, sizeof(xstate_fx_sw_bytes));
 342
 343         /*
 344          * Copy the xstate memory layout.
 345          */
 346         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 347                                   &target->thread.fpu.state->xsave, 0, -1);
 348         return ret;
 349 }
 350
 351 int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
 352                   unsigned int pos, unsigned int count,
 353                   const void *kbuf, const void __user *ubuf)
 354 {
 355         int ret;
 356         struct xsave_hdr_struct *xsave_hdr;
 357
 358         if (!cpu_has_xsave)
 359                 return -ENODEV;
 360
 361         ret = init_fpu(target);
 362         if (ret)
 363                 return ret;
 364
 365         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 366                                  &target->thread.fpu.state->xsave, 0, -1);
 367
 368         /*
 369          * mxcsr reserved bits must be masked to zero for security reasons.
 370          */
 371         target->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
 372
 373         xsave_hdr = &target->thread.fpu.state->xsave.xsave_hdr;
 374
 375         xsave_hdr->xstate_bv &= pcntxt_mask;
 376         /*
 377          * These bits must be zero.
 378          */
 379         memset(xsave_hdr->reserved, 0, 48);
 380
 381         return ret;
 382 }
 383
 384 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
 385
 386 /*
 387  * FPU tag word conversions.
 388  */
 389
 390 static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
 391 {
 392         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
 393
 394         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
 395         tmp = ~twd;
 396         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
 397         /* and move the valid bits to the lower byte. */
 398         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
 399         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
 400         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
 401
 402         return tmp;
 403 }
 404
 405 #define FPREG_ADDR(f, n)        ((void *)&(f)->st_space + (n) * 16)
 406 #define FP_EXP_TAG_VALID        0
 407 #define FP_EXP_TAG_ZERO         1
 408 #define FP_EXP_TAG_SPECIAL      2
 409 #define FP_EXP_TAG_EMPTY        3
 410
 411 static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
 412 {
 413         struct _fpxreg *st;
 414         u32 tos = (fxsave->swd >> 11) & 7;
 415         u32 twd = (unsigned long) fxsave->twd;
 416         u32 tag;
 417         u32 ret = 0xffff0000u;
 418         int i;
 419
 420         for (i = 0; i < 8; i++, twd >>= 1) {
 421                 if (twd & 0x1) {
 422                         st = FPREG_ADDR(fxsave, (i - tos) & 7);
 423
 424                         switch (st->exponent & 0x7fff) {
 425                         case 0x7fff:
 426                                 tag = FP_EXP_TAG_SPECIAL;
 427                                 break;
 428                         case 0x0000:
 429                                 if (!st->significand[0] &&
 430                                     !st->significand[1] &&
 431                                     !st->significand[2] &&
 432                                     !st->significand[3])
 433                                         tag = FP_EXP_TAG_ZERO;
 434                                 else
 435                                         tag = FP_EXP_TAG_SPECIAL;
 436                                 break;
 437                         default:
 438                                 if (st->significand[3] & 0x8000)
 439                                         tag = FP_EXP_TAG_VALID;
 440                                 else
 441                                         tag = FP_EXP_TAG_SPECIAL;
 442                                 break;
 443                         }
 444                 } else {
 445                         tag = FP_EXP_TAG_EMPTY;
 446                 }
 447                 ret |= tag << (2 * i);
 448         }
 449         return ret;
 450 }
 451
 452 /*
 453  * FXSR floating point environment conversions.
 454  */
 455
 456 void
 457 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
 458 {
 459         struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
 460         struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
 461         struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
 462         int i;
 463
 464         env->cwd = fxsave->cwd | 0xffff0000u;
 465         env->swd = fxsave->swd | 0xffff0000u;
 466         env->twd = twd_fxsr_to_i387(fxsave);
 467
 468 #ifdef CONFIG_X86_64
 469         env->fip = fxsave->rip;
 470         env->foo = fxsave->rdp;
 471         /*
 472          * should be actually ds/cs at fpu exception time, but
 473          * that information is not available in 64bit mode.
 474          */
 475         env->fcs = task_pt_regs(tsk)->cs;
 476         if (tsk == current) {
 477                 savesegment(ds, env->fos);
 478         } else {
 479                 env->fos = tsk->thread.ds;
 480         }
 481         env->fos |= 0xffff0000;
 482 #else
 483         env->fip = fxsave->fip;
 484         env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
 485         env->foo = fxsave->foo;
 486         env->fos = fxsave->fos;
 487 #endif
 488
 489         for (i = 0; i < 8; ++i)
 490                 memcpy(&to[i], &from[i], sizeof(to[0]));
 491 }
 492
 493 void convert_to_fxsr(struct task_struct *tsk,
 494                      const struct user_i387_ia32_struct *env)
 495
 496 {
 497         struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
 498         struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
 499         struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
 500         int i;
 501
 502         fxsave->cwd = env->cwd;
 503         fxsave->swd = env->swd;
 504         fxsave->twd = twd_i387_to_fxsr(env->twd);
 505         fxsave->fop = (u16) ((u32) env->fcs >> 16);
 506 #ifdef CONFIG_X86_64
 507         fxsave->rip = env->fip;
 508         fxsave->rdp = env->foo;
 509         /* cs and ds ignored */
 510 #else
 511         fxsave->fip = env->fip;
 512         fxsave->fcs = (env->fcs & 0xffff);
 513         fxsave->foo = env->foo;
 514         fxsave->fos = env->fos;
 515 #endif
 516
 517         for (i = 0; i < 8; ++i)
 518                 memcpy(&to[i], &from[i], sizeof(from[0]));
 519 }
 520
 521 int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 522                unsigned int pos, unsigned int count,
 523                void *kbuf, void __user *ubuf)
 524 {
 525         struct user_i387_ia32_struct env;
 526         int ret;
 527
 528         ret = init_fpu(target);
 529         if (ret)
 530                 return ret;
 531
 532         if (!static_cpu_has(X86_FEATURE_FPU))
 533                 return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
 534
 535         if (!cpu_has_fxsr)
 536                 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 537                                            &target->thread.fpu.state->fsave, 0,
 538                                            -1);
 539
 540         sanitize_i387_state(target);
 541
 542         if (kbuf && pos == 0 && count == sizeof(env)) {
 543                 convert_from_fxsr(kbuf, target);
 544                 return 0;
 545         }
 546
 547         convert_from_fxsr(&env, target);
 548
 549         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 550 }
 551
 552 int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 553                unsigned int pos, unsigned int count,
 554                const void *kbuf, const void __user *ubuf)
 555 {
 556         struct user_i387_ia32_struct env;
 557         int ret;
 558
 559         ret = init_fpu(target);
 560         if (ret)
 561                 return ret;
 562
 563         sanitize_i387_state(target);
 564
 565         if (!static_cpu_has(X86_FEATURE_FPU))
 566                 return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
 567
 568         if (!cpu_has_fxsr)
 569                 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 570                                           &target->thread.fpu.state->fsave, 0,
 571                                           -1);
 572
 573         if (pos > 0 || count < sizeof(env))
 574                 convert_from_fxsr(&env, target);
 575
 576         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 577         if (!ret)
 578                 convert_to_fxsr(target, &env);
 579
 580         /*
 581          * update the header bit in the xsave header, indicating the
 582          * presence of FP.
 583          */
 584         if (cpu_has_xsave)
 585                 target->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FP;
 586         return ret;
 587 }
 588
 589 /*
 590  * FPU state for core dumps.
 591  * This is only used for a.out dumps now.
 592  * It is declared generically using elf_fpregset_t (which is
 593  * struct user_i387_struct) but is in fact only used for 32-bit
 594  * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
 595  */
 596 int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
 597 {
 598         struct task_struct *tsk = current;
 599         int fpvalid;
 600
 601         fpvalid = !!used_math();
 602         if (fpvalid)
 603                 fpvalid = !fpregs_get(tsk, NULL,
 604                                       0, sizeof(struct user_i387_ia32_struct),
 605                                       fpu, NULL);
 606
 607         return fpvalid;
 608 }
 609 EXPORT_SYMBOL(dump_fpu);
 610
 611 #endif  /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */
 612
 613 static int __init no_387(char *s)
 614 {
 615         setup_clear_cpu_cap(X86_FEATURE_FPU);
 616         return 1;
 617 }
 618
 619 __setup("no387", no_387);
 620
 621 void fpu_detect(struct cpuinfo_x86 *c)
 622 {
 623         unsigned long cr0;
 624         u16 fsw, fcw;
 625
 626         fsw = fcw = 0xffff;
 627
 628         cr0 = read_cr0();
 629         cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
 630         write_cr0(cr0);
 631
 632         asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
 633                      : "+m" (fsw), "+m" (fcw));
 634
 635         if (fsw == 0 && (fcw & 0x103f) == 0x003f)
 636                 set_cpu_cap(c, X86_FEATURE_FPU);
 637         else
 638                 clear_cpu_cap(c, X86_FEATURE_FPU);
 639
 640         /* The final cr0 value is set in fpu_init() */
 641 }