arch/x86/include/asm/fpu/xstate.h

   1 #ifndef __ASM_X86_XSAVE_H
   2 #define __ASM_X86_XSAVE_H
   3
   4 #include <linux/types.h>
   5 #include <asm/processor.h>
   6 #include <linux/uaccess.h>
   7
   8 /* Bit 63 of XCR0 is reserved for future expansion */
   9 #define XSTATE_EXTEND_MASK      (~(XSTATE_FPSSE | (1ULL << 63)))
  10
  11 #define XSTATE_CPUID            0x0000000d
  12
  13 #define FXSAVE_SIZE     512
  14
  15 #define XSAVE_HDR_SIZE      64
  16 #define XSAVE_HDR_OFFSET    FXSAVE_SIZE
  17
  18 #define XSAVE_YMM_SIZE      256
  19 #define XSAVE_YMM_OFFSET    (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)
  20
  21 /* Supported features which support lazy state saving */
  22 #define XSTATE_LAZY     (XSTATE_FP | XSTATE_SSE | XSTATE_YMM                  \
  23                         | XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM)
  24
  25 /* Supported features which require eager state saving */
  26 #define XSTATE_EAGER    (XSTATE_BNDREGS | XSTATE_BNDCSR)
  27
  28 /* All currently supported features */
  29 #define XCNTXT_MASK     (XSTATE_LAZY | XSTATE_EAGER)
  30
  31 #ifdef CONFIG_X86_64
  32 #define REX_PREFIX      "0x48, "
  33 #else
  34 #define REX_PREFIX
  35 #endif
  36
  37 extern unsigned int xstate_size;
  38 extern u64 xfeatures_mask;
  39 extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
  40
  41 extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
  42
  43 /* These macros all use (%edi)/(%rdi) as the single memory argument. */
  44 #define XSAVE           ".byte " REX_PREFIX "0x0f,0xae,0x27"
  45 #define XSAVEOPT        ".byte " REX_PREFIX "0x0f,0xae,0x37"
  46 #define XSAVES          ".byte " REX_PREFIX "0x0f,0xc7,0x2f"
  47 #define XRSTOR          ".byte " REX_PREFIX "0x0f,0xae,0x2f"
  48 #define XRSTORS         ".byte " REX_PREFIX "0x0f,0xc7,0x1f"
  49
  50 /* xstate instruction fault handler: */
  51 #define xstate_fault(__err)             \
  52                                         \
  53         ".section .fixup,\"ax\"\n"      \
  54                                         \
  55         "3:  movl $-1,%[err]\n"         \
  56         "    jmp  2b\n"                 \
  57                                         \
  58         ".previous\n"                   \
  59                                         \
  60         _ASM_EXTABLE(1b, 3b)            \
  61         : [err] "=r" (__err)
  62
  63 /*
  64  * This function is called only during boot time when x86 caps are not set
  65  * up and alternative can not be used yet.
  66  */
  67 static inline int copy_xregs_to_kernel_booting(struct xregs_state *fx)
  68 {
  69         u64 mask = -1;
  70         u32 lmask = mask;
  71         u32 hmask = mask >> 32;
  72         int err = 0;
  73
  74         WARN_ON(system_state != SYSTEM_BOOTING);
  75
  76         if (boot_cpu_has(X86_FEATURE_XSAVES))
  77                 asm volatile("1:"XSAVES"\n\t"
  78                         "2:\n\t"
  79                              xstate_fault(err)
  80                         : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
  81                         :   "memory");
  82         else
  83                 asm volatile("1:"XSAVE"\n\t"
  84                         "2:\n\t"
  85                              xstate_fault(err)
  86                         : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
  87                         :   "memory");
  88         return err;
  89 }
  90
  91 /*
  92  * This function is called only during boot time when x86 caps are not set
  93  * up and alternative can not be used yet.
  94  */
  95 static inline int copy_kernel_to_xregs_booting(struct xregs_state *fx, u64 mask)
  96 {
  97         u32 lmask = mask;
  98         u32 hmask = mask >> 32;
  99         int err = 0;
 100
 101         WARN_ON(system_state != SYSTEM_BOOTING);
 102
 103         if (boot_cpu_has(X86_FEATURE_XSAVES))
 104                 asm volatile("1:"XRSTORS"\n\t"
 105                         "2:\n\t"
 106                              xstate_fault(err)
 107                         : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
 108                         :   "memory");
 109         else
 110                 asm volatile("1:"XRSTOR"\n\t"
 111                         "2:\n\t"
 112                              xstate_fault(err)
 113                         : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
 114                         :   "memory");
 115         return err;
 116 }
 117
 118 /*
 119  * Save processor xstate to xsave area.
 120  */
 121 static inline int copy_xregs_to_kernel(struct xregs_state *fx)
 122 {
 123         u64 mask = -1;
 124         u32 lmask = mask;
 125         u32 hmask = mask >> 32;
 126         int err = 0;
 127
 128         WARN_ON(!alternatives_patched);
 129
 130         /*
 131          * If xsaves is enabled, xsaves replaces xsaveopt because
 132          * it supports compact format and supervisor states in addition to
 133          * modified optimization in xsaveopt.
 134          *
 135          * Otherwise, if xsaveopt is enabled, xsaveopt replaces xsave
 136          * because xsaveopt supports modified optimization which is not
 137          * supported by xsave.
 138          *
 139          * If none of xsaves and xsaveopt is enabled, use xsave.
 140          */
 141         alternative_input_2(
 142                 "1:"XSAVE,
 143                 XSAVEOPT,
 144                 X86_FEATURE_XSAVEOPT,
 145                 XSAVES,
 146                 X86_FEATURE_XSAVES,
 147                 [fx] "D" (fx), "a" (lmask), "d" (hmask) :
 148                 "memory");
 149         asm volatile("2:\n\t"
 150                      xstate_fault(err)
 151                      : "0" (0)
 152                      : "memory");
 153
 154         return err;
 155 }
 156
 157 /*
 158  * Restore processor xstate from xsave area.
 159  */
 160 static inline int copy_kernel_to_xregs(struct xregs_state *fx, u64 mask)
 161 {
 162         int err = 0;
 163         u32 lmask = mask;
 164         u32 hmask = mask >> 32;
 165
 166         /*
 167          * Use xrstors to restore context if it is enabled. xrstors supports
 168          * compacted format of xsave area which is not supported by xrstor.
 169          */
 170         alternative_input(
 171                 "1: " XRSTOR,
 172                 XRSTORS,
 173                 X86_FEATURE_XSAVES,
 174                 "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
 175                 : "memory");
 176
 177         asm volatile("2:\n"
 178                      xstate_fault(err)
 179                      : "0" (0)
 180                      : "memory");
 181
 182         return err;
 183 }
 184
 185 /*
 186  * Save xstate to user space xsave area.
 187  *
 188  * We don't use modified optimization because xrstor/xrstors might track
 189  * a different application.
 190  *
 191  * We don't use compacted format xsave area for
 192  * backward compatibility for old applications which don't understand
 193  * compacted format of xsave area.
 194  */
 195 static inline int copy_xregs_to_user(struct xregs_state __user *buf)
 196 {
 197         int err;
 198
 199         /*
 200          * Clear the xsave header first, so that reserved fields are
 201          * initialized to zero.
 202          */
 203         err = __clear_user(&buf->header, sizeof(buf->header));
 204         if (unlikely(err))
 205                 return -EFAULT;
 206
 207         __asm__ __volatile__(ASM_STAC "\n"
 208                              "1:"XSAVE"\n"
 209                              "2: " ASM_CLAC "\n"
 210                              xstate_fault(err)
 211                              : "D" (buf), "a" (-1), "d" (-1), "0" (0)
 212                              : "memory");
 213         return err;
 214 }
 215
 216 /*
 217  * Restore xstate from user space xsave area.
 218  */
 219 static inline int copy_user_to_xregs(struct xregs_state __user *buf, u64 mask)
 220 {
 221         int err = 0;
 222         struct xregs_state *xstate = ((__force struct xregs_state *)buf);
 223         u32 lmask = mask;
 224         u32 hmask = mask >> 32;
 225
 226         __asm__ __volatile__(ASM_STAC "\n"
 227                              "1:"XRSTOR"\n"
 228                              "2: " ASM_CLAC "\n"
 229                              xstate_fault(err)
 230                              : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0)
 231                              : "memory");       /* memory required? */
 232         return err;
 233 }
 234
 235 void *get_xsave_addr(struct xregs_state *xsave, int xstate);
 236
 237 #endif