arch/x86/xen/xen-asm_32.S

   1 /*
   2  * Asm versions of Xen pv-ops, suitable for either direct use or
   3  * inlining.  The inline versions are the same as the direct-use
   4  * versions, with the pre- and post-amble chopped off.
   5  *
   6  * This code is encoded for size rather than absolute efficiency, with
   7  * a view to being able to inline as much as possible.
   8  *
   9  * We only bother with direct forms (ie, vcpu in pda) of the
  10  * operations here; the indirect forms are better handled in C, since
  11  * they're generally too large to inline anyway.
  12  */
  13
  14 #include <asm/thread_info.h>
  15 #include <asm/processor-flags.h>
  16 #include <asm/segment.h>
  17 #include <asm/asm.h>
  18
  19 #include <xen/interface/xen.h>
  20
  21 #include "xen-asm.h"
  22
  23 /*
  24  * Force an event check by making a hypercall, but preserve regs
  25  * before making the call.
  26  */
  27 check_events:
  28         push %eax
  29         push %ecx
  30         push %edx
  31         call xen_force_evtchn_callback
  32         pop %edx
  33         pop %ecx
  34         pop %eax
  35         ret
  36
  37 /*
  38  * We can't use sysexit directly, because we're not running in ring0.
  39  * But we can easily fake it up using iret.  Assuming xen_sysexit is
  40  * jumped to with a standard stack frame, we can just strip it back to
  41  * a standard iret frame and use iret.
  42  */
  43 ENTRY(xen_sysexit)
  44         movl PT_EAX(%esp), %eax                 /* Shouldn't be necessary? */
  45         orl $X86_EFLAGS_IF, PT_EFLAGS(%esp)
  46         lea PT_EIP(%esp), %esp
  47
  48         jmp xen_iret
  49 ENDPROC(xen_sysexit)
  50
  51 /*
  52  * This is run where a normal iret would be run, with the same stack setup:
  53  *      8: eflags
  54  *      4: cs
  55  *      esp-> 0: eip
  56  *
  57  * This attempts to make sure that any pending events are dealt with
  58  * on return to usermode, but there is a small window in which an
  59  * event can happen just before entering usermode.  If the nested
  60  * interrupt ends up setting one of the TIF_WORK_MASK pending work
  61  * flags, they will not be tested again before returning to
  62  * usermode. This means that a process can end up with pending work,
  63  * which will be unprocessed until the process enters and leaves the
  64  * kernel again, which could be an unbounded amount of time.  This
  65  * means that a pending signal or reschedule event could be
  66  * indefinitely delayed.
  67  *
  68  * The fix is to notice a nested interrupt in the critical window, and
  69  * if one occurs, then fold the nested interrupt into the current
  70  * interrupt stack frame, and re-process it iteratively rather than
  71  * recursively.  This means that it will exit via the normal path, and
  72  * all pending work will be dealt with appropriately.
  73  *
  74  * Because the nested interrupt handler needs to deal with the current
  75  * stack state in whatever form its in, we keep things simple by only
  76  * using a single register which is pushed/popped on the stack.
  77  */
  78 ENTRY(xen_iret)
  79         /* test eflags for special cases */
  80         testl $(X86_EFLAGS_VM | XEN_EFLAGS_NMI), 8(%esp)
  81         jnz hyper_iret
  82
  83         push %eax
  84         ESP_OFFSET=4    # bytes pushed onto stack
  85
  86         /*
  87          * Store vcpu_info pointer for easy access.  Do it this way to
  88          * avoid having to reload %fs
  89          */
  90 #ifdef CONFIG_SMP
  91         GET_THREAD_INFO(%eax)
  92         movl TI_cpu(%eax), %eax
  93         movl __per_cpu_offset(,%eax,4), %eax
  94         mov xen_vcpu(%eax), %eax
  95 #else
  96         movl xen_vcpu, %eax
  97 #endif
  98
  99         /* check IF state we're restoring */
 100         testb $X86_EFLAGS_IF>>8, 8+1+ESP_OFFSET(%esp)
 101
 102         /*
 103          * Maybe enable events.  Once this happens we could get a
 104          * recursive event, so the critical region starts immediately
 105          * afterwards.  However, if that happens we don't end up
 106          * resuming the code, so we don't have to be worried about
 107          * being preempted to another CPU.
 108          */
 109         setz XEN_vcpu_info_mask(%eax)
 110 xen_iret_start_crit:
 111
 112         /* check for unmasked and pending */
 113         cmpw $0x0001, XEN_vcpu_info_pending(%eax)
 114
 115         /*
 116          * If there's something pending, mask events again so we can
 117          * jump back into xen_hypervisor_callback. Otherwise do not
 118          * touch XEN_vcpu_info_mask.
 119          */
 120         jne 1f
 121         movb $1, XEN_vcpu_info_mask(%eax)
 122
 123 1:      popl %eax
 124
 125         /*
 126          * From this point on the registers are restored and the stack
 127          * updated, so we don't need to worry about it if we're
 128          * preempted
 129          */
 130 iret_restore_end:
 131
 132         /*
 133          * Jump to hypervisor_callback after fixing up the stack.
 134          * Events are masked, so jumping out of the critical region is
 135          * OK.
 136          */
 137         je xen_hypervisor_callback
 138
 139 1:      iret
 140 xen_iret_end_crit:
 141         _ASM_EXTABLE(1b, iret_exc)
 142
 143 hyper_iret:
 144         /* put this out of line since its very rarely used */
 145         jmp hypercall_page + __HYPERVISOR_iret * 32
 146
 147         .globl xen_iret_start_crit, xen_iret_end_crit
 148
 149 /*
 150  * This is called by xen_hypervisor_callback in entry.S when it sees
 151  * that the EIP at the time of interrupt was between
 152  * xen_iret_start_crit and xen_iret_end_crit.  We're passed the EIP in
 153  * %eax so we can do a more refined determination of what to do.
 154  *
 155  * The stack format at this point is:
 156  *      ----------------
 157  *       ss             : (ss/esp may be present if we came from usermode)
 158  *       esp            :
 159  *       eflags         }  outer exception info
 160  *       cs             }
 161  *       eip            }
 162  *      ---------------- <- edi (copy dest)
 163  *       eax            :  outer eax if it hasn't been restored
 164  *      ----------------
 165  *       eflags         }  nested exception info
 166  *       cs             }   (no ss/esp because we're nested
 167  *       eip            }    from the same ring)
 168  *       orig_eax       }<- esi (copy src)
 169  *       - - - - - - - -
 170  *       fs             }
 171  *       es             }
 172  *       ds             }  SAVE_ALL state
 173  *       eax            }
 174  *        :             :
 175  *       ebx            }<- esp
 176  *      ----------------
 177  *
 178  * In order to deliver the nested exception properly, we need to shift
 179  * everything from the return addr up to the error code so it sits
 180  * just under the outer exception info.  This means that when we
 181  * handle the exception, we do it in the context of the outer
 182  * exception rather than starting a new one.
 183  *
 184  * The only caveat is that if the outer eax hasn't been restored yet
 185  * (ie, it's still on stack), we need to insert its value into the
 186  * SAVE_ALL state before going on, since it's usermode state which we
 187  * eventually need to restore.
 188  */
 189 ENTRY(xen_iret_crit_fixup)
 190         /*
 191          * Paranoia: Make sure we're really coming from kernel space.
 192          * One could imagine a case where userspace jumps into the
 193          * critical range address, but just before the CPU delivers a
 194          * GP, it decides to deliver an interrupt instead.  Unlikely?
 195          * Definitely.  Easy to avoid?  Yes.  The Intel documents
 196          * explicitly say that the reported EIP for a bad jump is the
 197          * jump instruction itself, not the destination, but some
 198          * virtual environments get this wrong.
 199          */
 200         movl PT_CS(%esp), %ecx
 201         andl $SEGMENT_RPL_MASK, %ecx
 202         cmpl $USER_RPL, %ecx
 203         je 2f
 204
 205         lea PT_ORIG_EAX(%esp), %esi
 206         lea PT_EFLAGS(%esp), %edi
 207
 208         /*
 209          * If eip is before iret_restore_end then stack
 210          * hasn't been restored yet.
 211          */
 212         cmp $iret_restore_end, %eax
 213         jae 1f
 214
 215         movl 0+4(%edi), %eax            /* copy EAX (just above top of frame) */
 216         movl %eax, PT_EAX(%esp)
 217
 218         lea ESP_OFFSET(%edi), %edi      /* move dest up over saved regs */
 219
 220         /* set up the copy */
 221 1:      std
 222         mov $PT_EIP / 4, %ecx           /* saved regs up to orig_eax */
 223         rep movsl
 224         cld
 225
 226         lea 4(%edi), %esp               /* point esp to new frame */
 227 2:      jmp xen_do_upcall
 228