KVM: x86 emulator: access GPRs on demand

Instead of populating the entire register file, read in registers
as they are accessed, and write back only the modified ones.  This
saves a VMREAD and VMWRITE on Intel (for rsp, since it is not usually
used during emulation), and a two 128-byte copies for the registers.

Signed-off-by: Avi Kivity <avi@redhat.com>
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index c764f43b71c5c851f46be710023d4945f1df3c76..282aee5d6ac1f662f5791b536884dff21b63b279 100644 (file)
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -85,6 +85,19 @@ struct x86_instruction_info {
 #define X86EMUL_INTERCEPTED     6 /* Intercepted by nested VMCB/VMCS */
 
 struct x86_emulate_ops {
+       /*
+        * read_gpr: read a general purpose register (rax - r15)
+        *
+        * @reg: gpr number.
+        */
+       ulong (*read_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg);
+       /*
+        * write_gpr: write a general purpose register (rax - r15)
+        *
+        * @reg: gpr number.
+        * @val: value to write.
+        */
+       void (*write_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val);
        /*
         * read_std: Read bytes of standard (non-emulated/special) memory.
         *           Used for descriptor reading.
@@ -281,8 +294,10 @@ struct x86_emulate_ctxt {
        bool rip_relative;
        unsigned long _eip;
        struct operand memop;
+       u32 regs_valid;  /* bitmaps of registers in _regs[] that can be read */
+       u32 regs_dirty;  /* bitmaps of registers in _regs[] that have been written */
        /* Fields above regs are cleared together. */
-       unsigned long regs[NR_VCPU_REGS];
+       unsigned long _regs[NR_VCPU_REGS];
        struct operand *memopp;
        struct fetch_cache fetch;
        struct read_cache io_read;
@@ -394,4 +409,7 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
                         u16 tss_selector, int idt_index, int reason,
                         bool has_error_code, u32 error_code);
 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq);
+void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt);
+void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt);
+
 #endif /* _ASM_X86_KVM_X86_EMULATE_H */

diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index e8fb6c5c6c0a99742510ce060b27a64664503ee4..5e27ba532613ea6e5963289c33784f19c9e243b0 100644 (file)
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -202,6 +202,42 @@ struct gprefix {
 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
 #define EFLG_RESERVED_ONE_MASK 2
 
+static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
+{
+       if (!(ctxt->regs_valid & (1 << nr))) {
+               ctxt->regs_valid |= 1 << nr;
+               ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
+       }
+       return ctxt->_regs[nr];
+}
+
+static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
+{
+       ctxt->regs_valid |= 1 << nr;
+       ctxt->regs_dirty |= 1 << nr;
+       return &ctxt->_regs[nr];
+}
+
+static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
+{
+       reg_read(ctxt, nr);
+       return reg_write(ctxt, nr);
+}
+
+static void writeback_registers(struct x86_emulate_ctxt *ctxt)
+{
+       unsigned reg;
+
+       for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
+               ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
+}
+
+static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
+{
+       ctxt->regs_dirty = 0;
+       ctxt->regs_valid = 0;
+}
+
 /*
  * Instruction emulation:
  * Most instructions are emulated directly via a fragment of inline assembly
@@ -374,8 +410,8 @@ struct gprefix {
 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)                 \
        do {                                                            \
                unsigned long _tmp;                                     \
-               ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX];              \
-               ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX];              \
+               ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX);            \
+               ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX);            \
                                                                        \
                __asm__ __volatile__ (                                  \
                        _PRE_EFLAGS("0", "5", "1")                      \
@@ -494,7 +530,7 @@ register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, in
 
 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
 {
-       masked_increment(&ctxt->regs[VCPU_REGS_RSP], stack_mask(ctxt), inc);
+       masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
 }
 
 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
@@ -786,14 +822,15 @@ static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
  * pointer into the block that addresses the relevant register.
  * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
  */
-static void *decode_register(u8 modrm_reg, unsigned long *regs,
+static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
                             int highbyte_regs)
 {
        void *p;
 
-       p = &regs[modrm_reg];
        if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
-               p = (unsigned char *)&regs[modrm_reg & 3] + 1;
+               p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
+       else
+               p = reg_rmw(ctxt, modrm_reg);
        return p;
 }
 
@@ -982,10 +1019,10 @@ static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
 
        op->type = OP_REG;
        if (ctxt->d & ByteOp) {
-               op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
+               op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
                op->bytes = 1;
        } else {
-               op->addr.reg = decode_register(reg, ctxt->regs, 0);
+               op->addr.reg = decode_register(ctxt, reg, 0);
                op->bytes = ctxt->op_bytes;
        }
        fetch_register_operand(op);
@@ -1020,8 +1057,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
        if (ctxt->modrm_mod == 3) {
                op->type = OP_REG;
                op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
-               op->addr.reg = decode_register(ctxt->modrm_rm,
-                                              ctxt->regs, ctxt->d & ByteOp);
+               op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
                if (ctxt->d & Sse) {
                        op->type = OP_XMM;
                        op->bytes = 16;
@@ -1042,10 +1078,10 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
        op->type = OP_MEM;
 
        if (ctxt->ad_bytes == 2) {
-               unsigned bx = ctxt->regs[VCPU_REGS_RBX];
-               unsigned bp = ctxt->regs[VCPU_REGS_RBP];
-               unsigned si = ctxt->regs[VCPU_REGS_RSI];
-               unsigned di = ctxt->regs[VCPU_REGS_RDI];
+               unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
+               unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
+               unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
+               unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
 
                /* 16-bit ModR/M decode. */
                switch (ctxt->modrm_mod) {
@@ -1102,17 +1138,17 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
                        if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
                                modrm_ea += insn_fetch(s32, ctxt);
                        else {
-                               modrm_ea += ctxt->regs[base_reg];
+                               modrm_ea += reg_read(ctxt, base_reg);
                                adjust_modrm_seg(ctxt, base_reg);
                        }
                        if (index_reg != 4)
-                               modrm_ea += ctxt->regs[index_reg] << scale;
+                               modrm_ea += reg_read(ctxt, index_reg) << scale;
                } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
                        if (ctxt->mode == X86EMUL_MODE_PROT64)
                                ctxt->rip_relative = 1;
                } else {
                        base_reg = ctxt->modrm_rm;
-                       modrm_ea += ctxt->regs[base_reg];
+                       modrm_ea += reg_read(ctxt, base_reg);
                        adjust_modrm_seg(ctxt, base_reg);
                }
                switch (ctxt->modrm_mod) {
@@ -1250,10 +1286,10 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
        if (rc->pos == rc->end) { /* refill pio read ahead */
                unsigned int in_page, n;
                unsigned int count = ctxt->rep_prefix ?
-                       address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
+                       address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
                in_page = (ctxt->eflags & EFLG_DF) ?
-                       offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
-                       PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
+                       offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
+                       PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
                n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
                        count);
                if (n == 0)
@@ -1533,7 +1569,7 @@ static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
        struct segmented_address addr;
 
        rsp_increment(ctxt, -bytes);
-       addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt);
+       addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
        addr.seg = VCPU_SREG_SS;
 
        return segmented_write(ctxt, addr, data, bytes);
@@ -1552,7 +1588,7 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt,
        int rc;
        struct segmented_address addr;
 
-       addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt);
+       addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
        addr.seg = VCPU_SREG_SS;
        rc = segmented_read(ctxt, addr, dest, len);
        if (rc != X86EMUL_CONTINUE)
@@ -1620,26 +1656,28 @@ static int em_enter(struct x86_emulate_ctxt *ctxt)
        int rc;
        unsigned frame_size = ctxt->src.val;
        unsigned nesting_level = ctxt->src2.val & 31;
+       ulong rbp;
 
        if (nesting_level)
                return X86EMUL_UNHANDLEABLE;
 
-       rc = push(ctxt, &ctxt->regs[VCPU_REGS_RBP], stack_size(ctxt));
+       rbp = reg_read(ctxt, VCPU_REGS_RBP);
+       rc = push(ctxt, &rbp, stack_size(ctxt));
        if (rc != X86EMUL_CONTINUE)
                return rc;
-       assign_masked(&ctxt->regs[VCPU_REGS_RBP], ctxt->regs[VCPU_REGS_RSP],
+       assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
                      stack_mask(ctxt));
-       assign_masked(&ctxt->regs[VCPU_REGS_RSP],
-                     ctxt->regs[VCPU_REGS_RSP] - frame_size,
+       assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
+                     reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
                      stack_mask(ctxt));
        return X86EMUL_CONTINUE;
 }
 
 static int em_leave(struct x86_emulate_ctxt *ctxt)
 {
-       assign_masked(&ctxt->regs[VCPU_REGS_RSP], ctxt->regs[VCPU_REGS_RBP],
+       assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
                      stack_mask(ctxt));
-       return emulate_pop(ctxt, &ctxt->regs[VCPU_REGS_RBP], ctxt->op_bytes);
+       return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
 }
 
 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
@@ -1667,13 +1705,13 @@ static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
 
 static int em_pusha(struct x86_emulate_ctxt *ctxt)
 {
-       unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
+       unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
        int rc = X86EMUL_CONTINUE;
        int reg = VCPU_REGS_RAX;
 
        while (reg <= VCPU_REGS_RDI) {
                (reg == VCPU_REGS_RSP) ?
-               (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
+               (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
 
                rc = em_push(ctxt);
                if (rc != X86EMUL_CONTINUE)
@@ -1702,7 +1740,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
                        --reg;
                }
 
-               rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
+               rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
                if (rc != X86EMUL_CONTINUE)
                        break;
                --reg;
@@ -1710,7 +1748,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
        return rc;
 }
 
-int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
+static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
 {
        struct x86_emulate_ops *ops = ctxt->ops;
        int rc;
@@ -1759,11 +1797,22 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
        return rc;
 }
 
+int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
+{
+       int rc;
+
+       invalidate_registers(ctxt);
+       rc = __emulate_int_real(ctxt, irq);
+       if (rc == X86EMUL_CONTINUE)
+               writeback_registers(ctxt);
+       return rc;
+}
+
 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
 {
        switch(ctxt->mode) {
        case X86EMUL_MODE_REAL:
-               return emulate_int_real(ctxt, irq);
+               return __emulate_int_real(ctxt, irq);
        case X86EMUL_MODE_VM86:
        case X86EMUL_MODE_PROT16:
        case X86EMUL_MODE_PROT32:
@@ -1970,14 +2019,14 @@ static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
 {
        u64 old = ctxt->dst.orig_val64;
 
-       if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
-           ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
-               ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
-               ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
+       if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
+           ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
+               *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
+               *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
                ctxt->eflags &= ~EFLG_ZF;
        } else {
-               ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
-                       (u32) ctxt->regs[VCPU_REGS_RBX];
+               ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
+                       (u32) reg_read(ctxt, VCPU_REGS_RBX);
 
                ctxt->eflags |= EFLG_ZF;
        }
@@ -2013,7 +2062,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
 {
        /* Save real source value, then compare EAX against destination. */
        ctxt->src.orig_val = ctxt->src.val;
-       ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
+       ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
        emulate_2op_SrcV(ctxt, "cmp");
 
        if (ctxt->eflags & EFLG_ZF) {
@@ -2022,7 +2071,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
        } else {
                /* Failure: write the value we saw to EAX. */
                ctxt->dst.type = OP_REG;
-               ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
+               ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
        }
        return X86EMUL_CONTINUE;
 }
@@ -2159,10 +2208,10 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt)
        ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
        ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
 
-       ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
+       *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
        if (efer & EFER_LMA) {
 #ifdef CONFIG_X86_64
-               ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
+               *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
 
                ops->get_msr(ctxt,
                             ctxt->mode == X86EMUL_MODE_PROT64 ?
@@ -2241,7 +2290,7 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt)
        ctxt->_eip = msr_data;
 
        ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
-       ctxt->regs[VCPU_REGS_RSP] = msr_data;
+       *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
 
        return X86EMUL_CONTINUE;
 }
@@ -2291,8 +2340,8 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt)
        ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
        ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
 
-       ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
-       ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
+       ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
+       *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
 
        return X86EMUL_CONTINUE;
 }
@@ -2361,14 +2410,14 @@ static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
 {
        tss->ip = ctxt->_eip;
        tss->flag = ctxt->eflags;
-       tss->ax = ctxt->regs[VCPU_REGS_RAX];
-       tss->cx = ctxt->regs[VCPU_REGS_RCX];
-       tss->dx = ctxt->regs[VCPU_REGS_RDX];
-       tss->bx = ctxt->regs[VCPU_REGS_RBX];
-       tss->sp = ctxt->regs[VCPU_REGS_RSP];
-       tss->bp = ctxt->regs[VCPU_REGS_RBP];
-       tss->si = ctxt->regs[VCPU_REGS_RSI];
-       tss->di = ctxt->regs[VCPU_REGS_RDI];
+       tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
+       tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
+       tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
+       tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
+       tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
+       tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
+       tss->si = reg_read(ctxt, VCPU_REGS_RSI);
+       tss->di = reg_read(ctxt, VCPU_REGS_RDI);
 
        tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
        tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
@@ -2384,14 +2433,14 @@ static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
 
        ctxt->_eip = tss->ip;
        ctxt->eflags = tss->flag | 2;
-       ctxt->regs[VCPU_REGS_RAX] = tss->ax;
-       ctxt->regs[VCPU_REGS_RCX] = tss->cx;
-       ctxt->regs[VCPU_REGS_RDX] = tss->dx;
-       ctxt->regs[VCPU_REGS_RBX] = tss->bx;
-       ctxt->regs[VCPU_REGS_RSP] = tss->sp;
-       ctxt->regs[VCPU_REGS_RBP] = tss->bp;
-       ctxt->regs[VCPU_REGS_RSI] = tss->si;
-       ctxt->regs[VCPU_REGS_RDI] = tss->di;
+       *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
+       *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
+       *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
+       *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
+       *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
+       *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
+       *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
+       *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
 
        /*
         * SDM says that segment selectors are loaded before segment
@@ -2476,14 +2525,14 @@ static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
        tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
        tss->eip = ctxt->_eip;
        tss->eflags = ctxt->eflags;
-       tss->eax = ctxt->regs[VCPU_REGS_RAX];
-       tss->ecx = ctxt->regs[VCPU_REGS_RCX];
-       tss->edx = ctxt->regs[VCPU_REGS_RDX];
-       tss->ebx = ctxt->regs[VCPU_REGS_RBX];
-       tss->esp = ctxt->regs[VCPU_REGS_RSP];
-       tss->ebp = ctxt->regs[VCPU_REGS_RBP];
-       tss->esi = ctxt->regs[VCPU_REGS_RSI];
-       tss->edi = ctxt->regs[VCPU_REGS_RDI];
+       tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
+       tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
+       tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
+       tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
+       tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
+       tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
+       tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
+       tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
 
        tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
        tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
@@ -2505,14 +2554,14 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
        ctxt->eflags = tss->eflags | 2;
 
        /* General purpose registers */
-       ctxt->regs[VCPU_REGS_RAX] = tss->eax;
-       ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
-       ctxt->regs[VCPU_REGS_RDX] = tss->edx;
-       ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
-       ctxt->regs[VCPU_REGS_RSP] = tss->esp;
-       ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
-       ctxt->regs[VCPU_REGS_RSI] = tss->esi;
-       ctxt->regs[VCPU_REGS_RDI] = tss->edi;
+       *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
+       *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
+       *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
+       *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
+       *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
+       *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
+       *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
+       *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
 
        /*
         * SDM says that segment selectors are loaded before segment
@@ -2727,14 +2776,17 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
 {
        int rc;
 
+       invalidate_registers(ctxt);
        ctxt->_eip = ctxt->eip;
        ctxt->dst.type = OP_NONE;
 
        rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
                                     has_error_code, error_code);
 
-       if (rc == X86EMUL_CONTINUE)
+       if (rc == X86EMUL_CONTINUE) {
                ctxt->eip = ctxt->_eip;
+               writeback_registers(ctxt);
+       }
 
        return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
 }
@@ -2744,8 +2796,8 @@ static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
 {
        int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
 
-       register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
-       op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
+       register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
+       op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
        op->addr.mem.seg = seg;
 }
 
@@ -2921,7 +2973,7 @@ static int em_cwd(struct x86_emulate_ctxt *ctxt)
 {
        ctxt->dst.type = OP_REG;
        ctxt->dst.bytes = ctxt->src.bytes;
-       ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
+       ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
        ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
 
        return X86EMUL_CONTINUE;
@@ -2932,8 +2984,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
        u64 tsc = 0;
 
        ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
-       ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
-       ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
+       *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
+       *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
        return X86EMUL_CONTINUE;
 }
 
@@ -2941,10 +2993,10 @@ static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
 {
        u64 pmc;
 
-       if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
+       if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
                return emulate_gp(ctxt, 0);
-       ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
-       ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
+       *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
+       *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
        return X86EMUL_CONTINUE;
 }
 
@@ -2986,9 +3038,9 @@ static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
 {
        u64 msr_data;
 
-       msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
-               | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
-       if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
+       msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
+               | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
+       if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
                return emulate_gp(ctxt, 0);
 
        return X86EMUL_CONTINUE;
@@ -2998,11 +3050,11 @@ static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
 {
        u64 msr_data;
 
-       if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
+       if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
                return emulate_gp(ctxt, 0);
 
-       ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
-       ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
+       *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
+       *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
        return X86EMUL_CONTINUE;
 }
 
@@ -3182,8 +3234,8 @@ static int em_lmsw(struct x86_emulate_ctxt *ctxt)
 
 static int em_loop(struct x86_emulate_ctxt *ctxt)
 {
-       register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
-       if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
+       register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
+       if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
            (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
                jmp_rel(ctxt, ctxt->src.val);
 
@@ -3192,7 +3244,7 @@ static int em_loop(struct x86_emulate_ctxt *ctxt)
 
 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
 {
-       if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
+       if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
                jmp_rel(ctxt, ctxt->src.val);
 
        return X86EMUL_CONTINUE;
@@ -3280,20 +3332,20 @@ static int em_cpuid(struct x86_emulate_ctxt *ctxt)
 {
        u32 eax, ebx, ecx, edx;
 
-       eax = ctxt->regs[VCPU_REGS_RAX];
-       ecx = ctxt->regs[VCPU_REGS_RCX];
+       eax = reg_read(ctxt, VCPU_REGS_RAX);
+       ecx = reg_read(ctxt, VCPU_REGS_RCX);
        ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
-       ctxt->regs[VCPU_REGS_RAX] = eax;
-       ctxt->regs[VCPU_REGS_RBX] = ebx;
-       ctxt->regs[VCPU_REGS_RCX] = ecx;
-       ctxt->regs[VCPU_REGS_RDX] = edx;
+       *reg_write(ctxt, VCPU_REGS_RAX) = eax;
+       *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
+       *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
+       *reg_write(ctxt, VCPU_REGS_RDX) = edx;
        return X86EMUL_CONTINUE;
 }
 
 static int em_lahf(struct x86_emulate_ctxt *ctxt)
 {
-       ctxt->regs[VCPU_REGS_RAX] &= ~0xff00UL;
-       ctxt->regs[VCPU_REGS_RAX] |= (ctxt->eflags & 0xff) << 8;
+       *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
+       *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
        return X86EMUL_CONTINUE;
 }
 
@@ -3450,7 +3502,7 @@ static int check_svme(struct x86_emulate_ctxt *ctxt)
 
 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
 {
-       u64 rax = ctxt->regs[VCPU_REGS_RAX];
+       u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
 
        /* Valid physical address? */
        if (rax & 0xffff000000000000ULL)
@@ -3472,7 +3524,7 @@ static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
 {
        u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
-       u64 rcx = ctxt->regs[VCPU_REGS_RCX];
+       u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
 
        if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
            (rcx > 3))
@@ -3930,7 +3982,7 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
        case OpAcc:
                op->type = OP_REG;
                op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
-               op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
+               op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
                fetch_register_operand(op);
                op->orig_val = op->val;
                break;
@@ -3938,19 +3990,19 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
                op->type = OP_MEM;
                op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
                op->addr.mem.ea =
-                       register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
+                       register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
                op->addr.mem.seg = VCPU_SREG_ES;
                op->val = 0;
                break;
        case OpDX:
                op->type = OP_REG;
                op->bytes = 2;
-               op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
+               op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
                fetch_register_operand(op);
                break;
        case OpCL:
                op->bytes = 1;
-               op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
+               op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
                break;
        case OpImmByte:
                rc = decode_imm(ctxt, op, 1, true);
@@ -3981,7 +4033,7 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
                op->type = OP_MEM;
                op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
                op->addr.mem.ea =
-                       register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
+                       register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
                op->addr.mem.seg = seg_override(ctxt);
                op->val = 0;
                break;
@@ -4287,6 +4339,7 @@ static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
                read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
 }
 
+
 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 {
        struct x86_emulate_ops *ops = ctxt->ops;
@@ -4371,7 +4424,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 
        if (ctxt->rep_prefix && (ctxt->d & String)) {
                /* All REP prefixes have the same first termination condition */
-               if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
+               if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
                        ctxt->eip = ctxt->_eip;
                        goto done;
                }
@@ -4444,7 +4497,7 @@ special_insn:
                ctxt->dst.val = ctxt->src.addr.mem.ea;
                break;
        case 0x90 ... 0x97: /* nop / xchg reg, rax */
-               if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
+               if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
                        break;
                rc = em_xchg(ctxt);
                break;
@@ -4472,7 +4525,7 @@ special_insn:
                rc = em_grp2(ctxt);
                break;
        case 0xd2 ... 0xd3:     /* Grp2 */
-               ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
+               ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
                rc = em_grp2(ctxt);
                break;
        case 0xe9: /* jmp rel */
@@ -4527,14 +4580,14 @@ writeback:
 
        if (ctxt->rep_prefix && (ctxt->d & String)) {
                struct read_cache *r = &ctxt->io_read;
-               register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
+               register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
 
                if (!string_insn_completed(ctxt)) {
                        /*
                         * Re-enter guest when pio read ahead buffer is empty
                         * or, if it is not used, after each 1024 iteration.
                         */
-                       if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
+                       if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
                            (r->end == 0 || r->end != r->pos)) {
                                /*
                                 * Reset read cache. Usually happens before
@@ -4542,6 +4595,7 @@ writeback:
                                 * we have to do it here.
                                 */
                                ctxt->mem_read.end = 0;
+                               writeback_registers(ctxt);
                                return EMULATION_RESTART;
                        }
                        goto done; /* skip rip writeback */
@@ -4556,6 +4610,9 @@ done:
        if (rc == X86EMUL_INTERCEPTED)
                return EMULATION_INTERCEPTED;
 
+       if (rc == X86EMUL_CONTINUE)
+               writeback_registers(ctxt);
+
        return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
 
 twobyte_insn:
@@ -4628,3 +4685,13 @@ twobyte_insn:
 cannot_emulate:
        return EMULATION_FAILED;
 }
+
+void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
+{
+       invalidate_registers(ctxt);
+}
+
+void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
+{
+       writeback_registers(ctxt);
+}

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 42bbf4187d20c64febfc6609094fdb8cddd0cc30..e00050ce7a6aed214b2e49cc363e2619040d1d62 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4313,7 +4313,19 @@ static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
        kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx);
 }
 
+static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
+{
+       return kvm_register_read(emul_to_vcpu(ctxt), reg);
+}
+
+static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
+{
+       kvm_register_write(emul_to_vcpu(ctxt), reg, val);
+}
+
 static struct x86_emulate_ops emulate_ops = {
+       .read_gpr            = emulator_read_gpr,
+       .write_gpr           = emulator_write_gpr,
        .read_std            = kvm_read_guest_virt_system,
        .write_std           = kvm_write_guest_virt_system,
        .fetch               = kvm_fetch_guest_virt,
@@ -4348,14 +4360,6 @@ static struct x86_emulate_ops emulate_ops = {
        .get_cpuid           = emulator_get_cpuid,
 };
 
-static void cache_all_regs(struct kvm_vcpu *vcpu)
-{
-       kvm_register_read(vcpu, VCPU_REGS_RAX);
-       kvm_register_read(vcpu, VCPU_REGS_RSP);
-       kvm_register_read(vcpu, VCPU_REGS_RIP);
-       vcpu->arch.regs_dirty = ~0;
-}
-
 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
 {
        u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
@@ -4382,12 +4386,10 @@ static void inject_emulated_exception(struct kvm_vcpu *vcpu)
                kvm_queue_exception(vcpu, ctxt->exception.vector);
 }
 
-static void init_decode_cache(struct x86_emulate_ctxt *ctxt,
-                             const unsigned long *regs)
+static void init_decode_cache(struct x86_emulate_ctxt *ctxt)
 {
        memset(&ctxt->twobyte, 0,
-              (void *)&ctxt->regs - (void *)&ctxt->twobyte);
-       memcpy(ctxt->regs, regs, sizeof(ctxt->regs));
+              (void *)&ctxt->_regs - (void *)&ctxt->twobyte);
 
        ctxt->fetch.start = 0;
        ctxt->fetch.end = 0;
@@ -4402,14 +4404,6 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
        struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
        int cs_db, cs_l;
 
-       /*
-        * TODO: fix emulate.c to use guest_read/write_register
-        * instead of direct ->regs accesses, can save hundred cycles
-        * on Intel for instructions that don't read/change RSP, for
-        * for example.
-        */
-       cache_all_regs(vcpu);
-
        kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 
        ctxt->eflags = kvm_get_rflags(vcpu);
@@ -4421,7 +4415,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
                                                          X86EMUL_MODE_PROT16;
        ctxt->guest_mode = is_guest_mode(vcpu);
 
-       init_decode_cache(ctxt, vcpu->arch.regs);
+       init_decode_cache(ctxt);
        vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
 }
 
@@ -4441,7 +4435,6 @@ int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
                return EMULATE_FAIL;
 
        ctxt->eip = ctxt->_eip;
-       memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
        kvm_rip_write(vcpu, ctxt->eip);
        kvm_set_rflags(vcpu, ctxt->eflags);
 
@@ -4599,7 +4592,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
           changes registers values  during IO operation */
        if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
                vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
-               memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs);
+               emulator_invalidate_register_cache(ctxt);
        }
 
 restart:
@@ -4637,7 +4630,6 @@ restart:
                toggle_interruptibility(vcpu, ctxt->interruptibility);
                kvm_set_rflags(vcpu, ctxt->eflags);
                kvm_make_request(KVM_REQ_EVENT, vcpu);
-               memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
                vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
                kvm_rip_write(vcpu, ctxt->eip);
        } else
@@ -5591,8 +5583,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
                 * that usually, but some bad designed PV devices (vmware
                 * backdoor interface) need this to work
                 */
-               struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
-               memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
+               emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
                vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
        }
        regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
@@ -5723,6 +5714,7 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
 {
        struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
        int ret;
+       unsigned reg;
 
        init_emulate_ctxt(vcpu);
 
@@ -5732,7 +5724,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
        if (ret)
                return EMULATE_FAIL;
 
-       memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
        kvm_rip_write(vcpu, ctxt->eip);
        kvm_set_rflags(vcpu, ctxt->eflags);
        kvm_make_request(KVM_REQ_EVENT, vcpu);