Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[deliverable/linux.git] / drivers / xen / time.c

diff --git a/drivers/xen/time.c b/drivers/xen/time.c
index 71078425c9ea04641cfde0a8c944b055b056ec1d..ac5f23fcafc23939bb518375d853c3af99e896d9 100644 (file)
--- a/drivers/xen/time.c
+++ b/drivers/xen/time.c
@@ -6,6 +6,7 @@
 #include <linux/math64.h>
 #include <linux/gfp.h>
 
+#include <asm/paravirt.h>
 #include <asm/xen/hypervisor.h>
 #include <asm/xen/hypercall.h>
 
@@ -46,27 +47,31 @@ static u64 get64(const u64 *p)
        return ret;
 }
 
-/*
- * Runstate accounting
- */
-void xen_get_runstate_snapshot(struct vcpu_runstate_info *res)
+static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res,
+                                         unsigned int cpu)
 {
        u64 state_time;
        struct vcpu_runstate_info *state;
 
        BUG_ON(preemptible());
 
-       state = this_cpu_ptr(&xen_runstate);
+       state = per_cpu_ptr(&xen_runstate, cpu);
 
-       /*
-        * The runstate info is always updated by the hypervisor on
-        * the current CPU, so there's no need to use anything
-        * stronger than a compiler barrier when fetching it.
-        */
        do {
                state_time = get64(&state->state_entry_time);
+               rmb();  /* Hypervisor might update data. */
                *res = READ_ONCE(*state);
-       } while (get64(&state->state_entry_time) != state_time);
+               rmb();  /* Hypervisor might update data. */
+       } while (get64(&state->state_entry_time) != state_time ||
+                (state_time & XEN_RUNSTATE_UPDATE));
+}
+
+/*
+ * Runstate accounting
+ */
+void xen_get_runstate_snapshot(struct vcpu_runstate_info *res)
+{
+       xen_get_runstate_snapshot_cpu(res, smp_processor_id());
 }
 
 /* return true when a vcpu could run but has no real cpu to run on */
@@ -75,6 +80,14 @@ bool xen_vcpu_stolen(int vcpu)
        return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
 }
 
+u64 xen_steal_clock(int cpu)
+{
+       struct vcpu_runstate_info state;
+
+       xen_get_runstate_snapshot_cpu(&state, cpu);
+       return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
+}
+
 void xen_setup_runstate_info(int cpu)
 {
        struct vcpu_register_runstate_memory_area area;
@@ -82,7 +95,20 @@ void xen_setup_runstate_info(int cpu)
        area.addr.v = &per_cpu(xen_runstate, cpu);
 
        if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
-                              cpu, &area))
+                              xen_vcpu_nr(cpu), &area))
                BUG();
 }
 
+void __init xen_time_setup_guest(void)
+{
+       bool xen_runstate_remote;
+
+       xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable,
+                                       VMASST_TYPE_runstate_update_flag);
+
+       pv_time_ops.steal_clock = xen_steal_clock;
+
+       static_key_slow_inc(&paravirt_steal_enabled);
+       if (xen_runstate_remote)
+               static_key_slow_inc(&paravirt_steal_rq_enabled);
+}