2 * sched_clock for unstable cpu clocks
4 * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Ingo Molnar <mingo@redhat.com>
8 * Guillaume Chazarain <guichaz@gmail.com>
10 * Create a semi stable clock from a mixture of other events, including:
14 * - explicit idle events
16 * We use gtod as base and the unstable clock deltas. The deltas are filtered,
17 * making it monotonic and keeping it within an expected window. This window
18 * is set up using jiffies.
20 * Furthermore, explicit sleep and wakeup hooks allow us to account for time
21 * that is otherwise invisible (TSC gets stopped).
23 * The clock: sched_clock_cpu() is monotonic per cpu, and should be somewhat
24 * consistent between cpus (never more than 1 jiffies difference).
26 #include <linux/sched.h>
27 #include <linux/percpu.h>
28 #include <linux/spinlock.h>
29 #include <linux/ktime.h>
30 #include <linux/module.h>
33 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
35 struct sched_clock_data
{
37 * Raw spinlock - this is a special case: this might be called
38 * from within instrumentation code so we dont want to do any
39 * instrumentation ourselves.
43 unsigned long tick_jiffies
;
50 static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data
, sched_clock_data
);
52 static inline struct sched_clock_data
*this_scd(void)
54 return &__get_cpu_var(sched_clock_data
);
57 static inline struct sched_clock_data
*cpu_sdc(int cpu
)
59 return &per_cpu(sched_clock_data
, cpu
);
62 static __read_mostly
int sched_clock_running
;
64 void sched_clock_init(void)
66 u64 ktime_now
= ktime_to_ns(ktime_get());
67 unsigned long now_jiffies
= jiffies
;
70 for_each_possible_cpu(cpu
) {
71 struct sched_clock_data
*scd
= cpu_sdc(cpu
);
73 scd
->lock
= (raw_spinlock_t
)__RAW_SPIN_LOCK_UNLOCKED
;
74 scd
->tick_jiffies
= now_jiffies
;
77 scd
->tick_gtod
= ktime_now
;
78 scd
->clock
= ktime_now
;
81 sched_clock_running
= 1;
85 * update the percpu scd from the raw @now value
87 * - filter out backward motion
88 * - use jiffies to generate a min,max window to clip the raw values
90 static void __update_sched_clock(struct sched_clock_data
*scd
, u64 now
)
92 unsigned long now_jiffies
= jiffies
;
93 long delta_jiffies
= now_jiffies
- scd
->tick_jiffies
;
94 u64 clock
= scd
->clock
;
95 u64 min_clock
, max_clock
;
96 s64 delta
= now
- scd
->prev_raw
;
98 WARN_ON_ONCE(!irqs_disabled());
100 min_clock
= scd
->tick_gtod
+
101 (delta_jiffies
? delta_jiffies
- 1 : 0) * TICK_NSEC
;
103 if (unlikely(delta
< 0)) {
109 * The clock must stay within a jiffie of the gtod.
110 * But since we may be at the start of a jiffy or the end of one
111 * we add another jiffy buffer.
113 max_clock
= scd
->tick_gtod
+ (2 + delta_jiffies
) * TICK_NSEC
;
115 if (unlikely(clock
+ delta
> max_clock
)) {
116 if (clock
< max_clock
)
125 if (unlikely(clock
< min_clock
))
132 static void lock_double_clock(struct sched_clock_data
*data1
,
133 struct sched_clock_data
*data2
)
136 __raw_spin_lock(&data1
->lock
);
137 __raw_spin_lock(&data2
->lock
);
139 __raw_spin_lock(&data2
->lock
);
140 __raw_spin_lock(&data1
->lock
);
144 u64
sched_clock_cpu(int cpu
)
146 struct sched_clock_data
*scd
= cpu_sdc(cpu
);
149 if (unlikely(!sched_clock_running
))
152 WARN_ON_ONCE(!irqs_disabled());
155 if (cpu
!= raw_smp_processor_id()) {
157 * in order to update a remote cpu's clock based on our
158 * unstable raw time rebase it against:
159 * tick_raw (offset between raw counters)
160 * tick_gotd (tick offset between cpus)
162 struct sched_clock_data
*my_scd
= this_scd();
164 lock_double_clock(scd
, my_scd
);
166 now
-= my_scd
->tick_raw
;
167 now
+= scd
->tick_raw
;
169 now
-= my_scd
->tick_gtod
;
170 now
+= scd
->tick_gtod
;
172 __raw_spin_unlock(&my_scd
->lock
);
174 __raw_spin_lock(&scd
->lock
);
177 __update_sched_clock(scd
, now
);
180 __raw_spin_unlock(&scd
->lock
);
185 void sched_clock_tick(void)
187 struct sched_clock_data
*scd
= this_scd();
188 unsigned long now_jiffies
= jiffies
;
191 if (unlikely(!sched_clock_running
))
194 WARN_ON_ONCE(!irqs_disabled());
197 now_gtod
= ktime_to_ns(ktime_get());
199 __raw_spin_lock(&scd
->lock
);
200 __update_sched_clock(scd
, now
);
202 * update tick_gtod after __update_sched_clock() because that will
203 * already observe 1 new jiffy; adding a new tick_gtod to that would
204 * increase the clock 2 jiffies.
206 scd
->tick_jiffies
= now_jiffies
;
208 scd
->tick_gtod
= now_gtod
;
209 __raw_spin_unlock(&scd
->lock
);
213 * We are going deep-idle (irqs are disabled):
215 void sched_clock_idle_sleep_event(void)
217 sched_clock_cpu(smp_processor_id());
219 EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event
);
222 * We just idled delta nanoseconds (called with irqs disabled):
224 void sched_clock_idle_wakeup_event(u64 delta_ns
)
226 struct sched_clock_data
*scd
= this_scd();
227 u64 now
= sched_clock();
230 * Override the previous timestamp and ignore all
231 * sched_clock() deltas that occured while we idled,
232 * and use the PM-provided delta_ns to advance the
235 __raw_spin_lock(&scd
->lock
);
237 scd
->clock
+= delta_ns
;
238 __raw_spin_unlock(&scd
->lock
);
240 touch_softlockup_watchdog();
242 EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event
);
247 * Scheduler clock - returns current time in nanosec units.
248 * This is default implementation.
249 * Architectures and sub-architectures can override this.
251 unsigned long long __attribute__((weak
)) sched_clock(void)
253 return (unsigned long long)jiffies
* (NSEC_PER_SEC
/ HZ
);