2 * Copyright 2006 Andi Kleen, SUSE Labs.
3 * Subject to the GNU Public License, v.2
5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
7 * The code should have no internal unresolved relocations.
8 * Check with readelf after changing.
11 /* Disable profiling for userspace code: */
12 #define DISABLE_BRANCH_PROFILING
14 #include <linux/kernel.h>
15 #include <linux/posix-timers.h>
16 #include <linux/time.h>
17 #include <linux/string.h>
18 #include <asm/vsyscall.h>
19 #include <asm/fixmap.h>
20 #include <asm/vgtod.h>
21 #include <asm/timex.h>
23 #include <asm/unistd.h>
26 #define gtod (&VVAR(vsyscall_gtod_data))
28 notrace
static cycle_t
vread_tsc(void)
34 * Empirically, a fence (of type that depends on the CPU)
35 * before rdtsc is enough to ensure that rdtsc is ordered
36 * with respect to loads. The various CPU manuals are unclear
37 * as to whether rdtsc can be reordered with later loads,
38 * but no one has ever seen it happen.
41 ret
= (cycle_t
)vget_cycles();
43 last
= VVAR(vsyscall_gtod_data
).clock
.cycle_last
;
45 if (likely(ret
>= last
))
49 * GCC likes to generate cmov here, but this branch is extremely
50 * predictable (it's just a funciton of time and the likely is
51 * very likely) and there's a data dependence, so force GCC
52 * to generate a branch instead. I don't barrier() because
53 * we don't actually need a barrier, and if this function
54 * ever gets inlined it will generate worse code.
60 static notrace cycle_t
vread_hpet(void)
62 return readl((const void __iomem
*)fix_to_virt(VSYSCALL_HPET
) + 0xf0);
65 notrace
static long vdso_fallback_gettime(long clock
, struct timespec
*ts
)
68 asm("syscall" : "=a" (ret
) :
69 "0" (__NR_clock_gettime
),"D" (clock
), "S" (ts
) : "memory");
73 notrace
static long vdso_fallback_gtod(struct timeval
*tv
, struct timezone
*tz
)
77 asm("syscall" : "=a" (ret
) :
78 "0" (__NR_gettimeofday
), "D" (tv
), "S" (tz
) : "memory");
83 notrace
static inline u64
vgetsns(void)
87 if (gtod
->clock
.vclock_mode
== VCLOCK_TSC
)
89 else if (gtod
->clock
.vclock_mode
== VCLOCK_HPET
)
90 cycles
= vread_hpet();
93 v
= (cycles
- gtod
->clock
.cycle_last
) & gtod
->clock
.mask
;
94 return v
* gtod
->clock
.mult
;
97 /* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
98 notrace
static int __always_inline
do_realtime(struct timespec
*ts
)
106 seq
= read_seqcount_begin(>od
->seq
);
107 mode
= gtod
->clock
.vclock_mode
;
108 ts
->tv_sec
= gtod
->wall_time_sec
;
109 ns
= gtod
->wall_time_snsec
;
111 ns
>>= gtod
->clock
.shift
;
112 } while (unlikely(read_seqcount_retry(>od
->seq
, seq
)));
114 timespec_add_ns(ts
, ns
);
118 notrace
static int do_monotonic(struct timespec
*ts
)
126 seq
= read_seqcount_begin(>od
->seq
);
127 mode
= gtod
->clock
.vclock_mode
;
128 ts
->tv_sec
= gtod
->monotonic_time_sec
;
129 ns
= gtod
->monotonic_time_snsec
;
131 ns
>>= gtod
->clock
.shift
;
132 } while (unlikely(read_seqcount_retry(>od
->seq
, seq
)));
133 timespec_add_ns(ts
, ns
);
138 notrace
static int do_realtime_coarse(struct timespec
*ts
)
142 seq
= read_seqcount_begin(>od
->seq
);
143 ts
->tv_sec
= gtod
->wall_time_coarse
.tv_sec
;
144 ts
->tv_nsec
= gtod
->wall_time_coarse
.tv_nsec
;
145 } while (unlikely(read_seqcount_retry(>od
->seq
, seq
)));
149 notrace
static int do_monotonic_coarse(struct timespec
*ts
)
153 seq
= read_seqcount_begin(>od
->seq
);
154 ts
->tv_sec
= gtod
->monotonic_time_coarse
.tv_sec
;
155 ts
->tv_nsec
= gtod
->monotonic_time_coarse
.tv_nsec
;
156 } while (unlikely(read_seqcount_retry(>od
->seq
, seq
)));
161 notrace
int __vdso_clock_gettime(clockid_t clock
, struct timespec
*ts
)
163 int ret
= VCLOCK_NONE
;
167 ret
= do_realtime(ts
);
169 case CLOCK_MONOTONIC
:
170 ret
= do_monotonic(ts
);
172 case CLOCK_REALTIME_COARSE
:
173 return do_realtime_coarse(ts
);
174 case CLOCK_MONOTONIC_COARSE
:
175 return do_monotonic_coarse(ts
);
178 if (ret
== VCLOCK_NONE
)
179 return vdso_fallback_gettime(clock
, ts
);
182 int clock_gettime(clockid_t
, struct timespec
*)
183 __attribute__((weak
, alias("__vdso_clock_gettime")));
185 notrace
int __vdso_gettimeofday(struct timeval
*tv
, struct timezone
*tz
)
187 long ret
= VCLOCK_NONE
;
189 if (likely(tv
!= NULL
)) {
190 BUILD_BUG_ON(offsetof(struct timeval
, tv_usec
) !=
191 offsetof(struct timespec
, tv_nsec
) ||
192 sizeof(*tv
) != sizeof(struct timespec
));
193 ret
= do_realtime((struct timespec
*)tv
);
196 if (unlikely(tz
!= NULL
)) {
197 /* Avoid memcpy. Some old compilers fail to inline it */
198 tz
->tz_minuteswest
= gtod
->sys_tz
.tz_minuteswest
;
199 tz
->tz_dsttime
= gtod
->sys_tz
.tz_dsttime
;
202 if (ret
== VCLOCK_NONE
)
203 return vdso_fallback_gtod(tv
, tz
);
206 int gettimeofday(struct timeval
*, struct timezone
*)
207 __attribute__((weak
, alias("__vdso_gettimeofday")));
210 * This will break when the xtime seconds get inaccurate, but that is
213 notrace
time_t __vdso_time(time_t *t
)
215 /* This is atomic on x86_64 so we don't need any locks. */
216 time_t result
= ACCESS_ONCE(VVAR(vsyscall_gtod_data
).wall_time_sec
);
223 __attribute__((weak
, alias("__vdso_time")));
This page took 0.049108 seconds and 5 git commands to generate.