Implement mechanism to export some kernel timekeeping data to

usermode, using shared page.  The structures and functions have vdso
prefix, to indicate the intended location of the code in some future.

The versioned per-algorithm data is exported in the format of struct
vdso_timehands, which mostly repeats the content of in-kernel struct
timehands. Usermode reading of the structure can be lockless.
Compatibility export for 32bit processes on 64bit host is also
provided. Kernel also provides usermode with indication about
currently used timecounter, so that libc can fall back to syscall if
configured timecounter is unknown to usermode code.

The shared data updates are initiated both from the tc_windup(), where
a fast task is queued to do the update, and from sysctl handlers which
change timecounter. A manual override switch
kern.timecounter.fast_gettime allows to turn off the mechanism.

Only x86 architectures export the real algorithm data, and there, only
for tsc timecounter. HPET counters page could be exported as well, but
I prefer to not further glue the kernel and libc ABI there until
proper vdso-based solution is developed.

Minimal stubs neccessary for non-x86 architectures to still compile
are provided.

Discussed with:	bde
Reviewed by:	jhb
Tested by:	flo
MFC after:	1 month

21 files changed, 636 insertions, 0 deletions

diff --git a/sys/amd64/include/vdso.h b/sys/amd64/include/vdso.h
new file mode 100644
index 0000000..b81c455
--- /dev/null
+++ b/sys/amd64/include/vdso.h
@@ -0,0 +1,6 @@
+/*-
+ * This file is in the public domain.
+ */
+/* $FreeBSD$ */
+
+#include <x86/vdso.h>
diff --git a/sys/arm/include/vdso.h b/sys/arm/include/vdso.h
new file mode 100644
index 0000000..d6aa162
--- /dev/null
+++ b/sys/arm/include/vdso.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _ARM_VDSO_H
+#define	_ARM_VDSO_H
+
+#define	VDSO_TIMEHANDS_MD			\
+	uint32_t	th_res[8];
+
+#endif
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 95d3de6..d597296 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -79,6 +79,7 @@ libkern/umoddi3.c		standard
 #libkern/arm/strcmp.S		standard
 #libkern/arm/strncmp.S		standard
 #
+kern/subr_dummy_vdso_tc.c	standard
 board_id.h			standard				   \
 	dependency	"$S/arm/conf/genboardid.awk $S/arm/conf/mach-types" \
 	compile-with	"${AWK} -f $S/arm/conf/genboardid.awk $S/arm/conf/mach-types > board_id.h" \
diff --git a/sys/conf/files.ia64 b/sys/conf/files.ia64
index f9a62af..da08545 100644
--- a/sys/conf/files.ia64
+++ b/sys/conf/files.ia64
@@ -138,3 +138,4 @@ libkern/ia64/bswap16.S		standard
 libkern/ia64/bswap32.S		standard
 libkern/memmove.c		standard
 libkern/memset.c		standard
+kern/subr_dummy_vdso_tc.c	standard
diff --git a/sys/conf/files.mips b/sys/conf/files.mips
index 2ae3bc9..a7d650b 100644
--- a/sys/conf/files.mips
+++ b/sys/conf/files.mips
@@ -91,6 +91,7 @@ compat/freebsd32/freebsd32_misc.c	optional	compat_freebsd32
 compat/freebsd32/freebsd32_syscalls.c	optional	compat_freebsd32
 compat/freebsd32/freebsd32_sysent.c	optional	compat_freebsd32
 kern/imgact_elf32.c			optional	compat_freebsd32
+kern/subr_dummy_vdso_tc.c		standard
 mips/mips/freebsd32_machdep.c		optional	compat_freebsd32
 
 kern/kern_clocksource.c		standard
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index e145258..9862bac 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -61,6 +61,7 @@ dev/tsec/if_tsec.c		optional	tsec
 dev/tsec/if_tsec_fdt.c		optional	tsec fdt
 dev/uart/uart_cpu_powerpc.c	optional	uart aim
 kern/kern_clocksource.c		standard
+kern/subr_dummy_vdso_tc.c	standard
 kern/syscalls.c			optional	ktr
 libkern/ashldi3.c		optional	powerpc
 libkern/ashrdi3.c		optional	powerpc
diff --git a/sys/conf/files.sparc64 b/sys/conf/files.sparc64
index 68c0413..2907dd0 100644
--- a/sys/conf/files.sparc64
+++ b/sys/conf/files.sparc64
@@ -61,6 +61,7 @@ dev/syscons/scvtb.c		optional	sc
 dev/uart/uart_cpu_sparc64.c	optional	uart
 dev/uart/uart_kbd_sun.c		optional	uart sc
 kern/kern_clocksource.c		standard
+kern/subr_dummy_vdso_tc.c	standard
 kern/syscalls.c			optional	ktr
 libkern/ffs.c			standard
 libkern/ffsl.c			standard
diff --git a/sys/i386/include/vdso.h b/sys/i386/include/vdso.h
new file mode 100644
index 0000000..b81c455
--- /dev/null
+++ b/sys/i386/include/vdso.h
@@ -0,0 +1,6 @@
+/*-
+ * This file is in the public domain.
+ */
+/* $FreeBSD$ */
+
+#include <x86/vdso.h>
diff --git a/sys/ia64/include/vdso.h b/sys/ia64/include/vdso.h
new file mode 100644
index 0000000..4571daa
--- /dev/null
+++ b/sys/ia64/include/vdso.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _IA64_VDSO_H
+#define	_IA64_VDSO_H
+
+#define	VDSO_TIMEHANDS_MD			\
+	uint32_t	th_res[8];
+
+#ifdef _KERNEL
+#ifdef COMPAT_FREEBSD32
+
+#define	VDSO_TIMEHANDS_MD32	VDSO_TIMEHANDS_MD
+
+#endif
+#endif
+#endif
diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c
index f907526..5d0f494 100644
--- a/sys/kern/imgact_elf.c
+++ b/sys/kern/imgact_elf.c
@@ -1011,6 +1011,10 @@ __elfN(freebsd_fixup)(register_t **stack_base, struct image_params *imgp)
 		AUXARGS_ENTRY(pos, AT_PAGESIZES, imgp->pagesizes);
 		AUXARGS_ENTRY(pos, AT_PAGESIZESLEN, imgp->pagesizeslen);
 	}
+	if (imgp->sysent->sv_timekeep_base != 0) {
+		AUXARGS_ENTRY(pos, AT_TIMEKEEP,
+		    imgp->sysent->sv_timekeep_base);
+	}
 	AUXARGS_ENTRY(pos, AT_STACKPROT, imgp->sysent->sv_shared_page_obj
 	    != NULL && imgp->stack_prot != 0 ? imgp->stack_prot :
 	    imgp->sysent->sv_stackprot);
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index d9baab6..80502e3 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -28,6 +28,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_capsicum.h"
+#include "opt_compat.h"
 #include "opt_hwpmc_hooks.h"
 #include "opt_kdtrace.h"
 #include "opt_ktrace.h"
@@ -64,6 +65,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysent.h>
 #include <sys/shm.h>
 #include <sys/sysctl.h>
+#include <sys/vdso.h>
 #include <sys/vnode.h>
 #include <sys/stat.h>
 #ifdef KTRACE
@@ -1608,10 +1610,76 @@ shared_page_init(void *dummy __unused)
 SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init,
     NULL);
 
+static void
+timehands_update(void *arg)
+{
+	struct sysentvec *sv;
+	struct sf_buf *sf;
+	struct vdso_timehands th;
+	struct vdso_timekeep *tk;
+	uint32_t enabled, idx;
+
+	sv = arg;
+	sx_xlock(&shared_page_alloc_sx);
+	enabled = tc_fill_vdso_timehands(&th);
+	sf = shared_page_write_start(sv->sv_timekeep_off);
+	tk = (void *)(sf_buf_kva(sf) + (sv->sv_timekeep_off & PAGE_MASK));
+	idx = sv->sv_timekeep_curr;
+	atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0);
+	if (++idx >= VDSO_TH_NUM)
+		idx = 0;
+	sv->sv_timekeep_curr = idx;
+	if (++sv->sv_timekeep_gen == 0)
+		sv->sv_timekeep_gen = 1;
+	th.th_gen = 0;
+	if (enabled)
+		tk->tk_th[idx] = th;
+	tk->tk_enabled = enabled;
+	atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
+	tk->tk_current = idx;
+	shared_page_write_end(sf);
+	sx_xunlock(&shared_page_alloc_sx);
+}
+
+#ifdef COMPAT_FREEBSD32
+static void
+timehands_update32(void *arg)
+{
+	struct sysentvec *sv;
+	struct sf_buf *sf;
+	struct vdso_timekeep32 *tk;
+	struct vdso_timehands32 th;
+	uint32_t enabled, idx;
+
+	sv = arg;
+	sx_xlock(&shared_page_alloc_sx);
+	enabled = tc_fill_vdso_timehands32(&th);
+	sf = shared_page_write_start(sv->sv_timekeep_off);
+	tk = (void *)(sf_buf_kva(sf) + (sv->sv_timekeep_off & PAGE_MASK));
+	idx = sv->sv_timekeep_curr;
+	atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0);
+	if (++idx >= VDSO_TH_NUM)
+		idx = 0;
+	sv->sv_timekeep_curr = idx;
+	if (++sv->sv_timekeep_gen == 0)
+		sv->sv_timekeep_gen = 1;
+	th.th_gen = 0;
+	if (enabled)
+		tk->tk_th[idx] = th;
+	tk->tk_enabled = enabled;
+	atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
+	tk->tk_current = idx;
+	shared_page_write_end(sf);
+	sx_xunlock(&shared_page_alloc_sx);
+}
+#endif
+
 void
 exec_sysvec_init(void *param)
 {
 	struct sysentvec *sv;
+	int tk_base;
+	uint32_t tk_ver;
 
 	sv = (struct sysentvec *)param;
 
@@ -1620,4 +1688,29 @@ exec_sysvec_init(void *param)
 	sv->sv_shared_page_obj = shared_page_obj;
 	sv->sv_sigcode_base = sv->sv_shared_page_base +
 	    shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode);
+	tk_ver = VDSO_TK_VER_CURR;
+#ifdef COMPAT_FREEBSD32
+	if ((sv->sv_flags & SV_ILP32) != 0) {
+		tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) +
+		    sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16);
+		KASSERT(tk_base != -1, ("tk_base -1 for 32bit"));
+		EVENTHANDLER_REGISTER(tc_windup, timehands_update32, sv,
+		    EVENTHANDLER_PRI_ANY);
+		shared_page_write(tk_base + offsetof(struct vdso_timekeep32,
+		    tk_ver), sizeof(uint32_t), &tk_ver);
+	} else {
+#endif
+		tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) +
+		    sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16);
+		KASSERT(tk_base != -1, ("tk_base -1 for native"));
+		EVENTHANDLER_REGISTER(tc_windup, timehands_update, sv,
+		    EVENTHANDLER_PRI_ANY);
+		shared_page_write(tk_base + offsetof(struct vdso_timekeep,
+		    tk_ver), sizeof(uint32_t), &tk_ver);
+#ifdef COMPAT_FREEBSD32
+	}
+#endif
+	sv->sv_timekeep_base = sv->sv_shared_page_base + tk_base;
+	sv->sv_timekeep_off = tk_base;
+	EVENTHANDLER_INVOKE(tc_windup);
 }
diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index e272fdd..0b8fefe 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -16,6 +16,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_compat.h"
 #include "opt_ntp.h"
 #include "opt_ffclock.h"
 
@@ -30,8 +31,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/timeffc.h>
 #include <sys/timepps.h>
+#include <sys/taskqueue.h>
 #include <sys/timetc.h>
 #include <sys/timex.h>
+#include <sys/vdso.h>
 
 /*
  * A large step happens on boot.  This constant detects such steps.
@@ -118,8 +121,12 @@ SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
     &timestepwarnings, 0, "Log time steps");
 
 static void tc_windup(void);
+static void tc_windup_push_vdso(void *ctx, int pending);
 static void cpu_tick_calibrate(int);
 
+static struct task tc_windup_push_vdso_task = TASK_INITIALIZER(0,
+    tc_windup_push_vdso,  0);
+
 static int
 sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
 {
@@ -1360,6 +1367,7 @@ tc_windup(void)
 #endif
 
 	timehands = th;
+	taskqueue_enqueue_fast(taskqueue_fast, &tc_windup_push_vdso_task);
 }
 
 /* Report or change the active timecounter hardware. */
@@ -1386,6 +1394,7 @@ sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
 		(void)newtc->tc_get_timecount(newtc);
 
 		timecounter = newtc;
+		EVENTHANDLER_INVOKE(tc_windup);
 		return (0);
 	}
 	return (EINVAL);
@@ -1844,3 +1853,78 @@ cputick2usec(uint64_t tick)
 }
 
 cpu_tick_f	*cpu_ticks = tc_cpu_ticks;
+
+static int vdso_th_enable = 1;
+static int
+sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
+{
+	int old_vdso_th_enable, error;
+
+	old_vdso_th_enable = vdso_th_enable;
+	error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req);
+	if (error != 0)
+		return (error);
+	vdso_th_enable = old_vdso_th_enable;
+	EVENTHANDLER_INVOKE(tc_windup);
+	return (0);
+}
+SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
+    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
+    NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day");
+
+uint32_t
+tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
+{
+	struct timehands *th;
+	uint32_t enabled;
+	int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		vdso_th->th_algo = VDSO_TH_ALGO_1;
+		vdso_th->th_scale = th->th_scale;
+		vdso_th->th_offset_count = th->th_offset_count;
+		vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
+		vdso_th->th_offset = th->th_offset;
+		vdso_th->th_boottime = boottimebin;
+		enabled = cpu_fill_vdso_timehands(vdso_th);
+	} while (gen == 0 || timehands->th_generation != gen);
+	if (!vdso_th_enable)
+		enabled = 0;
+	return (enabled);
+}
+
+#ifdef COMPAT_FREEBSD32
+uint32_t
+tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
+{
+	struct timehands *th;
+	uint32_t enabled;
+	int gen;
+
+	do {
+		th = timehands;
+		gen = th->th_generation;
+		vdso_th32->th_algo = VDSO_TH_ALGO_1;
+		*(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale;
+		vdso_th32->th_offset_count = th->th_offset_count;
+		vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask;
+		vdso_th32->th_offset.sec = th->th_offset.sec;
+		*(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
+		vdso_th32->th_boottime.sec = boottimebin.sec;
+		*(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac;
+		enabled = cpu_fill_vdso_timehands32(vdso_th32);
+	} while (gen == 0 || timehands->th_generation != gen);
+	if (!vdso_th_enable)
+		enabled = 0;
+	return (enabled);
+}
+#endif
+
+static void
+tc_windup_push_vdso(void *ctx, int pending)
+{
+
+	EVENTHANDLER_INVOKE(tc_windup);
+}
diff --git a/sys/kern/subr_dummy_vdso_tc.c b/sys/kern/subr_dummy_vdso_tc.c
new file mode 100644
index 0000000..9c84501
--- /dev/null
+++ b/sys/kern/subr_dummy_vdso_tc.c
@@ -0,0 +1,49 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_compat.h"
+
+#include <sys/param.h>
+#include <sys/vdso.h>
+
+uint32_t
+cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th)
+{
+
+	return (0);
+}
+
+#ifdef COMPAT_FREEBSD32
+uint32_t
+cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
+{
+
+	return (0);
+}
+#endif
diff --git a/sys/mips/include/vdso.h b/sys/mips/include/vdso.h
new file mode 100644
index 0000000..082df1a
--- /dev/null
+++ b/sys/mips/include/vdso.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MIPS_VDSO_H
+#define	_MIPS_VDSO_H
+
+#define	VDSO_TIMEHANDS_MD			\
+	uint32_t	th_res[8];
+
+#ifdef _KERNEL
+#ifdef COMPAT_FREEBSD32
+
+#define	VDSO_TIMEHANDS_MD32	VDSO_TIMEHANDS_MD
+
+#endif
+#endif
+#endif
diff --git a/sys/pc98/include/vdso.h b/sys/pc98/include/vdso.h
new file mode 100644
index 0000000..b81c455
--- /dev/null
+++ b/sys/pc98/include/vdso.h
@@ -0,0 +1,6 @@
+/*-
+ * This file is in the public domain.
+ */
+/* $FreeBSD$ */
+
+#include <x86/vdso.h>
diff --git a/sys/powerpc/include/vdso.h b/sys/powerpc/include/vdso.h
new file mode 100644
index 0000000..0bdafd9
--- /dev/null
+++ b/sys/powerpc/include/vdso.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _POWERPC_VDSO_H
+#define	_POWERPC_VDSO_H
+
+#define	VDSO_TIMEHANDS_MD			\
+	uint32_t	th_res[8];
+
+#ifdef _KERNEL
+#ifdef COMPAT_FREEBSD32
+
+#define	VDSO_TIMEHANDS_MD32	VDSO_TIMEHANDS_MD
+
+#endif
+#endif
+#endif
diff --git a/sys/sparc64/include/vdso.h b/sys/sparc64/include/vdso.h
new file mode 100644
index 0000000..98b291e
--- /dev/null
+++ b/sys/sparc64/include/vdso.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SPARC64_VDSO_H
+#define	_SPARC64_VDSO_H
+
+#define	VDSO_TIMEHANDS_MD			\
+	uint32_t	th_res[8];
+
+#endif
diff --git a/sys/sys/sysent.h b/sys/sys/sysent.h
index 9dd5c1c..22769c2 100644
--- a/sys/sys/sysent.h
+++ b/sys/sys/sysent.h
@@ -124,6 +124,10 @@ struct sysentvec {
 	vm_offset_t	sv_shared_page_base;
 	vm_offset_t	sv_shared_page_len;
 	vm_offset_t	sv_sigcode_base;
+	vm_offset_t	sv_timekeep_base;
+	int		sv_timekeep_off;
+	int		sv_timekeep_curr;
+	uint32_t	sv_timekeep_gen;
 	void		*sv_shared_page_obj;
 	void		(*sv_schedtail)(struct thread *);
 };
diff --git a/sys/sys/vdso.h b/sys/sys/vdso.h
new file mode 100644
index 0000000..9f3f3af
--- /dev/null
+++ b/sys/sys/vdso.h
@@ -0,0 +1,124 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS_VDSO_H
+#define	_SYS_VDSO_H
+
+#include <sys/types.h>
+#include <sys/eventhandler.h>
+#include <machine/vdso.h>
+
+struct vdso_timehands {
+	uint32_t	th_algo;
+	uint32_t	th_gen;
+	uint64_t	th_scale;
+	uint32_t 	th_offset_count;
+	uint32_t	th_counter_mask;
+	struct bintime	th_offset;
+	struct bintime	th_boottime;
+	VDSO_TIMEHANDS_MD
+};
+
+struct vdso_timekeep {
+	uint32_t	tk_ver;
+	uint32_t	tk_enabled;
+	uint32_t	tk_current;
+	struct vdso_timehands	tk_th[];
+};
+
+#define	VDSO_TK_CURRENT_BUSY	0xffffffff
+#define	VDSO_TK_VER_1		0x1
+#define	VDSO_TK_VER_CURR	VDSO_TK_VER_1
+#define	VDSO_TH_ALGO_1		0x1
+
+#ifndef _KERNEL
+
+struct timespec;
+struct timeval;
+struct timezone;
+
+int __vdso_clock_gettime(clockid_t clock_id, struct timespec *ts);
+#pragma weak __vdso_clock_gettime
+
+int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
+#pragma weak __vdso_gettimeofday
+
+u_int __vdso_gettc(const struct vdso_timehands *vdso_th);
+#pragma weak __vdso_gettc
+
+#endif
+
+#ifdef _KERNEL
+
+uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);
+
+/*
+ * The cpu_fill_vdso_timehands() function should fill MD-part of the
+ * struct vdso_timehands, which is both machine- and
+ * timecounter-depended. The return value should be 1 if fast
+ * userspace timecounter is enabled by hardware, and 0 otherwise. The
+ * global sysctl enable override is handled by machine-independed code
+ * after cpu_fill_vdso_timehands() call is made.
+ */
+uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th);
+
+typedef void (*tc_windup_fn)(void *);
+EVENTHANDLER_DECLARE(tc_windup, tc_windup_fn);
+
+#define	VDSO_TH_NUM	4
+
+#ifdef COMPAT_FREEBSD32
+struct bintime32 {
+	uint32_t	sec;
+	uint32_t	frac[2];
+};
+
+struct vdso_timehands32 {
+	uint32_t	th_algo;
+	uint32_t	th_gen;
+	uint32_t	th_scale[2];
+	uint32_t 	th_offset_count;
+	uint32_t	th_counter_mask;
+	struct bintime32	th_offset;
+	struct bintime32	th_boottime;
+	VDSO_TIMEHANDS_MD32
+};
+
+struct vdso_timekeep32 {
+	uint32_t	tk_ver;
+	uint32_t	tk_enabled;
+	uint32_t	tk_current;
+	struct vdso_timehands32	tk_th[];
+};
+
+uint32_t tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32);
+uint32_t cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32);
+
+#endif
+#endif
+
+#endif
diff --git a/sys/x86/include/vdso.h b/sys/x86/include/vdso.h
new file mode 100644
index 0000000..1171420
--- /dev/null
+++ b/sys/x86/include/vdso.h
@@ -0,0 +1,42 @@
+/*-
+ * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _X86_VDSO_H
+#define	_X86_VDSO_H
+
+#define	VDSO_TIMEHANDS_MD			\
+	uint32_t	th_x86_shift;		\
+	uint32_t	th_res[7];
+
+#ifdef _KERNEL
+#ifdef COMPAT_FREEBSD32
+
+#define	VDSO_TIMEHANDS_MD32	VDSO_TIMEHANDS_MD
+
+#endif
+#endif
+#endif
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index 4d1618f..085c339 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -27,6 +27,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_compat.h"
 #include "opt_clock.h"
 
 #include <sys/param.h>
@@ -41,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/power.h>
 #include <sys/smp.h>
+#include <sys/vdso.h>
 #include <machine/clock.h>
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
@@ -604,3 +606,23 @@ tsc_get_timecount_low(struct timecounter *tc)
 	: "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx");
 	return (rv);
 }
+
+uint32_t
+cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th)
+{
+
+	vdso_th->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
+	bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
+	return (timecounter == &tsc_timecounter);
+}
+
+#ifdef COMPAT_FREEBSD32
+uint32_t
+cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
+{
+
+	vdso_th32->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
+	bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res));
+	return (timecounter == &tsc_timecounter);
+}
+#endif