diff options
author | kib <kib@FreeBSD.org> | 2012-06-22 07:06:40 +0000 |
---|---|---|
committer | kib <kib@FreeBSD.org> | 2012-06-22 07:06:40 +0000 |
commit | 7b36a081088572ce45af9b675b5992adc971506d (patch) | |
tree | 37498e393ce83d19f36a986f13f5aa2c5d5095e6 | |
parent | 860e30dae9add1199b159f743b020e0d9c533c5f (diff) | |
download | FreeBSD-src-7b36a081088572ce45af9b675b5992adc971506d.zip FreeBSD-src-7b36a081088572ce45af9b675b5992adc971506d.tar.gz |
Implement mechanism to export some kernel timekeeping data to
usermode, using shared page. The structures and functions have vdso
prefix, to indicate the intended location of the code in some future.
The versioned per-algorithm data is exported in the format of struct
vdso_timehands, which mostly repeats the content of in-kernel struct
timehands. Usermode reading of the structure can be lockless.
Compatibility export for 32bit processes on 64bit host is also
provided. Kernel also provides usermode with indication about
currently used timecounter, so that libc can fall back to syscall if
configured timecounter is unknown to usermode code.
The shared data updates are initiated both from the tc_windup(), where
a fast task is queued to do the update, and from sysctl handlers which
change timecounter. A manual override switch
kern.timecounter.fast_gettime allows to turn off the mechanism.
Only x86 architectures export the real algorithm data, and there, only
for tsc timecounter. HPET counters page could be exported as well, but
I prefer to not further glue the kernel and libc ABI there until
proper vdso-based solution is developed.
Minimal stubs neccessary for non-x86 architectures to still compile
are provided.
Discussed with: bde
Reviewed by: jhb
Tested by: flo
MFC after: 1 month
-rw-r--r-- | sys/amd64/include/vdso.h | 6 | ||||
-rw-r--r-- | sys/arm/include/vdso.h | 34 | ||||
-rw-r--r-- | sys/conf/files.arm | 1 | ||||
-rw-r--r-- | sys/conf/files.ia64 | 1 | ||||
-rw-r--r-- | sys/conf/files.mips | 1 | ||||
-rw-r--r-- | sys/conf/files.powerpc | 1 | ||||
-rw-r--r-- | sys/conf/files.sparc64 | 1 | ||||
-rw-r--r-- | sys/i386/include/vdso.h | 6 | ||||
-rw-r--r-- | sys/ia64/include/vdso.h | 41 | ||||
-rw-r--r-- | sys/kern/imgact_elf.c | 4 | ||||
-rw-r--r-- | sys/kern/kern_exec.c | 93 | ||||
-rw-r--r-- | sys/kern/kern_tc.c | 84 | ||||
-rw-r--r-- | sys/kern/subr_dummy_vdso_tc.c | 49 | ||||
-rw-r--r-- | sys/mips/include/vdso.h | 41 | ||||
-rw-r--r-- | sys/pc98/include/vdso.h | 6 | ||||
-rw-r--r-- | sys/powerpc/include/vdso.h | 41 | ||||
-rw-r--r-- | sys/sparc64/include/vdso.h | 34 | ||||
-rw-r--r-- | sys/sys/sysent.h | 4 | ||||
-rw-r--r-- | sys/sys/vdso.h | 124 | ||||
-rw-r--r-- | sys/x86/include/vdso.h | 42 | ||||
-rw-r--r-- | sys/x86/x86/tsc.c | 22 |
21 files changed, 636 insertions, 0 deletions
diff --git a/sys/amd64/include/vdso.h b/sys/amd64/include/vdso.h new file mode 100644 index 0000000..b81c455 --- /dev/null +++ b/sys/amd64/include/vdso.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include <x86/vdso.h> diff --git a/sys/arm/include/vdso.h b/sys/arm/include/vdso.h new file mode 100644 index 0000000..d6aa162 --- /dev/null +++ b/sys/arm/include/vdso.h @@ -0,0 +1,34 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM_VDSO_H +#define _ARM_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#endif diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 95d3de6..d597296 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -79,6 +79,7 @@ libkern/umoddi3.c standard #libkern/arm/strcmp.S standard #libkern/arm/strncmp.S standard # +kern/subr_dummy_vdso_tc.c standard board_id.h standard \ dependency "$S/arm/conf/genboardid.awk $S/arm/conf/mach-types" \ compile-with "${AWK} -f $S/arm/conf/genboardid.awk $S/arm/conf/mach-types > board_id.h" \ diff --git a/sys/conf/files.ia64 b/sys/conf/files.ia64 index f9a62af..da08545 100644 --- a/sys/conf/files.ia64 +++ b/sys/conf/files.ia64 @@ -138,3 +138,4 @@ libkern/ia64/bswap16.S standard libkern/ia64/bswap32.S standard libkern/memmove.c standard libkern/memset.c standard +kern/subr_dummy_vdso_tc.c standard diff --git a/sys/conf/files.mips b/sys/conf/files.mips index 2ae3bc9..a7d650b 100644 --- a/sys/conf/files.mips +++ b/sys/conf/files.mips @@ -91,6 +91,7 @@ compat/freebsd32/freebsd32_misc.c optional compat_freebsd32 compat/freebsd32/freebsd32_syscalls.c optional compat_freebsd32 compat/freebsd32/freebsd32_sysent.c optional compat_freebsd32 kern/imgact_elf32.c optional compat_freebsd32 +kern/subr_dummy_vdso_tc.c standard mips/mips/freebsd32_machdep.c optional compat_freebsd32 kern/kern_clocksource.c standard diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index e145258..9862bac 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -61,6 +61,7 @@ dev/tsec/if_tsec.c optional tsec dev/tsec/if_tsec_fdt.c optional tsec fdt dev/uart/uart_cpu_powerpc.c optional uart aim kern/kern_clocksource.c standard +kern/subr_dummy_vdso_tc.c standard kern/syscalls.c optional ktr libkern/ashldi3.c optional powerpc libkern/ashrdi3.c optional powerpc diff --git a/sys/conf/files.sparc64 b/sys/conf/files.sparc64 index 68c0413..2907dd0 100644 --- a/sys/conf/files.sparc64 +++ b/sys/conf/files.sparc64 @@ -61,6 +61,7 @@ dev/syscons/scvtb.c optional sc dev/uart/uart_cpu_sparc64.c optional uart dev/uart/uart_kbd_sun.c optional uart sc kern/kern_clocksource.c standard +kern/subr_dummy_vdso_tc.c standard kern/syscalls.c optional ktr libkern/ffs.c standard libkern/ffsl.c standard diff --git a/sys/i386/include/vdso.h b/sys/i386/include/vdso.h new file mode 100644 index 0000000..b81c455 --- /dev/null +++ b/sys/i386/include/vdso.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include <x86/vdso.h> diff --git a/sys/ia64/include/vdso.h b/sys/ia64/include/vdso.h new file mode 100644 index 0000000..4571daa --- /dev/null +++ b/sys/ia64/include/vdso.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _IA64_VDSO_H +#define _IA64_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index f907526..5d0f494 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -1011,6 +1011,10 @@ __elfN(freebsd_fixup)(register_t **stack_base, struct image_params *imgp) AUXARGS_ENTRY(pos, AT_PAGESIZES, imgp->pagesizes); AUXARGS_ENTRY(pos, AT_PAGESIZESLEN, imgp->pagesizeslen); } + if (imgp->sysent->sv_timekeep_base != 0) { + AUXARGS_ENTRY(pos, AT_TIMEKEEP, + imgp->sysent->sv_timekeep_base); + } AUXARGS_ENTRY(pos, AT_STACKPROT, imgp->sysent->sv_shared_page_obj != NULL && imgp->stack_prot != 0 ? imgp->stack_prot : imgp->sysent->sv_stackprot); diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index d9baab6..80502e3 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -28,6 +28,7 @@ __FBSDID("$FreeBSD$"); #include "opt_capsicum.h" +#include "opt_compat.h" #include "opt_hwpmc_hooks.h" #include "opt_kdtrace.h" #include "opt_ktrace.h" @@ -64,6 +65,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysent.h> #include <sys/shm.h> #include <sys/sysctl.h> +#include <sys/vdso.h> #include <sys/vnode.h> #include <sys/stat.h> #ifdef KTRACE @@ -1608,10 +1610,76 @@ shared_page_init(void *dummy __unused) SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init, NULL); +static void +timehands_update(void *arg) +{ + struct sysentvec *sv; + struct sf_buf *sf; + struct vdso_timehands th; + struct vdso_timekeep *tk; + uint32_t enabled, idx; + + sv = arg; + sx_xlock(&shared_page_alloc_sx); + enabled = tc_fill_vdso_timehands(&th); + sf = shared_page_write_start(sv->sv_timekeep_off); + tk = (void *)(sf_buf_kva(sf) + (sv->sv_timekeep_off & PAGE_MASK)); + idx = sv->sv_timekeep_curr; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0); + if (++idx >= VDSO_TH_NUM) + idx = 0; + sv->sv_timekeep_curr = idx; + if (++sv->sv_timekeep_gen == 0) + sv->sv_timekeep_gen = 1; + th.th_gen = 0; + if (enabled) + tk->tk_th[idx] = th; + tk->tk_enabled = enabled; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen); + tk->tk_current = idx; + shared_page_write_end(sf); + sx_xunlock(&shared_page_alloc_sx); +} + +#ifdef COMPAT_FREEBSD32 +static void +timehands_update32(void *arg) +{ + struct sysentvec *sv; + struct sf_buf *sf; + struct vdso_timekeep32 *tk; + struct vdso_timehands32 th; + uint32_t enabled, idx; + + sv = arg; + sx_xlock(&shared_page_alloc_sx); + enabled = tc_fill_vdso_timehands32(&th); + sf = shared_page_write_start(sv->sv_timekeep_off); + tk = (void *)(sf_buf_kva(sf) + (sv->sv_timekeep_off & PAGE_MASK)); + idx = sv->sv_timekeep_curr; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0); + if (++idx >= VDSO_TH_NUM) + idx = 0; + sv->sv_timekeep_curr = idx; + if (++sv->sv_timekeep_gen == 0) + sv->sv_timekeep_gen = 1; + th.th_gen = 0; + if (enabled) + tk->tk_th[idx] = th; + tk->tk_enabled = enabled; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen); + tk->tk_current = idx; + shared_page_write_end(sf); + sx_xunlock(&shared_page_alloc_sx); +} +#endif + void exec_sysvec_init(void *param) { struct sysentvec *sv; + int tk_base; + uint32_t tk_ver; sv = (struct sysentvec *)param; @@ -1620,4 +1688,29 @@ exec_sysvec_init(void *param) sv->sv_shared_page_obj = shared_page_obj; sv->sv_sigcode_base = sv->sv_shared_page_base + shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode); + tk_ver = VDSO_TK_VER_CURR; +#ifdef COMPAT_FREEBSD32 + if ((sv->sv_flags & SV_ILP32) != 0) { + tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) + + sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16); + KASSERT(tk_base != -1, ("tk_base -1 for 32bit")); + EVENTHANDLER_REGISTER(tc_windup, timehands_update32, sv, + EVENTHANDLER_PRI_ANY); + shared_page_write(tk_base + offsetof(struct vdso_timekeep32, + tk_ver), sizeof(uint32_t), &tk_ver); + } else { +#endif + tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) + + sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16); + KASSERT(tk_base != -1, ("tk_base -1 for native")); + EVENTHANDLER_REGISTER(tc_windup, timehands_update, sv, + EVENTHANDLER_PRI_ANY); + shared_page_write(tk_base + offsetof(struct vdso_timekeep, + tk_ver), sizeof(uint32_t), &tk_ver); +#ifdef COMPAT_FREEBSD32 + } +#endif + sv->sv_timekeep_base = sv->sv_shared_page_base + tk_base; + sv->sv_timekeep_off = tk_base; + EVENTHANDLER_INVOKE(tc_windup); } diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c index e272fdd..0b8fefe 100644 --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -16,6 +16,7 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); +#include "opt_compat.h" #include "opt_ntp.h" #include "opt_ffclock.h" @@ -30,8 +31,10 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/timeffc.h> #include <sys/timepps.h> +#include <sys/taskqueue.h> #include <sys/timetc.h> #include <sys/timex.h> +#include <sys/vdso.h> /* * A large step happens on boot. This constant detects such steps. @@ -118,8 +121,12 @@ SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, ×tepwarnings, 0, "Log time steps"); static void tc_windup(void); +static void tc_windup_push_vdso(void *ctx, int pending); static void cpu_tick_calibrate(int); +static struct task tc_windup_push_vdso_task = TASK_INITIALIZER(0, + tc_windup_push_vdso, 0); + static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) { @@ -1360,6 +1367,7 @@ tc_windup(void) #endif timehands = th; + taskqueue_enqueue_fast(taskqueue_fast, &tc_windup_push_vdso_task); } /* Report or change the active timecounter hardware. */ @@ -1386,6 +1394,7 @@ sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) (void)newtc->tc_get_timecount(newtc); timecounter = newtc; + EVENTHANDLER_INVOKE(tc_windup); return (0); } return (EINVAL); @@ -1844,3 +1853,78 @@ cputick2usec(uint64_t tick) } cpu_tick_f *cpu_ticks = tc_cpu_ticks; + +static int vdso_th_enable = 1; +static int +sysctl_fast_gettime(SYSCTL_HANDLER_ARGS) +{ + int old_vdso_th_enable, error; + + old_vdso_th_enable = vdso_th_enable; + error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req); + if (error != 0) + return (error); + vdso_th_enable = old_vdso_th_enable; + EVENTHANDLER_INVOKE(tc_windup); + return (0); +} +SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day"); + +uint32_t +tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) +{ + struct timehands *th; + uint32_t enabled; + int gen; + + do { + th = timehands; + gen = th->th_generation; + vdso_th->th_algo = VDSO_TH_ALGO_1; + vdso_th->th_scale = th->th_scale; + vdso_th->th_offset_count = th->th_offset_count; + vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; + vdso_th->th_offset = th->th_offset; + vdso_th->th_boottime = boottimebin; + enabled = cpu_fill_vdso_timehands(vdso_th); + } while (gen == 0 || timehands->th_generation != gen); + if (!vdso_th_enable) + enabled = 0; + return (enabled); +} + +#ifdef COMPAT_FREEBSD32 +uint32_t +tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) +{ + struct timehands *th; + uint32_t enabled; + int gen; + + do { + th = timehands; + gen = th->th_generation; + vdso_th32->th_algo = VDSO_TH_ALGO_1; + *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; + vdso_th32->th_offset_count = th->th_offset_count; + vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; + vdso_th32->th_offset.sec = th->th_offset.sec; + *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; + vdso_th32->th_boottime.sec = boottimebin.sec; + *(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac; + enabled = cpu_fill_vdso_timehands32(vdso_th32); + } while (gen == 0 || timehands->th_generation != gen); + if (!vdso_th_enable) + enabled = 0; + return (enabled); +} +#endif + +static void +tc_windup_push_vdso(void *ctx, int pending) +{ + + EVENTHANDLER_INVOKE(tc_windup); +} diff --git a/sys/kern/subr_dummy_vdso_tc.c b/sys/kern/subr_dummy_vdso_tc.c new file mode 100644 index 0000000..9c84501 --- /dev/null +++ b/sys/kern/subr_dummy_vdso_tc.c @@ -0,0 +1,49 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_compat.h" + +#include <sys/param.h> +#include <sys/vdso.h> + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th) +{ + + return (0); +} + +#ifdef COMPAT_FREEBSD32 +uint32_t +cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) +{ + + return (0); +} +#endif diff --git a/sys/mips/include/vdso.h b/sys/mips/include/vdso.h new file mode 100644 index 0000000..082df1a --- /dev/null +++ b/sys/mips/include/vdso.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MIPS_VDSO_H +#define _MIPS_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/pc98/include/vdso.h b/sys/pc98/include/vdso.h new file mode 100644 index 0000000..b81c455 --- /dev/null +++ b/sys/pc98/include/vdso.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include <x86/vdso.h> diff --git a/sys/powerpc/include/vdso.h b/sys/powerpc/include/vdso.h new file mode 100644 index 0000000..0bdafd9 --- /dev/null +++ b/sys/powerpc/include/vdso.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _POWERPC_VDSO_H +#define _POWERPC_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/sparc64/include/vdso.h b/sys/sparc64/include/vdso.h new file mode 100644 index 0000000..98b291e --- /dev/null +++ b/sys/sparc64/include/vdso.h @@ -0,0 +1,34 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPARC64_VDSO_H +#define _SPARC64_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#endif diff --git a/sys/sys/sysent.h b/sys/sys/sysent.h index 9dd5c1c..22769c2 100644 --- a/sys/sys/sysent.h +++ b/sys/sys/sysent.h @@ -124,6 +124,10 @@ struct sysentvec { vm_offset_t sv_shared_page_base; vm_offset_t sv_shared_page_len; vm_offset_t sv_sigcode_base; + vm_offset_t sv_timekeep_base; + int sv_timekeep_off; + int sv_timekeep_curr; + uint32_t sv_timekeep_gen; void *sv_shared_page_obj; void (*sv_schedtail)(struct thread *); }; diff --git a/sys/sys/vdso.h b/sys/sys/vdso.h new file mode 100644 index 0000000..9f3f3af --- /dev/null +++ b/sys/sys/vdso.h @@ -0,0 +1,124 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_VDSO_H +#define _SYS_VDSO_H + +#include <sys/types.h> +#include <sys/eventhandler.h> +#include <machine/vdso.h> + +struct vdso_timehands { + uint32_t th_algo; + uint32_t th_gen; + uint64_t th_scale; + uint32_t th_offset_count; + uint32_t th_counter_mask; + struct bintime th_offset; + struct bintime th_boottime; + VDSO_TIMEHANDS_MD +}; + +struct vdso_timekeep { + uint32_t tk_ver; + uint32_t tk_enabled; + uint32_t tk_current; + struct vdso_timehands tk_th[]; +}; + +#define VDSO_TK_CURRENT_BUSY 0xffffffff +#define VDSO_TK_VER_1 0x1 +#define VDSO_TK_VER_CURR VDSO_TK_VER_1 +#define VDSO_TH_ALGO_1 0x1 + +#ifndef _KERNEL + +struct timespec; +struct timeval; +struct timezone; + +int __vdso_clock_gettime(clockid_t clock_id, struct timespec *ts); +#pragma weak __vdso_clock_gettime + +int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz); +#pragma weak __vdso_gettimeofday + +u_int __vdso_gettc(const struct vdso_timehands *vdso_th); +#pragma weak __vdso_gettc + +#endif + +#ifdef _KERNEL + +uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th); + +/* + * The cpu_fill_vdso_timehands() function should fill MD-part of the + * struct vdso_timehands, which is both machine- and + * timecounter-depended. The return value should be 1 if fast + * userspace timecounter is enabled by hardware, and 0 otherwise. The + * global sysctl enable override is handled by machine-independed code + * after cpu_fill_vdso_timehands() call is made. + */ +uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th); + +typedef void (*tc_windup_fn)(void *); +EVENTHANDLER_DECLARE(tc_windup, tc_windup_fn); + +#define VDSO_TH_NUM 4 + +#ifdef COMPAT_FREEBSD32 +struct bintime32 { + uint32_t sec; + uint32_t frac[2]; +}; + +struct vdso_timehands32 { + uint32_t th_algo; + uint32_t th_gen; + uint32_t th_scale[2]; + uint32_t th_offset_count; + uint32_t th_counter_mask; + struct bintime32 th_offset; + struct bintime32 th_boottime; + VDSO_TIMEHANDS_MD32 +}; + +struct vdso_timekeep32 { + uint32_t tk_ver; + uint32_t tk_enabled; + uint32_t tk_current; + struct vdso_timehands32 tk_th[]; +}; + +uint32_t tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32); +uint32_t cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32); + +#endif +#endif + +#endif diff --git a/sys/x86/include/vdso.h b/sys/x86/include/vdso.h new file mode 100644 index 0000000..1171420 --- /dev/null +++ b/sys/x86/include/vdso.h @@ -0,0 +1,42 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _X86_VDSO_H +#define _X86_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_x86_shift; \ + uint32_t th_res[7]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c index 4d1618f..085c339 100644 --- a/sys/x86/x86/tsc.c +++ b/sys/x86/x86/tsc.c @@ -27,6 +27,7 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); +#include "opt_compat.h" #include "opt_clock.h" #include <sys/param.h> @@ -41,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/power.h> #include <sys/smp.h> +#include <sys/vdso.h> #include <machine/clock.h> #include <machine/cputypes.h> #include <machine/md_var.h> @@ -604,3 +606,23 @@ tsc_get_timecount_low(struct timecounter *tc) : "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx"); return (rv); } + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th) +{ + + vdso_th->th_x86_shift = (int)(intptr_t)timecounter->tc_priv; + bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); + return (timecounter == &tsc_timecounter); +} + +#ifdef COMPAT_FREEBSD32 +uint32_t +cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) +{ + + vdso_th32->th_x86_shift = (int)(intptr_t)timecounter->tc_priv; + bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res)); + return (timecounter == &tsc_timecounter); +} +#endif |