diff options
| -rw-r--r-- | share/man/man9/Makefile | 9 | ||||
| -rw-r--r-- | share/man/man9/counter.9 | 200 | ||||
| -rw-r--r-- | sys/amd64/include/counter.h | 51 | ||||
| -rw-r--r-- | sys/arm/include/counter.h | 54 | ||||
| -rw-r--r-- | sys/conf/files | 1 | ||||
| -rw-r--r-- | sys/i386/include/counter.h | 89 | ||||
| -rw-r--r-- | sys/ia64/include/counter.h | 54 | ||||
| -rw-r--r-- | sys/kern/subr_counter.c | 109 | ||||
| -rw-r--r-- | sys/mips/include/counter.h | 54 | ||||
| -rw-r--r-- | sys/pc98/include/counter.h | 6 | ||||
| -rw-r--r-- | sys/powerpc/include/counter.h | 82 | ||||
| -rw-r--r-- | sys/sparc64/include/counter.h | 54 | ||||
| -rw-r--r-- | sys/sys/counter.h | 42 | ||||
| -rw-r--r-- | sys/sys/sysctl.h | 14 |
14 files changed, 819 insertions, 0 deletions
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile index fa7cc8a..4691191 100644 --- a/share/man/man9/Makefile +++ b/share/man/man9/Makefile @@ -51,6 +51,7 @@ MAN= accept_filter.9 \ config_intrhook.9 \ contigmalloc.9 \ copy.9 \ + counter.9 \ cr_cansee.9 \ critical_enter.9 \ cr_seeothergids.9 \ @@ -569,6 +570,14 @@ MLINKS+=copy.9 copyin.9 \ copy.9 copyout.9 \ copy.9 copyout_nofault.9 \ copy.9 copystr.9 +MLINKS+=counter.9 counter_u64_alloc.9 \ + counter.9 counter_u64_free.9 \ + counter.9 counter_u64_add.9 \ + counter.9 counter_enter.9 \ + counter.9 counter_exit.9 \ + counter.9 counter_u64_add_protected.9 \ + counter.9 counter_u64_fetch.9 \ + counter.9 counter_u64_zero.9 MLINKS+=critical_enter.9 critical.9 \ critical_enter.9 critical_exit.9 MLINKS+=crypto.9 crypto_dispatch.9 \ diff --git a/share/man/man9/counter.9 b/share/man/man9/counter.9 new file mode 100644 index 0000000..8f77ce6 --- /dev/null +++ b/share/man/man9/counter.9 @@ -0,0 +1,200 @@ +.\"- +.\" Copyright (c) 2013 Gleb Smirnoff <glebius@FreeBSD.org> +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd April 3, 2013 +.Dt COUNTER 9 +.Os +.Sh NAME +.Nm counter +.Nd "SMP-friendly kernel counter implementation" +.Sh SYNOPSIS +.In sys/types.h +.In sys/counter.h +.Ft counter_u64_t +.Fn counter_u64_alloc "int wait" +.Ft void +.Fn counter_u64_free "counter_u64_t c" +.Ft void +.Fn counter_u64_add "counter_u64_t c" "int64_t v" +.Ft void +.Fn counter_enter +.Ft void +.Fn counter_exit +.Ft void +.Fn counter_u64_add_protected "counter_u64_t c" "int64_t v" +.Ft uint64_t +.Fn counter_u64_fetch "counter_u64_t c" +.Ft void +.Fn counter_u64_zero "counter_u64_t c" +.In sys/sysctl.h +.Fn SYSCTL_COUNTER_U64 parent nbr name access ptr val descr +.Fn SYSCTL_ADD_COUNTER_U64 ctx parent nbr name access ptr descr +.Sh DESCRIPTION +.Nm +is a generic facility to create counters +that can be utilized for any purpose (such as collecting statistical +data). +A +.Nm +is guaranteed to be lossless when several kernel threads do simultaneous +updates. +However, +.Nm +does not block the calling thread, +also no +.Xr atomic 9 +operations are used for the update, therefore the counters +can be used in any non-interrupt context. +Moreover, +.Nm +has special optimisations for SMP environments, making +.Nm +update faster than simple arithmetic on the global variable. +Thus +.Nm +is considered suitable for accounting in the performance-critical +code pathes. +.Bl -tag -width indent +.It Fn counter_u64_alloc how +Allocate a new 64-bit unsigned counter. +The +.Fa wait +argument is the +.Xr malloc 9 +wait flag, should be either +.Va M_NOWAIT +or +.Va M_WAITOK . +If +.Va M_WAITOK +is specified the operation may fail. +.It Fn counter_u64_free c +Free the previously allocated counter +.Fa c . +.It Fn counter_u64_add c v +Add +.Fa v +to +.Fa c . +The KPI does not guarantee any protection from wraparound. +.It Fn counter_enter +Enter mode that would allow to safely update several counters via +.Fn counter_u64_add_protected . +On some machines this expands to +.Xr critical 9 +section, while on other is a nop. +See +.Sx IMPLEMENTATION DETAILS . +.It Fn counter_exit +Exit mode for updating several counters. +.It Fn counter_u64_add_protected c v +Same as +.Fn counter_u64_add , +but should be preceded by +.Fn counter_enter . +.It Fn counter_u64_fetch c +Take a snapshot of counter +.Fa c . +The data obtained is not guaranteed to reflect the real cumulative +value for any moment. +.It Fn counter_u64_zero c +Clear the counter +.Fa c +and set it to zero. +.It Fn SYSCTL_COUNTER_U64 parent nbr name access ptr val descr +Declare a static +.Xr sysctl +oid that would represent a +.Nm . +The +.Fa ptr +argument should be a pointer to allocated +.Vt counter_u64_t . +A read of the oid returns value obtained through +.Fn counter_u64_fetch . +Any write to the oid zeroes it. +.It Fn SYSCTL_ADD_COUNTER_U64 ctx parent nbr name access ptr descr +Create a +.Xr sysctl +oid that would represent a +.Nm . +The +.Fa ptr +argument should be a pointer to allocated +.Vt counter_u64_t . +A read of the oid returns value obtained through +.Fn counter_u64_fetch . +Any write to the oid zeroes it. +.El +.Sh IMPLEMENTATION DETAILS +On all architectures +.Nm +is implemented using per-CPU data fields that are specially aligned +in memory, to avoid inter-CPU bus traffic due to shared use +of the variables between CPUs. +These are allocated using +.Va UMA_ZONE_PCPU +.Xr uma 9 +zone. +The update operation only touches the field that is private to current CPU. +Fetch operation loops through all per-CPU fields and obtains a snapshot +sum of all fields. +.Pp +On amd64 a +.Nm counter +update is implemented as a single instruction without lock semantics, +operating on the private data for the current CPU, +which is safe against preemption and interrupts. +.Pp +On i386 architecture, when machine supports the cmpxchg8 instruction, +this instruction is used. +The multi-instruction sequence provides the same guarantees as the +amd64 single-instruction implementation. +.Pp +On some architectures updating a counter require a +.Xr critical 9 +section. +.Sh SEE ALSO +.Xr atomic 9 , +.Xr critical 9 , +.Xr locking 9 , +.Xr malloc 9 , +.Xr sysctl 9 , +.Xr uma 9 +.Sh HISTORY +The +.Nm +facility first appeared in +.Fx 10.0 . +.Sh AUTHORS +.An -nosplit +The +.Nm +facility was written by +.An Gleb Smirnoff +and +.An Konstantin Belousov . diff --git a/sys/amd64/include/counter.h b/sys/amd64/include/counter.h new file mode 100644 index 0000000..3ed5f76 --- /dev/null +++ b/sys/amd64/include/counter.h @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> + +extern struct pcpu __pcpu[1]; + +#define counter_enter() do {} while (0) +#define counter_exit() do {} while (0) + +#define counter_u64_add_protected(c, i) counter_u64_add(c, i) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + __asm __volatile("addq\t%1,%%gs:(%0)" + : + : "r" ((char *)c - (char *)&__pcpu[0]), "r" (inc) + : "memory", "cc"); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/arm/include/counter.h b/sys/arm/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/arm/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/conf/files b/sys/conf/files index c83fb03..95ed2c6 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -2683,6 +2683,7 @@ kern/subr_bus.c standard kern/subr_bus_dma.c standard kern/subr_bufring.c standard kern/subr_clock.c standard +kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_eventhandler.c standard diff --git a/sys/i386/include/counter.h b/sys/i386/include/counter.h new file mode 100644 index 0000000..f45d953 --- /dev/null +++ b/sys/i386/include/counter.h @@ -0,0 +1,89 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif +#include <machine/md_var.h> +#include <machine/specialreg.h> + +#define counter_enter() do { \ + if ((cpu_feature & CPUID_CX8) == 0) \ + critical_enter(); \ +} while (0) + +#define counter_exit() do { \ + if ((cpu_feature & CPUID_CX8) == 0) \ + critical_exit(); \ +} while (0) + +static inline void +counter_64_inc_8b(uint64_t *p, int64_t inc) +{ + + __asm __volatile( + "movl %%fs:(%%esi),%%eax\n\t" + "movl %%fs:4(%%esi),%%edx\n" +"1:\n\t" + "movl %%eax,%%ebx\n\t" + "movl %%edx,%%ecx\n\t" + "addl (%%edi),%%ebx\n\t" + "adcl 4(%%edi),%%ecx\n\t" + "cmpxchg8b %%fs:(%%esi)\n\t" + "jnz 1b" + : + : "S" (p), "D" (&inc) + : "memory", "cc", "eax", "edx", "ebx", "ecx"); +} + +#define counter_u64_add_protected(c, inc) do { \ + if ((cpu_feature & CPUID_CX8) == 0) { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ + } else \ + counter_64_inc_8b((c), (inc)); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + if ((cpu_feature & CPUID_CX8) == 0) { + critical_enter(); + *(uint64_t *)zpcpu_get(c) += inc; + critical_exit(); + } else { + counter_64_inc_8b(c, inc); + } +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/ia64/include/counter.h b/sys/ia64/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/ia64/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/kern/subr_counter.c b/sys/kern/subr_counter.c new file mode 100644 index 0000000..5856cf0 --- /dev/null +++ b/sys/kern/subr_counter.c @@ -0,0 +1,109 @@ +/*- + * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/counter.h> +#include <sys/kernel.h> +#include <sys/smp.h> +#include <sys/sysctl.h> +#include <vm/uma.h> + +static uma_zone_t uint64_pcpu_zone; + +void +counter_u64_zero(counter_u64_t c) +{ + int i; + + for (i = 0; i < mp_ncpus; i++) + *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0; +} + +uint64_t +counter_u64_fetch(counter_u64_t c) +{ + uint64_t r; + int i; + + r = 0; + for (i = 0; i < mp_ncpus; i++) + r += *(uint64_t *)((char *)c + sizeof(struct pcpu) * i); + + return (r); +} + +counter_u64_t +counter_u64_alloc(int flags) +{ + counter_u64_t r; + + r = uma_zalloc(uint64_pcpu_zone, flags); + if (r != NULL) + counter_u64_zero(r); + + return (r); +} + +void +counter_u64_free(counter_u64_t c) +{ + + uma_zfree(uint64_pcpu_zone, c); +} + +int +sysctl_handle_counter_u64(SYSCTL_HANDLER_ARGS) +{ + uint64_t out; + int error; + + out = counter_u64_fetch(*(counter_u64_t *)arg1); + + error = SYSCTL_OUT(req, &out, sizeof(uint64_t)); + + if (error || !req->newptr) + return (error); + + /* + * Any write attempt to a counter zeroes it. + */ + counter_u64_zero(*(counter_u64_t *)arg1); + + return (0); +} + +static void +counter_startup(void) +{ + + uint64_pcpu_zone = uma_zcreate("uint64 pcpu", sizeof(uint64_t), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); +} +SYSINIT(counter, SI_SUB_KMEM, SI_ORDER_ANY, counter_startup, NULL); diff --git a/sys/mips/include/counter.h b/sys/mips/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/mips/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/pc98/include/counter.h b/sys/pc98/include/counter.h new file mode 100644 index 0000000..7b2bbad --- /dev/null +++ b/sys/pc98/include/counter.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include <i386/counter.h> diff --git a/sys/powerpc/include/counter.h b/sys/powerpc/include/counter.h new file mode 100644 index 0000000..94a7df6 --- /dev/null +++ b/sys/powerpc/include/counter.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2012, 2013 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#if defined(AIM) && defined(__powerpc64__) + +#define counter_enter() do {} while (0) +#define counter_exit() do {} while (0) + +#define counter_u64_add_protected(c, i) counter_u64_add(c, i) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + uint64_t ccpu, old; + + __asm __volatile("\n" + "1:\n\t" + "mfsprg %0, 0\n\t" + "ldarx %1, %0, %2\n\t" + "add %1, %1, %3\n\t" + "stdcx. %1, %0, %2\n\t" + "bne- 1b" + : "=&b" (ccpu), "=&r" (old) + : "r" ((char *)c - (char *)&__pcpu[0]), "r" (inc) + : "cc", "memory"); +} + +#else /* !AIM || !64bit */ + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* AIM 64bit */ + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/sparc64/include/counter.h b/sys/sparc64/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/sparc64/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/sys/counter.h b/sys/sys/counter.h new file mode 100644 index 0000000..ee500b5 --- /dev/null +++ b/sys/sys/counter.h @@ -0,0 +1,42 @@ +/*- + * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __SYS_COUNTER_H__ +#define __SYS_COUNTER_H__ + +typedef uint64_t *counter_u64_t; + +#include <machine/counter.h> + +counter_u64_t counter_u64_alloc(int); +void counter_u64_free(counter_u64_t); + +void counter_u64_zero(counter_u64_t); +uint64_t counter_u64_fetch(counter_u64_t); + +#endif /* ! __SYS_COUNTER_H__ */ diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h index e575f22..2fdeed2 100644 --- a/sys/sys/sysctl.h +++ b/sys/sys/sysctl.h @@ -184,6 +184,7 @@ int sysctl_handle_long(SYSCTL_HANDLER_ARGS); int sysctl_handle_64(SYSCTL_HANDLER_ARGS); int sysctl_handle_string(SYSCTL_HANDLER_ARGS); int sysctl_handle_opaque(SYSCTL_HANDLER_ARGS); +int sysctl_handle_counter_u64(SYSCTL_HANDLER_ARGS); int sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS); int sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS); @@ -379,6 +380,19 @@ SYSCTL_ALLOWED_TYPES(UINT64, uint64_t *a; unsigned long long *b; ); SYSCTL_ADD_ASSERT_TYPE(UINT64, ptr), 0, \ sysctl_handle_64, "QU", __DESCR(descr)) +/* Oid for a 64-bin unsigned counter(9). The pointer must be non NULL. */ +#define SYSCTL_COUNTER_U64(parent, nbr, name, access, ptr, val, descr) \ + SYSCTL_ASSERT_TYPE(UINT64, ptr, parent, name); \ + SYSCTL_OID(parent, nbr, name, \ + CTLTYPE_U64 | CTLFLAG_MPSAFE | (access), \ + ptr, val, sysctl_handle_counter_u64, "QU", descr) + +#define SYSCTL_ADD_COUNTER_U64(ctx, parent, nbr, name, access, ptr, descr)\ + sysctl_add_oid(ctx, parent, nbr, name, \ + CTLTYPE_U64 | CTLFLAG_MPSAFE | (access), \ + SYSCTL_ADD_ASSERT_TYPE(UINT64, ptr), 0, \ + sysctl_handle_counter_u64, "QU", __DESCR(descr)) + /* Oid for an opaque object. Specified by a pointer and a length. */ #define SYSCTL_OPAQUE(parent, nbr, name, access, ptr, len, fmt, descr) \ SYSCTL_OID(parent, nbr, name, CTLTYPE_OPAQUE|(access), \ |
