diff options
author | glebius <glebius@FreeBSD.org> | 2013-04-08 19:40:53 +0000 |
---|---|---|
committer | glebius <glebius@FreeBSD.org> | 2013-04-08 19:40:53 +0000 |
commit | 9cf64d6c35c62c34531273aadc2bfc0e0606fd77 (patch) | |
tree | de8f9f44d18df863668a203bd7eaeaea76185b92 /sys | |
parent | e01107c5d37d344c4a3ed663347b49eaf4527e4a (diff) | |
download | FreeBSD-src-9cf64d6c35c62c34531273aadc2bfc0e0606fd77.zip FreeBSD-src-9cf64d6c35c62c34531273aadc2bfc0e0606fd77.tar.gz |
Merge from projects/counters: counter(9).
Introduce counter(9) API, that implements fast and raceless counters,
provided (but not limited to) for gathering of statistical data.
See http://lists.freebsd.org/pipermail/freebsd-arch/2013-April/014204.html
for more details.
In collaboration with: kib
Reviewed by: luigi
Tested by: ae, ray
Sponsored by: Nginx, Inc.
Diffstat (limited to 'sys')
-rw-r--r-- | sys/amd64/include/counter.h | 51 | ||||
-rw-r--r-- | sys/arm/include/counter.h | 54 | ||||
-rw-r--r-- | sys/conf/files | 1 | ||||
-rw-r--r-- | sys/i386/include/counter.h | 89 | ||||
-rw-r--r-- | sys/ia64/include/counter.h | 54 | ||||
-rw-r--r-- | sys/kern/subr_counter.c | 109 | ||||
-rw-r--r-- | sys/mips/include/counter.h | 54 | ||||
-rw-r--r-- | sys/pc98/include/counter.h | 6 | ||||
-rw-r--r-- | sys/powerpc/include/counter.h | 82 | ||||
-rw-r--r-- | sys/sparc64/include/counter.h | 54 | ||||
-rw-r--r-- | sys/sys/counter.h | 42 | ||||
-rw-r--r-- | sys/sys/sysctl.h | 14 |
12 files changed, 610 insertions, 0 deletions
diff --git a/sys/amd64/include/counter.h b/sys/amd64/include/counter.h new file mode 100644 index 0000000..3ed5f76 --- /dev/null +++ b/sys/amd64/include/counter.h @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> + +extern struct pcpu __pcpu[1]; + +#define counter_enter() do {} while (0) +#define counter_exit() do {} while (0) + +#define counter_u64_add_protected(c, i) counter_u64_add(c, i) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + __asm __volatile("addq\t%1,%%gs:(%0)" + : + : "r" ((char *)c - (char *)&__pcpu[0]), "r" (inc) + : "memory", "cc"); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/arm/include/counter.h b/sys/arm/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/arm/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/conf/files b/sys/conf/files index c83fb03..95ed2c6 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -2683,6 +2683,7 @@ kern/subr_bus.c standard kern/subr_bus_dma.c standard kern/subr_bufring.c standard kern/subr_clock.c standard +kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard kern/subr_eventhandler.c standard diff --git a/sys/i386/include/counter.h b/sys/i386/include/counter.h new file mode 100644 index 0000000..f45d953 --- /dev/null +++ b/sys/i386/include/counter.h @@ -0,0 +1,89 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif +#include <machine/md_var.h> +#include <machine/specialreg.h> + +#define counter_enter() do { \ + if ((cpu_feature & CPUID_CX8) == 0) \ + critical_enter(); \ +} while (0) + +#define counter_exit() do { \ + if ((cpu_feature & CPUID_CX8) == 0) \ + critical_exit(); \ +} while (0) + +static inline void +counter_64_inc_8b(uint64_t *p, int64_t inc) +{ + + __asm __volatile( + "movl %%fs:(%%esi),%%eax\n\t" + "movl %%fs:4(%%esi),%%edx\n" +"1:\n\t" + "movl %%eax,%%ebx\n\t" + "movl %%edx,%%ecx\n\t" + "addl (%%edi),%%ebx\n\t" + "adcl 4(%%edi),%%ecx\n\t" + "cmpxchg8b %%fs:(%%esi)\n\t" + "jnz 1b" + : + : "S" (p), "D" (&inc) + : "memory", "cc", "eax", "edx", "ebx", "ecx"); +} + +#define counter_u64_add_protected(c, inc) do { \ + if ((cpu_feature & CPUID_CX8) == 0) { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ + } else \ + counter_64_inc_8b((c), (inc)); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + if ((cpu_feature & CPUID_CX8) == 0) { + critical_enter(); + *(uint64_t *)zpcpu_get(c) += inc; + critical_exit(); + } else { + counter_64_inc_8b(c, inc); + } +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/ia64/include/counter.h b/sys/ia64/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/ia64/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/kern/subr_counter.c b/sys/kern/subr_counter.c new file mode 100644 index 0000000..5856cf0 --- /dev/null +++ b/sys/kern/subr_counter.c @@ -0,0 +1,109 @@ +/*- + * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/counter.h> +#include <sys/kernel.h> +#include <sys/smp.h> +#include <sys/sysctl.h> +#include <vm/uma.h> + +static uma_zone_t uint64_pcpu_zone; + +void +counter_u64_zero(counter_u64_t c) +{ + int i; + + for (i = 0; i < mp_ncpus; i++) + *(uint64_t *)((char *)c + sizeof(struct pcpu) * i) = 0; +} + +uint64_t +counter_u64_fetch(counter_u64_t c) +{ + uint64_t r; + int i; + + r = 0; + for (i = 0; i < mp_ncpus; i++) + r += *(uint64_t *)((char *)c + sizeof(struct pcpu) * i); + + return (r); +} + +counter_u64_t +counter_u64_alloc(int flags) +{ + counter_u64_t r; + + r = uma_zalloc(uint64_pcpu_zone, flags); + if (r != NULL) + counter_u64_zero(r); + + return (r); +} + +void +counter_u64_free(counter_u64_t c) +{ + + uma_zfree(uint64_pcpu_zone, c); +} + +int +sysctl_handle_counter_u64(SYSCTL_HANDLER_ARGS) +{ + uint64_t out; + int error; + + out = counter_u64_fetch(*(counter_u64_t *)arg1); + + error = SYSCTL_OUT(req, &out, sizeof(uint64_t)); + + if (error || !req->newptr) + return (error); + + /* + * Any write attempt to a counter zeroes it. + */ + counter_u64_zero(*(counter_u64_t *)arg1); + + return (0); +} + +static void +counter_startup(void) +{ + + uint64_pcpu_zone = uma_zcreate("uint64 pcpu", sizeof(uint64_t), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); +} +SYSINIT(counter, SI_SUB_KMEM, SI_ORDER_ANY, counter_startup, NULL); diff --git a/sys/mips/include/counter.h b/sys/mips/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/mips/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/pc98/include/counter.h b/sys/pc98/include/counter.h new file mode 100644 index 0000000..7b2bbad --- /dev/null +++ b/sys/pc98/include/counter.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include <i386/counter.h> diff --git a/sys/powerpc/include/counter.h b/sys/powerpc/include/counter.h new file mode 100644 index 0000000..94a7df6 --- /dev/null +++ b/sys/powerpc/include/counter.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2012, 2013 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#if defined(AIM) && defined(__powerpc64__) + +#define counter_enter() do {} while (0) +#define counter_exit() do {} while (0) + +#define counter_u64_add_protected(c, i) counter_u64_add(c, i) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + uint64_t ccpu, old; + + __asm __volatile("\n" + "1:\n\t" + "mfsprg %0, 0\n\t" + "ldarx %1, %0, %2\n\t" + "add %1, %1, %3\n\t" + "stdcx. %1, %0, %2\n\t" + "bne- 1b" + : "=&b" (ccpu), "=&r" (old) + : "r" ((char *)c - (char *)&__pcpu[0]), "r" (inc) + : "cc", "memory"); +} + +#else /* !AIM || !64bit */ + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* AIM 64bit */ + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/sparc64/include/counter.h b/sys/sparc64/include/counter.h new file mode 100644 index 0000000..68f89e2 --- /dev/null +++ b/sys/sparc64/include/counter.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __MACHINE_COUNTER_H__ +#define __MACHINE_COUNTER_H__ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! __MACHINE_COUNTER_H__ */ diff --git a/sys/sys/counter.h b/sys/sys/counter.h new file mode 100644 index 0000000..ee500b5 --- /dev/null +++ b/sys/sys/counter.h @@ -0,0 +1,42 @@ +/*- + * Copyright (c) 2012 Gleb Smirnoff <glebius@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __SYS_COUNTER_H__ +#define __SYS_COUNTER_H__ + +typedef uint64_t *counter_u64_t; + +#include <machine/counter.h> + +counter_u64_t counter_u64_alloc(int); +void counter_u64_free(counter_u64_t); + +void counter_u64_zero(counter_u64_t); +uint64_t counter_u64_fetch(counter_u64_t); + +#endif /* ! __SYS_COUNTER_H__ */ diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h index e575f22..2fdeed2 100644 --- a/sys/sys/sysctl.h +++ b/sys/sys/sysctl.h @@ -184,6 +184,7 @@ int sysctl_handle_long(SYSCTL_HANDLER_ARGS); int sysctl_handle_64(SYSCTL_HANDLER_ARGS); int sysctl_handle_string(SYSCTL_HANDLER_ARGS); int sysctl_handle_opaque(SYSCTL_HANDLER_ARGS); +int sysctl_handle_counter_u64(SYSCTL_HANDLER_ARGS); int sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS); int sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS); @@ -379,6 +380,19 @@ SYSCTL_ALLOWED_TYPES(UINT64, uint64_t *a; unsigned long long *b; ); SYSCTL_ADD_ASSERT_TYPE(UINT64, ptr), 0, \ sysctl_handle_64, "QU", __DESCR(descr)) +/* Oid for a 64-bin unsigned counter(9). The pointer must be non NULL. */ +#define SYSCTL_COUNTER_U64(parent, nbr, name, access, ptr, val, descr) \ + SYSCTL_ASSERT_TYPE(UINT64, ptr, parent, name); \ + SYSCTL_OID(parent, nbr, name, \ + CTLTYPE_U64 | CTLFLAG_MPSAFE | (access), \ + ptr, val, sysctl_handle_counter_u64, "QU", descr) + +#define SYSCTL_ADD_COUNTER_U64(ctx, parent, nbr, name, access, ptr, descr)\ + sysctl_add_oid(ctx, parent, nbr, name, \ + CTLTYPE_U64 | CTLFLAG_MPSAFE | (access), \ + SYSCTL_ADD_ASSERT_TYPE(UINT64, ptr), 0, \ + sysctl_handle_counter_u64, "QU", __DESCR(descr)) + /* Oid for an opaque object. Specified by a pointer and a length. */ #define SYSCTL_OPAQUE(parent, nbr, name, access, ptr, len, fmt, descr) \ SYSCTL_OID(parent, nbr, name, CTLTYPE_OPAQUE|(access), \ |