diff options
| -rw-r--r-- | sys/kern/subr_counter.c | 15 | ||||
| -rw-r--r-- | sys/kern/subr_pcpu.c | 25 | ||||
| -rw-r--r-- | sys/sys/pcpu.h | 7 | ||||
| -rw-r--r-- | sys/vm/uma.h | 12 | ||||
| -rw-r--r-- | sys/vm/uma_core.c | 16 |
5 files changed, 57 insertions, 18 deletions
diff --git a/sys/kern/subr_counter.c b/sys/kern/subr_counter.c index b3ddc7a..ea2759c 100644 --- a/sys/kern/subr_counter.c +++ b/sys/kern/subr_counter.c @@ -41,8 +41,6 @@ __FBSDID("$FreeBSD$"); #define IN_SUBR_COUNTER_C #include <sys/counter.h> -static uma_zone_t uint64_pcpu_zone; - void counter_u64_zero(counter_u64_t c) { @@ -62,7 +60,7 @@ counter_u64_alloc(int flags) { counter_u64_t r; - r = uma_zalloc(uint64_pcpu_zone, flags); + r = uma_zalloc(pcpu_zone_64, flags); if (r != NULL) counter_u64_zero(r); @@ -73,7 +71,7 @@ void counter_u64_free(counter_u64_t c) { - uma_zfree(uint64_pcpu_zone, c); + uma_zfree(pcpu_zone_64, c); } int @@ -96,12 +94,3 @@ sysctl_handle_counter_u64(SYSCTL_HANDLER_ARGS) return (0); } - -static void -counter_startup(void) -{ - - uint64_pcpu_zone = uma_zcreate("uint64 pcpu", sizeof(uint64_t), - NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); -} -SYSINIT(counter, SI_SUB_KMEM, SI_ORDER_ANY, counter_startup, NULL); diff --git a/sys/kern/subr_pcpu.c b/sys/kern/subr_pcpu.c index 505a4df..a01f67a 100644 --- a/sys/kern/subr_pcpu.c +++ b/sys/kern/subr_pcpu.c @@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/smp.h> #include <sys/sx.h> +#include <vm/uma.h> #include <ddb/ddb.h> static MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting."); @@ -127,6 +128,30 @@ dpcpu_startup(void *dummy __unused) SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0); /* + * UMA_PCPU_ZONE zones, that are available for all kernel + * consumers. Right now 64 bit zone is used for counter(9) + * and pointer zone is used by flowtable. + */ + +uma_zone_t pcpu_zone_64; +uma_zone_t pcpu_zone_ptr; + +static void +pcpu_zones_startup(void) +{ + + pcpu_zone_64 = uma_zcreate("64 pcpu", sizeof(uint64_t), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); + + if (sizeof(uint64_t) == sizeof(void *)) + pcpu_zone_ptr = pcpu_zone_64; + else + pcpu_zone_ptr = uma_zcreate("ptr pcpu", sizeof(void *), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU); +} +SYSINIT(pcpu_zones, SI_SUB_KMEM, SI_ORDER_ANY, pcpu_zones_startup, NULL); + +/* * First-fit extent based allocator for allocating space in the per-cpu * region reserved for modules. This is only intended for use by the * kernel linkers to place module linker sets. diff --git a/sys/sys/pcpu.h b/sys/sys/pcpu.h index c73cc53..57ffdb0 100644 --- a/sys/sys/pcpu.h +++ b/sys/sys/pcpu.h @@ -210,6 +210,13 @@ zpcpu_get(void *base) return ((char *)(base) + sizeof(struct pcpu) * curcpu); } +static inline void * +zpcpu_get_cpu(void *base, int cpu) +{ + + return ((char *)(base) + sizeof(struct pcpu) * cpu); +} + /* * Machine dependent callouts. cpu_pcpu_init() is responsible for * initializing machine dependent fields of struct pcpu, and diff --git a/sys/vm/uma.h b/sys/vm/uma.h index b4fb4e8..5012d98 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -33,8 +33,8 @@ * */ -#ifndef VM_UMA_H -#define VM_UMA_H +#ifndef _VM_UMA_H_ +#define _VM_UMA_H_ #include <sys/param.h> /* For NULL */ #include <sys/malloc.h> /* For M_* */ @@ -636,6 +636,12 @@ int uma_zone_exhausted(uma_zone_t zone); int uma_zone_exhausted_nolock(uma_zone_t zone); /* + * Common UMA_ZONE_PCPU zones. + */ +extern uma_zone_t pcpu_zone_64; +extern uma_zone_t pcpu_zone_ptr; + +/* * Exported statistics structures to be used by user space monitoring tools. * Statistics stream consists of a uma_stream_header, followed by a series of * alternative uma_type_header and uma_type_stat structures. @@ -683,4 +689,4 @@ struct uma_percpu_stat { uint64_t _ups_reserved[5]; /* Reserved. */ }; -#endif +#endif /* _VM_UMA_H_ */ diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 5bfbb72..3350356 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -264,6 +264,7 @@ static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, int align, uint32_t flags); static int zone_import(uma_zone_t zone, void **bucket, int max, int flags); static void zone_release(uma_zone_t zone, void **bucket, int cnt); +static void uma_zero_item(void *item, uma_zone_t zone); void uma_print_zone(uma_zone_t); void uma_print_stats(void); @@ -2167,7 +2168,7 @@ zalloc_start: uma_dbg_alloc(zone, NULL, item); #endif if (flags & M_ZERO) - bzero(item, zone->uz_size); + uma_zero_item(item, zone); return (item); } @@ -2617,7 +2618,7 @@ zone_alloc_item(uma_zone_t zone, void *udata, int flags) uma_dbg_alloc(zone, NULL, item); #endif if (flags & M_ZERO) - bzero(item, zone->uz_size); + uma_zero_item(item, zone); return (item); @@ -3234,6 +3235,17 @@ uma_large_free(uma_slab_t slab) zone_free_item(slabzone, slab, NULL, SKIP_NONE); } +static void +uma_zero_item(void *item, uma_zone_t zone) +{ + + if (zone->uz_flags & UMA_ZONE_PCPU) { + for (int i = 0; i < mp_ncpus; i++) + bzero(zpcpu_get_cpu(item, i), zone->uz_size); + } else + bzero(item, zone->uz_size); +} + void uma_print_stats(void) { |
