Merge r261722, r261723, r261724, r261725 from head:

  several minor improvements for UMA_ZPCPU_ZONE zones.

5 files changed, 57 insertions, 18 deletions

diff --git a/sys/kern/subr_counter.c b/sys/kern/subr_counter.c
index b3ddc7a..ea2759c 100644
--- a/sys/kern/subr_counter.c
+++ b/sys/kern/subr_counter.c
@@ -41,8 +41,6 @@ __FBSDID("$FreeBSD$");
 #define IN_SUBR_COUNTER_C
 #include <sys/counter.h>
  
-static uma_zone_t uint64_pcpu_zone;
-
 void
 counter_u64_zero(counter_u64_t c)
 {
@@ -62,7 +60,7 @@ counter_u64_alloc(int flags)
 {
 	counter_u64_t r;
 
-	r = uma_zalloc(uint64_pcpu_zone, flags);
+	r = uma_zalloc(pcpu_zone_64, flags);
 	if (r != NULL)
 		counter_u64_zero(r);
 
@@ -73,7 +71,7 @@ void
 counter_u64_free(counter_u64_t c)
 {
 
-	uma_zfree(uint64_pcpu_zone, c);
+	uma_zfree(pcpu_zone_64, c);
 }
 
 int
@@ -96,12 +94,3 @@ sysctl_handle_counter_u64(SYSCTL_HANDLER_ARGS)
 
 	return (0);
 }
-
-static void
-counter_startup(void)
-{
-
-	uint64_pcpu_zone = uma_zcreate("uint64 pcpu", sizeof(uint64_t),
-	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU);
-}
-SYSINIT(counter, SI_SUB_KMEM, SI_ORDER_ANY, counter_startup, NULL);
diff --git a/sys/kern/subr_pcpu.c b/sys/kern/subr_pcpu.c
index 505a4df..a01f67a 100644
--- a/sys/kern/subr_pcpu.c
+++ b/sys/kern/subr_pcpu.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/smp.h>
 #include <sys/sx.h>
+#include <vm/uma.h>
 #include <ddb/ddb.h>
 
 static MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting.");
@@ -127,6 +128,30 @@ dpcpu_startup(void *dummy __unused)
 SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0);
 
 /*
+ * UMA_PCPU_ZONE zones, that are available for all kernel
+ * consumers. Right now 64 bit zone is used for counter(9)
+ * and pointer zone is used by flowtable.
+ */
+
+uma_zone_t pcpu_zone_64;
+uma_zone_t pcpu_zone_ptr;
+
+static void
+pcpu_zones_startup(void)
+{
+
+	pcpu_zone_64 = uma_zcreate("64 pcpu", sizeof(uint64_t),
+	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU);
+
+	if (sizeof(uint64_t) == sizeof(void *))
+		pcpu_zone_ptr = pcpu_zone_64;
+	else
+		pcpu_zone_ptr = uma_zcreate("ptr pcpu", sizeof(void *),
+		    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_PCPU);
+}
+SYSINIT(pcpu_zones, SI_SUB_KMEM, SI_ORDER_ANY, pcpu_zones_startup, NULL);
+
+/*
  * First-fit extent based allocator for allocating space in the per-cpu
  * region reserved for modules.  This is only intended for use by the
  * kernel linkers to place module linker sets.
diff --git a/sys/sys/pcpu.h b/sys/sys/pcpu.h
index c73cc53..57ffdb0 100644
--- a/sys/sys/pcpu.h
+++ b/sys/sys/pcpu.h
@@ -210,6 +210,13 @@ zpcpu_get(void *base)
 	return ((char *)(base) + sizeof(struct pcpu) * curcpu);
 }
 
+static inline void *
+zpcpu_get_cpu(void *base, int cpu)
+{
+
+	return ((char *)(base) + sizeof(struct pcpu) * cpu);
+}
+
 /*
  * Machine dependent callouts.  cpu_pcpu_init() is responsible for
  * initializing machine dependent fields of struct pcpu, and
diff --git a/sys/vm/uma.h b/sys/vm/uma.h
index b4fb4e8..5012d98 100644
--- a/sys/vm/uma.h
+++ b/sys/vm/uma.h
@@ -33,8 +33,8 @@
  *
 */
 
-#ifndef VM_UMA_H
-#define VM_UMA_H
+#ifndef _VM_UMA_H_
+#define _VM_UMA_H_
 
 #include <sys/param.h>		/* For NULL */
 #include <sys/malloc.h>		/* For M_* */
@@ -636,6 +636,12 @@ int uma_zone_exhausted(uma_zone_t zone);
 int uma_zone_exhausted_nolock(uma_zone_t zone);
 
 /*
+ * Common UMA_ZONE_PCPU zones.
+ */
+extern uma_zone_t pcpu_zone_64;
+extern uma_zone_t pcpu_zone_ptr;
+
+/*
  * Exported statistics structures to be used by user space monitoring tools.
  * Statistics stream consists of a uma_stream_header, followed by a series of
  * alternative uma_type_header and uma_type_stat structures.
@@ -683,4 +689,4 @@ struct uma_percpu_stat {
 	uint64_t	_ups_reserved[5];	/* Reserved. */
 };
 
-#endif
+#endif	/* _VM_UMA_H_ */
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 5bfbb72..3350356 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -264,6 +264,7 @@ static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
     uma_fini fini, int align, uint32_t flags);
 static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
 static void zone_release(uma_zone_t zone, void **bucket, int cnt);
+static void uma_zero_item(void *item, uma_zone_t zone);
 
 void uma_print_zone(uma_zone_t);
 void uma_print_stats(void);
@@ -2167,7 +2168,7 @@ zalloc_start:
 		uma_dbg_alloc(zone, NULL, item);
 #endif
 		if (flags & M_ZERO)
-			bzero(item, zone->uz_size);
+			uma_zero_item(item, zone);
 		return (item);
 	}
 
@@ -2617,7 +2618,7 @@ zone_alloc_item(uma_zone_t zone, void *udata, int flags)
 	uma_dbg_alloc(zone, NULL, item);
 #endif
 	if (flags & M_ZERO)
-		bzero(item, zone->uz_size);
+		uma_zero_item(item, zone);
 
 	return (item);
 
@@ -3234,6 +3235,17 @@ uma_large_free(uma_slab_t slab)
 	zone_free_item(slabzone, slab, NULL, SKIP_NONE);
 }
 
+static void
+uma_zero_item(void *item, uma_zone_t zone)
+{
+
+	if (zone->uz_flags & UMA_ZONE_PCPU) {
+		for (int i = 0; i < mp_ncpus; i++)
+			bzero(zpcpu_get_cpu(item, i), zone->uz_size);
+	} else
+		bzero(item, zone->uz_size);
+}
+
 void
 uma_print_stats(void)
 {