Add an API for jumbo mbuf cluster allocation and also provide

4k clusters in addition to 9k and 16k ones.

 struct mbuf *m_getjcl(int how, short type, int flags, int size)
 void *m_cljget(struct mbuf *m, int how, int size)

m_getjcl() returns an mbuf with a cluster of the specified size attached
like m_getcl() does for 2k clusters.

m_cljget() is different from m_clget() as it can allocate clusters
without attaching them to an mbuf.  In that case the return value
is the pointer to the cluster of the requested size.  If an mbuf was
specified, it gets the cluster attached to it and the return value
can be safely ignored.

For size both take MCLBYTES, MJUM4BYTES, MJUM9BYTES, MJUM16BYTES.

Reviewed by:	glebius
Tested by:	glebius
Sponsored by:	TCP/IP Optimization Fundraise 2005

4 files changed, 113 insertions, 6 deletions

diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 9e398cd..2ac069b 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -95,6 +95,7 @@ __FBSDID("$FreeBSD$");
  */
 
 int nmbclusters;		/* limits number of mbuf clusters */
+int nmbjumbo4;			/* limits number of 4k jumbo clusters */
 int nmbjumbo9;			/* limits number of 9k jumbo clusters */
 int nmbjumbo16;			/* limits number of 16k jumbo clusters */
 struct mbstat mbstat;
@@ -113,6 +114,8 @@ SYSCTL_DECL(_kern_ipc);
 /* XXX: These should be tuneables. Can't change UMA limits on the fly. */
 SYSCTL_INT(_kern_ipc, OID_AUTO, nmbclusters, CTLFLAG_RW, &nmbclusters, 0,
     "Maximum number of mbuf clusters allowed");
+SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo4, CTLFLAG_RW, &nmbjumbo4, 0,
+    "Maximum number of mbuf 4k jumbo clusters allowed");
 SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0,
     "Maximum number of mbuf 9k jumbo clusters allowed");
 SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0,
@@ -126,6 +129,7 @@ SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
 uma_zone_t	zone_mbuf;
 uma_zone_t	zone_clust;
 uma_zone_t	zone_pack;
+uma_zone_t	zone_jumbo4;
 uma_zone_t	zone_jumbo9;
 uma_zone_t	zone_jumbo16;
 uma_zone_t	zone_ext_refcnt;
@@ -182,7 +186,18 @@ mbuf_init(void *dummy)
 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
 	    mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
 
-	/* Make jumbo frame zone too. 9k and 16k. */
+	/* Make jumbo frame zone too. 4k, 9k and 16k. */
+	zone_jumbo4 = uma_zcreate(MBUF_JUMBO4_MEM_NAME, MJUM4BYTES,
+	    mb_ctor_clust, mb_dtor_clust,
+#ifdef INVARIANTS
+	    trash_init, trash_fini,
+#else
+	    NULL, NULL,
+#endif
+	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
+	if (nmbjumbo4 > 0)
+		uma_zone_set_max(zone_jumbo4, nmbjumbo4);
+
 	zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
 	    mb_ctor_clust, mb_dtor_clust,
 #ifdef INVARIANTS
@@ -363,6 +378,9 @@ mb_ctor_clust(void *mem, int size, void *arg, int how)
 		case MCLBYTES:
 			type = EXT_CLUSTER;
 			break;
+		case MJUM4BYTES:
+			type = EXT_JUMBO4;
+			break;
 		case MJUM9BYTES:
 			type = EXT_JUMBO9;
 			break;
diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index e4b5483..96a027c 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -225,6 +225,9 @@ mb_free_ext(struct mbuf *m)
 		case EXT_CLUSTER:
 			uma_zfree(zone_clust, m->m_ext.ext_buf);
 			break;
+		case EXT_JUMBO4:
+			uma_zfree(zone_jumbo4, m->m_ext.ext_buf);
+			break;
 		case EXT_JUMBO9:
 			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
 			break;
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index e807f54..05a4b9a 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -185,9 +185,10 @@ struct mbuf {
  */
 #define	EXT_CLUSTER	1	/* mbuf cluster */
 #define	EXT_SFBUF	2	/* sendfile(2)'s sf_bufs */
-#define	EXT_JUMBO9	3	/* jumbo cluster 9216 bytes */
-#define	EXT_JUMBO16	4	/* jumbo cluster 16184 bytes */
-#define	EXT_PACKET	5	/* mbuf+cluster from packet zone */
+#define	EXT_JUMBO4	3	/* jumbo cluster 4096 bytes */
+#define	EXT_JUMBO9	4	/* jumbo cluster 9216 bytes */
+#define	EXT_JUMBO16	5	/* jumbo cluster 16184 bytes */
+#define	EXT_PACKET	6	/* mbuf+cluster from packet zone */
 #define	EXT_NET_DRV	100	/* custom ext_buf provided by net driver(s) */
 #define	EXT_MOD_TYPE	200	/* custom module's ext_buf type */
 #define	EXT_DISPOSABLE	300	/* can throw this buffer away w/page flipping */
@@ -305,6 +306,7 @@ struct mbstat {
 #define	MBUF_MEM_NAME		"mbuf"
 #define	MBUF_CLUSTER_MEM_NAME	"mbuf_cluster"
 #define	MBUF_PACKET_MEM_NAME	"mbuf_packet"
+#define	MBUF_JUMBO4_MEM_NAME	"mbuf_jumbo_4k"
 #define	MBUF_JUMBO9_MEM_NAME	"mbuf_jumbo_9k"
 #define	MBUF_JUMBO16_MEM_NAME	"mbuf_jumbo_16k"
 #define	MBUF_TAG_MEM_NAME	"mbuf_tag"
@@ -331,6 +333,7 @@ struct mbstat {
 extern uma_zone_t	zone_mbuf;
 extern uma_zone_t	zone_clust;
 extern uma_zone_t	zone_pack;
+extern uma_zone_t	zone_jumbo4;
 extern uma_zone_t	zone_jumbo9;
 extern uma_zone_t	zone_jumbo16;
 extern uma_zone_t	zone_ext_refcnt;
@@ -338,9 +341,11 @@ extern uma_zone_t	zone_ext_refcnt;
 static __inline struct mbuf	*m_get(int how, short type);
 static __inline struct mbuf	*m_gethdr(int how, short type);
 static __inline struct mbuf	*m_getcl(int how, short type, int flags);
+static __inline struct mbuf	*m_getjcl(int how, short type, int flags, int size);
 static __inline struct mbuf	*m_getclr(int how, short type);	/* XXX */
 static __inline struct mbuf	*m_free(struct mbuf *m);
 static __inline void		 m_clget(struct mbuf *m, int how);
+static __inline void		*m_cljget(struct mbuf *m, int how, int size);
 static __inline void		 m_chtype(struct mbuf *m, short new_type);
 void				 mb_free_ext(struct mbuf *);
 
@@ -393,6 +398,47 @@ m_getcl(int how, short type, int flags)
 	return (uma_zalloc_arg(zone_pack, &args, how));
 }
 
+/*
+ * m_getjcl() returns an mbuf with a cluster of the specified size attached.
+ * For size it takes MCLBYTES, MJUM4BYTES, MJUM9BYTES, MJUM16BYTES.
+ */
+static __inline	/* XXX: This is rather large, should be real function maybe. */
+struct mbuf *
+m_getjcl(int how, short type, int flags, int size)
+{
+	struct mb_args args;
+	struct mbuf *m, *n;
+	uma_zone_t zone;
+
+	args.flags = flags;
+	args.type = type;
+
+	m = uma_zalloc_arg(zone_mbuf, &args, how);
+	if (m == NULL)
+		return NULL;
+
+	switch (size) {
+	case MCLBYTES:
+		zone = zone_clust;
+		break;
+	case MJUM4BYTES:
+		zone = zone_jumbo4;
+		break;
+	case MJUM9BYTES:
+		zone = zone_jumbo9;
+		break;
+	case MJUM16BYTES:
+		zone = zone_jumbo16;
+		break;
+	default:
+		panic("%s: m_getjcl: invalid cluster type", __func__);
+	}
+	n = uma_zalloc_arg(zone, m, how);
+	if (n == NULL)
+		uma_zfree(zone_mbuf, m);
+	return n;
+}
+
 static __inline
 struct mbuf *
 m_free(struct mbuf *m)
@@ -416,6 +462,45 @@ m_clget(struct mbuf *m, int how)
 	uma_zalloc_arg(zone_clust, m, how);
 }
 
+/*
+ * m_cljget() is different from m_clget() as it can allocate clusters
+ * without attaching them to an mbuf.  In that case the return value
+ * is the pointer to the cluster of the requested size.  If an mbuf was
+ * specified, it gets the cluster attached to it and the return value
+ * can be safely ignored.
+ * For size it takes MCLBYTES, MJUM4BYTES, MJUM9BYTES, MJUM16BYTES.
+ */
+static __inline
+void *
+m_cljget(struct mbuf *m, int how, int size)
+{
+	uma_zone_t zone;
+
+	if (m && m->m_flags & M_EXT)
+		printf("%s: %p mbuf already has cluster\n", __func__, m);
+	if (m != NULL)
+		m->m_ext.ext_buf = NULL;
+
+	switch (size) {
+	case MCLBYTES:
+		zone = zone_clust;
+		break;
+	case MJUM4BYTES:
+		zone = zone_jumbo4;
+		break;
+	case MJUM9BYTES:
+		zone = zone_jumbo9;
+		break;
+	case MJUM16BYTES:
+		zone = zone_jumbo16;
+		break;
+	default:
+		panic("%s: m_getjcl: invalid cluster type", __func__);
+	}
+	
+	return (uma_zalloc_arg(zone, m, how));
+}
+
 static __inline
 void
 m_chtype(struct mbuf *m, short new_type)
diff --git a/sys/sys/param.h b/sys/sys/param.h
index d0a93f9..c298a2a 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -147,8 +147,9 @@
 
 #define MCLBYTES	(1 << MCLSHIFT)	/* size of an mbuf cluster */
 
-#define	MJUM9BYTES	(9 * 1024)	/* jumbo frame 9k */
-#define	MJUM16BYTES	(16 * 1024)	/* jumbo frame 16k */
+#define	MJUM4BYTES	(4 * 1024)	/* jumbo cluster 4k */
+#define	MJUM9BYTES	(9 * 1024)	/* jumbo cluster 9k */
+#define	MJUM16BYTES	(16 * 1024)	/* jumbo cluster 16k */
 
 /*
  * Some macros for units conversion