promote fast ipsec's m_clone routine for public use; it is renamed

m_unshare and the caller can now control how mbufs are allocated

Reviewed by:	andre, luigi, mlaier
MFC after:	1 week

1 files changed, 153 insertions, 0 deletions

diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index 887db00..5c4c5bc 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -1679,3 +1679,156 @@ m_align(struct mbuf *m, int len)
 		adjust = MLEN - len;
 	m->m_data += adjust &~ (sizeof(long)-1);
 }
+
+/*
+ * Create a writable copy of the mbuf chain.  While doing this
+ * we compact the chain with a goal of producing a chain with
+ * at most two mbufs.  The second mbuf in this chain is likely
+ * to be a cluster.  The primary purpose of this work is to create
+ * a writable packet for encryption, compression, etc.  The
+ * secondary goal is to linearize the data so the data can be
+ * passed to crypto hardware in the most efficient manner possible.
+ */
+struct mbuf *
+m_unshare(struct mbuf *m0, int how)
+{
+	struct mbuf *m, *mprev;
+	struct mbuf *n, *mfirst, *mlast;
+	int len, off;
+
+	mprev = NULL;
+	for (m = m0; m != NULL; m = mprev->m_next) {
+		/*
+		 * Regular mbufs are ignored unless there's a cluster
+		 * in front of it that we can use to coalesce.  We do
+		 * the latter mainly so later clusters can be coalesced
+		 * also w/o having to handle them specially (i.e. convert
+		 * mbuf+cluster -> cluster).  This optimization is heavily
+		 * influenced by the assumption that we're running over
+		 * Ethernet where MCLBYTES is large enough that the max
+		 * packet size will permit lots of coalescing into a
+		 * single cluster.  This in turn permits efficient
+		 * crypto operations, especially when using hardware.
+		 */
+		if ((m->m_flags & M_EXT) == 0) {
+			if (mprev && (mprev->m_flags & M_EXT) &&
+			    m->m_len <= M_TRAILINGSPACE(mprev)) {
+				/* XXX: this ignores mbuf types */
+				memcpy(mtod(mprev, caddr_t) + mprev->m_len,
+				       mtod(m, caddr_t), m->m_len);
+				mprev->m_len += m->m_len;
+				mprev->m_next = m->m_next;	/* unlink from chain */
+				m_free(m);			/* reclaim mbuf */
+#if 0
+				newipsecstat.ips_mbcoalesced++;
+#endif
+			} else {
+				mprev = m;
+			}
+			continue;
+		}
+		/*
+		 * Writable mbufs are left alone (for now).
+		 */
+		if (M_WRITABLE(m)) {
+			mprev = m;
+			continue;
+		}
+
+		/*
+		 * Not writable, replace with a copy or coalesce with
+		 * the previous mbuf if possible (since we have to copy
+		 * it anyway, we try to reduce the number of mbufs and
+		 * clusters so that future work is easier).
+		 */
+		KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags));
+		/* NB: we only coalesce into a cluster or larger */
+		if (mprev != NULL && (mprev->m_flags & M_EXT) &&
+		    m->m_len <= M_TRAILINGSPACE(mprev)) {
+			/* XXX: this ignores mbuf types */
+			memcpy(mtod(mprev, caddr_t) + mprev->m_len,
+			       mtod(m, caddr_t), m->m_len);
+			mprev->m_len += m->m_len;
+			mprev->m_next = m->m_next;	/* unlink from chain */
+			m_free(m);			/* reclaim mbuf */
+#if 0
+			newipsecstat.ips_clcoalesced++;
+#endif
+			continue;
+		}
+
+		/*
+		 * Allocate new space to hold the copy...
+		 */
+		/* XXX why can M_PKTHDR be set past the first mbuf? */
+		if (mprev == NULL && (m->m_flags & M_PKTHDR)) {
+			/*
+			 * NB: if a packet header is present we must
+			 * allocate the mbuf separately from any cluster
+			 * because M_MOVE_PKTHDR will smash the data
+			 * pointer and drop the M_EXT marker.
+			 */
+			MGETHDR(n, how, m->m_type);
+			if (n == NULL) {
+				m_freem(m0);
+				return (NULL);
+			}
+			M_MOVE_PKTHDR(n, m);
+			MCLGET(n, how);
+			if ((n->m_flags & M_EXT) == 0) {
+				m_free(n);
+				m_freem(m0);
+				return (NULL);
+			}
+		} else {
+			n = m_getcl(how, m->m_type, m->m_flags);
+			if (n == NULL) {
+				m_freem(m0);
+				return (NULL);
+			}
+		}
+		/*
+		 * ... and copy the data.  We deal with jumbo mbufs
+		 * (i.e. m_len > MCLBYTES) by splitting them into
+		 * clusters.  We could just malloc a buffer and make
+		 * it external but too many device drivers don't know
+		 * how to break up the non-contiguous memory when
+		 * doing DMA.
+		 */
+		len = m->m_len;
+		off = 0;
+		mfirst = n;
+		mlast = NULL;
+		for (;;) {
+			int cc = min(len, MCLBYTES);
+			memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
+			n->m_len = cc;
+			if (mlast != NULL)
+				mlast->m_next = n;
+			mlast = n;	
+#if 0
+			newipsecstat.ips_clcopied++;
+#endif
+
+			len -= cc;
+			if (len <= 0)
+				break;
+			off += cc;
+
+			n = m_getcl(how, m->m_type, m->m_flags);
+			if (n == NULL) {
+				m_freem(mfirst);
+				m_freem(m0);
+				return (NULL);
+			}
+		}
+		n->m_next = m->m_next; 
+		if (mprev == NULL)
+			m0 = mfirst;		/* new head of chain */
+		else
+			mprev->m_next = mfirst;	/* replace old mbuf */
+		m_free(m);			/* release old mbuf */
+		mprev = mfirst;
+	}
+	return (m0);
+}