Rewrite sendfile's header support so that headers are now sent in the first

packet along with data, instead of in their own packet.  When serving files
of size (packetsize - headersize) or smaller, this will result in one less
packet crossing the network.  Quick testing with thttpd and http_load has
shown a noticeable performance improvement in this case (350 vs 330 fetches
per second.)

Included in this commit are two support routines, iov_to_uio, and m_uiotombuf;
these routines are used by sendfile to construct the header mbuf chain that
will be linked to the rest of the data in the socket buffer.

3 files changed, 141 insertions, 12 deletions

diff --git a/sys/kern/kern_subr.c b/sys/kern/kern_subr.c
index a078b11..69ce291 100644
--- a/sys/kern/kern_subr.c
+++ b/sys/kern/kern_subr.c
@@ -483,3 +483,45 @@ copyinstrfrom(const void * __restrict src, void * __restrict dst, size_t len,
 	}
 	return (error);
 }
+
+int
+iov_to_uio(struct iovec *iovp, u_int iovcnt, struct uio *auio)
+{
+	int error = 0, i;
+	u_int iovlen;
+	struct iovec *iov = NULL;
+
+	if (iovcnt < 0)
+		panic("iovcnt < 0!\n");
+
+        /* note: can't use iovlen until iovcnt is validated */
+        iovlen = iovcnt * sizeof (struct iovec);
+        if (iovcnt > UIO_MAXIOV) {
+               error = EINVAL;
+               goto done;
+        }
+        MALLOC(iov, struct iovec *, iovlen, M_IOV, M_WAITOK);
+        auio->uio_iov = iov;
+        auio->uio_iovcnt = iovcnt;
+        auio->uio_segflg = UIO_USERSPACE;
+        auio->uio_offset = -1;
+        if ((error = copyin(iovp, iov, iovlen)))
+                goto done;
+        auio->uio_resid = 0;
+        for (i = 0; i < iovcnt; i++) {
+                if (iov->iov_len > INT_MAX - auio->uio_resid) {
+                        error = EINVAL;
+                        goto done;
+                }
+                auio->uio_resid += iov->iov_len;
+                iov++;
+        }
+
+done:
+	if (error && auio->uio_iov) {
+		FREE(auio->uio_iov, M_IOV);
+		auio->uio_iov = NULL;
+	}
+	return (error);
+
+}
diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index 1af7f2c..4a53f17 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/limits.h>
 #include <sys/lock.h>
 #include <sys/mac.h>
 #include <sys/malloc.h>
@@ -50,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/domain.h>
 #include <sys/protosw.h>
+#include <sys/uio.h>
 
 int	max_linkhdr;
 int	max_protohdr;
@@ -1028,3 +1030,57 @@ nospace:
 }
 
 #endif
+
+struct mbuf *
+m_uiotombuf(struct uio *uio, int how, int len)
+{
+	struct mbuf	*m_new = NULL, *m_final = NULL;
+	int		progress = 0, error = 0, length, total;
+
+	if (len > 0)
+		total = min(uio->uio_resid, len);
+	else
+		total = uio->uio_resid;
+
+	if (total > MHLEN)
+		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
+	else
+		m_final = m_gethdr(how, MT_DATA);
+
+	if (m_final == NULL)
+		goto nospace;
+
+	m_new = m_final;
+
+	while (progress < total) {
+		length = total - progress;
+		if (length > MCLBYTES)
+			length = MCLBYTES;
+
+		if (m_new == NULL) {
+			if (length > MLEN)
+				m_new = m_getcl(how, MT_DATA, 0);
+			else
+				m_new = m_get(how, MT_DATA);
+			if (m_new == NULL)
+				goto nospace;
+		}
+
+		error = uiomove(mtod(m_new, void *), length, uio);
+		if (error)
+			goto nospace;
+		progress += length;
+		m_new->m_len = length;
+		if (m_new != m_final)
+			m_cat(m_final, m_new);
+		m_new = NULL;
+	}
+	m_fixhdr(m_final);
+	return (m_final);
+nospace:
+	if (m_new)
+		m_free(m_new);
+	if (m_final)
+		m_freem(m_final);
+	return (NULL);
+}
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 60da8ca..9bec360 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -1677,13 +1677,15 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
 	struct vnode *vp;
 	struct vm_object *obj;
 	struct socket *so = NULL;
-	struct mbuf *m;
+	struct mbuf *m, *m_header = NULL;
 	struct sf_buf *sf;
 	struct vm_page *pg;
 	struct writev_args nuap;
 	struct sf_hdtr hdtr;
+	struct uio hdr_uio;
 	off_t off, xfsize, hdtr_size, sbytes = 0;
-	int error, s;
+	int error, s, headersize = 0, headersent = 0;
+	struct iovec *hdr_iov = NULL;
 
 	mtx_lock(&Giant);
 
@@ -1731,19 +1733,25 @@ do_sendfile(struct thread *td, struct sendfile_args *uap, int compat)
 		if (error)
 			goto done;
 		/*
-		 * Send any headers. Wimp out and use writev(2).
+		 * Send any headers.
 		 */
 		if (hdtr.headers != NULL) {
-			nuap.fd = uap->s;
-			nuap.iovp = hdtr.headers;
-			nuap.iovcnt = hdtr.hdr_cnt;
-			error = writev(td, &nuap);
+			hdr_uio.uio_td = td;
+			hdr_uio.uio_rw = UIO_WRITE;
+			error = iov_to_uio(hdtr.headers, hdtr.hdr_cnt,
+				&hdr_uio);
 			if (error)
 				goto done;
-			if (compat)
-				sbytes += td->td_retval[0];
-			else
-				hdtr_size += td->td_retval[0];
+			/* Cache hdr_iov, m_uiotombuf may change it. */
+			hdr_iov = hdr_uio.uio_iov;
+			if (hdr_uio.uio_resid > 0) {
+				m_header = m_uiotombuf(&hdr_uio, M_DONTWAIT, 0);
+				if (m_header == NULL)
+					goto done;
+				headersize = m_header->m_pkthdr.len;
+				if (compat)
+					sbytes += headersize;
+			}
 		}
 	}
 
@@ -1901,7 +1909,10 @@ retry_lookup:
 		/*
 		 * Get an mbuf header and set it up as having external storage.
 		 */
-		MGETHDR(m, M_TRYWAIT, MT_DATA);
+		if (m_header)
+			MGET(m, M_TRYWAIT, MT_DATA);
+		else
+			MGETHDR(m, M_TRYWAIT, MT_DATA);
 		if (m == NULL) {
 			error = ENOBUFS;
 			sf_buf_free((void *)sf_buf_kva(sf), sf);
@@ -1915,6 +1926,14 @@ retry_lookup:
 		    EXT_SFBUF);
 		m->m_data = (char *)sf_buf_kva(sf) + pgoff;
 		m->m_pkthdr.len = m->m_len = xfsize;
+
+		if (m_header) {
+			m_cat(m_header, m);
+			m = m_header;
+			m_header = NULL;
+			m_fixhdr(m);
+		}
+
 		/*
 		 * Add the buffer to the socket buffer chain.
 		 */
@@ -1976,6 +1995,7 @@ retry_space:
 			sbunlock(&so->so_snd);
 			goto done;
 		}
+		headersent = 1;
 	}
 	sbunlock(&so->so_snd);
 
@@ -1996,6 +2016,13 @@ retry_space:
 	}
 
 done:
+	if (headersent) {
+		if (!compat)
+			hdtr_size += headersize;
+	} else {
+		if (compat)
+			sbytes -= headersize;
+	}
 	/*
 	 * If there was no error we have to clear td->td_retval[0]
 	 * because it may have been set by writev.
@@ -2012,6 +2039,10 @@ done:
 		vrele(vp);
 	if (so)
 		fputsock(so);
+	if (hdr_iov)
+		FREE(hdr_iov, M_IOV);
+	if (m_header)
+		m_freem(m_header);
 
 	mtx_unlock(&Giant);