Use the improved m_uiotombuf() function instead of home grown sosend_copyin()

to do the userland to kernel copying in sosend_generic() and sosend_dgram().

sosend_copyin() is retained for ZERO_COPY_SOCKETS which are not yet supported
by m_uiotombuf().

Benchmaring shows significant improvements (95% confidence):
 66% less cpu (or 2.9 times better) with new sosend vs. old sosend (non-TSO)
 65% less cpu (or 2.8 times better) with new sosend vs. old sosend (TSO)

(Sender AMD Opteron 852 (2.6GHz) with em(4) PCI-X-133 interface and receiver
DELL Poweredge SC1425 P-IV Xeon 3.2GHz with em(4) LOM connected back to back
at 1000Base-TX full duplex.)

Sponsored by:	TCP/IP Optimization Fundraise 2005
MFC after:	3 month

1 files changed, 29 insertions, 1 deletions

diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 341b54c..635a2bf 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -813,9 +813,11 @@ struct so_zerocopy_stats so_zerocp_stats = {0,0,0};
 #include <vm/vm.h>
 #include <vm/vm_page.h>
 #include <vm/vm_object.h>
-#endif /*ZERO_COPY_SOCKETS*/
 
 /*
+ * sosend_copyin() is only used if zero copy sockets are enabled.  Otherwise
+ * sosend_dgram() and sosend_generic() use m_uiotombuf().
+ * 
  * sosend_copyin() accepts a uio and prepares an mbuf chain holding part or
  * all of the data referenced by the uio.  If desired, it uses zero-copy.
  * *space will be updated to reflect data copied in.
@@ -939,6 +941,7 @@ out:
 	*retmp = top;
 	return (error);
 }
+#endif /*ZERO_COPY_SOCKETS*/
 
 #define	SBLOCKWAIT(f)	(((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
 
@@ -954,7 +957,9 @@ sosend_dgram(so, addr, uio, top, control, flags, td)
 {
 	long space, resid;
 	int clen = 0, error, dontroute;
+#ifdef ZERO_COPY_SOCKETS
 	int atomic = sosendallatonce(so) || top;
+#endif
 
 	KASSERT(so->so_type == SOCK_DGRAM, ("sodgram_send: !SOCK_DGRAM"));
 	KASSERT(so->so_proto->pr_flags & PR_ATOMIC,
@@ -1040,9 +1045,19 @@ sosend_dgram(so, addr, uio, top, control, flags, td)
 		if (flags & MSG_EOR)
 			top->m_flags |= M_EOR;
 	} else {
+#ifdef ZERO_COPY_SOCKETS
 		error = sosend_copyin(uio, &top, atomic, &space, flags);
 		if (error)
 			goto out;
+#else
+		top = m_uiotombuf(uio, M_WAITOK, space, max_hdr,
+		    (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0)));
+		if (top == NULL) {
+			error = EFAULT;	/* only possible error */
+			goto out;
+		}
+		space -= resid - uio->uio_resid;
+#endif
 		resid = uio->uio_resid;
 	}
 	KASSERT(resid == 0, ("sosend_dgram: resid != 0"));
@@ -1202,12 +1217,25 @@ restart:
 				if (flags & MSG_EOR)
 					top->m_flags |= M_EOR;
 			} else {
+#ifdef ZERO_COPY_SOCKETS
 				error = sosend_copyin(uio, &top, atomic,
 				    &space, flags);
 				if (error != 0) {
 					SOCKBUF_LOCK(&so->so_snd);
 					goto release;
 				}
+#else
+				top = m_uiotombuf(uio, M_WAITOK, space,
+				    (atomic ? max_hdr : 0),
+				    (atomic ? M_PKTHDR : 0) |
+				    ((flags & MSG_EOR) ? M_EOR : 0));
+				if (top == NULL) {
+					SOCKBUF_LOCK(&so->so_snd);
+					error = EFAULT; /* only possible error */
+					goto release;
+				}
+				space -= resid - uio->uio_resid;
+#endif
 				resid = uio->uio_resid;
 			}
 			if (dontroute) {