Add driver independent interface to offload active established TCP connections

Reviewed by: silby

6 files changed, 463 insertions, 1 deletions

diff --git a/sys/conf/files b/sys/conf/files
index f8894c5..1105225 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1888,6 +1888,7 @@ netinet/tcp_debug.c		optional tcpdebug
 netinet/tcp_hostcache.c		optional inet
 netinet/tcp_input.c		optional inet
 netinet/tcp_output.c		optional inet
+netinet/tcp_ofld.c		optional inet
 netinet/tcp_reass.c		optional inet
 netinet/tcp_sack.c		optional inet
 netinet/tcp_subr.c		optional inet
diff --git a/sys/netinet/tcp_ofld.c b/sys/netinet/tcp_ofld.c
new file mode 100644
index 0000000..62699a8
--- /dev/null
+++ b/sys/netinet/tcp_ofld.c
@@ -0,0 +1,126 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+***************************************************************************/
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/mbuf.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+
+#include <net/if.h>
+#include <net/if_types.h>
+#include <net/if_var.h>
+
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/in_pcb.h>
+#include <netinet/tcp.h>
+#include <netinet/tcp_var.h>
+#include <netinet/tcp_ofld.h>
+#include <netinet/toedev.h>
+
+int
+ofld_connect(struct socket *so, struct sockaddr *nam)
+{
+	struct ifnet *ifp;
+	struct toedev *tdev;
+	struct rtentry *rt;
+	int err;
+
+	rt = rtalloc1(nam, 1, 0);
+	if (rt)
+		RT_UNLOCK(rt);
+	else
+		return (EHOSTUNREACH);
+	
+	ifp = rt->rt_ifp;
+	tdev = TOEDEV(ifp);
+	if (tdev == NULL) 
+		return (EINVAL);
+	
+	if (tdev->can_offload(tdev, so) == 0)
+		return (EINVAL);
+	
+	if ((err = tdev->connect(tdev, so, ifp)))
+		return (err);
+		
+	return (0);
+}
+
+int
+ofld_send(struct tcpcb *tp)
+{
+	return tp->t_tu->tu_send(tp);
+}
+
+int
+ofld_rcvd(struct tcpcb *tp)
+{
+
+	return tp->t_tu->tu_rcvd(tp);
+}
+
+int
+ofld_disconnect(struct tcpcb *tp)
+{
+       	return tp->t_tu->tu_disconnect(tp);
+}
+
+int
+ofld_abort(struct tcpcb *tp)
+{
+	return tp->t_tu->tu_abort(tp);
+}
+
+void
+ofld_detach(struct tcpcb *tp)
+{	
+	tp->t_tu->tu_detach(tp);
+}
+
+void
+ofld_listen_open(struct tcpcb *tp)
+{
+	EVENTHANDLER_INVOKE(ofld_listen, OFLD_LISTEN_OPEN, tp);
+}
+
+void
+ofld_listen_close(struct tcpcb *tp)
+{
+	EVENTHANDLER_INVOKE(ofld_listen, OFLD_LISTEN_CLOSE, tp);
+}
+
+
diff --git a/sys/netinet/tcp_ofld.h b/sys/netinet/tcp_ofld.h
new file mode 100644
index 0000000..380c659
--- /dev/null
+++ b/sys/netinet/tcp_ofld.h
@@ -0,0 +1,198 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+$FreeBSD$
+
+***************************************************************************/
+
+#ifndef _NETINET_TCP_OFLD_H_
+#define _NETINET_TCP_OFLD_H_
+
+#define	SC_ENTRY_PRESENT	1
+#define	SC_DROP			2
+
+#define tp_offload(tp) ((tp)->t_flags & TF_TOE)
+#define SO_OFFLOADABLE(so) ((so->so_options & SO_NOOFFLOAD) == 0)
+
+int ofld_connect(struct socket *so, struct sockaddr *nam);
+int ofld_can_offload(struct tcpcb *tp, struct sockaddr *nam);
+
+int ofld_send(struct tcpcb *tp);
+int ofld_rcvd(struct tcpcb *tp);
+int ofld_disconnect(struct tcpcb *tp);
+int ofld_abort(struct tcpcb *tp);
+void ofld_detach(struct tcpcb *tp);
+void ofld_listen_open(struct tcpcb *tp);
+void ofld_listen_close(struct tcpcb *tp);
+
+#ifndef DISABLE_TCP_OFFLOAD
+static __inline int
+tcp_gen_connect(struct socket *so, struct sockaddr *nam)
+{
+	int error;
+	struct tcpcb *tp = sototcpcb(so);
+	
+	if (!SO_OFFLOADABLE(so) || (error = ofld_connect(so, nam)) != 0)
+		error = tcp_output(tp);
+	
+	return (error);
+}
+
+static __inline int
+tcp_gen_disconnect(struct tcpcb *tp)
+{
+	int error;
+
+	if (tp_offload(tp))
+		error = ofld_disconnect(tp);
+	else
+		error = tcp_output(tp);
+
+	return (error);
+}
+
+static __inline int
+tcp_gen_abort(struct tcpcb *tp)
+{
+	int error;
+
+	if (tp_offload(tp))
+		error = ofld_abort(tp);
+	else
+		error = tcp_output(tp);
+
+	return (error);
+}
+
+static __inline int
+tcp_gen_send(struct tcpcb *tp)
+{
+	int error;
+
+	if (tp_offload(tp))
+		error = ofld_send(tp);
+	else
+		error = tcp_output(tp);
+
+	return (error);
+}
+
+static __inline int
+tcp_gen_rcvd(struct tcpcb *tp)
+{
+	int error;
+
+	if (tp_offload(tp))
+		error = ofld_rcvd(tp);
+	else
+		error = tcp_output(tp);
+
+	return (error);
+}
+
+static __inline void
+tcp_gen_listen_open(struct tcpcb *tp)
+{
+	if (SO_OFFLOADABLE(tp->t_inpcb->inp_socket))
+	    ofld_listen_open(tp);
+}
+
+static __inline void
+tcp_gen_listen_close(struct tcpcb *tp)
+{
+	ofld_listen_close(tp);
+}
+
+static __inline void
+tcp_gen_detach(struct tcpcb *tp)
+{
+	if (tp_offload(tp))
+		ofld_detach(tp);
+}
+
+#else
+
+static __inline int
+tcp_gen_connect(struct socket *so, struct sockaddr *nam)
+{
+	return tcp_output(tp);
+}
+
+static __inline int
+tcp_gen_disconnect(struct tcpcb *tp)
+{
+	return tcp_output(tp);
+}
+
+static __inline int
+tcp_gen_abort(struct tcpcb *tp)
+{
+	return tcp_output(tp);
+}
+
+static __inline int
+tcp_gen_send(struct tcpcb *tp)
+{
+	return tcp_output(tp);
+}
+
+static __inline int
+tcp_gen_rcvd(struct tcpcb *tp)
+{
+	return tcp_output(tp);
+}
+
+static __inline void
+tcp_gen_listen_open(struct tcpcb *tp) {}
+
+static __inline void
+tcp_gen_listen_close(struct tcpcb *tp) {}
+
+static __inline void
+tcp_gen_detach(struct tcpcb *tp) {}
+
+	
+
+
+#endif
+
+struct toe_usrreqs {
+	int (*tu_send)(struct tcpcb *tp);
+	int (*tu_rcvd)(struct tcpcb *tp);
+	int (*tu_disconnect)(struct tcpcb *tp);
+	int (*tu_abort)(struct tcpcb *tp);
+	void (*tu_detach)(struct tcpcb *tp);
+	void (*tu_syncache_event)(int event, void *toep);
+
+};
+
+#define OFLD_LISTEN_OPEN       1
+#define OFLD_LISTEN_CLOSE      2
+typedef void (*ofld_listen_fn)(void *, int, struct tcpcb *);
+EVENTHANDLER_DECLARE(ofld_listen, ofld_listen_fn);
+
+#endif
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index 0999043..1d29a18 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -123,6 +123,7 @@ struct tcpcb {
 #define	TF_SIGNATURE	0x400000	/* require MD5 digests (RFC2385) */
 #define	TF_FORCEDATA	0x800000	/* force out a byte */
 #define	TF_TSO		0x1000000	/* TSO enabled on this connection */
+#define	TF_TOE		0x2000000	/* this connection is offloaded */
 
 	tcp_seq	snd_una;		/* send unacknowledged */
 	tcp_seq	snd_max;		/* highest sequence number sent;
@@ -206,7 +207,9 @@ struct tcpcb {
 	int	t_rttlow;		/* smallest observerved RTT */
 	u_int32_t	rfbuf_ts;	/* recv buffer autoscaling timestamp */
 	int	rfbuf_cnt;		/* recv buffer autoscaling byte count */
-	void	*t_pspare[5];		/* toe usrreqs / toepcb * / congestion algo / vimage / 1 general use */ 
+	void	*t_pspare[3];		/* toe usrreqs / toepcb * / congestion algo / vimage / 1 general use */
+	struct toe_usrreqs *t_tu;       /* offload operations vector */
+	void *t_toe;			/* TOE pcb pointer	     */
 };
 
 #define IN_FASTRECOVERY(tp)	(tp->t_flags & TF_FASTRECOVERY)
diff --git a/sys/netinet/toedev.h b/sys/netinet/toedev.h
new file mode 100644
index 0000000..adba95b
--- /dev/null
+++ b/sys/netinet/toedev.h
@@ -0,0 +1,132 @@
+
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+$FreeBSD$
+
+***************************************************************************/
+
+#ifndef _OFFLOAD_DEV_H_
+#define _OFFLOAD_DEV_H_
+
+
+/* Parameter values for offload_get_phys_egress() */
+enum {
+	TOE_OPEN,
+	TOE_FAILOVER,
+};
+
+/* Parameter values for toe_failover() */
+enum {
+	TOE_ACTIVE_SLAVE,
+	TOE_LINK_DOWN,
+	TOE_LINK_UP,
+	TOE_RELEASE,
+	TOE_RELEASE_ALL,
+};
+
+#define TOENAMSIZ 16
+
+/* Get the toedev associated with a ifnet */
+#define TOEDEV(ifp) ((ifp)->if_llsoftc)
+
+struct offload_id {
+	unsigned int id;
+	unsigned long data;
+};
+
+struct ifnet;
+struct rt_entry;
+struct tom_info;
+struct sysctl_oid;
+struct socket;
+struct mbuf;
+
+struct toedev {
+	TAILQ_ENTRY(toedev) entry;  
+	char name[TOENAMSIZ];       /* TOE device name */
+	unsigned int ttid;          /* TOE type id */
+	unsigned long flags;        /* device flags */
+	unsigned int mtu;           /* max size of TX offloaded data */
+	unsigned int nconn;         /* max # of offloaded connections */
+	struct ifnet *lldev;   /* LL device associated with TOE messages */
+	const struct tom_info *offload_mod; /* attached TCP offload module */
+	struct sysctl_oid *sysctl_root;    /* root of proc dir for this TOE */
+	int (*open)(struct toedev *dev);
+	int (*close)(struct toedev *dev);
+	int (*can_offload)(struct toedev *dev, struct socket *so);
+	int (*connect)(struct toedev *dev, struct socket *so,
+		       struct ifnet *egress_ifp);
+	int (*send)(struct toedev *dev, struct mbuf *m);
+	int (*recv)(struct toedev *dev, struct mbuf **m, int n);
+	int (*ctl)(struct toedev *dev, unsigned int req, void *data);
+	void (*arp_update)(struct toedev *dev, struct rtentry *neigh);
+	void (*failover)(struct toedev *dev, struct ifnet *bond_ifp,
+			 struct ifnet *ndev, int event);
+	void *priv;                 /* driver private data */
+	void *l2opt;                /* optional layer 2 data */
+	void *l3opt;                /* optional layer 3 data */
+	void *l4opt;                /* optional layer 4 data */
+	void *ulp;                  /* ulp stuff */
+};
+
+struct tom_info {
+	TAILQ_ENTRY(tom_info) entry;
+	int (*attach)(struct toedev *dev, const struct offload_id *entry);
+	int (*detach)(struct toedev *dev);
+	const char *name;
+	const struct offload_id *id_table;
+};
+
+static inline void init_offload_dev(struct toedev *dev)
+{
+
+}
+
+extern int register_tom(struct tom_info *t);
+extern int unregister_tom(struct tom_info *t);
+extern int register_toedev(struct toedev *dev, const char *name);
+extern int unregister_toedev(struct toedev *dev);
+extern int activate_offload(struct toedev *dev);
+extern int toe_send(struct toedev *dev, struct mbuf *m);
+extern void toe_arp_update(struct rtentry *rt);
+extern struct ifnet *offload_get_phys_egress(struct ifnet *dev,
+						  struct socket *so,
+						  int context);
+extern int toe_receive_mbuf(struct toedev *dev, struct mbuf **m, int n);
+	
+static inline void toe_neigh_update(struct ifnet *neigh) {}
+static inline void toe_failover(struct ifnet *bond_ifp,
+				struct ifnet *fail_ifp, int event)
+{}
+static inline int toe_enslave(struct ifnet *bond_ifp,
+			      struct ifnet *slave_ifp)
+{
+	return 0;
+}
+
+#endif /* _OFFLOAD_DEV_H_ */
diff --git a/sys/sys/socket.h b/sys/sys/socket.h
index 5b12357..0952331 100644
--- a/sys/sys/socket.h
+++ b/sys/sys/socket.h
@@ -118,6 +118,8 @@ typedef	__uid_t		uid_t;
 #define	SO_ACCEPTFILTER	0x1000		/* there is an accept filter */
 #define	SO_BINTIME	0x2000		/* timestamp received dgram traffic */
 #endif
+#define	SO_NOOFFLOAD	0x4000		/* socket cannot be offloaded */
+#define	SO_NO_DDP       0x8000		/* disable direct data placement */
 
 /*
  * Additional options, not kept in so_options.