Replace struct tcpopt with a proxy toeopt struct in the TOE driver interface to

the TCP syncache. This returns struct tcpopt to being private within the TCP
implementation, thus allowing it to be modified without ABI concerns.

The patch breaks the ABI. Bump __FreeBSD_version to 800103 accordingly. The cxgb
driver is the only TOE consumer affected by this change, and needs to be
recompiled along with the kernel.

Suggested by:	rwatson
Reviewed by:	rwatson, kmacy
Approved by:	re (kensmith), kensmith (mentor temporarily unavailable)

7 files changed, 58 insertions, 22 deletions

diff --git a/UPDATING b/UPDATING
index 99fd25b..411f56c 100644
--- a/UPDATING
+++ b/UPDATING
@@ -22,6 +22,13 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 8.x IS SLOW:
 	to maximize performance.  (To disable malloc debugging, run
 	ln -s aj /etc/malloc.conf.)
 
+20090713:
+	The TOE interface to the TCP syncache has been modified to remove struct
+	tcpopt (<netinet/tcp_var.h>) from the ABI of the network stack.  The
+	cxgb driver is the only TOE consumer affected by this change, and needs
+	to be recompiled along with the kernel. As this change breaks the ABI,
+	bump __FreeBSD_version to 800103.
+
 20090712:
 	Padding has been added to struct tcpcb, sackhint and tcpstat in
 	<netinet/tcp_var.h> to facilitate future MFCs and bug fixes whilst
diff --git a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
index 96a0dc8..f3c6d2b 100644
--- a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
+++ b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
@@ -3249,13 +3249,13 @@ static void
 syncache_add_accept_req(struct cpl_pass_accept_req *req, struct socket *lso, struct toepcb *toep)
 {
 	struct in_conninfo inc;
-	struct tcpopt to;
+	struct toeopt toeo;
 	struct tcphdr th;
 	struct inpcb *inp;
 	int mss, wsf, sack, ts;
 	uint32_t rcv_isn = ntohl(req->rcv_isn);
 	
-	bzero(&to, sizeof(struct tcpopt));
+	bzero(&toeo, sizeof(struct toeopt));
 	inp = so_sotoinpcb(lso);
 	
 	/*
@@ -3281,10 +3281,11 @@ syncache_add_accept_req(struct cpl_pass_accept_req *req, struct socket *lso, str
 	wsf = req->tcp_options.wsf;
 	ts = req->tcp_options.tstamp;
 	sack = req->tcp_options.sack;
-	to.to_mss = mss;
-	to.to_wscale = wsf;
-	to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0);
-	tcp_offload_syncache_add(&inc, &to, &th, inp, &lso, &cxgb_toe_usrreqs, toep);
+	toeo.to_mss = mss;
+	toeo.to_wscale = wsf;
+	toeo.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0);
+	tcp_offload_syncache_add(&inc, &toeo, &th, inp, &lso, &cxgb_toe_usrreqs,
+toep);
 }
 
 
@@ -3584,7 +3585,7 @@ syncache_expand_establish_req(struct cpl_pass_establish *req, struct socket **so
 {
 
 	struct in_conninfo inc;
-	struct tcpopt to;
+	struct toeopt to;
 	struct tcphdr th;
 	int mss, wsf, sack, ts;
 	struct mbuf *m = NULL;
@@ -3597,7 +3598,7 @@ syncache_expand_establish_req(struct cpl_pass_establish *req, struct socket **so
 	
 	opt = ntohs(req->tcp_opt);
 	
-	bzero(&to, sizeof(struct tcpopt));
+	bzero(&toeo, sizeof(struct toeopt));
 	
 	/*
 	 * Fill out information for entering us into the syncache
@@ -3617,15 +3618,15 @@ syncache_expand_establish_req(struct cpl_pass_establish *req, struct socket **so
 	ts   = G_TCPOPT_TSTAMP(opt);
 	sack = G_TCPOPT_SACK(opt);
 	
-	to.to_mss = mss;
-	to.to_wscale =  G_TCPOPT_SND_WSCALE(opt);
-	to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0);
+	toeo.to_mss = mss;
+	toeo.to_wscale =  G_TCPOPT_SND_WSCALE(opt);
+	toeo.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0);
 
 	DPRINTF("syncache expand of %d:%d %d:%d mss:%d wsf:%d ts:%d sack:%d\n",
 	    ntohl(req->local_ip), ntohs(req->local_port),
 	    ntohl(req->peer_ip), ntohs(req->peer_port),
 	    mss, wsf, ts, sack);
-	return tcp_offload_syncache_expand(&inc, &to, &th, so, m);
+	return tcp_offload_syncache_expand(&inc, &toeo, &th, so, m);
 }
 
 
diff --git a/sys/netinet/tcp_offload.h b/sys/netinet/tcp_offload.h
index d05ef46..48f35ff 100644
--- a/sys/netinet/tcp_offload.h
+++ b/sys/netinet/tcp_offload.h
@@ -184,6 +184,19 @@ struct toe_usrreqs {
 	void (*tu_syncache_event)(int event, void *toep);
 };
 
+/*
+ * Proxy for struct tcpopt between TOE drivers and TCP functions.
+ */
+struct toeopt {
+	u_int64_t	to_flags;	/* see tcpopt in tcp_var.h */
+	u_int16_t	to_mss;		/* maximum segment size */
+	u_int8_t	to_wscale;	/* window scaling */
+
+	u_int8_t	_pad1;		/* explicit pad for 64bit alignment */
+	u_int32_t	_pad2;		/* explicit pad for 64bit alignment */
+	u_int64_t	_pad3[4];	/* TBD */
+};
+
 #define	TOE_SC_ENTRY_PRESENT		1	/* 4-tuple already present */
 #define	TOE_SC_DROP			2	/* connection was timed out */
 
diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c
index dc05f46..440115d 100644
--- a/sys/netinet/tcp_syncache.c
+++ b/sys/netinet/tcp_syncache.c
@@ -943,14 +943,20 @@ failed:
 }
 
 int
-tcp_offload_syncache_expand(struct in_conninfo *inc, struct tcpopt *to,
+tcp_offload_syncache_expand(struct in_conninfo *inc, struct toeopt *toeo,
     struct tcphdr *th, struct socket **lsop, struct mbuf *m)
 {
 	INIT_VNET_INET(curvnet);
+	struct tcpopt to;
 	int rc;
+
+	bzero(&to, sizeof(struct tcpopt));
+	to.to_mss = toeo->to_mss;
+	to.to_wscale = toeo->to_wscale;
+	to.to_flags = toeo->to_flags;
 	
 	INP_INFO_WLOCK(&V_tcbinfo);
-	rc = syncache_expand(inc, to, th, lsop, m);
+	rc = syncache_expand(inc, &to, th, lsop, m);
 	INP_INFO_WUNLOCK(&V_tcbinfo);
 
 	return (rc);
@@ -1437,15 +1443,22 @@ syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th,
 }
 
 void
-tcp_offload_syncache_add(struct in_conninfo *inc, struct tcpopt *to,
+tcp_offload_syncache_add(struct in_conninfo *inc, struct toeopt *toeo,
     struct tcphdr *th, struct inpcb *inp, struct socket **lsop,
     struct toe_usrreqs *tu, void *toepcb)
 {
 	INIT_VNET_INET(curvnet);
+	struct tcpopt to;
+
+	bzero(&to, sizeof(struct tcpopt));
+	to.to_mss = toeo->to_mss;
+	to.to_wscale = toeo->to_wscale;
+	to.to_flags = toeo->to_flags;
 
 	INP_INFO_WLOCK(&V_tcbinfo);
 	INP_WLOCK(inp);
-	_syncache_add(inc, to, th, inp, lsop, NULL, tu, toepcb);
+
+	_syncache_add(inc, &to, th, inp, lsop, NULL, tu, toepcb);
 }
 
 /*
diff --git a/sys/netinet/tcp_syncache.h b/sys/netinet/tcp_syncache.h
index 5ed0c9e..93c7aaa 100644
--- a/sys/netinet/tcp_syncache.h
+++ b/sys/netinet/tcp_syncache.h
@@ -34,6 +34,8 @@
 #define _NETINET_TCP_SYNCACHE_H_
 #ifdef _KERNEL
 
+struct toeopt;
+
 void	 syncache_init(void);
 #ifdef VIMAGE
 void	syncache_destroy(void);
@@ -41,11 +43,11 @@ void	syncache_destroy(void);
 void	 syncache_unreach(struct in_conninfo *, struct tcphdr *);
 int	 syncache_expand(struct in_conninfo *, struct tcpopt *,
 	     struct tcphdr *, struct socket **, struct mbuf *);
-int	 tcp_offload_syncache_expand(struct in_conninfo *inc, struct tcpopt *to,
+int	 tcp_offload_syncache_expand(struct in_conninfo *inc, struct toeopt *toeo,
              struct tcphdr *th, struct socket **lsop, struct mbuf *m);
 void	 syncache_add(struct in_conninfo *, struct tcpopt *,
 	     struct tcphdr *, struct inpcb *, struct socket **, struct mbuf *);
-void	 tcp_offload_syncache_add(struct in_conninfo *, struct tcpopt *,
+void	 tcp_offload_syncache_add(struct in_conninfo *, struct toeopt *,
              struct tcphdr *, struct inpcb *, struct socket **,
              struct toe_usrreqs *tu, void *toepcb);
 
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index d018358..e802220 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -267,7 +267,7 @@ struct tcpcb {
  * options in tcp_addoptions.
  */
 struct tcpopt {
-	u_long		to_flags;	/* which options are present */
+	u_int64_t	to_flags;	/* which options are present */
 #define	TOF_MSS		0x0001		/* maximum segment size */
 #define	TOF_SCALE	0x0002		/* window scaling */
 #define	TOF_SACKPERM	0x0004		/* SACK permitted */
@@ -277,11 +277,11 @@ struct tcpopt {
 #define	TOF_MAXOPT	0x0100
 	u_int32_t	to_tsval;	/* new timestamp */
 	u_int32_t	to_tsecr;	/* reflected timestamp */
+	u_char		*to_sacks;	/* pointer to the first SACK blocks */
+	u_char		*to_signature;	/* pointer to the TCP-MD5 signature */
 	u_int16_t	to_mss;		/* maximum segment size */
 	u_int8_t	to_wscale;	/* window scaling */
 	u_int8_t	to_nsacks;	/* number of SACK blocks */
-	u_char		*to_sacks;	/* pointer to the first SACK blocks */
-	u_char		*to_signature;	/* pointer to the TCP-MD5 signature */
 };
 
 /*
diff --git a/sys/sys/param.h b/sys/sys/param.h
index 890a5c9..dffedb2 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -58,7 +58,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 800102	/* Master, propagated to newvers */
+#define __FreeBSD_version 800103	/* Master, propagated to newvers */
 
 #ifndef LOCORE
 #include <sys/types.h>