- Updated TOE support in the kernel.

- Stateful TCP offload drivers for Terminator 3 and 4 (T3 and T4) ASICs.
  These are available as t3_tom and t4_tom modules that augment cxgb(4)
  and cxgbe(4) respectively.  The cxgb/cxgbe drivers continue to work as
  usual with or without these extra features.

- iWARP driver for Terminator 3 ASIC (kernel verbs).  T4 iWARP in the
  works and will follow soon.

Build-tested with make universe.

30s overview
============
What interfaces support TCP offload?  Look for TOE4 and/or TOE6 in the
capabilities of an interface:
# ifconfig -m | grep TOE

Enable/disable TCP offload on an interface (just like any other ifnet
capability):
# ifconfig cxgbe0 toe
# ifconfig cxgbe0 -toe

Which connections are offloaded?  Look for toe4 and/or toe6 in the
output of netstat and sockstat:
# netstat -np tcp | grep toe
# sockstat -46c | grep toe

Reviewed by:	bz, gnn
Sponsored by:	Chelsio communications.
MFC after:	~3 months (after 9.1, and after ensuring MFC is feasible)

10 files changed, 64 insertions, 49 deletions

diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index 0e66091..80ac632 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -314,6 +314,7 @@ SUBDIR=	\
 	${_ti} \
 	tl \
 	tmpfs \
+	${_toecore} \
 	${_tpm} \
 	trm \
 	${_twa} \
@@ -392,6 +393,7 @@ _random=	random
 .if (${MK_INET_SUPPORT} != "no" || ${MK_INET6_SUPPORT} != "no") || \
 	defined(ALL_MODULES)
 _carp=	carp
+_toecore=	toecore
 .endif
 
 .if ${MK_INET_SUPPORT} != "no" || defined(ALL_MODULES)
diff --git a/sys/modules/cxgb/Makefile b/sys/modules/cxgb/Makefile
index a6af817..325c705 100644
--- a/sys/modules/cxgb/Makefile
+++ b/sys/modules/cxgb/Makefile
@@ -1,39 +1,12 @@
 # $FreeBSD$
 SUBDIR= cxgb
-SUBDIR+= ${_toecore}
+SUBDIR+= cxgb_t3fw
 SUBDIR+= ${_tom}
 SUBDIR+= ${_iw_cxgb}
-SUBDIR+= cxgb_t3fw
-
-.if defined(SYSDIR)
-_sysdir = ${SYSDIR}
-.endif
-
-# Based on bsd.kmod.mk but we don't modify SYSDIR in this one.
-.for _dir in ${.CURDIR}/../.. ${.CURDIR}/../../.. ${.CURDIR}/../../../.. \
-    /sys /usr/src/sys
-.if !defined(_sysdir) && exists(${_dir}/kern/) && exists(${_dir}/conf/kmod.mk)
-_sysdir = ${_dir}
-.endif
-.endfor
-.if !defined(_sysdir) || !exists(${_sysdir}/kern/) || \
-    !exists(${_sysdir}/conf/kmod.mk)
-.error "can't find kernel source tree"
-.endif
-
-_toe_header = ${_sysdir}/netinet/toedev.h
-
-.if exists(${_toe_header})
-_toecore = toecore
-#_tom = tom
-.endif
-
-.if ${MACHINE_CPUARCH} == "i386" && exists(${_toe_header})
-_iw_cxgb = iw_cxgb
-.endif
 
-.if ${MACHINE_CPUARCH} == "amd64" && exists(${_toe_header})
-_iw_cxgb = iw_cxgb
+.if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386"
+_tom=		tom
+_iw_cxgb=	iw_cxgb
 .endif
 
 .include <bsd.subdir.mk>
diff --git a/sys/modules/cxgb/cxgb/Makefile b/sys/modules/cxgb/cxgb/Makefile
index ed4e229..8807ece 100644
--- a/sys/modules/cxgb/cxgb/Makefile
+++ b/sys/modules/cxgb/cxgb/Makefile
@@ -8,7 +8,7 @@ CXGB = ${.CURDIR}/../../../dev/cxgb
 KMOD=	if_cxgb
 SRCS=	cxgb_mc5.c cxgb_vsc8211.c cxgb_ael1002.c cxgb_mv88e1xxx.c 
 SRCS+=	cxgb_xgmac.c cxgb_vsc7323.c cxgb_t3_hw.c cxgb_main.c cxgb_aq100x.c
-SRCS+=  cxgb_sge.c cxgb_offload.c cxgb_tn1010.c
+SRCS+=  cxgb_sge.c cxgb_tn1010.c
 SRCS+=	device_if.h bus_if.h pci_if.h
 SRCS+=	opt_inet.h opt_inet6.h opt_zero.h opt_sched.h
 SRCS+=	uipc_mvec.c
@@ -19,6 +19,7 @@ CFLAGS+= -g -DDEFAULT_JUMBO -I${CXGB}
 .if ${MK_INET_SUPPORT} != "no"
 opt_inet.h:
 	@echo "#define INET 1" > ${.TARGET}
+	@echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
 .endif
 
 .if ${MK_INET6_SUPPORT} != "no"
diff --git a/sys/modules/cxgb/iw_cxgb/Makefile b/sys/modules/cxgb/iw_cxgb/Makefile
index 2cf2ef8..f633bd5 100644
--- a/sys/modules/cxgb/iw_cxgb/Makefile
+++ b/sys/modules/cxgb/iw_cxgb/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 CXGB = ${.CURDIR}/../../../dev/cxgb
 .PATH: ${CXGB}/ulp/iw_cxgb
 
@@ -8,8 +10,15 @@ SRCS=   iw_cxgb.c iw_cxgb_cm.c iw_cxgb_hal.c
 SRCS+=  iw_cxgb_provider.c iw_cxgb_qp.c iw_cxgb_resource.c
 SRCS+=  iw_cxgb_ev.c iw_cxgb_mem.c iw_cxgb_dbg.c iw_cxgb_cq.c
 SRCS+=  bus_if.h device_if.h opt_sched.h pci_if.h pcib_if.h opt_ktr.h
-SRCS+=  opt_inet.h
-CFLAGS+= -g -I${CXGB}
-#CFLAGS+= -DDEBUG
+SRCS+=  opt_inet.h opt_ofed.h vnode_if.h
+CFLAGS+= -I${CXGB} -I${.CURDIR}/../../../ofed/include -DLINUX_TYPES_DEFINED
+
+.if !defined(KERNBUILDDIR)
+.if ${MK_INET_SUPPORT} != "no"
+opt_inet.h:
+	echo "#define INET 1" > ${.TARGET}
+	echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
+.endif
+.endif
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/cxgb/toecore/Makefile b/sys/modules/cxgb/toecore/Makefile
deleted file mode 100644
index 4342519..0000000
--- a/sys/modules/cxgb/toecore/Makefile
+++ /dev/null
@@ -1,11 +0,0 @@
-# $FreeBSD$
-
-CXGB = ${.CURDIR}/../../../dev/cxgb
-.PATH: ${CXGB}/ulp/toecore
-
-KMOD=	toecore
-SRCS=   toedev.c
-SRCS+=	device_if.h bus_if.h pci_if.h opt_sched.h opt_inet.h
-CFLAGS+= -g -I${CXGB}
-
-.include <bsd.kmod.mk>
diff --git a/sys/modules/cxgb/tom/Makefile b/sys/modules/cxgb/tom/Makefile
index 4c22670..8b08fe2 100644
--- a/sys/modules/cxgb/tom/Makefile
+++ b/sys/modules/cxgb/tom/Makefile
@@ -1,15 +1,25 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 CXGB = ${.CURDIR}/../../../dev/cxgb
 .PATH: ${CXGB}/ulp/tom
 
-KMOD=	tom
-SRCS=   cxgb_tom.c cxgb_cpl_io.c cxgb_listen.c cxgb_tom_sysctl.c cxgb_cpl_socket.c
-SRCS+=  cxgb_ddp.c cxgb_vm.c cxgb_l2t.c cxgb_tcp_offload.c
+KMOD=	t3_tom
+SRCS=   cxgb_tom.c cxgb_cpl_io.c cxgb_listen.c cxgb_l2t.c
 SRCS+=	opt_compat.h opt_inet.h opt_inet6.h opt_ipsec.h
 SRCS+=	opt_tcpdebug.h opt_ddb.h opt_sched.h opt_global.h opt_ktr.h
 SRCS+=	device_if.h bus_if.h pci_if.h
 CFLAGS+= -g -I${CXGB}
 
 #CFLAGS+= -DDEBUG_PRINT -DDEBUG
+
+.if !defined(KERNBUILDDIR)
+.if ${MK_INET_SUPPORT} != "no"
+opt_inet.h:
+	echo "#define INET 1" > ${.TARGET}
+	echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
+.endif
+.endif
+
 .include <bsd.kmod.mk>
diff --git a/sys/modules/cxgbe/Makefile b/sys/modules/cxgbe/Makefile
index 1d69f76..5ec8dcc 100644
--- a/sys/modules/cxgbe/Makefile
+++ b/sys/modules/cxgbe/Makefile
@@ -4,5 +4,10 @@
 
 SUBDIR = if_cxgbe
 SUBDIR+= firmware
+SUBDIR+= ${_tom}
+
+.if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386"
+_tom=		tom
+.endif
 
 .include <bsd.subdir.mk>
diff --git a/sys/modules/cxgbe/tom/Makefile b/sys/modules/cxgbe/tom/Makefile
new file mode 100644
index 0000000..647984d
--- /dev/null
+++ b/sys/modules/cxgbe/tom/Makefile
@@ -0,0 +1,15 @@
+#
+# $FreeBSD$
+#
+
+CXGBE = ${.CURDIR}/../../../dev/cxgbe
+.PATH: ${CXGBE}/tom
+
+KMOD = t4_tom
+SRCS = t4_tom.c t4_connect.c t4_listen.c t4_cpl_io.c t4_tom_l2t.c
+SRCS+= device_if.h bus_if.h pci_if.h
+SRCS+= opt_inet.h
+
+CFLAGS+= -I${CXGBE}
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/rdma/krping/Makefile b/sys/modules/rdma/krping/Makefile
index 5e4871b..ae6cc80 100644
--- a/sys/modules/rdma/krping/Makefile
+++ b/sys/modules/rdma/krping/Makefile
@@ -6,5 +6,7 @@ RDMA= ${.CURDIR}/../../../contrib/rdma/krping
 KMOD= krping
 SRCS= krping.c krping_dev.c getopt.c
 SRCS+=  bus_if.h device_if.h opt_sched.h pci_if.h pcib_if.h
+SRCS+=  vnode_if.h
+CFLAGS+= -I${.CURDIR}/../../../ofed/include 
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/toecore/Makefile b/sys/modules/toecore/Makefile
new file mode 100644
index 0000000..0b30ab0
--- /dev/null
+++ b/sys/modules/toecore/Makefile
@@ -0,0 +1,9 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../netinet
+
+KMOD=	toecore
+SRCS=	toecore.c
+SRCS+=	opt_ofed.h
+
+.include <bsd.kmod.mk>