- Updated TOE support in the kernel.

- Stateful TCP offload drivers for Terminator 3 and 4 (T3 and T4) ASICs.
  These are available as t3_tom and t4_tom modules that augment cxgb(4)
  and cxgbe(4) respectively.  The cxgb/cxgbe drivers continue to work as
  usual with or without these extra features.

- iWARP driver for Terminator 3 ASIC (kernel verbs).  T4 iWARP in the
  works and will follow soon.

Build-tested with make universe.

30s overview
============
What interfaces support TCP offload?  Look for TOE4 and/or TOE6 in the
capabilities of an interface:
# ifconfig -m | grep TOE

Enable/disable TCP offload on an interface (just like any other ifnet
capability):
# ifconfig cxgbe0 toe
# ifconfig cxgbe0 -toe

Which connections are offloaded?  Look for toe4 and/or toe6 in the
output of netstat and sockstat:
# netstat -np tcp | grep toe
# sockstat -46c | grep toe

Reviewed by:	bz, gnn
Sponsored by:	Chelsio communications.
MFC after:	~3 months (after 9.1, and after ensuring MFC is feasible)

5 files changed, 49 insertions, 16 deletions

diff --git a/sys/contrib/rdma/krping/krping.c b/sys/contrib/rdma/krping/krping.c
index d787965..99d1924 100644
--- a/sys/contrib/rdma/krping/krping.c
+++ b/sys/contrib/rdma/krping/krping.c
@@ -41,7 +41,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/socket.h>
-#include <sys/module.h>
 #include <sys/endian.h>
 #include <sys/limits.h>
 #include <sys/proc.h>
@@ -53,11 +52,13 @@ __FBSDID("$FreeBSD$");
 #include <sys/queue.h>
 #include <sys/taskqueue.h>
 #include <sys/syslog.h>
+#include <netinet/in.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
-#include <contrib/rdma/rdma_cm.h>
+#include <linux/types.h>
+#include <rdma/rdma_cm.h>
 
 #include "getopt.h"
 #include "krping.h"
@@ -83,6 +84,7 @@ static const struct krping_option krping_opts[] = {
 	{"bw", OPT_NOPARAM, 'B'},
 	{"tx-depth", OPT_INT, 't'},
   	{"poll", OPT_NOPARAM, 'P'},
+  	{"memlimit", OPT_INT, 'm'},
 	{NULL, 0, 0}
 };
 
@@ -254,10 +256,14 @@ static void krping_cq_event_handler(struct ib_cq *cq, void *ctx)
 		ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP);
 	while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) {
 		if (wc.status) {
-			if (wc.status != IB_WC_WR_FLUSH_ERR)
-				log(LOG_ERR, "cq completion failed status %d\n",
+			if (wc.status == IB_WC_WR_FLUSH_ERR) {
+				DEBUG_LOG("cq flushed\n");
+				continue;
+			} else {
+				log(LOG_CRIT, "cq completion failed status %d\n",
 					wc.status);
-			goto error;
+				goto error;
+			}
 		}
 
 		switch (wc.opcode) {
@@ -432,8 +438,17 @@ static int krping_setup_buffers(struct krping_cb *cb)
 		}
 	}
 
-	cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL,
-		PAGE_SIZE, 0);
+	/* RNIC adapters have a limit upto which it can register physical memory
+	 * If DMA-MR memory mode is set then normally driver registers maximum
+	 * supported memory. After that if contigmalloc allocates memory beyond the
+	 * specified RNIC limit then Krping may not work.
+	 */
+	if (cb->use_dmamr && cb->memlimit)
+		cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, cb->memlimit,
+					    PAGE_SIZE, 0);
+	else 
+		cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL,
+					    PAGE_SIZE, 0);
 
 	if (!cb->rdma_buf) {
 		log(LOG_ERR, "rdma_buf malloc failed\n");
@@ -458,8 +473,12 @@ static int krping_setup_buffers(struct krping_cb *cb)
 	}
 
 	if (!cb->server || cb->wlat || cb->rlat || cb->bw) {
-		cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK,
-			0, -1UL, PAGE_SIZE, 0);
+		if (cb->use_dmamr && cb->memlimit)
+			cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK,
+						     0, cb->memlimit, PAGE_SIZE, 0);
+		else
+			cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK,
+						     0, -1UL, PAGE_SIZE, 0);
 		if (!cb->start_buf) {
 			log(LOG_ERR, "start_buf malloc failed\n");
 			ret = ENOMEM;
@@ -1636,6 +1655,8 @@ int krping_doit(char *cmd)
 	cb->state = IDLE;
 	cb->size = 64;
 	cb->txdepth = RPING_SQ_DEPTH;
+	cb->use_dmamr = 1;
+	cb->memlimit = 0;
 	mtx_init(&cb->lock, "krping mtx", NULL, MTX_DUPOK|MTX_DEF);
 
 	while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg,
@@ -1713,6 +1734,15 @@ int krping_doit(char *cmd)
 		case 'd':
 			debug++;
 			break;
+		case 'm':
+                        cb->memlimit = optint;
+                        if (cb->memlimit < 1) {
+                                log(LOG_ERR, "Invalid memory limit %ju\n",
+				    cb->memlimit);
+                                ret = EINVAL;
+                        } else
+                                DEBUG_LOG(PFX "memory limit %d\n", (int)optint);
+                        break;
 		default:
 			log(LOG_ERR, "unknown opt %s\n", optarg);
 			ret = EINVAL;
diff --git a/sys/contrib/rdma/krping/krping.h b/sys/contrib/rdma/krping/krping.h
index 8578e7e..d234825 100644
--- a/sys/contrib/rdma/krping/krping.h
+++ b/sys/contrib/rdma/krping/krping.h
@@ -1,7 +1,7 @@
 /*
  * $FreeBSD$
  */
-#include <contrib/rdma/ib_verbs.h>
+#include <rdma/ib_verbs.h>
 #include <netinet/in.h>
 
 /*
@@ -92,6 +92,8 @@ struct krping_cb {
 	int count;			/* ping count */
 	int size;			/* ping data size */
 	int validate;			/* validate ping data */
+	uint64_t memlimit;		/* limit of the physical memory that
+					   can be registered with dma_mr mode */
 
 	/* CM stuff */
 	struct rdma_cm_id *cm_id;	/* connection on client side,*/
diff --git a/sys/contrib/rdma/krping/krping_dev.c b/sys/contrib/rdma/krping/krping_dev.c
index 448f197..92d954d 100644
--- a/sys/contrib/rdma/krping/krping_dev.c
+++ b/sys/contrib/rdma/krping/krping_dev.c
@@ -14,7 +14,6 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
-#include <sys/module.h>
 #include <sys/systm.h>  /* uprintf */
 #include <sys/errno.h>
 #include <sys/param.h>  /* defines used in kernel.h */
@@ -51,6 +50,9 @@ typedef struct s_krping {
 /* vars */
 static struct cdev *krping_dev;
 
+#undef MODULE_VERSION
+#include <sys/module.h>
+
 static int
 krping_loader(struct module *m, int what, void *arg)
 {
@@ -175,6 +177,4 @@ krping_write(struct cdev *dev, struct uio *uio, int ioflag)
 	return(err);
 }
 
-MODULE_DEPEND(krping, rdma_core, 1, 1, 1);
-MODULE_DEPEND(krping, rdma_cma, 1, 1, 1);
 DEV_MODULE(krping,krping_loader,NULL);
diff --git a/sys/contrib/rdma/rdma_addr.c b/sys/contrib/rdma/rdma_addr.c
index b3f7349..33ec5ac 100644
--- a/sys/contrib/rdma/rdma_addr.c
+++ b/sys/contrib/rdma/rdma_addr.c
@@ -117,7 +117,8 @@ int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev,
 		     const unsigned char *dst_dev_addr)
 {
 	dev_addr->dev_type = RDMA_NODE_RNIC;
-	memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), MAX_ADDR_LEN);
+	memset(dev_addr->src_dev_addr, 0, MAX_ADDR_LEN);
+	memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), dev->if_addrlen);
 	memcpy(dev_addr->broadcast, dev->if_broadcastaddr, MAX_ADDR_LEN);
 	if (dst_dev_addr)
 		memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN);
@@ -207,7 +208,7 @@ static int addr_resolve_remote(struct sockaddr_in *src_in,
 		goto put;
 	}
  	ret = arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL, 
-		rt_key(iproute.ro_rt), dmac, &lle);
+		(struct sockaddr *)dst_in, dmac, &lle);
 	if (ret) {
 		goto put;
 	}
diff --git a/sys/contrib/rdma/rdma_cache.c b/sys/contrib/rdma/rdma_cache.c
index dced8eb..1e516a1 100644
--- a/sys/contrib/rdma/rdma_cache.c
+++ b/sys/contrib/rdma/rdma_cache.c
@@ -132,7 +132,7 @@ int ib_find_cached_gid(struct ib_device *device,
 	for (p = 0; p <= end_port(device) - start_port(device); ++p) {
 		cache = device->cache.gid_cache[p];
 		for (i = 0; i < cache->table_len; ++i) {
-			if (!memcmp(gid, &cache->table[i], 6)) { /* XXX */
+			if (!memcmp(gid, &cache->table[i], sizeof *gid)) {
 				*port_num = p + start_port(device);
 				if (index)
 					*index = i;