- Updated TOE support in the kernel.

- Stateful TCP offload drivers for Terminator 3 and 4 (T3 and T4) ASICs.
  These are available as t3_tom and t4_tom modules that augment cxgb(4)
  and cxgbe(4) respectively.  The cxgb/cxgbe drivers continue to work as
  usual with or without these extra features.

- iWARP driver for Terminator 3 ASIC (kernel verbs).  T4 iWARP in the
  works and will follow soon.

Build-tested with make universe.

30s overview
============
What interfaces support TCP offload?  Look for TOE4 and/or TOE6 in the
capabilities of an interface:
# ifconfig -m | grep TOE

Enable/disable TCP offload on an interface (just like any other ifnet
capability):
# ifconfig cxgbe0 toe
# ifconfig cxgbe0 -toe

Which connections are offloaded?  Look for toe4 and/or toe6 in the
output of netstat and sockstat:
# netstat -np tcp | grep toe
# sockstat -46c | grep toe

Reviewed by:	bz, gnn
Sponsored by:	Chelsio communications.
MFC after:	~3 months (after 9.1, and after ensuring MFC is feasible)

5 files changed, 30 insertions, 13 deletions

diff --git a/sys/ofed/drivers/infiniband/core/cma.c b/sys/ofed/drivers/infiniband/core/cma.c
index 750d2d5..9867f10 100644
--- a/sys/ofed/drivers/infiniband/core/cma.c
+++ b/sys/ofed/drivers/infiniband/core/cma.c
@@ -59,10 +59,10 @@ static int tavor_quirk = 0;
 module_param_named(tavor_quirk, tavor_quirk, int, 0644);
 MODULE_PARM_DESC(tavor_quirk, "Tavor performance quirk: limit MTU to 1K if > 0");
 
-int unify_tcp_port_space = 0;
+int unify_tcp_port_space = 1;
 module_param(unify_tcp_port_space, int, 0644);
 MODULE_PARM_DESC(unify_tcp_port_space, "Unify the host TCP and RDMA port "
-		 "space allocation (default=0)");
+		 "space allocation (default=1)");
 
 #define CMA_CM_RESPONSE_TIMEOUT 20
 #define CMA_MAX_CM_RETRIES 15
@@ -1478,6 +1478,7 @@ static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
 	struct sockaddr_in *sin;
 
 	id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
+					    id_priv->sock,
 					    iw_conn_req_handler,
 					    id_priv);
 	if (IS_ERR(id_priv->cm_id.iw))
@@ -2055,7 +2056,16 @@ static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
 				((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
 		}
 	}
-	return rdma_bind_addr(id, src_addr);
+	if (!cma_any_addr(src_addr))
+		return rdma_bind_addr(id, src_addr);
+	else {
+		struct sockaddr_in addr_in;
+
+        	memset(&addr_in, 0, sizeof addr_in);
+        	addr_in.sin_family = dst_addr->sa_family;
+        	addr_in.sin_len = sizeof addr_in;
+        	return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
+	}
 }
 
 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
@@ -2247,6 +2257,7 @@ static int cma_get_tcp_port(struct rdma_id_private *id_priv)
 		sock_release(sock);
 		return ret;
 	}
+
 	size = ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr);
 	ret = sock_getname(sock,
 			(struct sockaddr *) &id_priv->id.route.addr.src_addr,
@@ -2255,6 +2266,7 @@ static int cma_get_tcp_port(struct rdma_id_private *id_priv)
 		sock_release(sock);
 		return ret;
 	}
+
 	id_priv->sock = sock;
 	return 0;
 }
@@ -2604,7 +2616,8 @@ static int cma_connect_iw(struct rdma_id_private *id_priv,
 	int ret;
 	struct iw_cm_conn_param iw_param;
 
-	cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
+	cm_id = iw_create_cm_id(id_priv->id.device, id_priv->sock,
+				cma_iw_handler, id_priv);
 	if (IS_ERR(cm_id)) {
 		ret = PTR_ERR(cm_id);
 		goto out;
diff --git a/sys/ofed/drivers/infiniband/core/iwcm.c b/sys/ofed/drivers/infiniband/core/iwcm.c
index 625fec5..b13e53a 100644
--- a/sys/ofed/drivers/infiniband/core/iwcm.c
+++ b/sys/ofed/drivers/infiniband/core/iwcm.c
@@ -189,6 +189,7 @@ static void rem_ref(struct iw_cm_id *cm_id)
 static int cm_event_handler(struct iw_cm_id *cm_id, struct iw_cm_event *event);
 
 struct iw_cm_id *iw_create_cm_id(struct ib_device *device,
+				 struct socket *so,
 				 iw_cm_handler cm_handler,
 				 void *context)
 {
@@ -205,6 +206,7 @@ struct iw_cm_id *iw_create_cm_id(struct ib_device *device,
 	cm_id_priv->id.event_handler = cm_event_handler;
 	cm_id_priv->id.add_ref = add_ref;
 	cm_id_priv->id.rem_ref = rem_ref;
+	cm_id_priv->id.so = so;
 	spin_lock_init(&cm_id_priv->lock);
 	atomic_set(&cm_id_priv->refcount, 1);
 	init_waitqueue_head(&cm_id_priv->connect_wait);
@@ -629,6 +631,7 @@ static void cm_conn_req_handler(struct iwcm_id_private *listen_id_priv,
 	spin_unlock_irqrestore(&listen_id_priv->lock, flags);
 
 	cm_id = iw_create_cm_id(listen_id_priv->id.device,
+				iw_event->so,
 				listen_id_priv->id.cm_handler,
 				listen_id_priv->id.context);
 	/* If the cm_id could not be created, ignore the request */
diff --git a/sys/ofed/include/linux/net.h b/sys/ofed/include/linux/net.h
index 6e2aff3..f47acf9 100644
--- a/sys/ofed/include/linux/net.h
+++ b/sys/ofed/include/linux/net.h
@@ -48,12 +48,12 @@ sock_getname(struct socket *so, struct sockaddr *addr, int *sockaddr_len,
 	int error;
 
 	nam = NULL;
-	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0)
-		return (-ENOTCONN);
+	if (peer) {
+		if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0)
+			return (-ENOTCONN);
 
-	if (peer)
 		error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, nam);
-	else
+	} else
 		error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, nam);
 	if (error)
 		return (-error);
diff --git a/sys/ofed/include/net/netevent.h b/sys/ofed/include/net/netevent.h
index db5b50e..c7bbc5f 100644
--- a/sys/ofed/include/net/netevent.h
+++ b/sys/ofed/include/net/netevent.h
@@ -42,7 +42,7 @@ enum netevent_notif_type {
 struct llentry;
 
 static inline void
-_handle_arp_update_event(void *arg, struct llentry *lle)
+_handle_arp_update_event(void *arg, struct llentry *lle, int evt __unused)
 {
 	struct notifier_block *nb;
 
@@ -54,7 +54,7 @@ static inline int
 register_netevent_notifier(struct notifier_block *nb)
 {
 	nb->tags[NETEVENT_NEIGH_UPDATE] = EVENTHANDLER_REGISTER(
-	    arp_update_event, _handle_arp_update_event, nb, 0);
+	    lle_event, _handle_arp_update_event, nb, 0);
 	return (0);
 }
 
@@ -62,8 +62,7 @@ static inline int
 unregister_netevent_notifier(struct notifier_block *nb)
 {
 
-	EVENTHANDLER_DEREGISTER(arp_update_event,
-	    nb->tags[NETEVENT_NEIGH_UPDATE]);
+	EVENTHANDLER_DEREGISTER(lle_event, nb->tags[NETEVENT_NEIGH_UPDATE]);
 
 	return (0);
 }
diff --git a/sys/ofed/include/rdma/iw_cm.h b/sys/ofed/include/rdma/iw_cm.h
index cbb822e..412320e 100644
--- a/sys/ofed/include/rdma/iw_cm.h
+++ b/sys/ofed/include/rdma/iw_cm.h
@@ -63,6 +63,7 @@ struct iw_cm_event {
 	void *private_data;
 	u8 private_data_len;
 	void *provider_data;
+	struct socket *so;
 };
 
 /**
@@ -98,6 +99,7 @@ struct iw_cm_id {
 	/* Used by provider to add and remove refs on IW cm_id */
 	void (*add_ref)(struct iw_cm_id *);
 	void (*rem_ref)(struct iw_cm_id *);
+	struct socket           *so;
 };
 
 struct iw_cm_conn_param {
@@ -139,7 +141,7 @@ struct iw_cm_verbs {
  *   returned IW CM identifier.
  * @context: User specified context associated with the id.
  */
-struct iw_cm_id *iw_create_cm_id(struct ib_device *device,
+struct iw_cm_id *iw_create_cm_id(struct ib_device *device, struct socket *so,
 				 iw_cm_handler cm_handler, void *context);
 
 /**