MFC 305925,305926,305960

305925
    hyperv/hn: Don't mess up RSS key and indirect table after attachment.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D7889

305926
    hyperv/hn: Add sysctls to dynamic adjust RSS key and indirect table

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D7890

305960
    hyperv/hn: Apply RSS indirect table fixup before configure RNDIS RSS.

    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D7916

4 files changed, 139 insertions, 9 deletions

diff --git a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
index fe68872..7f541be 100644
--- a/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
+++ b/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c
@@ -325,6 +325,8 @@ static int hn_rx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS);
 static int hn_tx_stat_ulong_sysctl(SYSCTL_HANDLER_ARGS);
 static int hn_tx_conf_int_sysctl(SYSCTL_HANDLER_ARGS);
 static int hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS);
+static int hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS);
+static int hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS);
 static int hn_check_iplen(const struct mbuf *, int);
 static int hn_create_tx_ring(struct hn_softc *, int);
 static void hn_destroy_tx_ring(struct hn_tx_ring *);
@@ -391,6 +393,62 @@ hn_get_txswq_depth(const struct hn_tx_ring *txr)
 }
 
 static int
+hn_rss_reconfig(struct hn_softc *sc)
+{
+	int error;
+
+	HN_LOCK_ASSERT(sc);
+
+	/*
+	 * Disable RSS first.
+	 *
+	 * NOTE:
+	 * Direct reconfiguration by setting the UNCHG flags does
+	 * _not_ work properly.
+	 */
+	if (bootverbose)
+		if_printf(sc->hn_ifp, "disable RSS\n");
+	error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_DISABLE);
+	if (error) {
+		if_printf(sc->hn_ifp, "RSS disable failed\n");
+		return (error);
+	}
+
+	/*
+	 * Reenable the RSS w/ the updated RSS key or indirect
+	 * table.
+	 */
+	if (bootverbose)
+		if_printf(sc->hn_ifp, "reconfig RSS\n");
+	error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE);
+	if (error) {
+		if_printf(sc->hn_ifp, "RSS reconfig failed\n");
+		return (error);
+	}
+	return (0);
+}
+
+static void
+hn_rss_ind_fixup(struct hn_softc *sc, int nchan)
+{
+	struct ndis_rssprm_toeplitz *rss = &sc->hn_rss;
+	int i;
+
+	/*
+	 * Check indirect table to make sure that all channels in it
+	 * can be used.
+	 */
+	for (i = 0; i < NDIS_HASH_INDCNT; ++i) {
+		if (rss->rss_ind[i] >= nchan) {
+			if_printf(sc->hn_ifp,
+			    "RSS indirect table %d fixup: %u -> %d\n",
+			    i, rss->rss_ind[i], nchan - 1);
+			rss->rss_ind[i] = nchan - 1;
+		}
+	}
+}
+
+static int
 hn_ifmedia_upd(struct ifnet *ifp __unused)
 {
 
@@ -583,6 +641,12 @@ netvsc_attach(device_t dev)
 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "ndis_version",
 	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
 	    hn_ndis_version_sysctl, "A", "NDIS version");
+	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_key",
+	    CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
+	    hn_rss_key_sysctl, "IU", "RSS key");
+	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rss_ind",
+	    CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
+	    hn_rss_ind_sysctl, "IU", "RSS indirect table");
 
 	/*
 	 * Setup the ifmedia, which has been initialized earlier.
@@ -2031,6 +2095,51 @@ hn_ndis_version_sysctl(SYSCTL_HANDLER_ARGS)
 }
 
 static int
+hn_rss_key_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct hn_softc *sc = arg1;
+	int error;
+
+	HN_LOCK(sc);
+
+	error = SYSCTL_OUT(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key));
+	if (error || req->newptr == NULL)
+		goto back;
+
+	error = SYSCTL_IN(req, sc->hn_rss.rss_key, sizeof(sc->hn_rss.rss_key));
+	if (error)
+		goto back;
+
+	error = hn_rss_reconfig(sc);
+back:
+	HN_UNLOCK(sc);
+	return (error);
+}
+
+static int
+hn_rss_ind_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct hn_softc *sc = arg1;
+	int error;
+
+	HN_LOCK(sc);
+
+	error = SYSCTL_OUT(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind));
+	if (error || req->newptr == NULL)
+		goto back;
+
+	error = SYSCTL_IN(req, sc->hn_rss.rss_ind, sizeof(sc->hn_rss.rss_ind));
+	if (error)
+		goto back;
+
+	hn_rss_ind_fixup(sc, sc->hn_rx_ring_inuse);
+	error = hn_rss_reconfig(sc);
+back:
+	HN_UNLOCK(sc);
+	return (error);
+}
+
+static int
 hn_check_iplen(const struct mbuf *m, int hoff)
 {
 	const struct ip *ip;
@@ -3159,15 +3268,34 @@ hn_synth_attach(struct hn_softc *sc, int mtu)
 	 * are allocated.
 	 */
 
-	/* Setup default RSS key. */
-	memcpy(rss->rss_key, hn_rss_key_default, sizeof(rss->rss_key));
+	if (!device_is_attached(sc->hn_dev)) {
+		/*
+		 * Setup default RSS key and indirect table for the
+		 * attach DEVMETHOD.  They can be altered later on,
+		 * so don't mess them up once this interface is attached.
+		 */
+		if (bootverbose) {
+			if_printf(sc->hn_ifp, "setup default RSS key and "
+			    "indirect table\n");
+		}
+
+		/* Setup default RSS key. */
+		memcpy(rss->rss_key, hn_rss_key_default, sizeof(rss->rss_key));
 
-	/* Setup default RSS indirect table. */
-	/* TODO: Take ndis_rss_caps.ndis_nind into account. */
-	for (i = 0; i < NDIS_HASH_INDCNT; ++i)
-		rss->rss_ind[i] = i % nchan;
+		/* Setup default RSS indirect table. */
+		/* TODO: Take ndis_rss_caps.ndis_nind into account. */
+		for (i = 0; i < NDIS_HASH_INDCNT; ++i)
+			rss->rss_ind[i] = i % nchan;
+	} else {
+		/*
+		 * # of usable channels may be changed, so we have to
+		 * make sure that all entries in RSS indirect table
+		 * are valid.
+		 */
+		hn_rss_ind_fixup(sc, nchan);
+	}
 
-	error = hn_rndis_conf_rss(sc);
+	error = hn_rndis_conf_rss(sc, NDIS_RSS_FLAG_NONE);
 	if (error) {
 		/*
 		 * Failed to configure RSS key or indirect table; only
diff --git a/sys/dev/hyperv/netvsc/hv_rndis_filter.c b/sys/dev/hyperv/netvsc/hv_rndis_filter.c
index ff7a84c..72ab5ad 100644
--- a/sys/dev/hyperv/netvsc/hv_rndis_filter.c
+++ b/sys/dev/hyperv/netvsc/hv_rndis_filter.c
@@ -837,7 +837,7 @@ hn_rndis_conf_offload(struct hn_softc *sc)
 }
 
 int
-hn_rndis_conf_rss(struct hn_softc *sc)
+hn_rndis_conf_rss(struct hn_softc *sc, uint16_t flags)
 {
 	struct ndis_rssprm_toeplitz *rss = &sc->hn_rss;
 	struct ndis_rss_params *prm = &rss->rss_params;
@@ -858,6 +858,7 @@ hn_rndis_conf_rss(struct hn_softc *sc)
 	prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS;
 	prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2;
 	prm->ndis_hdr.ndis_size = sizeof(*rss);
+	prm->ndis_flags = flags;
 	prm->ndis_hash = NDIS_HASH_FUNCTION_TOEPLITZ |
 	    NDIS_HASH_IPV4 | NDIS_HASH_TCP_IPV4 |
 	    NDIS_HASH_IPV6 | NDIS_HASH_TCP_IPV6;
diff --git a/sys/dev/hyperv/netvsc/if_hnvar.h b/sys/dev/hyperv/netvsc/if_hnvar.h
index 23626ee..38ff16e 100644
--- a/sys/dev/hyperv/netvsc/if_hnvar.h
+++ b/sys/dev/hyperv/netvsc/if_hnvar.h
@@ -118,7 +118,7 @@ uint32_t	hn_chim_alloc(struct hn_softc *sc);
 void		hn_chim_free(struct hn_softc *sc, uint32_t chim_idx);
 
 int		hn_rndis_attach(struct hn_softc *sc);
-int		hn_rndis_conf_rss(struct hn_softc *sc);
+int		hn_rndis_conf_rss(struct hn_softc *sc, uint16_t flags);
 void		*hn_rndis_pktinfo_append(struct rndis_packet_msg *,
 		    size_t pktsize, size_t pi_dlen, uint32_t pi_type);
 int		hn_rndis_get_rsscaps(struct hn_softc *sc, int *rxr_cnt);
diff --git a/sys/dev/hyperv/netvsc/ndis.h b/sys/dev/hyperv/netvsc/ndis.h
index 895f756..28920f8 100644
--- a/sys/dev/hyperv/netvsc/ndis.h
+++ b/sys/dev/hyperv/netvsc/ndis.h
@@ -188,6 +188,7 @@ struct ndis_rss_params {
 #define	NDIS_RSS_PARAMS_REV_1		1	/* NDIS 6.0 */
 #define	NDIS_RSS_PARAMS_REV_2		2	/* NDIS 6.20 */
 
+#define	NDIS_RSS_FLAG_NONE		0x0000
 #define	NDIS_RSS_FLAG_BCPU_UNCHG	0x0001
 #define	NDIS_RSS_FLAG_HASH_UNCHG	0x0002
 #define	NDIS_RSS_FLAG_IND_UNCHG		0x0004