MFC r330649:

Add support for per priority flow control, PFC, to mlx5en(4).

Add support for PFC and implement reading the per priority statistics
using the sysctl(8) interface. PFC is used together with VLAN priority
and can be enabled and disabled on a per priority basis.

Global pause frames and PFC are incompatible features and surrounding
logic has been added to warn the user about misconfiguration.

Update relevant mlx5core APIs for PFC configuration.

Sponsored by:	Mellanox Technologies

4 files changed, 251 insertions, 44 deletions

diff --git a/sys/dev/mlx5/mlx5_core/mlx5_port.c b/sys/dev/mlx5/mlx5_core/mlx5_port.c
index da08669..b06c939 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_port.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_port.c
@@ -352,18 +352,26 @@ int mlx5_query_port_max_mtu(struct mlx5_core_dev *dev, int *max_mtu)
 }
 EXPORT_SYMBOL_GPL(mlx5_query_port_max_mtu);
 
-int mlx5_set_port_pause(struct mlx5_core_dev *dev, u32 port,
-			u32 rx_pause, u32 tx_pause)
+int mlx5_set_port_pause_and_pfc(struct mlx5_core_dev *dev, u32 port,
+				u8 rx_pause, u8 tx_pause,
+				u8 pfc_en_rx, u8 pfc_en_tx)
 {
-	u32 in[MLX5_ST_SZ_DW(pfcc_reg)];
-	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
+	u32 out[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
 
-	memset(in, 0, sizeof(in));
-	memset(out, 0, sizeof(out));
+	if (pfc_en_rx || pfc_en_tx) {
+		/* PFC and global pauseframes are incompatible features */
+		if (tx_pause || rx_pause)
+			return -EINVAL;
+	}
 
 	MLX5_SET(pfcc_reg, in, local_port, port);
 	MLX5_SET(pfcc_reg, in, pptx, tx_pause);
 	MLX5_SET(pfcc_reg, in, pprx, rx_pause);
+	MLX5_SET(pfcc_reg, in, pfctx, pfc_en_tx);
+	MLX5_SET(pfcc_reg, in, pfcrx, pfc_en_rx);
+	MLX5_SET(pfcc_reg, in, prio_mask_tx, pfc_en_tx);
+	MLX5_SET(pfcc_reg, in, prio_mask_rx, pfc_en_rx);
 
 	return mlx5_core_access_reg(dev, in, sizeof(in), out,
 				   sizeof(out), MLX5_REG_PFCC, 0, 1);
@@ -392,25 +400,9 @@ int mlx5_query_port_pause(struct mlx5_core_dev *dev, u32 port,
 	return 0;
 }
 
-int mlx5_set_port_pfc(struct mlx5_core_dev *dev, u8 pfc_en_tx, u8 pfc_en_rx)
-{
-	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
-	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
-
-	MLX5_SET(pfcc_reg, in, local_port, 1);
-	MLX5_SET(pfcc_reg, in, pfctx, pfc_en_tx);
-	MLX5_SET(pfcc_reg, in, pfcrx, pfc_en_rx);
-	MLX5_SET_TO_ONES(pfcc_reg, in, prio_mask_tx);
-	MLX5_SET_TO_ONES(pfcc_reg, in, prio_mask_rx);
-
-	return mlx5_core_access_reg(dev, in, sizeof(in), out,
-				    sizeof(out), MLX5_REG_PFCC, 0, 1);
-}
-EXPORT_SYMBOL_GPL(mlx5_set_port_pfc);
-
 int mlx5_query_port_pfc(struct mlx5_core_dev *dev, u8 *pfc_en_tx, u8 *pfc_en_rx)
 {
-	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {0};
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)] = {};
 	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
 	int err;
 
@@ -420,12 +412,10 @@ int mlx5_query_port_pfc(struct mlx5_core_dev *dev, u8 *pfc_en_tx, u8 *pfc_en_rx)
 	if (err)
 		return err;
 
-	if (pfc_en_tx)
+	if (pfc_en_tx != NULL)
 		*pfc_en_tx = MLX5_GET(pfcc_reg, out, pfctx);
-
-	if (pfc_en_rx)
+	if (pfc_en_rx != NULL)
 		*pfc_en_rx = MLX5_GET(pfcc_reg, out, pfcrx);
-
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mlx5_query_port_pfc);
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 45295c2..49c626a 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -276,13 +276,55 @@ struct mlx5e_vport_stats {
   m(+1, u64 rs_corrected_symbols_lane2, "rs_corrected_symbols_lane2",			\
 			"FEC corrected symbol counter lane 2")				\
   m(+1, u64 rs_corrected_symbols_lane3, "rs_corrected_symbols_lane3",			\
-			"FEC corrected symbol counter lane 3")				\
+			"FEC corrected symbol counter lane 3")
+
+/* Per priority statistics for PFC */
+#define	MLX5E_PPORT_PER_PRIO_STATS_SUB(m,n,p)			\
+  m(n, p, +1, u64, rx_octets, "rx_octets", "Received octets")		\
+  m(n, p, +1, u64, reserved_0, "reserved_0", "Reserved")		\
+  m(n, p, +1, u64, reserved_1, "reserved_1", "Reserved")		\
+  m(n, p, +1, u64, reserved_2, "reserved_2", "Reserved")		\
+  m(n, p, +1, u64, rx_frames, "rx_frames", "Received frames")		\
+  m(n, p, +1, u64, tx_octets, "tx_octets", "Transmitted octets")	\
+  m(n, p, +1, u64, reserved_3, "reserved_3", "Reserved")		\
+  m(n, p, +1, u64, reserved_4, "reserved_4", "Reserved")		\
+  m(n, p, +1, u64, reserved_5, "reserved_5", "Reserved")		\
+  m(n, p, +1, u64, tx_frames, "tx_frames", "Transmitted frames")	\
+  m(n, p, +1, u64, rx_pause, "rx_pause", "Received pause frames")	\
+  m(n, p, +1, u64, rx_pause_duration, "rx_pause_duration",		\
+	"Received pause duration")					\
+  m(n, p, +1, u64, tx_pause, "tx_pause", "Transmitted pause frames")	\
+  m(n, p, +1, u64, tx_pause_duration, "tx_pause_duration",		\
+	"Transmitted pause duration")					\
+  m(n, p, +1, u64, rx_pause_transition, "rx_pause_transition",		\
+	"Received pause transitions")					\
+  m(n, p, +1, u64, rx_discards, "rx_discards", "Discarded received frames") \
+  m(n, p, +1, u64, device_stall_minor_watermark,			\
+	"device_stall_minor_watermark", "Device stall minor watermark")	\
+  m(n, p, +1, u64, device_stall_critical_watermark,			\
+	"device_stall_critical_watermark", "Device stall critical watermark")
+
+#define	MLX5E_PPORT_PER_PRIO_STATS_PREFIX(m,p,c,t,f,s,d) \
+  m(c, t pri_##p##_##f, "prio" #p "_" s, "Priority " #p " - " d)
+
+#define	MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO 8
+
+#define	MLX5E_PPORT_PER_PRIO_STATS(m) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,0) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,1) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,2) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,3) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,4) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,5) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,6) \
+  MLX5E_PPORT_PER_PRIO_STATS_SUB(MLX5E_PPORT_PER_PRIO_STATS_PREFIX,m,7)
 
 /*
  * Make sure to update mlx5e_update_pport_counters()
  * when adding a new MLX5E_PPORT_STATS block
  */
 #define	MLX5E_PPORT_STATS(m)			\
+  MLX5E_PPORT_PER_PRIO_STATS(m)		\
   MLX5E_PPORT_IEEE802_3_STATS(m)		\
   MLX5E_PPORT_RFC2819_STATS(m)
 
@@ -298,6 +340,8 @@ struct mlx5e_vport_stats {
 #define	MLX5E_PPORT_STATS_NUM \
   (0 MLX5E_PPORT_STATS(MLX5E_STATS_COUNT))
 
+#define	MLX5E_PPORT_PER_PRIO_STATS_NUM \
+  (0 MLX5E_PPORT_PER_PRIO_STATS(MLX5E_STATS_COUNT))
 #define	MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM \
   (0 MLX5E_PPORT_RFC2819_STATS_DEBUG(MLX5E_STATS_COUNT))
 #define	MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM \
@@ -391,8 +435,10 @@ struct mlx5e_params {
 	bool	cqe_zipping_en;
 	u32	lro_wqe_sz;
 	u16	rx_hash_log_tbl_sz;
-	u32	tx_pauseframe_control;
-	u32	rx_pauseframe_control;
+	u32	tx_pauseframe_control __aligned(4);
+	u32	rx_pauseframe_control __aligned(4);
+	u32	tx_priority_flow_control __aligned(4);
+	u32	rx_priority_flow_control __aligned(4);
 };
 
 #define	MLX5E_PARAMS(m)							\
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index cb5a907..c622dd2 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -228,6 +228,32 @@ mlx5e_find_link_mode(u32 subtype)
 }
 
 static int
+mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
+{
+	return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
+	    priv->params.rx_pauseframe_control,
+	    priv->params.tx_pauseframe_control,
+	    priv->params.rx_priority_flow_control,
+	    priv->params.tx_priority_flow_control));
+}
+
+static int
+mlx5e_set_port_pfc(struct mlx5e_priv *priv)
+{
+	int error;
+
+	if (priv->params.rx_pauseframe_control ||
+	    priv->params.tx_pauseframe_control) {
+		if_printf(priv->ifp,
+		    "Global pauseframes must be disabled before enabling PFC.\n");
+		error = -EINVAL;
+	} else {
+		error = mlx5e_set_port_pause_and_pfc(priv);
+	}
+	return (error);
+}
+
+static int
 mlx5e_media_change(struct ifnet *dev)
 {
 	struct mlx5e_priv *priv = dev->if_softc;
@@ -270,6 +296,15 @@ mlx5e_media_change(struct ifnet *dev)
 			goto done;
 		}
 	}
+	if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
+		/* check if PFC is enabled */
+		if (priv->params.rx_priority_flow_control ||
+		    priv->params.tx_priority_flow_control) {
+			if_printf(dev, "PFC must be disabled before enabling global pauseframes.\n");
+			error = EINVAL;
+			goto done;
+		}
+	}
 	/* update pauseframe control bits */
 	priv->params.rx_pauseframe_control =
 	    (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
@@ -282,9 +317,7 @@ mlx5e_media_change(struct ifnet *dev)
 	/* reconfigure the hardware */
 	mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
 	mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN);
-	mlx5_set_port_pause(mdev, 1,
-	    priv->params.rx_pauseframe_control,
-	    priv->params.tx_pauseframe_control);
+	error = -mlx5e_set_port_pause_and_pfc(priv);
 	if (was_opened)
 		mlx5_set_port_status(mdev, MLX5_PORT_UP);
 
@@ -324,6 +357,7 @@ mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 	unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 	unsigned x;
 	unsigned y;
+	unsigned z;
 
 	/* allocate firmware request structures */
 	in = mlx5_vzalloc(sz);
@@ -342,7 +376,8 @@ mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 	/* read IEEE802_3 counter group using predefined counter layout */
 	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
-	for (x = y = 0; x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
+	for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
+	     x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
 		s->arg[y] = be64toh(ptr[x]);
 
 	/* read RFC2819 counter group using predefined counter layout */
@@ -365,6 +400,20 @@ mlx5e_update_pport_counters(struct mlx5e_priv *priv)
 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 	for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
 		s_debug->arg[y] = be64toh(ptr[x]);
+
+	/* read per-priority counters */
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
+
+	/* iterate all the priorities */
+	for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
+		MLX5_SET(ppcnt_reg, in, prio_tc, z);
+		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+
+		/* read per priority stats counter group using predefined counter layout */
+		for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
+		    MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
+			s->arg[y] = be64toh(ptr[x]);
+	}
 free_out:
 	/* free firmware request structures */
 	kvfree(in);
@@ -3103,17 +3152,93 @@ mlx5e_add_hw_stats(struct mlx5e_priv *priv)
 	    "Board ID");
 }
 
+static int
+mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
+{
+	struct mlx5e_priv *priv = arg1;
+	uint32_t tx_pfc;
+	uint32_t value;
+	int error;
+
+	PRIV_LOCK(priv);
+
+	tx_pfc = priv->params.tx_priority_flow_control;
+
+	/* get current value */
+	value = (tx_pfc >> arg2) & 1;
+
+	error = sysctl_handle_32(oidp, &value, 0, req);
+
+	/* range check value */
+	if (value != 0)
+		priv->params.tx_priority_flow_control |= (1 << arg2);
+	else
+		priv->params.tx_priority_flow_control &= ~(1 << arg2);
+
+	/* check if update is required */
+	if (error == 0 && priv->gone == 0 &&
+	    tx_pfc != priv->params.tx_priority_flow_control) {
+		error = -mlx5e_set_port_pfc(priv);
+		/* restore previous value */
+		if (error != 0)
+			priv->params.tx_priority_flow_control= tx_pfc;
+	}
+	PRIV_UNLOCK(priv);
+
+	return (error);
+}
+
+static int
+mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
+{
+	struct mlx5e_priv *priv = arg1;
+	uint32_t rx_pfc;
+	uint32_t value;
+	int error;
+
+	PRIV_LOCK(priv);
+
+	rx_pfc = priv->params.rx_priority_flow_control;
+
+	/* get current value */
+	value = (rx_pfc >> arg2) & 1;
+
+	error = sysctl_handle_32(oidp, &value, 0, req);
+
+	/* range check value */
+	if (value != 0)
+		priv->params.rx_priority_flow_control |= (1 << arg2);
+	else
+		priv->params.rx_priority_flow_control &= ~(1 << arg2);
+
+	/* check if update is required */
+	if (error == 0 && priv->gone == 0 &&
+	    rx_pfc != priv->params.rx_priority_flow_control) {
+		error = -mlx5e_set_port_pfc(priv);
+		/* restore previous value */
+		if (error != 0)
+			priv->params.rx_priority_flow_control= rx_pfc;
+	}
+	PRIV_UNLOCK(priv);
+
+	return (error);
+}
+
 static void
 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
 {
-#if (__FreeBSD_version < 1100000)
-	char path[64];
+	unsigned int x;
+	char path[96];
+	int error;
 
-#endif
 	/* Only receiving pauseframes is enabled by default */
 	priv->params.tx_pauseframe_control = 0;
 	priv->params.rx_pauseframe_control = 1;
 
+	/* disable ports flow control, PFC, by default */
+	priv->params.tx_priority_flow_control = 0;
+	priv->params.rx_priority_flow_control = 0;
+
 #if (__FreeBSD_version < 1100000)
 	/* compute path for sysctl */
 	snprintf(path, sizeof(path), "dev.mce.%d.tx_pauseframe_control",
@@ -3128,9 +3253,28 @@ mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
 
 	/* try to fetch tunable, if any */
 	TUNABLE_INT_FETCH(path, &priv->params.rx_pauseframe_control);
+
+	for (x = 0; x != 8; x++) {
+
+		/* compute path for sysctl */
+		snprintf(path, sizeof(path), "dev.mce.%d.tx_priority_flow_control_%u",
+		    device_get_unit(priv->mdev->pdev->dev.bsddev), x);
+
+		/* try to fetch tunable, if any */
+		if (TUNABLE_INT_FETCH(path, &value) == 0 && value != 0)
+			priv->params.tx_priority_flow_control |= 1 << x;
+
+		/* compute path for sysctl */
+		snprintf(path, sizeof(path), "dev.mce.%d.rx_priority_flow_control_%u",
+		    device_get_unit(priv->mdev->pdev->dev.bsddev), x);
+
+		/* try to fetch tunable, if any */
+		if (TUNABLE_INT_FETCH(path, &value) == 0 && value != 0)
+			priv->params.rx_priority_flow_control |= 1 << x;
+	}
 #endif
 
-	/* register pausframe SYSCTLs */
+	/* register pauseframe SYSCTLs */
 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
 	    OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
 	    &priv->params.tx_pauseframe_control, 0,
@@ -3141,6 +3285,25 @@ mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
 	    &priv->params.rx_pauseframe_control, 0,
 	    "Set to enable RX pause frames. Clear to disable.");
 
+	/* register priority_flow control, PFC, SYSCTLs */
+	for (x = 0; x != 8; x++) {
+		snprintf(path, sizeof(path), "tx_priority_flow_control_%u", x);
+
+		SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
+		    OID_AUTO, path, CTLTYPE_UINT | CTLFLAG_RWTUN |
+		    CTLFLAG_MPSAFE, priv, x, &mlx5e_sysctl_tx_priority_flow_control, "IU",
+		    "Set to enable TX ports flow control frames for given priority. Clear to disable.");
+
+		snprintf(path, sizeof(path), "rx_priority_flow_control_%u", x);
+
+		SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
+		    OID_AUTO, path, CTLTYPE_UINT | CTLFLAG_RWTUN |
+		    CTLFLAG_MPSAFE, priv, x, &mlx5e_sysctl_rx_priority_flow_control, "IU",
+		    "Set to enable RX ports flow control frames for given priority. Clear to disable.");
+	}
+
+	PRIV_LOCK(priv);
+
 	/* range check */
 	priv->params.tx_pauseframe_control =
 	    priv->params.tx_pauseframe_control ? 1 : 0;
@@ -3148,9 +3311,17 @@ mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
 	    priv->params.rx_pauseframe_control ? 1 : 0;
 
 	/* update firmware */
-	mlx5_set_port_pause(priv->mdev, 1,
-	    priv->params.rx_pauseframe_control,
-	    priv->params.tx_pauseframe_control);
+	error = mlx5e_set_port_pause_and_pfc(priv);
+	if (error == -EINVAL) {
+		if_printf(priv->ifp,
+		    "Global pauseframes must be disabled before enabling PFC.\n");
+		priv->params.rx_priority_flow_control = 0;
+		priv->params.tx_priority_flow_control = 0;
+
+		/* update firmware */
+		(void) mlx5e_set_port_pause_and_pfc(priv);
+	}
+	PRIV_UNLOCK(priv);
 }
 
 static void *
diff --git a/sys/dev/mlx5/port.h b/sys/dev/mlx5/port.h
index 6d88d2c..394afc0 100644
--- a/sys/dev/mlx5/port.h
+++ b/sys/dev/mlx5/port.h
@@ -127,11 +127,11 @@ int mlx5_set_port_status(struct mlx5_core_dev *dev,
 int mlx5_query_port_status(struct mlx5_core_dev *dev, u8 *status);
 int mlx5_query_port_admin_status(struct mlx5_core_dev *dev,
 				 enum mlx5_port_status *status);
-int mlx5_set_port_pause(struct mlx5_core_dev *dev, u32 port,
-			u32 rx_pause, u32 tx_pause);
+int mlx5_set_port_pause_and_pfc(struct mlx5_core_dev *dev, u32 port,
+				u8 rx_pause, u8 tx_pause,
+				u8 pfc_en_rx, u8 pfc_en_tx);
 int mlx5_query_port_pause(struct mlx5_core_dev *dev, u32 port,
 			  u32 *rx_pause, u32 *tx_pause);
-int mlx5_set_port_pfc(struct mlx5_core_dev *dev, u8 pfc_en_tx, u8 pfc_en_rx);
 int mlx5_query_port_pfc(struct mlx5_core_dev *dev, u8 *pfc_en_tx, u8 *pfc_en_rx);
 
 int mlx5_set_port_mtu(struct mlx5_core_dev *dev, int mtu);