6 files changed, 269 insertions, 75 deletions

diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
index 4203f5c..0db15bd 100644
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -372,6 +372,7 @@ struct mlx5e_params {
 	u16	tx_cq_moderation_pkts;
 	u16	min_rx_wqes;
 	bool	hw_lro_en;
+	bool	cqe_zipping_en;
 	u32	lro_wqe_sz;
 	u16	rx_hash_log_tbl_sz;
 };
@@ -392,7 +393,8 @@ struct mlx5e_params {
   m(+1, u64 tx_coalesce_usecs, "tx_coalesce_usecs", "Limit in usec for joining tx packets") \
   m(+1, u64 tx_coalesce_pkts, "tx_coalesce_pkts", "Maximum number of tx packets to join") \
   m(+1, u64 tx_coalesce_mode, "tx_coalesce_mode", "0: EQE mode 1: CQE mode") \
-  m(+1, u64 hw_lro, "hw_lro", "set to enable hw_lro")
+  m(+1, u64 hw_lro, "hw_lro", "set to enable hw_lro") \
+  m(+1, u64 cqe_zipping, "cqe_zipping", "0 : CQE zipping disabled")
 
 #define	MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT))
 
@@ -698,7 +700,7 @@ enum mlx5e_link_mode {
 	MLX5E_56GBASE_R4 = 8,
 	MLX5E_10GBASE_CR = 12,
 	MLX5E_10GBASE_SR = 13,
-	MLX5E_10GBASE_ER = 14,
+	MLX5E_10GBASE_LR = 14,
 	MLX5E_40GBASE_SR4 = 15,
 	MLX5E_40GBASE_LR4 = 16,
 	MLX5E_100GBASE_CR4 = 20,
@@ -787,5 +789,6 @@ void	mlx5e_create_stats(struct sysctl_ctx_list *,
     struct sysctl_oid_list *, const char *,
     const char **, unsigned, u64 *);
 void	mlx5e_send_nop(struct mlx5e_sq *, u32, bool);
+int	mlx5e_refresh_channel_params(struct mlx5e_priv *);
 
 #endif					/* _MLX5_EN_H_ */
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
index eede82f..e389a07 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
@@ -58,19 +58,60 @@ mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS)
 
 	PRIV_LOCK(priv);
 	value = priv->params_ethtool.arg[arg2];
-	error = sysctl_handle_64(oidp, &value, 0, req);
-	if (error || req->newptr == NULL ||
-	    value == priv->params_ethtool.arg[arg2])
-		goto done;
-
-	/* assign new value */
-	priv->params_ethtool.arg[arg2] = value;
+	if (req != NULL) {
+		error = sysctl_handle_64(oidp, &value, 0, req);
+		if (error || req->newptr == NULL ||
+		    value == priv->params_ethtool.arg[arg2])
+			goto done;
 
+		/* assign new value */
+		priv->params_ethtool.arg[arg2] = value;
+	} else {
+		error = 0;
+	}
 	/* check if device is gone */
 	if (priv->gone) {
 		error = ENXIO;
 		goto done;
 	}
+	/* import RX coal time */
+	if (priv->params_ethtool.rx_coalesce_usecs < 1)
+		priv->params_ethtool.rx_coalesce_usecs = 0;
+	else if (priv->params_ethtool.rx_coalesce_usecs >
+	    MLX5E_FLD_MAX(cqc, cq_period)) {
+		priv->params_ethtool.rx_coalesce_usecs =
+		    MLX5E_FLD_MAX(cqc, cq_period);
+	}
+	priv->params.rx_cq_moderation_usec = priv->params_ethtool.rx_coalesce_usecs;
+
+	/* import RX coal pkts */
+	if (priv->params_ethtool.rx_coalesce_pkts < 1)
+		priv->params_ethtool.rx_coalesce_pkts = 0;
+	else if (priv->params_ethtool.rx_coalesce_pkts >
+	    MLX5E_FLD_MAX(cqc, cq_max_count)) {
+		priv->params_ethtool.rx_coalesce_pkts =
+		    MLX5E_FLD_MAX(cqc, cq_max_count);
+	}
+	priv->params.rx_cq_moderation_pkts = priv->params_ethtool.rx_coalesce_pkts;
+
+	/* import TX coal time */
+	if (priv->params_ethtool.tx_coalesce_usecs < 1)
+		priv->params_ethtool.tx_coalesce_usecs = 0;
+	else if (priv->params_ethtool.tx_coalesce_usecs >
+	    MLX5E_FLD_MAX(cqc, cq_period)) {
+		priv->params_ethtool.tx_coalesce_usecs =
+		    MLX5E_FLD_MAX(cqc, cq_period);
+	}
+	priv->params.tx_cq_moderation_usec = priv->params_ethtool.tx_coalesce_usecs;
+
+	/* import TX coal pkts */
+	if (priv->params_ethtool.tx_coalesce_pkts < 1)
+		priv->params_ethtool.tx_coalesce_pkts = 0;
+	else if (priv->params_ethtool.tx_coalesce_pkts >
+	    MLX5E_FLD_MAX(cqc, cq_max_count)) {
+		priv->params_ethtool.tx_coalesce_pkts = MLX5E_FLD_MAX(cqc, cq_max_count);
+	}
+	priv->params.tx_cq_moderation_pkts = priv->params_ethtool.tx_coalesce_pkts;
 
 	if (&priv->params_ethtool.arg[arg2] == &priv->params_ethtool.rx_pauseframe_control ||
 	    &priv->params_ethtool.arg[arg2] == &priv->params_ethtool.tx_pauseframe_control) {
@@ -88,9 +129,19 @@ mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS)
 	}
 
 	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
-	if (was_opened)
+	if (was_opened) {
+		u64 *xarg = priv->params_ethtool.arg + arg2;
+
+		if (xarg == &priv->params_ethtool.tx_coalesce_pkts ||
+		    xarg == &priv->params_ethtool.rx_coalesce_pkts ||
+		    xarg == &priv->params_ethtool.tx_coalesce_usecs ||
+		    xarg == &priv->params_ethtool.rx_coalesce_usecs) {
+			/* avoid downing and upping the network interface */
+			error = mlx5e_refresh_channel_params(priv);
+			goto done;
+		}
 		mlx5e_close_locked(priv->ifp);
-
+	}
 	/* import TX queue size */
 	if (priv->params_ethtool.tx_queue_size <
 	    (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)) {
@@ -141,45 +192,6 @@ mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS)
 		priv->params_ethtool.tx_coalesce_mode = 1;
 	priv->params.tx_cq_moderation_mode = priv->params_ethtool.tx_coalesce_mode;
 
-	/* import RX coal time */
-	if (priv->params_ethtool.rx_coalesce_usecs < 1)
-		priv->params_ethtool.rx_coalesce_usecs = 0;
-	else if (priv->params_ethtool.rx_coalesce_usecs >
-	    MLX5E_FLD_MAX(cqc, cq_period)) {
-		priv->params_ethtool.rx_coalesce_usecs =
-		    MLX5E_FLD_MAX(cqc, cq_period);
-	}
-	priv->params.rx_cq_moderation_usec = priv->params_ethtool.rx_coalesce_usecs;
-
-	/* import RX coal pkts */
-	if (priv->params_ethtool.rx_coalesce_pkts < 1)
-		priv->params_ethtool.rx_coalesce_pkts = 0;
-	else if (priv->params_ethtool.rx_coalesce_pkts >
-	    MLX5E_FLD_MAX(cqc, cq_max_count)) {
-		priv->params_ethtool.rx_coalesce_pkts =
-		    MLX5E_FLD_MAX(cqc, cq_max_count);
-	}
-	priv->params.rx_cq_moderation_pkts = priv->params_ethtool.rx_coalesce_pkts;
-
-	/* import TX coal time */
-	if (priv->params_ethtool.tx_coalesce_usecs < 1)
-		priv->params_ethtool.tx_coalesce_usecs = 0;
-	else if (priv->params_ethtool.tx_coalesce_usecs >
-	    MLX5E_FLD_MAX(cqc, cq_period)) {
-		priv->params_ethtool.tx_coalesce_usecs =
-		    MLX5E_FLD_MAX(cqc, cq_period);
-	}
-	priv->params.tx_cq_moderation_usec = priv->params_ethtool.tx_coalesce_usecs;
-
-	/* import TX coal pkts */
-	if (priv->params_ethtool.tx_coalesce_pkts < 1)
-		priv->params_ethtool.tx_coalesce_pkts = 0;
-	else if (priv->params_ethtool.tx_coalesce_pkts >
-	    MLX5E_FLD_MAX(cqc, cq_max_count)) {
-		priv->params_ethtool.tx_coalesce_pkts = MLX5E_FLD_MAX(cqc, cq_max_count);
-	}
-	priv->params.tx_cq_moderation_pkts = priv->params_ethtool.tx_coalesce_pkts;
-
 	/* we always agree to turn off HW LRO - but not always to turn on */
 	if (priv->params_ethtool.hw_lro) {
 		if (priv->params_ethtool.hw_lro != 1) {
@@ -200,6 +212,18 @@ mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS)
 		priv->params.hw_lro_en = false;
 	}
 
+	if (&priv->params_ethtool.arg[arg2] ==
+	    &priv->params_ethtool.cqe_zipping) {
+		if (priv->params_ethtool.cqe_zipping &&
+		    MLX5_CAP_GEN(priv->mdev, cqe_compression)) {
+			priv->params.cqe_zipping_en = true;
+			priv->params_ethtool.cqe_zipping = 1;
+		} else {
+			priv->params.cqe_zipping_en = false;
+			priv->params_ethtool.cqe_zipping = 0;
+		}
+	}
+
 	if (was_opened)
 		mlx5e_open_locked(priv->ifp);
 done:
@@ -319,34 +343,33 @@ mlx5e_get_eeprom(struct mlx5e_priv *priv, struct mlx5e_eeprom *ee)
 static void
 mlx5e_print_eeprom(struct mlx5e_eeprom *eeprom)
 {
-	int i, j = 0;
-	int row = 0;
+	int row;
+	int index_in_row;
+	int byte_to_write = 0;
+	int line_length = 16;
 
 	printf("\nOffset\t\tValues\n");
-	printf("------\t\t------\n");
-	while (row < eeprom->len) {
-		printf("0x%04x\t\t", row);
-		for (i = 0; i < 16; i++) {
-			printf("%02x ", ((u8 *)eeprom->data)[j]);
-			j++;
-			row++;
+	printf("------\t\t------");
+	while (byte_to_write < eeprom->len) {
+		printf("\n0x%04X\t\t", byte_to_write);
+		for (index_in_row = 0; index_in_row < line_length; index_in_row++) {
+			printf("%02X ", ((u8 *)eeprom->data)[byte_to_write]);
+			byte_to_write++;
 		}
-		printf("\n");
 	}
 
 	if (eeprom->page_valid) {
 		row = MLX5E_EEPROM_HIGH_PAGE_OFFSET;
-		printf("\nUpper Page 0x03\n");
+		printf("\n\nUpper Page 0x03\n");
 		printf("\nOffset\t\tValues\n");
-		printf("------\t\t------\n");
+		printf("------\t\t------");
 		while (row < MLX5E_EEPROM_PAGE_LENGTH) {
-			printf("0x%04x\t\t", row);
-			for (i = 0; i < 16; i++) {
-				printf("%02x ", ((u8 *)eeprom->data)[j]);
-				j++;
+			printf("\n0x%04X\t\t", row);
+			for (index_in_row = 0; index_in_row < line_length; index_in_row++) {
+				printf("%02X ", ((u8 *)eeprom->data)[byte_to_write]);
+				byte_to_write++;
 				row++;
 			}
-			printf("\n");
 		}
 	}
 }
@@ -469,6 +492,7 @@ mlx5e_create_ethtool(struct mlx5e_priv *priv)
 	priv->params_ethtool.tx_coalesce_usecs = priv->params.tx_cq_moderation_usec;
 	priv->params_ethtool.tx_coalesce_pkts = priv->params.tx_cq_moderation_pkts;
 	priv->params_ethtool.hw_lro = priv->params.hw_lro_en;
+	priv->params_ethtool.cqe_zipping = priv->params.cqe_zipping_en;
 
 	/* create root node */
 	node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
@@ -484,10 +508,30 @@ mlx5e_create_ethtool(struct mlx5e_priv *priv)
 			    CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU",
 			    mlx5e_params_desc[2 * x + 1]);
 		} else {
+#if (__FreeBSD_version < 1100000)
+			char path[64];
+#endif
+			/*
+			 * NOTE: In FreeBSD-11 and newer the
+			 * CTLFLAG_RWTUN flag will take care of
+			 * loading default sysctl value from the
+			 * kernel environment, if any:
+			 */
 			SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO,
 			    mlx5e_params_desc[2 * x], CTLTYPE_U64 | CTLFLAG_RWTUN |
 			    CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU",
 			    mlx5e_params_desc[2 * x + 1]);
+
+#if (__FreeBSD_version < 1100000)
+			/* compute path for sysctl */
+			snprintf(path, sizeof(path), "dev.mce.%d.conf.%s",
+			    device_get_unit(priv->mdev->pdev->dev.bsddev),
+			    mlx5e_params_desc[2 * x]);
+
+			/* try to fetch tunable, if any */
+			if (TUNABLE_QUAD_FETCH(path, &priv->params_ethtool.arg[x]))
+				mlx5e_ethtool_handler(NULL, priv, x, NULL);
+#endif
 		}
 	}
 
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
index 77aaf7b..82ea69d 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -106,8 +106,8 @@ static const struct {
 		.subtype = IFM_10G_SR,
 		.baudrate = IF_Gbps(10ULL),
 	},
-	[MLX5E_10GBASE_ER] = {
-		.subtype = IFM_10G_ER,
+	[MLX5E_10GBASE_LR] = {
+		.subtype = IFM_10G_LR,
 		.baudrate = IF_Gbps(10ULL),
 	},
 	[MLX5E_40GBASE_SR4] = {
@@ -1604,6 +1604,16 @@ mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
 {
 	void *cqc = param->cqc;
 
+
+	/*
+	 * TODO The sysctl to control on/off is a bool value for now, which means
+	 * we only support CSUM, once HASH is implemnted we'll need to address that.
+	 */
+	if (priv->params.cqe_zipping_en) {
+		MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
+		MLX5_SET(cqc, cqc, cqe_compression_en, 1);
+	}
+
 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
 	MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
 	MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
@@ -1702,6 +1712,62 @@ mlx5e_close_channels(struct mlx5e_priv *priv)
 }
 
 static int
+mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
+{
+	return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
+	    priv->params.tx_cq_moderation_usec,
+	    priv->params.tx_cq_moderation_pkts));
+}
+
+static int
+mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
+{
+	return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
+	    priv->params.rx_cq_moderation_usec,
+	    priv->params.rx_cq_moderation_pkts));
+}
+
+static int
+mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
+{
+	int err;
+	int i;
+
+	if (c == NULL)
+		return (EINVAL);
+
+	err = mlx5e_refresh_rq_params(priv, &c->rq);
+	if (err)
+		goto done;
+
+	for (i = 0; i != c->num_tc; i++) {
+		err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
+		if (err)
+			goto done;
+	}
+done:
+	return (err);
+}
+
+int
+mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
+{
+	int i;
+
+	if (priv->channel == NULL)
+		return (EINVAL);
+
+	for (i = 0; i < priv->params.num_channels; i++) {
+		int err;
+
+		err = mlx5e_refresh_channel_params_sub(priv, priv->channel[i]);
+		if (err)
+			return (err);
+	}
+	return (0);
+}
+
+static int
 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
@@ -2288,6 +2354,7 @@ mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 	int size_read = 0;
 	int module_num;
 	int max_mtu;
+	uint8_t read_addr;
 
 	priv = ifp->if_softc;
 
@@ -2474,11 +2541,21 @@ out:
 		}
 
 		/*
-		 * Note that we ignore i2c.addr here. The driver hardcodes
-		 * the address to 0x50, while standard expects it to be 0xA0.
+		 * Currently 0XA0 and 0xA2 are the only addresses permitted.
+		 * The internal conversion is as follows:
 		 */
+		if (i2c.dev_addr == 0xA0)
+			read_addr = MLX5E_I2C_ADDR_LOW;
+		else if (i2c.dev_addr == 0xA2)
+			read_addr = MLX5E_I2C_ADDR_HIGH;
+		else {
+			if_printf(ifp, "Query eeprom failed, "
+			    "Invalid Address: %X\n", i2c.dev_addr);
+			error = EINVAL;
+			goto err_i2c;
+		}
 		error = mlx5_query_eeprom(priv->mdev,
-		    MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE,
+		    read_addr, MLX5E_EEPROM_LOW_PAGE,
 		    (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
 		    (uint32_t *)i2c.data, &size_read);
 		if (error) {
@@ -2489,7 +2566,7 @@ out:
 
 		if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
 			error = mlx5_query_eeprom(priv->mdev,
-			    MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE,
+			    read_addr, MLX5E_EEPROM_LOW_PAGE,
 			    (uint32_t)(i2c.offset + size_read),
 			    (uint32_t)(i2c.len - size_read), module_num,
 			    (uint32_t *)(i2c.data + size_read), &size_read);
@@ -2571,6 +2648,8 @@ mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
 	priv->params.hw_lro_en = false;
 	priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
 
+	priv->params.cqe_zipping_en = !!MLX5_CAP_GEN(mdev, cqe_compression);
+
 	priv->mdev = mdev;
 	priv->params.num_channels = num_comp_vectors;
 	priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
index 87fc524..059b5ef 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -248,6 +248,69 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
 	}
 }
 
+static inline void
+mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
+{
+	memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)),
+	    sizeof(struct mlx5_cqe64));
+}
+
+static inline void
+mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data)
+{
+	memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1),
+	    data, sizeof(struct mlx5_cqe64));
+}
+
+static inline void
+mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title,
+    struct mlx5_mini_cqe8 *mini,
+    u16 wqe_counter, int i)
+{
+	title->byte_cnt = mini->byte_cnt;
+	title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1);
+	title->check_sum = mini->checksum;
+	title->op_own = (title->op_own & 0xf0) |
+	    (((cq->wq.cc + i) >> cq->wq.log_sz) & 1);
+}
+
+#define MLX5E_MINI_ARRAY_SZ 8
+/* Make sure structs are not packet differently */
+CTASSERT(sizeof(struct mlx5_cqe64) ==
+    sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ);
+static void
+mlx5e_decompress_cqes(struct mlx5e_cq *cq)
+{
+	struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ];
+	struct mlx5_cqe64 title;
+	u32 cqe_count;
+	u32 i = 0;
+	u16 title_wqe_counter;
+
+	mlx5e_read_cqe_slot(cq, cq->wq.cc, &title);
+	title_wqe_counter = be16_to_cpu(title.wqe_counter);
+	cqe_count = be32_to_cpu(title.byte_cnt);
+
+	/* Make sure we won't overflow */
+	KASSERT(cqe_count <= cq->wq.sz_m1,
+	    ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__,
+	    cqe_count, cq->wq.sz_m1));
+
+	mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array);
+	while (true) {
+		mlx5e_decompress_cqe(cq, &title,
+		    &mini_array[i % MLX5E_MINI_ARRAY_SZ],
+		    title_wqe_counter, i);
+		mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title);
+		i++;
+
+		if (i == cqe_count)
+			break;
+		if (i % MLX5E_MINI_ARRAY_SZ == 0)
+			mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array);
+	}
+}
+
 static int
 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
 {
@@ -268,6 +331,11 @@ mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
 		if (!cqe)
 			break;
 
+		if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED)
+			mlx5e_decompress_cqes(&rq->cq);
+
+		mlx5_cqwq_pop(&rq->cq.wq);
+
 		wqe_counter_be = cqe->wqe_counter;
 		wqe_counter = be16_to_cpu(wqe_counter_be);
 		wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
index cb08727..483a7e1 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -383,6 +383,8 @@ mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
 		if (!cqe)
 			break;
 
+		mlx5_cqwq_pop(&sq->cq.wq);
+
 		ci = sqcc & sq->wq.sz_m1;
 		mb = sq->mbuf[ci].mbuf;
 		sq->mbuf[ci].mbuf = NULL;	/* Safety clear */
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c
index 7836bfe..f46639d 100644
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c
@@ -37,8 +37,6 @@ mlx5e_get_cqe(struct mlx5e_cq *cq)
 	if ((cqe->op_own ^ mlx5_cqwq_get_wrap_cnt(&cq->wq)) & MLX5_CQE_OWNER_MASK)
 		return (NULL);
 
-	mlx5_cqwq_pop(&cq->wq);
-
 	/* ensure cqe content is read after cqe ownership bit */
 	rmb();