diff options
| -rw-r--r-- | sys/modules/mlxen/Makefile | 2 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/en_ethtool.c | 1616 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/en_main.c | 11 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/en_netdev.c | 276 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/en_port.c | 1 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/en_rx.c | 420 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/en_selftest.c | 178 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/en_tx.c | 852 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/main.c | 88 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/mlx4.h | 10 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/mlx4_en.h | 83 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/mlx4_stats.h | 1 | ||||
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/port.c | 7 | ||||
| -rw-r--r-- | sys/ofed/include/linux/mlx4/cq.h | 2 | ||||
| -rw-r--r-- | sys/ofed/include/linux/mlx4/device.h | 7 | ||||
| -rw-r--r-- | sys/ofed/include/linux/mlx4/doorbell.h | 2 | ||||
| -rw-r--r-- | sys/ofed/include/linux/mlx4/qp.h | 8 |
17 files changed, 1019 insertions, 2545 deletions
diff --git a/sys/modules/mlxen/Makefile b/sys/modules/mlxen/Makefile index 9c8c0b3..b75e0d0 100644 --- a/sys/modules/mlxen/Makefile +++ b/sys/modules/mlxen/Makefile @@ -26,4 +26,4 @@ opt_inet6.h: .include <bsd.kmod.mk> -CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS} +CFLAGS+= ${GCC_MS_EXTENSIONS} diff --git a/sys/ofed/drivers/net/mlx4/en_ethtool.c b/sys/ofed/drivers/net/mlx4/en_ethtool.c deleted file mode 100644 index 3ebeaf4..0000000 --- a/sys/ofed/drivers/net/mlx4/en_ethtool.c +++ /dev/null @@ -1,1616 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include <linux/kernel.h> -#include <linux/netdevice.h> -#include <linux/mlx4/driver.h> -#include <linux/in.h> -#include <net/ip.h> -#include <linux/bitmap.h> - -#include "mlx4_en.h" -#include "en_port.h" - -#define EN_ETHTOOL_QP_ATTACH (1ull << 63) - -union mlx4_ethtool_flow_union { - struct ethtool_tcpip4_spec tcp_ip4_spec; - struct ethtool_tcpip4_spec udp_ip4_spec; - struct ethtool_tcpip4_spec sctp_ip4_spec; - struct ethtool_ah_espip4_spec ah_ip4_spec; - struct ethtool_ah_espip4_spec esp_ip4_spec; - struct ethtool_usrip4_spec usr_ip4_spec; - struct ethhdr ether_spec; - __u8 hdata[52]; -}; - -struct mlx4_ethtool_flow_ext { - __u8 padding[2]; - unsigned char h_dest[ETH_ALEN]; - __be16 vlan_etype; - __be16 vlan_tci; - __be32 data[2]; -}; - -struct mlx4_ethtool_rx_flow_spec { - __u32 flow_type; - union mlx4_ethtool_flow_union h_u; - struct mlx4_ethtool_flow_ext h_ext; - union mlx4_ethtool_flow_union m_u; - struct mlx4_ethtool_flow_ext m_ext; - __u64 ring_cookie; - __u32 location; -}; - -struct mlx4_ethtool_rxnfc { - __u32 cmd; - __u32 flow_type; - __u64 data; - struct mlx4_ethtool_rx_flow_spec fs; - __u32 rule_cnt; - __u32 rule_locs[0]; -}; - -#ifndef FLOW_MAC_EXT -#define FLOW_MAC_EXT 0x40000000 -#endif - -static void -mlx4_en_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, DRV_VERSION " (" DRV_RELDATE ")", - sizeof(drvinfo->version)); - snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), - "%d.%d.%d", - (u16) (mdev->dev->caps.fw_ver >> 32), - (u16) ((mdev->dev->caps.fw_ver >> 16) & 0xffff), - (u16) (mdev->dev->caps.fw_ver & 0xffff)); - strlcpy(drvinfo->bus_info, pci_name(mdev->dev->pdev), - sizeof(drvinfo->bus_info)); - drvinfo->n_stats = 0; - drvinfo->regdump_len = 0; - drvinfo->eedump_len = 0; -} - -static const char main_strings[][ETH_GSTRING_LEN] = { - /* packet statistics */ - "rx_packets", - "rx_bytes", - "rx_multicast_packets", - "rx_broadcast_packets", - "rx_errors", - "rx_dropped", - "rx_length_errors", - "rx_over_errors", - "rx_crc_errors", - "rx_jabbers", - "rx_in_range_length_error", - "rx_out_range_length_error", - "rx_lt_64_bytes_packets", - "rx_127_bytes_packets", - "rx_255_bytes_packets", - "rx_511_bytes_packets", - "rx_1023_bytes_packets", - "rx_1518_bytes_packets", - "rx_1522_bytes_packets", - "rx_1548_bytes_packets", - "rx_gt_1548_bytes_packets", - "tx_packets", - "tx_bytes", - "tx_multicast_packets", - "tx_broadcast_packets", - "tx_errors", - "tx_dropped", - "tx_lt_64_bytes_packets", - "tx_127_bytes_packets", - "tx_255_bytes_packets", - "tx_511_bytes_packets", - "tx_1023_bytes_packets", - "tx_1518_bytes_packets", - "tx_1522_bytes_packets", - "tx_1548_bytes_packets", - "tx_gt_1548_bytes_packets", - "rx_prio_0_packets", "rx_prio_0_bytes", - "rx_prio_1_packets", "rx_prio_1_bytes", - "rx_prio_2_packets", "rx_prio_2_bytes", - "rx_prio_3_packets", "rx_prio_3_bytes", - "rx_prio_4_packets", "rx_prio_4_bytes", - "rx_prio_5_packets", "rx_prio_5_bytes", - "rx_prio_6_packets", "rx_prio_6_bytes", - "rx_prio_7_packets", "rx_prio_7_bytes", - "rx_novlan_packets", "rx_novlan_bytes", - "tx_prio_0_packets", "tx_prio_0_bytes", - "tx_prio_1_packets", "tx_prio_1_bytes", - "tx_prio_2_packets", "tx_prio_2_bytes", - "tx_prio_3_packets", "tx_prio_3_bytes", - "tx_prio_4_packets", "tx_prio_4_bytes", - "tx_prio_5_packets", "tx_prio_5_bytes", - "tx_prio_6_packets", "tx_prio_6_bytes", - "tx_prio_7_packets", "tx_prio_7_bytes", - "tx_novlan_packets", "tx_novlan_bytes", - - /* flow control statistics */ - "rx_pause_prio_0", "rx_pause_duration_prio_0", - "rx_pause_transition_prio_0", "tx_pause_prio_0", - "tx_pause_duration_prio_0", "tx_pause_transition_prio_0", - "rx_pause_prio_1", "rx_pause_duration_prio_1", - "rx_pause_transition_prio_1", "tx_pause_prio_1", - "tx_pause_duration_prio_1", "tx_pause_transition_prio_1", - "rx_pause_prio_2", "rx_pause_duration_prio_2", - "rx_pause_transition_prio_2", "tx_pause_prio_2", - "tx_pause_duration_prio_2", "tx_pause_transition_prio_2", - "rx_pause_prio_3", "rx_pause_duration_prio_3", - "rx_pause_transition_prio_3", "tx_pause_prio_3", - "tx_pause_duration_prio_3", "tx_pause_transition_prio_3", - "rx_pause_prio_4", "rx_pause_duration_prio_4", - "rx_pause_transition_prio_4", "tx_pause_prio_4", - "tx_pause_duration_prio_4", "tx_pause_transition_prio_4", - "rx_pause_prio_5", "rx_pause_duration_prio_5", - "rx_pause_transition_prio_5", "tx_pause_prio_5", - "tx_pause_duration_prio_5", "tx_pause_transition_prio_5", - "rx_pause_prio_6", "rx_pause_duration_prio_6", - "rx_pause_transition_prio_6", "tx_pause_prio_6", - "tx_pause_duration_prio_6", "tx_pause_transition_prio_6", - "rx_pause_prio_7", "rx_pause_duration_prio_7", - "rx_pause_transition_prio_7", "tx_pause_prio_7", - "tx_pause_duration_prio_7", "tx_pause_transition_prio_7", - - /* VF statistics */ - "rx_packets", - "rx_bytes", - "rx_multicast_packets", - "rx_broadcast_packets", - "rx_errors", - "rx_dropped", - "tx_packets", - "tx_bytes", - "tx_multicast_packets", - "tx_broadcast_packets", - "tx_errors", - - /* VPort statistics */ - "vport_rx_unicast_packets", - "vport_rx_unicast_bytes", - "vport_rx_multicast_packets", - "vport_rx_multicast_bytes", - "vport_rx_broadcast_packets", - "vport_rx_broadcast_bytes", - "vport_rx_dropped", - "vport_rx_errors", - "vport_tx_unicast_packets", - "vport_tx_unicast_bytes", - "vport_tx_multicast_packets", - "vport_tx_multicast_bytes", - "vport_tx_broadcast_packets", - "vport_tx_broadcast_bytes", - "vport_tx_errors", - - /* port statistics */ - "tx_tso_packets", - "tx_queue_stopped", "tx_wake_queue", "tx_timeout", "rx_alloc_failed", - "rx_csum_good", "rx_csum_none", "tx_chksum_offload", -}; - -static const char mlx4_en_test_names[][ETH_GSTRING_LEN]= { - "Interrupt Test", - "Link Test", - "Speed Test", - "Register Test", - "Loopback Test", -}; - -static u32 mlx4_en_get_msglevel(struct net_device *dev) -{ - return ((struct mlx4_en_priv *) netdev_priv(dev))->msg_enable; -} - -static void mlx4_en_set_msglevel(struct net_device *dev, u32 val) -{ - ((struct mlx4_en_priv *) netdev_priv(dev))->msg_enable = val; -} - -static void mlx4_en_get_wol(struct net_device *netdev, - struct ethtool_wolinfo *wol) -{ - struct mlx4_en_priv *priv = netdev_priv(netdev); - int err = 0; - u64 config = 0; - u64 mask; - - if ((priv->port < 1) || (priv->port > 2)) { - en_err(priv, "Failed to get WoL information\n"); - return; - } - - mask = (priv->port == 1) ? MLX4_DEV_CAP_FLAG_WOL_PORT1 : - MLX4_DEV_CAP_FLAG_WOL_PORT2; - - if (!(priv->mdev->dev->caps.flags & mask)) { - wol->supported = 0; - wol->wolopts = 0; - return; - } - - err = mlx4_wol_read(priv->mdev->dev, &config, priv->port); - if (err) { - en_err(priv, "Failed to get WoL information\n"); - return; - } - - if (config & MLX4_EN_WOL_MAGIC) - wol->supported = WAKE_MAGIC; - else - wol->supported = 0; - - if (config & MLX4_EN_WOL_ENABLED) - wol->wolopts = WAKE_MAGIC; - else - wol->wolopts = 0; -} - -static int mlx4_en_set_wol(struct net_device *netdev, - struct ethtool_wolinfo *wol) -{ - struct mlx4_en_priv *priv = netdev_priv(netdev); - u64 config = 0; - int err = 0; - u64 mask; - - if ((priv->port < 1) || (priv->port > 2)) - return -EOPNOTSUPP; - - mask = (priv->port == 1) ? MLX4_DEV_CAP_FLAG_WOL_PORT1 : - MLX4_DEV_CAP_FLAG_WOL_PORT2; - - if (!(priv->mdev->dev->caps.flags & mask)) - return -EOPNOTSUPP; - - if (wol->supported & ~WAKE_MAGIC) - return -EINVAL; - - err = mlx4_wol_read(priv->mdev->dev, &config, priv->port); - if (err) { - en_err(priv, "Failed to get WoL info, unable to modify\n"); - return err; - } - - if (wol->wolopts & WAKE_MAGIC) { - config |= MLX4_EN_WOL_DO_MODIFY | MLX4_EN_WOL_ENABLED | - MLX4_EN_WOL_MAGIC; - } else { - config &= ~(MLX4_EN_WOL_ENABLED | MLX4_EN_WOL_MAGIC); - config |= MLX4_EN_WOL_DO_MODIFY; - } - - err = mlx4_wol_write(priv->mdev->dev, config, priv->port); - if (err) - en_err(priv, "Failed to set WoL information\n"); - - return err; -} - -struct bitmap_sim_iterator { - bool advance_array; - unsigned long *stats_bitmap; - unsigned int count; - unsigned int j; -}; - -static inline void bitmap_sim_iterator_init(struct bitmap_sim_iterator *h, - unsigned long *stats_bitmap, - int count) -{ - h->j = 0; - h->advance_array = !bitmap_empty(stats_bitmap, count); - h->count = h->advance_array ? bitmap_weight(stats_bitmap, count) - : count; - h->stats_bitmap = stats_bitmap; -} - -static inline int bitmap_sim_iterator_test(struct bitmap_sim_iterator *h) -{ - return !h->advance_array ? 1 : test_bit(h->j, h->stats_bitmap); -} - -static inline int bitmap_sim_iterator_inc(struct bitmap_sim_iterator *h) -{ - return h->j++; -} - -static inline unsigned int bitmap_sim_iterator_count( - struct bitmap_sim_iterator *h) -{ - return h->count; -} - -int mlx4_en_get_sset_count(struct net_device *dev, int sset) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - switch (sset) { - case ETH_SS_STATS: - return bitmap_sim_iterator_count(&it) + - (priv->tx_ring_num * 2) + -#ifdef LL_EXTENDED_STATS - (priv->rx_ring_num * 5); -#else - (priv->rx_ring_num * 2); -#endif - case ETH_SS_TEST: - return MLX4_EN_NUM_SELF_TEST - !(priv->mdev->dev->caps.flags - & MLX4_DEV_CAP_FLAG_UC_LOOPBACK) * 2; - default: - return -EOPNOTSUPP; - } -} - -void mlx4_en_get_ethtool_stats(struct net_device *dev, - struct ethtool_stats *stats, u64 *data) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int index = 0; - int i; - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - if (!data || !priv->port_up) - return; - - spin_lock_bh(&priv->stats_lock); - - for (i = 0; i < NUM_PKT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->pkstats)[i]; - for (i = 0; i < NUM_FLOW_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((u64 *)&priv->flowstats)[i]; - for (i = 0; i < NUM_VF_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->vf_stats)[i]; - for (i = 0; i < NUM_VPORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->vport_stats)[i]; - for (i = 0; i < NUM_PORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - data[index++] = - ((unsigned long *)&priv->port_stats)[i]; - - for (i = 0; i < priv->tx_ring_num; i++) { - data[index++] = priv->tx_ring[i]->packets; - data[index++] = priv->tx_ring[i]->bytes; - } - for (i = 0; i < priv->rx_ring_num; i++) { - data[index++] = priv->rx_ring[i]->packets; - data[index++] = priv->rx_ring[i]->bytes; -#ifdef LL_EXTENDED_STATS - data[index++] = priv->rx_ring[i]->yields; - data[index++] = priv->rx_ring[i]->misses; - data[index++] = priv->rx_ring[i]->cleaned; -#endif - } - spin_unlock_bh(&priv->stats_lock); - -} - -void mlx4_en_restore_ethtool_stats(struct mlx4_en_priv *priv, u64 *data) -{ - int index = 0; - int i; - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - if (!data || !priv->port_up) - return; - - spin_lock_bh(&priv->stats_lock); - - for (i = 0; i < NUM_PKT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->pkstats)[i] = - data[index++]; - for (i = 0; i < NUM_FLOW_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) - if (bitmap_sim_iterator_test(&it)) - ((u64 *)&priv->flowstats)[i] = - data[index++]; - for (i = 0; i < NUM_VF_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->vf_stats)[i] = - data[index++]; - for (i = 0; i < NUM_VPORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->vport_stats)[i] = - data[index++]; - for (i = 0; i < NUM_PORT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - ((unsigned long *)&priv->port_stats)[i] = - data[index++]; - - for (i = 0; i < priv->tx_ring_num; i++) { - priv->tx_ring[i]->packets = data[index++]; - priv->tx_ring[i]->bytes = data[index++]; - } - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_ring[i]->packets = data[index++]; - priv->rx_ring[i]->bytes = data[index++]; - } - spin_unlock_bh(&priv->stats_lock); -} - -static void mlx4_en_self_test(struct net_device *dev, - struct ethtool_test *etest, u64 *buf) -{ - mlx4_en_ex_selftest(dev, &etest->flags, buf); -} - -static void mlx4_en_get_strings(struct net_device *dev, - uint32_t stringset, uint8_t *data) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int index = 0; - int i, k; - struct bitmap_sim_iterator it; - - int num_of_stats = NUM_ALL_STATS - - ((priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) ? 0 : NUM_FLOW_STATS); - - bitmap_sim_iterator_init(&it, priv->stats_bitmap, num_of_stats); - - switch (stringset) { - case ETH_SS_TEST: - for (i = 0; i < MLX4_EN_NUM_SELF_TEST - 2; i++) - strcpy(data + i * ETH_GSTRING_LEN, mlx4_en_test_names[i]); - if (priv->mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UC_LOOPBACK) - for (; i < MLX4_EN_NUM_SELF_TEST; i++) - strcpy(data + i * ETH_GSTRING_LEN, mlx4_en_test_names[i]); - break; - - case ETH_SS_STATS: - /* Add main counters */ - for (i = 0; i < NUM_PKT_STATS; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - strcpy(data + (index++) * ETH_GSTRING_LEN, - main_strings[i]); - - for (k = 0; k < NUM_FLOW_STATS; k++, - bitmap_sim_iterator_inc(&it)) - if (priv->mdev->dev->caps.flags2 & - MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN) - if (bitmap_sim_iterator_test(&it)) - strcpy(data + (index++) * - ETH_GSTRING_LEN, - main_strings[i + k]); - - for (; (i + k) < num_of_stats; i++, - bitmap_sim_iterator_inc(&it)) - if (bitmap_sim_iterator_test(&it)) - strcpy(data + (index++) * ETH_GSTRING_LEN, - main_strings[i + k]); - - for (i = 0; i < priv->tx_ring_num; i++) { - sprintf(data + (index++) * ETH_GSTRING_LEN, - "tx%d_packets", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "tx%d_bytes", i); - } - for (i = 0; i < priv->rx_ring_num; i++) { - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_packets", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_bytes", i); -#ifdef LL_EXTENDED_STATS - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_napi_yield", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_misses", i); - sprintf(data + (index++) * ETH_GSTRING_LEN, - "rx%d_cleaned", i); -#endif - } - break; - } -} - -static u32 mlx4_en_autoneg_get(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - u32 autoneg = AUTONEG_DISABLE; - - if ((mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETH_BACKPL_AN_REP) && - priv->port_state.autoneg) { - autoneg = AUTONEG_ENABLE; - } - - return autoneg; -} - -static int mlx4_en_get_settings(struct net_device *dev, struct ethtool_cmd *cmd) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int trans_type; - - /* SUPPORTED_1000baseT_Half isn't supported */ - cmd->supported = SUPPORTED_1000baseT_Full - |SUPPORTED_10000baseT_Full; - - cmd->advertising = ADVERTISED_1000baseT_Full - |ADVERTISED_10000baseT_Full; - - cmd->supported |= SUPPORTED_1000baseKX_Full - |SUPPORTED_10000baseKX4_Full - |SUPPORTED_10000baseKR_Full - |SUPPORTED_10000baseR_FEC - |SUPPORTED_40000baseKR4_Full - |SUPPORTED_40000baseCR4_Full - |SUPPORTED_40000baseSR4_Full - |SUPPORTED_40000baseLR4_Full; - - /* ADVERTISED_1000baseT_Half isn't advertised */ - cmd->advertising |= ADVERTISED_1000baseKX_Full - |ADVERTISED_10000baseKX4_Full - |ADVERTISED_10000baseKR_Full - |ADVERTISED_10000baseR_FEC - |ADVERTISED_40000baseKR4_Full - |ADVERTISED_40000baseCR4_Full - |ADVERTISED_40000baseSR4_Full - |ADVERTISED_40000baseLR4_Full; - - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return -ENOMEM; - - cmd->autoneg = mlx4_en_autoneg_get(dev); - if (cmd->autoneg == AUTONEG_ENABLE) { - cmd->supported |= SUPPORTED_Autoneg; - cmd->advertising |= ADVERTISED_Autoneg; - } - - trans_type = priv->port_state.transciver; - if (netif_carrier_ok(dev)) { - ethtool_cmd_speed_set(cmd, priv->port_state.link_speed); - cmd->duplex = DUPLEX_FULL; - } else { - ethtool_cmd_speed_set(cmd, -1); - cmd->duplex = -1; - } - - if (trans_type > 0 && trans_type <= 0xC) { - cmd->port = PORT_FIBRE; - cmd->transceiver = XCVR_EXTERNAL; - cmd->supported |= SUPPORTED_FIBRE; - cmd->advertising |= ADVERTISED_FIBRE; - } else if (trans_type == 0x80 || trans_type == 0) { - cmd->port = PORT_TP; - cmd->transceiver = XCVR_INTERNAL; - cmd->supported |= SUPPORTED_TP; - cmd->advertising |= ADVERTISED_TP; - } else { - cmd->port = -1; - cmd->transceiver = -1; - } - return 0; -} - -static const char *mlx4_en_duplex_to_string(int duplex) -{ - switch (duplex) { - case DUPLEX_FULL: - return "FULL"; - case DUPLEX_HALF: - return "HALF"; - default: - break; - } - return "UNKNOWN"; -} - -static int mlx4_en_set_settings(struct net_device *dev, struct ethtool_cmd *cmd) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_port_state *port_state = &priv->port_state; - - if ((cmd->autoneg != port_state->autoneg) || - (ethtool_cmd_speed(cmd) != port_state->link_speed) || - (cmd->duplex != DUPLEX_FULL)) { - en_info(priv, "Changing port state properties (auto-negotiation" - " , speed/duplex) is not supported. Current:" - " auto-negotiation=%d speed/duplex=%d/%s\n", - port_state->autoneg, port_state->link_speed, - mlx4_en_duplex_to_string(DUPLEX_FULL)); - return -EOPNOTSUPP; - } - - /* User provided same port state properties that are currently set. - * Nothing to change - */ - return 0; -} - -static int mlx4_en_get_coalesce(struct net_device *dev, - struct ethtool_coalesce *coal) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - coal->tx_coalesce_usecs = priv->tx_usecs; - coal->tx_max_coalesced_frames = priv->tx_frames; - coal->rx_coalesce_usecs = priv->rx_usecs; - coal->rx_max_coalesced_frames = priv->rx_frames; - - coal->pkt_rate_low = priv->pkt_rate_low; - coal->rx_coalesce_usecs_low = priv->rx_usecs_low; - coal->pkt_rate_high = priv->pkt_rate_high; - coal->rx_coalesce_usecs_high = priv->rx_usecs_high; - coal->rate_sample_interval = priv->sample_interval; - coal->use_adaptive_rx_coalesce = priv->adaptive_rx_coal; - return 0; -} - -static int mlx4_en_set_coalesce(struct net_device *dev, - struct ethtool_coalesce *coal) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - int err, i; - - priv->rx_frames = (coal->rx_max_coalesced_frames == - MLX4_EN_AUTO_CONF) ? - MLX4_EN_RX_COAL_TARGET / - priv->dev->mtu + 1 : - coal->rx_max_coalesced_frames; - priv->rx_usecs = (coal->rx_coalesce_usecs == - MLX4_EN_AUTO_CONF) ? - MLX4_EN_RX_COAL_TIME : - coal->rx_coalesce_usecs; - - /* Setting TX coalescing parameters */ - if (coal->tx_coalesce_usecs != priv->tx_usecs || - coal->tx_max_coalesced_frames != priv->tx_frames) { - priv->tx_usecs = coal->tx_coalesce_usecs; - priv->tx_frames = coal->tx_max_coalesced_frames; - if (priv->port_up) { - for (i = 0; i < priv->tx_ring_num; i++) { - priv->tx_cq[i]->moder_cnt = priv->tx_frames; - priv->tx_cq[i]->moder_time = priv->tx_usecs; - if (mlx4_en_set_cq_moder(priv, priv->tx_cq[i])) - en_warn(priv, "Failed changing moderation for TX cq %d\n", i); - } - } - } - - /* Set adaptive coalescing params */ - priv->pkt_rate_low = coal->pkt_rate_low; - priv->rx_usecs_low = coal->rx_coalesce_usecs_low; - priv->pkt_rate_high = coal->pkt_rate_high; - priv->rx_usecs_high = coal->rx_coalesce_usecs_high; - priv->sample_interval = coal->rate_sample_interval; - priv->adaptive_rx_coal = coal->use_adaptive_rx_coalesce; - if (priv->adaptive_rx_coal) - return 0; - - if (priv->port_up) { - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_cq[i]->moder_cnt = priv->rx_frames; - priv->rx_cq[i]->moder_time = priv->rx_usecs; - priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; - err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); - if (err) - return err; - } - } - - return 0; -} - -static int mlx4_en_set_pauseparam(struct net_device *dev, - struct ethtool_pauseparam *pause) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int err; - - if (pause->autoneg) - return -EOPNOTSUPP; - - priv->prof->tx_pause = pause->tx_pause != 0; - priv->prof->rx_pause = pause->rx_pause != 0; - err = mlx4_SET_PORT_general(mdev->dev, priv->port, - priv->rx_skb_size + ETH_FCS_LEN, - priv->prof->tx_pause, - priv->prof->tx_ppp, - priv->prof->rx_pause, - priv->prof->rx_ppp); - if (err) - en_err(priv, "Failed setting pause params\n"); - - return err; -} - -static void mlx4_en_get_pauseparam(struct net_device *dev, - struct ethtool_pauseparam *pause) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - pause->tx_pause = priv->prof->tx_pause; - pause->rx_pause = priv->prof->rx_pause; - pause->autoneg = mlx4_en_autoneg_get(dev); -} - -/* rtnl lock must be taken before calling */ -int mlx4_en_pre_config(struct mlx4_en_priv *priv) -{ -#ifdef CONFIG_RFS_ACCEL - struct cpu_rmap *rmap; - - if (!priv->dev->rx_cpu_rmap) - return 0; - - /* Disable RFS events - * Must have all RFS jobs flushed before freeing resources - */ - rmap = priv->dev->rx_cpu_rmap; - priv->dev->rx_cpu_rmap = NULL; - - rtnl_unlock(); - free_irq_cpu_rmap(rmap); - rtnl_lock(); - - if (priv->dev->rx_cpu_rmap) - return -EBUSY; /* another configuration completed while lock - * was free - */ - - /* Make sure all currently running filter_work are being processed - * Other work will return immediatly because of disable_rfs - */ - flush_workqueue(priv->mdev->workqueue); - -#endif - - return 0; -} - -static int mlx4_en_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - u32 rx_size, tx_size; - int port_up = 0; - int err = 0; - int i, n_stats; - u64 *data = NULL; - - if (!priv->port_up) - return -ENOMEM; - - if (param->rx_jumbo_pending || param->rx_mini_pending) - return -EINVAL; - - rx_size = roundup_pow_of_two(param->rx_pending); - rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE); - rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE); - tx_size = roundup_pow_of_two(param->tx_pending); - tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE); - tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE); - - if (rx_size == (priv->port_up ? priv->rx_ring[0]->actual_size : - priv->rx_ring[0]->size) && - tx_size == priv->tx_ring[0]->size) - return 0; - err = mlx4_en_pre_config(priv); - if (err) - return err; - - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); - } - - /* Cache port statistics */ - n_stats = mlx4_en_get_sset_count(dev, ETH_SS_STATS); - if (n_stats > 0) { - data = kmalloc(n_stats * sizeof(u64), GFP_KERNEL); - if (data) - mlx4_en_get_ethtool_stats(dev, NULL, data); - } - - mlx4_en_free_resources(priv); - - priv->prof->tx_ring_size = tx_size; - priv->prof->rx_ring_size = rx_size; - - err = mlx4_en_alloc_resources(priv); - if (err) { - en_err(priv, "Failed reallocating port resources\n"); - goto out; - } - - /* Restore port statistics */ - if (n_stats > 0 && data) - mlx4_en_restore_ethtool_stats(priv, data); - - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) { - en_err(priv, "Failed starting port\n"); - goto out; - } - - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_cq[i]->moder_cnt = priv->rx_frames; - priv->rx_cq[i]->moder_time = priv->rx_usecs; - priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; - err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); - if (err) - goto out; - } - } - -out: - kfree(data); - mutex_unlock(&mdev->state_lock); - return err; -} - -static void mlx4_en_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - if (!priv->port_up) - return; - - memset(param, 0, sizeof(*param)); - param->rx_max_pending = MLX4_EN_MAX_RX_SIZE; - param->tx_max_pending = MLX4_EN_MAX_TX_SIZE; - param->rx_pending = priv->port_up ? - priv->rx_ring[0]->actual_size : priv->rx_ring[0]->size; - param->tx_pending = priv->tx_ring[0]->size; -} - -static u32 mlx4_en_get_rxfh_indir_size(struct net_device *dev) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - return priv->rx_ring_num; -} - -static int mlx4_en_get_rxfh_indir(struct net_device *dev, u32 *ring_index) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_rss_map *rss_map = &priv->rss_map; - int rss_rings; - size_t n = priv->rx_ring_num; - int err = 0; - - rss_rings = priv->prof->rss_rings ?: priv->rx_ring_num; - rss_rings = 1 << ilog2(rss_rings); - - while (n--) { - ring_index[n] = rss_map->qps[n % rss_rings].qpn - - rss_map->base_qpn; - } - - return err; -} - -static int mlx4_en_set_rxfh_indir(struct net_device *dev, - const u32 *ring_index) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int port_up = 0; - int err = 0; - int i; - int rss_rings = 0; - - /* Calculate RSS table size and make sure flows are spread evenly - * between rings - */ - for (i = 0; i < priv->rx_ring_num; i++) { - if (i > 0 && !ring_index[i] && !rss_rings) - rss_rings = i; - - if (ring_index[i] != (i % (rss_rings ?: priv->rx_ring_num))) - return -EINVAL; - } - - if (!rss_rings) - rss_rings = priv->rx_ring_num; - - /* RSS table size must be an order of 2 */ - if (!is_power_of_2(rss_rings)) - return -EINVAL; - - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); - } - - priv->prof->rss_rings = rss_rings; - - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) - en_err(priv, "Failed starting port\n"); - } - - mutex_unlock(&mdev->state_lock); - return err; -} - -#define all_zeros_or_all_ones(field) \ - ((field) == 0 || (field) == (__force typeof(field))-1) - -static int mlx4_en_validate_flow(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd) -{ - struct ethtool_usrip4_spec *l3_mask; - struct ethtool_tcpip4_spec *l4_mask; - struct ethhdr *eth_mask; - - if (cmd->fs.location >= MAX_NUM_OF_FS_RULES) - return -EINVAL; - - if (cmd->fs.flow_type & FLOW_MAC_EXT) { - /* dest mac mask must be ff:ff:ff:ff:ff:ff */ - if (!is_broadcast_ether_addr(cmd->fs.m_ext.h_dest)) - return -EINVAL; - } - - switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { - case TCP_V4_FLOW: - case UDP_V4_FLOW: - if (cmd->fs.m_u.tcp_ip4_spec.tos) - return -EINVAL; - l4_mask = &cmd->fs.m_u.tcp_ip4_spec; - /* don't allow mask which isn't all 0 or 1 */ - if (!all_zeros_or_all_ones(l4_mask->ip4src) || - !all_zeros_or_all_ones(l4_mask->ip4dst) || - !all_zeros_or_all_ones(l4_mask->psrc) || - !all_zeros_or_all_ones(l4_mask->pdst)) - return -EINVAL; - break; - case IP_USER_FLOW: - l3_mask = &cmd->fs.m_u.usr_ip4_spec; - if (l3_mask->l4_4_bytes || l3_mask->tos || l3_mask->proto || - cmd->fs.h_u.usr_ip4_spec.ip_ver != ETH_RX_NFC_IP4 || - (!l3_mask->ip4src && !l3_mask->ip4dst) || - !all_zeros_or_all_ones(l3_mask->ip4src) || - !all_zeros_or_all_ones(l3_mask->ip4dst)) - return -EINVAL; - break; - case ETHER_FLOW: - eth_mask = &cmd->fs.m_u.ether_spec; - /* source mac mask must not be set */ - if (!is_zero_ether_addr(eth_mask->h_source)) - return -EINVAL; - - /* dest mac mask must be ff:ff:ff:ff:ff:ff */ - if (!is_broadcast_ether_addr(eth_mask->h_dest)) - return -EINVAL; - - if (!all_zeros_or_all_ones(eth_mask->h_proto)) - return -EINVAL; - break; - default: - return -EINVAL; - } - - if ((cmd->fs.flow_type & FLOW_EXT)) { - if (cmd->fs.m_ext.vlan_etype || - !(cmd->fs.m_ext.vlan_tci == 0 || - cmd->fs.m_ext.vlan_tci == cpu_to_be16(0xfff))) - return -EINVAL; - if (cmd->fs.m_ext.vlan_tci) { - if (be16_to_cpu(cmd->fs.h_ext.vlan_tci) < - VLAN_MIN_VALUE || - be16_to_cpu(cmd->fs.h_ext.vlan_tci) > - VLAN_MAX_VALUE) - return -EINVAL; - } - } - - return 0; -} - -static int mlx4_en_ethtool_add_mac_rule(struct mlx4_ethtool_rxnfc *cmd, - struct list_head *rule_list_h, - struct mlx4_spec_list *spec_l2, - unsigned char *mac) -{ - int err = 0; - __be64 mac_msk = cpu_to_be64(MLX4_MAC_MASK << 16); - - spec_l2->id = MLX4_NET_TRANS_RULE_ID_ETH; - memcpy(spec_l2->eth.dst_mac_msk, &mac_msk, ETH_ALEN); - memcpy(spec_l2->eth.dst_mac, mac, ETH_ALEN); - - if ((cmd->fs.flow_type & FLOW_EXT) && cmd->fs.m_ext.vlan_tci) { - spec_l2->eth.vlan_id = cmd->fs.h_ext.vlan_tci; - spec_l2->eth.vlan_id_msk = cpu_to_be16(0xfff); - } - - list_add_tail(&spec_l2->list, rule_list_h); - - return err; -} - -static int mlx4_en_ethtool_add_mac_rule_by_ipv4(struct mlx4_en_priv *priv, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *rule_list_h, - struct mlx4_spec_list *spec_l2, - __be32 ipv4_dst) -{ - unsigned char mac[ETH_ALEN]; - - if (!ipv4_is_multicast(ipv4_dst)) { - if (cmd->fs.flow_type & FLOW_MAC_EXT) - memcpy(&mac, cmd->fs.h_ext.h_dest, ETH_ALEN); - else - memcpy(&mac, priv->dev->dev_addr, ETH_ALEN); - } else { - ip_eth_mc_map(ipv4_dst, mac); - } - - return mlx4_en_ethtool_add_mac_rule(cmd, rule_list_h, spec_l2, &mac[0]); -} - -static int add_ip_rule(struct mlx4_en_priv *priv, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *list_h) -{ - struct mlx4_spec_list *spec_l2 = NULL; - struct mlx4_spec_list *spec_l3 = NULL; - struct ethtool_usrip4_spec *l3_mask = &cmd->fs.m_u.usr_ip4_spec; - - spec_l3 = kzalloc(sizeof(*spec_l3), GFP_KERNEL); - spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); - if (!spec_l2 || !spec_l3) { - en_err(priv, "Fail to alloc ethtool rule.\n"); - kfree(spec_l2); - kfree(spec_l3); - return -ENOMEM; - } - - mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, spec_l2, - cmd->fs.h_u. - usr_ip4_spec.ip4dst); - spec_l3->id = MLX4_NET_TRANS_RULE_ID_IPV4; - spec_l3->ipv4.src_ip = cmd->fs.h_u.usr_ip4_spec.ip4src; - if (l3_mask->ip4src) - spec_l3->ipv4.src_ip_msk = MLX4_BE_WORD_MASK; - spec_l3->ipv4.dst_ip = cmd->fs.h_u.usr_ip4_spec.ip4dst; - if (l3_mask->ip4dst) - spec_l3->ipv4.dst_ip_msk = MLX4_BE_WORD_MASK; - list_add_tail(&spec_l3->list, list_h); - - return 0; -} - -static int add_tcp_udp_rule(struct mlx4_en_priv *priv, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *list_h, int proto) -{ - struct mlx4_spec_list *spec_l2 = NULL; - struct mlx4_spec_list *spec_l3 = NULL; - struct mlx4_spec_list *spec_l4 = NULL; - struct ethtool_tcpip4_spec *l4_mask = &cmd->fs.m_u.tcp_ip4_spec; - - spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); - spec_l3 = kzalloc(sizeof(*spec_l3), GFP_KERNEL); - spec_l4 = kzalloc(sizeof(*spec_l4), GFP_KERNEL); - if (!spec_l2 || !spec_l3 || !spec_l4) { - en_err(priv, "Fail to alloc ethtool rule.\n"); - kfree(spec_l2); - kfree(spec_l3); - kfree(spec_l4); - return -ENOMEM; - } - - spec_l3->id = MLX4_NET_TRANS_RULE_ID_IPV4; - - if (proto == TCP_V4_FLOW) { - mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, - spec_l2, - cmd->fs.h_u. - tcp_ip4_spec.ip4dst); - spec_l4->id = MLX4_NET_TRANS_RULE_ID_TCP; - spec_l3->ipv4.src_ip = cmd->fs.h_u.tcp_ip4_spec.ip4src; - spec_l3->ipv4.dst_ip = cmd->fs.h_u.tcp_ip4_spec.ip4dst; - spec_l4->tcp_udp.src_port = cmd->fs.h_u.tcp_ip4_spec.psrc; - spec_l4->tcp_udp.dst_port = cmd->fs.h_u.tcp_ip4_spec.pdst; - } else { - mlx4_en_ethtool_add_mac_rule_by_ipv4(priv, cmd, list_h, - spec_l2, - cmd->fs.h_u. - udp_ip4_spec.ip4dst); - spec_l4->id = MLX4_NET_TRANS_RULE_ID_UDP; - spec_l3->ipv4.src_ip = cmd->fs.h_u.udp_ip4_spec.ip4src; - spec_l3->ipv4.dst_ip = cmd->fs.h_u.udp_ip4_spec.ip4dst; - spec_l4->tcp_udp.src_port = cmd->fs.h_u.udp_ip4_spec.psrc; - spec_l4->tcp_udp.dst_port = cmd->fs.h_u.udp_ip4_spec.pdst; - } - - if (l4_mask->ip4src) - spec_l3->ipv4.src_ip_msk = MLX4_BE_WORD_MASK; - if (l4_mask->ip4dst) - spec_l3->ipv4.dst_ip_msk = MLX4_BE_WORD_MASK; - - if (l4_mask->psrc) - spec_l4->tcp_udp.src_port_msk = MLX4_BE_SHORT_MASK; - if (l4_mask->pdst) - spec_l4->tcp_udp.dst_port_msk = MLX4_BE_SHORT_MASK; - - list_add_tail(&spec_l3->list, list_h); - list_add_tail(&spec_l4->list, list_h); - - return 0; -} - -static int mlx4_en_ethtool_to_net_trans_rule(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd, - struct list_head *rule_list_h) -{ - int err; - struct ethhdr *eth_spec; - struct mlx4_spec_list *spec_l2; - struct mlx4_en_priv *priv = netdev_priv(dev); - - err = mlx4_en_validate_flow(dev, cmd); - if (err) - return err; - - switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) { - case ETHER_FLOW: - spec_l2 = kzalloc(sizeof(*spec_l2), GFP_KERNEL); - if (!spec_l2) - return -ENOMEM; - - eth_spec = &cmd->fs.h_u.ether_spec; - mlx4_en_ethtool_add_mac_rule(cmd, rule_list_h, spec_l2, ð_spec->h_dest[0]); - spec_l2->eth.ether_type = eth_spec->h_proto; - if (eth_spec->h_proto) - spec_l2->eth.ether_type_enable = 1; - break; - case IP_USER_FLOW: - err = add_ip_rule(priv, cmd, rule_list_h); - break; - case TCP_V4_FLOW: - err = add_tcp_udp_rule(priv, cmd, rule_list_h, TCP_V4_FLOW); - break; - case UDP_V4_FLOW: - err = add_tcp_udp_rule(priv, cmd, rule_list_h, UDP_V4_FLOW); - break; - } - - return err; -} - -static int mlx4_en_flow_replace(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd) -{ - int err; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct ethtool_flow_id *loc_rule; - struct mlx4_spec_list *spec, *tmp_spec; - u32 qpn; - u64 reg_id; - - struct mlx4_net_trans_rule rule = { - .queue_mode = MLX4_NET_TRANS_Q_FIFO, - .exclusive = 0, - .allow_loopback = 1, - .promisc_mode = MLX4_FS_REGULAR, - }; - - rule.port = priv->port; - rule.priority = MLX4_DOMAIN_ETHTOOL | cmd->fs.location; - INIT_LIST_HEAD(&rule.list); - - /* Allow direct QP attaches if the EN_ETHTOOL_QP_ATTACH flag is set */ - if (cmd->fs.ring_cookie == RX_CLS_FLOW_DISC) - qpn = priv->drop_qp.qpn; - else if (cmd->fs.ring_cookie & EN_ETHTOOL_QP_ATTACH) { - qpn = cmd->fs.ring_cookie & (EN_ETHTOOL_QP_ATTACH - 1); - } else { - if (cmd->fs.ring_cookie >= priv->rx_ring_num) { - en_warn(priv, "rxnfc: RX ring (%llu) doesn't exist.\n", - cmd->fs.ring_cookie); - return -EINVAL; - } - qpn = priv->rss_map.qps[cmd->fs.ring_cookie].qpn; - if (!qpn) { - en_warn(priv, "rxnfc: RX ring (%llu) is inactive.\n", - cmd->fs.ring_cookie); - return -EINVAL; - } - } - rule.qpn = qpn; - err = mlx4_en_ethtool_to_net_trans_rule(dev, cmd, &rule.list); - if (err) - goto out_free_list; - - mutex_lock(&mdev->state_lock); - loc_rule = &priv->ethtool_rules[cmd->fs.location]; - if (loc_rule->id) { - err = mlx4_flow_detach(priv->mdev->dev, loc_rule->id); - if (err) { - en_err(priv, "Fail to detach network rule at location %d. registration id = %llx\n", - cmd->fs.location, loc_rule->id); - goto unlock; - } - loc_rule->id = 0; - memset(&loc_rule->flow_spec, 0, - sizeof(struct ethtool_rx_flow_spec)); - list_del(&loc_rule->list); - } - err = mlx4_flow_attach(priv->mdev->dev, &rule, ®_id); - if (err) { - en_err(priv, "Fail to attach network rule at location %d.\n", - cmd->fs.location); - goto unlock; - } - loc_rule->id = reg_id; - memcpy(&loc_rule->flow_spec, &cmd->fs, - sizeof(struct ethtool_rx_flow_spec)); - list_add_tail(&loc_rule->list, &priv->ethtool_list); - -unlock: - mutex_unlock(&mdev->state_lock); -out_free_list: - list_for_each_entry_safe(spec, tmp_spec, &rule.list, list) { - list_del(&spec->list); - kfree(spec); - } - return err; -} - -static int mlx4_en_flow_detach(struct net_device *dev, - struct mlx4_ethtool_rxnfc *cmd) -{ - int err = 0; - struct ethtool_flow_id *rule; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - - if (cmd->fs.location >= MAX_NUM_OF_FS_RULES) - return -EINVAL; - - mutex_lock(&mdev->state_lock); - rule = &priv->ethtool_rules[cmd->fs.location]; - if (!rule->id) { - err = -ENOENT; - goto out; - } - - err = mlx4_flow_detach(priv->mdev->dev, rule->id); - if (err) { - en_err(priv, "Fail to detach network rule at location %d. registration id = 0x%llx\n", - cmd->fs.location, rule->id); - goto out; - } - rule->id = 0; - memset(&rule->flow_spec, 0, sizeof(struct ethtool_rx_flow_spec)); - - list_del(&rule->list); -out: - mutex_unlock(&mdev->state_lock); - return err; - -} - -static int mlx4_en_get_flow(struct net_device *dev, struct mlx4_ethtool_rxnfc *cmd, - int loc) -{ - int err = 0; - struct ethtool_flow_id *rule; - struct mlx4_en_priv *priv = netdev_priv(dev); - - if (loc < 0 || loc >= MAX_NUM_OF_FS_RULES) - return -EINVAL; - - rule = &priv->ethtool_rules[loc]; - if (rule->id) - memcpy(&cmd->fs, &rule->flow_spec, - sizeof(struct ethtool_rx_flow_spec)); - else - err = -ENOENT; - - return err; -} - -static int mlx4_en_get_num_flows(struct mlx4_en_priv *priv) -{ - - int i, res = 0; - for (i = 0; i < MAX_NUM_OF_FS_RULES; i++) { - if (priv->ethtool_rules[i].id) - res++; - } - return res; - -} - -static int mlx4_en_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *c, - u32 *rule_locs) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int err = 0; - int i = 0, priority = 0; - struct mlx4_ethtool_rxnfc *cmd = (struct mlx4_ethtool_rxnfc *)c; - - if ((cmd->cmd == ETHTOOL_GRXCLSRLCNT || - cmd->cmd == ETHTOOL_GRXCLSRULE || - cmd->cmd == ETHTOOL_GRXCLSRLALL) && - (mdev->dev->caps.steering_mode != - MLX4_STEERING_MODE_DEVICE_MANAGED || !priv->port_up)) - return -EINVAL; - - switch (cmd->cmd) { - case ETHTOOL_GRXRINGS: - cmd->data = priv->rx_ring_num; - break; - case ETHTOOL_GRXCLSRLCNT: - cmd->rule_cnt = mlx4_en_get_num_flows(priv); - break; - case ETHTOOL_GRXCLSRULE: - err = mlx4_en_get_flow(dev, cmd, cmd->fs.location); - break; - case ETHTOOL_GRXCLSRLALL: - while ((!err || err == -ENOENT) && priority < cmd->rule_cnt) { - err = mlx4_en_get_flow(dev, cmd, i); - if (!err) - rule_locs[priority++] = i; - i++; - } - err = 0; - break; - default: - err = -EOPNOTSUPP; - break; - } - - return err; -} - -static int mlx4_en_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *c) -{ - int err = 0; - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_ethtool_rxnfc *cmd = (struct mlx4_ethtool_rxnfc *)c; - - if (mdev->dev->caps.steering_mode != - MLX4_STEERING_MODE_DEVICE_MANAGED || !priv->port_up) - return -EINVAL; - - switch (cmd->cmd) { - case ETHTOOL_SRXCLSRLINS: - err = mlx4_en_flow_replace(dev, cmd); - break; - case ETHTOOL_SRXCLSRLDEL: - err = mlx4_en_flow_detach(dev, cmd); - break; - default: - en_warn(priv, "Unsupported ethtool command. (%d)\n", cmd->cmd); - return -EINVAL; - } - - return err; -} - -static void mlx4_en_get_channels(struct net_device *dev, - struct ethtool_channels *channel) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - - memset(channel, 0, sizeof(*channel)); - - channel->max_rx = MAX_RX_RINGS; - channel->max_tx = MLX4_EN_MAX_TX_RING_P_UP; - - channel->rx_count = priv->rx_ring_num; - channel->tx_count = priv->tx_ring_num / MLX4_EN_NUM_UP; -} - -static int mlx4_en_set_channels(struct net_device *dev, - struct ethtool_channels *channel) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int port_up = 0; - int i; - int err = 0; - - if (channel->other_count || channel->combined_count || - channel->tx_count > MLX4_EN_MAX_TX_RING_P_UP || - channel->rx_count > MAX_RX_RINGS || - !channel->tx_count || !channel->rx_count) - return -EINVAL; - - err = mlx4_en_pre_config(priv); - if (err) - return err; - - mutex_lock(&mdev->state_lock); - if (priv->port_up) { - port_up = 1; - mlx4_en_stop_port(dev); - } - - mlx4_en_free_resources(priv); - - priv->num_tx_rings_p_up = channel->tx_count; - priv->tx_ring_num = channel->tx_count * MLX4_EN_NUM_UP; - priv->rx_ring_num = channel->rx_count; - - err = mlx4_en_alloc_resources(priv); - if (err) { - en_err(priv, "Failed reallocating port resources\n"); - goto out; - } - - netif_set_real_num_tx_queues(dev, priv->tx_ring_num); - netif_set_real_num_rx_queues(dev, priv->rx_ring_num); - - mlx4_en_setup_tc(dev, MLX4_EN_NUM_UP); - - en_warn(priv, "Using %d TX rings\n", priv->tx_ring_num); - en_warn(priv, "Using %d RX rings\n", priv->rx_ring_num); - - if (port_up) { - err = mlx4_en_start_port(dev); - if (err) - en_err(priv, "Failed starting port\n"); - - for (i = 0; i < priv->rx_ring_num; i++) { - priv->rx_cq[i]->moder_cnt = priv->rx_frames; - priv->rx_cq[i]->moder_time = priv->rx_usecs; - priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; - err = mlx4_en_set_cq_moder(priv, priv->rx_cq[i]); - if (err) - goto out; - } - } - -out: - mutex_unlock(&mdev->state_lock); - return err; -} - -static int mlx4_en_get_ts_info(struct net_device *dev, - struct ethtool_ts_info *info) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int ret; - - ret = ethtool_op_get_ts_info(dev, info); - if (ret) - return ret; - - if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) { - info->so_timestamping |= - SOF_TIMESTAMPING_TX_HARDWARE | - SOF_TIMESTAMPING_RX_HARDWARE | - SOF_TIMESTAMPING_RAW_HARDWARE; - - info->tx_types = - (1 << HWTSTAMP_TX_OFF) | - (1 << HWTSTAMP_TX_ON); - - info->rx_filters = - (1 << HWTSTAMP_FILTER_NONE) | - (1 << HWTSTAMP_FILTER_ALL); - } - - return ret; -} - -const struct ethtool_ops mlx4_en_ethtool_ops = { - .get_drvinfo = mlx4_en_get_drvinfo, - .get_settings = mlx4_en_get_settings, - .set_settings = mlx4_en_set_settings, - .get_link = ethtool_op_get_link, - .get_strings = mlx4_en_get_strings, - .get_sset_count = mlx4_en_get_sset_count, - .get_ethtool_stats = mlx4_en_get_ethtool_stats, - .self_test = mlx4_en_self_test, - .get_wol = mlx4_en_get_wol, - .set_wol = mlx4_en_set_wol, - .get_msglevel = mlx4_en_get_msglevel, - .set_msglevel = mlx4_en_set_msglevel, - .get_coalesce = mlx4_en_get_coalesce, - .set_coalesce = mlx4_en_set_coalesce, - .get_pauseparam = mlx4_en_get_pauseparam, - .set_pauseparam = mlx4_en_set_pauseparam, - .get_ringparam = mlx4_en_get_ringparam, - .set_ringparam = mlx4_en_set_ringparam, - .get_rxnfc = mlx4_en_get_rxnfc, - .set_rxnfc = mlx4_en_set_rxnfc, - .get_rxfh_indir_size = mlx4_en_get_rxfh_indir_size, - .get_rxfh_indir = mlx4_en_get_rxfh_indir, - .set_rxfh_indir = mlx4_en_set_rxfh_indir, - .get_channels = mlx4_en_get_channels, - .set_channels = mlx4_en_set_channels, - .get_ts_info = mlx4_en_get_ts_info, -}; - - - - - diff --git a/sys/ofed/drivers/net/mlx4/en_main.c b/sys/ofed/drivers/net/mlx4/en_main.c index 757afe0..09c1824 100644 --- a/sys/ofed/drivers/net/mlx4/en_main.c +++ b/sys/ofed/drivers/net/mlx4/en_main.c @@ -42,14 +42,7 @@ #include "mlx4_en.h" -MODULE_AUTHOR("Liran Liss, Yevgeny Petrilin"); -MODULE_DESCRIPTION("Mellanox ConnectX HCA Ethernet driver"); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_VERSION(DRV_VERSION " ("DRV_RELDATE")"); - -static const char mlx4_en_version[] = - DRV_NAME ": Mellanox ConnectX HCA Ethernet driver v" - DRV_VERSION " (" DRV_RELDATE ")\n"; +/* Mellanox ConnectX HCA Ethernet driver */ #define MLX4_EN_PARM_INT(X, def_val, desc) \ static unsigned int X = def_val;\ @@ -174,8 +167,6 @@ static void *mlx4_en_add(struct mlx4_dev *dev) int i; int err; - printk_once(KERN_INFO "%s", mlx4_en_version); - mdev = kzalloc(sizeof *mdev, GFP_KERNEL); if (!mdev) { dev_err(&dev->pdev->dev, "Device struct alloc failed, " diff --git a/sys/ofed/drivers/net/mlx4/en_netdev.c b/sys/ofed/drivers/net/mlx4/en_netdev.c index d8b2f4f..0c7b300 100644 --- a/sys/ofed/drivers/net/mlx4/en_netdev.c +++ b/sys/ofed/drivers/net/mlx4/en_netdev.c @@ -659,8 +659,10 @@ static void mlx4_en_cache_mclist(struct net_device *dev) continue; /* Make sure the list didn't grow. */ tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); - if (tmp == NULL) + if (tmp == NULL) { + en_err(priv, "Failed to allocate multicast list\n"); break; + } memcpy(tmp->addr, LLADDR((struct sockaddr_dl *)ifma->ifma_addr), ETH_ALEN); list_add_tail(&tmp->list, &priv->mc_list); @@ -971,12 +973,12 @@ static void mlx4_en_do_set_rx_mode(struct work_struct *work) if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { if (priv->port_state.link_state) { priv->last_link_state = MLX4_DEV_EVENT_PORT_UP; - /* Important note: the following call for if_link_state_change - * is needed for interface up scenario (start port, link state - * change) */ /* update netif baudrate */ priv->dev->if_baudrate = IF_Mbps(priv->port_state.link_speed); + /* Important note: the following call for if_link_state_change + * is needed for interface up scenario (start port, link state + * change) */ if_link_state_change(priv->dev, LINK_STATE_UP); en_dbg(HW, priv, "Link Up\n"); } @@ -1196,8 +1198,8 @@ static void mlx4_en_linkstate(struct work_struct *work) /* update netif baudrate */ priv->dev->if_baudrate = 0; - /* make sure the port is up before notifying the OS. - * This is tricky since we get here on INIT_PORT and + /* make sure the port is up before notifying the OS. + * This is tricky since we get here on INIT_PORT and * in such case we can't tell the OS the port is up. * To solve this there is a call to if_link_state_change * in set_rx_mode. @@ -1246,7 +1248,6 @@ int mlx4_en_start_port(struct net_device *dev) PAGE_SIZE); priv->rx_alloc_order = get_order(priv->rx_alloc_size); priv->rx_buf_size = roundup_pow_of_two(priv->rx_mb_size); - priv->log_rx_info = ROUNDUP_LOG2(sizeof(struct mlx4_en_rx_buf)); en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_mb_size); /* Configure rx cq's and rings */ @@ -1575,6 +1576,7 @@ static void mlx4_en_clear_stats(struct net_device *dev) priv->tx_ring[i]->bytes = 0; priv->tx_ring[i]->packets = 0; priv->tx_ring[i]->tx_csum = 0; + priv->tx_ring[i]->oversized_packets = 0; } for (i = 0; i < priv->rx_ring_num; i++) { priv->rx_ring[i]->bytes = 0; @@ -1644,8 +1646,6 @@ void mlx4_en_free_resources(struct mlx4_en_priv *priv) if (priv->sysctl) sysctl_ctx_free(&priv->stat_ctx); - - } int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) @@ -1730,8 +1730,11 @@ void mlx4_en_destroy_netdev(struct net_device *dev) EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); /* Unregister device - this will close the port if it was up */ - if (priv->registered) + if (priv->registered) { + mutex_lock(&mdev->state_lock); ether_ifdetach(dev); + mutex_unlock(&mdev->state_lock); + } if (priv->allocated) mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); @@ -1809,13 +1812,6 @@ static int mlx4_en_calc_media(struct mlx4_en_priv *priv) active = IFM_ETHER; if (priv->last_link_state == MLX4_DEV_EVENT_PORT_DOWN) return (active); - /* - * [ShaharK] mlx4_en_QUERY_PORT sleeps and cannot be called under a - * non-sleepable lock. - * I moved it to the periodic mlx4_en_do_get_stats. - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return (active); - */ active |= IFM_FDX; trans_type = priv->port_state.transciver; /* XXX I don't know all of the transceiver values. */ @@ -1948,12 +1944,55 @@ static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) case SIOCSIFCAP: mutex_lock(&mdev->state_lock); mask = ifr->ifr_reqcap ^ dev->if_capenable; - if (mask & IFCAP_HWCSUM) - dev->if_capenable ^= IFCAP_HWCSUM; - if (mask & IFCAP_TSO4) + if (mask & IFCAP_TXCSUM) { + dev->if_capenable ^= IFCAP_TXCSUM; + dev->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); + + if (IFCAP_TSO4 & dev->if_capenable && + !(IFCAP_TXCSUM & dev->if_capenable)) { + dev->if_capenable &= ~IFCAP_TSO4; + dev->if_hwassist &= ~CSUM_IP_TSO; + if_printf(dev, + "tso4 disabled due to -txcsum.\n"); + } + } + if (mask & IFCAP_TXCSUM_IPV6) { + dev->if_capenable ^= IFCAP_TXCSUM_IPV6; + dev->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); + + if (IFCAP_TSO6 & dev->if_capenable && + !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { + dev->if_capenable &= ~IFCAP_TSO6; + dev->if_hwassist &= ~CSUM_IP6_TSO; + if_printf(dev, + "tso6 disabled due to -txcsum6.\n"); + } + } + if (mask & IFCAP_RXCSUM) + dev->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + dev->if_capenable ^= IFCAP_RXCSUM_IPV6; + + if (mask & IFCAP_TSO4) { + if (!(IFCAP_TSO4 & dev->if_capenable) && + !(IFCAP_TXCSUM & dev->if_capenable)) { + if_printf(dev, "enable txcsum first.\n"); + error = EAGAIN; + goto out; + } dev->if_capenable ^= IFCAP_TSO4; - if (mask & IFCAP_TSO6) + dev->if_hwassist ^= CSUM_IP_TSO; + } + if (mask & IFCAP_TSO6) { + if (!(IFCAP_TSO6 & dev->if_capenable) && + !(IFCAP_TXCSUM_IPV6 & dev->if_capenable)) { + if_printf(dev, "enable txcsum6 first.\n"); + error = EAGAIN; + goto out; + } dev->if_capenable ^= IFCAP_TSO6; + dev->if_hwassist ^= CSUM_IP6_TSO; + } if (mask & IFCAP_LRO) dev->if_capenable ^= IFCAP_LRO; if (mask & IFCAP_VLAN_HWTAGGING) @@ -1964,9 +2003,11 @@ static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) dev->if_capenable ^= IFCAP_WOL_MAGIC; if (dev->if_drv_flags & IFF_DRV_RUNNING) mlx4_en_start_port(dev); +out: mutex_unlock(&mdev->state_lock); VLAN_CAPABILITIES(dev); break; +#if __FreeBSD_version >= 1100036 case SIOCGI2C: { struct ifi2creq i2c; @@ -1990,6 +2031,7 @@ static int mlx4_en_ioctl(struct ifnet *dev, u_long command, caddr_t data) error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); break; } +#endif default: error = ether_ioctl(dev, command, data); break; @@ -2049,8 +2091,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, priv->port = port; priv->port_up = false; priv->flags = prof->flags; - priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | - MLX4_WQE_CTRL_SOLICITED); priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up; priv->tx_ring_num = prof->tx_ring_num; @@ -2066,7 +2106,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, err = -ENOMEM; goto out; } - + priv->rx_ring_num = prof->rx_ring_num; priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; priv->mac_index = -1; @@ -2089,7 +2129,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) INIT_HLIST_HEAD(&priv->mac_hash[i]); - /* Query for default mac and max mtu */ priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; priv->mac = mdev->dev->caps.def_mac[priv->port]; @@ -2105,8 +2144,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, goto out; } - - priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + DS_SIZE); @@ -2128,7 +2165,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, /* * Set driver features */ - dev->if_capabilities |= IFCAP_RXCSUM | IFCAP_TXCSUM; + dev->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6; dev->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; dev->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; dev->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; @@ -2137,10 +2174,12 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, if (mdev->LSO_support) dev->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTSO; +#if __FreeBSD_version >= 1100000 /* set TSO limits so that we don't have to drop TX packets */ - dev->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); - dev->if_hw_tsomaxsegcount = 16; - dev->if_hw_tsomaxsegsize = 65536; /* XXX can do up to 4GByte */ + dev->if_hw_tsomax = MLX4_EN_TX_MAX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN) /* hdr */; + dev->if_hw_tsomaxsegcount = MLX4_EN_TX_MAX_MBUF_FRAGS - 1 /* hdr */; + dev->if_hw_tsomaxsegsize = MLX4_EN_TX_MAX_MBUF_SIZE; +#endif dev->if_capenable = dev->if_capabilities; @@ -2149,6 +2188,8 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, dev->if_hwassist |= CSUM_TSO; if (dev->if_capenable & IFCAP_TXCSUM) dev->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); + if (dev->if_capenable & IFCAP_TXCSUM_IPV6) + dev->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); /* Register for VLAN events */ @@ -2211,8 +2252,6 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) queue_delayed_work(mdev->workqueue, &priv->service_task, SERVICE_TASK_DELAY); - - return 0; out: @@ -2294,6 +2333,162 @@ static int mlx4_en_set_tx_ring_size(SYSCTL_HANDLER_ARGS) return (error); } +static int mlx4_en_get_module_info(struct net_device *dev, + struct ethtool_modinfo *modinfo) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int ret; + u8 data[4]; + + /* Read first 2 bytes to get Module & REV ID */ + ret = mlx4_get_module_info(mdev->dev, priv->port, + 0/*offset*/, 2/*size*/, data); + + if (ret < 2) { + en_err(priv, "Failed to read eeprom module first two bytes, error: 0x%x\n", -ret); + return -EIO; + } + + switch (data[0] /* identifier */) { + case MLX4_MODULE_ID_QSFP: + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + break; + case MLX4_MODULE_ID_QSFP_PLUS: + if (data[1] >= 0x3) { /* revision id */ + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + } else { + modinfo->type = ETH_MODULE_SFF_8436; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + } + break; + case MLX4_MODULE_ID_QSFP28: + modinfo->type = ETH_MODULE_SFF_8636; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + break; + case MLX4_MODULE_ID_SFP: + modinfo->type = ETH_MODULE_SFF_8472; + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; + break; + default: + en_err(priv, "mlx4_en_get_module_info : Not recognized cable type\n"); + return -EINVAL; + } + + return 0; +} + +static int mlx4_en_get_module_eeprom(struct net_device *dev, + struct ethtool_eeprom *ee, + u8 *data) +{ + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_dev *mdev = priv->mdev; + int offset = ee->offset; + int i = 0, ret; + + if (ee->len == 0) + return -EINVAL; + + memset(data, 0, ee->len); + + while (i < ee->len) { + en_dbg(DRV, priv, + "mlx4_get_module_info i(%d) offset(%d) len(%d)\n", + i, offset, ee->len - i); + + ret = mlx4_get_module_info(mdev->dev, priv->port, + offset, ee->len - i, data + i); + + if (!ret) /* Done reading */ + return 0; + + if (ret < 0) { + en_err(priv, + "mlx4_get_module_info i(%d) offset(%d) bytes_to_read(%d) - FAILED (0x%x)\n", + i, offset, ee->len - i, ret); + return -1; + } + + i += ret; + offset += ret; + } + return 0; +} + +static void mlx4_en_print_eeprom(u8 *data, __u32 len) +{ + int i; + int j = 0; + int row = 0; + const int NUM_OF_BYTES = 16; + + printf("\nOffset\t\tValues\n"); + printf("------\t\t------\n"); + while(row < len){ + printf("0x%04x\t\t",row); + for(i=0; i < NUM_OF_BYTES; i++){ + printf("%02x ", data[j]); + row++; + j++; + } + printf("\n"); + } +} + +/* Read cable EEPROM module information by first inspecting the first + * two bytes to get the length and then read the rest of the information. + * The information is printed to dmesg. */ +static int mlx4_en_read_eeprom(SYSCTL_HANDLER_ARGS) +{ + + u8* data; + int error; + int result = 0; + struct mlx4_en_priv *priv; + struct net_device *dev; + struct ethtool_modinfo modinfo; + struct ethtool_eeprom ee; + + error = sysctl_handle_int(oidp, &result, 0, req); + if (error || !req->newptr) + return (error); + + if (result == 1) { + priv = arg1; + dev = priv->dev; + data = kmalloc(PAGE_SIZE, GFP_KERNEL); + + error = mlx4_en_get_module_info(dev, &modinfo); + if (error) { + en_err(priv, + "mlx4_en_get_module_info returned with error - FAILED (0x%x)\n", + -error); + goto out; + } + + ee.len = modinfo.eeprom_len; + ee.offset = 0; + + error = mlx4_en_get_module_eeprom(dev, &ee, data); + if (error) { + en_err(priv, + "mlx4_en_get_module_eeprom returned with error - FAILED (0x%x)\n", + -error); + /* Continue printing partial information in case of an error */ + } + + /* EEPROM information will be printed in dmesg */ + mlx4_en_print_eeprom(data, ee.len); +out: + kfree(data); + } + /* Return zero to prevent sysctl failure. */ + return (0); +} + static int mlx4_en_set_tx_ppp(SYSCTL_HANDLER_ARGS) { struct mlx4_en_priv *priv; @@ -2419,7 +2614,7 @@ static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv) /* Add coalescer configuration. */ coal = SYSCTL_ADD_NODE(ctx, node_list, OID_AUTO, "coalesce", CTLFLAG_RD, NULL, "Interrupt coalesce configuration"); - coal_list = SYSCTL_CHILDREN(node); + coal_list = SYSCTL_CHILDREN(coal); SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "pkt_rate_low", CTLFLAG_RW, &priv->pkt_rate_low, 0, "Packets per-second for minimum delay"); @@ -2438,11 +2633,14 @@ static void mlx4_en_sysctl_conf(struct mlx4_en_priv *priv) SYSCTL_ADD_UINT(ctx, coal_list, OID_AUTO, "adaptive_rx_coal", CTLFLAG_RW, &priv->adaptive_rx_coal, 0, "Enable adaptive rx coalescing"); + /* EEPROM support */ + SYSCTL_ADD_PROC(ctx, node_list, OID_AUTO, "eeprom_info", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx4_en_read_eeprom, "I", "EEPROM information"); } static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) { - struct net_device *dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *node; struct sysctl_oid_list *node_list; @@ -2453,8 +2651,6 @@ static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) char namebuf[128]; int i; - dev = priv->dev; - ctx = &priv->stat_ctx; sysctl_ctx_init(ctx); node = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(priv->sysctl), OID_AUTO, @@ -2482,6 +2678,8 @@ static void mlx4_en_sysctl_stat(struct mlx4_en_priv *priv) &priv->port_stats.wake_queue, "Queue resumed after full"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_timeout", CTLFLAG_RD, &priv->port_stats.tx_timeout, "Transmit timeouts"); + SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_oversized_packets", CTLFLAG_RD, + &priv->port_stats.oversized_packets, "TX oversized packets, m_defrag failed"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_alloc_failed", CTLFLAG_RD, &priv->port_stats.rx_alloc_failed, "RX failed to allocate mbuf"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "rx_chksum_good", CTLFLAG_RD, @@ -2565,7 +2763,7 @@ struct mlx4_en_pkt_stats { SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_packets", CTLFLAG_RD, &priv->pkstats.tx_packets, "TX packets"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_bytes", CTLFLAG_RD, - &priv->pkstats.tx_packets, "TX Bytes"); + &priv->pkstats.tx_bytes, "TX Bytes"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_multicast_packets", CTLFLAG_RD, &priv->pkstats.tx_multicast_packets, "TX Multicast Packets"); SYSCTL_ADD_ULONG(ctx, node_list, OID_AUTO, "tx_broadcast_packets", CTLFLAG_RD, @@ -2606,8 +2804,8 @@ struct mlx4_en_pkt_stats { CTLFLAG_RD, &tx_ring->packets, "TX packets"); SYSCTL_ADD_ULONG(ctx, ring_list, OID_AUTO, "bytes", CTLFLAG_RD, &tx_ring->bytes, "TX bytes"); - } + for (i = 0; i < priv->rx_ring_num; i++) { rx_ring = priv->rx_ring[i]; snprintf(namebuf, sizeof(namebuf), "rx_ring%d", i); diff --git a/sys/ofed/drivers/net/mlx4/en_port.c b/sys/ofed/drivers/net/mlx4/en_port.c index 645c9c4..6c71962 100644 --- a/sys/ofed/drivers/net/mlx4/en_port.c +++ b/sys/ofed/drivers/net/mlx4/en_port.c @@ -194,6 +194,7 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) priv->port_stats.tx_chksum_offload += priv->tx_ring[i]->tx_csum; priv->port_stats.queue_stopped += priv->tx_ring[i]->queue_stopped; priv->port_stats.wake_queue += priv->tx_ring[i]->wake_queue; + priv->port_stats.oversized_packets += priv->tx_ring[i]->oversized_packets; } /* RX Statistics */ priv->pkstats.rx_packets = be64_to_cpu(mlx4_en_stats->RTOT_prio_0) + diff --git a/sys/ofed/drivers/net/mlx4/en_rx.c b/sys/ofed/drivers/net/mlx4/en_rx.c index e011b87..b57bd26 100644 --- a/sys/ofed/drivers/net/mlx4/en_rx.c +++ b/sys/ofed/drivers/net/mlx4/en_rx.c @@ -49,106 +49,135 @@ static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index; + struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (ring->stride * index)); int possible_frags; int i; - /* Set size and memtype fields */ - for (i = 0; i < priv->num_frags; i++) { - rx_desc->data[i].byte_count = - cpu_to_be32(priv->frag_info[i].frag_size); - rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key); - } - - /* If the number of used fragments does not fill up the ring stride, - * * remaining (unused) fragments must be padded with null address/size - * * and a special memory key */ + rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size); + rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key); + + /* + * If the number of used fragments does not fill up the ring + * stride, remaining (unused) fragments must be padded with + * null address/size and a special memory key: + */ possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE; - for (i = priv->num_frags; i < possible_frags; i++) { + for (i = 1; i < possible_frags; i++) { rx_desc->data[i].byte_count = 0; rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD); rx_desc->data[i].addr = 0; } - } -static int mlx4_en_alloc_buf(struct mlx4_en_priv *priv, - struct mlx4_en_rx_desc *rx_desc, - struct mbuf **mb_list, - int i) +static int +mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring, + __be64 *pdma, struct mlx4_en_rx_mbuf *mb_list) { - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_frag_info *frag_info = &priv->frag_info[i]; + bus_dma_segment_t segs[1]; + bus_dmamap_t map; struct mbuf *mb; - dma_addr_t dma; + int nsegs; + int err; - if (i == 0) - mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, frag_info->frag_size); - else - mb = m_getjcl(M_NOWAIT, MT_DATA, 0, frag_info->frag_size); - if (mb == NULL) { - priv->port_stats.rx_alloc_failed++; - return -ENOMEM; + /* try to allocate a new spare mbuf */ + if (unlikely(ring->spare.mbuf == NULL)) { + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (unlikely(mb == NULL)) + return (-ENOMEM); + /* setup correct length */ + mb->m_len = ring->rx_mb_size; + + /* load spare mbuf into BUSDMA */ + err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (unlikely(err != 0)) { + m_freem(mb); + return (err); + } + + /* store spare info */ + ring->spare.mbuf = mb; + ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr); + + bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, + BUS_DMASYNC_PREREAD); + } + + /* synchronize and unload the current mbuf, if any */ + if (likely(mb_list->mbuf != NULL)) { + bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, mb_list->dma_map); } - dma = pci_map_single(mdev->pdev, mb->m_data, frag_info->frag_size, - PCI_DMA_FROMDEVICE); - rx_desc->data[i].addr = cpu_to_be64(dma); - mb_list[i] = mb; - return 0; -} + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size); + if (unlikely(mb == NULL)) + goto use_spare; -static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, int index) -{ - struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride); - struct mbuf **mb_list = ring->rx_info + (index << priv->log_rx_info); - int i; + /* setup correct length */ + mb->m_len = ring->rx_mb_size; - for (i = 0; i < priv->num_frags; i++) - if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, i)) - goto err; + err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (unlikely(err != 0)) { + m_freem(mb); + goto use_spare; + } + + *pdma = cpu_to_be64(segs[0].ds_addr); + mb_list->mbuf = mb; + + bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD); + return (0); - return 0; +use_spare: + /* swap DMA maps */ + map = mb_list->dma_map; + mb_list->dma_map = ring->spare.dma_map; + ring->spare.dma_map = map; -err: - while (i--) - m_free(mb_list[i]); - return -ENOMEM; + /* swap MBUFs */ + mb_list->mbuf = ring->spare.mbuf; + ring->spare.mbuf = NULL; + + /* store physical address */ + *pdma = ring->spare.paddr_be; + return (0); } -static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) +static void +mlx4_en_free_buf(struct mlx4_en_rx_ring *ring, struct mlx4_en_rx_mbuf *mb_list) { - *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); + bus_dmamap_t map = mb_list->dma_map; + bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, map); + m_freem(mb_list->mbuf); + mb_list->mbuf = NULL; /* safety clearing */ } -static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, - int index) +static int +mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *ring, int index) { - struct mlx4_en_frag_info *frag_info; - struct mlx4_en_dev *mdev = priv->mdev; - struct mbuf **mb_list; - struct mlx4_en_rx_desc *rx_desc = ring->buf + (index << ring->log_stride); - dma_addr_t dma; - int nr; - - mb_list = ring->rx_info + (index << priv->log_rx_info); - for (nr = 0; nr < priv->num_frags; nr++) { - en_dbg(DRV, priv, "Freeing fragment:%d\n", nr); - frag_info = &priv->frag_info[nr]; - dma = be64_to_cpu(rx_desc->data[nr].addr); - -#if BITS_PER_LONG == 64 - en_dbg(DRV, priv, "Unmaping buffer at dma:0x%lx\n", (u64) dma); -#elif BITS_PER_LONG == 32 - en_dbg(DRV, priv, "Unmaping buffer at dma:0x%llx\n", (u64) dma); -#endif - pci_unmap_single(mdev->pdev, dma, frag_info->frag_size, - PCI_DMA_FROMDEVICE); - m_free(mb_list[nr]); + struct mlx4_en_rx_desc *rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (index * ring->stride)); + struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index; + + mb_list->mbuf = NULL; + + if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) { + priv->port_stats.rx_alloc_failed++; + return (-ENOMEM); } + return (0); +} + +static inline void +mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring) +{ + *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff); } static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv) @@ -191,7 +220,8 @@ reduce_rings: while (ring->actual_size > new_size) { ring->actual_size--; ring->prod--; - mlx4_en_free_rx_desc(priv, ring, ring->actual_size); + mlx4_en_free_buf(ring, + ring->mbuf + ring->actual_size); } } @@ -211,100 +241,106 @@ static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, while (ring->cons != ring->prod) { index = ring->cons & ring->size_mask; en_dbg(DRV, priv, "Processing descriptor:%d\n", index); - mlx4_en_free_rx_desc(priv, ring, index); + mlx4_en_free_buf(ring, ring->mbuf + index); ++ring->cons; } } -#if MLX4_EN_MAX_RX_FRAGS == 3 -static int frag_sizes[] = { - FRAG_SZ0, - FRAG_SZ1, - FRAG_SZ2, -}; -#elif MLX4_EN_MAX_RX_FRAGS == 2 -static int frag_sizes[] = { - FRAG_SZ0, - FRAG_SZ1, -}; -#else -#error "Unknown MAX_RX_FRAGS" -#endif - void mlx4_en_calc_rx_buf(struct net_device *dev) { struct mlx4_en_priv *priv = netdev_priv(dev); int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN; - int buf_size = 0; - int i, frag; - for (i = 0, frag = 0; buf_size < eff_mtu; frag++, i++) { - /* - * Allocate small to large but only as much as is needed for - * the tail. - */ - while (i > 0 && eff_mtu - buf_size <= frag_sizes[i - 1]) - i--; - priv->frag_info[frag].frag_size = frag_sizes[i]; - priv->frag_info[frag].frag_prefix_size = buf_size; - buf_size += priv->frag_info[frag].frag_size; - } + if (eff_mtu > MJUM16BYTES) { + en_err(priv, "MTU(%d) is too big\n", (int)dev->if_mtu); + eff_mtu = MJUM16BYTES; + } else if (eff_mtu > MJUM9BYTES) { + eff_mtu = MJUM16BYTES; + } else if (eff_mtu > MJUMPAGESIZE) { + eff_mtu = MJUM9BYTES; + } else if (eff_mtu > MCLBYTES) { + eff_mtu = MJUMPAGESIZE; + } else { + eff_mtu = MCLBYTES; + } - priv->num_frags = frag; priv->rx_mb_size = eff_mtu; - priv->log_rx_info = - ROUNDUP_LOG2(priv->num_frags * sizeof(struct mbuf *)); - en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d " - "num_frags:%d):\n", eff_mtu, priv->num_frags); - for (i = 0; i < priv->num_frags; i++) { - en_dbg(DRV, priv, " frag:%d - size:%d prefix:%d\n", i, - priv->frag_info[i].frag_size, - priv->frag_info[i].frag_prefix_size); - } + en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu); } - int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, int node) { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring; - int err = -ENOMEM; + int err; int tmp; + uint32_t x; ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL); if (!ring) { en_err(priv, "Failed to allocate RX ring structure\n"); return -ENOMEM; } - + + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MJUM16BYTES, /* maxsize */ + 1, /* nsegments */ + MJUM16BYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->dma_tag))) { + en_err(priv, "Failed to create DMA tag\n"); + goto err_ring; + } + ring->prod = 0; ring->cons = 0; ring->size = size; ring->size_mask = size - 1; - ring->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE * MLX4_EN_MAX_RX_FRAGS); + ring->stride = roundup_pow_of_two( + sizeof(struct mlx4_en_rx_desc) + DS_SIZE); ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride + TXBB_SIZE; - tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * - sizeof(struct mbuf *)); + tmp = size * sizeof(struct mlx4_en_rx_mbuf); - ring->rx_info = kmalloc(tmp, GFP_KERNEL); - if (!ring->rx_info) { + ring->mbuf = kzalloc(tmp, GFP_KERNEL); + if (ring->mbuf == NULL) { err = -ENOMEM; - goto err_ring; + goto err_dma_tag; } - en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n", - ring->rx_info, tmp); + err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map); + if (err != 0) + goto err_info; + + for (x = 0; x != size; x++) { + err = -bus_dmamap_create(ring->dma_tag, 0, + &ring->mbuf[x].dma_map); + if (err != 0) { + while (x--) + bus_dmamap_destroy(ring->dma_tag, + ring->mbuf[x].dma_map); + goto err_info; + } + } + en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n", + ring->mbuf, tmp); err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size, 2 * PAGE_SIZE); if (err) - goto err_info; + goto err_dma_map; err = mlx4_en_map_buffer(&ring->wqres.buf); if (err) { @@ -317,23 +353,29 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, err_hwq: mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); +err_dma_map: + for (x = 0; x != size; x++) { + bus_dmamap_destroy(ring->dma_tag, + ring->mbuf[x].dma_map); + } + bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); err_info: - vfree(ring->rx_info); + vfree(ring->mbuf); +err_dma_tag: + bus_dma_tag_destroy(ring->dma_tag); err_ring: kfree(ring); - - return err; + return (err); } - int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv) { struct mlx4_en_rx_ring *ring; int i; int ring_ind; int err; - int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + - DS_SIZE * priv->num_frags); + int stride = roundup_pow_of_two( + sizeof(struct mlx4_en_rx_desc) + DS_SIZE); for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) { ring = priv->rx_ring[ring_ind]; @@ -409,10 +451,22 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_rx_ring *ring = *pring; + uint32_t x; mlx4_en_unmap_buffer(&ring->wqres.buf); mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE); - vfree(ring->rx_info); + for (x = 0; x != size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map); + /* free spare mbuf, if any */ + if (ring->spare.mbuf != NULL) { + bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map); + m_freem(ring->spare.mbuf); + } + bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map); + vfree(ring->mbuf); + bus_dma_tag_destroy(ring->dma_tag); kfree(ring); *pring = NULL; #ifdef CONFIG_RFS_ACCEL @@ -420,7 +474,6 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, #endif } - void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { @@ -469,69 +522,27 @@ static inline int invalid_cqe(struct mlx4_en_priv *priv, return 0; } - -/* Unmap a completed descriptor and free unused pages */ -static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_rx_desc *rx_desc, - struct mbuf **mb_list, - int length) +static struct mbuf * +mlx4_en_rx_mb(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, + struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_mbuf *mb_list, + int length) { - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_frag_info *frag_info; - dma_addr_t dma; struct mbuf *mb; - int nr; - - mb = mb_list[0]; - mb->m_pkthdr.len = length; - /* Collect used fragments while replacing them in the HW descirptors */ - for (nr = 0; nr < priv->num_frags; nr++) { - frag_info = &priv->frag_info[nr]; - if (length <= frag_info->frag_prefix_size) - break; - if (nr) - mb->m_next = mb_list[nr]; - mb = mb_list[nr]; - mb->m_len = frag_info->frag_size; - dma = be64_to_cpu(rx_desc->data[nr].addr); - - /* Allocate a replacement page */ - if (mlx4_en_alloc_buf(priv, rx_desc, mb_list, nr)) - goto fail; - - /* Unmap buffer */ - pci_unmap_single(mdev->pdev, dma, frag_info->frag_size, - PCI_DMA_FROMDEVICE); - } - /* Adjust size of last fragment to match actual length */ - mb->m_len = length - priv->frag_info[nr - 1].frag_prefix_size; - mb->m_next = NULL; - return 0; -fail: - /* Drop all accumulated fragments (which have already been replaced in - * the descriptor) of this packet; remaining fragments are reused... */ - while (nr > 0) { - nr--; - m_free(mb_list[nr]); - } - return -ENOMEM; - -} + /* get mbuf */ + mb = mb_list->mbuf; -static struct mbuf *mlx4_en_rx_mb(struct mlx4_en_priv *priv, - struct mlx4_en_rx_desc *rx_desc, - struct mbuf **mb_list, - unsigned int length) -{ - struct mbuf *mb; + /* collect used fragment while atomically replacing it */ + if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) + return (NULL); - mb = mb_list[0]; - /* Move relevant fragments to mb */ - if (unlikely(mlx4_en_complete_rx_desc(priv, rx_desc, mb_list, length))) - return NULL; + /* range check hardware computed value */ + if (unlikely(length > mb->m_len)) + length = mb->m_len; - return mb; + /* update total packet length in packet header */ + mb->m_len = mb->m_pkthdr.len = length; + return (mb); } /* For cpu arch with cache line of 64B the performance is better when cqe size==64B @@ -545,7 +556,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_cqe *cqe; struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring]; - struct mbuf **mb_list; + struct mlx4_en_rx_mbuf *mb_list; struct mlx4_en_rx_desc *rx_desc; struct mbuf *mb; struct mlx4_cq *mcq = &cq->mcq; @@ -573,8 +584,9 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud /* Process all completed CQEs */ while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK, cons_index & size)) { - mb_list = ring->rx_info + (index << priv->log_rx_info); - rx_desc = ring->buf + (index << ring->log_stride); + mb_list = ring->mbuf + index; + rx_desc = (struct mlx4_en_rx_desc *) + (ring->buf + (index << ring->log_stride)); /* * make sure we read the CQE after we read the ownership bit @@ -589,8 +601,9 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud */ length = be32_to_cpu(cqe->byte_cnt); length -= ring->fcs_del; - mb = mlx4_en_rx_mb(priv, rx_desc, mb_list, length); - if (!mb) { + + mb = mlx4_en_rx_mb(priv, ring, rx_desc, mb_list, length); + if (unlikely(!mb)) { ring->errors++; goto next; } @@ -611,7 +624,8 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->sl_vid); mb->m_flags |= M_VLANTAG; } - if (likely(dev->if_capabilities & IFCAP_RXCSUM) && + if (likely(dev->if_capenable & + (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) && (cqe->checksum == cpu_to_be16(0xffff))) { priv->port_stats.rx_chksum_good++; @@ -692,6 +706,7 @@ void mlx4_en_rx_irq(struct mlx4_cq *mcq) // Because there is no NAPI in freeBSD done = mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET); if (priv->port_up && (done == MLX4_EN_RX_BUDGET) ) { + cq->curr_poll_rx_cpu_id = curcpu; taskqueue_enqueue(cq->tq, &cq->cq_task); } else { @@ -702,8 +717,15 @@ void mlx4_en_rx_irq(struct mlx4_cq *mcq) void mlx4_en_rx_que(void *context, int pending) { struct mlx4_en_cq *cq; + struct thread *td; cq = context; + td = curthread; + + thread_lock(td); + sched_bind(td, cq->curr_poll_rx_cpu_id); + thread_unlock(td); + while (mlx4_en_poll_rx_cq(cq, MLX4_EN_RX_BUDGET) == MLX4_EN_RX_BUDGET); mlx4_en_arm_cq(cq->dev->if_softc, cq); @@ -841,8 +863,8 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) else rss_rings = priv->prof->rss_rings; - ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path) - + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; + ptr = ((u8 *)&context) + offsetof(struct mlx4_qp_context, pri_path) + + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH; rss_context = ptr; rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 | (rss_map->base_qpn)); diff --git a/sys/ofed/drivers/net/mlx4/en_selftest.c b/sys/ofed/drivers/net/mlx4/en_selftest.c deleted file mode 100644 index fb13bd6..0000000 --- a/sys/ofed/drivers/net/mlx4/en_selftest.c +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include <linux/kernel.h> -#include <linux/netdevice.h> -#include <linux/delay.h> -#include <linux/mlx4/driver.h> - -#include "mlx4_en.h" - - -static int mlx4_en_test_registers(struct mlx4_en_priv *priv) -{ - return mlx4_cmd(priv->mdev->dev, 0, 0, 0, MLX4_CMD_HW_HEALTH_CHECK, - MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); -} - -static int mlx4_en_test_loopback_xmit(struct mlx4_en_priv *priv) -{ - struct sk_buff *skb; - struct ethhdr *ethh; - unsigned char *packet; - unsigned int packet_size = MLX4_LOOPBACK_TEST_PAYLOAD; - unsigned int i; - int err; - - - /* build the pkt before xmit */ - skb = netdev_alloc_skb(priv->dev, MLX4_LOOPBACK_TEST_PAYLOAD + ETH_HLEN + NET_IP_ALIGN); - if (!skb) { - en_err(priv, "-LOOPBACK_TEST_XMIT- failed to create skb for xmit\n"); - return -ENOMEM; - } - skb_reserve(skb, NET_IP_ALIGN); - - ethh = (struct ethhdr *)skb_put(skb, sizeof(struct ethhdr)); - packet = (unsigned char *)skb_put(skb, packet_size); - memcpy(ethh->h_dest, priv->dev->dev_addr, ETH_ALEN); - memset(ethh->h_source, 0, ETH_ALEN); - ethh->h_proto = htons(ETH_P_ARP); - skb_set_mac_header(skb, 0); - for (i = 0; i < packet_size; ++i) /* fill our packet */ - packet[i] = (unsigned char)(i & 0xff); - - /* xmit the pkt */ - err = mlx4_en_xmit(skb, priv->dev); - return err; -} - -static int mlx4_en_test_loopback(struct mlx4_en_priv *priv) -{ - u32 loopback_ok = 0; - int i; - - - priv->loopback_ok = 0; - priv->validate_loopback = 1; - - mlx4_en_update_loopback_state(priv->dev, priv->dev->features); - - /* xmit */ - if (mlx4_en_test_loopback_xmit(priv)) { - en_err(priv, "Transmitting loopback packet failed\n"); - goto mlx4_en_test_loopback_exit; - } - - /* polling for result */ - for (i = 0; i < MLX4_EN_LOOPBACK_RETRIES; ++i) { - msleep(MLX4_EN_LOOPBACK_TIMEOUT); - if (priv->loopback_ok) { - loopback_ok = 1; - break; - } - } - if (!loopback_ok) - en_err(priv, "Loopback packet didn't arrive\n"); - -mlx4_en_test_loopback_exit: - - priv->validate_loopback = 0; - mlx4_en_update_loopback_state(priv->dev, priv->dev->features); - return !loopback_ok; -} - - -static int mlx4_en_test_link(struct mlx4_en_priv *priv) -{ - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return -ENOMEM; - if (priv->port_state.link_state == 1) - return 0; - else - return 1; -} - -static int mlx4_en_test_speed(struct mlx4_en_priv *priv) -{ - - if (mlx4_en_QUERY_PORT(priv->mdev, priv->port)) - return -ENOMEM; - - /* The device supports 1G, 10G and 40G speed */ - if (priv->port_state.link_speed != MLX4_EN_LINK_SPEED_1G && - priv->port_state.link_speed != MLX4_EN_LINK_SPEED_10G && - priv->port_state.link_speed != MLX4_EN_LINK_SPEED_40G) - return priv->port_state.link_speed; - return 0; -} - - -void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf) -{ - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - int i, carrier_ok; - - memset(buf, 0, sizeof(u64) * MLX4_EN_NUM_SELF_TEST); - - if (*flags & ETH_TEST_FL_OFFLINE) { - /* disable the interface */ - carrier_ok = netif_carrier_ok(dev); - - netif_carrier_off(dev); - /* Wait until all tx queues are empty. - * there should not be any additional incoming traffic - * since we turned the carrier off */ - msleep(200); - - if (priv->mdev->dev->caps.flags & - MLX4_DEV_CAP_FLAG_UC_LOOPBACK) { - buf[3] = mlx4_en_test_registers(priv); - if (priv->port_up) - buf[4] = mlx4_en_test_loopback(priv); - } - - if (carrier_ok) - netif_carrier_on(dev); - - } - buf[0] = mlx4_test_interrupts(mdev->dev); - buf[1] = mlx4_en_test_link(priv); - buf[2] = mlx4_en_test_speed(priv); - - for (i = 0; i < MLX4_EN_NUM_SELF_TEST; i++) { - if (buf[i]) - *flags |= ETH_TEST_FL_FAILED; - } -} diff --git a/sys/ofed/drivers/net/mlx4/en_tx.c b/sys/ofed/drivers/net/mlx4/en_tx.c index 115dd0b..c147dd3 100644 --- a/sys/ofed/drivers/net/mlx4/en_tx.c +++ b/sys/ofed/drivers/net/mlx4/en_tx.c @@ -68,6 +68,7 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_ring *ring; + uint32_t x; int tmp; int err; @@ -80,11 +81,26 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, } } + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MLX4_EN_TX_MAX_PAYLOAD_SIZE, /* maxsize */ + MLX4_EN_TX_MAX_MBUF_FRAGS, /* nsegments */ + MLX4_EN_TX_MAX_MBUF_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &ring->dma_tag))) + goto done; + ring->size = size; ring->size_mask = size - 1; ring->stride = stride; - ring->full_size = ring->size - HEADROOM - MAX_DESC_TXBBS; - ring->inline_thold = min(inline_thold, MAX_INLINE); + ring->inline_thold = MAX(MIN_PKT_LEN, MIN(inline_thold, MAX_INLINE)); mtx_init(&ring->tx_lock.m, "mlx4 tx", NULL, MTX_DEF); mtx_init(&ring->comp_lock.m, "mlx4 comp", NULL, MTX_DEF); @@ -93,30 +109,36 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, M_WAITOK, &ring->tx_lock.m); if (ring->br == NULL) { en_err(priv, "Failed allocating tx_info ring\n"); - return -ENOMEM; + err = -ENOMEM; + goto err_free_dma_tag; } tmp = size * sizeof(struct mlx4_en_tx_info); - ring->tx_info = vmalloc_node(tmp, node); + ring->tx_info = kzalloc_node(tmp, GFP_KERNEL, node); if (!ring->tx_info) { - ring->tx_info = vmalloc(tmp); + ring->tx_info = kzalloc(tmp, GFP_KERNEL); if (!ring->tx_info) { err = -ENOMEM; goto err_ring; } } - en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", - ring->tx_info, tmp); - - ring->bounce_buf = kmalloc_node(MAX_DESC_SIZE, GFP_KERNEL, node); - if (!ring->bounce_buf) { - ring->bounce_buf = kmalloc(MAX_DESC_SIZE, GFP_KERNEL); - if (!ring->bounce_buf) { - err = -ENOMEM; + /* Create DMA descriptor MAPs */ + for (x = 0; x != size; x++) { + err = -bus_dmamap_create(ring->dma_tag, 0, + &ring->tx_info[x].dma_map); + if (err != 0) { + while (x--) { + bus_dmamap_destroy(ring->dma_tag, + ring->tx_info[x].dma_map); + } goto err_info; } } + + en_dbg(DRV, priv, "Allocated tx_info ring at addr:%p size:%d\n", + ring->tx_info, tmp); + ring->buf_size = ALIGN(size * ring->stride, MLX4_EN_PAGE_SIZE); /* Allocate HW buffers on provided NUMA node */ @@ -124,7 +146,7 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, 2 * PAGE_SIZE); if (err) { en_err(priv, "Failed allocating hwq resources\n"); - goto err_bounce; + goto err_dma_map; } err = mlx4_en_map_buffer(&ring->wqres.buf); @@ -174,12 +196,16 @@ err_map: mlx4_en_unmap_buffer(&ring->wqres.buf); err_hwq_res: mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); -err_bounce: - kfree(ring->bounce_buf); +err_dma_map: + for (x = 0; x != size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); err_info: vfree(ring->tx_info); err_ring: buf_ring_free(ring->br, M_DEVBUF); +err_free_dma_tag: + bus_dma_tag_destroy(ring->dma_tag); +done: kfree(ring); return err; } @@ -189,6 +215,7 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, { struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_ring *ring = *pring; + uint32_t x; en_dbg(DRV, priv, "Destroying tx ring, qpn: %d\n", ring->qpn); buf_ring_free(ring->br, M_DEVBUF); @@ -199,10 +226,12 @@ void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, mlx4_qp_release_range(priv->mdev->dev, ring->qpn, 1); mlx4_en_unmap_buffer(&ring->wqres.buf); mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size); - kfree(ring->bounce_buf); + for (x = 0; x != ring->size; x++) + bus_dmamap_destroy(ring->dma_tag, ring->tx_info[x].dma_map); vfree(ring->tx_info); mtx_destroy(&ring->tx_lock.m); mtx_destroy(&ring->comp_lock.m); + bus_dma_tag_destroy(ring->dma_tag); kfree(ring); *pring = NULL; } @@ -220,7 +249,6 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, ring->last_nr_txbb = 1; ring->poll_cnt = 0; ring->blocked = 0; - memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info)); memset(ring->buf, 0, ring->buf_size); ring->qp_state = MLX4_QP_STATE_RST; @@ -245,96 +273,63 @@ void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, MLX4_QP_STATE_RST, NULL, 0, 0, &ring->qp); } -static void mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - int index, u8 owner) +static volatile struct mlx4_wqe_data_seg * +mlx4_en_store_inline_lso_data(volatile struct mlx4_wqe_data_seg *dseg, + struct mbuf *mb, int len, __be32 owner_bit) { - struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; - struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; - void *end = ring->buf + ring->buf_size; - __be32 *ptr = (__be32 *)tx_desc; - __be32 stamp = cpu_to_be32(STAMP_VAL | (!!owner << STAMP_SHIFT)); - int i; - - /* Optimize the common case when there are no wraparounds */ - if (likely((void *)tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) - /* Stamp the freed descriptor */ - for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { - *ptr = stamp; - ptr += STAMP_DWORDS; - } - else - /* Stamp the freed descriptor */ - for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { - *ptr = stamp; - ptr += STAMP_DWORDS; - if ((void *)ptr >= end) { - ptr = ring->buf; - stamp ^= cpu_to_be32(0x80000000); - } - } + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + + /* copy data into place */ + m_copydata(mb, 0, len, inl + 4); + dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); + return (dseg); } -static u32 mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - int index, u8 owner, u64 timestamp) +static void +mlx4_en_store_inline_lso_header(volatile struct mlx4_wqe_data_seg *dseg, + int len, __be32 owner_bit) +{ +} + +static void +mlx4_en_stamp_wqe(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, u32 index, u8 owner) { - struct mlx4_en_dev *mdev = priv->mdev; struct mlx4_en_tx_info *tx_info = &ring->tx_info[index]; - struct mlx4_en_tx_desc *tx_desc = ring->buf + index * TXBB_SIZE; - struct mlx4_wqe_data_seg *data = (void *) tx_desc + tx_info->data_offset; - struct mbuf *mb = tx_info->mb; - void *end = ring->buf + ring->buf_size; - int frags = tx_info->nr_segs;; - int i; - - /* Optimize the common case when there are no wraparounds */ - if (likely((void *) tx_desc + tx_info->nr_txbb * TXBB_SIZE <= end)) { - if (!tx_info->inl) { - if (tx_info->linear) { - dma_unmap_single(priv->ddev, - (dma_addr_t) be64_to_cpu(data->addr), - be32_to_cpu(data->byte_count), - PCI_DMA_TODEVICE); - ++data; - } + struct mlx4_en_tx_desc *tx_desc = (struct mlx4_en_tx_desc *) + (ring->buf + (index * TXBB_SIZE)); + volatile __be32 *ptr = (__be32 *)tx_desc; + const __be32 stamp = cpu_to_be32(STAMP_VAL | + ((u32)owner << STAMP_SHIFT)); + u32 i; + + /* Stamp the freed descriptor */ + for (i = 0; i < tx_info->nr_txbb * TXBB_SIZE; i += STAMP_STRIDE) { + *ptr = stamp; + ptr += STAMP_DWORDS; + } +} - for (i = 0; i < frags; i++) { - pci_unmap_single(mdev->pdev, - (dma_addr_t) be64_to_cpu(data[i].addr), - data[i].byte_count, PCI_DMA_TODEVICE); - } - } - } else { - if (!tx_info->inl) { - if ((void *) data >= end) { - data = ring->buf + ((void *)data - end); - } +static u32 +mlx4_en_free_tx_desc(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring *ring, u32 index) +{ + struct mlx4_en_tx_info *tx_info; + struct mbuf *mb; - if (tx_info->linear) { - dma_unmap_single(priv->ddev, - (dma_addr_t) be64_to_cpu(data->addr), - be32_to_cpu(data->byte_count), - PCI_DMA_TODEVICE); - ++data; - } + tx_info = &ring->tx_info[index]; + mb = tx_info->mb; + + if (mb == NULL) + goto done; + + bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(ring->dma_tag, tx_info->dma_map); - for (i = 0; i < frags; i++) { - /* Check for wraparound before unmapping */ - if ((void *) data >= end) - data = ring->buf; - pci_unmap_single(mdev->pdev, - (dma_addr_t) be64_to_cpu(data->addr), - data->byte_count, PCI_DMA_TODEVICE); - ++data; - } - } - } - /* Send a copy of the frame to the BPF listener */ - if (priv->dev && priv->dev->if_bpf) - ETHER_BPF_MTAP(priv->dev, mb); m_freem(mb); - return tx_info->nr_txbb; +done: + return (tx_info->nr_txbb); } int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) @@ -354,8 +349,7 @@ int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) while (ring->cons != ring->prod) { ring->last_nr_txbb = mlx4_en_free_tx_desc(priv, ring, - ring->cons & ring->size_mask, - !!(ring->cons & ring->size), 0); + ring->cons & ring->size_mask); ring->cons += ring->last_nr_txbb; cnt++; } @@ -366,6 +360,14 @@ int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring) return cnt; } +static bool +mlx4_en_tx_ring_is_full(struct mlx4_en_tx_ring *ring) +{ + int wqs; + wqs = ring->size - (ring->prod - ring->cons); + return (wqs < (HEADROOM + (2 * MLX4_EN_TX_WQE_MAX_WQEBBS))); +} + static int mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq) { @@ -381,12 +383,7 @@ static int mlx4_en_process_tx_cq(struct net_device *dev, int size = cq->size; u32 size_mask = ring->size_mask; struct mlx4_cqe *buf = cq->buf; - u32 packets = 0; - u32 bytes = 0; int factor = priv->cqe_factor; - u64 timestamp = 0; - int done = 0; - if (!priv->port_up) return 0; @@ -421,16 +418,12 @@ static int mlx4_en_process_tx_cq(struct net_device *dev, ring_index = (ring_index + ring->last_nr_txbb) & size_mask; /* free next descriptor */ ring->last_nr_txbb = mlx4_en_free_tx_desc( - priv, ring, ring_index, - !!((ring->cons + txbbs_skipped) & - ring->size), timestamp); + priv, ring, ring_index); mlx4_en_stamp_wqe(priv, ring, stamp_index, !!((ring->cons + txbbs_stamp) & ring->size)); stamp_index = ring_index; txbbs_stamp = txbbs_skipped; - packets++; - bytes += ring->tx_info[ring_index].nr_bytes; } while (ring_index != new_index); ++cons_index; @@ -449,15 +442,14 @@ static int mlx4_en_process_tx_cq(struct net_device *dev, ring->cons += txbbs_skipped; /* Wakeup Tx queue if it was stopped and ring is not full */ - if (unlikely(ring->blocked) && - (ring->prod - ring->cons) <= ring->full_size) { + if (unlikely(ring->blocked) && !mlx4_en_tx_ring_is_full(ring)) { ring->blocked = 0; if (atomic_fetchadd_int(&priv->blocked, -1) == 1) atomic_clear_int(&dev->if_drv_flags ,IFF_DRV_OACTIVE); ring->wake_queue++; priv->port_stats.wake_queue++; } - return done; + return (0); } void mlx4_en_tx_irq(struct mlx4_cq *mcq) @@ -498,34 +490,6 @@ void mlx4_en_poll_tx_cq(unsigned long data) spin_unlock(&ring->comp_lock); } -static struct mlx4_en_tx_desc *mlx4_en_bounce_to_desc(struct mlx4_en_priv *priv, - struct mlx4_en_tx_ring *ring, - u32 index, - unsigned int desc_size) -{ - u32 copy = (ring->size - index) * TXBB_SIZE; - int i; - - for (i = desc_size - copy - 4; i >= 0; i -= 4) { - if ((i & (TXBB_SIZE - 1)) == 0) - wmb(); - - *((u32 *) (ring->buf + i)) = - *((u32 *) (ring->bounce_buf + copy + i)); - } - - for (i = copy - 4; i >= 4 ; i -= 4) { - if ((i & (TXBB_SIZE - 1)) == 0) - wmb(); - - *((u32 *) (ring->buf + index * TXBB_SIZE + i)) = - *((u32 *) (ring->bounce_buf + i)); - } - - /* Return real descriptor location */ - return ring->buf + index * TXBB_SIZE; -} - static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) { struct mlx4_en_cq *cq = priv->tx_cq[tx_ind]; @@ -544,30 +508,22 @@ static inline void mlx4_en_xmit_poll(struct mlx4_en_priv *priv, int tx_ind) } } -static int is_inline(struct mbuf *mb, int thold) +static u16 +mlx4_en_get_inline_hdr_size(struct mlx4_en_tx_ring *ring, struct mbuf *mb) { - if (thold && mb->m_pkthdr.len <= thold && - (mb->m_pkthdr.csum_flags & CSUM_TSO) == 0) - return 1; + u16 retval; - return 0; -} - -static int inline_size(struct mbuf *mb) -{ - int len; + /* only copy from first fragment, if possible */ + retval = MIN(ring->inline_thold, mb->m_len); - len = mb->m_pkthdr.len; - if (len + CTRL_SIZE + sizeof(struct mlx4_wqe_inline_seg) - <= MLX4_INLINE_ALIGN) - return ALIGN(len + CTRL_SIZE + - sizeof(struct mlx4_wqe_inline_seg), 16); - else - return ALIGN(len + CTRL_SIZE + 2 * - sizeof(struct mlx4_wqe_inline_seg), 16); + /* check for too little data */ + if (unlikely(retval < MIN_PKT_LEN)) + retval = MIN(ring->inline_thold, mb->m_pkthdr.len); + return (retval); } -static int get_head_size(struct mbuf *mb) +static int +mlx4_en_get_header_size(struct mbuf *mb) { struct ether_vlan_header *eh; struct tcphdr *th; @@ -620,83 +576,48 @@ static int get_head_size(struct mbuf *mb) return (eth_hdr_len); } -static int get_real_size(struct mbuf *mb, struct net_device *dev, int *p_n_segs, - int *lso_header_size, int inl) +static volatile struct mlx4_wqe_data_seg * +mlx4_en_store_inline_data(volatile struct mlx4_wqe_data_seg *dseg, + struct mbuf *mb, int len, __be32 owner_bit) { - struct mbuf *m; - int nr_segs = 0; - - for (m = mb; m != NULL; m = m->m_next) - if (m->m_len) - nr_segs++; - - if (mb->m_pkthdr.csum_flags & CSUM_TSO) { - *lso_header_size = get_head_size(mb); - if (*lso_header_size) { - if (mb->m_len == *lso_header_size) - nr_segs--; - *p_n_segs = nr_segs; - return CTRL_SIZE + nr_segs * DS_SIZE + - ALIGN(*lso_header_size + 4, DS_SIZE); - } - } else - *lso_header_size = 0; - *p_n_segs = nr_segs; - if (inl) - return inline_size(mb); - return (CTRL_SIZE + nr_segs * DS_SIZE); -} - -static struct mbuf *mb_copy(struct mbuf *mb, int *offp, char *data, int len) -{ - int bytes; - int off; - - off = *offp; - while (len) { - bytes = min(mb->m_len - off, len); - if (bytes) - memcpy(data, mb->m_data + off, bytes); - len -= bytes; - data += bytes; - off += bytes; - if (off == mb->m_len) { - off = 0; - mb = mb->m_next; - } - } - *offp = off; - return (mb); + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; + + if (unlikely(len < MIN_PKT_LEN)) { + m_copydata(mb, 0, len, inl + 4); + memset(inl + 4 + len, 0, MIN_PKT_LEN - len); + dseg += DIV_ROUND_UP(4 + MIN_PKT_LEN, DS_SIZE_ALIGNMENT); + } else if (len <= spc) { + m_copydata(mb, 0, len, inl + 4); + dseg += DIV_ROUND_UP(4 + len, DS_SIZE_ALIGNMENT); + } else { + m_copydata(mb, 0, spc, inl + 4); + m_copydata(mb, spc, len - spc, inl + 8 + spc); + dseg += DIV_ROUND_UP(8 + len, DS_SIZE_ALIGNMENT); + } + return (dseg); } -static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc, struct mbuf *mb, - int real_size, u16 *vlan_tag, int tx_ind) +static void +mlx4_en_store_inline_header(volatile struct mlx4_wqe_data_seg *dseg, + int len, __be32 owner_bit) { - struct mlx4_wqe_inline_seg *inl = &tx_desc->inl; - int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl; - int len; - int off; - - off = 0; - len = mb->m_pkthdr.len; - if (len <= spc) { - inl->byte_count = cpu_to_be32(1 << 31 | - (max_t(typeof(len), len, MIN_PKT_LEN))); - mb_copy(mb, &off, (void *)(inl + 1), len); - if (len < MIN_PKT_LEN) - memset(((void *)(inl + 1)) + len, 0, - MIN_PKT_LEN - len); + uint8_t *inl = __DEVOLATILE(uint8_t *, dseg); + const int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - 4; + + if (unlikely(len < MIN_PKT_LEN)) { + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | MIN_PKT_LEN); + } else if (len <= spc) { + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | len); } else { - inl->byte_count = cpu_to_be32(1 << 31 | spc); - mb = mb_copy(mb, &off, (void *)(inl + 1), spc); - inl = (void *) (inl + 1) + spc; - mb_copy(mb, &off, (void *)(inl + 1), len - spc); + *(volatile uint32_t *)(inl + 4 + spc) = + SET_BYTE_COUNT((1 << 31) | (len - spc)); wmb(); - inl->byte_count = cpu_to_be32(1 << 31 | (len - spc)); + *(volatile uint32_t *)inl = + SET_BYTE_COUNT((1 << 31) | spc); } - tx_desc->ctrl.vlan_tag = cpu_to_be16(*vlan_tag); - tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * !!(*vlan_tag); - tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; } static unsigned long hashrandom; @@ -720,18 +641,14 @@ u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb) up = (vlan_tag >> 13) % MLX4_EN_NUM_UP; } #endif - /* check if flowid is set */ - if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) - queue_index = mb->m_pkthdr.flowid; - else - queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); + queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); return ((queue_index % rings_p_up) + (up * rings_p_up)); } -static void mlx4_bf_copy(void __iomem *dst, unsigned long *src, unsigned bytecnt) +static void mlx4_bf_copy(void __iomem *dst, volatile unsigned long *src, unsigned bytecnt) { - __iowrite64_copy(dst, src, bytecnt / 8); + __iowrite64_copy(dst, __DEVOLATILE(void *, src), bytecnt / 8); } static u64 mlx4_en_mac_to_u64(u8 *addr) @@ -746,168 +663,263 @@ static u64 mlx4_en_mac_to_u64(u8 *addr) return mac; } -static int mlx4_en_xmit(struct net_device *dev, int tx_ind, struct mbuf **mbp) +static int mlx4_en_xmit(struct mlx4_en_priv *priv, int tx_ind, struct mbuf **mbp) { - struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; - struct mlx4_en_tx_ring *ring; - struct mlx4_en_cq *cq; - struct mlx4_en_tx_desc *tx_desc; - struct mlx4_wqe_data_seg *data; + enum { + DS_FACT = TXBB_SIZE / DS_SIZE_ALIGNMENT, + CTRL_FLAGS = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | + MLX4_WQE_CTRL_SOLICITED), + }; + bus_dma_segment_t segs[MLX4_EN_TX_MAX_MBUF_FRAGS]; + volatile struct mlx4_wqe_data_seg *dseg; + volatile struct mlx4_wqe_data_seg *dseg_inline; + volatile struct mlx4_en_tx_desc *tx_desc; + struct mlx4_en_tx_ring *ring = priv->tx_ring[tx_ind]; + struct ifnet *ifp = priv->dev; struct mlx4_en_tx_info *tx_info; + struct mbuf *mb = *mbp; struct mbuf *m; - int nr_txbb; + __be32 owner_bit; int nr_segs; - int desc_size; - int real_size; - dma_addr_t dma; - u32 index, bf_index, ring_size; - __be32 op_own; - u16 vlan_tag = 0; - int i; - int lso_header_size; - bool bounce = false; - bool inl = false; - struct mbuf *mb; - mb = *mbp; - int defrag = 1; - - if (!priv->port_up) - goto tx_drop; - - ring = priv->tx_ring[tx_ind]; - ring_size = ring->size; - inl = is_inline(mb, ring->inline_thold); - -retry: - real_size = get_real_size(mb, dev, &nr_segs, &lso_header_size, inl); - if (unlikely(!real_size)) - goto tx_drop; + int pad; + int err; + u32 bf_size; + u32 bf_prod; + u32 opcode; + u16 index; + u16 ds_cnt; + u16 ihs; - /* Align descriptor to TXBB size */ - desc_size = ALIGN(real_size, TXBB_SIZE); - nr_txbb = desc_size / TXBB_SIZE; - if (unlikely(nr_txbb > MAX_DESC_TXBBS)) { - if (defrag) { - mb = m_defrag(*mbp, M_NOWAIT); - if (mb == NULL) { - mb = *mbp; - goto tx_drop; - } - *mbp = mb; - defrag = 0; - goto retry; - } - en_warn(priv, "Oversized header or SG list\n"); + if (unlikely(!priv->port_up)) { + err = EINVAL; goto tx_drop; } - /* Obtain VLAN information if present */ - if (mb->m_flags & M_VLANTAG) { - vlan_tag = mb->m_pkthdr.ether_vtag; - } - - /* Check available TXBBs and 2K spare for prefetch - * Even if netif_tx_stop_queue() will be called - * driver will send current packet to ensure - * that at least one completion will be issued after - * stopping the queue - */ - if (unlikely((int)(ring->prod - ring->cons) > ring->full_size)) { - /* every full Tx ring stops queue */ - if (ring->blocked == 0) - atomic_add_int(&priv->blocked, 1); - /* Set HW-queue-is-full flag */ - atomic_set_int(&dev->if_drv_flags, IFF_DRV_OACTIVE); + /* check if TX ring is full */ + if (unlikely(mlx4_en_tx_ring_is_full(ring))) { + /* every full native Tx ring stops queue */ + if (ring->blocked == 0) + atomic_add_int(&priv->blocked, 1); + /* Set HW-queue-is-full flag */ + atomic_set_int(&ifp->if_drv_flags, IFF_DRV_OACTIVE); + priv->port_stats.queue_stopped++; ring->blocked = 1; priv->port_stats.queue_stopped++; ring->queue_stopped++; /* Use interrupts to find out when queue opened */ - cq = priv->tx_cq[tx_ind]; - mlx4_en_arm_cq(priv, cq); - return EBUSY; + mlx4_en_arm_cq(priv, priv->tx_cq[tx_ind]); + return (ENOBUFS); } + /* sanity check we are not wrapping around */ + KASSERT(((~ring->prod) & ring->size_mask) >= + (MLX4_EN_TX_WQE_MAX_WQEBBS - 1), ("Wrapping around TX ring")); + /* Track current inflight packets for performance analysis */ AVG_PERF_COUNTER(priv->pstats.inflight_avg, (u32) (ring->prod - ring->cons - 1)); - /* Packet is good - grab an index and transmit it */ + /* Track current mbuf packet header length */ + AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); + + /* Grab an index and try to transmit packet */ + owner_bit = (ring->prod & ring->size) ? + cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0; index = ring->prod & ring->size_mask; - bf_index = ring->prod; + tx_desc = (volatile struct mlx4_en_tx_desc *) + (ring->buf + index * TXBB_SIZE); + tx_info = &ring->tx_info[index]; + dseg = &tx_desc->data; - /* See if we have enough space for whole descriptor TXBB for setting - * SW ownership on next descriptor; if not, use a bounce buffer. */ - if (likely(index + nr_txbb <= ring_size)) - tx_desc = ring->buf + index * TXBB_SIZE; - else { - tx_desc = (struct mlx4_en_tx_desc *) ring->bounce_buf; - bounce = true; + /* send a copy of the frame to the BPF listener, if any */ + if (ifp != NULL && ifp->if_bpf != NULL) + ETHER_BPF_MTAP(ifp, mb); + + /* get default flags */ + tx_desc->ctrl.srcrb_flags = CTRL_FLAGS; + + if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); + + if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | + CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) + tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); + + /* do statistics */ + if (likely(tx_desc->ctrl.srcrb_flags != CTRL_FLAGS)) { + priv->port_stats.tx_chksum_offload++; + ring->tx_csum++; } - /* Save mb in tx_info ring */ - tx_info = &ring->tx_info[index]; - tx_info->mb = mb; - tx_info->nr_txbb = nr_txbb; - tx_info->nr_segs = nr_segs; - - if (lso_header_size) { - memcpy(tx_desc->lso.header, mb->m_data, lso_header_size); - data = ((void *)&tx_desc->lso + ALIGN(lso_header_size + 4, - DS_SIZE)); - /* lso header is part of m_data. - * need to omit when mapping DMA */ - mb->m_data += lso_header_size; - mb->m_len -= lso_header_size; + /* check for VLAN tag */ + if (mb->m_flags & M_VLANTAG) { + tx_desc->ctrl.vlan_tag = cpu_to_be16(mb->m_pkthdr.ether_vtag); + tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN; + } else { + tx_desc->ctrl.vlan_tag = 0; + tx_desc->ctrl.ins_vlan = 0; } - else - data = &tx_desc->data; - /* valid only for none inline segments */ - tx_info->data_offset = (void *)data - (void *)tx_desc; + /* clear immediate field */ + tx_desc->ctrl.imm = 0; - if (inl) { - tx_info->inl = 1; + /* Handle LSO (TSO) packets */ + if (mb->m_pkthdr.csum_flags & CSUM_TSO) { + u32 payload_len; + u32 mss = mb->m_pkthdr.tso_segsz; + u32 num_pkts; + + opcode = cpu_to_be32(MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR) | + owner_bit; + ihs = mlx4_en_get_header_size(mb); + if (unlikely(ihs > MAX_INLINE)) { + ring->oversized_packets++; + err = EINVAL; + goto tx_drop; + } + tx_desc->lso.mss_hdr_size = cpu_to_be32((mss << 16) | ihs); + payload_len = mb->m_pkthdr.len - ihs; + if (unlikely(payload_len == 0)) + num_pkts = 1; + else + num_pkts = DIV_ROUND_UP(payload_len, mss); + ring->bytes += payload_len + (num_pkts * ihs); + ring->packets += num_pkts; + priv->port_stats.tso_packets++; + /* store pointer to inline header */ + dseg_inline = dseg; + /* copy data inline */ + dseg = mlx4_en_store_inline_lso_data(dseg, + mb, ihs, owner_bit); } else { - for (i = 0, m = mb; i < nr_segs; i++, m = m->m_next) { - if (m->m_len == 0) { - i--; - continue; - } - dma = pci_map_single(mdev->dev->pdev, m->m_data, - m->m_len, PCI_DMA_TODEVICE); - data->addr = cpu_to_be64(dma); - data->lkey = cpu_to_be32(mdev->mr.key); - wmb(); - data->byte_count = cpu_to_be32(m->m_len); - data++; - } - if (lso_header_size) { - mb->m_data -= lso_header_size; - mb->m_len += lso_header_size; - } - tx_info->inl = 0; + opcode = cpu_to_be32(MLX4_OPCODE_SEND) | + owner_bit; + ihs = mlx4_en_get_inline_hdr_size(ring, mb); + ring->bytes += max_t (unsigned int, + mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); + ring->packets++; + /* store pointer to inline header */ + dseg_inline = dseg; + /* copy data inline */ + dseg = mlx4_en_store_inline_data(dseg, + mb, ihs, owner_bit); + } + m_adj(mb, ihs); + + /* trim off empty mbufs */ + while (mb->m_len == 0) { + mb = m_free(mb); + /* check if all data has been inlined */ + if (mb == NULL) { + nr_segs = 0; + goto skip_dma; + } } + err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, + mb, segs, &nr_segs, BUS_DMA_NOWAIT); + if (unlikely(err == EFBIG)) { + /* Too many mbuf fragments */ + m = m_defrag(mb, M_NOWAIT); + if (m == NULL) { + ring->oversized_packets++; + goto tx_drop; + } + mb = m; + /* Try again */ + err = bus_dmamap_load_mbuf_sg(ring->dma_tag, tx_info->dma_map, + mb, segs, &nr_segs, BUS_DMA_NOWAIT); + } + /* catch errors */ + if (unlikely(err != 0)) { + ring->oversized_packets++; + goto tx_drop; + } + /* make sure all mbuf data is written to RAM */ + bus_dmamap_sync(ring->dma_tag, tx_info->dma_map, + BUS_DMASYNC_PREWRITE); - /* Prepare ctrl segement apart opcode+ownership, which depends on - * whether LSO is used */ - tx_desc->ctrl.vlan_tag = cpu_to_be16(vlan_tag); - tx_desc->ctrl.ins_vlan = MLX4_WQE_CTRL_INS_VLAN * - !!vlan_tag; - tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; - tx_desc->ctrl.srcrb_flags = priv->ctrl_flags; - if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO | - CSUM_TCP | CSUM_UDP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) { - if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); - if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | - CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_TCP_UDP_CSUM); - priv->port_stats.tx_chksum_offload++; - ring->tx_csum++; - } +skip_dma: + /* compute number of DS needed */ + ds_cnt = (dseg - ((volatile struct mlx4_wqe_data_seg *)tx_desc)) + nr_segs; + + /* + * Check if the next request can wrap around and fill the end + * of the current request with zero immediate data: + */ + pad = DIV_ROUND_UP(ds_cnt, DS_FACT); + pad = (~(ring->prod + pad)) & ring->size_mask; + + if (unlikely(pad < (MLX4_EN_TX_WQE_MAX_WQEBBS - 1))) { + /* + * Compute the least number of DS blocks we need to + * pad in order to achieve a TX ring wraparound: + */ + pad = (DS_FACT * (pad + 1)); + } else { + /* + * The hardware will automatically jump to the next + * TXBB. No need for padding. + */ + pad = 0; + } + + /* compute total number of DS blocks */ + ds_cnt += pad; + /* + * When modifying this code, please ensure that the following + * computation is always less than or equal to 0x3F: + * + * ((MLX4_EN_TX_WQE_MAX_WQEBBS - 1) * DS_FACT) + + * (MLX4_EN_TX_WQE_MAX_WQEBBS * DS_FACT) + * + * Else the "ds_cnt" variable can become too big. + */ + tx_desc->ctrl.fence_size = (ds_cnt & 0x3f); + + /* store pointer to mbuf */ + tx_info->mb = mb; + tx_info->nr_txbb = DIV_ROUND_UP(ds_cnt, DS_FACT); + bf_size = ds_cnt * DS_SIZE_ALIGNMENT; + bf_prod = ring->prod; + + /* compute end of "dseg" array */ + dseg += nr_segs + pad; + + /* pad using zero immediate dseg */ + while (pad--) { + dseg--; + dseg->addr = 0; + dseg->lkey = 0; + wmb(); + dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); + } + + /* fill segment list */ + while (nr_segs--) { + if (unlikely(segs[nr_segs].ds_len == 0)) { + dseg--; + dseg->addr = 0; + dseg->lkey = 0; + wmb(); + dseg->byte_count = SET_BYTE_COUNT((1 << 31)|0); + } else { + dseg--; + dseg->addr = cpu_to_be64((uint64_t)segs[nr_segs].ds_addr); + dseg->lkey = cpu_to_be32(priv->mdev->mr.key); + wmb(); + dseg->byte_count = SET_BYTE_COUNT((uint32_t)segs[nr_segs].ds_len); + } + } + + wmb(); + + /* write owner bits in reverse order */ + if ((opcode & cpu_to_be32(0x1F)) == cpu_to_be32(MLX4_OPCODE_LSO)) + mlx4_en_store_inline_lso_header(dseg_inline, ihs, owner_bit); + else + mlx4_en_store_inline_header(dseg_inline, ihs, owner_bit); if (unlikely(priv->validate_loopback)) { /* Copy dst mac address to wqe */ @@ -925,77 +937,46 @@ retry: } } - /* Handle LSO (TSO) packets */ - if (lso_header_size) { - int segsz; - /* Mark opcode as LSO */ - op_own = cpu_to_be32(MLX4_OPCODE_LSO | (1 << 6)) | - ((ring->prod & ring_size) ? - cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); - - /* Fill in the LSO prefix */ - tx_desc->lso.mss_hdr_size = cpu_to_be32( - mb->m_pkthdr.tso_segsz << 16 | lso_header_size); - - priv->port_stats.tso_packets++; - segsz = mb->m_pkthdr.tso_segsz; - i = ((mb->m_pkthdr.len - lso_header_size + segsz - 1) / segsz); - tx_info->nr_bytes= mb->m_pkthdr.len + (i - 1) * lso_header_size; - ring->packets += i; - } else { - /* Normal (Non LSO) packet */ - op_own = cpu_to_be32(MLX4_OPCODE_SEND) | - ((ring->prod & ring_size) ? - cpu_to_be32(MLX4_EN_BIT_DESC_OWN) : 0); - tx_info->nr_bytes = max(mb->m_pkthdr.len, - (unsigned int)ETHER_MIN_LEN - ETHER_CRC_LEN); - ring->packets++; - - } - ring->bytes += tx_info->nr_bytes; - AVG_PERF_COUNTER(priv->pstats.tx_pktsz_avg, mb->m_pkthdr.len); + /* update producer counter */ + ring->prod += tx_info->nr_txbb; - if (tx_info->inl) { - build_inline_wqe(tx_desc, mb, real_size, &vlan_tag, tx_ind); - tx_info->inl = 1; - } + if (ring->bf_enabled && bf_size <= MAX_BF && + (tx_desc->ctrl.ins_vlan != MLX4_WQE_CTRL_INS_VLAN)) { - ring->prod += nr_txbb; + /* store doorbell number */ + *(volatile __be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); + /* or in producer number for this WQE */ + opcode |= cpu_to_be32((bf_prod & 0xffff) << 8); - /* If we used a bounce buffer then copy descriptor back into place */ - if (unlikely(bounce)) - tx_desc = mlx4_en_bounce_to_desc(priv, ring, index, desc_size); - if (ring->bf_enabled && desc_size <= MAX_BF && !bounce && !vlan_tag) { - *(__be32 *) (&tx_desc->ctrl.vlan_tag) |= cpu_to_be32(ring->doorbell_qpn); - op_own |= htonl((bf_index & 0xffff) << 8); - /* Ensure new descirptor hits memory - * before setting ownership of this descriptor to HW */ + /* + * Ensure the new descriptor hits memory before + * setting ownership of this descriptor to HW: + */ wmb(); - tx_desc->ctrl.owner_opcode = op_own; - + tx_desc->ctrl.owner_opcode = opcode; wmb(); - - mlx4_bf_copy(ring->bf.reg + ring->bf.offset, (unsigned long *) &tx_desc->ctrl, - desc_size); - + mlx4_bf_copy(((u8 *)ring->bf.reg) + ring->bf.offset, + (volatile unsigned long *) &tx_desc->ctrl, bf_size); wmb(); - ring->bf.offset ^= ring->bf.buf_size; } else { - /* Ensure new descirptor hits memory - * before setting ownership of this descriptor to HW */ + /* + * Ensure the new descriptor hits memory before + * setting ownership of this descriptor to HW: + */ wmb(); - tx_desc->ctrl.owner_opcode = op_own; + tx_desc->ctrl.owner_opcode = opcode; wmb(); - writel(cpu_to_be32(ring->doorbell_qpn), ring->bf.uar->map + MLX4_SEND_DOORBELL); + writel(cpu_to_be32(ring->doorbell_qpn), + ((u8 *)ring->bf.uar->map) + MLX4_SEND_DOORBELL); } - return 0; + return (0); tx_drop: *mbp = NULL; m_freem(mb); - return EINVAL; + return (err); } static int @@ -1015,13 +996,16 @@ mlx4_en_transmit_locked(struct ifnet *dev, int tx_ind, struct mbuf *m) } enqueued = 0; - if (m != NULL) { - if ((err = drbr_enqueue(dev, ring->br, m)) != 0) - return (err); - } + if (m != NULL) + /* + * If we can't insert mbuf into drbr, try to xmit anyway. + * We keep the error we got so we could return that after xmit. + */ + err = drbr_enqueue(dev, ring->br, m); + /* Process the queue */ while ((next = drbr_peek(dev, ring->br)) != NULL) { - if ((err = mlx4_en_xmit(dev, tx_ind, &next)) != 0) { + if (mlx4_en_xmit(priv, tx_ind, &next) != 0) { if (next == NULL) { drbr_advance(dev, ring->br); } else { @@ -1072,10 +1056,14 @@ mlx4_en_transmit(struct ifnet *dev, struct mbuf *m) int i, err = 0; /* Compute which queue to use */ - i = mlx4_en_select_queue(dev, m); + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { + i = m->m_pkthdr.flowid % priv->tx_ring_num; + } + else { + i = mlx4_en_select_queue(dev, m); + } ring = priv->tx_ring[i]; - if (spin_trylock(&ring->tx_lock)) { err = mlx4_en_transmit_locked(dev, i, m); spin_unlock(&ring->tx_lock); diff --git a/sys/ofed/drivers/net/mlx4/main.c b/sys/ofed/drivers/net/mlx4/main.c index ddcd7d1..217220a 100644 --- a/sys/ofed/drivers/net/mlx4/main.c +++ b/sys/ofed/drivers/net/mlx4/main.c @@ -33,7 +33,7 @@ * SOFTWARE. */ -#include <linux/kmod.h> +#include <linux/kmod.h> /* * kmod.h must be included before module.h since it includes (indirectly) sys/module.h * To use the FBSD macro sys/module.h should define MODULE_VERSION before linux/module does. @@ -57,9 +57,7 @@ #include "icm.h" #include "mlx4_stats.h" -MODULE_AUTHOR("Roland Dreier"); -MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver"); -MODULE_LICENSE("Dual BSD/GPL"); +/* Mellanox ConnectX HCA low-level driver */ struct workqueue_struct *mlx4_wq; @@ -177,7 +175,7 @@ MODULE_PARM_DESC(enable_64b_cqe_eqe, #define PF_CONTEXT_BEHAVIOUR_MASK MLX4_FUNC_CAP_64B_EQE_CQE static char mlx4_version[] __devinitdata = - DRV_NAME ": Mellanox ConnectX core driver v" + DRV_NAME ": Mellanox ConnectX VPI driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; static int log_num_mac = 7; @@ -608,7 +606,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) if (dev_cap->min_page_sz > PAGE_SIZE) { mlx4_err(dev, "HCA minimum page size of %d bigger than " "kernel PAGE_SIZE of %d, aborting.\n", - dev_cap->min_page_sz, PAGE_SIZE); + dev_cap->min_page_sz, (int)PAGE_SIZE); return -ENODEV; } if (dev_cap->num_ports > MLX4_MAX_PORTS) { @@ -979,7 +977,7 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) if (page_size > PAGE_SIZE) { mlx4_err(dev, "HCA minimum page size of %d bigger than " "kernel PAGE_SIZE of %d, aborting.\n", - page_size, PAGE_SIZE); + page_size, (int)PAGE_SIZE); return -ENODEV; } @@ -989,7 +987,7 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) /* TODO: relax this assumption */ if (dev->caps.uar_page_size != PAGE_SIZE) { mlx4_err(dev, "UAR size:%d != kernel PAGE_SIZE of %d\n", - dev->caps.uar_page_size, PAGE_SIZE); + dev->caps.uar_page_size, (int)PAGE_SIZE); return -ENODEV; } @@ -1299,6 +1297,43 @@ static inline int ibta_mtu_to_int(enum ibta_mtu mtu) } } +static ssize_t +show_board(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, + board_attr); + struct mlx4_dev *mdev = info->dev; + + return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN, + mdev->board_id); +} + +static ssize_t +show_hca(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, + hca_attr); + struct mlx4_dev *mdev = info->dev; + + return sprintf(buf, "MT%d\n", mdev->pdev->device); +} + +static ssize_t +show_firmware_version(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mlx4_hca_info *info = container_of(attr, struct mlx4_hca_info, + firmware_attr); + struct mlx4_dev *mdev = info->dev; + + return sprintf(buf, "%d.%d.%d\n", (int)(mdev->caps.fw_ver >> 32), + (int)(mdev->caps.fw_ver >> 16) & 0xffff, + (int)mdev->caps.fw_ver & 0xffff); +} + static ssize_t show_port_ib_mtu(struct device *dev, struct device_attribute *attr, char *buf) @@ -2941,6 +2976,30 @@ no_msi: no_irq: dev->caps.num_comp_vectors = 0; dev->caps.comp_pool = 0; + return; +} + +static void +mlx4_init_hca_info(struct mlx4_dev *dev) +{ + struct mlx4_hca_info *info = &mlx4_priv(dev)->hca_info; + + info->dev = dev; + + info->firmware_attr = (struct device_attribute)__ATTR(fw_ver, S_IRUGO, + show_firmware_version, NULL); + if (device_create_file(&dev->pdev->dev, &info->firmware_attr)) + mlx4_err(dev, "Failed to add file firmware version"); + + info->hca_attr = (struct device_attribute)__ATTR(hca, S_IRUGO, show_hca, + NULL); + if (device_create_file(&dev->pdev->dev, &info->hca_attr)) + mlx4_err(dev, "Failed to add file hca type"); + + info->board_attr = (struct device_attribute)__ATTR(board_id, S_IRUGO, + show_board, NULL); + if (device_create_file(&dev->pdev->dev, &info->board_attr)) + mlx4_err(dev, "Failed to add file board id type"); } static int mlx4_init_port_info(struct mlx4_dev *dev, int port) @@ -2994,6 +3053,14 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port) return err; } +static void +mlx4_cleanup_hca_info(struct mlx4_hca_info *info) +{ + device_remove_file(&info->dev->pdev->dev, &info->firmware_attr); + device_remove_file(&info->dev->pdev->dev, &info->board_attr); + device_remove_file(&info->dev->pdev->dev, &info->hca_attr); +} + static void mlx4_cleanup_port_info(struct mlx4_port_info *info) { if (info->port < 0) @@ -3351,6 +3418,7 @@ slave_start: goto err_steer; mlx4_init_quotas(dev); + mlx4_init_hca_info(dev); for (port = 1; port <= dev->caps.num_ports; port++) { err = mlx4_init_port_info(dev, port); @@ -3443,8 +3511,7 @@ err_disable_pdev: static int __devinit mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id) { - printk_once(KERN_INFO "%s", mlx4_version); - + device_set_desc(pdev->dev.bsddev, mlx4_version); return __mlx4_init_one(pdev, id->driver_data); } @@ -3464,6 +3531,7 @@ static void mlx4_remove_one(struct pci_dev *pdev) mlx4_stop_sense(dev); mlx4_unregister_device(dev); + mlx4_cleanup_hca_info(&priv->hca_info); for (p = 1; p <= dev->caps.num_ports; p++) { mlx4_cleanup_port_info(&priv->port[p]); mlx4_CLOSE_PORT(dev, p); diff --git a/sys/ofed/drivers/net/mlx4/mlx4.h b/sys/ofed/drivers/net/mlx4/mlx4.h index 47fde25..bef2e00 100644 --- a/sys/ofed/drivers/net/mlx4/mlx4.h +++ b/sys/ofed/drivers/net/mlx4/mlx4.h @@ -51,7 +51,7 @@ #define DRV_NAME "mlx4_core" #define PFX DRV_NAME ": " -#define DRV_VERSION "2.1" +#define DRV_VERSION "2.1.6" #define DRV_RELDATE __DATE__ #define DRV_STACK_NAME "Linux-MLNX_OFED" @@ -755,6 +755,13 @@ struct mlx4_set_port_rqp_calc_context { __be32 mcast; }; +struct mlx4_hca_info { + struct mlx4_dev *dev; + struct device_attribute firmware_attr; + struct device_attribute hca_attr; + struct device_attribute board_attr; +}; + struct mlx4_port_info { struct mlx4_dev *dev; int port; @@ -845,6 +852,7 @@ struct mlx4_priv { struct mlx4_uar driver_uar; void __iomem *kar; struct mlx4_port_info port[MLX4_MAX_PORTS + 1]; + struct mlx4_hca_info hca_info; struct mlx4_sense sense; struct mutex port_mutex; struct mlx4_msix_ctl msix_ctl; diff --git a/sys/ofed/drivers/net/mlx4/mlx4_en.h b/sys/ofed/drivers/net/mlx4/mlx4_en.h index b96563d..c776b0e 100644 --- a/sys/ofed/drivers/net/mlx4/mlx4_en.h +++ b/sys/ofed/drivers/net/mlx4/mlx4_en.h @@ -59,8 +59,6 @@ #include "mlx4_stats.h" #define DRV_NAME "mlx4_en" -#define DRV_VERSION "2.1" -#define DRV_RELDATE __DATE__ #define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN) @@ -95,10 +93,6 @@ #define VLAN_MIN_VALUE 1 #define VLAN_MAX_VALUE 4094 -/* Typical TSO descriptor with 16 gather entries is 352 bytes... */ -#define MAX_DESC_SIZE 512 -#define MAX_DESC_TXBBS (MAX_DESC_SIZE / TXBB_SIZE) - /* * OS related constants and tunables */ @@ -113,26 +107,6 @@ enum mlx4_en_alloc_type { MLX4_EN_ALLOC_REPLACEMENT = 1, }; -/* Receive fragment sizes; we use at most 3 fragments (for 9600 byte MTU - * and 4K allocations) */ -#if MJUMPAGESIZE == 4096 -enum { - FRAG_SZ0 = MCLBYTES, - FRAG_SZ1 = MJUMPAGESIZE, - FRAG_SZ2 = MJUMPAGESIZE, -}; -#define MLX4_EN_MAX_RX_FRAGS 3 -#elif MJUMPAGESIZE == 8192 -enum { - FRAG_SZ0 = MCLBYTES, - FRAG_SZ1 = MJUMPAGESIZE, -}; -#define MLX4_EN_MAX_RX_FRAGS 2 -#elif MJUMPAGESIZE == 8192 -#else -#error "Unknown PAGE_SIZE" -#endif - /* Maximum ring sizes */ #define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 @@ -154,7 +128,7 @@ enum { #define MLX4_EN_NUM_UP 1 #define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \ - MLX4_EN_NUM_UP) + MLX4_EN_NUM_UP) #define MLX4_EN_DEF_TX_RING_SIZE 1024 #define MLX4_EN_DEF_RX_RING_SIZE 1024 @@ -235,16 +209,10 @@ enum cq_type { #define ILLEGAL_MAC(addr) (addr == 0xffffffffffffULL || addr == 0x0) struct mlx4_en_tx_info { + bus_dmamap_t dma_map; struct mbuf *mb; u32 nr_txbb; u32 nr_bytes; - u8 linear; - u8 nr_segs; - u8 data_offset; - u8 inl; -#if 0 - u8 ts_requested; -#endif }; @@ -265,14 +233,22 @@ struct mlx4_en_tx_desc { #define MLX4_EN_USE_SRQ 0x01000000 -#define MLX4_EN_TX_BUDGET 64*4 //Compensate for no NAPI in freeBSD - might need some fine tunning in the future. #define MLX4_EN_RX_BUDGET 64 +#define MLX4_EN_TX_MAX_DESC_SIZE 512 /* bytes */ +#define MLX4_EN_TX_MAX_MBUF_SIZE 65536 /* bytes */ +#define MLX4_EN_TX_MAX_PAYLOAD_SIZE 65536 /* bytes */ +#define MLX4_EN_TX_MAX_MBUF_FRAGS \ + ((MLX4_EN_TX_MAX_DESC_SIZE - 128) / DS_SIZE_ALIGNMENT) /* units */ +#define MLX4_EN_TX_WQE_MAX_WQEBBS \ + (MLX4_EN_TX_MAX_DESC_SIZE / TXBB_SIZE) /* units */ + #define MLX4_EN_CX3_LOW_ID 0x1000 #define MLX4_EN_CX3_HIGH_ID 0x1005 struct mlx4_en_tx_ring { spinlock_t tx_lock; + bus_dma_tag_t dma_tag; struct mlx4_hwq_resources wqres; u32 size ; /* number of TXBBs */ u32 size_mask; @@ -282,11 +258,10 @@ struct mlx4_en_tx_ring { u32 cons; u32 buf_size; u32 doorbell_qpn; - void *buf; + u8 *buf; u16 poll_cnt; int blocked; struct mlx4_en_tx_info *tx_info; - u8 *bounce_buf; u8 queue_index; cpuset_t affinity_mask; struct buf_ring *br; @@ -300,13 +275,12 @@ struct mlx4_en_tx_ring { unsigned long packets; unsigned long tx_csum; unsigned long queue_stopped; + unsigned long oversized_packets; unsigned long wake_queue; struct mlx4_bf bf; bool bf_enabled; - struct netdev_queue *tx_queue; int hwtstamp_tx_type; spinlock_t comp_lock; - int full_size; int inline_thold; u64 watchdog_time; }; @@ -316,14 +290,21 @@ struct mlx4_en_rx_desc { struct mlx4_wqe_data_seg data[0]; }; -struct mlx4_en_rx_buf { - dma_addr_t dma; - struct page *page; - unsigned int page_offset; +struct mlx4_en_rx_mbuf { + bus_dmamap_t dma_map; + struct mbuf *mbuf; +}; + +struct mlx4_en_rx_spare { + bus_dmamap_t dma_map; + struct mbuf *mbuf; + u64 paddr_be; }; struct mlx4_en_rx_ring { struct mlx4_hwq_resources wqres; + bus_dma_tag_t dma_tag; + struct mlx4_en_rx_spare spare; u32 size ; /* number of Rx descs*/ u32 actual_size; u32 size_mask; @@ -339,8 +320,8 @@ struct mlx4_en_rx_ring { u32 rx_buf_size; u32 rx_mb_size; int qpn; - void *buf; - void *rx_info; + u8 *buf; + struct mlx4_en_rx_mbuf *mbuf; unsigned long errors; unsigned long bytes; unsigned long packets; @@ -400,6 +381,7 @@ struct mlx4_en_cq { #define MLX4_EN_OPCODE_ERROR 0x1e u32 tot_rx; u32 tot_tx; + u32 curr_poll_rx_cpu_id; #ifdef CONFIG_NET_RX_BUSY_POLL unsigned int state; @@ -524,12 +506,6 @@ struct en_port { u8 vport_num; }; -struct mlx4_en_frag_info { - u16 frag_size; - u16 frag_prefix_size; -}; - - struct mlx4_en_priv { struct mlx4_en_dev *mdev; struct mlx4_en_port_profile *prof; @@ -575,18 +551,14 @@ struct mlx4_en_priv { int cqe_factor; struct mlx4_en_rss_map rss_map; - __be32 ctrl_flags; u32 flags; u8 num_tx_rings_p_up; u32 tx_ring_num; u32 rx_ring_num; u32 rx_mb_size; - struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS]; u16 rx_alloc_order; u32 rx_alloc_size; u32 rx_buf_size; - u16 num_frags; - u16 log_rx_info; struct mlx4_en_tx_ring **tx_ring; struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS]; @@ -640,7 +612,6 @@ struct mlx4_en_priv { unsigned long last_ifq_jiffies; u64 if_counters_rx_errors; u64 if_counters_rx_no_buffer; - }; enum mlx4_en_wol { diff --git a/sys/ofed/drivers/net/mlx4/mlx4_stats.h b/sys/ofed/drivers/net/mlx4/mlx4_stats.h index 0270cef..ef0b1c0 100644 --- a/sys/ofed/drivers/net/mlx4/mlx4_stats.h +++ b/sys/ofed/drivers/net/mlx4/mlx4_stats.h @@ -124,6 +124,7 @@ struct mlx4_en_port_stats { unsigned long queue_stopped; unsigned long wake_queue; unsigned long tx_timeout; + unsigned long oversized_packets; unsigned long rx_alloc_failed; unsigned long rx_chksum_good; unsigned long rx_chksum_none; diff --git a/sys/ofed/drivers/net/mlx4/port.c b/sys/ofed/drivers/net/mlx4/port.c index 4d7aa0b..9ef63f7 100644 --- a/sys/ofed/drivers/net/mlx4/port.c +++ b/sys/ofed/drivers/net/mlx4/port.c @@ -1145,12 +1145,17 @@ int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, u16 offset, size = MODULE_INFO_MAX_READ; inbox = mlx4_alloc_cmd_mailbox(dev); - if (IS_ERR(inbox)) + if (IS_ERR(inbox)) { + mlx4_err(dev, + "mlx4_alloc_cmd_mailbox returned with error(%lx)", PTR_ERR(inbox)); return PTR_ERR(inbox); + } outbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(outbox)) { mlx4_free_cmd_mailbox(dev, inbox); + mlx4_err(dev, + "mlx4_alloc_cmd_mailbox returned with error(%lx)", PTR_ERR(outbox)); return PTR_ERR(outbox); } diff --git a/sys/ofed/include/linux/mlx4/cq.h b/sys/ofed/include/linux/mlx4/cq.h index ed6bec1..b39910a 100644 --- a/sys/ofed/include/linux/mlx4/cq.h +++ b/sys/ofed/include/linux/mlx4/cq.h @@ -139,7 +139,7 @@ enum { }; static inline void mlx4_cq_arm(struct mlx4_cq *cq, u32 cmd, - void __iomem *uar_page, + u8 __iomem *uar_page, spinlock_t *doorbell_lock) { __be32 doorbell[2]; diff --git a/sys/ofed/include/linux/mlx4/device.h b/sys/ofed/include/linux/mlx4/device.h index 536c421..aca4aef 100644 --- a/sys/ofed/include/linux/mlx4/device.h +++ b/sys/ofed/include/linux/mlx4/device.h @@ -413,6 +413,13 @@ enum { #define MSTR_SM_CHANGE_MASK (MLX4_EQ_PORT_INFO_MSTR_SM_SL_CHANGE_MASK | \ MLX4_EQ_PORT_INFO_MSTR_SM_LID_CHANGE_MASK) +enum mlx4_module_id { + MLX4_MODULE_ID_SFP = 0x3, + MLX4_MODULE_ID_QSFP = 0xC, + MLX4_MODULE_ID_QSFP_PLUS = 0xD, + MLX4_MODULE_ID_QSFP28 = 0x11, +}; + static inline u64 mlx4_fw_ver(u64 major, u64 minor, u64 subminor) { return (major << 32) | (minor << 16) | subminor; diff --git a/sys/ofed/include/linux/mlx4/doorbell.h b/sys/ofed/include/linux/mlx4/doorbell.h index f31bba2..6724e5e 100644 --- a/sys/ofed/include/linux/mlx4/doorbell.h +++ b/sys/ofed/include/linux/mlx4/doorbell.h @@ -77,7 +77,7 @@ static inline void mlx4_write64(__be32 val[2], void __iomem *dest, spin_lock_irqsave(doorbell_lock, flags); __raw_writel((__force u32) val[0], dest); - __raw_writel((__force u32) val[1], dest + 4); + __raw_writel((__force u32) val[1], (u8 *)dest + 4); spin_unlock_irqrestore(doorbell_lock, flags); } diff --git a/sys/ofed/include/linux/mlx4/qp.h b/sys/ofed/include/linux/mlx4/qp.h index 9b78167..c8b6cb1 100644 --- a/sys/ofed/include/linux/mlx4/qp.h +++ b/sys/ofed/include/linux/mlx4/qp.h @@ -39,6 +39,12 @@ #define MLX4_INVALID_LKEY 0x100 +#define DS_SIZE_ALIGNMENT 16 + +#define SET_BYTE_COUNT(byte_count) cpu_to_be32(byte_count) +#define SET_LSO_MSS(mss_hdr_size) cpu_to_be32(mss_hdr_size) +#define DS_BYTE_COUNT_MASK cpu_to_be32(0x7fffffff) + enum ib_m_qp_attr_mask { IB_M_EXT_CLASS_1 = 1 << 28, IB_M_EXT_CLASS_2 = 1 << 29, @@ -266,7 +272,9 @@ enum { /* param3 */ #define MLX4_FW_VER_WQE_CTRL_NEC mlx4_fw_ver(2, 2, 232) enum { + MLX4_WQE_CTRL_OWN = 1 << 31, MLX4_WQE_CTRL_NEC = 1 << 29, + MLX4_WQE_CTRL_RR = 1 << 6, MLX4_WQE_CTRL_FENCE = 1 << 6, MLX4_WQE_CTRL_CQ_UPDATE = 3 << 2, MLX4_WQE_CTRL_SOLICITED = 1 << 1, |
