diff options
author | hselasky <hselasky@FreeBSD.org> | 2015-11-10 12:20:22 +0000 |
---|---|---|
committer | hselasky <hselasky@FreeBSD.org> | 2015-11-10 12:20:22 +0000 |
commit | 5c8e2d7ca19ebbae70a9f0d49ee07477f4be5aeb (patch) | |
tree | 7c4ab6c945e4031786f0bbf3a76cb0ea6f5d5167 /sys/dev/mlx5/mlx5_en | |
parent | 60024cd7aad6a43a5dd84afd90b915b12449c421 (diff) | |
download | FreeBSD-src-5c8e2d7ca19ebbae70a9f0d49ee07477f4be5aeb.zip FreeBSD-src-5c8e2d7ca19ebbae70a9f0d49ee07477f4be5aeb.tar.gz |
Add mlx5 and mlx5en driver(s) for ConnectX-4 and ConnectX-4LX cards
from Mellanox Technologies. The current driver supports ethernet
speeds up to and including 100 GBit/s. Infiniband support will be
done later.
The code added is not compiled by default, which will be done by a
separate commit.
Sponsored by: Mellanox Technologies
MFC after: 2 weeks
Diffstat (limited to 'sys/dev/mlx5/mlx5_en')
-rw-r--r-- | sys/dev/mlx5/mlx5_en/en.h | 781 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c | 493 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c | 870 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_main.c | 2902 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_rx.c | 340 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_tx.c | 485 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c | 58 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/tcp_tlro.c | 697 | ||||
-rw-r--r-- | sys/dev/mlx5/mlx5_en/tcp_tlro.h | 83 |
9 files changed, 6709 insertions, 0 deletions
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h new file mode 100644 index 0000000..2988db3 --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/en.h @@ -0,0 +1,781 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MLX5_EN_H_ +#define _MLX5_EN_H_ + +#include <linux/kmod.h> +#include <linux/page.h> +#include <linux/slab.h> +#include <linux/if_vlan.h> +#include <linux/if_ether.h> +#include <linux/vmalloc.h> +#include <linux/moduleparam.h> +#include <linux/delay.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> + +#include <netinet/in_systm.h> +#include <netinet/in.h> +#include <netinet/if_ether.h> +#include <netinet/ip.h> +#include <netinet/ip6.h> +#include <netinet/tcp.h> +#include <netinet/tcp_lro.h> +#include <netinet/udp.h> +#include <net/ethernet.h> +#include <sys/buf_ring.h> + +#include <machine/bus.h> + +#ifdef HAVE_TURBO_LRO +#include "tcp_tlro.h" +#endif + +#include <dev/mlx5/driver.h> +#include <dev/mlx5/qp.h> +#include <dev/mlx5/cq.h> +#include <dev/mlx5/vport.h> + +#include <dev/mlx5/mlx5_core/wq.h> +#include <dev/mlx5/mlx5_core/transobj.h> +#include <dev/mlx5/mlx5_core/mlx5_core.h> + +#define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x7 +#define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa +#define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd + +#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE 0x7 +#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa +#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd + +/* freeBSD HW LRO is limited by 16KB - the size of max mbuf */ +#define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ MJUM16BYTES +#define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10 +#define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3 +#define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20 +#define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC 0x10 +#define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS 0x20 +#define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES 0x80 +#define MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ 0x7 +#define MLX5E_CACHELINE_SIZE CACHE_LINE_SIZE +#define MLX5E_HW2SW_MTU(hwmtu) \ + ((hwmtu) - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN)) +#define MLX5E_SW2HW_MTU(swmtu) \ + ((swmtu) + (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN)) +#define MLX5E_SW2MB_MTU(swmtu) \ + (MLX5E_SW2HW_MTU(swmtu) + MLX5E_NET_IP_ALIGN) +#define MLX5E_MTU_MIN 72 /* Min MTU allowed by the kernel */ +#define MLX5E_MTU_MAX MIN(ETHERMTU_JUMBO, MJUM16BYTES) /* Max MTU of Ethernet jumbo frames */ + +#define MLX5E_BUDGET_MAX 8192 /* RX and TX */ +#define MLX5E_RX_BUDGET_MAX 256 +#define MLX5E_SQ_BF_BUDGET 16 +#define MLX5E_SQ_TX_QUEUE_SIZE 4096 /* SQ drbr queue size */ + +#define MLX5E_MAX_TX_NUM_TC 8 /* units */ +#define MLX5E_MAX_TX_HEADER 128 /* bytes */ +#define MLX5E_MAX_TX_PAYLOAD_SIZE 65536 /* bytes */ +#define MLX5E_MAX_TX_MBUF_SIZE 65536 /* bytes */ +#define MLX5E_MAX_TX_MBUF_FRAGS \ + ((MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS) - \ + (MLX5E_MAX_TX_HEADER / MLX5_SEND_WQE_DS)) /* units */ +#define MLX5E_MAX_TX_INLINE \ + (MLX5E_MAX_TX_HEADER - sizeof(struct mlx5e_tx_wqe) + \ + sizeof(((struct mlx5e_tx_wqe *)0)->eth.inline_hdr_start)) /* bytes */ + +MALLOC_DECLARE(M_MLX5EN); + +struct mlx5_core_dev; +struct mlx5e_cq; + +typedef void (mlx5e_cq_comp_t)(struct mlx5_core_cq *); + +#define MLX5E_STATS_COUNT(a,b,c,d) a +#define MLX5E_STATS_VAR(a,b,c,d) b; +#define MLX5E_STATS_DESC(a,b,c,d) c, d, + +#define MLX5E_VPORT_STATS(m) \ + /* HW counters */ \ + m(+1, u64 rx_packets, "rx_packets", "Received packets") \ + m(+1, u64 rx_bytes, "rx_bytes", "Received bytes") \ + m(+1, u64 tx_packets, "tx_packets", "Transmitted packets") \ + m(+1, u64 tx_bytes, "tx_bytes", "Transmitted bytes") \ + m(+1, u64 rx_error_packets, "rx_error_packets", "Received error packets") \ + m(+1, u64 rx_error_bytes, "rx_error_bytes", "Received error bytes") \ + m(+1, u64 tx_error_packets, "tx_error_packets", "Transmitted error packets") \ + m(+1, u64 tx_error_bytes, "tx_error_bytes", "Transmitted error bytes") \ + m(+1, u64 rx_unicast_packets, "rx_unicast_packets", "Received unicast packets") \ + m(+1, u64 rx_unicast_bytes, "rx_unicast_bytes", "Received unicast bytes") \ + m(+1, u64 tx_unicast_packets, "tx_unicast_packets", "Transmitted unicast packets") \ + m(+1, u64 tx_unicast_bytes, "tx_unicast_bytes", "Transmitted unicast bytes") \ + m(+1, u64 rx_multicast_packets, "rx_multicast_packets", "Received multicast packets") \ + m(+1, u64 rx_multicast_bytes, "rx_multicast_bytes", "Received multicast bytes") \ + m(+1, u64 tx_multicast_packets, "tx_multicast_packets", "Transmitted multicast packets") \ + m(+1, u64 tx_multicast_bytes, "tx_multicast_bytes", "Transmitted multicast bytes") \ + m(+1, u64 rx_broadcast_packets, "rx_broadcast_packets", "Received broadcast packets") \ + m(+1, u64 rx_broadcast_bytes, "rx_broadcast_bytes", "Received broadcast bytes") \ + m(+1, u64 tx_broadcast_packets, "tx_broadcast_packets", "Transmitted broadcast packets") \ + m(+1, u64 tx_broadcast_bytes, "tx_broadcast_bytes", "Transmitted broadcast bytes") \ + /* SW counters */ \ + m(+1, u64 tso_packets, "tso_packets", "Transmitted TSO packets") \ + m(+1, u64 tso_bytes, "tso_bytes", "Transmitted TSO bytes") \ + m(+1, u64 lro_packets, "lro_packets", "Received LRO packets") \ + m(+1, u64 lro_bytes, "lro_bytes", "Received LRO bytes") \ + m(+1, u64 sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO") \ + m(+1, u64 sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO") \ + m(+1, u64 rx_csum_good, "rx_csum_good", "Received checksum valid packets") \ + m(+1, u64 rx_csum_none, "rx_csum_none", "Received no checksum packets") \ + m(+1, u64 tx_csum_offload, "tx_csum_offload", "Transmit checksum offload packets") \ + m(+1, u64 tx_queue_dropped, "tx_queue_dropped", "Transmit queue dropped") \ + m(+1, u64 tx_defragged, "tx_defragged", "Transmit queue defragged") \ + m(+1, u64 rx_wqe_err, "rx_wqe_err", "Receive WQE errors") + +#define MLX5E_VPORT_STATS_NUM (0 MLX5E_VPORT_STATS(MLX5E_STATS_COUNT)) + +struct mlx5e_vport_stats { + struct sysctl_ctx_list ctx; + u64 arg [0]; + MLX5E_VPORT_STATS(MLX5E_STATS_VAR) +}; + +#define MLX5E_PPORT_IEEE802_3_STATS(m) \ + m(+1, u64 frames_tx, "frames_tx", "Frames transmitted") \ + m(+1, u64 frames_rx, "frames_rx", "Frames received") \ + m(+1, u64 check_seq_err, "check_seq_err", "Sequence errors") \ + m(+1, u64 alignment_err, "alignment_err", "Alignment errors") \ + m(+1, u64 octets_tx, "octets_tx", "Bytes transmitted") \ + m(+1, u64 octets_received, "octets_received", "Bytes received") \ + m(+1, u64 multicast_xmitted, "multicast_xmitted", "Multicast transmitted") \ + m(+1, u64 broadcast_xmitted, "broadcast_xmitted", "Broadcast transmitted") \ + m(+1, u64 multicast_rx, "multicast_rx", "Multicast received") \ + m(+1, u64 broadcast_rx, "broadcast_rx", "Broadcast received") \ + m(+1, u64 in_range_len_errors, "in_range_len_errors", "In range length errors") \ + m(+1, u64 out_of_range_len, "out_of_range_len", "Out of range length errors") \ + m(+1, u64 too_long_errors, "too_long_errors", "Too long errors") \ + m(+1, u64 symbol_err, "symbol_err", "Symbol errors") \ + m(+1, u64 mac_control_tx, "mac_control_tx", "MAC control transmitted") \ + m(+1, u64 mac_control_rx, "mac_control_rx", "MAC control received") \ + m(+1, u64 unsupported_op_rx, "unsupported_op_rx", "Unsupported operation received") \ + m(+1, u64 pause_ctrl_rx, "pause_ctrl_rx", "Pause control received") \ + m(+1, u64 pause_ctrl_tx, "pause_ctrl_tx", "Pause control transmitted") + +#define MLX5E_PPORT_RFC2819_STATS(m) \ + m(+1, u64 drop_events, "drop_events", "Dropped events") \ + m(+1, u64 octets, "octets", "Octets") \ + m(+1, u64 pkts, "pkts", "Packets") \ + m(+1, u64 broadcast_pkts, "broadcast_pkts", "Broadcast packets") \ + m(+1, u64 multicast_pkts, "multicast_pkts", "Multicast packets") \ + m(+1, u64 crc_align_errors, "crc_align_errors", "CRC alignment errors") \ + m(+1, u64 undersize_pkts, "undersize_pkts", "Undersized packets") \ + m(+1, u64 oversize_pkts, "oversize_pkts", "Oversized packets") \ + m(+1, u64 fragments, "fragments", "Fragments") \ + m(+1, u64 jabbers, "jabbers", "Jabbers") \ + m(+1, u64 collisions, "collisions", "Collisions") + +#define MLX5E_PPORT_RFC2819_STATS_DEBUG(m) \ + m(+1, u64 p64octets, "p64octets", "Bytes") \ + m(+1, u64 p65to127octets, "p65to127octets", "Bytes") \ + m(+1, u64 p128to255octets, "p128to255octets", "Bytes") \ + m(+1, u64 p256to511octets, "p256to511octets", "Bytes") \ + m(+1, u64 p512to1023octets, "p512to1023octets", "Bytes") \ + m(+1, u64 p1024to1518octets, "p1024to1518octets", "Bytes") \ + m(+1, u64 p1519to2047octets, "p1519to2047octets", "Bytes") \ + m(+1, u64 p2048to4095octets, "p2048to4095octets", "Bytes") \ + m(+1, u64 p4096to8191octets, "p4096to8191octets", "Bytes") \ + m(+1, u64 p8192to10239octets, "p8192to10239octets", "Bytes") + +#define MLX5E_PPORT_RFC2863_STATS_DEBUG(m) \ + m(+1, u64 in_octets, "in_octets", "In octets") \ + m(+1, u64 in_ucast_pkts, "in_ucast_pkts", "In unicast packets") \ + m(+1, u64 in_discards, "in_discards", "In discards") \ + m(+1, u64 in_errors, "in_errors", "In errors") \ + m(+1, u64 in_unknown_protos, "in_unknown_protos", "In unknown protocols") \ + m(+1, u64 out_octets, "out_octets", "Out octets") \ + m(+1, u64 out_ucast_pkts, "out_ucast_pkts", "Out unicast packets") \ + m(+1, u64 out_discards, "out_discards", "Out discards") \ + m(+1, u64 out_errors, "out_errors", "Out errors") \ + m(+1, u64 in_multicast_pkts, "in_multicast_pkts", "In multicast packets") \ + m(+1, u64 in_broadcast_pkts, "in_broadcast_pkts", "In broadcast packets") \ + m(+1, u64 out_multicast_pkts, "out_multicast_pkts", "Out multicast packets") \ + m(+1, u64 out_broadcast_pkts, "out_broadcast_pkts", "Out broadcast packets") + +#define MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(m) \ + m(+1, u64 time_since_last_clear, "time_since_last_clear", \ + "Time since the last counters clear event (msec)") \ + m(+1, u64 symbol_errors, "symbol_errors", "Symbol errors") \ + m(+1, u64 sync_headers_errors, "sync_headers_errors", "Sync header error counter") \ + m(+1, u64 bip_errors_lane0, "edpl_bip_errors_lane0", \ + "Indicates the number of PRBS errors on lane 0") \ + m(+1, u64 bip_errors_lane1, "edpl_bip_errors_lane1", \ + "Indicates the number of PRBS errors on lane 1") \ + m(+1, u64 bip_errors_lane2, "edpl_bip_errors_lane2", \ + "Indicates the number of PRBS errors on lane 2") \ + m(+1, u64 bip_errors_lane3, "edpl_bip_errors_lane3", \ + "Indicates the number of PRBS errors on lane 3") \ + m(+1, u64 fc_corrected_blocks_lane0, "fc_corrected_blocks_lane0", \ + "FEC correctable block counter lane 0") \ + m(+1, u64 fc_corrected_blocks_lane1, "fc_corrected_blocks_lane1", \ + "FEC correctable block counter lane 1") \ + m(+1, u64 fc_corrected_blocks_lane2, "fc_corrected_blocks_lane2", \ + "FEC correctable block counter lane 2") \ + m(+1, u64 fc_corrected_blocks_lane3, "fc_corrected_blocks_lane3", \ + "FEC correctable block counter lane 3") \ + m(+1, u64 rs_corrected_blocks, "rs_corrected_blocks", \ + "FEC correcable block counter") \ + m(+1, u64 rs_uncorrectable_blocks, "rs_uncorrectable_blocks", \ + "FEC uncorrecable block counter") \ + m(+1, u64 rs_no_errors_blocks, "rs_no_errors_blocks", \ + "The number of RS-FEC blocks received that had no errors") \ + m(+1, u64 rs_single_error_blocks, "rs_single_error_blocks", \ + "The number of corrected RS-FEC blocks received that had" \ + "exactly 1 error symbol") \ + m(+1, u64 rs_corrected_symbols_total, "rs_corrected_symbols_total", \ + "Port FEC corrected symbol counter") \ + m(+1, u64 rs_corrected_symbols_lane0, "rs_corrected_symbols_lane0", \ + "FEC corrected symbol counter lane 0") \ + m(+1, u64 rs_corrected_symbols_lane1, "rs_corrected_symbols_lane1", \ + "FEC corrected symbol counter lane 1") \ + m(+1, u64 rs_corrected_symbols_lane2, "rs_corrected_symbols_lane2", \ + "FEC corrected symbol counter lane 2") \ + m(+1, u64 rs_corrected_symbols_lane3, "rs_corrected_symbols_lane3", \ + "FEC corrected symbol counter lane 3") \ + +#define MLX5E_PPORT_Q_CONTERS(m) \ + m(+1, u64 out_of_rx_buffer, "out_of_rx_buffer", "out of rx buffers aka no recv wqes events") + +/* + * Make sure to update mlx5e_update_pport_counters() + * when adding a new MLX5E_PPORT_STATS block + */ +#define MLX5E_PPORT_STATS(m) \ + MLX5E_PPORT_IEEE802_3_STATS(m) \ + MLX5E_PPORT_RFC2819_STATS(m) \ + MLX5E_PPORT_Q_CONTERS(m) + +#define MLX5E_PORT_STATS_DEBUG(m) \ + MLX5E_PPORT_RFC2819_STATS_DEBUG(m) \ + MLX5E_PPORT_RFC2863_STATS_DEBUG(m) \ + MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(m) + +#define MLX5E_PPORT_IEEE802_3_STATS_NUM \ + (0 MLX5E_PPORT_IEEE802_3_STATS(MLX5E_STATS_COUNT)) +#define MLX5E_PPORT_RFC2819_STATS_NUM \ + (0 MLX5E_PPORT_RFC2819_STATS(MLX5E_STATS_COUNT)) +#define MLX5E_PPORT_STATS_NUM \ + (0 MLX5E_PPORT_STATS(MLX5E_STATS_COUNT)) + +#define MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM \ + (0 MLX5E_PPORT_RFC2819_STATS_DEBUG(MLX5E_STATS_COUNT)) +#define MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM \ + (0 MLX5E_PPORT_RFC2863_STATS_DEBUG(MLX5E_STATS_COUNT)) +#define MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM \ + (0 MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(MLX5E_STATS_COUNT)) +#define MLX5E_PORT_STATS_DEBUG_NUM \ + (0 MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_COUNT)) + +struct mlx5e_pport_stats { + struct sysctl_ctx_list ctx; + u64 arg [0]; + MLX5E_PPORT_STATS(MLX5E_STATS_VAR) +}; + +struct mlx5e_port_stats_debug { + struct sysctl_ctx_list ctx; + u64 arg [0]; + MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_VAR) +}; + +#define MLX5E_RQ_STATS(m) \ + m(+1, u64 packets, "packets", "Received packets") \ + m(+1, u64 csum_none, "csum_none", "Received packets") \ + m(+1, u64 lro_packets, "lro_packets", "Received packets") \ + m(+1, u64 lro_bytes, "lro_bytes", "Received packets") \ + m(+1, u64 sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO") \ + m(+1, u64 sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO") \ + m(+1, u64 wqe_err, "wqe_err", "Received packets") + +#define MLX5E_RQ_STATS_NUM (0 MLX5E_RQ_STATS(MLX5E_STATS_COUNT)) + +struct mlx5e_rq_stats { + struct sysctl_ctx_list ctx; + u64 arg [0]; + MLX5E_RQ_STATS(MLX5E_STATS_VAR) +}; + +#define MLX5E_SQ_STATS(m) \ + m(+1, u64 packets, "packets", "Transmitted packets") \ + m(+1, u64 tso_packets, "tso_packets", "Transmitted packets") \ + m(+1, u64 tso_bytes, "tso_bytes", "Transmitted bytes") \ + m(+1, u64 csum_offload_none, "csum_offload_none", "Transmitted packets") \ + m(+1, u64 defragged, "defragged", "Transmitted packets") \ + m(+1, u64 dropped, "dropped", "Transmitted packets") \ + m(+1, u64 nop, "nop", "Transmitted packets") + +#define MLX5E_SQ_STATS_NUM (0 MLX5E_SQ_STATS(MLX5E_STATS_COUNT)) + +struct mlx5e_sq_stats { + struct sysctl_ctx_list ctx; + u64 arg [0]; + MLX5E_SQ_STATS(MLX5E_STATS_VAR) +}; + +struct mlx5e_stats { + struct mlx5e_vport_stats vport; + struct mlx5e_pport_stats pport; + struct mlx5e_port_stats_debug port_stats_debug; +}; + +struct mlx5e_params { + u8 log_sq_size; + u8 log_rq_size; + u16 num_channels; + u8 default_vlan_prio; + u8 num_tc; + u8 rx_cq_moderation_mode; + u16 rx_cq_moderation_usec; + u16 rx_cq_moderation_pkts; + u16 tx_cq_moderation_usec; + u16 tx_cq_moderation_pkts; + u16 min_rx_wqes; + bool hw_lro_en; + u32 lro_wqe_sz; + u16 rx_hash_log_tbl_sz; +}; + +#define MLX5E_PARAMS(m) \ + m(+1, u64 tx_pauseframe_control, "tx_pauseframe_control", "Set to enable TX pause frames. Clear to disable.") \ + m(+1, u64 rx_pauseframe_control, "rx_pauseframe_control", "Set to enable RX pause frames. Clear to disable.") \ + m(+1, u64 tx_queue_size_max, "tx_queue_size_max", "Max send queue size") \ + m(+1, u64 rx_queue_size_max, "rx_queue_size_max", "Max receive queue size") \ + m(+1, u64 tx_queue_size, "tx_queue_size", "Default send queue size") \ + m(+1, u64 rx_queue_size, "rx_queue_size", "Default receive queue size") \ + m(+1, u64 channels, "channels", "Default number of channels") \ + m(+1, u64 coalesce_usecs_max, "coalesce_usecs_max", "Maximum usecs for joining packets") \ + m(+1, u64 coalesce_pkts_max, "coalesce_pkts_max", "Maximum packets to join") \ + m(+1, u64 rx_coalesce_usecs, "rx_coalesce_usecs", "Limit in usec for joining rx packets") \ + m(+1, u64 rx_coalesce_pkts, "rx_coalesce_pkts", "Maximum number of rx packets to join") \ + m(+1, u64 rx_coalesce_mode, "rx_coalesce_mode", "0: EQE mode 1: CQE mode") \ + m(+1, u64 tx_coalesce_usecs, "tx_coalesce_usecs", "Limit in usec for joining tx packets") \ + m(+1, u64 tx_coalesce_pkts, "tx_coalesce_pkts", "Maximum number of tx packets to join") \ + m(+1, u64 hw_lro, "hw_lro", "set to enable hw_lro") + +#define MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT)) + +struct mlx5e_params_ethtool { + u64 arg [0]; + MLX5E_PARAMS(MLX5E_STATS_VAR) +}; + +/* EEPROM Standards for plug in modules */ +#ifndef MLX5E_ETH_MODULE_SFF_8472 +#define MLX5E_ETH_MODULE_SFF_8472 0x1 +#define MLX5E_ETH_MODULE_SFF_8472_LEN 128 +#endif + +#ifndef MLX5E_ETH_MODULE_SFF_8636 +#define MLX5E_ETH_MODULE_SFF_8636 0x2 +#define MLX5E_ETH_MODULE_SFF_8636_LEN 256 +#endif + +#ifndef MLX5E_ETH_MODULE_SFF_8436 +#define MLX5E_ETH_MODULE_SFF_8436 0x3 +#define MLX5E_ETH_MODULE_SFF_8436_LEN 256 +#endif + +/* EEPROM I2C Addresses */ +#define MLX5E_I2C_ADDR_LOW 0x50 +#define MLX5E_I2C_ADDR_HIGH 0x51 + +#define MLX5E_EEPROM_LOW_PAGE 0x0 +#define MLX5E_EEPROM_HIGH_PAGE 0x3 + +#define MLX5E_EEPROM_HIGH_PAGE_OFFSET 128 +#define MLX5E_EEPROM_PAGE_LENGTH 256 + +#define MLX5E_EEPROM_INFO_BYTES 0x3 + +struct mlx5e_cq { + /* data path - accessed per cqe */ + struct mlx5_cqwq wq; + + /* data path - accessed per HW polling */ + struct mlx5_core_cq mcq; + struct mlx5e_channel *channel; + + /* control */ + struct mlx5_wq_ctrl wq_ctrl; +} __aligned(MLX5E_CACHELINE_SIZE); + +struct mlx5e_rq_mbuf { + bus_dmamap_t dma_map; + caddr_t data; + struct mbuf *mbuf; +}; + +struct mlx5e_rq { + /* data path */ + struct mlx5_wq_ll wq; + struct mtx mtx; + bus_dma_tag_t dma_tag; + u32 wqe_sz; + struct mlx5e_rq_mbuf *mbuf; + struct device *pdev; + struct ifnet *ifp; + struct mlx5e_rq_stats stats; + struct mlx5e_cq cq; +#ifdef HAVE_TURBO_LRO + struct tlro_ctrl lro; +#else + struct lro_ctrl lro; +#endif + volatile int enabled; + int ix; + + /* control */ + struct mlx5_wq_ctrl wq_ctrl; + u32 rqn; + struct mlx5e_channel *channel; +} __aligned(MLX5E_CACHELINE_SIZE); + +struct mlx5e_sq_mbuf { + bus_dmamap_t dma_map; + struct mbuf *mbuf; + u32 num_bytes; + u32 num_wqebbs; +}; + +enum { + MLX5E_SQ_READY, + MLX5E_SQ_FULL +}; + +struct mlx5e_sq { + /* data path */ + struct mtx lock; + bus_dma_tag_t dma_tag; + struct mtx comp_lock; + + /* dirtied @completion */ + u16 cc; + + /* dirtied @xmit */ + u16 pc __aligned(MLX5E_CACHELINE_SIZE); + u16 bf_offset; + struct mlx5e_sq_stats stats; + + struct mlx5e_cq cq; + struct task sq_task; + struct taskqueue *sq_tq; + + /* pointers to per packet info: write@xmit, read@completion */ + struct mlx5e_sq_mbuf *mbuf; + struct buf_ring *br; + + /* read only */ + struct mlx5_wq_cyc wq; + void __iomem *uar_map; + void __iomem *uar_bf_map; + u32 sqn; + u32 bf_buf_size; + struct device *pdev; + u32 mkey_be; + + /* control path */ + struct mlx5_wq_ctrl wq_ctrl; + struct mlx5_uar uar; + struct mlx5e_channel *channel; + int tc; + unsigned int queue_state; +} __aligned(MLX5E_CACHELINE_SIZE); + +static inline bool +mlx5e_sq_has_room_for(struct mlx5e_sq *sq, u16 n) +{ + return ((sq->wq.sz_m1 & (sq->cc - sq->pc)) >= n || + sq->cc == sq->pc); +} + +struct mlx5e_channel { + /* data path */ + struct mlx5e_rq rq; + struct mlx5e_sq sq[MLX5E_MAX_TX_NUM_TC]; + struct device *pdev; + struct ifnet *ifp; + u32 mkey_be; + u8 num_tc; + + /* control */ + struct mlx5e_priv *priv; + int ix; + int cpu; +} __aligned(MLX5E_CACHELINE_SIZE); + +enum mlx5e_traffic_types { + MLX5E_TT_IPV4_TCP, + MLX5E_TT_IPV6_TCP, + MLX5E_TT_IPV4_UDP, + MLX5E_TT_IPV6_UDP, + MLX5E_TT_IPV4_IPSEC_AH, + MLX5E_TT_IPV6_IPSEC_AH, + MLX5E_TT_IPV4_IPSEC_ESP, + MLX5E_TT_IPV6_IPSEC_ESP, + MLX5E_TT_IPV4, + MLX5E_TT_IPV6, + MLX5E_TT_ANY, + MLX5E_NUM_TT, +}; + +enum { + MLX5E_RQT_SPREADING = 0, + MLX5E_RQT_DEFAULT_RQ = 1, + MLX5E_NUM_RQT = 2, +}; + +struct mlx5e_eth_addr_info { + u8 addr [ETH_ALEN + 2]; + u32 tt_vec; + u32 ft_ix[MLX5E_NUM_TT]; /* flow table index per traffic type */ +}; + +#define MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE) + +struct mlx5e_eth_addr_hash_node; + +struct mlx5e_eth_addr_hash_head { + struct mlx5e_eth_addr_hash_node *lh_first; +}; + +struct mlx5e_eth_addr_db { + struct mlx5e_eth_addr_hash_head if_uc[MLX5E_ETH_ADDR_HASH_SIZE]; + struct mlx5e_eth_addr_hash_head if_mc[MLX5E_ETH_ADDR_HASH_SIZE]; + struct mlx5e_eth_addr_info broadcast; + struct mlx5e_eth_addr_info allmulti; + struct mlx5e_eth_addr_info promisc; + bool broadcast_enabled; + bool allmulti_enabled; + bool promisc_enabled; +}; + +enum { + MLX5E_STATE_ASYNC_EVENTS_ENABLE, + MLX5E_STATE_OPENED, +}; + +struct mlx5e_vlan_db { + unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; + u32 active_vlans_ft_ix[VLAN_N_VID]; + u32 untagged_rule_ft_ix; + u32 any_vlan_rule_ft_ix; + bool filter_disabled; +}; + +struct mlx5e_flow_table { + void *vlan; + void *main; +}; + +struct mlx5e_priv { + /* priv data path fields - start */ + int order_base_2_num_channels; + int queue_mapping_channel_mask; + int num_tc; + int default_vlan_prio; + /* priv data path fields - end */ + + unsigned long state; + int gone; +#define PRIV_LOCK(priv) sx_xlock(&(priv)->state_lock) +#define PRIV_UNLOCK(priv) sx_xunlock(&(priv)->state_lock) +#define PRIV_LOCKED(priv) sx_xlocked(&(priv)->state_lock) + struct sx state_lock; /* Protects Interface state */ + struct mlx5_uar cq_uar; + u32 pdn; + u32 tdn; + struct mlx5_core_mr mr; + + struct mlx5e_channel * volatile *channel; + u32 tisn[MLX5E_MAX_TX_NUM_TC]; + u32 rqtn; + u32 tirn[MLX5E_NUM_TT]; + + struct mlx5e_flow_table ft; + struct mlx5e_eth_addr_db eth_addr; + struct mlx5e_vlan_db vlan; + + struct mlx5e_params params; + struct mlx5e_params_ethtool params_ethtool; + struct mtx async_events_mtx; /* sync hw events */ + struct work_struct update_stats_work; + struct work_struct update_carrier_work; + struct work_struct set_rx_mode_work; + + struct mlx5_core_dev *mdev; + struct ifnet *ifp; + struct sysctl_ctx_list sysctl_ctx; + struct sysctl_oid *sysctl_ifnet; + struct sysctl_oid *sysctl_hw; + int sysctl_debug; + struct mlx5e_stats stats; + int counter_set_id; + + eventhandler_tag vlan_detach; + eventhandler_tag vlan_attach; + struct ifmedia media; + int media_status_last; + int media_active_last; + + struct callout watchdog; +}; + +#define MLX5E_NET_IP_ALIGN 2 + +struct mlx5e_tx_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_eth_seg eth; +}; + +struct mlx5e_rx_wqe { + struct mlx5_wqe_srq_next_seg next; + struct mlx5_wqe_data_seg data; +}; + +struct mlx5e_eeprom { + int lock_bit; + int i2c_addr; + int page_num; + int device_addr; + int module_num; + int len; + int type; + int page_valid; + u32 *data; +}; + +enum mlx5e_link_mode { + MLX5E_1000BASE_CX_SGMII = 0, + MLX5E_1000BASE_KX = 1, + MLX5E_10GBASE_CX4 = 2, + MLX5E_10GBASE_KX4 = 3, + MLX5E_10GBASE_KR = 4, + MLX5E_20GBASE_KR2 = 5, + MLX5E_40GBASE_CR4 = 6, + MLX5E_40GBASE_KR4 = 7, + MLX5E_56GBASE_R4 = 8, + MLX5E_10GBASE_CR = 12, + MLX5E_10GBASE_SR = 13, + MLX5E_10GBASE_ER = 14, + MLX5E_40GBASE_SR4 = 15, + MLX5E_40GBASE_LR4 = 16, + MLX5E_100GBASE_CR4 = 20, + MLX5E_100GBASE_SR4 = 21, + MLX5E_100GBASE_KR4 = 22, + MLX5E_100GBASE_LR4 = 23, + MLX5E_100BASE_TX = 24, + MLX5E_100BASE_T = 25, + MLX5E_10GBASE_T = 26, + MLX5E_25GBASE_CR = 27, + MLX5E_25GBASE_KR = 28, + MLX5E_25GBASE_SR = 29, + MLX5E_50GBASE_CR2 = 30, + MLX5E_50GBASE_KR2 = 31, + MLX5E_LINK_MODES_NUMBER, +}; + +#define MLX5E_PROT_MASK(link_mode) (1 << (link_mode)) +#define MLX5E_FLD_MAX(typ, fld) ((1ULL << __mlx5_bit_sz(typ, fld)) - 1ULL) + +int mlx5e_xmit(struct ifnet *, struct mbuf *); + +int mlx5e_open_locked(struct ifnet *); +int mlx5e_close_locked(struct ifnet *); + +void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, int event); +void mlx5e_rx_cq_comp(struct mlx5_core_cq *); +void mlx5e_tx_cq_comp(struct mlx5_core_cq *); +struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq); +void mlx5e_tx_que(void *context, int pending); + +int mlx5e_open_flow_table(struct mlx5e_priv *priv); +void mlx5e_close_flow_table(struct mlx5e_priv *priv); +void mlx5e_set_rx_mode_core(struct mlx5e_priv *priv); +void mlx5e_set_rx_mode_work(struct work_struct *work); + +void mlx5e_vlan_rx_add_vid(void *, struct ifnet *, u16); +void mlx5e_vlan_rx_kill_vid(void *, struct ifnet *, u16); +void mlx5e_enable_vlan_filter(struct mlx5e_priv *priv); +void mlx5e_disable_vlan_filter(struct mlx5e_priv *priv); +int mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv); +void mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv); + +static inline void +mlx5e_tx_notify_hw(struct mlx5e_sq *sq, + struct mlx5e_tx_wqe *wqe, int bf_sz) +{ + u16 ofst = MLX5_BF_OFFSET + sq->bf_offset; + + /* ensure wqe is visible to device before updating doorbell record */ + wmb(); + + *sq->wq.db = cpu_to_be32(sq->pc); + + /* + * Ensure the doorbell record is visible to device before ringing + * the doorbell: + */ + wmb(); + + if (bf_sz) { + __iowrite64_copy(sq->uar_bf_map + ofst, &wqe->ctrl, bf_sz); + + /* flush the write-combining mapped buffer */ + wmb(); + + } else { + mlx5_write64((__be32 *)&wqe->ctrl, sq->uar_map + ofst, NULL); + } + + sq->bf_offset ^= sq->bf_buf_size; +} + +static inline void +mlx5e_cq_arm(struct mlx5e_cq *cq) +{ + struct mlx5_core_cq *mcq; + + mcq = &cq->mcq; + mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, NULL, cq->wq.cc); +} + +extern const struct ethtool_ops mlx5e_ethtool_ops; +void mlx5e_create_ethtool(struct mlx5e_priv *); +void mlx5e_create_stats(struct sysctl_ctx_list *, + struct sysctl_oid_list *, const char *, + const char **, unsigned, u64 *); +void mlx5e_send_nop(struct mlx5e_sq *, u32, bool); + +#endif /* _MLX5_EN_H_ */ diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c new file mode 100644 index 0000000..2ad2255 --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c @@ -0,0 +1,493 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "en.h" +#include <net/sff8472.h> + +void +mlx5e_create_stats(struct sysctl_ctx_list *ctx, + struct sysctl_oid_list *parent, const char *buffer, + const char **desc, unsigned num, u64 * arg) +{ + struct sysctl_oid *node; + unsigned x; + + sysctl_ctx_init(ctx); + + node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, + buffer, CTLFLAG_RD, NULL, "Statistics"); + if (node == NULL) + return; + for (x = 0; x != num; x++) { + SYSCTL_ADD_UQUAD(ctx, SYSCTL_CHILDREN(node), OID_AUTO, + desc[2 * x], CTLFLAG_RD, arg + x, desc[2 * x + 1]); + } +} + +static int +mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS) +{ + struct mlx5e_priv *priv = arg1; + uint64_t value; + int was_opened; + int error; + + PRIV_LOCK(priv); + value = priv->params_ethtool.arg[arg2]; + error = sysctl_handle_64(oidp, &value, 0, req); + if (error || req->newptr == NULL || + value == priv->params_ethtool.arg[arg2]) + goto done; + + /* assign new value */ + priv->params_ethtool.arg[arg2] = value; + + /* check if device is gone */ + if (priv->gone) { + error = ENXIO; + goto done; + } + + if (&priv->params_ethtool.arg[arg2] == &priv->params_ethtool.rx_pauseframe_control || + &priv->params_ethtool.arg[arg2] == &priv->params_ethtool.tx_pauseframe_control) { + /* range check parameters */ + priv->params_ethtool.rx_pauseframe_control = + priv->params_ethtool.rx_pauseframe_control ? 1 : 0; + priv->params_ethtool.tx_pauseframe_control = + priv->params_ethtool.tx_pauseframe_control ? 1 : 0; + + /* update firmware */ + error = -mlx5_set_port_pause(priv->mdev, 1, + priv->params_ethtool.rx_pauseframe_control, + priv->params_ethtool.tx_pauseframe_control); + goto done; + } + + was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (was_opened) + mlx5e_close_locked(priv->ifp); + + /* import TX queue size */ + if (priv->params_ethtool.tx_queue_size < + (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)) { + priv->params_ethtool.tx_queue_size = + (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE); + } else if (priv->params_ethtool.tx_queue_size > + priv->params_ethtool.tx_queue_size_max) { + priv->params_ethtool.tx_queue_size = + priv->params_ethtool.tx_queue_size_max; + } + priv->params.log_sq_size = + order_base_2(priv->params_ethtool.tx_queue_size); + + /* import RX queue size */ + if (priv->params_ethtool.rx_queue_size < + (1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE)) { + priv->params_ethtool.rx_queue_size = + (1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE); + } else if (priv->params_ethtool.rx_queue_size > + priv->params_ethtool.rx_queue_size_max) { + priv->params_ethtool.rx_queue_size = + priv->params_ethtool.rx_queue_size_max; + } + priv->params.log_rq_size = + order_base_2(priv->params_ethtool.rx_queue_size); + + priv->params.min_rx_wqes = min_t (u16, + priv->params_ethtool.rx_queue_size - 1, + MLX5E_PARAMS_DEFAULT_MIN_RX_WQES); + + /* import number of channels */ + if (priv->params_ethtool.channels < 1) + priv->params_ethtool.channels = 1; + else if (priv->params_ethtool.channels > + (u64) priv->mdev->priv.eq_table.num_comp_vectors) { + priv->params_ethtool.channels = + (u64) priv->mdev->priv.eq_table.num_comp_vectors; + } + priv->params.num_channels = priv->params_ethtool.channels; + + /* import RX mode */ + if (priv->params_ethtool.rx_coalesce_mode != 0) + priv->params_ethtool.rx_coalesce_mode = 1; + priv->params.rx_cq_moderation_mode = priv->params_ethtool.rx_coalesce_mode; + + /* import RX coal time */ + if (priv->params_ethtool.rx_coalesce_usecs < 1) + priv->params_ethtool.rx_coalesce_usecs = 0; + else if (priv->params_ethtool.rx_coalesce_usecs > + MLX5E_FLD_MAX(cqc, cq_period)) { + priv->params_ethtool.rx_coalesce_usecs = + MLX5E_FLD_MAX(cqc, cq_period); + } + priv->params.rx_cq_moderation_usec = priv->params_ethtool.rx_coalesce_usecs; + + /* import RX coal pkts */ + if (priv->params_ethtool.rx_coalesce_pkts < 1) + priv->params_ethtool.rx_coalesce_pkts = 0; + else if (priv->params_ethtool.rx_coalesce_pkts > + MLX5E_FLD_MAX(cqc, cq_max_count)) { + priv->params_ethtool.rx_coalesce_pkts = + MLX5E_FLD_MAX(cqc, cq_max_count); + } + priv->params.rx_cq_moderation_pkts = priv->params_ethtool.rx_coalesce_pkts; + + /* import TX coal time */ + if (priv->params_ethtool.tx_coalesce_usecs < 1) + priv->params_ethtool.tx_coalesce_usecs = 0; + else if (priv->params_ethtool.tx_coalesce_usecs > + MLX5E_FLD_MAX(cqc, cq_period)) { + priv->params_ethtool.tx_coalesce_usecs = + MLX5E_FLD_MAX(cqc, cq_period); + } + priv->params.tx_cq_moderation_usec = priv->params_ethtool.tx_coalesce_usecs; + + /* import TX coal pkts */ + if (priv->params_ethtool.tx_coalesce_pkts < 1) + priv->params_ethtool.tx_coalesce_pkts = 0; + else if (priv->params_ethtool.tx_coalesce_pkts > + MLX5E_FLD_MAX(cqc, cq_max_count)) { + priv->params_ethtool.tx_coalesce_pkts = MLX5E_FLD_MAX(cqc, cq_max_count); + } + priv->params.tx_cq_moderation_pkts = priv->params_ethtool.tx_coalesce_pkts; + + /* we always agree to turn off HW LRO - but not always to turn on */ + if (priv->params_ethtool.hw_lro) { + if (priv->params_ethtool.hw_lro != 1) { + priv->params_ethtool.hw_lro = priv->params.hw_lro_en; + error = EINVAL; + goto done; + } + if (priv->ifp->if_capenable & IFCAP_LRO) + priv->params.hw_lro_en = !!MLX5_CAP_ETH(priv->mdev, lro_cap); + } + else { + priv->params.hw_lro_en = false; + } + + if (was_opened) + mlx5e_open_locked(priv->ifp); +done: + PRIV_UNLOCK(priv); + return (error); +} + +/* + * Read the first three bytes of the eeprom in order to get the needed info + * for the whole reading. + * Byte 0 - Identifier byte + * Byte 1 - Revision byte + * Byte 2 - Status byte + */ +static int +mlx5e_get_eeprom_info(struct mlx5e_priv *priv, struct mlx5e_eeprom *eeprom) +{ + struct mlx5_core_dev *dev = priv->mdev; + u32 data = 0; + int size_read = 0; + int ret; + + ret = mlx5_query_module_num(dev, &eeprom->module_num); + if (ret) { + if_printf(priv->ifp, "%s:%d: Failed query module error=%d\n", + __func__, __LINE__, ret); + return (ret); + } + + /* Read the first three bytes to get Identifier, Revision and Status */ + ret = mlx5_query_eeprom(dev, eeprom->i2c_addr, eeprom->page_num, + eeprom->device_addr, MLX5E_EEPROM_INFO_BYTES, eeprom->module_num, &data, + &size_read); + if (ret) { + if_printf(priv->ifp, "%s:%d: Failed query eeprom module error=0x%x\n", + __func__, __LINE__, ret); + return (ret); + } + + switch (data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK) { + case SFF_8024_ID_QSFP: + eeprom->type = MLX5E_ETH_MODULE_SFF_8436; + eeprom->len = MLX5E_ETH_MODULE_SFF_8436_LEN; + break; + case SFF_8024_ID_QSFPPLUS: + case SFF_8024_ID_QSFP28: + if ((data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK) == SFF_8024_ID_QSFP28 || + ((data & MLX5_EEPROM_REVISION_ID_BYTE_MASK) >> 8) >= 0x3) { + eeprom->type = MLX5E_ETH_MODULE_SFF_8636; + eeprom->len = MLX5E_ETH_MODULE_SFF_8636_LEN; + } else { + eeprom->type = MLX5E_ETH_MODULE_SFF_8436; + eeprom->len = MLX5E_ETH_MODULE_SFF_8436_LEN; + } + if ((data & MLX5_EEPROM_PAGE_3_VALID_BIT_MASK) == 0) + eeprom->page_valid = 1; + break; + case SFF_8024_ID_SFP: + eeprom->type = MLX5E_ETH_MODULE_SFF_8472; + eeprom->len = MLX5E_ETH_MODULE_SFF_8472_LEN; + break; + default: + if_printf(priv->ifp, "%s:%d: Not recognized cable type = 0x%x\n", + __func__, __LINE__, data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK); + return (EINVAL); + } + return (0); +} + +/* Read both low and high pages of the eeprom */ +static int +mlx5e_get_eeprom(struct mlx5e_priv *priv, struct mlx5e_eeprom *ee) +{ + struct mlx5_core_dev *dev = priv->mdev; + int size_read = 0; + int ret; + + if (ee->len == 0) + return (EINVAL); + + /* Read low page of the eeprom */ + while (ee->device_addr < ee->len) { + ret = mlx5_query_eeprom(dev, ee->i2c_addr, ee->page_num, ee->device_addr, + ee->len - ee->device_addr, ee->module_num, + ee->data + (ee->device_addr/4), &size_read); + if (ret) { + if_printf(priv->ifp, "%s:%d: Failed reading eeprom, " + "error = 0x%02x\n", __func__, __LINE__, ret); + return (ret); + } + ee->device_addr += size_read; + } + + /* Read high page of the eeprom */ + if (ee->page_valid) { + ee->device_addr = MLX5E_EEPROM_HIGH_PAGE_OFFSET; + ee->page_num = MLX5E_EEPROM_HIGH_PAGE; + size_read = 0; + while (ee->device_addr < MLX5E_EEPROM_PAGE_LENGTH) { + ret = mlx5_query_eeprom(dev, ee->i2c_addr, ee->page_num, + ee->device_addr, MLX5E_EEPROM_PAGE_LENGTH - ee->device_addr, + ee->module_num, ee->data + (ee->len/4) + + ((ee->device_addr - MLX5E_EEPROM_HIGH_PAGE_OFFSET)/4), + &size_read); + if (ret) { + if_printf(priv->ifp, "%s:%d: Failed reading eeprom, " + "error = 0x%02x\n", __func__, __LINE__, ret); + return (ret); + } + ee->device_addr += size_read; + } + } + return (0); +} + +static void +mlx5e_print_eeprom(struct mlx5e_eeprom *eeprom) +{ + int i, j = 0; + int row = 0; + + printf("\nOffset\t\tValues\n"); + printf("------\t\t------\n"); + while (row < eeprom->len) { + printf("0x%04x\t\t",row); + for (i = 0; i < 16; i++) { + printf("%02x ", ((u8*)eeprom->data)[j]); + j++; + row++; + } + printf("\n"); + } + + if (eeprom->page_valid) { + row = MLX5E_EEPROM_HIGH_PAGE_OFFSET; + printf("\nUpper Page 0x03\n"); + printf("\nOffset\t\tValues\n"); + printf("------\t\t------\n"); + while (row < MLX5E_EEPROM_PAGE_LENGTH) { + printf("0x%04x\t\t",row); + for (i = 0; i < 16; i++) { + printf("%02x ", ((u8*)eeprom->data)[j]); + j++; + row++; + } + printf("\n"); + } + } +} + +/* + * Read cable EEPROM module information by first inspecting the first + * three bytes to get the initial information for a whole reading. + * Information will be printed to dmesg. + */ +static int +mlx5e_read_eeprom(SYSCTL_HANDLER_ARGS) +{ + struct mlx5e_priv *priv = arg1; + struct mlx5e_eeprom eeprom; + int error; + int result = 0; + + PRIV_LOCK(priv); + error = sysctl_handle_int(oidp, &result, 0, req); + if (error || !req->newptr) + goto done; + + /* Check if device is gone */ + if (priv->gone) { + error = ENXIO; + goto done; + } + + if (result == 1) { + eeprom.i2c_addr = MLX5E_I2C_ADDR_LOW; + eeprom.device_addr = 0; + eeprom.page_num = MLX5E_EEPROM_LOW_PAGE; + eeprom.page_valid = 0; + + /* Read three first bytes to get important info */ + error = mlx5e_get_eeprom_info(priv, &eeprom); + if (error) { + if_printf(priv->ifp, "%s:%d: Failed reading eeprom's " + "initial information\n", __func__, __LINE__); + error = 0; + goto done; + } + + /* Allocate needed length buffer and additional space for the 3rd */ + eeprom.data = malloc(eeprom.len + MLX5E_EEPROM_PAGE_LENGTH, + M_MLX5EN, M_WAITOK | M_ZERO); + + /* Read the whole eeprom information */ + error = mlx5e_get_eeprom(priv, &eeprom); + if (error) { + if_printf(priv->ifp, "%s:%d: Failed reading eeprom\n", + __func__, __LINE__); + error = 0; + /* Continue printing partial information in case of an error */ + } + + mlx5e_print_eeprom(&eeprom); + free(eeprom.data, M_MLX5EN); + } +done: + PRIV_UNLOCK(priv); + return (error); +} + +static const char *mlx5e_params_desc[] = { + MLX5E_PARAMS(MLX5E_STATS_DESC) +}; + +static const char *mlx5e_port_stats_debug_desc[] = { + MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_DESC) +}; + +static int +mlx5e_ethtool_debug_stats(SYSCTL_HANDLER_ARGS) +{ + struct mlx5e_priv *priv = arg1; + int error; + int sys_debug; + + sys_debug = priv->sysctl_debug; + error = sysctl_handle_int(oidp, &priv->sysctl_debug, 0, req); + if (error || !req->newptr) + return (error); + priv->sysctl_debug = !!priv->sysctl_debug; + if (sys_debug == priv->sysctl_debug) + return (error); + if (priv->sysctl_debug) + mlx5e_create_stats(&priv->stats.port_stats_debug.ctx, + SYSCTL_CHILDREN(priv->sysctl_ifnet), "debug_stats", + mlx5e_port_stats_debug_desc, MLX5E_PORT_STATS_DEBUG_NUM, + priv->stats.port_stats_debug.arg); + else + sysctl_ctx_free(&priv->stats.port_stats_debug.ctx); + return (error); +} + +void +mlx5e_create_ethtool(struct mlx5e_priv *priv) +{ + struct sysctl_oid *node; + const char *pnameunit; + unsigned x; + + /* set some defaults */ + priv->params_ethtool.tx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; + priv->params_ethtool.rx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE; + priv->params_ethtool.tx_queue_size = 1 << priv->params.log_sq_size; + priv->params_ethtool.rx_queue_size = 1 << priv->params.log_rq_size; + priv->params_ethtool.channels = priv->params.num_channels; + priv->params_ethtool.coalesce_pkts_max = MLX5E_FLD_MAX(cqc, cq_max_count); + priv->params_ethtool.coalesce_usecs_max = MLX5E_FLD_MAX(cqc, cq_period); + priv->params_ethtool.rx_coalesce_mode = priv->params.rx_cq_moderation_mode; + priv->params_ethtool.rx_coalesce_usecs = priv->params.rx_cq_moderation_usec; + priv->params_ethtool.rx_coalesce_pkts = priv->params.rx_cq_moderation_pkts; + priv->params_ethtool.tx_coalesce_usecs = priv->params.tx_cq_moderation_usec; + priv->params_ethtool.tx_coalesce_pkts = priv->params.tx_cq_moderation_pkts; + priv->params_ethtool.hw_lro = priv->params.hw_lro_en; + + /* create root node */ + node = SYSCTL_ADD_NODE(&priv->sysctl_ctx, + SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO, + "conf", CTLFLAG_RW, NULL, "Configuration"); + if (node == NULL) + return; + for (x = 0; x != MLX5E_PARAMS_NUM; x++) { + /* check for read-only parameter */ + if (strstr(mlx5e_params_desc[2 * x], "_max") != NULL) { + SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, + mlx5e_params_desc[2 * x], CTLTYPE_U64 | CTLFLAG_RD | + CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU", + mlx5e_params_desc[2 * x + 1]); + } else { + SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, + mlx5e_params_desc[2 * x], CTLTYPE_U64 | CTLFLAG_RWTUN | + CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU", + mlx5e_params_desc[2 * x + 1]); + } + } + + SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, + "debug_stats", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, + 0, &mlx5e_ethtool_debug_stats, "I", "Extended debug statistics"); + + pnameunit = device_get_nameunit(priv->mdev->pdev->dev.bsddev); + + SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), + OID_AUTO, "device_name", CTLFLAG_RD, + __DECONST(void *, pnameunit), 0, + "PCI device name"); + + /* EEPROM support */ + SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "eeprom_info", + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0, + mlx5e_read_eeprom, "I", "EEPROM information"); +} + diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c new file mode 100644 index 0000000..ab9ea73 --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c @@ -0,0 +1,870 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "en.h" + +#include <linux/list.h> +#include <dev/mlx5/flow_table.h> + +enum { + MLX5E_FULLMATCH = 0, + MLX5E_ALLMULTI = 1, + MLX5E_PROMISC = 2, +}; + +enum { + MLX5E_UC = 0, + MLX5E_MC_IPV4 = 1, + MLX5E_MC_IPV6 = 2, + MLX5E_MC_OTHER = 3, +}; + +enum { + MLX5E_ACTION_NONE = 0, + MLX5E_ACTION_ADD = 1, + MLX5E_ACTION_DEL = 2, +}; + +struct mlx5e_eth_addr_hash_node { + LIST_ENTRY(mlx5e_eth_addr_hash_node) hlist; + u8 action; + struct mlx5e_eth_addr_info ai; +}; + +static inline int +mlx5e_hash_eth_addr(const u8 * addr) +{ + return (addr[5]); +} + +static void +mlx5e_add_eth_addr_to_hash(struct mlx5e_eth_addr_hash_head *hash, + const u8 * addr) +{ + struct mlx5e_eth_addr_hash_node *hn; + int ix = mlx5e_hash_eth_addr(addr); + + LIST_FOREACH(hn, &hash[ix], hlist) { + if (bcmp(hn->ai.addr, addr, ETHER_ADDR_LEN) == 0) { + if (hn->action == MLX5E_ACTION_DEL) + hn->action = MLX5E_ACTION_NONE; + return; + } + } + + hn = malloc(sizeof(*hn), M_MLX5EN, M_NOWAIT | M_ZERO); + if (hn == NULL) + return; + + ether_addr_copy(hn->ai.addr, addr); + hn->action = MLX5E_ACTION_ADD; + + LIST_INSERT_HEAD(&hash[ix], hn, hlist); +} + +static void +mlx5e_del_eth_addr_from_hash(struct mlx5e_eth_addr_hash_node *hn) +{ + LIST_REMOVE(hn, hlist); + free(hn, M_MLX5EN); +} + +static void +mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv, + struct mlx5e_eth_addr_info *ai) +{ + void *ft = priv->ft.main; + + if (ai->tt_vec & (1 << MLX5E_TT_IPV6_TCP)) + mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV6_TCP]); + + if (ai->tt_vec & (1 << MLX5E_TT_IPV4_TCP)) + mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV4_TCP]); + + if (ai->tt_vec & (1 << MLX5E_TT_IPV6_UDP)) + mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV6_UDP]); + + if (ai->tt_vec & (1 << MLX5E_TT_IPV4_UDP)) + mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV4_UDP]); + + if (ai->tt_vec & (1 << MLX5E_TT_IPV6)) + mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV6]); + + if (ai->tt_vec & (1 << MLX5E_TT_IPV4)) + mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV4]); + + if (ai->tt_vec & (1 << MLX5E_TT_ANY)) + mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_ANY]); +} + +static int +mlx5e_get_eth_addr_type(const u8 * addr) +{ + if (ETHER_IS_MULTICAST(addr) == 0) + return (MLX5E_UC); + + if ((addr[0] == 0x01) && + (addr[1] == 0x00) && + (addr[2] == 0x5e) && + !(addr[3] & 0x80)) + return (MLX5E_MC_IPV4); + + if ((addr[0] == 0x33) && + (addr[1] == 0x33)) + return (MLX5E_MC_IPV6); + + return (MLX5E_MC_OTHER); +} + +static u32 +mlx5e_get_tt_vec(struct mlx5e_eth_addr_info *ai, int type) +{ + int eth_addr_type; + u32 ret; + + switch (type) { + case MLX5E_FULLMATCH: + eth_addr_type = mlx5e_get_eth_addr_type(ai->addr); + switch (eth_addr_type) { + case MLX5E_UC: + ret = + (1 << MLX5E_TT_IPV4_TCP) | + (1 << MLX5E_TT_IPV6_TCP) | + (1 << MLX5E_TT_IPV4_UDP) | + (1 << MLX5E_TT_IPV6_UDP) | + (1 << MLX5E_TT_IPV4) | + (1 << MLX5E_TT_IPV6) | + (1 << MLX5E_TT_ANY) | + 0; + break; + + case MLX5E_MC_IPV4: + ret = + (1 << MLX5E_TT_IPV4_UDP) | + (1 << MLX5E_TT_IPV4) | + 0; + break; + + case MLX5E_MC_IPV6: + ret = + (1 << MLX5E_TT_IPV6_UDP) | + (1 << MLX5E_TT_IPV6) | + 0; + break; + + default: + ret = + (1 << MLX5E_TT_ANY) | + 0; + break; + } + break; + + case MLX5E_ALLMULTI: + ret = + (1 << MLX5E_TT_IPV4_UDP) | + (1 << MLX5E_TT_IPV6_UDP) | + (1 << MLX5E_TT_IPV4) | + (1 << MLX5E_TT_IPV6) | + (1 << MLX5E_TT_ANY) | + 0; + break; + + default: /* MLX5E_PROMISC */ + ret = + (1 << MLX5E_TT_IPV4_TCP) | + (1 << MLX5E_TT_IPV6_TCP) | + (1 << MLX5E_TT_IPV4_UDP) | + (1 << MLX5E_TT_IPV6_UDP) | + (1 << MLX5E_TT_IPV4) | + (1 << MLX5E_TT_IPV6) | + (1 << MLX5E_TT_ANY) | + 0; + break; + } + + return (ret); +} + +static int +mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv, + struct mlx5e_eth_addr_info *ai, int type, + void *flow_context, void *match_criteria) +{ + u8 match_criteria_enable = 0; + void *match_value; + void *dest; + u8 *dmac; + u8 *match_criteria_dmac; + void *ft = priv->ft.main; + u32 *tirn = priv->tirn; + u32 tt_vec; + int err; + + match_value = MLX5_ADDR_OF(flow_context, flow_context, match_value); + dmac = MLX5_ADDR_OF(fte_match_param, match_value, + outer_headers.dmac_47_16); + match_criteria_dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, + outer_headers.dmac_47_16); + dest = MLX5_ADDR_OF(flow_context, flow_context, destination); + + MLX5_SET(flow_context, flow_context, action, + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); + MLX5_SET(flow_context, flow_context, destination_list_size, 1); + MLX5_SET(dest_format_struct, dest, destination_type, + MLX5_FLOW_CONTEXT_DEST_TYPE_TIR); + + switch (type) { + case MLX5E_FULLMATCH: + match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + memset(match_criteria_dmac, 0xff, ETH_ALEN); + ether_addr_copy(dmac, ai->addr); + break; + + case MLX5E_ALLMULTI: + match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + match_criteria_dmac[0] = 0x01; + dmac[0] = 0x01; + break; + + case MLX5E_PROMISC: + break; + default: + break; + } + + tt_vec = mlx5e_get_tt_vec(ai, type); + + if (tt_vec & (1 << MLX5E_TT_ANY)) { + MLX5_SET(dest_format_struct, dest, destination_id, + tirn[MLX5E_TT_ANY]); + err = mlx5_add_flow_table_entry(ft, match_criteria_enable, + match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_ANY]); + if (err) { + mlx5e_del_eth_addr_from_flow_table(priv, ai); + return (err); + } + ai->tt_vec |= (1 << MLX5E_TT_ANY); + } + + match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, match_criteria, + outer_headers.ethertype); + + if (tt_vec & (1 << MLX5E_TT_IPV4)) { + MLX5_SET(fte_match_param, match_value, outer_headers.ethertype, + ETHERTYPE_IP); + MLX5_SET(dest_format_struct, dest, destination_id, + tirn[MLX5E_TT_IPV4]); + err = mlx5_add_flow_table_entry(ft, match_criteria_enable, + match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV4]); + if (err) { + mlx5e_del_eth_addr_from_flow_table(priv, ai); + return (err); + } + ai->tt_vec |= (1 << MLX5E_TT_IPV4); + } + + if (tt_vec & (1 << MLX5E_TT_IPV6)) { + MLX5_SET(fte_match_param, match_value, outer_headers.ethertype, + ETHERTYPE_IPV6); + MLX5_SET(dest_format_struct, dest, destination_id, + tirn[MLX5E_TT_IPV6]); + err = mlx5_add_flow_table_entry(ft, match_criteria_enable, + match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV6]); + if (err) { + mlx5e_del_eth_addr_from_flow_table(priv, ai); + return (err); + } + ai->tt_vec |= (1 << MLX5E_TT_IPV6); + } + MLX5_SET_TO_ONES(fte_match_param, match_criteria, + outer_headers.ip_protocol); + MLX5_SET(fte_match_param, match_value, outer_headers.ip_protocol, + IPPROTO_UDP); + + if (tt_vec & (1 << MLX5E_TT_IPV4_UDP)) { + MLX5_SET(fte_match_param, match_value, outer_headers.ethertype, + ETHERTYPE_IP); + MLX5_SET(dest_format_struct, dest, destination_id, + tirn[MLX5E_TT_IPV4_UDP]); + err = mlx5_add_flow_table_entry(ft, match_criteria_enable, + match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV4_UDP]); + if (err) { + mlx5e_del_eth_addr_from_flow_table(priv, ai); + return (err); + } + ai->tt_vec |= (1 << MLX5E_TT_IPV4_UDP); + } + if (tt_vec & (1 << MLX5E_TT_IPV6_UDP)) { + MLX5_SET(fte_match_param, match_value, outer_headers.ethertype, + ETHERTYPE_IPV6); + MLX5_SET(dest_format_struct, dest, destination_id, + tirn[MLX5E_TT_IPV6_UDP]); + err = mlx5_add_flow_table_entry(ft, match_criteria_enable, + match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV6_UDP]); + if (err) { + mlx5e_del_eth_addr_from_flow_table(priv, ai); + return (err); + } + ai->tt_vec |= (1 << MLX5E_TT_IPV6_UDP); + } + MLX5_SET(fte_match_param, match_value, outer_headers.ip_protocol, + IPPROTO_TCP); + + if (tt_vec & (1 << MLX5E_TT_IPV4_TCP)) { + MLX5_SET(fte_match_param, match_value, outer_headers.ethertype, + ETHERTYPE_IP); + MLX5_SET(dest_format_struct, dest, destination_id, + tirn[MLX5E_TT_IPV4_TCP]); + err = mlx5_add_flow_table_entry(ft, match_criteria_enable, + match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV4_TCP]); + if (err) { + mlx5e_del_eth_addr_from_flow_table(priv, ai); + return (err); + } + ai->tt_vec |= (1 << MLX5E_TT_IPV4_TCP); + } + if (tt_vec & (1 << MLX5E_TT_IPV6_TCP)) { + MLX5_SET(fte_match_param, match_value, outer_headers.ethertype, + ETHERTYPE_IPV6); + MLX5_SET(dest_format_struct, dest, destination_id, + tirn[MLX5E_TT_IPV6_TCP]); + err = mlx5_add_flow_table_entry(ft, match_criteria_enable, + match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV6_TCP]); + if (err) { + mlx5e_del_eth_addr_from_flow_table(priv, ai); + return (err); + } + ai->tt_vec |= (1 << MLX5E_TT_IPV6_TCP); + } + return (0); +} + +static int +mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv, + struct mlx5e_eth_addr_info *ai, int type) +{ + u32 *flow_context; + u32 *match_criteria; + int err; + + flow_context = mlx5_vzalloc(MLX5_ST_SZ_BYTES(flow_context) + + MLX5_ST_SZ_BYTES(dest_format_struct)); + match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); + if (!flow_context || !match_criteria) { + if_printf(priv->ifp, "%s: alloc failed\n", __func__); + err = -ENOMEM; + goto add_eth_addr_rule_out; + } + + err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, flow_context, + match_criteria); + if (err) + if_printf(priv->ifp, "%s: failed\n", __func__); + +add_eth_addr_rule_out: + kvfree(match_criteria); + kvfree(flow_context); + return (err); +} + +enum mlx5e_vlan_rule_type { + MLX5E_VLAN_RULE_TYPE_UNTAGGED, + MLX5E_VLAN_RULE_TYPE_ANY_VID, + MLX5E_VLAN_RULE_TYPE_MATCH_VID, +}; + +static int +mlx5e_add_vlan_rule(struct mlx5e_priv *priv, + enum mlx5e_vlan_rule_type rule_type, u16 vid) +{ + u8 match_criteria_enable = 0; + u32 *flow_context; + void *match_value; + void *dest; + u32 *match_criteria; + u32 *ft_ix; + int err; + + flow_context = mlx5_vzalloc(MLX5_ST_SZ_BYTES(flow_context) + + MLX5_ST_SZ_BYTES(dest_format_struct)); + match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param)); + if (!flow_context || !match_criteria) { + if_printf(priv->ifp, "%s: alloc failed\n", __func__); + err = -ENOMEM; + goto add_vlan_rule_out; + } + match_value = MLX5_ADDR_OF(flow_context, flow_context, match_value); + dest = MLX5_ADDR_OF(flow_context, flow_context, destination); + + MLX5_SET(flow_context, flow_context, action, + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); + MLX5_SET(flow_context, flow_context, destination_list_size, 1); + MLX5_SET(dest_format_struct, dest, destination_type, + MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE); + MLX5_SET(dest_format_struct, dest, destination_id, + mlx5_get_flow_table_id(priv->ft.main)); + + match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, match_criteria, + outer_headers.vlan_tag); + + switch (rule_type) { + case MLX5E_VLAN_RULE_TYPE_UNTAGGED: + ft_ix = &priv->vlan.untagged_rule_ft_ix; + break; + case MLX5E_VLAN_RULE_TYPE_ANY_VID: + ft_ix = &priv->vlan.any_vlan_rule_ft_ix; + MLX5_SET(fte_match_param, match_value, outer_headers.vlan_tag, + 1); + break; + default: /* MLX5E_VLAN_RULE_TYPE_MATCH_VID */ + ft_ix = &priv->vlan.active_vlans_ft_ix[vid]; + MLX5_SET(fte_match_param, match_value, outer_headers.vlan_tag, + 1); + MLX5_SET_TO_ONES(fte_match_param, match_criteria, + outer_headers.first_vid); + MLX5_SET(fte_match_param, match_value, outer_headers.first_vid, + vid); + break; + } + + err = mlx5_add_flow_table_entry(priv->ft.vlan, match_criteria_enable, + match_criteria, flow_context, ft_ix); + if (err) + if_printf(priv->ifp, "%s: failed\n", __func__); + +add_vlan_rule_out: + kvfree(match_criteria); + kvfree(flow_context); + return (err); +} + +static void +mlx5e_del_vlan_rule(struct mlx5e_priv *priv, + enum mlx5e_vlan_rule_type rule_type, u16 vid) +{ + switch (rule_type) { + case MLX5E_VLAN_RULE_TYPE_UNTAGGED: + mlx5_del_flow_table_entry(priv->ft.vlan, + priv->vlan.untagged_rule_ft_ix); + break; + case MLX5E_VLAN_RULE_TYPE_ANY_VID: + mlx5_del_flow_table_entry(priv->ft.vlan, + priv->vlan.any_vlan_rule_ft_ix); + break; + case MLX5E_VLAN_RULE_TYPE_MATCH_VID: + mlx5_del_flow_table_entry(priv->ft.vlan, + priv->vlan.active_vlans_ft_ix[vid]); + break; + } +} + +void +mlx5e_enable_vlan_filter(struct mlx5e_priv *priv) +{ + if (priv->vlan.filter_disabled) { + priv->vlan.filter_disabled = false; + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) + mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID, + 0); + } +} + +void +mlx5e_disable_vlan_filter(struct mlx5e_priv *priv) +{ + if (!priv->vlan.filter_disabled) { + priv->vlan.filter_disabled = true; + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) + mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID, + 0); + } +} + +void +mlx5e_vlan_rx_add_vid(void *arg, struct ifnet *ifp, u16 vid) +{ + struct mlx5e_priv *priv = arg; + + if (ifp != priv->ifp) + return; + + PRIV_LOCK(priv); + set_bit(vid, priv->vlan.active_vlans); + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) + mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid); + PRIV_UNLOCK(priv); +} + +void +mlx5e_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, u16 vid) +{ + struct mlx5e_priv *priv = arg; + + if (ifp != priv->ifp) + return; + + PRIV_LOCK(priv); + clear_bit(vid, priv->vlan.active_vlans); + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) + mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid); + PRIV_UNLOCK(priv); +} + +int +mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv) +{ + u16 vid; + int err; + + for_each_set_bit(vid, priv->vlan.active_vlans, VLAN_N_VID) { + err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, + vid); + if (err) + return (err); + } + + err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); + if (err) + return (err); + + if (priv->vlan.filter_disabled) { + err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID, + 0); + if (err) + return (err); + } + return (0); +} + +void +mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv) +{ + u16 vid; + + if (priv->vlan.filter_disabled) + mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID, 0); + + mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); + + for_each_set_bit(vid, priv->vlan.active_vlans, VLAN_N_VID) + mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid); +} + +#define mlx5e_for_each_hash_node(hn, tmp, hash, i) \ + for (i = 0; i < MLX5E_ETH_ADDR_HASH_SIZE; i++) \ + LIST_FOREACH_SAFE(hn, &(hash)[i], hlist, tmp) + +static void +mlx5e_execute_action(struct mlx5e_priv *priv, + struct mlx5e_eth_addr_hash_node *hn) +{ + switch (hn->action) { + case MLX5E_ACTION_ADD: + mlx5e_add_eth_addr_rule(priv, &hn->ai, MLX5E_FULLMATCH); + hn->action = MLX5E_ACTION_NONE; + break; + + case MLX5E_ACTION_DEL: + mlx5e_del_eth_addr_from_flow_table(priv, &hn->ai); + mlx5e_del_eth_addr_from_hash(hn); + break; + + default: + break; + } +} + +static void +mlx5e_sync_ifp_addr(struct mlx5e_priv *priv) +{ + struct ifnet *ifp = priv->ifp; + struct ifaddr *ifa; + struct ifmultiaddr *ifma; + + /* XXX adding this entry might not be needed */ + mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_uc, + LLADDR((struct sockaddr_dl *)(ifp->if_addr->ifa_addr))); + + if_addr_rlock(ifp); + TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + if (ifa->ifa_addr->sa_family != AF_LINK) + continue; + mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_uc, + LLADDR((struct sockaddr_dl *)ifa->ifa_addr)); + } + if_addr_runlock(ifp); + + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_mc, + LLADDR((struct sockaddr_dl *)ifma->ifma_addr)); + } + if_maddr_runlock(ifp); +} + +static void +mlx5e_apply_ifp_addr(struct mlx5e_priv *priv) +{ + struct mlx5e_eth_addr_hash_node *hn; + struct mlx5e_eth_addr_hash_node *tmp; + int i; + + mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i) + mlx5e_execute_action(priv, hn); + + mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i) + mlx5e_execute_action(priv, hn); +} + +static void +mlx5e_handle_ifp_addr(struct mlx5e_priv *priv) +{ + struct mlx5e_eth_addr_hash_node *hn; + struct mlx5e_eth_addr_hash_node *tmp; + int i; + + mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i) + hn->action = MLX5E_ACTION_DEL; + mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i) + hn->action = MLX5E_ACTION_DEL; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) + mlx5e_sync_ifp_addr(priv); + + mlx5e_apply_ifp_addr(priv); +} + +void +mlx5e_set_rx_mode_core(struct mlx5e_priv *priv) +{ + struct mlx5e_eth_addr_db *ea = &priv->eth_addr; + struct ifnet *ndev = priv->ifp; + + bool rx_mode_enable = test_bit(MLX5E_STATE_OPENED, &priv->state); + bool promisc_enabled = rx_mode_enable && (ndev->if_flags & IFF_PROMISC); + bool allmulti_enabled = rx_mode_enable && (ndev->if_flags & IFF_ALLMULTI); + bool broadcast_enabled = rx_mode_enable; + + bool enable_promisc = !ea->promisc_enabled && promisc_enabled; + bool disable_promisc = ea->promisc_enabled && !promisc_enabled; + bool enable_allmulti = !ea->allmulti_enabled && allmulti_enabled; + bool disable_allmulti = ea->allmulti_enabled && !allmulti_enabled; + bool enable_broadcast = !ea->broadcast_enabled && broadcast_enabled; + bool disable_broadcast = ea->broadcast_enabled && !broadcast_enabled; + + /* update broadcast address */ + ether_addr_copy(priv->eth_addr.broadcast.addr, + priv->ifp->if_broadcastaddr); + + if (enable_promisc) + mlx5e_add_eth_addr_rule(priv, &ea->promisc, MLX5E_PROMISC); + if (enable_allmulti) + mlx5e_add_eth_addr_rule(priv, &ea->allmulti, MLX5E_ALLMULTI); + if (enable_broadcast) + mlx5e_add_eth_addr_rule(priv, &ea->broadcast, MLX5E_FULLMATCH); + + mlx5e_handle_ifp_addr(priv); + + if (disable_broadcast) + mlx5e_del_eth_addr_from_flow_table(priv, &ea->broadcast); + if (disable_allmulti) + mlx5e_del_eth_addr_from_flow_table(priv, &ea->allmulti); + if (disable_promisc) + mlx5e_del_eth_addr_from_flow_table(priv, &ea->promisc); + + ea->promisc_enabled = promisc_enabled; + ea->allmulti_enabled = allmulti_enabled; + ea->broadcast_enabled = broadcast_enabled; +} + +void +mlx5e_set_rx_mode_work(struct work_struct *work) +{ + struct mlx5e_priv *priv = + container_of(work, struct mlx5e_priv, set_rx_mode_work); + + PRIV_LOCK(priv); + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) + mlx5e_set_rx_mode_core(priv); + PRIV_UNLOCK(priv); +} + +static int +mlx5e_create_main_flow_table(struct mlx5e_priv *priv) +{ + struct mlx5_flow_table_group *g; + u8 *dmac; + + g = malloc(9 * sizeof(*g), M_MLX5EN, M_WAITOK | M_ZERO); + if (g == NULL) + return (-ENOMEM); + + g[0].log_sz = 2; + g[0].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria, + outer_headers.ethertype); + MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria, + outer_headers.ip_protocol); + + g[1].log_sz = 1; + g[1].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, g[1].match_criteria, + outer_headers.ethertype); + + g[2].log_sz = 0; + + g[3].log_sz = 14; + g[3].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + dmac = MLX5_ADDR_OF(fte_match_param, g[3].match_criteria, + outer_headers.dmac_47_16); + memset(dmac, 0xff, ETH_ALEN); + MLX5_SET_TO_ONES(fte_match_param, g[3].match_criteria, + outer_headers.ethertype); + MLX5_SET_TO_ONES(fte_match_param, g[3].match_criteria, + outer_headers.ip_protocol); + + g[4].log_sz = 13; + g[4].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + dmac = MLX5_ADDR_OF(fte_match_param, g[4].match_criteria, + outer_headers.dmac_47_16); + memset(dmac, 0xff, ETH_ALEN); + MLX5_SET_TO_ONES(fte_match_param, g[4].match_criteria, + outer_headers.ethertype); + + g[5].log_sz = 11; + g[5].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + dmac = MLX5_ADDR_OF(fte_match_param, g[5].match_criteria, + outer_headers.dmac_47_16); + memset(dmac, 0xff, ETH_ALEN); + + g[6].log_sz = 2; + g[6].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + dmac = MLX5_ADDR_OF(fte_match_param, g[6].match_criteria, + outer_headers.dmac_47_16); + dmac[0] = 0x01; + MLX5_SET_TO_ONES(fte_match_param, g[6].match_criteria, + outer_headers.ethertype); + MLX5_SET_TO_ONES(fte_match_param, g[6].match_criteria, + outer_headers.ip_protocol); + + g[7].log_sz = 1; + g[7].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + dmac = MLX5_ADDR_OF(fte_match_param, g[7].match_criteria, + outer_headers.dmac_47_16); + dmac[0] = 0x01; + MLX5_SET_TO_ONES(fte_match_param, g[7].match_criteria, + outer_headers.ethertype); + + g[8].log_sz = 0; + g[8].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + dmac = MLX5_ADDR_OF(fte_match_param, g[8].match_criteria, + outer_headers.dmac_47_16); + dmac[0] = 0x01; + priv->ft.main = mlx5_create_flow_table(priv->mdev, 1, + MLX5_FLOW_TABLE_TYPE_NIC_RCV, + 0, 9, g); + free(g, M_MLX5EN); + + return (priv->ft.main ? 0 : -ENOMEM); +} + +static void +mlx5e_destroy_main_flow_table(struct mlx5e_priv *priv) +{ + mlx5_destroy_flow_table(priv->ft.main); + priv->ft.main = NULL; +} + +static int +mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv) +{ + struct mlx5_flow_table_group *g; + + g = malloc(2 * sizeof(*g), M_MLX5EN, M_WAITOK | M_ZERO); + if (g == NULL) + return (-ENOMEM); + + g[0].log_sz = 12; + g[0].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria, + outer_headers.vlan_tag); + MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria, + outer_headers.first_vid); + + /* untagged + any vlan id */ + g[1].log_sz = 1; + g[1].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, g[1].match_criteria, + outer_headers.vlan_tag); + + priv->ft.vlan = mlx5_create_flow_table(priv->mdev, 0, + MLX5_FLOW_TABLE_TYPE_NIC_RCV, + 0, 2, g); + free(g, M_MLX5EN); + + return (priv->ft.vlan ? 0 : -ENOMEM); +} + +static void +mlx5e_destroy_vlan_flow_table(struct mlx5e_priv *priv) +{ + mlx5_destroy_flow_table(priv->ft.vlan); + priv->ft.vlan = NULL; +} + +int +mlx5e_open_flow_table(struct mlx5e_priv *priv) +{ + int err; + + err = mlx5e_create_main_flow_table(priv); + if (err) + return (err); + + err = mlx5e_create_vlan_flow_table(priv); + if (err) + goto err_destroy_main_flow_table; + + return (0); + +err_destroy_main_flow_table: + mlx5e_destroy_main_flow_table(priv); + + return (err); +} + +void +mlx5e_close_flow_table(struct mlx5e_priv *priv) +{ + mlx5e_destroy_vlan_flow_table(priv); + mlx5e_destroy_main_flow_table(priv); +} diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c new file mode 100644 index 0000000..e50252c --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c @@ -0,0 +1,2902 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "en.h" + +#include <sys/sockio.h> +#include <machine/atomic.h> + +#define ETH_DRIVER_VERSION "3.1.0-dev" +char mlx5e_version[] = "Mellanox Ethernet driver" + " (" ETH_DRIVER_VERSION ")"; + +struct mlx5e_rq_param { + u32 rqc [MLX5_ST_SZ_DW(rqc)]; + struct mlx5_wq_param wq; +}; + +struct mlx5e_sq_param { + u32 sqc [MLX5_ST_SZ_DW(sqc)]; + struct mlx5_wq_param wq; +}; + +struct mlx5e_cq_param { + u32 cqc [MLX5_ST_SZ_DW(cqc)]; + struct mlx5_wq_param wq; + u16 eq_ix; +}; + +struct mlx5e_channel_param { + struct mlx5e_rq_param rq; + struct mlx5e_sq_param sq; + struct mlx5e_cq_param rx_cq; + struct mlx5e_cq_param tx_cq; +}; + +static const struct { + u32 subtype; + u64 baudrate; +} mlx5e_mode_table[MLX5E_LINK_MODES_NUMBER] = { + + [MLX5E_1000BASE_CX_SGMII] = { + .subtype = IFM_1000_CX_SGMII, + .baudrate = IF_Mbps(1000ULL), + }, + [MLX5E_1000BASE_KX] = { + .subtype = IFM_1000_KX, + .baudrate = IF_Mbps(1000ULL), + }, + [MLX5E_10GBASE_CX4] = { + .subtype = IFM_10G_CX4, + .baudrate = IF_Gbps(10ULL), + }, + [MLX5E_10GBASE_KX4] = { + .subtype = IFM_10G_KX4, + .baudrate = IF_Gbps(10ULL), + }, + [MLX5E_10GBASE_KR] = { + .subtype = IFM_10G_KR, + .baudrate = IF_Gbps(10ULL), + }, + [MLX5E_20GBASE_KR2] = { + .subtype = IFM_20G_KR2, + .baudrate = IF_Gbps(20ULL), + }, + [MLX5E_40GBASE_CR4] = { + .subtype = IFM_40G_CR4, + .baudrate = IF_Gbps(40ULL), + }, + [MLX5E_40GBASE_KR4] = { + .subtype = IFM_40G_KR4, + .baudrate = IF_Gbps(40ULL), + }, + [MLX5E_56GBASE_R4] = { + .subtype = IFM_56G_R4, + .baudrate = IF_Gbps(56ULL), + }, + [MLX5E_10GBASE_CR] = { + .subtype = IFM_10G_CR1, + .baudrate = IF_Gbps(10ULL), + }, + [MLX5E_10GBASE_SR] = { + .subtype = IFM_10G_SR, + .baudrate = IF_Gbps(10ULL), + }, + [MLX5E_10GBASE_ER] = { + .subtype = IFM_10G_ER, + .baudrate = IF_Gbps(10ULL), + }, + [MLX5E_40GBASE_SR4] = { + .subtype = IFM_40G_SR4, + .baudrate = IF_Gbps(40ULL), + }, + [MLX5E_40GBASE_LR4] = { + .subtype = IFM_40G_LR4, + .baudrate = IF_Gbps(40ULL), + }, + [MLX5E_100GBASE_CR4] = { + .subtype = IFM_100G_CR4, + .baudrate = IF_Gbps(100ULL), + }, + [MLX5E_100GBASE_SR4] = { + .subtype = IFM_100G_SR4, + .baudrate = IF_Gbps(100ULL), + }, + [MLX5E_100GBASE_KR4] = { + .subtype = IFM_100G_KR4, + .baudrate = IF_Gbps(100ULL), + }, + [MLX5E_100GBASE_LR4] = { + .subtype = IFM_100G_LR4, + .baudrate = IF_Gbps(100ULL), + }, + [MLX5E_100BASE_TX] = { + .subtype = IFM_100_TX, + .baudrate = IF_Mbps(100ULL), + }, + [MLX5E_100BASE_T] = { + .subtype = IFM_100_T, + .baudrate = IF_Mbps(100ULL), + }, + [MLX5E_10GBASE_T] = { + .subtype = IFM_10G_T, + .baudrate = IF_Gbps(10ULL), + }, + [MLX5E_25GBASE_CR] = { + .subtype = IFM_25G_CR, + .baudrate = IF_Gbps(25ULL), + }, + [MLX5E_25GBASE_KR] = { + .subtype = IFM_25G_KR, + .baudrate = IF_Gbps(25ULL), + }, + [MLX5E_25GBASE_SR] = { + .subtype = IFM_25G_SR, + .baudrate = IF_Gbps(25ULL), + }, + [MLX5E_50GBASE_CR2] = { + .subtype = IFM_50G_CR2, + .baudrate = IF_Gbps(50ULL), + }, + [MLX5E_50GBASE_KR2] = { + .subtype = IFM_50G_KR2, + .baudrate = IF_Gbps(50ULL), + }, +}; + +MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet"); + +static void +mlx5e_update_carrier(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + u32 out[MLX5_ST_SZ_DW(ptys_reg)]; + u32 eth_proto_oper; + int error; + u8 port_state; + u8 i; + + port_state = mlx5_query_vport_state(mdev, + MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT); + + if (port_state == VPORT_STATE_UP) { + priv->media_status_last |= IFM_ACTIVE; + } else { + priv->media_status_last &= ~IFM_ACTIVE; + priv->media_active_last = IFM_ETHER; + if_link_state_change(priv->ifp, LINK_STATE_DOWN); + return; + } + + error = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN); + if (error) { + priv->media_active_last = IFM_ETHER; + priv->ifp->if_baudrate = 1; + if_printf(priv->ifp, "%s: query port ptys failed: 0x%x\n", + __func__, error); + return; + } + eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper); + + for (i = 0; i != MLX5E_LINK_MODES_NUMBER; i++) { + if (mlx5e_mode_table[i].baudrate == 0) + continue; + if (MLX5E_PROT_MASK(i) & eth_proto_oper) { + priv->ifp->if_baudrate = + mlx5e_mode_table[i].baudrate; + priv->media_active_last = + mlx5e_mode_table[i].subtype | IFM_ETHER | IFM_FDX; + } + } + if_link_state_change(priv->ifp, LINK_STATE_UP); +} + +static void +mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr) +{ + struct mlx5e_priv *priv = dev->if_softc; + + ifmr->ifm_status = priv->media_status_last; + ifmr->ifm_active = priv->media_active_last | + (priv->params_ethtool.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) | + (priv->params_ethtool.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0); + +} + +static u32 +mlx5e_find_link_mode(u32 subtype) +{ + u32 i; + u32 link_mode = 0; + + for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) { + if (mlx5e_mode_table[i].baudrate == 0) + continue; + if (mlx5e_mode_table[i].subtype == subtype) + link_mode |= MLX5E_PROT_MASK(i); + } + + return (link_mode); +} + +static int +mlx5e_media_change(struct ifnet *dev) +{ + struct mlx5e_priv *priv = dev->if_softc; + struct mlx5_core_dev *mdev = priv->mdev; + u32 eth_proto_cap; + u32 link_mode; + int locked; + int error; + + locked = PRIV_LOCKED(priv); + if (!locked) + PRIV_LOCK(priv); + + if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) { + error = EINVAL; + goto done; + } + + link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media)); + + error = mlx5_query_port_proto_cap(mdev, ð_proto_cap, MLX5_PTYS_EN); + if (error) { + if_printf(dev, "Query port media capability failed\n"); + goto done; + } + if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) + link_mode = eth_proto_cap; + else + link_mode = link_mode & eth_proto_cap; + + if (!link_mode) { + if_printf(dev, "Not supported link mode requested\n"); + error = EINVAL; + goto done; + } + + mlx5_set_port_status(mdev, MLX5_PORT_DOWN); + mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN); + mlx5_set_port_status(mdev, MLX5_PORT_UP); + +done: + if (!locked) + PRIV_UNLOCK(priv); + return (error); +} + +static void +mlx5e_update_carrier_work(struct work_struct *work) +{ + struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, + update_carrier_work); + + PRIV_LOCK(priv); + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) + mlx5e_update_carrier(priv); + PRIV_UNLOCK(priv); +} + +static void +mlx5e_update_pport_counters(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_pport_stats *s = &priv->stats.pport; + struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug; + u32 *in; + u32 *out; + u64 *ptr; + unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg); + unsigned x; + unsigned y; + + in = mlx5_vzalloc(sz); + out = mlx5_vzalloc(sz); + if (in == NULL || out == NULL) + goto free_out; + + ptr = (uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set); + + MLX5_SET(ppcnt_reg, in, local_port, 1); + + MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP); + mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); + for (x = y = 0; x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++) + s->arg[y] = be64toh(ptr[x]); + + MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP); + mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); + for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++) + s->arg[y] = be64toh(ptr[x]); + for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM + + MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++) + s_debug->arg[y] = be64toh(ptr[x]); + + MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP); + mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); + for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++) + s_debug->arg[y] = be64toh(ptr[x]); + + MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP); + mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); + for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++) + s_debug->arg[y] = be64toh(ptr[x]); +free_out: + kvfree(in); + kvfree(out); +} + +static void +mlx5e_update_stats_work(struct work_struct *work) +{ + struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, + update_stats_work); + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_vport_stats *s = &priv->stats.vport; + struct mlx5e_rq_stats *rq_stats; + struct mlx5e_sq_stats *sq_stats; + struct buf_ring *sq_br; +#if (__FreeBSD_version < 1100000) + struct ifnet *ifp = priv->ifp; +#endif + u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)]; + u32 *out; + int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out); + u64 tso_packets = 0; + u64 tso_bytes = 0; + u64 tx_queue_dropped = 0; + u64 tx_defragged = 0; + u64 tx_offload_none = 0; + u64 lro_packets = 0; + u64 lro_bytes = 0; + u64 sw_lro_queued = 0; + u64 sw_lro_flushed = 0; + u64 rx_csum_none = 0; + u64 rx_wqe_err = 0; + u32 out_of_rx_buffer = 0; + int i; + int j; + + PRIV_LOCK(priv); + out = mlx5_vzalloc(outlen); + if (out == NULL) + goto free_out; + if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0) + goto free_out; + + /* Collect firts the SW counters and then HW for consistency */ + for (i = 0; i < priv->params.num_channels; i++) { + struct mlx5e_rq *rq = &priv->channel[i]->rq; + + rq_stats = &priv->channel[i]->rq.stats; + + /* collect stats from LRO */ + rq_stats->sw_lro_queued = rq->lro.lro_queued; + rq_stats->sw_lro_flushed = rq->lro.lro_flushed; + sw_lro_queued += rq_stats->sw_lro_queued; + sw_lro_flushed += rq_stats->sw_lro_flushed; + lro_packets += rq_stats->lro_packets; + lro_bytes += rq_stats->lro_bytes; + rx_csum_none += rq_stats->csum_none; + rx_wqe_err += rq_stats->wqe_err; + + for (j = 0; j < priv->num_tc; j++) { + sq_stats = &priv->channel[i]->sq[j].stats; + sq_br = priv->channel[i]->sq[j].br; + + tso_packets += sq_stats->tso_packets; + tso_bytes += sq_stats->tso_bytes; + tx_queue_dropped += sq_stats->dropped; + tx_queue_dropped += sq_br->br_drops; + tx_defragged += sq_stats->defragged; + tx_offload_none += sq_stats->csum_offload_none; + } + } + + /* update counters */ + s->tso_packets = tso_packets; + s->tso_bytes = tso_bytes; + s->tx_queue_dropped = tx_queue_dropped; + s->tx_defragged = tx_defragged; + s->lro_packets = lro_packets; + s->lro_bytes = lro_bytes; + s->sw_lro_queued = sw_lro_queued; + s->sw_lro_flushed = sw_lro_flushed; + s->rx_csum_none = rx_csum_none; + s->rx_wqe_err = rx_wqe_err; + + /* HW counters */ + memset(in, 0, sizeof(in)); + + MLX5_SET(query_vport_counter_in, in, opcode, + MLX5_CMD_OP_QUERY_VPORT_COUNTER); + MLX5_SET(query_vport_counter_in, in, op_mod, 0); + MLX5_SET(query_vport_counter_in, in, other_vport, 0); + + memset(out, 0, outlen); + + if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen)) + goto free_out; + +#define MLX5_GET_CTR(out, x) \ + MLX5_GET64(query_vport_counter_out, out, x) + + s->rx_error_packets = + MLX5_GET_CTR(out, received_errors.packets); + s->rx_error_bytes = + MLX5_GET_CTR(out, received_errors.octets); + s->tx_error_packets = + MLX5_GET_CTR(out, transmit_errors.packets); + s->tx_error_bytes = + MLX5_GET_CTR(out, transmit_errors.octets); + + s->rx_unicast_packets = + MLX5_GET_CTR(out, received_eth_unicast.packets); + s->rx_unicast_bytes = + MLX5_GET_CTR(out, received_eth_unicast.octets); + s->tx_unicast_packets = + MLX5_GET_CTR(out, transmitted_eth_unicast.packets); + s->tx_unicast_bytes = + MLX5_GET_CTR(out, transmitted_eth_unicast.octets); + + s->rx_multicast_packets = + MLX5_GET_CTR(out, received_eth_multicast.packets); + s->rx_multicast_bytes = + MLX5_GET_CTR(out, received_eth_multicast.octets); + s->tx_multicast_packets = + MLX5_GET_CTR(out, transmitted_eth_multicast.packets); + s->tx_multicast_bytes = + MLX5_GET_CTR(out, transmitted_eth_multicast.octets); + + s->rx_broadcast_packets = + MLX5_GET_CTR(out, received_eth_broadcast.packets); + s->rx_broadcast_bytes = + MLX5_GET_CTR(out, received_eth_broadcast.octets); + s->tx_broadcast_packets = + MLX5_GET_CTR(out, transmitted_eth_broadcast.packets); + s->tx_broadcast_bytes = + MLX5_GET_CTR(out, transmitted_eth_broadcast.octets); + + s->rx_packets = + s->rx_unicast_packets + + s->rx_multicast_packets + + s->rx_broadcast_packets; + s->rx_bytes = + s->rx_unicast_bytes + + s->rx_multicast_bytes + + s->rx_broadcast_bytes; + s->tx_packets = + s->tx_unicast_packets + + s->tx_multicast_packets + + s->tx_broadcast_packets; + s->tx_bytes = + s->tx_unicast_bytes + + s->tx_multicast_bytes + + s->tx_broadcast_bytes; + + /* Update calculated offload counters */ + s->tx_csum_offload = s->tx_packets - tx_offload_none; + s->rx_csum_good = s->rx_packets - s->rx_csum_none; + +#if (__FreeBSD_version < 1100000) + /* no get_counters interface in fbsd 10 */ + ifp->if_ipackets = s->rx_packets; + ifp->if_ierrors = s->rx_error_packets; + ifp->if_opackets = s->tx_packets; + ifp->if_oerrors = s->tx_error_packets; + ifp->if_snd.ifq_drops = s->tx_queue_dropped; + ifp->if_ibytes = s->rx_bytes; + ifp->if_obytes = s->tx_bytes; +#endif + + mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id, + &out_of_rx_buffer); + + /* Update per port counters */ + mlx5e_update_pport_counters(priv); + priv->stats.pport.out_of_rx_buffer = (u64)out_of_rx_buffer; +free_out: + kvfree(out); + PRIV_UNLOCK(priv); +} + +static void +mlx5e_update_stats(void *arg) +{ + struct mlx5e_priv *priv = arg; + + schedule_work(&priv->update_stats_work); + + callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv); +} + +static void +mlx5e_async_event_sub(struct mlx5e_priv *priv, + enum mlx5_dev_event event) +{ + switch (event) { + case MLX5_DEV_EVENT_PORT_UP: + case MLX5_DEV_EVENT_PORT_DOWN: + schedule_work(&priv->update_carrier_work); + break; + + default: + break; + } +} + +static void +mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv, + enum mlx5_dev_event event, unsigned long param) +{ + struct mlx5e_priv *priv = vpriv; + + mtx_lock(&priv->async_events_mtx); + if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state)) + mlx5e_async_event_sub(priv, event); + mtx_unlock(&priv->async_events_mtx); +} + +static void +mlx5e_enable_async_events(struct mlx5e_priv *priv) +{ + set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state); +} + +static void +mlx5e_disable_async_events(struct mlx5e_priv *priv) +{ + mtx_lock(&priv->async_events_mtx); + clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state); + mtx_unlock(&priv->async_events_mtx); +} + +static const char *mlx5e_rq_stats_desc[] = { + MLX5E_RQ_STATS(MLX5E_STATS_DESC) +}; + +static int +mlx5e_create_rq(struct mlx5e_channel *c, + struct mlx5e_rq_param *param, + struct mlx5e_rq *rq) +{ + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + char buffer[16]; + void *rqc = param->rqc; + void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); + int wq_sz; + int err; + int i; + + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MJUM16BYTES, /* maxsize */ + 1, /* nsegments */ + MJUM16BYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &rq->dma_tag))) + goto done; + + err = mlx5_wq_ll_create(mdev, ¶m->wq, rqc_wq, &rq->wq, + &rq->wq_ctrl); + if (err) + goto err_free_dma_tag; + + rq->wq.db = &rq->wq.db[MLX5_RCV_DBR]; + + if (priv->params.hw_lro_en) { + rq->wqe_sz = priv->params.lro_wqe_sz; + } + else { + rq->wqe_sz = MLX5E_SW2MB_MTU(priv->ifp->if_mtu); + } + if (rq->wqe_sz > MJUM16BYTES) { + err = -ENOMEM; + goto err_rq_wq_destroy; + } else if (rq->wqe_sz > MJUM9BYTES) { + rq->wqe_sz = MJUM16BYTES; + } else if (rq->wqe_sz > MJUMPAGESIZE) { + rq->wqe_sz = MJUM9BYTES; + } else if (rq->wqe_sz > MCLBYTES) { + rq->wqe_sz = MJUMPAGESIZE; + } else { + rq->wqe_sz = MCLBYTES; + } + + wq_sz = mlx5_wq_ll_get_size(&rq->wq); + rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO); + if (rq->mbuf == NULL) { + err = -ENOMEM; + goto err_rq_wq_destroy; + } + + for (i = 0; i != wq_sz; i++) { + struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i); + uint32_t byte_count = rq->wqe_sz - MLX5E_NET_IP_ALIGN; + + err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map); + if (err != 0) { + while (i--) + bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map); + goto err_rq_mbuf_free; + } + wqe->data.lkey = c->mkey_be; + wqe->data.byte_count = cpu_to_be32(byte_count | MLX5_HW_START_PADDING); + } + + rq->pdev = c->pdev; + rq->ifp = c->ifp; + rq->channel = c; + rq->ix = c->ix; + + snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix); + mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), + buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM, + rq->stats.arg); + +#ifdef HAVE_TURBO_LRO + if (tcp_tlro_init(&rq->lro, c->ifp, MLX5E_BUDGET_MAX) != 0) + rq->lro.mbuf = NULL; +#else + if (tcp_lro_init(&rq->lro)) + rq->lro.lro_cnt = 0; + else + rq->lro.ifp = c->ifp; +#endif + return (0); + +err_rq_mbuf_free: + free(rq->mbuf, M_MLX5EN); +err_rq_wq_destroy: + mlx5_wq_destroy(&rq->wq_ctrl); +err_free_dma_tag: + bus_dma_tag_destroy(rq->dma_tag); +done: + return (err); +} + +static void +mlx5e_destroy_rq(struct mlx5e_rq *rq) +{ + int wq_sz; + int i; + + /* destroy all sysctl nodes */ + sysctl_ctx_free(&rq->stats.ctx); + + /* free leftover LRO packets, if any */ +#ifdef HAVE_TURBO_LRO + tcp_tlro_free(&rq->lro); +#else + tcp_lro_free(&rq->lro); +#endif + wq_sz = mlx5_wq_ll_get_size(&rq->wq); + for (i = 0; i != wq_sz; i++) { + if (rq->mbuf[i].mbuf != NULL) { + bus_dmamap_unload(rq->dma_tag, + rq->mbuf[i].dma_map); + m_freem(rq->mbuf[i].mbuf); + } + bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map); + } + free(rq->mbuf, M_MLX5EN); + mlx5_wq_destroy(&rq->wq_ctrl); +} + +static int +mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) +{ + struct mlx5e_channel *c = rq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + + void *in; + void *rqc; + void *wq; + int inlen; + int err; + + inlen = MLX5_ST_SZ_BYTES(create_rq_in) + + sizeof(u64) * rq->wq_ctrl.buf.npages; + in = mlx5_vzalloc(inlen); + if (in == NULL) + return (-ENOMEM); + + rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); + wq = MLX5_ADDR_OF(rqc, rqc, wq); + + memcpy(rqc, param->rqc, sizeof(param->rqc)); + + MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn); + MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); + MLX5_SET(rqc, rqc, flush_in_error_en, 1); + if (priv->counter_set_id >= 0) + MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id); + MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - + PAGE_SHIFT); + MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); + + mlx5_fill_page_array(&rq->wq_ctrl.buf, + (__be64 *) MLX5_ADDR_OF(wq, wq, pas)); + + err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); + + kvfree(in); + + return (err); +} + +static int +mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state) +{ + struct mlx5e_channel *c = rq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + + void *in; + void *rqc; + int inlen; + int err; + + inlen = MLX5_ST_SZ_BYTES(modify_rq_in); + in = mlx5_vzalloc(inlen); + if (in == NULL) + return (-ENOMEM); + + rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); + + MLX5_SET(modify_rq_in, in, rqn, rq->rqn); + MLX5_SET(modify_rq_in, in, rq_state, curr_state); + MLX5_SET(rqc, rqc, state, next_state); + + err = mlx5_core_modify_rq(mdev, in, inlen); + + kvfree(in); + + return (err); +} + +static void +mlx5e_disable_rq(struct mlx5e_rq *rq) +{ + struct mlx5e_channel *c = rq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + + mlx5_core_destroy_rq(mdev, rq->rqn); +} + +static int +mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq) +{ + struct mlx5e_channel *c = rq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_wq_ll *wq = &rq->wq; + int i; + + for (i = 0; i < 1000; i++) { + if (wq->cur_sz >= priv->params.min_rx_wqes) + return (0); + + msleep(4); + } + return (-ETIMEDOUT); +} + +static int +mlx5e_open_rq(struct mlx5e_channel *c, + struct mlx5e_rq_param *param, + struct mlx5e_rq *rq) +{ + int err; + int i; + + err = mlx5e_create_rq(c, param, rq); + if (err) + return (err); + + err = mlx5e_enable_rq(rq, param); + if (err) + goto err_destroy_rq; + + err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); + if (err) + goto err_disable_rq; + + c->rq.enabled = 1; + + /* + * Test send queues, which will trigger + * "mlx5e_post_rx_wqes()": + */ + for (i = 0; i != c->num_tc; i++) + mlx5e_send_nop(&c->sq[i], 1, true); + return (0); + +err_disable_rq: + mlx5e_disable_rq(rq); +err_destroy_rq: + mlx5e_destroy_rq(rq); + + return (err); +} + +static void +mlx5e_close_rq(struct mlx5e_rq *rq) +{ + rq->enabled = 0; + mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR); +} + +static void +mlx5e_close_rq_wait(struct mlx5e_rq *rq) +{ + /* wait till RQ is empty */ + while (!mlx5_wq_ll_is_empty(&rq->wq)) { + msleep(4); + rq->cq.mcq.comp(&rq->cq.mcq); + } + + mlx5e_disable_rq(rq); + mlx5e_destroy_rq(rq); +} + +static void +mlx5e_free_sq_db(struct mlx5e_sq *sq) +{ + int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); + int x; + + for (x = 0; x != wq_sz; x++) + bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map); + free(sq->mbuf, M_MLX5EN); +} + +static int +mlx5e_alloc_sq_db(struct mlx5e_sq *sq) +{ + int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); + int err; + int x; + + sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO); + if (sq->mbuf == NULL) + return (-ENOMEM); + + /* Create DMA descriptor MAPs */ + for (x = 0; x != wq_sz; x++) { + err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map); + if (err != 0) { + while (x--) + bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map); + free(sq->mbuf, M_MLX5EN); + return (err); + } + } + return (0); +} + +static const char *mlx5e_sq_stats_desc[] = { + MLX5E_SQ_STATS(MLX5E_STATS_DESC) +}; + +static int +mlx5e_create_sq(struct mlx5e_channel *c, + int tc, + struct mlx5e_sq_param *param, + struct mlx5e_sq *sq) +{ + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + char buffer[16]; + + void *sqc = param->sqc; + void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq); + int err; + + /* Create DMA descriptor TAG */ + if ((err = -bus_dma_tag_create( + bus_get_dma_tag(mdev->pdev->dev.bsddev), + 1, /* any alignment */ + 0, /* no boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MLX5E_MAX_TX_PAYLOAD_SIZE, /* maxsize */ + MLX5E_MAX_TX_MBUF_FRAGS, /* nsegments */ + MLX5E_MAX_TX_MBUF_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockfuncarg */ + &sq->dma_tag))) + goto done; + + err = mlx5_alloc_map_uar(mdev, &sq->uar); + if (err) + goto err_free_dma_tag; + + err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, &sq->wq, + &sq->wq_ctrl); + if (err) + goto err_unmap_free_uar; + + sq->wq.db = &sq->wq.db[MLX5_SND_DBR]; + sq->uar_map = sq->uar.map; + sq->uar_bf_map = sq->uar.bf_map; + sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2; + + err = mlx5e_alloc_sq_db(sq); + if (err) + goto err_sq_wq_destroy; + + sq->pdev = c->pdev; + sq->mkey_be = c->mkey_be; + sq->channel = c; + sq->tc = tc; + + sq->br = buf_ring_alloc(MLX5E_SQ_TX_QUEUE_SIZE, M_MLX5EN, + M_WAITOK, &sq->lock); + if (sq->br == NULL) { + if_printf(c->ifp, "%s: Failed allocating sq drbr buffer\n", + __func__); + err = -ENOMEM; + goto err_free_sq_db; + } + + sq->sq_tq = taskqueue_create_fast("mlx5e_que", M_WAITOK, + taskqueue_thread_enqueue, &sq->sq_tq); + if (sq->sq_tq == NULL) { + if_printf(c->ifp, "%s: Failed allocating taskqueue\n", + __func__); + err = -ENOMEM; + goto err_free_drbr; + } + TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq); + taskqueue_start_threads(&sq->sq_tq, 1, PI_NET, "%s tx sq", + c->ifp->if_xname); + + + snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc); + mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), + buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM, + sq->stats.arg); + + return (0); + +err_free_drbr: + buf_ring_free(sq->br, M_MLX5EN); +err_free_sq_db: + mlx5e_free_sq_db(sq); +err_sq_wq_destroy: + mlx5_wq_destroy(&sq->wq_ctrl); + +err_unmap_free_uar: + mlx5_unmap_free_uar(mdev, &sq->uar); + +err_free_dma_tag: + bus_dma_tag_destroy(sq->dma_tag); +done: + return (err); +} + +static void +mlx5e_destroy_sq(struct mlx5e_sq *sq) +{ + struct mlx5e_channel *c = sq->channel; + struct mlx5e_priv *priv = c->priv; + + /* destroy all sysctl nodes */ + sysctl_ctx_free(&sq->stats.ctx); + + mlx5e_free_sq_db(sq); + mlx5_wq_destroy(&sq->wq_ctrl); + mlx5_unmap_free_uar(priv->mdev, &sq->uar); + taskqueue_drain(sq->sq_tq, &sq->sq_task); + taskqueue_free(sq->sq_tq); + buf_ring_free(sq->br, M_MLX5EN); +} + +static int +mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param) +{ + struct mlx5e_channel *c = sq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + + void *in; + void *sqc; + void *wq; + int inlen; + int err; + + inlen = MLX5_ST_SZ_BYTES(create_sq_in) + + sizeof(u64) * sq->wq_ctrl.buf.npages; + in = mlx5_vzalloc(inlen); + if (in == NULL) + return (-ENOMEM); + + sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); + wq = MLX5_ADDR_OF(sqc, sqc, wq); + + memcpy(sqc, param->sqc, sizeof(param->sqc)); + + MLX5_SET(sqc, sqc, tis_num_0, priv->tisn[sq->tc]); + MLX5_SET(sqc, sqc, cqn, c->sq[sq->tc].cq.mcq.cqn); + MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); + MLX5_SET(sqc, sqc, tis_lst_sz, 1); + MLX5_SET(sqc, sqc, flush_in_error_en, 1); + + MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); + MLX5_SET(wq, wq, uar_page, sq->uar.index); + MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - + PAGE_SHIFT); + MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); + + mlx5_fill_page_array(&sq->wq_ctrl.buf, + (__be64 *) MLX5_ADDR_OF(wq, wq, pas)); + + err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn); + + kvfree(in); + + return (err); +} + +static int +mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state) +{ + struct mlx5e_channel *c = sq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + + void *in; + void *sqc; + int inlen; + int err; + + inlen = MLX5_ST_SZ_BYTES(modify_sq_in); + in = mlx5_vzalloc(inlen); + if (in == NULL) + return (-ENOMEM); + + sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); + + MLX5_SET(modify_sq_in, in, sqn, sq->sqn); + MLX5_SET(modify_sq_in, in, sq_state, curr_state); + MLX5_SET(sqc, sqc, state, next_state); + + err = mlx5_core_modify_sq(mdev, in, inlen); + + kvfree(in); + + return (err); +} + +static void +mlx5e_disable_sq(struct mlx5e_sq *sq) +{ + struct mlx5e_channel *c = sq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + + mlx5_core_destroy_sq(mdev, sq->sqn); +} + +static int +mlx5e_open_sq(struct mlx5e_channel *c, + int tc, + struct mlx5e_sq_param *param, + struct mlx5e_sq *sq) +{ + int err; + + err = mlx5e_create_sq(c, tc, param, sq); + if (err) + return (err); + + err = mlx5e_enable_sq(sq, param); + if (err) + goto err_destroy_sq; + + err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY); + if (err) + goto err_disable_sq; + + atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_READY); + + return (0); + +err_disable_sq: + mlx5e_disable_sq(sq); +err_destroy_sq: + mlx5e_destroy_sq(sq); + + return (err); +} + +static void +mlx5e_close_sq(struct mlx5e_sq *sq) +{ + + /* ensure hw is notified of all pending wqes */ + if (mlx5e_sq_has_room_for(sq, 1)) + mlx5e_send_nop(sq, 1, true); + + mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR); +} + +static void +mlx5e_close_sq_wait(struct mlx5e_sq *sq) +{ + /* wait till SQ is empty */ + while (sq->cc != sq->pc) { + msleep(4); + sq->cq.mcq.comp(&sq->cq.mcq); + } + + mlx5e_disable_sq(sq); + mlx5e_destroy_sq(sq); +} + +static int +mlx5e_create_cq(struct mlx5e_channel *c, + struct mlx5e_cq_param *param, + struct mlx5e_cq *cq, + mlx5e_cq_comp_t *comp) +{ + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_core_cq *mcq = &cq->mcq; + int eqn_not_used; + int irqn; + int err; + u32 i; + + param->wq.buf_numa_node = 0; + param->wq.db_numa_node = 0; + param->eq_ix = c->ix; + + err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, + &cq->wq_ctrl); + if (err) + return (err); + + mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn); + + mcq->cqe_sz = 64; + mcq->set_ci_db = cq->wq_ctrl.db.db; + mcq->arm_db = cq->wq_ctrl.db.db + 1; + *mcq->set_ci_db = 0; + *mcq->arm_db = 0; + mcq->vector = param->eq_ix; + mcq->comp = comp; + mcq->event = mlx5e_cq_error_event; + mcq->irqn = irqn; + mcq->uar = &priv->cq_uar; + + for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { + struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); + + cqe->op_own = 0xf1; + } + + cq->channel = c; + + return (0); +} + +static void +mlx5e_destroy_cq(struct mlx5e_cq *cq) +{ + mlx5_wq_destroy(&cq->wq_ctrl); +} + +static int +mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, + u8 moderation_mode) +{ + struct mlx5e_channel *c = cq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_core_cq *mcq = &cq->mcq; + void *in; + void *cqc; + int inlen; + int irqn_not_used; + int eqn; + int err; + + inlen = MLX5_ST_SZ_BYTES(create_cq_in) + + sizeof(u64) * cq->wq_ctrl.buf.npages; + in = mlx5_vzalloc(inlen); + if (in == NULL) + return (-ENOMEM); + + cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); + + memcpy(cqc, param->cqc, sizeof(param->cqc)); + + mlx5_fill_page_array(&cq->wq_ctrl.buf, + (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas)); + + mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used); + + MLX5_SET(cqc, cqc, cq_period_mode, moderation_mode); + MLX5_SET(cqc, cqc, c_eqn, eqn); + MLX5_SET(cqc, cqc, uar_page, mcq->uar->index); + MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - + PAGE_SHIFT); + MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); + + err = mlx5_core_create_cq(mdev, mcq, in, inlen); + + kvfree(in); + + if (err) + return (err); + + mlx5e_cq_arm(cq); + + return (0); +} + +static void +mlx5e_disable_cq(struct mlx5e_cq *cq) +{ + struct mlx5e_channel *c = cq->channel; + struct mlx5e_priv *priv = c->priv; + struct mlx5_core_dev *mdev = priv->mdev; + + mlx5_core_destroy_cq(mdev, &cq->mcq); +} + +static int +mlx5e_open_cq(struct mlx5e_channel *c, + struct mlx5e_cq_param *param, + struct mlx5e_cq *cq, + mlx5e_cq_comp_t *comp, + u8 moderation_mode) +{ + int err; + + err = mlx5e_create_cq(c, param, cq, comp); + if (err) + return (err); + + err = mlx5e_enable_cq(cq, param, moderation_mode); + if (err) + goto err_destroy_cq; + + return (0); + +err_destroy_cq: + mlx5e_destroy_cq(cq); + + return (err); +} + +static void +mlx5e_close_cq(struct mlx5e_cq *cq) +{ + mlx5e_disable_cq(cq); + mlx5e_destroy_cq(cq); +} + +static int +mlx5e_open_tx_cqs(struct mlx5e_channel *c, + struct mlx5e_channel_param *cparam) +{ + int err; + int tc; + + for (tc = 0; tc < c->num_tc; tc++) { + /* open completion queue */ + err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq, + &mlx5e_tx_cq_comp, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); + if (err) + goto err_close_tx_cqs; + } + return (0); + +err_close_tx_cqs: + for (tc--; tc >= 0; tc--) + mlx5e_close_cq(&c->sq[tc].cq); + + return (err); +} + +static void +mlx5e_close_tx_cqs(struct mlx5e_channel *c) +{ + int tc; + + for (tc = 0; tc < c->num_tc; tc++) + mlx5e_close_cq(&c->sq[tc].cq); +} + +static int +mlx5e_open_sqs(struct mlx5e_channel *c, + struct mlx5e_channel_param *cparam) +{ + int err; + int tc; + + for (tc = 0; tc < c->num_tc; tc++) { + err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]); + if (err) + goto err_close_sqs; + } + + return (0); + +err_close_sqs: + for (tc--; tc >= 0; tc--) { + mlx5e_close_sq(&c->sq[tc]); + mlx5e_close_sq_wait(&c->sq[tc]); + } + + return (err); +} + +static void +mlx5e_close_sqs(struct mlx5e_channel *c) +{ + int tc; + + for (tc = 0; tc < c->num_tc; tc++) + mlx5e_close_sq(&c->sq[tc]); +} + +static void +mlx5e_close_sqs_wait(struct mlx5e_channel *c) +{ + int tc; + + for (tc = 0; tc < c->num_tc; tc++) + mlx5e_close_sq_wait(&c->sq[tc]); +} + +static void +mlx5e_chan_mtx_init(struct mlx5e_channel *c) +{ + int tc; + + mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF); + + for (tc = 0; tc < c->num_tc; tc++) { + mtx_init(&c->sq[tc].lock, "mlx5tx", MTX_NETWORK_LOCK, MTX_DEF); + mtx_init(&c->sq[tc].comp_lock, "mlx5comp", MTX_NETWORK_LOCK, + MTX_DEF); + } +} + +static void +mlx5e_chan_mtx_destroy(struct mlx5e_channel *c) +{ + int tc; + + mtx_destroy(&c->rq.mtx); + + for (tc = 0; tc < c->num_tc; tc++) { + mtx_destroy(&c->sq[tc].lock); + mtx_destroy(&c->sq[tc].comp_lock); + } +} + +static int +mlx5e_open_channel(struct mlx5e_priv *priv, int ix, + struct mlx5e_channel_param *cparam, + struct mlx5e_channel * volatile *cp) +{ + struct mlx5e_channel *c; + u8 rx_moderation_mode; + int err; + + c = malloc(sizeof(*c), M_MLX5EN, M_WAITOK | M_ZERO); + if (c == NULL) + return (-ENOMEM); + + c->priv = priv; + c->ix = ix; + c->cpu = 0; + c->pdev = &priv->mdev->pdev->dev; + c->ifp = priv->ifp; + c->mkey_be = cpu_to_be32(priv->mr.key); + c->num_tc = priv->num_tc; + + /* init mutexes */ + mlx5e_chan_mtx_init(c); + + /* open transmit completion queue */ + err = mlx5e_open_tx_cqs(c, cparam); + if (err) + goto err_free; + + switch (priv->params.rx_cq_moderation_mode) { + case 0: + rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; + break; + default: + if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe)) + rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE; + else + rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; + break; + } + + /* open receive completion queue */ + err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq, + &mlx5e_rx_cq_comp, rx_moderation_mode); + if (err) + goto err_close_tx_cqs; + + err = mlx5e_open_sqs(c, cparam); + if (err) + goto err_close_rx_cq; + + err = mlx5e_open_rq(c, &cparam->rq, &c->rq); + if (err) + goto err_close_sqs; + + /* store channel pointer */ + *cp = c; + + /* poll receive queue initially */ + c->rq.cq.mcq.comp(&c->rq.cq.mcq); + + return (0); + +err_close_sqs: + mlx5e_close_sqs(c); + mlx5e_close_sqs_wait(c); + +err_close_rx_cq: + mlx5e_close_cq(&c->rq.cq); + +err_close_tx_cqs: + mlx5e_close_tx_cqs(c); + +err_free: + /* destroy mutexes */ + mlx5e_chan_mtx_destroy(c); + free(c, M_MLX5EN); + return (err); +} + +static void +mlx5e_close_channel(struct mlx5e_channel * volatile *pp) +{ + struct mlx5e_channel *c = *pp; + + /* check if channel is already closed */ + if (c == NULL) + return; + mlx5e_close_rq(&c->rq); + mlx5e_close_sqs(c); +} + +static void +mlx5e_close_channel_wait(struct mlx5e_channel * volatile *pp) +{ + struct mlx5e_channel *c = *pp; + + /* check if channel is already closed */ + if (c == NULL) + return; + /* ensure channel pointer is no longer used */ + *pp = NULL; + + mlx5e_close_rq_wait(&c->rq); + mlx5e_close_sqs_wait(c); + mlx5e_close_cq(&c->rq.cq); + mlx5e_close_tx_cqs(c); + /* destroy mutexes */ + mlx5e_chan_mtx_destroy(c); + free(c, M_MLX5EN); +} + +static void +mlx5e_build_rq_param(struct mlx5e_priv *priv, + struct mlx5e_rq_param *param) +{ + void *rqc = param->rqc; + void *wq = MLX5_ADDR_OF(rqc, rqc, wq); + + MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST); + MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN); + MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe))); + MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size); + MLX5_SET(wq, wq, pd, priv->pdn); + + param->wq.buf_numa_node = 0; + param->wq.db_numa_node = 0; + param->wq.linear = 1; +} + +static void +mlx5e_build_sq_param(struct mlx5e_priv *priv, + struct mlx5e_sq_param *param) +{ + void *sqc = param->sqc; + void *wq = MLX5_ADDR_OF(sqc, sqc, wq); + + MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size); + MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); + MLX5_SET(wq, wq, pd, priv->pdn); + + param->wq.buf_numa_node = 0; + param->wq.db_numa_node = 0; + param->wq.linear = 1; +} + +static void +mlx5e_build_common_cq_param(struct mlx5e_priv *priv, + struct mlx5e_cq_param *param) +{ + void *cqc = param->cqc; + + MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index); +} + +static void +mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, + struct mlx5e_cq_param *param) +{ + void *cqc = param->cqc; + + MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size); + MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec); + MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts); + + mlx5e_build_common_cq_param(priv, param); +} + +static void +mlx5e_build_tx_cq_param(struct mlx5e_priv *priv, + struct mlx5e_cq_param *param) +{ + void *cqc = param->cqc; + + MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size); + MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec); + MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts); + + mlx5e_build_common_cq_param(priv, param); +} + +static void +mlx5e_build_channel_param(struct mlx5e_priv *priv, + struct mlx5e_channel_param *cparam) +{ + memset(cparam, 0, sizeof(*cparam)); + + mlx5e_build_rq_param(priv, &cparam->rq); + mlx5e_build_sq_param(priv, &cparam->sq); + mlx5e_build_rx_cq_param(priv, &cparam->rx_cq); + mlx5e_build_tx_cq_param(priv, &cparam->tx_cq); +} + +static int +mlx5e_open_channels(struct mlx5e_priv *priv) +{ + struct mlx5e_channel_param cparam; + void *ptr; + int err; + int i; + int j; + + priv->channel = malloc(priv->params.num_channels * + sizeof(struct mlx5e_channel *), M_MLX5EN, M_WAITOK | M_ZERO); + if (priv->channel == NULL) + return (-ENOMEM); + + mlx5e_build_channel_param(priv, &cparam); + for (i = 0; i < priv->params.num_channels; i++) { + err = mlx5e_open_channel(priv, i, &cparam, &priv->channel[i]); + if (err) + goto err_close_channels; + } + + for (j = 0; j < priv->params.num_channels; j++) { + err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq); + if (err) + goto err_close_channels; + } + + return (0); + +err_close_channels: + for (i--; i >= 0; i--) { + mlx5e_close_channel(&priv->channel[i]); + mlx5e_close_channel_wait(&priv->channel[i]); + } + + /* remove "volatile" attribute from "channel" pointer */ + ptr = __DECONST(void *, priv->channel); + priv->channel = NULL; + + free(ptr, M_MLX5EN); + + return (err); +} + +static void +mlx5e_close_channels(struct mlx5e_priv *priv) +{ + void *ptr; + int i; + + if (priv->channel == NULL) + return; + + for (i = 0; i < priv->params.num_channels; i++) + mlx5e_close_channel(&priv->channel[i]); + for (i = 0; i < priv->params.num_channels; i++) + mlx5e_close_channel_wait(&priv->channel[i]); + + /* remove "volatile" attribute from "channel" pointer */ + ptr = __DECONST(void *, priv->channel); + priv->channel = NULL; + + free(ptr, M_MLX5EN); +} + +static int +mlx5e_open_tis(struct mlx5e_priv *priv, int tc) +{ + struct mlx5_core_dev *mdev = priv->mdev; + u32 in[MLX5_ST_SZ_DW(create_tis_in)]; + void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + + memset(in, 0, sizeof(in)); + + MLX5_SET(tisc, tisc, prio, tc); + MLX5_SET(tisc, tisc, transport_domain, priv->tdn); + + return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc])); +} + +static void +mlx5e_close_tis(struct mlx5e_priv *priv, int tc) +{ + mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]); +} + +static int +mlx5e_open_tises(struct mlx5e_priv *priv) +{ + int num_tc = priv->num_tc; + int err; + int tc; + + for (tc = 0; tc < num_tc; tc++) { + err = mlx5e_open_tis(priv, tc); + if (err) + goto err_close_tises; + } + + return (0); + +err_close_tises: + for (tc--; tc >= 0; tc--) + mlx5e_close_tis(priv, tc); + + return (err); +} + +static void +mlx5e_close_tises(struct mlx5e_priv *priv) +{ + int num_tc = priv->num_tc; + int tc; + + for (tc = 0; tc < num_tc; tc++) + mlx5e_close_tis(priv, tc); +} + +static int +mlx5e_open_rqt(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + u32 *in; + u32 out[MLX5_ST_SZ_DW(create_rqt_out)]; + void *rqtc; + int inlen; + int err; + int sz; + int i; + + sz = 1 << priv->params.rx_hash_log_tbl_sz; + + inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; + in = mlx5_vzalloc(inlen); + if (in == NULL) + return (-ENOMEM); + rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); + + MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); + MLX5_SET(rqtc, rqtc, rqt_max_size, sz); + + for (i = 0; i < sz; i++) { + int ix = i % priv->params.num_channels; + + MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn); + } + + MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT); + + memset(out, 0, sizeof(out)); + err = mlx5_cmd_exec_check_status(mdev, in, inlen, out, sizeof(out)); + if (!err) + priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn); + + kvfree(in); + + return (err); +} + +static void +mlx5e_close_rqt(struct mlx5e_priv *priv) +{ + u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)]; + u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)]; + + memset(in, 0, sizeof(in)); + + MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT); + MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn); + + mlx5_cmd_exec_check_status(priv->mdev, in, sizeof(in), out, + sizeof(out)); +} + +static void +mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt) +{ + void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); + __be32 *hkey; + + MLX5_SET(tirc, tirc, transport_domain, priv->tdn); + +#define ROUGH_MAX_L2_L3_HDR_SZ 256 + +#define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\ + MLX5_HASH_FIELD_SEL_DST_IP) + +#define MLX5_HASH_ALL (MLX5_HASH_FIELD_SEL_SRC_IP |\ + MLX5_HASH_FIELD_SEL_DST_IP |\ + MLX5_HASH_FIELD_SEL_L4_SPORT |\ + MLX5_HASH_FIELD_SEL_L4_DPORT) + +#define MLX5_HASH_IP_IPSEC_SPI (MLX5_HASH_FIELD_SEL_SRC_IP |\ + MLX5_HASH_FIELD_SEL_DST_IP |\ + MLX5_HASH_FIELD_SEL_IPSEC_SPI) + + if (priv->params.hw_lro_en) { + MLX5_SET(tirc, tirc, lro_enable_mask, + MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | + MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO); + MLX5_SET(tirc, tirc, lro_max_msg_sz, + (priv->params.lro_wqe_sz - + ROUGH_MAX_L2_L3_HDR_SZ) >> 8); + /* TODO: add the option to choose timer value dynamically */ + MLX5_SET(tirc, tirc, lro_timeout_period_usecs, + MLX5_CAP_ETH(priv->mdev, + lro_timer_supported_periods[2])); + } + + + switch (tt) { + case MLX5E_TT_ANY: + MLX5_SET(tirc, tirc, disp_type, + MLX5_TIRC_DISP_TYPE_DIRECT); + MLX5_SET(tirc, tirc, inline_rqn, + priv->channel[0]->rq.rqn); + break; + default: + MLX5_SET(tirc, tirc, disp_type, + MLX5_TIRC_DISP_TYPE_INDIRECT); + MLX5_SET(tirc, tirc, indirect_table, + priv->rqtn); + MLX5_SET(tirc, tirc, rx_hash_fn, + MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ); + MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); + hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); + hkey[0] = cpu_to_be32(0xD181C62C); + hkey[1] = cpu_to_be32(0xF7F4DB5B); + hkey[2] = cpu_to_be32(0x1983A2FC); + hkey[3] = cpu_to_be32(0x943E1ADB); + hkey[4] = cpu_to_be32(0xD9389E6B); + hkey[5] = cpu_to_be32(0xD1039C2C); + hkey[6] = cpu_to_be32(0xA74499AD); + hkey[7] = cpu_to_be32(0x593D56D9); + hkey[8] = cpu_to_be32(0xF3253C06); + hkey[9] = cpu_to_be32(0x2ADC1FFC); + break; + } + + switch (tt) { + case MLX5E_TT_IPV4_TCP: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV4); + MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, + MLX5_L4_PROT_TYPE_TCP); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_ALL); + break; + + case MLX5E_TT_IPV6_TCP: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV6); + MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, + MLX5_L4_PROT_TYPE_TCP); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_ALL); + break; + + case MLX5E_TT_IPV4_UDP: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV4); + MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, + MLX5_L4_PROT_TYPE_UDP); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_ALL); + break; + + case MLX5E_TT_IPV6_UDP: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV6); + MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, + MLX5_L4_PROT_TYPE_UDP); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_ALL); + break; + + case MLX5E_TT_IPV4_IPSEC_AH: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV4); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP_IPSEC_SPI); + break; + + case MLX5E_TT_IPV6_IPSEC_AH: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV6); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP_IPSEC_SPI); + break; + + case MLX5E_TT_IPV4_IPSEC_ESP: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV4); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP_IPSEC_SPI); + break; + + case MLX5E_TT_IPV6_IPSEC_ESP: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV6); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP_IPSEC_SPI); + break; + + case MLX5E_TT_IPV4: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV4); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP); + break; + + case MLX5E_TT_IPV6: + MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, + MLX5_L3_PROT_TYPE_IPV6); + MLX5_SET(rx_hash_field_select, hfso, selected_fields, + MLX5_HASH_IP); + break; + + default: + break; + } +} + +static int +mlx5e_open_tir(struct mlx5e_priv *priv, int tt) +{ + struct mlx5_core_dev *mdev = priv->mdev; + u32 *in; + void *tirc; + int inlen; + int err; + + inlen = MLX5_ST_SZ_BYTES(create_tir_in); + in = mlx5_vzalloc(inlen); + if (in == NULL) + return (-ENOMEM); + tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context); + + mlx5e_build_tir_ctx(priv, tirc, tt); + + err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]); + + kvfree(in); + + return (err); +} + +static void +mlx5e_close_tir(struct mlx5e_priv *priv, int tt) +{ + mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]); +} + +static int +mlx5e_open_tirs(struct mlx5e_priv *priv) +{ + int err; + int i; + + for (i = 0; i < MLX5E_NUM_TT; i++) { + err = mlx5e_open_tir(priv, i); + if (err) + goto err_close_tirs; + } + + return (0); + +err_close_tirs: + for (i--; i >= 0; i--) + mlx5e_close_tir(priv, i); + + return (err); +} + +static void +mlx5e_close_tirs(struct mlx5e_priv *priv) +{ + int i; + + for (i = 0; i < MLX5E_NUM_TT; i++) + mlx5e_close_tir(priv, i); +} + +/* + * SW MTU does not include headers, + * HW MTU includes all headers and checksums. + */ +static int +mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu) +{ + struct mlx5e_priv *priv = ifp->if_softc; + struct mlx5_core_dev *mdev = priv->mdev; + int hw_mtu; + int min_mtu; + int err; + + /* + * Trying to set MTU to zero, in order + * to find out the FW's minimal MTU + */ + err = mlx5_set_port_mtu(mdev, 0); + if (err) + return (err); + err = mlx5_query_port_oper_mtu(mdev, &min_mtu); + if (err) { + if_printf(ifp, "Query port minimal MTU failed\n"); + return (err); + } + + if (sw_mtu < MLX5E_HW2SW_MTU(min_mtu)) { + ifp->if_mtu = sw_mtu; + return (0); + } + + err = mlx5_set_port_mtu(mdev, MLX5E_SW2HW_MTU(sw_mtu)); + if (err) + return (err); + + err = mlx5_query_port_oper_mtu(mdev, &hw_mtu); + if (!err) { + ifp->if_mtu = MLX5E_HW2SW_MTU(hw_mtu); + + if (ifp->if_mtu != sw_mtu) { + if_printf(ifp, "Port MTU %d is different than " + "ifp mtu %d\n", sw_mtu, (int)ifp->if_mtu); + } + } else { + if_printf(ifp, "Query port MTU, after setting new " + "MTU value, failed\n"); + ifp->if_mtu = sw_mtu; + } + return (0); +} + +int +mlx5e_open_locked(struct ifnet *ifp) +{ + struct mlx5e_priv *priv = ifp->if_softc; + int err; + + /* check if already opened */ + if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0) + return (0); + + err = mlx5e_open_tises(priv); + if (err) { + if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n", + __func__, err); + return (err); + } + err = mlx5_vport_alloc_q_counter(priv->mdev, &priv->counter_set_id); + if (err) { + if_printf(priv->ifp, + "%s: mlx5_vport_alloc_q_counter failed: %d\n", + __func__, err); + goto err_close_tises; + } + err = mlx5e_open_channels(priv); + if (err) { + if_printf(ifp, "%s: mlx5e_open_channels failed, %d\n", + __func__, err); + goto err_dalloc_q_counter; + } + err = mlx5e_open_rqt(priv); + if (err) { + if_printf(ifp, "%s: mlx5e_open_rqt failed, %d\n", + __func__, err); + goto err_close_channels; + } + err = mlx5e_open_tirs(priv); + if (err) { + if_printf(ifp, "%s: mlx5e_open_tir failed, %d\n", + __func__, err); + goto err_close_rqls; + } + err = mlx5e_open_flow_table(priv); + if (err) { + if_printf(ifp, "%s: mlx5e_open_flow_table failed, %d\n", + __func__, err); + goto err_close_tirs; + } + err = mlx5e_add_all_vlan_rules(priv); + if (err) { + if_printf(ifp, "%s: mlx5e_add_all_vlan_rules failed, %d\n", + __func__, err); + goto err_close_flow_table; + } + set_bit(MLX5E_STATE_OPENED, &priv->state); + + mlx5e_update_carrier(priv); + mlx5e_set_rx_mode_core(priv); + + return (0); + +err_close_flow_table: + mlx5e_close_flow_table(priv); + +err_close_tirs: + mlx5e_close_tirs(priv); + +err_close_rqls: + mlx5e_close_rqt(priv); + +err_close_channels: + mlx5e_close_channels(priv); + +err_dalloc_q_counter: + mlx5_vport_dealloc_q_counter(priv->mdev, priv->counter_set_id); + +err_close_tises: + mlx5e_close_tises(priv); + + return (err); +} + +static void +mlx5e_open(void *arg) +{ + struct mlx5e_priv *priv = arg; + + PRIV_LOCK(priv); + if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP)) + if_printf(priv->ifp, + "%s: Setting port status to up failed\n", + __func__); + + mlx5e_open_locked(priv->ifp); + priv->ifp->if_drv_flags |= IFF_DRV_RUNNING; + PRIV_UNLOCK(priv); +} + +int +mlx5e_close_locked(struct ifnet *ifp) +{ + struct mlx5e_priv *priv = ifp->if_softc; + + /* check if already closed */ + if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0) + return (0); + + clear_bit(MLX5E_STATE_OPENED, &priv->state); + + mlx5e_set_rx_mode_core(priv); + mlx5e_del_all_vlan_rules(priv); + if_link_state_change(priv->ifp, LINK_STATE_DOWN); + mlx5e_close_flow_table(priv); + mlx5e_close_tirs(priv); + mlx5e_close_rqt(priv); + mlx5e_close_channels(priv); + mlx5_vport_dealloc_q_counter(priv->mdev, priv->counter_set_id); + mlx5e_close_tises(priv); + + return (0); +} + +#if (__FreeBSD_version >= 1100000) +static uint64_t +mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt) +{ + struct mlx5e_priv *priv = ifp->if_softc; + u64 retval; + + /* PRIV_LOCK(priv); XXX not allowed */ + switch (cnt) { + case IFCOUNTER_IPACKETS: + retval = priv->stats.vport.rx_packets; + break; + case IFCOUNTER_IERRORS: + retval = priv->stats.vport.rx_error_packets; + break; + case IFCOUNTER_OPACKETS: + retval = priv->stats.vport.tx_packets; + break; + case IFCOUNTER_OERRORS: + retval = priv->stats.vport.tx_error_packets; + break; + case IFCOUNTER_IBYTES: + retval = priv->stats.vport.rx_bytes; + break; + case IFCOUNTER_OBYTES: + retval = priv->stats.vport.tx_bytes; + break; + case IFCOUNTER_IMCASTS: + retval = priv->stats.vport.rx_multicast_packets; + break; + case IFCOUNTER_OMCASTS: + retval = priv->stats.vport.tx_multicast_packets; + break; + case IFCOUNTER_OQDROPS: + retval = priv->stats.vport.tx_queue_dropped; + break; + default: + retval = if_get_counter_default(ifp, cnt); + break; + } + /* PRIV_UNLOCK(priv); XXX not allowed */ + return (retval); +} +#endif + +static void +mlx5e_set_rx_mode(struct ifnet *ifp) +{ + struct mlx5e_priv *priv = ifp->if_softc; + + schedule_work(&priv->set_rx_mode_work); +} + +static int +mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data) +{ + struct mlx5e_priv *priv; + struct ifreq *ifr; + struct ifi2creq i2c; + int error = 0; + int mask = 0; + int size_read = 0; + int module_num; + int max_mtu; + + priv = ifp->if_softc; + + /* check if detaching */ + if (priv == NULL || priv->gone != 0) + return (ENXIO); + + switch (command) { + case SIOCSIFMTU: + ifr = (struct ifreq *)data; + + PRIV_LOCK(priv); + mlx5_query_port_max_mtu(priv->mdev, &max_mtu); + + if (ifr->ifr_mtu >= MLX5E_MTU_MIN && + ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) { + int was_opened; + + was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (was_opened) + mlx5e_close_locked(ifp); + + /* set new MTU */ + mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu); + + if (was_opened) + mlx5e_open_locked(ifp); + } else { + error = EINVAL; + if_printf(ifp, "Invalid MTU value. Min val: %d, Max val: %d\n", + MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu)); + } + PRIV_UNLOCK(priv); + break; + case SIOCSIFFLAGS: + if ((ifp->if_flags & IFF_UP) && + (ifp->if_drv_flags & IFF_DRV_RUNNING)) { + mlx5e_set_rx_mode(ifp); + break; + } + PRIV_LOCK(priv); + if (ifp->if_flags & IFF_UP) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { + if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0) + mlx5e_open_locked(ifp); + ifp->if_drv_flags |= IFF_DRV_RUNNING; + mlx5_set_port_status(priv->mdev, MLX5_PORT_UP); + } + } else { + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + mlx5_set_port_status(priv->mdev, + MLX5_PORT_DOWN); + if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0) + mlx5e_close_locked(ifp); + mlx5e_update_carrier(priv); + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; + } + } + PRIV_UNLOCK(priv); + break; + case SIOCADDMULTI: + case SIOCDELMULTI: + mlx5e_set_rx_mode(ifp); + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + case SIOCGIFXMEDIA: + ifr = (struct ifreq *)data; + error = ifmedia_ioctl(ifp, ifr, &priv->media, command); + break; + case SIOCSIFCAP: + ifr = (struct ifreq *)data; + PRIV_LOCK(priv); + mask = ifr->ifr_reqcap ^ ifp->if_capenable; + + if (mask & IFCAP_TXCSUM) { + ifp->if_capenable ^= IFCAP_TXCSUM; + ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); + + if (IFCAP_TSO4 & ifp->if_capenable && + !(IFCAP_TXCSUM & ifp->if_capenable)) { + ifp->if_capenable &= ~IFCAP_TSO4; + ifp->if_hwassist &= ~CSUM_IP_TSO; + if_printf(ifp, + "tso4 disabled due to -txcsum.\n"); + } + } + if (mask & IFCAP_TXCSUM_IPV6) { + ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; + ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); + + if (IFCAP_TSO6 & ifp->if_capenable && + !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { + ifp->if_capenable &= ~IFCAP_TSO6; + ifp->if_hwassist &= ~CSUM_IP6_TSO; + if_printf(ifp, + "tso6 disabled due to -txcsum6.\n"); + } + } + if (mask & IFCAP_RXCSUM) + ifp->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; + + if (mask & IFCAP_TSO4) { + if (!(IFCAP_TSO4 & ifp->if_capenable) && + !(IFCAP_TXCSUM & ifp->if_capenable)) { + if_printf(ifp, "enable txcsum first.\n"); + error = EAGAIN; + goto out; + } + ifp->if_capenable ^= IFCAP_TSO4; + ifp->if_hwassist ^= CSUM_IP_TSO; + } + if (mask & IFCAP_TSO6) { + if (!(IFCAP_TSO6 & ifp->if_capenable) && + !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { + if_printf(ifp, "enable txcsum6 first.\n"); + error = EAGAIN; + goto out; + } + ifp->if_capenable ^= IFCAP_TSO6; + ifp->if_hwassist ^= CSUM_IP6_TSO; + } + + if (mask & IFCAP_VLAN_HWFILTER) { + if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) + mlx5e_disable_vlan_filter(priv); + else + mlx5e_enable_vlan_filter(priv); + + ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; + } + if (mask & IFCAP_VLAN_HWTAGGING) + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + + if (mask & IFCAP_WOL_MAGIC) + ifp->if_capenable ^= IFCAP_WOL_MAGIC; + + VLAN_CAPABILITIES(ifp); + /* turn off LRO means also turn of HW LRO - if it's on */ + if (mask & IFCAP_LRO ) { + int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + bool need_restart = false; + + ifp->if_capenable ^= IFCAP_LRO; + if (!(ifp->if_capenable & IFCAP_LRO)) { + if (priv->params.hw_lro_en) { + priv->params.hw_lro_en = false; + need_restart = true; + /* Not sure this is the correct way */ + priv->params_ethtool.hw_lro = priv->params.hw_lro_en; + } + } + if (was_opened && need_restart) { + mlx5e_close_locked(ifp); + mlx5e_open_locked(ifp); + } + } +out: + PRIV_UNLOCK(priv); + break; + + case SIOCGI2C: + ifr = (struct ifreq *)data; + + /* Copy from the user-space address ifr_data to the kernel-space address i2c */ + error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); + if (error) + break; + + if (i2c.len > sizeof(i2c.data)) { + error = EINVAL; + break; + } + + PRIV_LOCK(priv); + /* Get module_num which is required for the query_eeprom */ + error = mlx5_query_module_num(priv->mdev, &module_num); + if (error) { + if_printf(ifp, "Query module num failed, eeprom " + "reading is not supported\n"); + goto err_i2c; + } + + /* + * Note that we ignore i2c.addr here. The driver hardcodes + * the address to 0x50, while standard expects it to be 0xA0. + */ + error = mlx5_query_eeprom(priv->mdev, + MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE, + (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num, + (uint32_t *)i2c.data, &size_read); + if (error) { + if_printf(ifp, "Query eeprom failed, eeprom " + "reading is not supported\n"); + goto err_i2c; + } + + if (i2c.len > MLX5_EEPROM_MAX_BYTES) { + error = mlx5_query_eeprom(priv->mdev, + MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE, + (uint32_t)(i2c.offset + size_read), + (uint32_t)(i2c.len - size_read), module_num, + (uint32_t *)(i2c.data + size_read), &size_read); + } + if (error) { + if_printf(ifp, "Query eeprom failed, eeprom " + "reading is not supported\n"); + goto err_i2c; + } + + error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); +err_i2c: + PRIV_UNLOCK(priv); + break; + + default: + error = ether_ioctl(ifp, command, data); + break; + } + return (error); +} + +static int +mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev) +{ + /* + * TODO: uncoment once FW really sets all these bits if + * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap || + * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap || + * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return + * -ENOTSUPP; + */ + + /* TODO: add more must-to-have features */ + + return (0); +} + +static void +mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev, + struct mlx5e_priv *priv, + int num_comp_vectors) +{ + /* + * TODO: Consider link speed for setting "log_sq_size", + * "log_rq_size" and "cq_moderation_xxx": + */ + priv->params.log_sq_size = + MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; + priv->params.log_rq_size = + MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; + priv->params.rx_cq_moderation_usec = + MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? + MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE : + MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC; + priv->params.rx_cq_moderation_mode = + MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0; + priv->params.rx_cq_moderation_pkts = + MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS; + priv->params.tx_cq_moderation_usec = + MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC; + priv->params.tx_cq_moderation_pkts = + MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS; + priv->params.min_rx_wqes = + MLX5E_PARAMS_DEFAULT_MIN_RX_WQES; + priv->params.rx_hash_log_tbl_sz = + (order_base_2(num_comp_vectors) > + MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ? + order_base_2(num_comp_vectors) : + MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ; + priv->params.num_tc = 1; + priv->params.default_vlan_prio = 0; + priv->counter_set_id = -1; + + /* + * hw lro is currently defaulted to off. + * when it won't anymore we will consider the + * HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)" + */ + priv->params.hw_lro_en = false; + priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ; + + priv->mdev = mdev; + priv->params.num_channels = num_comp_vectors; + priv->order_base_2_num_channels = order_base_2(num_comp_vectors); + priv->queue_mapping_channel_mask = + roundup_pow_of_two(num_comp_vectors) - 1; + priv->num_tc = priv->params.num_tc; + priv->default_vlan_prio = priv->params.default_vlan_prio; + + INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); + INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); + INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); +} + +static int +mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn, + struct mlx5_core_mr *mr) +{ + struct ifnet *ifp = priv->ifp; + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_create_mkey_mbox_in *in; + int err; + + in = mlx5_vzalloc(sizeof(*in)); + if (in == NULL) { + if_printf(ifp, "%s: failed to allocate inbox\n", __func__); + return (-ENOMEM); + } + in->seg.flags = MLX5_PERM_LOCAL_WRITE | + MLX5_PERM_LOCAL_READ | + MLX5_ACCESS_MODE_PA; + in->seg.flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64); + in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8); + + err = mlx5_core_create_mkey(mdev, mr, in, sizeof(*in), NULL, NULL, + NULL); + if (err) + if_printf(ifp, "%s: mlx5_core_create_mkey failed, %d\n", + __func__, err); + + kvfree(in); + + return (err); +} + +static const char *mlx5e_vport_stats_desc[] = { + MLX5E_VPORT_STATS(MLX5E_STATS_DESC) +}; + +static const char *mlx5e_pport_stats_desc[] = { + MLX5E_PPORT_STATS(MLX5E_STATS_DESC) +}; + +static void +mlx5e_priv_mtx_init(struct mlx5e_priv *priv) +{ + mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF); + sx_init(&priv->state_lock, "mlx5state"); + callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0); +} + +static void +mlx5e_priv_mtx_destroy(struct mlx5e_priv *priv) +{ + mtx_destroy(&priv->async_events_mtx); + sx_destroy(&priv->state_lock); +} + +static int +sysctl_firmware(SYSCTL_HANDLER_ARGS) +{ + /* %d.%d%.d the string format. + * fw_rev_{maj,min,sub} return u16, 2^16 = 65536. + * We need at most 5 chars to store that. + * it also has: two "." and NULL at the end. + * Which means we need 18 (5*3 + 3) chars at most. + */ + char fw[18]; + struct mlx5e_priv *priv = arg1; + int error; + + snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev), + fw_rev_sub(priv->mdev)); + error = sysctl_handle_string(oidp, fw, sizeof(fw), req); + return (error); +} + +static void +mlx5e_add_hw_stats(struct mlx5e_priv *priv) +{ + SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw), + OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, priv, 0, + sysctl_firmware, "A", "HCA firmware version"); + + SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw), + OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0, + "Board ID"); +} + +static void * +mlx5e_create_ifp(struct mlx5_core_dev *mdev) +{ + static volatile int mlx5_en_unit; + struct ifnet *ifp; + struct mlx5e_priv *priv; + u8 dev_addr[ETHER_ADDR_LEN] __aligned(4); + struct sysctl_oid_list *child; + int ncv = mdev->priv.eq_table.num_comp_vectors; + char unit[16]; + int err; + int i; + u32 eth_proto_cap; + + if (mlx5e_check_required_hca_cap(mdev)) { + mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n"); + return (NULL); + } + priv = malloc(sizeof(*priv), M_MLX5EN, M_WAITOK | M_ZERO); + if (priv == NULL) { + mlx5_core_err(mdev, "malloc() failed\n"); + return (NULL); + } + mlx5e_priv_mtx_init(priv); + + ifp = priv->ifp = if_alloc(IFT_ETHER); + if (ifp == NULL) { + mlx5_core_err(mdev, "if_alloc() failed\n"); + goto err_free_priv; + } + ifp->if_softc = priv; + if_initname(ifp, "mce", atomic_fetchadd_int(&mlx5_en_unit, 1)); + ifp->if_mtu = ETHERMTU; + ifp->if_init = mlx5e_open; + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_ioctl = mlx5e_ioctl; + ifp->if_transmit = mlx5e_xmit; + ifp->if_qflush = if_qflush; +#if (__FreeBSD_version >= 1100000) + ifp->if_get_counter = mlx5e_get_counter; +#endif + ifp->if_snd.ifq_maxlen = ifqmaxlen; + /* + * Set driver features + */ + ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6; + ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING; + ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER; + ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU; + ifp->if_capabilities |= IFCAP_LRO; + ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO; + + /* set TSO limits so that we don't have to drop TX packets */ + ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); + ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */; + ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE; + + ifp->if_capenable = ifp->if_capabilities; + ifp->if_hwassist = 0; + if (ifp->if_capenable & IFCAP_TSO) + ifp->if_hwassist |= CSUM_TSO; + if (ifp->if_capenable & IFCAP_TXCSUM) + ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); + if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) + ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); + + /* ifnet sysctl tree */ + sysctl_ctx_init(&priv->sysctl_ctx); + priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev), + OID_AUTO, ifp->if_dname, CTLFLAG_RD, 0, "MLX5 ethernet - interface name"); + if (priv->sysctl_ifnet == NULL) { + mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n"); + goto err_free_sysctl; + } + snprintf(unit, sizeof(unit), "%d", ifp->if_dunit); + priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), + OID_AUTO, unit, CTLFLAG_RD, 0, "MLX5 ethernet - interface unit"); + if (priv->sysctl_ifnet == NULL) { + mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n"); + goto err_free_sysctl; + } + /* HW sysctl tree */ + child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev)); + priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child, + OID_AUTO, "hw", CTLFLAG_RD, 0, "MLX5 ethernet dev hw"); + if (priv->sysctl_hw == NULL) { + mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n"); + goto err_free_sysctl; + } + + mlx5e_build_ifp_priv(mdev, priv, ncv); + + err = mlx5_alloc_map_uar(mdev, &priv->cq_uar); + if (err) { + if_printf(ifp, "%s: mlx5_alloc_map_uar failed, %d\n", + __func__, err); + goto err_free_sysctl; + } + err = mlx5_core_alloc_pd(mdev, &priv->pdn); + if (err) { + if_printf(ifp, "%s: mlx5_core_alloc_pd failed, %d\n", + __func__, err); + goto err_unmap_free_uar; + } + + err = mlx5_alloc_transport_domain(mdev, &priv->tdn); + + if (err) { + if_printf(ifp, "%s: mlx5_alloc_transport_domain failed, %d\n", + __func__, err); + goto err_dealloc_pd; + } + + err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr); + if (err) { + if_printf(ifp, "%s: mlx5e_create_mkey failed, %d\n", + __func__, err); + goto err_dealloc_transport_domain; + } + mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr); + + /* set default MTU */ + mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu); + + /* Set desc */ + device_set_desc(mdev->pdev->dev.bsddev, mlx5e_version); + + /* Set default media status */ + priv->media_status_last = IFM_AVALID; + priv->media_active_last = IFM_ETHER | IFM_AUTO; + + /* Pauseframes are enabled by default */ + priv->params_ethtool.tx_pauseframe_control = 1; + priv->params_ethtool.rx_pauseframe_control = 1; + + err = mlx5_query_port_proto_cap(mdev, ð_proto_cap, MLX5_PTYS_EN); + if (err) { + eth_proto_cap = 0; + if_printf(ifp, "%s: Query port media capability failed, %d\n", + __func__, err); + } + + /* Setup supported medias */ + ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK, + mlx5e_media_change, mlx5e_media_status); + + for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) { + if (mlx5e_mode_table[i].baudrate == 0) + continue; + if (MLX5E_PROT_MASK(i) & eth_proto_cap) + ifmedia_add(&priv->media, + IFM_ETHER | mlx5e_mode_table[i].subtype | + IFM_FDX, 0, NULL); + } + + ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO); + ether_ifattach(ifp, dev_addr); + + /* Register for VLAN events */ + priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, + mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST); + priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, + mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST); + + /* Link is down by default */ + if_link_state_change(ifp, LINK_STATE_DOWN); + + mlx5e_enable_async_events(priv); + + mlx5e_add_hw_stats(priv); + + mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), + "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM, + priv->stats.vport.arg); + + mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), + "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM, + priv->stats.pport.arg); + + mlx5e_create_ethtool(priv); + + mtx_lock(&priv->async_events_mtx); + mlx5e_update_stats(priv); + mtx_unlock(&priv->async_events_mtx); + + return (priv); + +err_dealloc_transport_domain: + mlx5_dealloc_transport_domain(mdev, priv->tdn); + +err_dealloc_pd: + mlx5_core_dealloc_pd(mdev, priv->pdn); + +err_unmap_free_uar: + mlx5_unmap_free_uar(mdev, &priv->cq_uar); + +err_free_sysctl: + sysctl_ctx_free(&priv->sysctl_ctx); + + if_free(ifp); + +err_free_priv: + mlx5e_priv_mtx_destroy(priv); + free(priv, M_MLX5EN); + return (NULL); +} + +static void +mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv) +{ + struct mlx5e_priv *priv = vpriv; + struct ifnet *ifp = priv->ifp; + + /* don't allow more IOCTLs */ + priv->gone = 1; + + /* XXX wait a bit to allow IOCTL handlers to complete */ + pause("W", hz); + + /* stop watchdog timer */ + callout_drain(&priv->watchdog); + + if (priv->vlan_attach != NULL) + EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach); + if (priv->vlan_detach != NULL) + EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach); + + /* make sure device gets closed */ + PRIV_LOCK(priv); + mlx5e_close_locked(ifp); + PRIV_UNLOCK(priv); + + /* unregister device */ + ifmedia_removeall(&priv->media); + ether_ifdetach(ifp); + if_free(ifp); + + /* destroy all remaining sysctl nodes */ + if (priv->sysctl_debug) + sysctl_ctx_free(&priv->stats.port_stats_debug.ctx); + sysctl_ctx_free(&priv->stats.vport.ctx); + sysctl_ctx_free(&priv->stats.pport.ctx); + sysctl_ctx_free(&priv->sysctl_ctx); + + mlx5_core_destroy_mkey(priv->mdev, &priv->mr); + mlx5_dealloc_transport_domain(priv->mdev, priv->tdn); + mlx5_core_dealloc_pd(priv->mdev, priv->pdn); + mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar); + mlx5e_disable_async_events(priv); + flush_scheduled_work(); + mlx5e_priv_mtx_destroy(priv); + free(priv, M_MLX5EN); +} + +static void * +mlx5e_get_ifp(void *vpriv) +{ + struct mlx5e_priv *priv = vpriv; + + return (priv->ifp); +} + +static struct mlx5_interface mlx5e_interface = { + .add = mlx5e_create_ifp, + .remove = mlx5e_destroy_ifp, + .event = mlx5e_async_event, + .protocol = MLX5_INTERFACE_PROTOCOL_ETH, + .get_dev = mlx5e_get_ifp, +}; + +void +mlx5e_init(void) +{ + mlx5_register_interface(&mlx5e_interface); +} + +void +mlx5e_cleanup(void) +{ + mlx5_unregister_interface(&mlx5e_interface); +} + +module_init_order(mlx5e_init, SI_ORDER_THIRD); +module_exit_order(mlx5e_cleanup, SI_ORDER_THIRD); + +#if (__FreeBSD_version >= 1100000) +MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1); +#endif +MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1); +MODULE_VERSION(mlx5en, 1); diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c new file mode 100644 index 0000000..bce4915 --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -0,0 +1,340 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "en.h" +#include <machine/in_cksum.h> + +static inline int +mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, + struct mlx5e_rx_wqe *wqe, u16 ix) +{ + bus_dma_segment_t segs[1]; + struct mbuf *mb; + int nsegs; + int err; + + if (rq->mbuf[ix].mbuf != NULL) + return (0); + + mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); + if (unlikely(!mb)) + return (-ENOMEM); + + /* set initial mbuf length */ + mb->m_pkthdr.len = mb->m_len = rq->wqe_sz; + + /* get IP header aligned */ + m_adj(mb, MLX5E_NET_IP_ALIGN); + + err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (err != 0) + goto err_free_mbuf; + if (unlikely(nsegs != 1)) { + bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map); + err = -ENOMEM; + goto err_free_mbuf; + } + wqe->data.addr = cpu_to_be64(segs[0].ds_addr); + + rq->mbuf[ix].mbuf = mb; + rq->mbuf[ix].data = mb->m_data; + + bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map, + BUS_DMASYNC_PREREAD); + return (0); + +err_free_mbuf: + m_freem(mb); + return (err); +} + +static void +mlx5e_post_rx_wqes(struct mlx5e_rq *rq) +{ + if (unlikely(rq->enabled == 0)) + return; + + while (!mlx5_wq_ll_is_full(&rq->wq)) { + struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head); + + if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) + break; + + mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index)); + } + + /* ensure wqes are visible to device before updating doorbell record */ + wmb(); + + mlx5_wq_ll_update_db_record(&rq->wq); +} + +static void +mlx5e_lro_update_hdr(struct mbuf* mb, struct mlx5_cqe64 *cqe) +{ + /* TODO: consider vlans, ip options, ... */ + struct ether_header *eh; + uint16_t eh_type; + struct ip6_hdr *ip6 = NULL; + struct ip *ip4 = NULL; + struct tcphdr *th; + uint32_t *ts_ptr; + + eh = mtod(mb, struct ether_header *); + eh_type = ntohs(eh->ether_type); + + u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe); + int tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || + (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); + + /* TODO: consider vlan */ + u16 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN; + + switch (eh_type) { + case ETHERTYPE_IP: + ip4 = (struct ip *)(eh + 1); + th = (struct tcphdr *)(ip4 + 1); + break; + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(eh + 1); + th = (struct tcphdr *)(ip6 + 1); + break; + default: + return; + } + + ts_ptr = (uint32_t *)(th + 1); + + if (get_cqe_lro_tcppsh(cqe)) + th->th_flags |= TH_PUSH; + + if (tcp_ack) { + th->th_flags |= TH_ACK; + th->th_ack = cqe->lro_ack_seq_num; + th->th_win = cqe->lro_tcp_win; + + /* FreeBSD handles only 32bit aligned timestamp + * right after the TCP hdr + * +--------+--------+--------+--------+ + * | NOP | NOP | TSopt | 10 | + * +--------+--------+--------+--------+ + * | TSval timestamp | + * +--------+--------+--------+--------+ + * | TSecr timestamp | + * +--------+--------+--------+--------+ + */ + if (get_cqe_lro_timestamp_valid(cqe) && + (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 | + TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 | + TCPOLEN_TIMESTAMP))) { + /* cqe->timestamp is 64bit long. + * [0-31] - timestamp. + * [32-64] - timestamp echo replay. + */ + ts_ptr[1] = *(uint32_t *)&cqe->timestamp; + ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1); + } + } + + if (ip4) { + ip4->ip_ttl = cqe->lro_min_ttl; + ip4->ip_len = cpu_to_be16(tot_len); + ip4->ip_sum = 0; + ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2); + } else { + ip6->ip6_hlim = cqe->lro_min_ttl; + ip6->ip6_plen = cpu_to_be16(tot_len - + sizeof(struct ip6_hdr)); + } + /* TODO: handle tcp checksum */ +} + +static inline void +mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, + struct mlx5e_rq *rq, struct mbuf *mb, + u32 cqe_bcnt) +{ + struct ifnet *ifp = rq->ifp; + int lro_num_seg; /* HW LRO session aggregated packets counter */ + + lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; + if (lro_num_seg > 1) { + mlx5e_lro_update_hdr(mb, cqe); + rq->stats.lro_packets++; + rq->stats.lro_bytes += cqe_bcnt; + } + + mb->m_pkthdr.len = mb->m_len = cqe_bcnt; + /* check if a Toeplitz hash was computed */ + if (cqe->rss_hash_type != 0) + mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); + else + mb->m_pkthdr.flowid = rq->ix; + M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); + mb->m_pkthdr.rcvif = ifp; + + if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && + ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) == + (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) { + mb->m_pkthdr.csum_flags = + CSUM_IP_CHECKED | CSUM_IP_VALID | + CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + mb->m_pkthdr.csum_data = htons(0xffff); + } else { + rq->stats.csum_none++; + } + + if (cqe_has_vlan(cqe)) { + mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info); + mb->m_flags |= M_VLANTAG; + } +} + +static int +mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) +{ +#ifndef HAVE_TURBO_LRO + struct lro_entry *queued; +#endif + int i; + + for (i = 0; i < budget; i++) { + struct mlx5e_rx_wqe *wqe; + struct mlx5_cqe64 *cqe; + struct mbuf *mb; + __be16 wqe_counter_be; + u16 wqe_counter; + u32 byte_cnt; + + cqe = mlx5e_get_cqe(&rq->cq); + if (!cqe) + break; + + wqe_counter_be = cqe->wqe_counter; + wqe_counter = be16_to_cpu(wqe_counter_be); + wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); + byte_cnt = be32_to_cpu(cqe->byte_cnt); + + bus_dmamap_sync(rq->dma_tag, + rq->mbuf[wqe_counter].dma_map, + BUS_DMASYNC_POSTREAD); + + if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { + rq->stats.wqe_err++; + goto wq_ll_pop; + } + + if (MHLEN >= byte_cnt && + (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { + bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t), + byte_cnt); + } else { + mb = rq->mbuf[wqe_counter].mbuf; + rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */ + + bus_dmamap_unload(rq->dma_tag, + rq->mbuf[wqe_counter].dma_map); + } + + mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt); + rq->stats.packets++; +#ifdef HAVE_TURBO_LRO + if (mb->m_pkthdr.csum_flags == 0 || + (rq->ifp->if_capenable & IFCAP_LRO) == 0 || + rq->lro.mbuf == NULL) { + /* normal input */ + rq->ifp->if_input(rq->ifp, mb); + } else { + tcp_tlro_rx(&rq->lro, mb); + } +#else + if (mb->m_pkthdr.csum_flags == 0 || + (rq->ifp->if_capenable & IFCAP_LRO) == 0 || + rq->lro.lro_cnt == 0 || + tcp_lro_rx(&rq->lro, mb, 0) != 0) { + rq->ifp->if_input(rq->ifp, mb); + } +#endif +wq_ll_pop: + mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, + &wqe->next.next_wqe_index); + } + + mlx5_cqwq_update_db_record(&rq->cq.wq); + + /* ensure cq space is freed before enabling more cqes */ + wmb(); +#ifndef HAVE_TURBO_LRO + while ((queued = SLIST_FIRST(&rq->lro.lro_active)) != NULL) { + SLIST_REMOVE_HEAD(&rq->lro.lro_active, next); + tcp_lro_flush(&rq->lro, queued); + } +#endif + return (i); +} + +void +mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq) +{ + struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); + int i = 0; + +#ifdef HAVE_PER_CQ_EVENT_PACKET + struct mbuf *mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); + if (mb != NULL) { + /* this code is used for debugging purpose only */ + mb->m_pkthdr.len = mb->m_len = 15; + memset(mb->m_data, 255, 14); + mb->m_data[14] = rq->ix; + mb->m_pkthdr.rcvif = rq->ifp; + rq->ifp->if_input(rq->ifp, mb); + } +#endif + + mtx_lock(&rq->mtx); + + /* + * Polling the entire CQ without posting new WQEs results in + * lack of receive WQEs during heavy traffic scenarios. + */ + while (1) { + if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != + MLX5E_RX_BUDGET_MAX) + break; + i += MLX5E_RX_BUDGET_MAX; + if (i >= MLX5E_BUDGET_MAX) + break; + mlx5e_post_rx_wqes(rq); + } + mlx5e_post_rx_wqes(rq); + mlx5e_cq_arm(&rq->cq); +#ifdef HAVE_TURBO_LRO + tcp_tlro_flush(&rq->lro, 1); +#endif + mtx_unlock(&rq->mtx); +} diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c new file mode 100644 index 0000000..3b69f84 --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c @@ -0,0 +1,485 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "en.h" +#include <machine/atomic.h> + +void +mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt, bool notify_hw) +{ + u16 pi = sq->pc & sq->wq.sz_m1; + struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); + + memset(&wqe->ctrl, 0, sizeof(wqe->ctrl)); + + wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP); + wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); + wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; + + sq->mbuf[pi].mbuf = NULL; + sq->mbuf[pi].num_bytes = 0; + sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); + sq->pc += sq->mbuf[pi].num_wqebbs; + if (notify_hw) + mlx5e_tx_notify_hw(sq, wqe, 0); +} + +#if (__FreeBSD_version >= 1100000) +static uint32_t mlx5e_hash_value; + +static void +mlx5e_hash_init(void *arg) +{ + mlx5e_hash_value = m_ether_tcpip_hash_init(); +} + +/* Make kernel call mlx5e_hash_init after the random stack finished initializing */ +SYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL); +#endif + +static struct mlx5e_sq * +mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb) +{ + struct mlx5e_priv *priv = ifp->if_softc; + u32 ch; + u32 tc; + + /* check if channels are successfully opened */ + if (unlikely(priv->channel == NULL)) + return (NULL); + + /* obtain VLAN information if present */ + if (mb->m_flags & M_VLANTAG) { + tc = (mb->m_pkthdr.ether_vtag >> 13); + if (tc >= priv->num_tc) + tc = priv->default_vlan_prio; + } else { + tc = priv->default_vlan_prio; + } + + ch = priv->params.num_channels; + + /* check if flowid is set */ + if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) { + ch = (mb->m_pkthdr.flowid % 128) % ch; + } else { +#if (__FreeBSD_version >= 1100000) + ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | + MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch; +#else + /* + * m_ether_tcpip_hash not present in stable, so just + * throw unhashed mbufs on queue 0 + */ + ch = 0; +#endif + } + + /* check if channel is allocated */ + if (unlikely(priv->channel[ch] == NULL)) + return (NULL); + + return (&priv->channel[ch]->sq[tc]); +} + +static inline u16 +mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb) +{ + return (MIN(MLX5E_MAX_TX_INLINE, mb->m_len)); +} + +static int +mlx5e_get_header_size(struct mbuf *mb) +{ + struct ether_vlan_header *eh; + struct tcphdr *th; + struct ip *ip; + int ip_hlen, tcp_hlen; + struct ip6_hdr *ip6; + uint16_t eth_type; + int eth_hdr_len; + + eh = mtod(mb, struct ether_vlan_header *); + if (mb->m_len < ETHER_HDR_LEN) + return (0); + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + eth_type = ntohs(eh->evl_proto); + eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + eth_type = ntohs(eh->evl_encap_proto); + eth_hdr_len = ETHER_HDR_LEN; + } + if (mb->m_len < eth_hdr_len) + return (0); + switch (eth_type) { + case ETHERTYPE_IP: + ip = (struct ip *)(mb->m_data + eth_hdr_len); + if (mb->m_len < eth_hdr_len + sizeof(*ip)) + return (0); + if (ip->ip_p != IPPROTO_TCP) + return (0); + ip_hlen = ip->ip_hl << 2; + eth_hdr_len += ip_hlen; + break; + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); + if (mb->m_len < eth_hdr_len + sizeof(*ip6)) + return (0); + if (ip6->ip6_nxt != IPPROTO_TCP) + return (0); + eth_hdr_len += sizeof(*ip6); + break; + default: + return (0); + } + if (mb->m_len < eth_hdr_len + sizeof(*th)) + return (0); + th = (struct tcphdr *)(mb->m_data + eth_hdr_len); + tcp_hlen = th->th_off << 2; + eth_hdr_len += tcp_hlen; + if (mb->m_len < eth_hdr_len) + return (0); + return (eth_hdr_len); +} + +/* The return value is not going back to the stack because of + * the drbr */ +static int +mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp) +{ + bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS]; + struct mlx5_wqe_data_seg *dseg; + struct mlx5e_tx_wqe *wqe; + struct ifnet *ifp; + int nsegs; + int err; + int x; + struct mbuf *mb = *mbp; + u16 ds_cnt; + u16 ihs; + u16 pi; + u8 opcode; + + /* Return ENOBUFS if the queue is full, this may trigger reinsertion + * of the mbuf into the drbr (see mlx5e_xmit_locked) */ + if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) { + return (ENOBUFS); + } + + /* Align SQ edge with NOPs to avoid WQE wrap around */ + pi = ((~sq->pc) & sq->wq.sz_m1); + if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) { + /* send one multi NOP message instead of many */ + mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS, false); + pi = ((~sq->pc) & sq->wq.sz_m1); + if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) { + m_freem(mb); + return (ENOMEM); + } + } + + /* Setup local variables */ + pi = sq->pc & sq->wq.sz_m1; + wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); + ifp = sq->channel->ifp; + + memset(wqe, 0, sizeof(*wqe)); + + /* send a copy of the frame to the BPF listener, if any */ + if (ifp != NULL && ifp->if_bpf != NULL) + ETHER_BPF_MTAP(ifp, mb); + + if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) { + wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM; + } + if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) { + wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM; + } + if ( wqe->eth.cs_flags == 0 ) { + sq->stats.csum_offload_none++; + } + + if (mb->m_pkthdr.csum_flags & CSUM_TSO) { + u32 payload_len; + u32 mss = mb->m_pkthdr.tso_segsz; + u32 num_pkts; + + wqe->eth.mss = cpu_to_be16(mss); + opcode = MLX5_OPCODE_LSO; + ihs = mlx5e_get_header_size(mb); + payload_len = mb->m_pkthdr.len - ihs; + if (payload_len == 0) + num_pkts = 1; + else + num_pkts = DIV_ROUND_UP(payload_len, mss); + sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs); + + sq->stats.tso_packets++; + sq->stats.tso_bytes += payload_len; + } else { + opcode = MLX5_OPCODE_SEND; + ihs = mlx5e_get_inline_hdr_size(sq, mb); + sq->mbuf[pi].num_bytes = max_t (unsigned int, + mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); + } + if (mb->m_flags & M_VLANTAG) { + struct ether_vlan_header *eh = + (struct ether_vlan_header *)wqe->eth.inline_hdr_start; + /* range checks */ + if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN)) + ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN); + else if (ihs < ETHER_HDR_LEN) { + err = EINVAL; + goto tx_drop; + } + m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); + m_adj(mb, ETHER_HDR_LEN); + /* insert 4 bytes VLAN tag into data stream */ + eh->evl_proto = eh->evl_encap_proto; + eh->evl_encap_proto = htons(ETHERTYPE_VLAN); + eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); + /* copy rest of header data, if any */ + m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1)); + m_adj(mb, ihs - ETHER_HDR_LEN); + /* extend header by 4 bytes */ + ihs += ETHER_VLAN_ENCAP_LEN; + } else { + m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); + m_adj(mb, ihs); + } + + wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); + + ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; + if (likely(ihs > sizeof(wqe->eth.inline_hdr_start))) { + ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start), + MLX5_SEND_WQE_DS); + } + dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt; + + /* trim off empty mbufs */ + while (mb->m_len == 0) { + mb = m_free(mb); + /* check if all data has been inlined */ + if (mb == NULL) + goto skip_dma; + } + + err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + if (err == EFBIG) { + /* Update *mbp before defrag in case it was trimmed in the loop above */ + *mbp = mb; + /* Update statistics */ + sq->stats.defragged++; + /* Too many mbuf fragments */ + mb = m_defrag(*mbp, M_NOWAIT); + if (mb == NULL) { + mb = *mbp; + goto tx_drop; + } + /* Try again */ + err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, + mb, segs, &nsegs, BUS_DMA_NOWAIT); + } + /* catch errors */ + if (err != 0) { + goto tx_drop; + } + *mbp = mb; + + for (x = 0; x != nsegs; x++) { + if (segs[x].ds_len == 0) + continue; + dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr); + dseg->lkey = sq->mkey_be; + dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len); + dseg++; + } +skip_dma: + ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl)); + + wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); + wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); + wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; + + /* store pointer to mbuf */ + sq->mbuf[pi].mbuf = mb; + sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); + sq->pc += sq->mbuf[pi].num_wqebbs; + + /* make sure all mbuf data is written to RAM */ + if (mb != NULL) + bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map, BUS_DMASYNC_PREWRITE); + + mlx5e_tx_notify_hw(sq, wqe, 0); + + sq->stats.packets++; + return (0); + +tx_drop: + sq->stats.dropped++; + *mbp = NULL; + m_freem(mb); + return err; +} + +static void +mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget) +{ + u16 sqcc; + + /* + * sq->cc must be updated only after mlx5_cqwq_update_db_record(), + * otherwise a cq overrun may occur + */ + sqcc = sq->cc; + + while (budget--) { + struct mlx5_cqe64 *cqe; + struct mbuf *mb; + u16 ci; + + cqe = mlx5e_get_cqe(&sq->cq); + if (!cqe) + break; + + ci = sqcc & sq->wq.sz_m1; + mb = sq->mbuf[ci].mbuf; + sq->mbuf[ci].mbuf = NULL; /* safety clear */ + + if (mb == NULL) { + if (sq->mbuf[ci].num_bytes == 0) { + /* NOP */ + sq->stats.nop++; + } + } else { + bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map); + + /* free transmitted mbuf */ + m_freem(mb); + } + sqcc += sq->mbuf[ci].num_wqebbs; + } + + mlx5_cqwq_update_db_record(&sq->cq.wq); + + /* ensure cq space is freed before enabling more cqes */ + wmb(); + + sq->cc = sqcc; + + if (atomic_cmpset_int(&sq->queue_state, MLX5E_SQ_FULL, MLX5E_SQ_READY)) + taskqueue_enqueue(sq->sq_tq, &sq->sq_task); +} + +static int +mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb) +{ + struct mbuf *next; + int err = 0; + + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { + if (mb) + err = drbr_enqueue(ifp, sq->br, mb); + return (err); + } + + if (mb != NULL) + /* If we can't insert mbuf into drbr, try to xmit anyway. + * We keep the error we got so we could return that after xmit. + */ + err = drbr_enqueue(ifp, sq->br, mb); + + /* Process the queue */ + while ((next = drbr_peek(ifp, sq->br)) != NULL) { + if (mlx5e_sq_xmit(sq, &next) != 0) { + if (next == NULL) { + drbr_advance(ifp, sq->br); + } else { + drbr_putback(ifp, sq->br, next); + atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_FULL); + } + break; + } + drbr_advance(ifp, sq->br); + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + break; + } + return (err); +} + +int +mlx5e_xmit(struct ifnet *ifp, struct mbuf *mb) +{ + struct mlx5e_sq *sq; + int ret; + + sq = mlx5e_select_queue(ifp, mb); + if (unlikely(sq == NULL)) { + /* invalid send queue */ + m_freem(mb); + return (ENXIO); + } + + if (mtx_trylock(&sq->lock)) { + ret = mlx5e_xmit_locked(ifp, sq, mb); + mtx_unlock(&sq->lock); + } else { + ret = drbr_enqueue(ifp, sq->br, mb); + taskqueue_enqueue(sq->sq_tq, &sq->sq_task); + } + + return (ret); +} + +void +mlx5e_tx_cq_comp(struct mlx5_core_cq *mcq) +{ + struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq); + + mtx_lock(&sq->comp_lock); + mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX); + mlx5e_cq_arm(&sq->cq); + mtx_unlock(&sq->comp_lock); +} + +void +mlx5e_tx_que(void *context, int pending) +{ + struct mlx5e_sq *sq = context; + struct ifnet *ifp = sq->channel->ifp; + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + mtx_lock(&sq->lock); + if (!drbr_empty(ifp, sq->br)) + mlx5e_xmit_locked(ifp, sq, NULL); + mtx_unlock(&sq->lock); + } +} diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c new file mode 100644 index 0000000..7836bfe --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c @@ -0,0 +1,58 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "en.h" + +struct mlx5_cqe64 * +mlx5e_get_cqe(struct mlx5e_cq *cq) +{ + struct mlx5_cqe64 *cqe; + + cqe = mlx5_cqwq_get_wqe(&cq->wq, mlx5_cqwq_get_ci(&cq->wq)); + + if ((cqe->op_own ^ mlx5_cqwq_get_wrap_cnt(&cq->wq)) & MLX5_CQE_OWNER_MASK) + return (NULL); + + mlx5_cqwq_pop(&cq->wq); + + /* ensure cqe content is read after cqe ownership bit */ + rmb(); + + return (cqe); +} + +void +mlx5e_cq_error_event(struct mlx5_core_cq *mcq, int event) +{ + struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq); + struct mlx5e_channel *c = cq->channel; + struct mlx5e_priv *priv = c->priv; + struct ifnet *ifp = priv->ifp; + + if_printf(ifp, "%s: cqn=0x%.6x event=0x%.2x\n", + __func__, mcq->cqn, event); +} diff --git a/sys/dev/mlx5/mlx5_en/tcp_tlro.c b/sys/dev/mlx5/mlx5_en/tcp_tlro.c new file mode 100644 index 0000000..27e861e --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/tcp_tlro.c @@ -0,0 +1,697 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include <sys/param.h> +#include <sys/libkern.h> +#include <sys/mbuf.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> +#include <sys/malloc.h> +#include <sys/kernel.h> +#include <sys/endian.h> +#include <sys/socket.h> +#include <sys/sockopt.h> +#include <sys/smp.h> + +#include <net/if.h> +#include <net/if_var.h> +#include <net/ethernet.h> + +#if defined(INET) || defined(INET6) +#include <netinet/in.h> +#endif + +#ifdef INET +#include <netinet/ip.h> +#endif + +#ifdef INET6 +#include <netinet/ip6.h> +#endif + +#include <netinet/tcp_var.h> + +#include "tcp_tlro.h" + +#ifndef M_HASHTYPE_LRO_TCP +#ifndef KLD_MODULE +#warning "M_HASHTYPE_LRO_TCP is not defined" +#endif +#define M_HASHTYPE_LRO_TCP 254 +#endif + +static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, tlro, + CTLFLAG_RW, 0, "TCP turbo LRO parameters"); + +static MALLOC_DEFINE(M_TLRO, "TLRO", "Turbo LRO"); + +static int tlro_min_rate = 20; /* Hz */ + +SYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, min_rate, CTLFLAG_RWTUN, + &tlro_min_rate, 0, "Minimum serving rate in Hz"); + +static int tlro_max_packet = IP_MAXPACKET; + +SYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, max_packet, CTLFLAG_RWTUN, + &tlro_max_packet, 0, "Maximum packet size in bytes"); + +typedef struct { + uint32_t value; +} __packed uint32_p_t; + +static uint16_t +tcp_tlro_csum(const uint32_p_t *p, size_t l) +{ + const uint32_p_t *pend = p + (l / 4); + uint64_t cs; + + for (cs = 0; p != pend; p++) + cs += le32toh(p->value); + while (cs > 0xffff) + cs = (cs >> 16) + (cs & 0xffff); + return (cs); +} + +static void * +tcp_tlro_get_header(const struct mbuf *m, const u_int off, + const u_int len) +{ + if (m->m_len < (off + len)) + return (NULL); + return (mtod(m, char *) + off); +} + +static uint8_t +tcp_tlro_info_save_timestamp(struct tlro_mbuf_data *pinfo) +{ + struct tcphdr *tcp = pinfo->tcp; + uint32_t *ts_ptr; + + if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2)) + return (0); + + ts_ptr = (uint32_t *)(tcp + 1); + if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | + (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) + return (0); + + /* save timestamps */ + pinfo->tcp_ts = ts_ptr[1]; + pinfo->tcp_ts_reply = ts_ptr[2]; + return (1); +} + +static void +tcp_tlro_info_restore_timestamp(struct tlro_mbuf_data *pinfoa, + struct tlro_mbuf_data *pinfob) +{ + struct tcphdr *tcp = pinfoa->tcp; + uint32_t *ts_ptr; + + if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2)) + return; + + ts_ptr = (uint32_t *)(tcp + 1); + if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | + (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) + return; + + /* restore timestamps */ + ts_ptr[1] = pinfob->tcp_ts; + ts_ptr[2] = pinfob->tcp_ts_reply; +} + +static void +tcp_tlro_extract_header(struct tlro_mbuf_data *pinfo, struct mbuf *m, int seq) +{ + uint8_t *phdr = (uint8_t *)pinfo->buf; + struct ether_header *eh; + struct ether_vlan_header *vlan; +#ifdef INET + struct ip *ip; +#endif +#ifdef INET6 + struct ip6_hdr *ip6; +#endif + struct tcphdr *tcp; + uint16_t etype; + int diff; + int off; + + /* fill in information */ + pinfo->head = m; + pinfo->last_tick = ticks; + pinfo->sequence = seq; + pinfo->pprev = &m_last(m)->m_next; + + off = sizeof(*eh); + if (m->m_len < off) + goto error; + eh = tcp_tlro_get_header(m, 0, sizeof(*eh)); + if (eh == NULL) + goto error; + memcpy(phdr, &eh->ether_dhost, ETHER_ADDR_LEN); + phdr += ETHER_ADDR_LEN; + memcpy(phdr, &eh->ether_type, sizeof(eh->ether_type)); + phdr += sizeof(eh->ether_type); + etype = ntohs(eh->ether_type); + + if (etype == ETHERTYPE_VLAN) { + vlan = tcp_tlro_get_header(m, off, sizeof(*vlan)); + if (vlan == NULL) + goto error; + memcpy(phdr, &vlan->evl_tag, sizeof(vlan->evl_tag) + + sizeof(vlan->evl_proto)); + phdr += sizeof(vlan->evl_tag) + sizeof(vlan->evl_proto); + etype = ntohs(vlan->evl_proto); + off += sizeof(*vlan) - sizeof(*eh); + } + switch (etype) { +#ifdef INET + case ETHERTYPE_IP: + /* + * Cannot LRO: + * - Non-IP packets + * - Fragmented packets + * - Packets with IPv4 options + * - Non-TCP packets + */ + ip = tcp_tlro_get_header(m, off, sizeof(*ip)); + if (ip == NULL || + (ip->ip_off & htons(IP_MF | IP_OFFMASK)) != 0 || + (ip->ip_p != IPPROTO_TCP) || + (ip->ip_hl << 2) != sizeof(*ip)) + goto error; + + /* Legacy IP has a header checksum that needs to be correct */ + if (!(m->m_pkthdr.csum_flags & CSUM_IP_CHECKED)) { + /* Verify IP header */ + if (tcp_tlro_csum((uint32_p_t *)ip, sizeof(*ip)) != 0xFFFF) + m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; + else + m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | + CSUM_IP_VALID; + } + /* Only accept valid checksums */ + if (!(m->m_pkthdr.csum_flags & CSUM_IP_VALID) || + !(m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) + goto error; + memcpy(phdr, &ip->ip_src, sizeof(ip->ip_src) + + sizeof(ip->ip_dst)); + phdr += sizeof(ip->ip_src) + sizeof(ip->ip_dst); + if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP) + pinfo->ip_len = m->m_pkthdr.len - off; + else + pinfo->ip_len = ntohs(ip->ip_len); + pinfo->ip_hdrlen = sizeof(*ip); + pinfo->ip.v4 = ip; + pinfo->ip_version = 4; + off += sizeof(*ip); + break; +#endif +#ifdef INET6 + case ETHERTYPE_IPV6: + /* + * Cannot LRO: + * - Non-IP packets + * - Packets with IPv6 options + * - Non-TCP packets + */ + ip6 = tcp_tlro_get_header(m, off, sizeof(*ip6)); + if (ip6 == NULL || ip6->ip6_nxt != IPPROTO_TCP) + goto error; + if (!(m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) + goto error; + memcpy(phdr, &ip6->ip6_src, sizeof(struct in6_addr) + + sizeof(struct in6_addr)); + phdr += sizeof(struct in6_addr) + sizeof(struct in6_addr); + if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP) + pinfo->ip_len = m->m_pkthdr.len - off; + else + pinfo->ip_len = ntohs(ip6->ip6_plen) + sizeof(*ip6); + pinfo->ip_hdrlen = sizeof(*ip6); + pinfo->ip.v6 = ip6; + pinfo->ip_version = 6; + off += sizeof(*ip6); + break; +#endif + default: + goto error; + } + tcp = tcp_tlro_get_header(m, off, sizeof(*tcp)); + if (tcp == NULL) + goto error; + memcpy(phdr, &tcp->th_sport, sizeof(tcp->th_sport) + + sizeof(tcp->th_dport)); + phdr += sizeof(tcp->th_sport) + + sizeof(tcp->th_dport); + /* store TCP header length */ + *phdr++ = tcp->th_off; + if (tcp->th_off < (sizeof(*tcp) >> 2)) + goto error; + + /* compute offset to data payload */ + pinfo->tcp_len = (tcp->th_off << 2); + off += pinfo->tcp_len; + + /* store more info */ + pinfo->data_off = off; + pinfo->tcp = tcp; + + /* try to save timestamp, if any */ + *phdr++ = tcp_tlro_info_save_timestamp(pinfo); + + /* verify offset and IP/TCP length */ + if (off > m->m_pkthdr.len || + pinfo->ip_len < pinfo->tcp_len) + goto error; + + /* compute data payload length */ + pinfo->data_len = (pinfo->ip_len - pinfo->tcp_len - pinfo->ip_hdrlen); + + /* trim any padded data */ + diff = (m->m_pkthdr.len - off) - pinfo->data_len; + if (diff != 0) { + if (diff < 0) + goto error; + else + m_adj(m, -diff); + } + /* compute header length */ + pinfo->buf_length = phdr - (uint8_t *)pinfo->buf; + /* zero-pad rest of buffer */ + memset(phdr, 0, TLRO_MAX_HEADER - pinfo->buf_length); + return; +error: + pinfo->buf_length = 0; +} + +static int +tcp_tlro_cmp64(const uint64_t *pa, const uint64_t *pb) +{ + int64_t diff = 0; + unsigned x; + + for (x = 0; x != TLRO_MAX_HEADER / 8; x++) { + /* + * NOTE: Endianness does not matter in this + * comparisation: + */ + diff = pa[x] - pb[x]; + if (diff != 0) + goto done; + } +done: + if (diff < 0) + return (-1); + else if (diff > 0) + return (1); + return (0); +} + +static int +tcp_tlro_compare_header(const void *_ppa, const void *_ppb) +{ + const struct tlro_mbuf_ptr *ppa = _ppa; + const struct tlro_mbuf_ptr *ppb = _ppb; + struct tlro_mbuf_data *pinfoa = ppa->data; + struct tlro_mbuf_data *pinfob = ppb->data; + int ret; + + ret = (pinfoa->head == NULL) - (pinfob->head == NULL); + if (ret != 0) + goto done; + + ret = pinfoa->buf_length - pinfob->buf_length; + if (ret != 0) + goto done; + if (pinfoa->buf_length != 0) { + ret = tcp_tlro_cmp64(pinfoa->buf, pinfob->buf); + if (ret != 0) + goto done; + ret = ntohl(pinfoa->tcp->th_seq) - ntohl(pinfob->tcp->th_seq); + if (ret != 0) + goto done; + ret = ntohl(pinfoa->tcp->th_ack) - ntohl(pinfob->tcp->th_ack); + if (ret != 0) + goto done; + ret = pinfoa->sequence - pinfob->sequence; + if (ret != 0) + goto done; + } +done: + return (ret); +} + +static void +tcp_tlro_sort(struct tlro_ctrl *tlro) +{ + if (tlro->curr == 0) + return; + + qsort(tlro->mbuf, tlro->curr, sizeof(struct tlro_mbuf_ptr), + &tcp_tlro_compare_header); +} + +static int +tcp_tlro_get_ticks(void) +{ + int to = tlro_min_rate; + + if (to < 1) + to = 1; + to = hz / to; + if (to < 1) + to = 1; + return (to); +} + +static void +tcp_tlro_combine(struct tlro_ctrl *tlro, int force) +{ + struct tlro_mbuf_data *pinfoa; + struct tlro_mbuf_data *pinfob; + uint32_t cs; + int curr_ticks = ticks; + int ticks_limit = tcp_tlro_get_ticks(); + unsigned x; + unsigned y; + unsigned z; + int temp; + + if (tlro->curr == 0) + return; + + for (y = 0; y != tlro->curr;) { + struct mbuf *m; + + pinfoa = tlro->mbuf[y].data; + for (x = y + 1; x != tlro->curr; x++) { + pinfob = tlro->mbuf[x].data; + if (pinfoa->buf_length != pinfob->buf_length || + tcp_tlro_cmp64(pinfoa->buf, pinfob->buf) != 0) + break; + } + if (pinfoa->buf_length == 0) { + /* forward traffic which cannot be combined */ + for (z = y; z != x; z++) { + /* just forward packets */ + pinfob = tlro->mbuf[z].data; + + m = pinfob->head; + + /* reset info structure */ + pinfob->head = NULL; + pinfob->buf_length = 0; + + /* do stats */ + tlro->lro_flushed++; + + /* input packet to network layer */ + (*tlro->ifp->if_input) (tlro->ifp, m); + } + y = z; + continue; + } + + /* compute current checksum subtracted some header parts */ + temp = (pinfoa->ip_len - pinfoa->ip_hdrlen); + cs = ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) + + tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len); + + /* append all fragments into one block */ + for (z = y + 1; z != x; z++) { + + pinfob = tlro->mbuf[z].data; + + /* check for command packets */ + if ((pinfoa->tcp->th_flags & ~(TH_ACK | TH_PUSH)) || + (pinfob->tcp->th_flags & ~(TH_ACK | TH_PUSH))) + break; + + /* check if there is enough space */ + if ((pinfoa->ip_len + pinfob->data_len) > tlro_max_packet) + break; + + /* try to append the new segment */ + temp = ntohl(pinfoa->tcp->th_seq) + pinfoa->data_len; + if (temp != (int)ntohl(pinfob->tcp->th_seq)) + break; + + temp = pinfob->ip_len - pinfob->ip_hdrlen; + cs += ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) + + tcp_tlro_csum((uint32_p_t *)pinfob->tcp, pinfob->tcp_len); + /* remove fields which appear twice */ + cs += (IPPROTO_TCP << 8); + if (pinfob->ip_version == 4) { + cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_src, 4); + cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_dst, 4); + } else { + cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_src, 16); + cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_dst, 16); + } + /* remainder computation */ + while (cs > 0xffff) + cs = (cs >> 16) + (cs & 0xffff); + + /* update window and ack sequence number */ + pinfoa->tcp->th_ack = pinfob->tcp->th_ack; + pinfoa->tcp->th_win = pinfob->tcp->th_win; + + /* check if we should restore the timestamp */ + tcp_tlro_info_restore_timestamp(pinfoa, pinfob); + + /* accumulate TCP flags */ + pinfoa->tcp->th_flags |= pinfob->tcp->th_flags; + + /* update lengths */ + pinfoa->ip_len += pinfob->data_len; + pinfoa->data_len += pinfob->data_len; + + /* clear mbuf pointer - packet is accumulated */ + m = pinfob->head; + + /* reset info structure */ + pinfob->head = NULL; + pinfob->buf_length = 0; + + /* append data to mbuf [y] */ + m_adj(m, pinfob->data_off); + /* delete mbuf tags, if any */ + m_tag_delete_chain(m, NULL); + /* clear packet header flag */ + m->m_flags &= ~M_PKTHDR; + + /* concat mbuf(s) to end of list */ + pinfoa->pprev[0] = m; + m = m_last(m); + pinfoa->pprev = &m->m_next; + pinfoa->head->m_pkthdr.len += pinfob->data_len; + } + /* compute new TCP header checksum */ + pinfoa->tcp->th_sum = 0; + + temp = pinfoa->ip_len - pinfoa->ip_hdrlen; + cs = (cs ^ 0xFFFF) + + tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len) + + ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8); + + /* remainder computation */ + while (cs > 0xffff) + cs = (cs >> 16) + (cs & 0xffff); + + /* update new checksum */ + pinfoa->tcp->th_sum = ~htole16(cs); + + /* update IP length, if any */ + if (pinfoa->ip_version == 4) { + if (pinfoa->ip_len > IP_MAXPACKET) { + M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP); + pinfoa->ip.v4->ip_len = htons(IP_MAXPACKET); + } else { + pinfoa->ip.v4->ip_len = htons(pinfoa->ip_len); + } + } else { + if (pinfoa->ip_len > (IP_MAXPACKET + sizeof(*pinfoa->ip.v6))) { + M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP); + pinfoa->ip.v6->ip6_plen = htons(IP_MAXPACKET); + } else { + temp = pinfoa->ip_len - sizeof(*pinfoa->ip.v6); + pinfoa->ip.v6->ip6_plen = htons(temp); + } + } + + temp = curr_ticks - pinfoa->last_tick; + /* check if packet should be forwarded */ + if (force != 0 || z != x || temp >= ticks_limit || + pinfoa->data_len == 0) { + + /* compute new IPv4 header checksum */ + if (pinfoa->ip_version == 4) { + pinfoa->ip.v4->ip_sum = 0; + cs = tcp_tlro_csum((uint32_p_t *)pinfoa->ip.v4, + sizeof(*pinfoa->ip.v4)); + pinfoa->ip.v4->ip_sum = ~htole16(cs); + } + /* forward packet */ + m = pinfoa->head; + + /* reset info structure */ + pinfoa->head = NULL; + pinfoa->buf_length = 0; + + /* do stats */ + tlro->lro_flushed++; + + /* input packet to network layer */ + (*tlro->ifp->if_input) (tlro->ifp, m); + } + y = z; + } + + /* cleanup all NULL heads */ + for (y = 0; y != tlro->curr; y++) { + if (tlro->mbuf[y].data->head == NULL) { + for (z = y + 1; z != tlro->curr; z++) { + struct tlro_mbuf_ptr ptemp; + if (tlro->mbuf[z].data->head == NULL) + continue; + ptemp = tlro->mbuf[y]; + tlro->mbuf[y] = tlro->mbuf[z]; + tlro->mbuf[z] = ptemp; + y++; + } + break; + } + } + tlro->curr = y; +} + +static void +tcp_tlro_cleanup(struct tlro_ctrl *tlro) +{ + while (tlro->curr != 0 && + tlro->mbuf[tlro->curr - 1].data->head == NULL) + tlro->curr--; +} + +void +tcp_tlro_flush(struct tlro_ctrl *tlro, int force) +{ + if (tlro->curr == 0) + return; + + tcp_tlro_sort(tlro); + tcp_tlro_cleanup(tlro); + tcp_tlro_combine(tlro, force); +} + +int +tcp_tlro_init(struct tlro_ctrl *tlro, struct ifnet *ifp, + int max_mbufs) +{ + ssize_t size; + uint32_t x; + + /* set zero defaults */ + memset(tlro, 0, sizeof(*tlro)); + + /* compute size needed for data */ + size = (sizeof(struct tlro_mbuf_ptr) * max_mbufs) + + (sizeof(struct tlro_mbuf_data) * max_mbufs); + + /* range check */ + if (max_mbufs <= 0 || size <= 0 || ifp == NULL) + return (EINVAL); + + /* setup tlro control structure */ + tlro->mbuf = malloc(size, M_TLRO, M_WAITOK | M_ZERO); + tlro->max = max_mbufs; + tlro->ifp = ifp; + + /* setup pointer array */ + for (x = 0; x != tlro->max; x++) { + tlro->mbuf[x].data = ((struct tlro_mbuf_data *) + &tlro->mbuf[max_mbufs]) + x; + } + return (0); +} + +void +tcp_tlro_free(struct tlro_ctrl *tlro) +{ + struct tlro_mbuf_data *pinfo; + struct mbuf *m; + uint32_t y; + + /* check if not setup */ + if (tlro->mbuf == NULL) + return; + /* free MBUF array and any leftover MBUFs */ + for (y = 0; y != tlro->max; y++) { + + pinfo = tlro->mbuf[y].data; + + m = pinfo->head; + + /* reset info structure */ + pinfo->head = NULL; + pinfo->buf_length = 0; + + m_freem(m); + } + free(tlro->mbuf, M_TLRO); + /* reset buffer */ + memset(tlro, 0, sizeof(*tlro)); +} + +void +tcp_tlro_rx(struct tlro_ctrl *tlro, struct mbuf *m) +{ + if (m->m_len > 0 && tlro->curr < tlro->max) { + /* do stats */ + tlro->lro_queued++; + + /* extract header */ + tcp_tlro_extract_header(tlro->mbuf[tlro->curr++].data, + m, tlro->sequence++); + } else if (tlro->ifp != NULL) { + /* do stats */ + tlro->lro_flushed++; + + /* input packet to network layer */ + (*tlro->ifp->if_input) (tlro->ifp, m); + } else { + /* packet drop */ + m_freem(m); + } +} diff --git a/sys/dev/mlx5/mlx5_en/tcp_tlro.h b/sys/dev/mlx5/mlx5_en/tcp_tlro.h new file mode 100644 index 0000000..1e605d5 --- /dev/null +++ b/sys/dev/mlx5/mlx5_en/tcp_tlro.h @@ -0,0 +1,83 @@ +/*- + * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _TCP_TLRO_H_ +#define _TCP_TLRO_H_ + +#define TLRO_MAX_HEADER 64 /* bytes */ + +struct ip; +struct ip6_hdr; +struct tcphdr; + +struct tlro_mbuf_data { + union { +#ifdef INET + struct ip *v4; +#endif +#ifdef INET6 + struct ip6_hdr *v6; +#endif + } ip; + struct tcphdr *tcp; + struct mbuf *head; + struct mbuf **pprev; + int last_tick; + int sequence; + int data_len; + int data_off; + int ip_hdrlen; + int ip_len; + uint32_t tcp_ts; + uint32_t tcp_ts_reply; + uint16_t tcp_len; + uint8_t ip_version; + uint8_t buf_length; /* in 32-bit words */ + uint64_t buf[TLRO_MAX_HEADER / 8]; +} __aligned(256); + +struct tlro_mbuf_ptr { + struct tlro_mbuf_data *data; +}; + +/* NB: This is part of driver structs */ +struct tlro_ctrl { + struct ifnet *ifp; + struct tlro_mbuf_ptr *mbuf; + uint64_t lro_queued; + uint64_t lro_flushed; + uint32_t max; + uint32_t curr; + int sequence; +}; + +int tcp_tlro_init(struct tlro_ctrl *, struct ifnet *, int); +void tcp_tlro_free(struct tlro_ctrl *); +void tcp_tlro_flush(struct tlro_ctrl *, int); +void tcp_tlro_rx(struct tlro_ctrl *, struct mbuf *); + +#endif /* _TCP_TLRO_H_ */ |